in src/Lucene.Net/Index/IndexWriter.cs [5407:5763]
private int MergeMiddle(MergePolicy.OneMerge merge)
{
merge.CheckAborted(directory);
string mergedName = merge.info.Info.Name;
IList<SegmentCommitInfo> sourceSegments = merge.Segments;
IOContext context = new IOContext(merge.MergeInfo);
CheckAbort checkAbort = new CheckAbort(merge, directory);
TrackingDirectoryWrapper dirWrapper = new TrackingDirectoryWrapper(directory);
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "merging " + SegString(merge.Segments));
}
merge.readers = new JCG.List<SegmentReader>();
// this is try/finally to make sure merger's readers are
// closed:
bool success = false;
try
{
int segUpto = 0;
while (segUpto < sourceSegments.Count)
{
SegmentCommitInfo info = sourceSegments[segUpto];
// Hold onto the "live" reader; we will use this to
// commit merged deletes
ReadersAndUpdates rld = readerPool.Get(info, true);
// Carefully pull the most recent live docs and reader
SegmentReader reader;
IBits liveDocs;
int delCount;
UninterruptableMonitor.Enter(this);
try
{
// Must sync to ensure BufferedDeletesStream cannot change liveDocs,
// pendingDeleteCount and field updates while we pull a copy:
reader = rld.GetReaderForMerge(context);
liveDocs = rld.GetReadOnlyLiveDocs();
delCount = rld.PendingDeleteCount + info.DelCount;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(reader != null);
Debugging.Assert(rld.VerifyDocCounts());
}
if (infoStream.IsEnabled("IW"))
{
if (rld.PendingDeleteCount != 0)
{
infoStream.Message("IW", "seg=" + SegString(info) + " delCount=" + info.DelCount + " pendingDelCount=" + rld.PendingDeleteCount);
}
else if (info.DelCount != 0)
{
infoStream.Message("IW", "seg=" + SegString(info) + " delCount=" + info.DelCount);
}
else
{
infoStream.Message("IW", "seg=" + SegString(info) + " no deletes");
}
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
// Deletes might have happened after we pulled the merge reader and
// before we got a read-only copy of the segment's actual live docs
// (taking pending deletes into account). In that case we need to
// make a new reader with updated live docs and del count.
if (reader.NumDeletedDocs != delCount)
{
// fix the reader's live docs and del count
if (Debugging.AssertsEnabled) Debugging.Assert(delCount > reader.NumDeletedDocs); // beware of zombies
SegmentReader newReader = new SegmentReader(info, reader, liveDocs, info.Info.DocCount - delCount);
bool released = false;
try
{
rld.Release(reader);
released = true;
}
finally
{
if (!released)
{
newReader.DecRef();
}
}
reader = newReader;
}
merge.readers.Add(reader);
if (Debugging.AssertsEnabled) Debugging.Assert(delCount <= info.Info.DocCount, "delCount={0} info.DocCount={1} rld.PendingDeleteCount={2} info.DelCount=", delCount, info.Info.DocCount, rld.PendingDeleteCount, info.DelCount);
segUpto++;
}
// System.out.println("[" + Thread.currentThread().getName() + "] IW.mergeMiddle: merging " + merge.getMergeReaders());
// we pass merge.getMergeReaders() instead of merge.readers to allow the
// OneMerge to return a view over the actual segments to merge
SegmentMerger merger = new SegmentMerger(merge.GetMergeReaders(), merge.info.Info, infoStream, dirWrapper, config.TermIndexInterval, checkAbort, globalFieldNumberMap, context, config.CheckIntegrityAtMerge);
merge.CheckAborted(directory);
// this is where all the work happens:
MergeState mergeState;
bool success3 = false;
try
{
if (!merger.ShouldMerge)
{
// would result in a 0 document segment: nothing to merge!
mergeState = new MergeState(new JCG.List<AtomicReader>(), merge.info.Info, infoStream, checkAbort);
}
else
{
mergeState = merger.Merge();
}
success3 = true;
}
finally
{
if (!success3)
{
UninterruptableMonitor.Enter(this);
try
{
deleter.Refresh(merge.info.Info.Name);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
}
if (Debugging.AssertsEnabled) Debugging.Assert(mergeState.SegmentInfo == merge.info.Info);
merge.info.Info.SetFiles(new JCG.HashSet<string>(dirWrapper.CreatedFiles));
// Record which codec was used to write the segment
if (infoStream.IsEnabled("IW"))
{
if (merge.info.Info.DocCount == 0)
{
infoStream.Message("IW", "merge away fully deleted segments");
}
else
{
infoStream.Message("IW", "merge codec=" + codec + " docCount=" + merge.info.Info.DocCount + "; merged segment has " + (mergeState.FieldInfos.HasVectors ? "vectors" : "no vectors") + "; " + (mergeState.FieldInfos.HasNorms ? "norms" : "no norms") + "; " + (mergeState.FieldInfos.HasDocValues ? "docValues" : "no docValues") + "; " + (mergeState.FieldInfos.HasProx ? "prox" : "no prox") + "; " + (mergeState.FieldInfos.HasProx ? "freqs" : "no freqs"));
}
}
// Very important to do this before opening the reader
// because codec must know if prox was written for
// this segment:
//System.out.println("merger set hasProx=" + merger.hasProx() + " seg=" + merge.info.name);
bool useCompoundFile;
UninterruptableMonitor.Enter(this); // Guard segmentInfos
try
{
useCompoundFile = mergePolicy.UseCompoundFile(segmentInfos, merge.info);
}
finally
{
UninterruptableMonitor.Exit(this);
}
if (useCompoundFile)
{
success = false;
ICollection<string> filesToRemove = merge.info.GetFiles();
try
{
filesToRemove = CreateCompoundFile(infoStream, directory, checkAbort, merge.info.Info, context);
success = true;
}
catch (Exception ioe) when (ioe.IsIOException())
{
UninterruptableMonitor.Enter(this);
try
{
if (merge.IsAborted)
{
// this can happen if rollback or close(false)
// is called -- fall through to logic below to
// remove the partially created CFS:
}
else
{
HandleMergeException(ioe, merge);
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
catch (Exception t) when (t.IsThrowable())
{
HandleMergeException(t, merge);
}
finally
{
if (!success)
{
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "hit exception creating compound file during merge");
}
UninterruptableMonitor.Enter(this);
try
{
deleter.DeleteFile(Lucene.Net.Index.IndexFileNames.SegmentFileName(mergedName, "", Lucene.Net.Index.IndexFileNames.COMPOUND_FILE_EXTENSION));
deleter.DeleteFile(Lucene.Net.Index.IndexFileNames.SegmentFileName(mergedName, "", Lucene.Net.Index.IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
deleter.DeleteNewFiles(merge.info.GetFiles());
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
}
// So that, if we hit exc in deleteNewFiles (next)
// or in commitMerge (later), we close the
// per-segment readers in the finally clause below:
success = false;
UninterruptableMonitor.Enter(this);
try
{
// delete new non cfs files directly: they were never
// registered with IFD
deleter.DeleteNewFiles(filesToRemove);
if (merge.IsAborted)
{
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "abort merge after building CFS");
}
deleter.DeleteFile(Lucene.Net.Index.IndexFileNames.SegmentFileName(mergedName, "", Lucene.Net.Index.IndexFileNames.COMPOUND_FILE_EXTENSION));
deleter.DeleteFile(Lucene.Net.Index.IndexFileNames.SegmentFileName(mergedName, "", Lucene.Net.Index.IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
return 0;
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
merge.info.Info.UseCompoundFile = true;
}
else
{
// So that, if we hit exc in commitMerge (later),
// we close the per-segment readers in the finally
// clause below:
success = false;
}
// Have codec write SegmentInfo. Must do this after
// creating CFS so that 1) .si isn't slurped into CFS,
// and 2) .si reflects useCompoundFile=true change
// above:
bool success2 = false;
try
{
codec.SegmentInfoFormat.SegmentInfoWriter.Write(directory, merge.info.Info, mergeState.FieldInfos, context);
success2 = true;
}
finally
{
if (!success2)
{
UninterruptableMonitor.Enter(this);
try
{
deleter.DeleteNewFiles(merge.info.GetFiles());
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
}
// TODO: ideally we would freeze merge.info here!!
// because any changes after writing the .si will be
// lost...
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", string.Format(CultureInfo.InvariantCulture, "merged segment size={0:n3} MB vs estimate={1:n3} MB", merge.info.GetSizeInBytes() / 1024.0 / 1024.0, merge.EstimatedMergeBytes / 1024 / 1024.0));
}
IndexReaderWarmer mergedSegmentWarmer = config.MergedSegmentWarmer;
if (poolReaders && mergedSegmentWarmer != null && merge.info.Info.DocCount != 0)
{
ReadersAndUpdates rld = readerPool.Get(merge.info, true);
SegmentReader sr = rld.GetReader(IOContext.READ);
try
{
mergedSegmentWarmer.Warm(sr);
}
finally
{
UninterruptableMonitor.Enter(this);
try
{
rld.Release(sr);
readerPool.Release(rld);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
}
// Force READ context because we merge deletes onto
// this reader:
if (!CommitMerge(merge, mergeState))
{
// commitMerge will return false if this merge was
// aborted
return 0;
}
success = true;
}
finally
{
// Readers are already closed in commitMerge if we didn't hit
// an exc:
if (!success)
{
CloseMergeReaders(merge, true);
}
}
return merge.info.Info.DocCount;
}