Apache Lucene open-source search software
Streamline segment info logged by infostream, esp. when merging (#15885)
Michael Sokolov03fc503a49c1
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 4aaf30e7287..6fa4c282674 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -338,6 +338,8 @@ Other* GITHUB#15867: Use IllegalCallerException to signal wrong caller in internal classes(VectorizationProvider, TestSecrets). (Uwe Schindler)+* GITHUB#15885: Slenderize SegmentInfo.toString() and use beefy version in infostream only once+* GITHUB#15891: Add the ability to pass DataAccessHints in Lucene104ScalarQuantizedVectorsReader (Navneet Verma)======================= Lucene 10.4.0 =======================diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.javaindex 4ab42e74587..30ff07c3932 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java@@ -1047,7 +1047,6 @@ public class IndexWriter// Must clone because we don't want the incoming NRT reader to "see" any changes this writer// now makes:segmentInfos = reader.segmentInfos.clone();-SegmentInfos lastCommit;try {lastCommit = SegmentInfos.readCommit(directoryOrig, segmentInfos.getSegmentsFileName());@@ -1111,7 +1110,9 @@ public class IndexWriterrollbackSegments = segmentInfos.createBackupSegmentInfos();}-+ if (infoStream.isEnabled("IW")) {+ infoStream.message("IW", "init " + segmentInfos.toStringVerbose());+ }commitUserData = new HashMap<>(segmentInfos.getUserData()).entrySet();pendingNumDocs.set(segmentInfos.totalMaxDoc());@@ -2844,7 +2845,7 @@ public class IndexWriterensureOpen(false);if (infoStream.isEnabled("IW")) {- infoStream.message("IW", "publishFlushedSegment " + newSegment);+ infoStream.message("IW", "publishFlushedSegment " + newSegment.toStringVerbose());}if (globalPacket != null && globalPacket.any()) {@@ -3343,6 +3344,13 @@ public class IndexWritertry {writer.addIndexesReaderMerge(merge);success = true;+ if (infoStream != null && infoStream.isEnabled("IW")) {+ if (merge.info == null) {+ infoStream.message("IW", "dropped 100% deleted segment");+ } else {+ infoStream.message("IW", "merged new segment " + merge.info.toStringVerbose());+ }+ }} catch (Throwable t) {handleMergeException(t, merge);} finally {@@ -4775,6 +4783,7 @@ public class IndexWriterif (merge.info != null && merge.isAborted() == false) {if (infoStream.isEnabled("IW")) {+ infoStream.message("IW", "merged new segment " + merge.info.toStringVerbose());infoStream.message("IW","merge time "diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.javaindex 08222107d2d..ed101ecda9e 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java@@ -349,7 +349,21 @@ public class SegmentCommitInfo {/** Returns a description of this segment. */public String toString(int pendingDelCount) {- String s = info.toString(delCount + pendingDelCount);+ return info.toString(delCount + pendingDelCount);+ }++ /** Returns a description of this segment. */+ @Override+ public String toString() {+ return info.toString();+ }++ /** Returns a more verbose description of this segment for use in infostream logging */+ String toStringVerbose() {+ String s = info.toStringVerbose();+ if (this.id != null) {+ s += " :id=" + StringHelper.idToString(id);+ }if (delGen != -1) {s += ":delGen=" + delGen;}@@ -362,18 +376,9 @@ public class SegmentCommitInfo {if (softDelCount > 0) {s += " :softDel=" + softDelCount;}- if (this.id != null) {- s += " :id=" + StringHelper.idToString(id);- }-return s;}- @Override- public String toString() {- return toString(0);- }-@Overridepublic SegmentCommitInfo clone() {SegmentCommitInfo other =diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.javaindex b85a6cb9e63..95f9ef9601b 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java@@ -211,31 +211,41 @@ public final class SegmentInfo {return Collections.unmodifiableSet(setFiles);}- @Override- public String toString() {- return toString(0);- }-/*** Used for debugging. Format may suddenly change.*- * <p>Current format looks like <code>_a(3.1):c45/4:[sorter=<long: "timestamp">!]</code>,- * which means the segment's name is <code>_a</code>; it was created with Lucene 3.1 (or '?' if- * it's unknown); it's using compound file format (would be <code>C</code> if not compound); it- * has 45 documents; it has 4 deletions (this part is left off when there are no deletions); it is- * sorted by the timestamp field in descending order (this part is omitted for unsorted segments).+ * <p>Current format looks like <code>_a(3.1):c45/4</code>, which means the segment's name is+ * <code>_a</code>; it was created with Lucene 3.1 (or '?' if it's unknown); it's using compound+ * file format (would be <code>C</code> if not compound); it has 45 documents; it has 4 deletions+ * (this part is left off when there are no deletions).+ *+ * <p>There is also {@link #toStringVerbose()} that includes more info about the segment. It is be+ * used to print detailed info when the segment is first written or read.*/public String toString(int delCount) {StringBuilder s = new StringBuilder();s.append(name).append('(').append(version == null ? "?" : version).append(')').append(':');char cfs = getUseCompoundFile() ? 'c' : 'C';s.append(cfs);-s.append(maxDoc);-if (delCount != 0) {s.append('/').append(delCount);}+ return s.toString();+ }++ @Override+ public String toString() {+ return toString(0);+ }++ /**+ * Logged by InfoStream when the segment is first flushed. Format may change. Appends more+ * information to the basic toString().+ */+ String toStringVerbose() {+ StringBuilder s = new StringBuilder();+ s.append(toString(0));if (indexSort != null) {s.append(":[indexSort=");diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.javaindex 62639d9faed..79077902eb8 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java@@ -985,6 +985,14 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo/** Returns readable description of this segment. */@Overridepublic String toString() {+ return toString(false);+ }++ String toStringVerbose() {+ return toString(true);+ }++ String toString(boolean verbose) {StringBuilder buffer = new StringBuilder();buffer.append(getSegmentsFileName()).append(": ");final int count = size();@@ -992,8 +1000,11 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfoif (i > 0) {buffer.append(' ');}- final SegmentCommitInfo info = info(i);- buffer.append(info.toString(0));+ if (verbose) {+ buffer.append(info(i).toStringVerbose());+ } else {+ buffer.append(info(i).toString(0));+ }}return buffer.toString();}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java b/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.javaindex 80681ee126b..27b426e416e 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java@@ -17,6 +17,10 @@package org.apache.lucene.index;import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import java.util.Locale;+import java.util.Set;import java.util.concurrent.atomic.AtomicBoolean;import org.apache.lucene.document.Document;import org.apache.lucene.store.Directory;@@ -86,4 +90,64 @@ public class TestInfoStream extends LuceneTestCase {dir.close();assertTrue(seenTestPoint.get());}++ public void testMergeSmallIndex() throws Exception {+ // examine info stream output from IW, MP, MS+ IndexWriterConfig iwc = new IndexWriterConfig(null);+ Set<String> components = Set.of("IW", "MP", "MS", "BD", "BU");+ List<String> infoStream = new ArrayList<>();+ iwc.setInfoStream(+ new InfoStream() {+ @Override+ public void close() throws IOException {}++ @Override+ public void message(String component, String message) {+ if (components.contains(component)) {+ infoStream.add(String.format(Locale.ROOT, "[%s] %s\n", component, message));+ }+ }++ @Override+ public boolean isEnabled(String component) {+ return components.contains(component);+ }+ });+ try (Directory dir = newDirectory();+ IndexWriter iw = new IndexWriter(dir, iwc)) {+ iw.addDocument(new Document());+ iw.commit();+ iw.addDocument(new Document());+ iw.addDocument(new Document());+ try (IndexReader reader = DirectoryReader.open(iw)) {+ assertTrue(iw.tryDeleteDocument(reader, 2) > 0);+ }+ iw.commit();+ iw.forceMerge(1);+ }+ boolean foundInit = false;+ int flushedCount = 0;+ int mergedCount = 0;+ for (String message : infoStream) {+ // we don't want to be printing full diagnostics every time a segment is mentioned in the log,+ // only when it is first flushed+ if (message.contains("diagnostics")) {+ if (message.startsWith("[IW] publishFlushedSegment")) {+ flushedCount++;+ } else if (message.startsWith("[IW] merged new segment")) {+ mergedCount++;+ } else {+ fail("message contains diagnostics: " + message);+ }+ }+ if (message.contains("init segments")) {+ assertEquals("[IW] init segments: \n", message);+ foundInit = true;+ }+ // System.out.print(message);+ }+ assertTrue("init message not found", foundInit);+ assertEquals(2, flushedCount);+ assertEquals(1, mergedCount);+ }}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.javaindex 0743715ebb1..76439967b70 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java@@ -184,7 +184,8 @@ public class TestSegmentInfos extends LuceneTestCase {Sort.INDEXORDER);assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000" + ":[indexSort=<doc>]",- si.toString());+ si.toStringVerbose());+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());// diagnostics O, attributes Xsi =@@ -210,7 +211,8 @@ public class TestSegmentInfos extends LuceneTestCase {+ ":[diagnostics="+ diagnostics+ "]",- si.toString());+ si.toStringVerbose());+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());// diagnostics X, attributes Osi =@@ -236,7 +238,8 @@ public class TestSegmentInfos extends LuceneTestCase {+ ":[attributes="+ attributes+ "]",- si.toString());+ si.toStringVerbose());+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());// diagnostics O, attributes Osi =@@ -265,7 +268,8 @@ public class TestSegmentInfos extends LuceneTestCase {+ ":[attributes="+ attributes+ "]",- si.toString());+ si.toStringVerbose());+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());dir.close();}