Skip to content

Apache Lucene open-source search software

Streamline segment info logged by infostream, esp. when merging (#15885)

Michael Sokolov03fc503a49c1
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4aaf30e7287..6fa4c282674 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -338,6 +338,8 @@ Other
* GITHUB#15867: Use IllegalCallerException to signal wrong caller in internal classes
(VectorizationProvider, TestSecrets). (Uwe Schindler)
+* GITHUB#15885: Slenderize SegmentInfo.toString() and use beefy version in infostream only once
+
* GITHUB#15891: Add the ability to pass DataAccessHints in Lucene104ScalarQuantizedVectorsReader (Navneet Verma)
======================= Lucene 10.4.0 =======================
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 4ab42e74587..30ff07c3932 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1047,7 +1047,6 @@ public class IndexWriter
// Must clone because we don't want the incoming NRT reader to "see" any changes this writer
// now makes:
segmentInfos = reader.segmentInfos.clone();
-
SegmentInfos lastCommit;
try {
lastCommit = SegmentInfos.readCommit(directoryOrig, segmentInfos.getSegmentsFileName());
@@ -1111,7 +1110,9 @@ public class IndexWriter
rollbackSegments = segmentInfos.createBackupSegmentInfos();
}
-
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "init " + segmentInfos.toStringVerbose());
+ }
commitUserData = new HashMap<>(segmentInfos.getUserData()).entrySet();
pendingNumDocs.set(segmentInfos.totalMaxDoc());
@@ -2844,7 +2845,7 @@ public class IndexWriter
ensureOpen(false);
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "publishFlushedSegment " + newSegment);
+ infoStream.message("IW", "publishFlushedSegment " + newSegment.toStringVerbose());
}
if (globalPacket != null && globalPacket.any()) {
@@ -3343,6 +3344,13 @@ public class IndexWriter
try {
writer.addIndexesReaderMerge(merge);
success = true;
+ if (infoStream != null && infoStream.isEnabled("IW")) {
+ if (merge.info == null) {
+ infoStream.message("IW", "dropped 100% deleted segment");
+ } else {
+ infoStream.message("IW", "merged new segment " + merge.info.toStringVerbose());
+ }
+ }
} catch (Throwable t) {
handleMergeException(t, merge);
} finally {
@@ -4775,6 +4783,7 @@ public class IndexWriter
if (merge.info != null && merge.isAborted() == false) {
if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "merged new segment " + merge.info.toStringVerbose());
infoStream.message(
"IW",
"merge time "
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
index 08222107d2d..ed101ecda9e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
@@ -349,7 +349,21 @@ public class SegmentCommitInfo {
/** Returns a description of this segment. */
public String toString(int pendingDelCount) {
- String s = info.toString(delCount + pendingDelCount);
+ return info.toString(delCount + pendingDelCount);
+ }
+
+ /** Returns a description of this segment. */
+ @Override
+ public String toString() {
+ return info.toString();
+ }
+
+ /** Returns a more verbose description of this segment for use in infostream logging */
+ String toStringVerbose() {
+ String s = info.toStringVerbose();
+ if (this.id != null) {
+ s += " :id=" + StringHelper.idToString(id);
+ }
if (delGen != -1) {
s += ":delGen=" + delGen;
}
@@ -362,18 +376,9 @@ public class SegmentCommitInfo {
if (softDelCount > 0) {
s += " :softDel=" + softDelCount;
}
- if (this.id != null) {
- s += " :id=" + StringHelper.idToString(id);
- }
-
return s;
}
- @Override
- public String toString() {
- return toString(0);
- }
-
@Override
public SegmentCommitInfo clone() {
SegmentCommitInfo other =
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
index b85a6cb9e63..95f9ef9601b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
@@ -211,31 +211,41 @@ public final class SegmentInfo {
return Collections.unmodifiableSet(setFiles);
}
- @Override
- public String toString() {
- return toString(0);
- }
-
/**
* Used for debugging. Format may suddenly change.
*
- * <p>Current format looks like <code>_a(3.1):c45/4:[sorter=&lt;long: "timestamp"&gt;!]</code>,
- * which means the segment's name is <code>_a</code>; it was created with Lucene 3.1 (or '?' if
- * it's unknown); it's using compound file format (would be <code>C</code> if not compound); it
- * has 45 documents; it has 4 deletions (this part is left off when there are no deletions); it is
- * sorted by the timestamp field in descending order (this part is omitted for unsorted segments).
+ * <p>Current format looks like <code>_a(3.1):c45/4</code>, which means the segment's name is
+ * <code>_a</code>; it was created with Lucene 3.1 (or '?' if it's unknown); it's using compound
+ * file format (would be <code>C</code> if not compound); it has 45 documents; it has 4 deletions
+ * (this part is left off when there are no deletions).
+ *
+ * <p>There is also {@link #toStringVerbose()} that includes more info about the segment. It is be
+ * used to print detailed info when the segment is first written or read.
*/
public String toString(int delCount) {
StringBuilder s = new StringBuilder();
s.append(name).append('(').append(version == null ? "?" : version).append(')').append(':');
char cfs = getUseCompoundFile() ? 'c' : 'C';
s.append(cfs);
-
s.append(maxDoc);
-
if (delCount != 0) {
s.append('/').append(delCount);
}
+ return s.toString();
+ }
+
+ @Override
+ public String toString() {
+ return toString(0);
+ }
+
+ /**
+ * Logged by InfoStream when the segment is first flushed. Format may change. Appends more
+ * information to the basic toString().
+ */
+ String toStringVerbose() {
+ StringBuilder s = new StringBuilder();
+ s.append(toString(0));
if (indexSort != null) {
s.append(":[indexSort=");
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
index 62639d9faed..79077902eb8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
@@ -985,6 +985,14 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
/** Returns readable description of this segment. */
@Override
public String toString() {
+ return toString(false);
+ }
+
+ String toStringVerbose() {
+ return toString(true);
+ }
+
+ String toString(boolean verbose) {
StringBuilder buffer = new StringBuilder();
buffer.append(getSegmentsFileName()).append(": ");
final int count = size();
@@ -992,8 +1000,11 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
if (i > 0) {
buffer.append(' ');
}
- final SegmentCommitInfo info = info(i);
- buffer.append(info.toString(0));
+ if (verbose) {
+ buffer.append(info(i).toStringVerbose());
+ } else {
+ buffer.append(info(i).toString(0));
+ }
}
return buffer.toString();
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java b/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java
index 80681ee126b..27b426e416e 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java
@@ -17,6 +17,10 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.Directory;
@@ -86,4 +90,64 @@ public class TestInfoStream extends LuceneTestCase {
dir.close();
assertTrue(seenTestPoint.get());
}
+
+ public void testMergeSmallIndex() throws Exception {
+ // examine info stream output from IW, MP, MS
+ IndexWriterConfig iwc = new IndexWriterConfig(null);
+ Set<String> components = Set.of("IW", "MP", "MS", "BD", "BU");
+ List<String> infoStream = new ArrayList<>();
+ iwc.setInfoStream(
+ new InfoStream() {
+ @Override
+ public void close() throws IOException {}
+
+ @Override
+ public void message(String component, String message) {
+ if (components.contains(component)) {
+ infoStream.add(String.format(Locale.ROOT, "[%s] %s\n", component, message));
+ }
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ return components.contains(component);
+ }
+ });
+ try (Directory dir = newDirectory();
+ IndexWriter iw = new IndexWriter(dir, iwc)) {
+ iw.addDocument(new Document());
+ iw.commit();
+ iw.addDocument(new Document());
+ iw.addDocument(new Document());
+ try (IndexReader reader = DirectoryReader.open(iw)) {
+ assertTrue(iw.tryDeleteDocument(reader, 2) > 0);
+ }
+ iw.commit();
+ iw.forceMerge(1);
+ }
+ boolean foundInit = false;
+ int flushedCount = 0;
+ int mergedCount = 0;
+ for (String message : infoStream) {
+ // we don't want to be printing full diagnostics every time a segment is mentioned in the log,
+ // only when it is first flushed
+ if (message.contains("diagnostics")) {
+ if (message.startsWith("[IW] publishFlushedSegment")) {
+ flushedCount++;
+ } else if (message.startsWith("[IW] merged new segment")) {
+ mergedCount++;
+ } else {
+ fail("message contains diagnostics: " + message);
+ }
+ }
+ if (message.contains("init segments")) {
+ assertEquals("[IW] init segments: \n", message);
+ foundInit = true;
+ }
+ // System.out.print(message);
+ }
+ assertTrue("init message not found", foundInit);
+ assertEquals(2, flushedCount);
+ assertEquals(1, mergedCount);
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
index 0743715ebb1..76439967b70 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
@@ -184,7 +184,8 @@ public class TestSegmentInfos extends LuceneTestCase {
Sort.INDEXORDER);
assertEquals(
"TEST(" + Version.LATEST.toString() + ")" + ":C10000" + ":[indexSort=<doc>]",
- si.toString());
+ si.toStringVerbose());
+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());
// diagnostics O, attributes X
si =
@@ -210,7 +211,8 @@ public class TestSegmentInfos extends LuceneTestCase {
+ ":[diagnostics="
+ diagnostics
+ "]",
- si.toString());
+ si.toStringVerbose());
+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());
// diagnostics X, attributes O
si =
@@ -236,7 +238,8 @@ public class TestSegmentInfos extends LuceneTestCase {
+ ":[attributes="
+ attributes
+ "]",
- si.toString());
+ si.toStringVerbose());
+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());
// diagnostics O, attributes O
si =
@@ -265,7 +268,8 @@ public class TestSegmentInfos extends LuceneTestCase {
+ ":[attributes="
+ attributes
+ "]",
- si.toString());
+ si.toStringVerbose());
+ assertEquals("TEST(" + Version.LATEST.toString() + ")" + ":C10000", si.toString());
dir.close();
}