Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9535: Commit DWPT bytes used before locking indexing (#1918)

Simon Willnauerc258905bd01f
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
index d368e88c323..8b652827849 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
@@ -145,20 +145,6 @@ final class DocumentsWriterFlushControl implements Accountable, Closeable {
return true;
}
- private synchronized void commitPerThreadBytes(DocumentsWriterPerThread perThread) {
- final long delta = perThread.commitLastBytesUsed();
- /*
- * We need to differentiate here if we are pending since setFlushPending
- * moves the perThread memory to the flushBytes and we could be set to
- * pending during a delete
- */
- if (perThread.isFlushPending()) {
- flushBytes += delta;
- } else {
- activeBytes += delta;
- }
- assert updatePeaks(delta);
- }
// only for asserts
private boolean updatePeaks(long delta) {
@@ -170,25 +156,42 @@ final class DocumentsWriterFlushControl implements Accountable, Closeable {
return true;
}
- synchronized DocumentsWriterPerThread doAfterDocument(DocumentsWriterPerThread perThread, boolean isUpdate) {
- try {
- commitPerThreadBytes(perThread);
- if (!perThread.isFlushPending()) {
- if (isUpdate) {
- flushPolicy.onUpdate(this, perThread);
+ DocumentsWriterPerThread doAfterDocument(DocumentsWriterPerThread perThread, boolean isUpdate) {
+ final long delta = perThread.getCommitLastBytesUsedDelta();
+ synchronized (this) {
+ // we need to commit this under lock but calculate it outside of the lock to minimize the time this lock is held
+ // per document. The reason we update this under lock is that we mark DWPTs as pending without acquiring it's
+ // lock in #setFlushPending and this also reads the committed bytes and modifies the flush/activeBytes.
+ // In the future we can clean this up to be more intuitive.
+ perThread.commitLastBytesUsed(delta);
+ try {
+ /*
+ * We need to differentiate here if we are pending since setFlushPending
+ * moves the perThread memory to the flushBytes and we could be set to
+ * pending during a delete
+ */
+ if (perThread.isFlushPending()) {
+ flushBytes += delta;
+ assert updatePeaks(delta);
} else {
- flushPolicy.onInsert(this, perThread);
- }
- if (!perThread.isFlushPending() && perThread.ramBytesUsed() > hardMaxBytesPerDWPT) {
- // Safety check to prevent a single DWPT exceeding its RAM limit. This
- // is super important since we can not address more than 2048 MB per DWPT
- setFlushPending(perThread);
+ activeBytes += delta;
+ assert updatePeaks(delta);
+ if (isUpdate) {
+ flushPolicy.onUpdate(this, perThread);
+ } else {
+ flushPolicy.onInsert(this, perThread);
+ }
+ if (!perThread.isFlushPending() && perThread.ramBytesUsed() > hardMaxBytesPerDWPT) {
+ // Safety check to prevent a single DWPT exceeding its RAM limit. This
+ // is super important since we can not address more than 2048 MB per DWPT
+ setFlushPending(perThread);
+ }
}
+ return checkout(perThread, false);
+ } finally {
+ boolean stalled = updateStallState();
+ assert assertNumDocsSinceStalled(stalled) && assertMemory();
}
- return checkout(perThread, false);
- } finally {
- boolean stalled = updateStallState();
- assert assertNumDocsSinceStalled(stalled) && assertMemory();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
index d8ae3754ec0..70cb04def0e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
@@ -540,12 +540,21 @@ final class DocumentsWriterPerThread implements Accountable {
/**
* Commits the current {@link #ramBytesUsed()} and stores it's value for later reuse.
* The last committed bytes used can be retrieved via {@link #getLastCommittedBytesUsed()}
+ */
+ void commitLastBytesUsed(long delta) {
+ assert isHeldByCurrentThread();
+ assert getCommitLastBytesUsedDelta() == delta : "delta has changed";
+ lastCommittedBytesUsed += delta;
+ }
+
+ /**
+ * Calculates the delta between the last committed bytes used and the currently used ram.
+ * @see #commitLastBytesUsed(long)
* @return the delta between the current {@link #ramBytesUsed()} and the current {@link #getLastCommittedBytesUsed()}
*/
- long commitLastBytesUsed() {
+ long getCommitLastBytesUsedDelta() {
assert isHeldByCurrentThread();
long delta = ramBytesUsed() - lastCommittedBytesUsed;
- lastCommittedBytesUsed += delta;
return delta;
}