Apache Solr open-source search software
Compute RAM usage ByteBuffersDataOutput on the fly. (#1919)
Adrien Grandc3f97fbdc11c
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.javaindex ac1dc7579e5..7711302c5e4 100644--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java@@ -117,6 +117,9 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountab*/private final ArrayDeque<ByteBuffer> blocks = new ArrayDeque<>();+ /** Cumulative RAM usage across all blocks. */+ private long ramBytesUsed;+/*** The current-or-next write block.*/@@ -400,13 +403,8 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountabpublic long ramBytesUsed() {// Return a rough estimation for allocated blocks. Note that we do not make// any special distinction for direct memory buffers.- ByteBuffer first = blocks.peek();- if (first == null) {- return 0L;- } else {- // All blocks have the same capacity.- return (first.capacity() + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * blocks.size();- }+ assert ramBytesUsed == blocks.stream().mapToLong(ByteBuffer::capacity).sum() + blocks.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;+ return ramBytesUsed;}/**@@ -422,6 +420,7 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountabblocks.forEach(blockReuse);}blocks.clear();+ ramBytesUsed = 0;currentBlock = EMPTY;}@@ -455,6 +454,7 @@ public final class ByteBuffersDataOutput extends DataOutput implements AccountabcurrentBlock = blockAllocate.apply(requiredBlockSize);assert currentBlock.capacity() == requiredBlockSize;blocks.add(currentBlock);+ ramBytesUsed += RamUsageEstimator.NUM_BYTES_OBJECT_REF + currentBlock.capacity();}private void rewriteToBlockSize(int targetBlockBits) {@@ -476,6 +476,7 @@ public final class ByteBuffersDataOutput extends DataOutput implements Accountabassert blocks.isEmpty();this.blockBits = targetBlockBits;blocks.addAll(cloned.blocks);+ ramBytesUsed = cloned.ramBytesUsed;}private static int computeBlockSizeBitsFor(long bytes) {diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.javaindex 8a9012acc48..d67e9de0267 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java@@ -20,10 +20,12 @@ import static org.junit.Assert.*;import java.io.IOException;import java.nio.ByteBuffer;+import java.util.List;import java.util.Random;import java.util.concurrent.atomic.AtomicInteger;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.LuceneTestCase;+import org.apache.lucene.util.RamUsageEstimator;import org.junit.Assert;import org.junit.Test;@@ -159,4 +161,44 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<ByteassertTrue(bb.hasArray()); // heap-based by default, so array should be there.}}++ @Test+ public void testRamBytesUsed() {+ ByteBuffersDataOutput out = new ByteBuffersDataOutput();+ // Empty output requires no RAM+ assertEquals(0, out.ramBytesUsed());++ // Non-empty buffer requires RAM+ out.writeInt(4);+ assertEquals(out.ramBytesUsed(), computeRamBytesUsed(out));++ // Make sure this keeps working with multiple backing buffers+ while (out.toBufferList().size() < 2) {+ out.writeLong(42);+ }+ assertEquals(out.ramBytesUsed(), computeRamBytesUsed(out));++ // Make sure this keeps working when increasing the block size+ int currentBlockCapacity = out.toBufferList().get(0).capacity();+ do {+ out.writeLong(42);+ } while (out.toBufferList().get(0).capacity() == currentBlockCapacity);+ assertEquals(out.ramBytesUsed(), computeRamBytesUsed(out));++ // Back to zero after a clear+ out.reset();+ assertEquals(0, out.ramBytesUsed());++ // And back to non-empty+ out.writeInt(4);+ assertEquals(out.ramBytesUsed(), computeRamBytesUsed(out));+ }++ private static long computeRamBytesUsed(ByteBuffersDataOutput out) {+ if (out.size() == 0) {+ return 0;+ }+ List<ByteBuffer> buffers = out.toBufferList();+ return buffers.stream().mapToLong(ByteBuffer::capacity).sum() + buffers.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;+ }}