Apache Lucene open-source search software
Avoid a blocking direct read when creating a DirectIODirectory slice (#16619)
John Maurice1f5d30938bf1
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 4121b695d7c..e77c3057b65 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -212,6 +212,9 @@ Optimizations* GITHUB#12324: ELiminate redundant work in AbstractKnnVectorQuery.rewrite (Mike Sokolov)+* GITHUB#16619: DirectIODirectory's slice() no longer performs a blocking direct read when+ creating a slice; the first read resolves the deferred start position instead. (John Maurice)+Bug Fixes---------------------* GITHUB#16553: Fix stored-fields corruption when PerField#finish throws: the stored-fields framediff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java b/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.javaindex 94723546b6c..f76bda39eac 100644--- a/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java+++ b/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java@@ -300,7 +300,7 @@ public class DirectIODirectory extends FilterDirectory {}}- private static final class DirectIOIndexInput extends IndexInput {+ static final class DirectIOIndexInput extends IndexInput { // package-private, visible for testprivate final ByteBuffer buffer;private final FileChannel channel;private final int blockSize;@@ -310,6 +310,17 @@ public class DirectIODirectory extends FilterDirectory {private boolean isOpen;private long filePos;+ /**+ * Where a lazily positioned clone/slice starts, until its first fill: -1 when nothing is+ * pending, otherwise the number of bytes from the start of the block at {@code filePos ++ * buffer.capacity()} to this input's position 0. Consumed by the first {@link #refill(int)};+ * dropped by a repositioning {@link #seek(long)} or {@link #seekInternal(long)}. The sentinel+ * is -1 rather than 0 because 0 is a legal pending value (a block-aligned start), and such a+ * slice must still take the first-fill path so that its EOF check uses the pending value rather+ * than the length of the caller's read.+ */+ private int pendingDelta = -1;+/*** Creates a new instance of DirectIOIndexInput for reading index input with direct IO bypassing* OS buffer@@ -344,10 +355,19 @@ public class DirectIODirectory extends FilterDirectory {this.isClosable = false;this.length = length;this.offset = offset;- this.filePos = -bufferSize;+ // Remember the start position instead of reading it now; the first refill() resolves it, so+ // slice() does no I/O even when offset is not block-aligned.+ final long alignedStart = offset - (offset % this.blockSize);+ this.filePos = alignedStart - bufferSize;+ this.pendingDelta = (int) (offset - alignedStart);buffer.limit(0);}+ // visible for test: true until the first fill+ boolean isDeferred() {+ return pendingDelta >= 0;+ }+private static ByteBuffer allocateBuffer(int bufferSize, int blockSize) {return ByteBuffer.allocateDirect(bufferSize + blockSize - 1).alignedSlice(blockSize)@@ -370,7 +390,9 @@ public class DirectIODirectory extends FilterDirectory {// refill) first,// will result in negative value equal to bufferSize being returned,// due to the initialization method filePos = -bufferSize used in constructor.- assert filePointer == -buffer.capacity() - offset || filePointer >= 0+ // Before its first fill a lazily positioned clone/slice is further offset by its pending+ // delta, because its filePos starts at alignedStart - bufferSize rather than -bufferSize.+ assert filePointer == -buffer.capacity() - Math.max(pendingDelta, 0) || filePointer >= 0: "filePointer should either be initial value equal to negative buffer capacity, or larger than or equal to 0";return Math.max(filePointer, 0);}@@ -378,6 +400,15 @@ public class DirectIODirectory extends FilterDirectory {@Overridepublic void seek(long pos) throws IOException {if (pos != getFilePointer()) {+ if (pendingDelta >= 0) {+ // A slice that has not been read yet fills first, so the window check below sees the+ // same buffer the eager code had: a seek past this input's length that lands in that+ // block repositions within it instead of failing, as it always has.+ final int parked = pendingDelta;+ pendingDelta = -1;+ fill(parked);+ buffer.position(parked);+ }final long absolutePos = pos + offset;if (absolutePos >= filePos && absolutePos <= filePos + buffer.limit()) {// the new position is within the existing buffer@@ -390,6 +421,10 @@ public class DirectIODirectory extends FilterDirectory {}private void seekInternal(long pos) throws IOException {+ // An explicit seek replaces a pending start, and its refill must check EOF against this+ // position rather than the pending one. Only clone() gets here with a pending start: it+ // positions a fresh clone at the parent's file pointer.+ pendingDelta = -1;final long absPos = pos + offset;final long alignedPos = absPos - (absPos % blockSize);filePos = alignedPos - buffer.capacity();@@ -441,6 +476,23 @@ public class DirectIODirectory extends FilterDirectory {}private void refill(int bytesToRead) throws IOException {+ final int parked = pendingDelta;+ if (parked >= 0) {+ // First fill of a slice with a pending start: fill and position exactly as the constructor+ // used to, so the EOF check uses the pending offset and an over-long first read behaves as+ // before. If that leaves nothing to read (a zero-length slice at the very end of the file),+ // fall through so the caller's own read throws EOFException as it always did.+ pendingDelta = -1;+ fill(parked);+ buffer.position(parked);+ if (buffer.hasRemaining()) {+ return;+ }+ }+ fill(bytesToRead);+ }++ private void fill(int bytesToRead) throws IOException {filePos += buffer.capacity();// BaseDirectoryTestCase#testSeekPastEOF test for consecutive read past EOF,@@ -553,9 +605,9 @@ public class DirectIODirectory extends FilterDirectory {throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + this);}- var slice = new DirectIOIndexInput(sliceDescription, this, this.offset + offset, length);- slice.seekInternal(0L);- return slice;+ // The constructor remembers the start position and the first read resolves it, so+ // constructing a slice performs no I/O.+ return new DirectIOIndexInput(sliceDescription, this, this.offset + offset, length);}}}diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.javaindex 32f375cd28f..6b533bca257 100644--- a/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.java+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.java@@ -16,6 +16,8 @@*/package org.apache.lucene.misc.store;+import static org.hamcrest.Matchers.greaterThanOrEqualTo;+import com.carrotsearch.randomizedtesting.RandomizedTest;import java.io.EOFException;import java.io.IOException;@@ -38,6 +40,7 @@ import org.apache.lucene.store.IndexOutput;import org.apache.lucene.store.MergeInfo;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.store.BaseDirectoryTestCase;+import org.apache.lucene.util.ArrayUtil;import org.junit.BeforeClass;public class TestDirectIODirectory extends BaseDirectoryTestCase {@@ -218,6 +221,263 @@ public class TestDirectIODirectory extends BaseDirectoryTestCase {}}+ private static byte[] writeRandomFile(Directory dir, String name, int size) throws IOException {+ byte[] bytes = new byte[size];+ random().nextBytes(bytes);+ try (IndexOutput o = dir.createOutput(name, IOContext.DEFAULT)) {+ o.writeBytes(bytes, 0, size);+ }+ return bytes;+ }++ /**+ * Slice offsets to exercise: the start of the file, a few unaligned offsets inside the first+ * block (the shape of a codec header), two block starts and an unaligned offset after one, the+ * last four bytes of the file, plus random offsets. Offsets stop at {@code fileSize - 2} so every+ * slice is at least two bytes long, which the correctness matrix relies on for its inner-slice+ * and clone cases.+ */+ private static long[] sliceOffsets(int blockSize, int fileSize) {+ assert fileSize >= 4 * blockSize;+ long[] fixed = {0, 1, 7, 96, blockSize, blockSize + 188, 2L * blockSize, fileSize - 4};+ long[] offsets = ArrayUtil.growExact(fixed, fixed.length + atLeast(3));+ for (int i = fixed.length; i < offsets.length; i++) {+ offsets[i] = RandomizedTest.randomLongBetween(0, fileSize - 2);+ }+ return offsets;+ }++ public void testSliceDefersIOAtEveryOffset() throws Exception {+ Path path = createTempDir("testSliceDefersIOAtEveryOffset");+ final int blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize());+ final int fileSize = 4 * blockSize;+ try (Directory dir = getDirectory(path)) {+ byte[] bytes = writeRandomFile(dir, "out", fileSize);+ final long[] offsets = sliceOffsets(blockSize, fileSize);+ try (IndexInput in = dir.openInput("out", IOContext.DEFAULT)) {+ for (long offset : offsets) {+ IndexInput slice = in.slice("slice@" + offset, offset, fileSize - offset);+ DirectIODirectory.DirectIOIndexInput directSlice =+ (DirectIODirectory.DirectIOIndexInput) slice;+ assertTrue(+ "slice at offset " + offset + " did not defer its first fill",+ directSlice.isDeferred());+ assertEquals(0L, slice.getFilePointer());+ assertEquals(+ "first byte of slice at offset " + offset, bytes[(int) offset], slice.readByte());+ assertFalse(directSlice.isDeferred());+ }+ }+ }+ }++ public void testSliceCorrectnessMatrix() throws Exception {+ Path path = createTempDir("testSliceCorrectnessMatrix");+ final int blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize());+ final int fileSize = 4 * blockSize;+ try (Directory dir = getDirectory(path)) {+ byte[] bytes = writeRandomFile(dir, "out", fileSize);+ final long[] offsets = sliceOffsets(blockSize, fileSize);+ try (IndexInput in = dir.openInput("out", IOContext.DEFAULT)) {+ for (long offset : offsets) {+ final long[] lengths = {1, 4, blockSize - 7, blockSize, blockSize + 3, fileSize - offset};+ for (long length : lengths) {+ if (length <= 0 || length > fileSize - offset) {+ continue;+ }+ final int len = (int) length;+ // full read-through of the slice equals the same range of the parent+ IndexInput slice = in.slice("s", offset, length);+ byte[] actual = new byte[len];+ slice.readBytes(actual, 0, len);+ assertArrayEquals(+ "offset " + offset + " length " + length,+ ArrayUtil.copyOfSubArray(bytes, (int) offset, (int) offset + len),+ actual);+ if (len >= 2) {+ // slice-of-slice at a non-zero inner offset+ IndexInput inner = in.slice("s", offset, length).slice("ss", 1, length - 1);+ byte[] innerBytes = new byte[len - 1];+ inner.readBytes(innerBytes, 0, len - 1);+ assertArrayEquals(+ "inner slice at offset " + offset + " length " + length,+ ArrayUtil.copyOfSubArray(bytes, (int) offset + 1, (int) offset + len),+ innerBytes);+ // clone-after-partial-read starts at the parent's position+ IndexInput partial = in.slice("s", offset, length);+ partial.readBytes(new byte[len / 2], 0, len / 2);+ IndexInput clone = partial.clone();+ assertEquals(len / 2, clone.getFilePointer());+ byte[] rest = new byte[len - len / 2];+ clone.readBytes(rest, 0, rest.length);+ assertArrayEquals(+ "clone at offset " + offset + " length " + length,+ ArrayUtil.copyOfSubArray(bytes, (int) offset + len / 2, (int) offset + len),+ rest);+ }+ }+ }+ }+ }+ }++ public void testSeekResolvesFreshParkedSlice() throws Exception {+ Path path = createTempDir("testSeekResolvesFreshParkedSlice");+ final int blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize());+ final int fileSize = DirectIODirectory.DEFAULT_MERGE_BUFFER_SIZE + 3 * blockSize;+ try (Directory dir = getDirectory(path)) {+ byte[] bytes = writeRandomFile(dir, "out", fileSize);+ final long[] offsets = {1, 96, blockSize + 188};+ try (IndexInput in = dir.openInput("out", IOContext.DEFAULT)) {+ for (long offset : offsets) {+ final long length =+ Math.min(DirectIODirectory.DEFAULT_MERGE_BUFFER_SIZE, fileSize - offset);+ IndexInput slice = in.slice("s", offset, length);+ // seek() on a slice that has not been read yet must position it correctly: the byte+ // read is the slice's own last byte, not one relative to the wrong base.+ slice.seek(length - 1);+ assertEquals("offset " + offset, bytes[(int) (offset + length - 1)], slice.readByte());+ assertEquals(length, slice.getFilePointer());+ }+ }+ }+ }++ public void testCloneOfPartiallyReadUnalignedSliceResolvesToItsPosition() throws Exception {+ // clone() positions a fresh clone at the parent's file pointer through seekInternal, and is+ // the only caller that reaches it with a pending start. If seekInternal did not drop that+ // pending start first, a clone taken after a partial read of a short unaligned slice would+ // read the wrong bytes or throw EOFException.+ Path path = createTempDir("testCloneOfPartiallyReadSlice");+ final int blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize());+ final int fileSize = 4 * blockSize;+ try (Directory dir = getDirectory(path)) {+ byte[] bytes = writeRandomFile(dir, "out", fileSize);+ final long[] offsets = {1, 96, blockSize + 188, 3L * blockSize - 4};+ try (IndexInput in = dir.openInput("out", IOContext.DEFAULT)) {+ for (long offset : offsets) {+ final long length = Math.min(blockSize - 7, fileSize - offset);+ IndexInput partial = in.slice("s", offset, length);+ final int half = (int) length / 2;+ partial.readBytes(new byte[half], 0, half);+ IndexInput clone = partial.clone();+ assertEquals("offset " + offset, half, clone.getFilePointer());+ byte[] rest = new byte[(int) length - half];+ clone.readBytes(rest, 0, rest.length);+ assertArrayEquals(+ "offset " + offset,+ ArrayUtil.copyOfSubArray(bytes, (int) offset + half, (int) offset + (int) length),+ rest);+ }+ }+ }+ }++ /**+ * A characterization test, not an endorsement. Today a read past a slice's own length is served+ * from the buffered block as long as it stays inside the window, and only a read that runs off+ * the end of the window throws {@link EOFException}. That quirk predates this change; the lazy+ * first fill must reproduce it exactly rather than "fix" it in passing. If the quirk is ever+ * removed on purpose, this test should change with it.+ */+ public void testFirstFillEofSemanticsMatchEagerFill() throws Exception {+ Path path = createTempDir("testFirstFillEofSemantics");+ final int blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize());+ final int fileSize = 4 * blockSize;+ // the oracle below relies on the whole file fitting in one buffer window+ assumeTrue(+ "file must fit in one buffer window",+ fileSize <= DirectIODirectory.DEFAULT_MERGE_BUFFER_SIZE);+ try (Directory dir = getDirectory(path)) {+ byte[] bytes = writeRandomFile(dir, "out", fileSize);+ final long[] offsets = {0, 96, blockSize, fileSize - 5};+ final long[] lengths = {0, 1, 4, blockSize};+ try (IndexInput in = dir.openInput("out", IOContext.DEFAULT)) {+ for (long offset : offsets) {+ for (long l : lengths) {+ final long length = Math.min(l, fileSize - offset);+ final int[] readLens = {1, 4, (int) length + 2, blockSize + 32, fileSize};+ for (int readLen : readLens) {+ IndexInput slice = in.slice("s", offset, length);+ final String cell = "offset=" + offset + " length=" + length + " readLen=" + readLen;+ // if the first fill ever checked EOF against the caller's read length instead of+ // the pending offset, the in-window cells below would fail+ if (readLen <= fileSize - offset) {+ byte[] dst = new byte[readLen];+ slice.readBytes(dst, 0, readLen);+ assertArrayEquals(+ cell,+ ArrayUtil.copyOfSubArray(bytes, (int) offset, (int) offset + readLen),+ dst);+ } else {+ expectThrows(+ EOFException.class, cell, () -> slice.readBytes(new byte[readLen], 0, readLen));+ }+ }+ }+ }+ }+ }+ }++ public void testZeroLengthSliceAtFileEndThrowsEOF() throws Exception {+ Path path = createTempDir("testZeroLengthSliceAtFileEnd");+ final int blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize());+ // one file whose length is a multiple of the block size (pending offset 0) and one whose+ // length is not+ final int[] fileSizes = {2 * blockSize, 2 * blockSize + 300};+ try (Directory dir = getDirectory(path)) {+ for (int fileSize : fileSizes) {+ String name = "out" + fileSize;+ writeRandomFile(dir, name, fileSize);+ try (IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {+ IndexInput slice = in.slice("end", in.length(), 0);+ // the exception type matters: a first fill that finds nothing to read must fall+ // through to the caller's own read and throw EOFException, not a+ // BufferUnderflowException from an empty buffer+ expectThrows(EOFException.class, "file size " + fileSize, slice::readByte);+ }+ }+ }+ }++ public void testSeekBeyondSliceLengthMatchesEagerBehaviour() throws Exception {+ Path path = createTempDir("testSeekBeyondSliceLength");+ final int blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize());+ final int fileSize = 4 * blockSize;+ assumeTrue(+ "file must fit in one buffer window",+ fileSize <= DirectIODirectory.DEFAULT_MERGE_BUFFER_SIZE);+ try (Directory dir = getDirectory(path)) {+ byte[] bytes = writeRandomFile(dir, "out", fileSize);+ try (IndexInput in = dir.openInput("out", IOContext.DEFAULT)) {+ // seek targets inside the block the eager code buffered at construction: today the+ // seek silently repositions and the read serves the byte beyond the slice's end. A+ // fresh lazy slice must do the same rather than throw EOFException, and+ // getFilePointer() must return normally afterwards (assertions are enabled)+ final long[] offsets = {blockSize, blockSize + 188};+ final long[] seekTargets = {10, blockSize + 50L};+ for (long offset : offsets) {+ for (long target : seekTargets) {+ IndexInput slice = in.slice("s", offset, 4);+ slice.seek(target);+ assertEquals(+ "offset " + offset + " target " + target,+ bytes[(int) (offset + target)],+ slice.readByte());+ assertEquals(target + 1, slice.getFilePointer());+ }+ }+ // a target outside the buffered window: the refill checks EOF against the slice's own+ // end and throws, as today; getFilePointer() must still return normally afterwards+ IndexInput far = in.slice("s", blockSize + 188, 4);+ expectThrows(+ EOFException.class, () -> far.seek(DirectIODirectory.DEFAULT_MERGE_BUFFER_SIZE + 100L));+ assertThat(far.getFilePointer(), greaterThanOrEqualTo(0L));+ }+ }+ }+// Ping-pong seeks should be really fast, since the position should be within buffer.// The test should complete within sub-second times, not minutes.public void testSeekSmall() throws IOException {