Apache Lucene open-source search software
Make Directory#copyFrom abstract, route FilterDirectory through createOutput (#16530)
Tim Allison7fbd268187fb
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 0bedecf21b4..79eda724d4b 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -111,6 +111,12 @@ API Changes* GITHUB#16518: Changes the signature of ScorerSupplier#cost to throw an IOException. (Ignacio Vera)+* GITHUB#16530: Directory#copyFrom is now abstract so that implementations choose copy behavior+ explicitly. BaseDirectory provides the standard routing implementation via the new protected+ final copyThroughCreateOutput helper. FilterDirectory#copyFrom now routes through createOutput+ so that per-file bookkeeping in subclasses applies to copied files; subclasses that want an+ optimized copy (e.g. HardlinkCopyDirectoryWrapper) may override to delegate. (Tim Allison)+New Features---------------------* GITHUB#16418: Incremental doc-values updates. A set-only doc-values update (NUMERIC or BINARY) is written as a sparsediff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.mdindex 75a260d3aa8..378f6611bb8 100644--- a/lucene/MIGRATE.md+++ b/lucene/MIGRATE.md@@ -48,6 +48,19 @@ called `LuceneTestCaseParent` but you should reference them eitherwithout an explicit type or via the type of the parent classfor your test framework. The parent class may be removed in the future.+### Directory#copyFrom is now abstract (GITHUB#16530)++`Directory#copyFrom` no longer has a default implementation, so classes that extend+`Directory` directly must now implement it. Subclasses of `BaseDirectory` inherit the+standard implementation, which routes the copy through `createOutput`; it is also+available to subclasses as the protected final helper `Directory#copyThroughCreateOutput`.++`FilterDirectory#copyFrom` no longer delegates to the wrapped directory's `copyFrom`.+It instead routes through `createOutput`, so filter directories that override+`createOutput` for per-file bookkeeping now see copied files as well. Filter directories+that relied on the wrapped directory's optimized copy should override `copyFrom` and+delegate explicitly, as `HardlinkCopyDirectoryWrapper` does.+### Relaxed Index Upgrade Policy (GITHUB#13797)Starting with Lucene 11.0.0, the index upgrade policy has been relaxed to allow safe upgrades across multiple major version numbers without reindexing when no format breaks occur.diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.java b/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.javaindex 362b5b3880f..419e33a3306 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.java@@ -65,6 +65,12 @@ public abstract class CompoundDirectory extends Directory {@Overridepublic final void syncMetaData() {}+ @Override+ public final void copyFrom(Directory from, String src, String dest, IOContext context)+ throws IOException {+ throw new UnsupportedOperationException();+ }+@Overridepublic final IndexOutput createOutput(String name, IOContext context) throws IOException {throw new UnsupportedOperationException();diff --git a/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.javaindex 06272ba6b4e..e1dbf6168a0 100644--- a/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.java+++ b/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.java@@ -25,6 +25,7 @@ import org.apache.lucene.store.FilterDirectory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.IOUtils;final class TrackingTmpOutputDirectoryWrapper extends FilterDirectory {private final Map<String, String> fileNames = new HashMap<>();@@ -47,6 +48,23 @@ final class TrackingTmpOutputDirectoryWrapper extends FilterDirectory {return super.openInput(tmpName, context);}+ @Override+ public void copyFrom(Directory from, String src, String dest, IOContext context)+ throws IOException {+ // the inherited failure cleanup would delete dest, but createOutput() redirects dest to a+ // temp file; on failure remove the mapping and delete the temp file instead+ try (IndexInput is = from.openInput(src, IOContext.READONCE);+ IndexOutput os = createOutput(dest, context)) {+ os.copyBytes(is, is.length());+ } catch (Throwable t) {+ String tmpName = fileNames.remove(dest);+ if (tmpName != null) {+ IOUtils.deleteFilesSuppressingExceptions(t, in, tmpName);+ }+ throw t;+ }+ }+public Map<String, String> getTemporaryFiles() {return fileNames;}diff --git a/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.javaindex 21e87835268..5b00f51948d 100644--- a/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java+++ b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java@@ -51,6 +51,12 @@ public abstract class BaseDirectory extends Directory {}}+ @Override+ public void copyFrom(Directory from, String src, String dest, IOContext context)+ throws IOException {+ copyThroughCreateOutput(from, src, dest, context);+ }+@Overridepublic String toString() {return super.toString() + " lockFactory=" + lockFactory;diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.javaindex 80236f5c186..f9c1a01033e 100644--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java@@ -173,9 +173,21 @@ public abstract class Directory implements Closeable {/*** Copies an existing {@code src} file from directory {@code from} to a non-existent file {@code* dest} in this directory. The given IOContext is only used for opening the destination file.+ *+ * <p>Any per-file bookkeeping performed by {@link #createOutput} must also cover copied files;+ * most implementations should delegate to {@link #copyThroughCreateOutput}.+ */+ public abstract void copyFrom(Directory from, String src, String dest, IOContext context)+ throws IOException;++ /**+ * Copies {@code src} from {@code from} to {@code dest} in this directory via {@link+ * #createOutput}. On failure, deletes the file named {@code dest} from {@code this} — subclasses+ * whose {@link #createOutput} maps {@code dest} to a different physical file must override {@link+ * #copyFrom} and clean up that file instead.*/- public void copyFrom(Directory from, String src, String dest, IOContext context)- throws IOException {+ protected final void copyThroughCreateOutput(+ Directory from, String src, String dest, IOContext context) throws IOException {try (IndexInput is = from.openInput(src, IOContext.READONCE);IndexOutput os = createOutput(dest, context)) {os.copyBytes(is, is.length());diff --git a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.javaindex 76ff9cfca44..562e3edd205 100644--- a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java+++ b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java@@ -237,6 +237,12 @@ public class FileSwitchDirectory extends Directory {return getDirectory(name).openInput(name, context);}+ @Override+ public void copyFrom(Directory from, String src, String dest, IOContext context)+ throws IOException {+ getDirectory(dest).copyFrom(from, src, dest, context);+ }+@Overridepublic Set<String> getPendingDeletions() throws IOException {Set<String> primaryDeletions = primaryDir.getPendingDeletions();diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.javaindex 3d5c1a5f51b..c3571ba9b76 100644--- a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java+++ b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java@@ -116,10 +116,16 @@ public abstract class FilterDirectory extends Directory {return getClass().getSimpleName() + "(" + in.toString() + ")";}+ /**+ * Not delegated to {@link #in}: routes through {@link #createOutput} so per-file bookkeeping+ * there covers copied files too. Subclasses that don't override {@link #createOutput} and want an+ * optimized copy (e.g. {@code HardlinkCopyDirectoryWrapper}) may override to delegate: {@code+ * in.copyFrom(from, src, dest, context)}.+ */@Overridepublic void copyFrom(Directory from, String src, String dest, IOContext context)throws IOException {- in.copyFrom(from, src, dest, context);+ copyThroughCreateOutput(from, src, dest, context);}@Overridediff --git a/lucene/core/src/test/org/apache/lucene/index/TestTrackingTmpOutputDirectoryWrapper.java b/lucene/core/src/test/org/apache/lucene/index/TestTrackingTmpOutputDirectoryWrapper.javanew file mode 100644index 00000000000..8ddc262a920--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/index/TestTrackingTmpOutputDirectoryWrapper.java@@ -0,0 +1,85 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.index;++import java.io.IOException;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FilterDirectory;+import org.apache.lucene.store.FilterIndexInput;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;++public class TestTrackingTmpOutputDirectoryWrapper extends LuceneTestCase {++ public void testCopyFromTracksLogicalNameInTemporaryFiles() throws IOException {+ try (Directory source = newDirectory();+ Directory backing = newDirectory()) {+ try (IndexOutput out = source.createOutput("src", IOContext.DEFAULT)) {+ out.writeBytes(new byte[] {1, 2, 3}, 3);+ }+ TrackingTmpOutputDirectoryWrapper wrapper = new TrackingTmpOutputDirectoryWrapper(backing);+ wrapper.copyFrom(source, "src", "dest", IOContext.DEFAULT);++ assertTrue(+ "copyFrom must register dest in getTemporaryFiles() via createOutput()",+ wrapper.getTemporaryFiles().containsKey("dest"));+ String tmpName = wrapper.getTemporaryFiles().get("dest");+ assertNotEquals("temp file name must differ from logical name", "dest", tmpName);+ try (IndexInput in = wrapper.openInput("dest", IOContext.DEFAULT)) {+ assertEquals(3L, in.length());+ }+ }+ }++ public void testCopyFromCleansUpOnFailure() throws IOException {+ try (Directory source = newDirectory();+ Directory backing = newDirectory()) {+ try (IndexOutput out = source.createOutput("src", IOContext.DEFAULT)) {+ out.writeBytes(new byte[] {1, 2, 3}, 3);+ }+ // Source whose reads always throw, so copyBytes fails after createOutput succeeds.+ Directory failingSource =+ new FilterDirectory(source) {+ @Override+ public IndexInput openInput(String name, IOContext context) throws IOException {+ return new FilterIndexInput("failing:" + name, super.openInput(name, context)) {+ @Override+ public void readBytes(byte[] b, int offset, int len) throws IOException {+ throw new IOException("simulated read failure");+ }+ };+ }+ };++ TrackingTmpOutputDirectoryWrapper wrapper = new TrackingTmpOutputDirectoryWrapper(backing);+ expectThrows(+ IOException.class,+ () -> wrapper.copyFrom(failingSource, "src", "dest", IOContext.DEFAULT));++ assertFalse(+ "dest must not remain in getTemporaryFiles() after failed copyFrom",+ wrapper.getTemporaryFiles().containsKey("dest"));+ assertEquals(+ "temp file must be deleted from backing dir after failed copyFrom",+ 0,+ backing.listAll().length);+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.javaindex 9eb987fb729..7b9708902a6 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java@@ -42,25 +42,25 @@ public class TestFilterDirectory extends BaseDirectoryTestCase {}}- public void testCopyFromDelegates() throws IOException {+ public void testCopyFromRoutesThroughCreateOutput() throws IOException {try (Directory srcDir = new ByteBuffersDirectory();Directory destDir = new ByteBuffersDirectory()) {try (IndexOutput out = srcDir.createOutput("test.txt", IOContext.DEFAULT)) {out.writeString("hello");}- boolean[] delegated = {false};- FilterDirectory wrappedDestDir =+ Set<String> created = new HashSet<>();+ FilterDirectory filterDestDir =new FilterDirectory(destDir) {@Override- public void copyFrom(Directory from, String src, String dest, IOContext context)- throws IOException {- delegated[0] = true;- in.copyFrom(from, src, dest, context);+ public IndexOutput createOutput(String name, IOContext context) throws IOException {+ created.add(name);+ return in.createOutput(name, context);}};- FilterDirectory filterDestDir = new FilterDirectory(wrappedDestDir) {};filterDestDir.copyFrom(srcDir, "test.txt", "copied.txt", IOContext.DEFAULT);- assertTrue("copyFrom should delegate to wrapped directory", delegated[0]);+ assertTrue(+ "copyFrom should route through the wrapper's createOutput",+ created.contains("copied.txt"));assertTrue(slowFileExists(destDir, "copied.txt"));try (IndexInput in = destDir.openInput("copied.txt", IOContext.DEFAULT)) {assertEquals("hello", in.readString());diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.javaindex 70c2d959b49..295f16bd552 100644--- a/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.java+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.java@@ -105,6 +105,20 @@ public class TestByteWritesTrackingDirectoryWrapper extends BaseDirectoryTestCasassertEquals(expectedMergeBytes, dir.getMergedBytes());}+ public void testCopyFromTracksBytes() throws Exception {+ ByteWritesTrackingDirectoryWrapper dir =+ new ByteWritesTrackingDirectoryWrapper(new ByteBuffersDirectory());+ int expectedMergeBytes = 1 + random().nextInt(100);+ try (Directory srcDir = new ByteBuffersDirectory()) {+ try (IndexOutput out = srcDir.createOutput("src", IOContext.DEFAULT)) {+ out.writeBytes(new byte[expectedMergeBytes], expectedMergeBytes);+ }+ dir.copyFrom(+ srcDir, "src", "dest", IOContext.merge(new MergeInfo(10, expectedMergeBytes, false, 2)));+ assertEquals(expectedMergeBytes, dir.getMergedBytes());+ }+ }+@Overrideprotected Directory getDirectory(Path path) throws IOException {return new ByteWritesTrackingDirectoryWrapper(new ByteBuffersDirectory());diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectoryCopyFrom.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectoryCopyFrom.javanew file mode 100644index 00000000000..e7291834192--- /dev/null+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectoryCopyFrom.java@@ -0,0 +1,52 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.misc.store;++import java.util.HashSet;+import java.util.OptionalLong;+import java.util.Set;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FSDirectory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.store.MergeInfo;+import org.apache.lucene.tests.util.LuceneTestCase;++/** Tests {@link DirectIODirectory#copyFrom} without requiring actual O_DIRECT support. */+public class TestDirectIODirectoryCopyFrom extends LuceneTestCase {++ public void testCopyFromConsultsUseDirectIO() throws Exception {+ Set<String> consulted = new HashSet<>();+ try (Directory srcDir = FSDirectory.open(createTempDir("src"));+ DirectIODirectory dir =+ new DirectIODirectory(FSDirectory.open(createTempDir("dest"))) {+ @Override+ protected boolean useDirectIO(+ String name, IOContext context, OptionalLong fileLength) {+ consulted.add(name);+ return false;+ }+ }) {+ try (IndexOutput out = srcDir.createOutput("src", IOContext.DEFAULT)) {+ out.writeBytes(new byte[8], 8);+ }+ dir.copyFrom(srcDir, "src", "dest", IOContext.merge(new MergeInfo(10, 8, false, 2)));+ assertTrue(+ "copyFrom must route through createOutput/useDirectIO", consulted.contains("dest"));+ }+ }+}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.javaindex a94412f942d..50084539f5c 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.java@@ -302,6 +302,11 @@ public abstract class BaseMergePolicyTestCase extends LuceneTestCase {throw new UnsupportedOperationException();}+ @Override+ public void copyFrom(Directory from, String src, String dest, IOContext context) {+ throw new UnsupportedOperationException();+ }+@Overridepublic IndexInput openInput(String name, IOContext context) throws IOException {throw new UnsupportedOperationException();diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.javaindex 0037c02f926..ee37b234d13 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java@@ -1120,8 +1120,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {}}- // don't override optional methods like copyFrom: we need the default impl for things like disk- // full checks. we randomly exercise "raw" directories anyway. We ensure default impls are used:+ // Pin the FilterDirectory default impls below so subclasses can't accidentally override them;+ // copyFrom routes through createOutput, which is where disk-full simulation happens.@Overridepublic final ChecksumIndexInput openChecksumInput(String name) throws IOException {