Skip to content

Apache Lucene open-source search software

Make Directory#copyFrom abstract, route FilterDirectory through createOutput (#16530)

Tim Allison7fbd268187fb
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0bedecf21b4..79eda724d4b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -111,6 +111,12 @@ API Changes
* GITHUB#16518: Changes the signature of ScorerSupplier#cost to throw an IOException. (Ignacio Vera)
+* GITHUB#16530: Directory#copyFrom is now abstract so that implementations choose copy behavior
+ explicitly. BaseDirectory provides the standard routing implementation via the new protected
+ final copyThroughCreateOutput helper. FilterDirectory#copyFrom now routes through createOutput
+ so that per-file bookkeeping in subclasses applies to copied files; subclasses that want an
+ optimized copy (e.g. HardlinkCopyDirectoryWrapper) may override to delegate. (Tim Allison)
+
New Features
---------------------
* GITHUB#16418: Incremental doc-values updates. A set-only doc-values update (NUMERIC or BINARY) is written as a sparse
diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md
index 75a260d3aa8..378f6611bb8 100644
--- a/lucene/MIGRATE.md
+++ b/lucene/MIGRATE.md
@@ -48,6 +48,19 @@ called `LuceneTestCaseParent` but you should reference them either
without an explicit type or via the type of the parent class
for your test framework. The parent class may be removed in the future.
+### Directory#copyFrom is now abstract (GITHUB#16530)
+
+`Directory#copyFrom` no longer has a default implementation, so classes that extend
+`Directory` directly must now implement it. Subclasses of `BaseDirectory` inherit the
+standard implementation, which routes the copy through `createOutput`; it is also
+available to subclasses as the protected final helper `Directory#copyThroughCreateOutput`.
+
+`FilterDirectory#copyFrom` no longer delegates to the wrapped directory's `copyFrom`.
+It instead routes through `createOutput`, so filter directories that override
+`createOutput` for per-file bookkeeping now see copied files as well. Filter directories
+that relied on the wrapped directory's optimized copy should override `copyFrom` and
+delegate explicitly, as `HardlinkCopyDirectoryWrapper` does.
+
### Relaxed Index Upgrade Policy (GITHUB#13797)
Starting with Lucene 11.0.0, the index upgrade policy has been relaxed to allow safe upgrades across multiple major version numbers without reindexing when no format breaks occur.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.java b/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.java
index 362b5b3880f..419e33a3306 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/CompoundDirectory.java
@@ -65,6 +65,12 @@ public abstract class CompoundDirectory extends Directory {
@Override
public final void syncMetaData() {}
+ @Override
+ public final void copyFrom(Directory from, String src, String dest, IOContext context)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public final IndexOutput createOutput(String name, IOContext context) throws IOException {
throw new UnsupportedOperationException();
diff --git a/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.java
index 06272ba6b4e..e1dbf6168a0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TrackingTmpOutputDirectoryWrapper.java
@@ -25,6 +25,7 @@ import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
final class TrackingTmpOutputDirectoryWrapper extends FilterDirectory {
private final Map<String, String> fileNames = new HashMap<>();
@@ -47,6 +48,23 @@ final class TrackingTmpOutputDirectoryWrapper extends FilterDirectory {
return super.openInput(tmpName, context);
}
+ @Override
+ public void copyFrom(Directory from, String src, String dest, IOContext context)
+ throws IOException {
+ // the inherited failure cleanup would delete dest, but createOutput() redirects dest to a
+ // temp file; on failure remove the mapping and delete the temp file instead
+ try (IndexInput is = from.openInput(src, IOContext.READONCE);
+ IndexOutput os = createOutput(dest, context)) {
+ os.copyBytes(is, is.length());
+ } catch (Throwable t) {
+ String tmpName = fileNames.remove(dest);
+ if (tmpName != null) {
+ IOUtils.deleteFilesSuppressingExceptions(t, in, tmpName);
+ }
+ throw t;
+ }
+ }
+
public Map<String, String> getTemporaryFiles() {
return fileNames;
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
index 21e87835268..5b00f51948d 100644
--- a/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java
@@ -51,6 +51,12 @@ public abstract class BaseDirectory extends Directory {
}
}
+ @Override
+ public void copyFrom(Directory from, String src, String dest, IOContext context)
+ throws IOException {
+ copyThroughCreateOutput(from, src, dest, context);
+ }
+
@Override
public String toString() {
return super.toString() + " lockFactory=" + lockFactory;
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index 80236f5c186..f9c1a01033e 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -173,9 +173,21 @@ public abstract class Directory implements Closeable {
/**
* Copies an existing {@code src} file from directory {@code from} to a non-existent file {@code
* dest} in this directory. The given IOContext is only used for opening the destination file.
+ *
+ * <p>Any per-file bookkeeping performed by {@link #createOutput} must also cover copied files;
+ * most implementations should delegate to {@link #copyThroughCreateOutput}.
+ */
+ public abstract void copyFrom(Directory from, String src, String dest, IOContext context)
+ throws IOException;
+
+ /**
+ * Copies {@code src} from {@code from} to {@code dest} in this directory via {@link
+ * #createOutput}. On failure, deletes the file named {@code dest} from {@code this} — subclasses
+ * whose {@link #createOutput} maps {@code dest} to a different physical file must override {@link
+ * #copyFrom} and clean up that file instead.
*/
- public void copyFrom(Directory from, String src, String dest, IOContext context)
- throws IOException {
+ protected final void copyThroughCreateOutput(
+ Directory from, String src, String dest, IOContext context) throws IOException {
try (IndexInput is = from.openInput(src, IOContext.READONCE);
IndexOutput os = createOutput(dest, context)) {
os.copyBytes(is, is.length());
diff --git a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
index 76ff9cfca44..562e3edd205 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
@@ -237,6 +237,12 @@ public class FileSwitchDirectory extends Directory {
return getDirectory(name).openInput(name, context);
}
+ @Override
+ public void copyFrom(Directory from, String src, String dest, IOContext context)
+ throws IOException {
+ getDirectory(dest).copyFrom(from, src, dest, context);
+ }
+
@Override
public Set<String> getPendingDeletions() throws IOException {
Set<String> primaryDeletions = primaryDir.getPendingDeletions();
diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
index 3d5c1a5f51b..c3571ba9b76 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
@@ -116,10 +116,16 @@ public abstract class FilterDirectory extends Directory {
return getClass().getSimpleName() + "(" + in.toString() + ")";
}
+ /**
+ * Not delegated to {@link #in}: routes through {@link #createOutput} so per-file bookkeeping
+ * there covers copied files too. Subclasses that don't override {@link #createOutput} and want an
+ * optimized copy (e.g. {@code HardlinkCopyDirectoryWrapper}) may override to delegate: {@code
+ * in.copyFrom(from, src, dest, context)}.
+ */
@Override
public void copyFrom(Directory from, String src, String dest, IOContext context)
throws IOException {
- in.copyFrom(from, src, dest, context);
+ copyThroughCreateOutput(from, src, dest, context);
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTrackingTmpOutputDirectoryWrapper.java b/lucene/core/src/test/org/apache/lucene/index/TestTrackingTmpOutputDirectoryWrapper.java
new file mode 100644
index 00000000000..8ddc262a920
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTrackingTmpOutputDirectoryWrapper.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestTrackingTmpOutputDirectoryWrapper extends LuceneTestCase {
+
+ public void testCopyFromTracksLogicalNameInTemporaryFiles() throws IOException {
+ try (Directory source = newDirectory();
+ Directory backing = newDirectory()) {
+ try (IndexOutput out = source.createOutput("src", IOContext.DEFAULT)) {
+ out.writeBytes(new byte[] {1, 2, 3}, 3);
+ }
+ TrackingTmpOutputDirectoryWrapper wrapper = new TrackingTmpOutputDirectoryWrapper(backing);
+ wrapper.copyFrom(source, "src", "dest", IOContext.DEFAULT);
+
+ assertTrue(
+ "copyFrom must register dest in getTemporaryFiles() via createOutput()",
+ wrapper.getTemporaryFiles().containsKey("dest"));
+ String tmpName = wrapper.getTemporaryFiles().get("dest");
+ assertNotEquals("temp file name must differ from logical name", "dest", tmpName);
+ try (IndexInput in = wrapper.openInput("dest", IOContext.DEFAULT)) {
+ assertEquals(3L, in.length());
+ }
+ }
+ }
+
+ public void testCopyFromCleansUpOnFailure() throws IOException {
+ try (Directory source = newDirectory();
+ Directory backing = newDirectory()) {
+ try (IndexOutput out = source.createOutput("src", IOContext.DEFAULT)) {
+ out.writeBytes(new byte[] {1, 2, 3}, 3);
+ }
+ // Source whose reads always throw, so copyBytes fails after createOutput succeeds.
+ Directory failingSource =
+ new FilterDirectory(source) {
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ return new FilterIndexInput("failing:" + name, super.openInput(name, context)) {
+ @Override
+ public void readBytes(byte[] b, int offset, int len) throws IOException {
+ throw new IOException("simulated read failure");
+ }
+ };
+ }
+ };
+
+ TrackingTmpOutputDirectoryWrapper wrapper = new TrackingTmpOutputDirectoryWrapper(backing);
+ expectThrows(
+ IOException.class,
+ () -> wrapper.copyFrom(failingSource, "src", "dest", IOContext.DEFAULT));
+
+ assertFalse(
+ "dest must not remain in getTemporaryFiles() after failed copyFrom",
+ wrapper.getTemporaryFiles().containsKey("dest"));
+ assertEquals(
+ "temp file must be deleted from backing dir after failed copyFrom",
+ 0,
+ backing.listAll().length);
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
index 9eb987fb729..7b9708902a6 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
@@ -42,25 +42,25 @@ public class TestFilterDirectory extends BaseDirectoryTestCase {
}
}
- public void testCopyFromDelegates() throws IOException {
+ public void testCopyFromRoutesThroughCreateOutput() throws IOException {
try (Directory srcDir = new ByteBuffersDirectory();
Directory destDir = new ByteBuffersDirectory()) {
try (IndexOutput out = srcDir.createOutput("test.txt", IOContext.DEFAULT)) {
out.writeString("hello");
}
- boolean[] delegated = {false};
- FilterDirectory wrappedDestDir =
+ Set<String> created = new HashSet<>();
+ FilterDirectory filterDestDir =
new FilterDirectory(destDir) {
@Override
- public void copyFrom(Directory from, String src, String dest, IOContext context)
- throws IOException {
- delegated[0] = true;
- in.copyFrom(from, src, dest, context);
+ public IndexOutput createOutput(String name, IOContext context) throws IOException {
+ created.add(name);
+ return in.createOutput(name, context);
}
};
- FilterDirectory filterDestDir = new FilterDirectory(wrappedDestDir) {};
filterDestDir.copyFrom(srcDir, "test.txt", "copied.txt", IOContext.DEFAULT);
- assertTrue("copyFrom should delegate to wrapped directory", delegated[0]);
+ assertTrue(
+ "copyFrom should route through the wrapper's createOutput",
+ created.contains("copied.txt"));
assertTrue(slowFileExists(destDir, "copied.txt"));
try (IndexInput in = destDir.openInput("copied.txt", IOContext.DEFAULT)) {
assertEquals("hello", in.readString());
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.java
index 70c2d959b49..295f16bd552 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/TestByteWritesTrackingDirectoryWrapper.java
@@ -105,6 +105,20 @@ public class TestByteWritesTrackingDirectoryWrapper extends BaseDirectoryTestCas
assertEquals(expectedMergeBytes, dir.getMergedBytes());
}
+ public void testCopyFromTracksBytes() throws Exception {
+ ByteWritesTrackingDirectoryWrapper dir =
+ new ByteWritesTrackingDirectoryWrapper(new ByteBuffersDirectory());
+ int expectedMergeBytes = 1 + random().nextInt(100);
+ try (Directory srcDir = new ByteBuffersDirectory()) {
+ try (IndexOutput out = srcDir.createOutput("src", IOContext.DEFAULT)) {
+ out.writeBytes(new byte[expectedMergeBytes], expectedMergeBytes);
+ }
+ dir.copyFrom(
+ srcDir, "src", "dest", IOContext.merge(new MergeInfo(10, expectedMergeBytes, false, 2)));
+ assertEquals(expectedMergeBytes, dir.getMergedBytes());
+ }
+ }
+
@Override
protected Directory getDirectory(Path path) throws IOException {
return new ByteWritesTrackingDirectoryWrapper(new ByteBuffersDirectory());
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectoryCopyFrom.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectoryCopyFrom.java
new file mode 100644
index 00000000000..e7291834192
--- /dev/null
+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectoryCopyFrom.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.misc.store;
+
+import java.util.HashSet;
+import java.util.OptionalLong;
+import java.util.Set;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MergeInfo;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+/** Tests {@link DirectIODirectory#copyFrom} without requiring actual O_DIRECT support. */
+public class TestDirectIODirectoryCopyFrom extends LuceneTestCase {
+
+ public void testCopyFromConsultsUseDirectIO() throws Exception {
+ Set<String> consulted = new HashSet<>();
+ try (Directory srcDir = FSDirectory.open(createTempDir("src"));
+ DirectIODirectory dir =
+ new DirectIODirectory(FSDirectory.open(createTempDir("dest"))) {
+ @Override
+ protected boolean useDirectIO(
+ String name, IOContext context, OptionalLong fileLength) {
+ consulted.add(name);
+ return false;
+ }
+ }) {
+ try (IndexOutput out = srcDir.createOutput("src", IOContext.DEFAULT)) {
+ out.writeBytes(new byte[8], 8);
+ }
+ dir.copyFrom(srcDir, "src", "dest", IOContext.merge(new MergeInfo(10, 8, false, 2)));
+ assertTrue(
+ "copyFrom must route through createOutput/useDirectIO", consulted.contains("dest"));
+ }
+ }
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.java
index a94412f942d..50084539f5c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseMergePolicyTestCase.java
@@ -302,6 +302,11 @@ public abstract class BaseMergePolicyTestCase extends LuceneTestCase {
throw new UnsupportedOperationException();
}
+ @Override
+ public void copyFrom(Directory from, String src, String dest, IOContext context) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
throw new UnsupportedOperationException();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
index 0037c02f926..ee37b234d13 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
@@ -1120,8 +1120,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
}
}
- // don't override optional methods like copyFrom: we need the default impl for things like disk
- // full checks. we randomly exercise "raw" directories anyway. We ensure default impls are used:
+ // Pin the FilterDirectory default impls below so subclasses can't accidentally override them;
+ // copyFrom routes through createOutput, which is where disk-full simulation happens.
@Override
public final ChecksumIndexInput openChecksumInput(String name) throws IOException {