Apache Lucene open-source search software
Fix WindowsFS to support Windows (#14627)
Uwe Schindlerccc8f7e245e7
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex e06262af574..22d8551bada 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -121,6 +121,10 @@ Other* GITHUB#14516: Move sloppySin into SloppyMath from GeoUtils (Ankit Jain)+* GITHUB#14627, GITHUB#11920: Fix mock filesystem WindowsFS to also work on Windows.+ This is required to make tests pass on Windows 11 which no longer has+ all limitations of previous Windows versions. (Uwe Schindler)+======================= Lucene 10.2.1 =======================diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.javaindex e6490108071..0072dfdde71 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java@@ -89,7 +89,6 @@ import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.store.LockObtainFailedException;-import org.apache.lucene.store.MMapDirectory;import org.apache.lucene.store.NIOFSDirectory;import org.apache.lucene.store.NoLockFactory;import org.apache.lucene.store.SimpleFSLockFactory;@@ -108,7 +107,6 @@ import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.Constants;import org.apache.lucene.util.IOSupplier;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.InfoStream;@@ -1243,15 +1241,9 @@ public class TestIndexWriter extends LuceneTestCase {WindowsFS provider = new WindowsFS(path.getFileSystem());Path indexPath = provider.wrapPath(path);- // NOTE: on Unix, we cannot use MMapDir, because WindowsFS doesn't see/think it keeps file- // handles open. Yet, on Windows, we MUST use MMapDir because the Windows OS will in fact- // prevent file deletion for us, and fails otherwise:- FSDirectory dir;- if (Constants.WINDOWS) {- dir = new MMapDirectory(indexPath);- } else {- dir = new NIOFSDirectory(indexPath);- }+ // NOTE: We cannot use MMapDir, because WindowsFS doesn't see/think it keeps file+ // handles open.+ FSDirectory dir = new NIOFSDirectory(indexPath);MergePolicy mergePolicy = newLogMergePolicy(true);@@ -2879,9 +2871,6 @@ public class TestIndexWriter extends LuceneTestCase {}public void testPendingDeleteDVGeneration() throws IOException {- // irony: currently we don't emulate windows well enough to work on windows!- assumeFalse("windows is not supported", Constants.WINDOWS);-Path path = createTempDir();// Use WindowsFS to prevent open files from being deleted:@@ -2947,9 +2936,6 @@ public class TestIndexWriter extends LuceneTestCase {}public void testPendingDeletionsRollbackWithReader() throws IOException {- // irony: currently we don't emulate Windows well enough to work on Windows!- assumeFalse("Windows is not supported", Constants.WINDOWS);-Path path = createTempDir();// Use WindowsFS to prevent open files from being deleted:@@ -2986,9 +2972,6 @@ public class TestIndexWriter extends LuceneTestCase {}public void testWithPendingDeletions() throws Exception {- // irony: currently we don't emulate Windows well enough to work on Windows!- assumeFalse("Windows is not supported", Constants.WINDOWS);-Path path = createTempDir();// Use WindowsFS to prevent open files from being deleted:@@ -3038,8 +3021,6 @@ public class TestIndexWriter extends LuceneTestCase {public void testPendingDeletesAlreadyWrittenFiles() throws IOException {Path path = createTempDir();- // irony: currently we don't emulate Windows well enough to work on Windows!- assumeFalse("Windows is not supported", Constants.WINDOWS);// Use WindowsFS to prevent open files from being deleted:WindowsFS provider = new WindowsFS(path.getFileSystem());diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.javaindex aff4fe2e273..bc9ec12774e 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java@@ -36,7 +36,6 @@ import org.apache.lucene.tests.mockfile.WindowsFS;import org.apache.lucene.tests.store.BaseDirectoryTestCase;import org.apache.lucene.tests.store.MockDirectoryWrapper;import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.Constants;public class TestFileSwitchDirectory extends BaseDirectoryTestCase {@@ -172,7 +171,6 @@ public class TestFileSwitchDirectory extends BaseDirectoryTestCase {public void testDeleteAndList() throws IOException {// relies on windows semanticsPath path = createTempDir();- assumeFalse("Irony we seem to not emulate windows well enough", Constants.WINDOWS);WindowsFS provider = new WindowsFS(path.getFileSystem());Path indexPath = provider.wrapPath(path);try (final FileSwitchDirectory dir =diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestHardLinkCopyDirectoryWrapper.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestHardLinkCopyDirectoryWrapper.javaindex 2e7c8927fa2..937912f4b8b 100644--- a/lucene/misc/src/test/org/apache/lucene/misc/store/TestHardLinkCopyDirectoryWrapper.java+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/TestHardLinkCopyDirectoryWrapper.java@@ -33,7 +33,6 @@ import org.apache.lucene.store.IndexOutput;import org.apache.lucene.store.NIOFSDirectory;import org.apache.lucene.tests.mockfile.WindowsFS;import org.apache.lucene.tests.store.BaseDirectoryTestCase;-import org.apache.lucene.util.Constants;import org.apache.lucene.util.IOUtils;// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows@@ -104,8 +103,6 @@ public class TestHardLinkCopyDirectoryWrapper extends BaseDirectoryTestCase {}public void testRenameWithHardLink() throws Exception {- // irony: currently we don't emulate windows well enough to work on windows!- assumeFalse("windows is not supported", Constants.WINDOWS);Path path = createTempDir();WindowsFS provider = new WindowsFS(path.getFileSystem());Directory dir1 = new NIOFSDirectory(provider.wrapPath(path));diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsFS.java b/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsFS.javaindex 0dd8f7f2ba6..efa08af253a 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsFS.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsFS.java@@ -21,10 +21,10 @@ import java.nio.file.CopyOption;import java.nio.file.FileSystem;import java.nio.file.Files;import java.nio.file.Path;-import java.nio.file.attribute.BasicFileAttributeView;import java.nio.file.attribute.BasicFileAttributes;import java.util.HashMap;import java.util.Map;+import java.util.Optional;/*** FileSystem that (imperfectly) acts like windows.@@ -55,10 +55,10 @@ public class WindowsFS extends HandleTrackingFS {/** Returns file "key" (e.g. inode) for the specified path */private Object getKey(Path existing) throws IOException {- BasicFileAttributeView view =- Files.getFileAttributeView(existing, BasicFileAttributeView.class);- BasicFileAttributes attributes = view.readAttributes();- return attributes.fileKey();+ // the key may be null, e.g. on real Windows!+ // in that case we fallback to the file path as key.+ return Optional.ofNullable(Files.readAttributes(existing, BasicFileAttributes.class).fileKey())+ .orElse(existing);}@Overridediff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsPath.java b/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsPath.javaindex 5ee16cd58a3..5c13475a08f 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsPath.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/mockfile/WindowsPath.java@@ -18,20 +18,17 @@ package org.apache.lucene.tests.mockfile;import java.nio.file.InvalidPathException;import java.nio.file.Path;-import java.util.Arrays;-import java.util.HashSet;+import java.util.Set;class WindowsPath extends FilterPath {- static final HashSet<Character> RESERVED_CHARACTERS =- new HashSet<>(Arrays.asList('<', '>', ':', '\"', '\\', '|', '?', '*'));+ static final Set<Character> RESERVED_CHARACTERS =+ Set.of('<', '>', ':', '\"', '\\', '|', '?', '*');- static final HashSet<String> RESERVED_NAMES =- new HashSet<>(- Arrays.asList(- "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7",- "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8",- "LPT9"));+ static final Set<String> RESERVED_NAMES =+ Set.of(+ "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7",+ "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9");WindowsPath(Path path, FilterFileSystem fileSystem) {super(path, fileSystem);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleTemporaryFilesCleanup.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleTemporaryFilesCleanup.javaindex 6d1b6ad1f3a..517fcdb99cb 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleTemporaryFilesCleanup.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleTemporaryFilesCleanup.java@@ -43,7 +43,6 @@ import org.apache.lucene.tests.mockfile.WindowsFS;import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;import org.apache.lucene.tests.util.LuceneTestCase.SuppressFsync;import org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks;-import org.apache.lucene.util.Constants;import org.apache.lucene.util.IOUtils;/**@@ -157,11 +156,8 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {fs = new HandleLimitFS(fs, limit).getFileSystem(null);}// windows is currently slow- if (random.nextInt(10) == 0) {- // don't try to emulate windows on windows: they don't get along- if (!Constants.WINDOWS && allowed(avoid, WindowsFS.class)) {- fs = new WindowsFS(fs).getFileSystem(null);- }+ if (random.nextInt(10) == 0 && allowed(avoid, WindowsFS.class)) {+ fs = new WindowsFS(fs).getFileSystem(null);}if (allowed(avoid, ExtrasFS.class)) {fs = new ExtrasFS(fs, random.nextInt(4) == 0, random.nextBoolean()).getFileSystem(null);diff --git a/lucene/test-framework/src/test/org/apache/lucene/tests/mockfile/TestWindowsFS.java b/lucene/test-framework/src/test/org/apache/lucene/tests/mockfile/TestWindowsFS.javaindex 9c3f53cdede..48141b98e6f 100644--- a/lucene/test-framework/src/test/org/apache/lucene/tests/mockfile/TestWindowsFS.java+++ b/lucene/test-framework/src/test/org/apache/lucene/tests/mockfile/TestWindowsFS.java@@ -29,6 +29,7 @@ import java.nio.file.StandardCopyOption;import java.util.Random;import java.util.concurrent.CyclicBarrier;import java.util.concurrent.atomic.AtomicBoolean;+import java.util.function.Predicate;import org.apache.lucene.util.Constants;/** Basic tests for WindowsFS */@@ -37,11 +38,6 @@ public class TestWindowsFS extends MockFileSystemTestCase {@Overridepublic void setUp() throws Exception {super.setUp();- // irony: currently we don't emulate windows well enough to work on windows!- // TODO: Can we fork this class and create a new class with only those tests that can run on- // Windows and then check if- // the Lucene WindowsFS error is same as the one OG Windows throws- assumeFalse("windows is not supported", Constants.WINDOWS);}@Override@@ -188,8 +184,12 @@ public class TestWindowsFS extends MockFileSystemTestCase {}public void testFileName() {- Character[] reservedCharacters = WindowsPath.RESERVED_CHARACTERS.toArray(new Character[0]);- String[] reservedNames = WindowsPath.RESERVED_NAMES.toArray(new String[0]);+ Character[] reservedCharacters =+ WindowsPath.RESERVED_CHARACTERS.stream()+ .filter(Predicate.not(Character.valueOf('\\')::equals))+ .sorted()+ .toArray(Character[]::new);+ String[] reservedNames = WindowsPath.RESERVED_NAMES.stream().sorted().toArray(String[]::new);String fileName;Random r = random();Path dir = wrap(createTempDir());@@ -211,7 +211,11 @@ public class TestWindowsFS extends MockFileSystemTestCase {expectThrows(InvalidPathException.class, () -> dir.resolve("foo:bar:tar"));expectThrows(InvalidPathException.class, () -> dir.resolve("foo?bar"));expectThrows(InvalidPathException.class, () -> dir.resolve("foo<bar"));- expectThrows(InvalidPathException.class, () -> dir.resolve("foo\\bar"));+ if (!Constants.WINDOWS) {+ // we need to exclude that test on Windows, because the backslash is resolved before our code+ // checks the filename:+ expectThrows(InvalidPathException.class, () -> dir.resolve("foo\\bar"));+ }expectThrows(InvalidPathException.class, () -> dir.resolve("foo*bar|tar"));expectThrows(InvalidPathException.class, () -> dir.resolve("foo|bar?tar"));}