Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9576: nuke SSD detection, modernize CMS defaults

Robert Muir85b58c262a75
diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy
index 35b3e8407b1..6b8b89f5f14 100644
--- a/gradle/testing/randomization/policies/solr-tests.policy
+++ b/gradle/testing/randomization/policies/solr-tests.policy
@@ -67,8 +67,6 @@ grant {
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
- // needed for test of IOUtils.spins (maybe it can be avoided)
- permission java.lang.RuntimePermission "getFileStoreAttributes";
// analyzers/uima: needed by lucene expressions' JavascriptCompiler
permission java.lang.RuntimePermission "createClassLoader";
// needed to test unmap hack on platforms that support it
diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy
index c71841149e6..c6f3f4b5359 100644
--- a/gradle/testing/randomization/policies/tests.policy
+++ b/gradle/testing/randomization/policies/tests.policy
@@ -50,8 +50,6 @@ grant {
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
- // needed for test of IOUtils.spins (maybe it can be avoided)
- permission java.lang.RuntimePermission "getFileStoreAttributes";
// analyzers/uima: needed by lucene expressions' JavascriptCompiler
permission java.lang.RuntimePermission "createClassLoader";
// needed to test unmap hack on platforms that support it
@@ -77,7 +75,6 @@ grant {
// CMS randomization
permission java.util.PropertyPermission "lucene.cms.override_core_count", "write";
- permission java.util.PropertyPermission "lucene.cms.override_spins", "write";
// used by nested tests? (e.g. TestLeaveFilesIfTestFails). TODO: look into this
permission java.util.PropertyPermission "tests.runnested", "write";
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 365a3e23512..122c3ed2fca 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -74,6 +74,14 @@ API Changes
Improvements
+* LUCENE-9576: Improve ConcurrentMergeScheduler settings by default, assuming modern I/O.
+ Previously Lucene was too conservative, jumping through hoops to detect if disks were SSD-backed.
+ In many common modern cases (VMs, RAID arrays, containers, encrypted mounts, non-Linux OS),
+ the pessimistic heuristics were wrong, resulting in slower indexing performance. Heuristics were
+ also complex and would trigger JDK issues even on unrelated mount points. Merge scheduler defaults
+ are now modernized and the heuristics removed. Users with spinning disks that want to maximize I/O
+ performance should tweak ConcurrentMergeScheduler. (Robert Muir)
+
* LUCENE-9463: Query match region retrieval component, passage scoring and formatting
for building custom highlighters. (Alan Woodward, Dawid Weiss)
diff --git a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
index cece58228d9..c9eb8d492fa 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
@@ -32,7 +32,6 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RateLimitedIndexOutput;
import org.apache.lucene.store.RateLimiter;
import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -52,25 +51,18 @@ import org.apache.lucene.util.ThreadInterruptedException;
* incoming threads by pausing until one more merges
* complete.</p>
*
- * <p>This class attempts to detect whether the index is
- * on rotational storage (traditional hard drive) or not
- * (e.g. solid-state disk) and changes the default max merge
- * and thread count accordingly. This detection is currently
- * Linux-only, and relies on the OS to put the right value
- * into /sys/block/&lt;dev&gt;/block/rotational. For all
- * other operating systems it currently assumes a rotational
- * disk for backwards compatibility. To enable default
- * settings for spinning or solid state disks for such
- * operating systems, use {@link #setDefaultMaxMergesAndThreads(boolean)}.
+ * <p>This class sets defaults based on Java's view of the
+ * cpu count, and it assumes a solid state disk (or similar).
+ * If you have a spinning disk and want to maximize performance,
+ * use {@link #setDefaultMaxMergesAndThreads(boolean)}.
*/
public class ConcurrentMergeScheduler extends MergeScheduler {
/** Dynamic default for {@code maxThreadCount} and {@code maxMergeCount},
- * used to detect whether the index is backed by an SSD or rotational disk and
- * set {@code maxThreadCount} accordingly. If it's an SSD,
- * {@code maxThreadCount} is set to {@code max(1, min(4, cpuCoreCount/2))},
- * otherwise 1. Note that detection only currently works on
- * Linux; other platforms will assume the index is not on an SSD. */
+ * based on CPU core count.
+ * {@code maxThreadCount} is set to {@code max(1, min(4, cpuCoreCount/2))}.
+ * {@code maxMergeCount} is set to {@code maxThreadCount + 5}.
+ */
public static final int AUTO_DETECT_MERGES_AND_THREADS = -1;
/** Used for testing.
@@ -78,11 +70,6 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
* @lucene.internal */
public static final String DEFAULT_CPU_CORE_COUNT_PROPERTY = "lucene.cms.override_core_count";
- /** Used for testing.
- *
- * @lucene.internal */
- public static final String DEFAULT_SPINS_PROPERTY = "lucene.cms.override_spins";
-
/** List of currently active {@link MergeThread}s. */
protected final List<MergeThread> mergeThreads = new ArrayList<>();
@@ -410,21 +397,9 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
private synchronized void initDynamicDefaults(Directory directory) throws IOException {
if (maxThreadCount == AUTO_DETECT_MERGES_AND_THREADS) {
- boolean spins = IOUtils.spins(directory);
-
- // Let tests override this to help reproducing a failure on a machine that has a different
- // core count than the one where the test originally failed:
- try {
- String value = System.getProperty(DEFAULT_SPINS_PROPERTY);
- if (value != null) {
- spins = Boolean.parseBoolean(value);
- }
- } catch (Exception ignored) {
- // that's fine we might hit a SecurityException etc. here just continue
- }
- setDefaultMaxMergesAndThreads(spins);
+ setDefaultMaxMergesAndThreads(false);
if (verbose()) {
- message("initDynamicDefaults spins=" + spins + " maxThreadCount=" + maxThreadCount + " maxMergeCount=" + maxMergeCount);
+ message("initDynamicDefaults maxThreadCount=" + maxThreadCount + " maxMergeCount=" + maxMergeCount);
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
index a417fdd8ace..fc3704489f1 100644
--- a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
+++ b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
@@ -27,8 +27,6 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
-import java.nio.file.DirectoryStream;
-import java.nio.file.FileStore;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
@@ -42,11 +40,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
-import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.FileSwitchDirectory;
-import org.apache.lucene.store.FilterDirectory;
/** This class emulates the new Java 7 "Try-With-Resources" statement.
* Remove once Lucene is on Java 7.
@@ -483,147 +477,6 @@ public final class IOUtils {
}
}
- /** If the dir is an {@link FSDirectory} or wraps one via possibly
- * nested {@link FilterDirectory} or {@link FileSwitchDirectory},
- * this returns {@link #spins(Path)} for the wrapped directory,
- * else, true.
- *
- * @throws IOException if {@code path} does not exist.
- *
- * @lucene.internal */
- public static boolean spins(Directory dir) throws IOException {
- dir = FilterDirectory.unwrap(dir);
- if (dir instanceof FileSwitchDirectory) {
- FileSwitchDirectory fsd = (FileSwitchDirectory) dir;
- // Spinning is contagious:
- return spins(fsd.getPrimaryDir()) || spins(fsd.getSecondaryDir());
- } else if (dir instanceof ByteBuffersDirectory) {
- return false;
- } else if (dir instanceof FSDirectory) {
- return spins(((FSDirectory) dir).getDirectory());
- } else {
- return true;
- }
- }
-
- /** Rough Linux-only heuristics to determine whether the provided
- * {@code Path} is backed by spinning storage. For example, this
- * returns false if the disk is a solid-state disk.
- *
- * @param path a location to check which must exist. the mount point will be determined from this location.
- * @return false if the storage is non-rotational (e.g. an SSD), or true if it is spinning or could not be determined
- * @throws IOException if {@code path} does not exist.
- *
- * @lucene.internal */
- public static boolean spins(Path path) throws IOException {
- // resolve symlinks (this will throw exception if the path does not exist)
- path = path.toRealPath();
-
- // Super cowboy approach, but seems to work!
- if (!Constants.LINUX) {
- return true; // no detection
- }
-
- try {
- return spinsLinux(path);
- } catch (Exception exc) {
- // our crazy heuristics can easily trigger SecurityException, AIOOBE, etc ...
- return true;
- }
- }
-
- // following methods are package-private for testing ONLY
-
- // note: requires a real or fake linux filesystem!
- static boolean spinsLinux(Path path) throws IOException {
- FileStore store = getFileStore(path);
-
- // if fs type is tmpfs, it doesn't spin.
- // this won't have a corresponding block device
- if ("tmpfs".equals(store.type())) {
- return false;
- }
-
- // get block device name
- String devName = store.name();
-
- // not a device (e.g. NFS server)
- if (!devName.startsWith("/")) {
- return true;
- }
-
- // resolve any symlinks to real block device (e.g. LVM)
- // /dev/sda0 -> sda0
- // /devices/XXX -> sda0
- devName = path.getRoot().resolve(devName).toRealPath().getFileName().toString();
-
- // now try to find the longest matching device folder in /sys/block
- // (that starts with our dev name):
- Path sysinfo = path.getRoot().resolve("sys").resolve("block");
- Path devsysinfo = null;
- int matchlen = 0;
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(sysinfo)) {
- for (Path device : stream) {
- String name = device.getFileName().toString();
- if (name.length() > matchlen && devName.startsWith(name)) {
- devsysinfo = device;
- matchlen = name.length();
- }
- }
- }
-
- if (devsysinfo == null) {
- return true; // give up
- }
-
- // read first byte from rotational, it's a 1 if it spins.
- Path rotational = devsysinfo.resolve("queue").resolve("rotational");
- try (InputStream stream = Files.newInputStream(rotational)) {
- return stream.read() == '1';
- }
- }
-
- // Files.getFileStore(Path) useless here!
- // don't complain, just try it yourself
- static FileStore getFileStore(Path path) throws IOException {
- FileStore store = Files.getFileStore(path);
- String mount = getMountPoint(store);
-
- // find the "matching" FileStore from system list, it's the one we want, but only return
- // that if it's unambiguous (only one matching):
- FileStore sameMountPoint = null;
- for (FileStore fs : path.getFileSystem().getFileStores()) {
- if (mount.equals(getMountPoint(fs))) {
- if (sameMountPoint == null) {
- sameMountPoint = fs;
- } else {
- // more than one filesystem has the same mount point; something is wrong!
- // fall back to crappy one we got from Files.getFileStore
- return store;
- }
- }
- }
-
- if (sameMountPoint != null) {
- // ok, we found only one, use it:
- return sameMountPoint;
- } else {
- // fall back to crappy one we got from Files.getFileStore
- return store;
- }
- }
-
- // these are hacks that are not guaranteed, may change across JVM versions, etc.
- static String getMountPoint(FileStore store) {
- String desc = store.toString();
- int index = desc.lastIndexOf(" (");
- if (index != -1) {
- return desc.substring(0, index);
- } else {
- return desc;
- }
- }
-
/**
* Returns the second throwable if the first is null otherwise adds the second as suppressed to the first
* and returns it.
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
index cddbe4590ae..9abbb0ce8f4 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
@@ -23,32 +23,19 @@ import java.net.URI;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
-import java.nio.file.FileStore;
import java.nio.file.FileSystem;
import java.nio.file.Files;
-import java.nio.file.LinkOption;
import java.nio.file.NoSuchFileException;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
-import java.nio.file.attribute.FileAttributeView;
-import java.nio.file.attribute.FileStoreAttributeView;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
import java.util.Objects;
-import java.util.Random;
import java.util.Set;
-import java.util.UUID;
-import org.apache.lucene.mockfile.FilterFileSystem;
import org.apache.lucene.mockfile.FilterFileSystemProvider;
import org.apache.lucene.mockfile.FilterPath;
-import org.junit.AssumptionViolatedException;
/** Simple test methods for IOUtils */
public class TestIOUtils extends LuceneTestCase {
@@ -109,378 +96,6 @@ public class TestIOUtils extends LuceneTestCase {
// actually deletes file2
}
- public void testSpinsBasics() throws Exception {
- Path dir = createTempDir();
- // no exception, directory exists
- IOUtils.spins(dir);
- Path file = dir.resolve("exists");
- Files.createFile(file);
- // no exception, file exists
- IOUtils.spins(file);
-
- // exception: file doesn't exist
- Path fake = dir.resolve("nonexistent");
- expectThrows(IOException.class, () -> {
- IOUtils.spins(fake);
- });
- }
-
- // fake up a filestore to test some underlying methods
- static class MockFileStore extends FileStore {
- final String description;
- final String type;
- final String name;
-
- MockFileStore(String description, String type, String name) {
- this.description = description;
- this.type = type;
- this.name = name;
- }
-
- @Override
- public String type() {
- return type;
- }
-
- @Override
- public String name() {
- return name;
- }
-
- @Override
- public String toString() {
- return description;
- }
-
-
- // TODO: we can enable mocking of these when we need them later:
-
- @Override
- public boolean isReadOnly() {
- return false;
- }
-
- @Override
- public long getTotalSpace() throws IOException {
- return 1000;
- }
-
- @Override
- public long getUsableSpace() throws IOException {
- return 800;
- }
-
- @Override
- public long getUnallocatedSpace() throws IOException {
- return 1000;
- }
-
- @Override
- public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type) {
- return false;
- }
-
- @Override
- public boolean supportsFileAttributeView(String name) {
- return false;
- }
-
- @Override
- public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type) {
- return null;
- }
-
- @Override
- public Object getAttribute(String attribute) throws IOException {
- return null;
- }
- }
-
- public void testGetMountPoint() throws Exception {
- assertEquals("/", IOUtils.getMountPoint(new MockFileStore("/ (/dev/sda1)", "ext4", "/dev/sda1")));
- assertEquals("/test/ space(((trash)))/", IOUtils.getMountPoint(new MockFileStore("/test/ space(((trash)))/ (/dev/sda1)", "ext3", "/dev/sda1")));
- assertEquals("/", IOUtils.getMountPoint(new MockFileStore("/ (notreal)", "ext2", "notreal")));
- }
-
- /** mock linux that takes mappings of test files, to their associated filesystems.
- * it will chroot /dev and /sys requests to root, so you can mock those too.
- * <p>
- * It is hacky by definition, so don't try putting it around a complex chain or anything.
- */
- static class MockLinuxFileSystemProvider extends FilterFileSystemProvider {
- final Map<String,FileStore> filesToStore;
- final Path root;
-
- public MockLinuxFileSystemProvider(FileSystem delegateInstance, final Map<String,FileStore> filesToStore, Path root) {
- super("mocklinux://", delegateInstance);
- if (mockedPath(root)) {
- throw new AssumptionViolatedException("can't mock /sys and /dev inside of /sys or /dev!");
- }
- final Collection<FileStore> allStores = new HashSet<>(filesToStore.values());
- this.fileSystem = new FilterFileSystem(this, delegateInstance) {
- @Override
- public Iterable<FileStore> getFileStores() {
- return allStores;
- }
-
- @Override
- public Path getPath(String first, String... more) {
- return new MockLinuxPath(delegateInstance.getPath(first, more), this);
- }
- };
- this.filesToStore = filesToStore;
- this.root = new MockLinuxPath(root, this.fileSystem);
- }
-
- @Override
- public FileStore getFileStore(Path path) throws IOException {
- FileStore ret = filesToStore.get(path.toString());
- if (ret == null) {
- throw new IllegalArgumentException("this mock doesnt know wtf to do with: " + path);
- }
- // act like the linux fs provider here, return a crazy rootfs one
- if (ret.toString().startsWith(root + " (")) {
- return new MockFileStore(root + " (rootfs)", "rootfs", "rootfs");
- }
-
- return ret;
- }
-
- static boolean mockedPath(Path path) {
- return path.toAbsolutePath().startsWith("/sys") || path.toAbsolutePath().startsWith("/dev");
- }
-
- Path maybeChroot(Path path) {
- if (mockedPath(path)) {
- // map to our chrooted location;
- return path.getRoot().resolve(root).resolve(path.toString().substring(1));
- } else {
- return path;
- }
- }
-
- @Override
- protected Path toDelegate(Path path) {
- return super.toDelegate(maybeChroot(path));
- }
-
- class MockLinuxPath extends FilterPath {
- MockLinuxPath(Path delegate, FileSystem fileSystem) {
- super(delegate, fileSystem);
- }
-
- @Override
- public Path toRealPath(LinkOption... options) throws IOException {
- Path p = maybeChroot(this);
- if (p == this) {
- return super.toRealPath(options);
- } else {
- return p.toRealPath(options);
- }
- }
-
- @Override
- protected Path wrap(Path other) {
- return new MockLinuxPath(other, fileSystem);
- }
- }
- }
-
- public void testGetFileStore() throws Exception {
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // now we can create some fake mount points:
- FileStore root = new MockFileStore(dir.toString() + " (/dev/sda1)", "ntfs", "/dev/sda1");
- FileStore usr = new MockFileStore(dir.resolve("usr").toString() + " (/dev/sda2)", "xfs", "/dev/sda2");
-
- // associate some preset files to these
- Map<String,FileStore> mappings = new HashMap<>();
- mappings.put(dir.toString(), root);
- mappings.put(dir.resolve("foo.txt").toString(), root);
- mappings.put(dir.resolve("usr").toString(), usr);
- mappings.put(dir.resolve("usr/bar.txt").toString(), usr);
-
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
- Path mockPath = mockLinux.getPath(dir.toString());
-
- // sanity check our mock:
- assertSame(usr, Files.getFileStore(mockPath.resolve("usr")));
- assertSame(usr, Files.getFileStore(mockPath.resolve("usr/bar.txt")));
- // for root filesystem we get a crappy one
- assertNotSame(root, Files.getFileStore(mockPath));
- assertNotSame(usr, Files.getFileStore(mockPath));
- assertNotSame(root, Files.getFileStore(mockPath.resolve("foo.txt")));
- assertNotSame(usr, Files.getFileStore(mockPath.resolve("foo.txt")));
-
- // now test our method:
- assertSame(usr, IOUtils.getFileStore(mockPath.resolve("usr")));
- assertSame(usr, IOUtils.getFileStore(mockPath.resolve("usr/bar.txt")));
- assertSame(root, IOUtils.getFileStore(mockPath));
- assertSame(root, IOUtils.getFileStore(mockPath.resolve("foo.txt")));
- }
-
- public void testTmpfsDoesntSpin() throws Exception {
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // fake tmpfs
- FileStore root = new MockFileStore(dir.toString() + " (/dev/sda1)", "tmpfs", "/dev/sda1");
- Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
-
- Path mockPath = mockLinux.getPath(dir.toString());
- assertFalse(IOUtils.spinsLinux(mockPath));
- }
-
- public void testNfsSpins() throws Exception {
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // fake nfs
- FileStore root = new MockFileStore(dir.toString() + " (somenfsserver:/some/mount)", "nfs", "somenfsserver:/some/mount");
- Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
-
- Path mockPath = mockLinux.getPath(dir.toString());
- assertTrue(IOUtils.spinsLinux(mockPath));
- }
-
- public void testSSD() throws Exception {
- assumeFalse("windows is not supported", Constants.WINDOWS);
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // fake ssd
- FileStore root = new MockFileStore(dir.toString() + " (/dev/zzz1)", "btrfs", "/dev/zzz1");
- // make a fake /dev/zzz1 for it
- Path devdir = dir.resolve("dev");
- Files.createDirectories(devdir);
- Files.createFile(devdir.resolve("zzz1"));
- // make a fake /sys/block/zzz/queue/rotational file for it
- Path sysdir = dir.resolve("sys").resolve("block").resolve("zzz").resolve("queue");
- Files.createDirectories(sysdir);
- try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
- o.write("0\n".getBytes(StandardCharsets.US_ASCII));
- }
- Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
-
- Path mockPath = mockLinux.getPath(dir.toString());
- assertFalse(IOUtils.spinsLinux(mockPath));
- }
-
- public void testNVME() throws Exception {
- assumeFalse("windows is not supported", Constants.WINDOWS);
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // fake ssd
- FileStore root = new MockFileStore(dir.toString() + " (/dev/nvme0n1p1)", "btrfs", "/dev/nvme0n1p1");
- // make a fake /dev/nvme0n1p1 for it
- Path devdir = dir.resolve("dev");
- Files.createDirectories(devdir);
- Files.createFile(devdir.resolve("nvme0n1p1"));
- // make a fake /sys/block/nvme0n1/queue/rotational file for it
- Path sysdir = dir.resolve("sys").resolve("block").resolve("nvme0n1").resolve("queue");
- Files.createDirectories(sysdir);
- try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
- o.write("0\n".getBytes(StandardCharsets.US_ASCII));
- }
- // As test for the longest path match, add some other devices (that have no queue/rotational), too:
- Files.createFile(dir.resolve("sys").resolve("block").resolve("nvme0"));
- Files.createFile(dir.resolve("sys").resolve("block").resolve("dummy"));
- Files.createFile(dir.resolve("sys").resolve("block").resolve("nvm"));
- Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
-
- Path mockPath = mockLinux.getPath(dir.toString());
- assertFalse(IOUtils.spinsLinux(mockPath));
- }
-
- public void testRotatingPlatters() throws Exception {
- assumeFalse("windows is not supported", Constants.WINDOWS);
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // fake ssd
- FileStore root = new MockFileStore(dir.toString() + " (/dev/zzz1)", "reiser4", "/dev/zzz1");
- // make a fake /dev/zzz1 for it
- Path devdir = dir.resolve("dev");
- Files.createDirectories(devdir);
- Files.createFile(devdir.resolve("zzz1"));
- // make a fake /sys/block/zzz/queue/rotational file for it
- Path sysdir = dir.resolve("sys").resolve("block").resolve("zzz").resolve("queue");
- Files.createDirectories(sysdir);
- try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
- o.write("1\n".getBytes(StandardCharsets.US_ASCII));
- }
- Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
-
- Path mockPath = mockLinux.getPath(dir.toString());
- assertTrue(IOUtils.spinsLinux(mockPath));
- }
-
- public void testManyPartitions() throws Exception {
- assumeFalse("windows is not supported", Constants.WINDOWS);
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // fake ssd
- FileStore root = new MockFileStore(dir.toString() + " (/dev/zzz12)", "zfs", "/dev/zzz12");
- // make a fake /dev/zzz11 for it
- Path devdir = dir.resolve("dev");
- Files.createDirectories(devdir);
- Files.createFile(devdir.resolve("zzz12"));
- // make a fake /sys/block/zzz/queue/rotational file for it
- Path sysdir = dir.resolve("sys").resolve("block").resolve("zzz").resolve("queue");
- Files.createDirectories(sysdir);
- try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
- o.write("0\n".getBytes(StandardCharsets.US_ASCII));
- }
- Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
-
- Path mockPath = mockLinux.getPath(dir.toString());
- assertFalse(IOUtils.spinsLinux(mockPath));
- }
-
- public void testSymlinkSSD() throws Exception {
- assumeFalse("windows is not supported", Constants.WINDOWS);
- Path dir = createTempDir();
- dir = FilterPath.unwrap(dir).toRealPath();
-
- // fake SSD with a symlink mount (Ubuntu-like):
- Random rnd = random();
- String partitionUUID = new UUID(rnd.nextLong(), rnd.nextLong()).toString();
- FileStore root = new MockFileStore(dir.toString() + " (/dev/disk/by-uuid/"+partitionUUID+")", "btrfs", "/dev/disk/by-uuid/"+partitionUUID);
- // make a fake /dev/sda1 for it
- Path devdir = dir.resolve("dev");
- Files.createDirectories(devdir);
- Path deviceFile = devdir.resolve("sda1");
- Files.createFile(deviceFile);
- // create a symlink to the above device file
- Path symlinkdir = devdir.resolve("disk").resolve("by-uuid");
- Files.createDirectories(symlinkdir);
- try {
- Files.createSymbolicLink(symlinkdir.resolve(partitionUUID), deviceFile);
- } catch (UnsupportedOperationException | IOException e) {
- assumeNoException("test requires filesystem that supports symbolic links", e);
- }
- // make a fake /sys/block/sda/queue/rotational file for it
- Path sysdir = dir.resolve("sys").resolve("block").resolve("sda").resolve("queue");
- Files.createDirectories(sysdir);
- try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
- o.write("0\n".getBytes(StandardCharsets.US_ASCII));
- }
- Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
- FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
-
- Path mockPath = mockLinux.getPath(dir.toString());
- assertFalse(IOUtils.spinsLinux(mockPath));
- }
-
public void testFsyncDirectory() throws Exception {
Path dir = createTempDir();
dir = FilterPath.unwrap(dir).toRealPath();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 5343eaefb3c..d6b884223f0 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -1865,19 +1865,6 @@ public abstract class LuceneTestCase extends Assert {
System.clearProperty(ConcurrentMergeScheduler.DEFAULT_CPU_CORE_COUNT_PROPERTY);
}
- @BeforeClass
- public static void setupSpins() {
- // Randomize IOUtils.spins() count so CMS varies its dynamic defaults, and this also "fixes" core
- // count from the master seed so it will always be the same on reproduce:
- boolean spins = random().nextBoolean();
- System.setProperty(ConcurrentMergeScheduler.DEFAULT_SPINS_PROPERTY, Boolean.toString(spins));
- }
-
- @AfterClass
- public static void restoreSpins() {
- System.clearProperty(ConcurrentMergeScheduler.DEFAULT_SPINS_PROPERTY);
- }
-
/**
* Create a new searcher over the reader. This searcher might randomly use
* threads.
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 21fb0489f75..c3414692483 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -777,14 +777,7 @@ public class CoreContainer {
true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs");
solrMetricsContext.gauge(() -> dataHome.toString(),
true, "path", SolrInfoBean.Category.CONTAINER.toString(), "fs");
- solrMetricsContext.gauge(() -> {
- try {
- return org.apache.lucene.util.IOUtils.spins(dataHome);
- } catch (IOException e) {
- // default to spinning
- return true;
- }
- },
+ solrMetricsContext.gauge(() -> false,
true, "spins", SolrInfoBean.Category.CONTAINER.toString(), "fs");
solrMetricsContext.gauge(() -> cfg.getCoreRootDirectory().toFile().getTotalSpace(),
true, "totalSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
@@ -792,14 +785,7 @@ public class CoreContainer {
true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
solrMetricsContext.gauge(() -> cfg.getCoreRootDirectory().toString(),
true, "path", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
- solrMetricsContext.gauge(() -> {
- try {
- return org.apache.lucene.util.IOUtils.spins(cfg.getCoreRootDirectory());
- } catch (IOException e) {
- // default to spinning
- return true;
- }
- },
+ solrMetricsContext.gauge(() -> false,
true, "spins", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
// add version information
solrMetricsContext.gauge(() -> this.getClass().getPackage().getSpecificationVersion(),
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 9ec9b467565..85555918f29 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -1228,14 +1228,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
parentContext.gauge(() -> dataDirFile.getTotalSpace(), true, "totalSpace", Category.CORE.toString(), "fs");
parentContext.gauge(() -> dataDirFile.getUsableSpace(), true, "usableSpace", Category.CORE.toString(), "fs");
parentContext.gauge(() -> dataDirPath.toAbsolutePath().toString(), true, "path", Category.CORE.toString(), "fs");
- parentContext.gauge(() -> {
- try {
- return org.apache.lucene.util.IOUtils.spins(dataDirPath.toAbsolutePath());
- } catch (IOException e) {
- // default to spinning
- return true;
- }
- }, true, "spins", Category.CORE.toString(), "fs");
+ parentContext.gauge(() -> false, true, "spins", Category.CORE.toString(), "fs");
}
public String getMetricTag() {
diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java
index f6eb498062b..992b973165f 100644
--- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java
@@ -27,7 +27,6 @@ import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.commons.io.FileUtils;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.core.CoreContainer;
@@ -181,12 +180,12 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 {
assertTrue(metrics.containsKey("CONTAINER.version.implementation"));
Gauge<?> g = (Gauge<?>)metrics.get("CONTAINER.fs.path");
assertEquals(g.getValue(), cc.getSolrHome());
- boolean spins = IOUtils.spins(cc.getCoreRootDirectory());
+ boolean spins = false;
g = (Gauge<?>)metrics.get("CONTAINER.fs.coreRoot.spins");
assertEquals(spins, g.getValue());
g = (Gauge<?>)metrics.get("CONTAINER.fs.spins");
if (cc.getConfig().getSolrDataHome() != null) {
- spins = IOUtils.spins(cc.getConfig().getSolrDataHome());
+ spins = false;
assertEquals(spins, g.getValue());
} else {
assertEquals(spins, g.getValue());
diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policy
index 10303472764..4a6b30fa9db 100644
--- a/solr/server/etc/security.policy
+++ b/solr/server/etc/security.policy
@@ -73,8 +73,6 @@ grant {
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
- // needed for test of IOUtils.spins (maybe it can be avoided)
- permission java.lang.RuntimePermission "getFileStoreAttributes";
// analyzers/uima: needed by lucene expressions' JavascriptCompiler
permission java.lang.RuntimePermission "createClassLoader";
// needed to test unmap hack on platforms that support it
diff --git a/solr/solr-ref-guide/src/taking-solr-to-production.adoc b/solr/solr-ref-guide/src/taking-solr-to-production.adoc
index 9303c7a1889..7e79b12d42d 100644
--- a/solr/solr-ref-guide/src/taking-solr-to-production.adoc
+++ b/solr/solr-ref-guide/src/taking-solr-to-production.adoc
@@ -167,15 +167,9 @@ If the `status` command is not successful, look for error messages in `/var/solr
The Merge Scheduler is configured in `solrconfig.xml` and defaults to `ConcurrentMergeScheduler`. This scheduler uses multiple threads to merge Lucene segments in the background.
-By default, the `ConcurrentMergeScheduler` auto-detects whether the underlying disk drive is rotational or a SSD and sets defaults for `maxThreadCount` and `maxMergeCount` accordingly. If the disk drive is determined to be rotational then the `maxThreadCount` is set to 1 and `maxMergeCount` is set to 6. Otherwise, `maxThreadCount` is set to 4 or half the number of processors available to the JVM whichever is greater and `maxMergeCount` is set to `maxThreadCount+5`.
+By default, the `ConcurrentMergeScheduler` auto-detects defaults for `maxThreadCount` and `maxMergeCount` accordingly. `maxThreadCount` is set to 4 or half the number of processors available to the JVM whichever is greater and `maxMergeCount` is set to `maxThreadCount+5`.
-This auto-detection works only on Linux and even then it is not guaranteed to be correct. On all other platforms, the disk is assumed to be rotational. Therefore, if the auto-detection fails or is incorrect then indexing performance can suffer badly due to the wrong defaults.
-
-The auto-detected value is exposed by the <<metrics-reporting.adoc#metrics-api, Metrics API>> with the key `solr.node:CONTAINER.fs.coreRoot.spins`. A value of `true` denotes that the disk is detected to be a rotational or spinning disk.
-
-It is safer to explicitly set values for `maxThreadCount` and `maxMergeCount` in the <<indexconfig-in-solrconfig.adoc#mergescheduler, IndexConfig section of SolrConfig.xml>> so that values appropriate to your hardware are used.
-
-Alternatively, the boolean system property `lucene.cms.override_spins` can be set in the `SOLR_OPTS` variable in the include file to override the auto-detected value. Similarly, the system property `lucene.cms.override_core_count` can be set to the number of CPU cores to override the auto-detected processor count.
+If you have a spinning disk, it is best to explicitly set values for `maxThreadCount` and `maxMergeCount` in the <<indexconfig-in-solrconfig.adoc#mergescheduler, IndexConfig section of SolrConfig.xml>> so that values appropriate to your hardware are used.
=== Memory and GC Settings