Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-18008: Remnant core files prevent Collections for working (#3904)

Eric Pugh2b70060ac3ed
diff --git a/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml b/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml
new file mode 100644
index 00000000000..45d804cccd6
--- /dev/null
+++ b/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml
@@ -0,0 +1,9 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Add solr.cloud.delete.unknown.cores.enabled setting for removing unknown but existing core data when a core is created in SolrCloud mode.
+type: changed # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+ - name: Eric Pugh
+ - name: David Smiley
+links:
+- name: SOLR-18008
+ url: https://issues.apache.org/jira/browse/SOLR-18008
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index bb06a189b9f..50dd516f5e3 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -94,6 +94,7 @@ import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.CollectionUtil;
+import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectCache;
@@ -1672,11 +1673,11 @@ public class CoreContainer {
} catch (Exception e) {
coreInitFailures.put(dcore.getName(), new CoreLoadFailure(dcore, e));
if (e instanceof ZkController.NotInClusterStateException && !newCollection) {
- // this mostly happens when the core is deleted when this node is down
+ // this mostly happens when the core is deleted when this node is down,
// but it can also happen if connecting to the wrong zookeeper
final boolean deleteUnknownCores =
- Boolean.parseBoolean(
- System.getProperty("solr.cloud.startup.delete.unknown.cores.enabled", "false"));
+ EnvUtils.getPropertyAsBool("solr.cloud.startup.delete.unknown.cores.enabled", false);
+
log.error(
"SolrCore {} in {} is not in cluster state.{}",
dcore.getName(),
@@ -1684,7 +1685,7 @@ public class CoreContainer {
(deleteUnknownCores
? " It will be deleted. See SOLR-13396 for more information."
: ""));
- // We alreday have an ongoing CoreOp, so do not wait to start another one
+ // We already have an ongoing CoreOp, so do not wait to start another one
unloadWithoutCoreOp(
dcore.getName(), deleteUnknownCores, deleteUnknownCores, deleteUnknownCores);
throw e;
diff --git a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
index c060fc5d3cd..2fe356dfca8 100644
--- a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
+++ b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
@@ -39,6 +39,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +66,7 @@ public class CorePropertiesLocator implements CoresLocator {
@Override
public void create(CoreContainer cc, CoreDescriptor... coreDescriptors) {
for (CoreDescriptor cd : coreDescriptors) {
+ checkForExistingCore(cd);
Path propertiesFile = cd.getInstanceDir().resolve(PROPERTIES_FILENAME);
if (Files.exists(propertiesFile))
throw new SolrException(
@@ -240,4 +242,23 @@ public class CorePropertiesLocator implements CoresLocator {
p.putAll(cd.getPersistableUserProperties());
return p;
}
+
+ protected void checkForExistingCore(CoreDescriptor cd) {
+ if (cd.getCloudDescriptor() != null && Files.exists(cd.getInstanceDir())) {
+ final boolean deleteUnknownCores =
+ EnvUtils.getPropertyAsBool("solr.cloud.delete.unknown.cores.enabled", false);
+ if (deleteUnknownCores) {
+ log.warn(
+ "Automatically deleting existing directory at [{}] for core [{}] because solr.cloud.delete.unknown.cores.enabled is true",
+ cd.getInstanceDir().toAbsolutePath(),
+ cd.getName());
+ SolrCore.deleteUnloadedCore(cd, true, true);
+ } else {
+ log.warn(
+ "Directory at [{}] for core[{}] already exists may prevent create operation. Set solr.cloud.delete.unknown.cores.enabled=true to delete directory. (SOLR-18008)",
+ cd.getInstanceDir().toAbsolutePath(),
+ cd.getName());
+ }
+ }
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java
new file mode 100644
index 00000000000..57cea18d24e
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Optional;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.response.json.JsonMapResponseParser;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test "solr.cloud.delete.unknown.cores.enabled" property that can be used if Solr has an
+ * inconsistent state with its cores lifecycle where remnant files are left on disk after various
+ * operations that delete a core.
+ */
+public class DeleteCoreRemnantsOnCreateTest extends SolrCloudTestCase {
+ private static final String DELETE_UNKNOWN_CORES_PROP = "solr.cloud.delete.unknown.cores.enabled";
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(1).addConfig("conf", configset("cloud-minimal")).configure();
+ }
+
+ /**
+ * Shared setup for testing collection creation with remnants. Creates a collection, deletes it,
+ * and then leaves behind a remnant directory.
+ */
+ private String setupCollectionRemnant(String collectionName) throws Exception {
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ CollectionAdminRequest.Create createRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ createRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ Replica primaryReplica = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ JettySolrRunner primaryJetty = cluster.getReplicaJetty(primaryReplica);
+ String originalCoreName = primaryReplica.getCoreName();
+ Path remnantInstanceDir;
+ try (SolrCore core = primaryJetty.getCoreContainer().getCore(originalCoreName)) {
+ CoreDescriptor cd = core.getCoreDescriptor();
+ remnantInstanceDir = cd.getInstanceDir();
+ }
+
+ CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
+ waitForState("Expected collection deletion", collectionName, (n, c) -> c == null);
+
+ // Simulate a core remnant still exists by creating the directory and core.properties
+ Files.createDirectories(remnantInstanceDir);
+ String propertiesContent = "";
+ Files.writeString(
+ remnantInstanceDir.resolve("core.properties"), propertiesContent, StandardCharsets.UTF_8);
+
+ return originalCoreName;
+ }
+
+ /**
+ * Shared setup for testing replica addition with remnants. Creates a collection, then simulates a
+ * remnant directory on the single node that will impact the next addReplica command.
+ */
+ private void setupReplicaRemnant(String collectionName) throws Exception {
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ CollectionAdminRequest.Create createRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ createRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ int nextReplicaIndex = 3; // Yep, from 1 to 3 due to how we count in ZK and setup.
+ String expectedNewReplicaName = collectionName + "_shard1_replica_n" + nextReplicaIndex;
+
+ // Simulate a core remnant on the single node adjacent to the existing replica instance path
+ Replica existing = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ try (SolrCore core =
+ cluster.getReplicaJetty(existing).getCoreContainer().getCore(existing.getCoreName())) {
+ Path siblingDir = core.getInstancePath().getParent().resolve(expectedNewReplicaName);
+ Files.createDirectories(siblingDir);
+ Files.writeString(
+ siblingDir.resolve("core.properties"),
+ "name="
+ + expectedNewReplicaName
+ + "_remnant\n"
+ + "collection="
+ + collectionName
+ + "_remnant\n"
+ + "shard=shard1\n"
+ + "coreNodeName=core_node_remnant\n",
+ StandardCharsets.UTF_8);
+ }
+ }
+
+ /**
+ * Shared setup for testing DeleteCore admin API with remnants. Creates a collection, deletes it,
+ * and then leaves behind a remnant core directory.
+ */
+ private String setupCoreRemnantForUnloadCoreOperation(String collectionName) throws Exception {
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ CollectionAdminRequest.Create createRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ createRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ Replica primaryReplica = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ JettySolrRunner primaryJetty = cluster.getReplicaJetty(primaryReplica);
+ String originalCoreName = primaryReplica.getCoreName();
+ Path remnantInstanceDir;
+ try (SolrCore core = primaryJetty.getCoreContainer().getCore(originalCoreName)) {
+ CoreDescriptor cd = core.getCoreDescriptor();
+ remnantInstanceDir = cd.getInstanceDir();
+ }
+
+ CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
+ waitForState("Expected collection deletion", collectionName, (n, c) -> c == null);
+
+ // Simulate a core remnant still exists by creating the directory and core.properties
+ Files.createDirectories(remnantInstanceDir);
+ Files.writeString(
+ remnantInstanceDir.resolve("core.properties"),
+ "name=" + originalCoreName + "\n",
+ StandardCharsets.UTF_8);
+
+ return originalCoreName;
+ }
+
+ @Test
+ public void testCreateCollectionWithRemnantsFailsWithoutSetting() throws Exception {
+ assertNull(
+ "Property should not be set by default", System.getProperty(DELETE_UNKNOWN_CORES_PROP));
+
+ String collectionName = "coreRemnantCreateNoSetting";
+ setupCollectionRemnant(collectionName);
+
+ // Try to create the collection again - this demonstrates the behavior without the setting
+ // In typical environments, this might fail, but behavior depends on configuration
+ CollectionAdminRequest.Create recreateRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+
+ // The request to create a collection SHOULD fail based on the remnant file, if it does not it
+ // means we've changed Solr's behavior when creating a core and
+ // remnants exist, and therefore we should rethink the utility of this setting.
+ Exception e =
+ assertThrows(
+ "This request to recreate the collection should have failed due to remnant files.",
+ Exception.class,
+ () -> recreateRequest.process(cluster.getSolrClient()));
+
+ assertTrue(
+ "Verify the exception was due to core creation failed.",
+ e.getMessage().contains("Underlying core creation failed"));
+ }
+
+ @Test
+ public void testCreateCollectionWithRemnantsWithSetting() throws Exception {
+ System.setProperty(DELETE_UNKNOWN_CORES_PROP, "true");
+
+ String collectionName = "coreRemnantCreateWithSetting";
+ setupCollectionRemnant(collectionName);
+
+ // With the setting enabled, collection creation should succeed despite remnants
+ CollectionAdminRequest.Create recreateRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ recreateRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected recreated collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ // Verify collection was created successfully
+ DocCollection collection = getCollectionState(collectionName);
+ assertNotNull("Collection should exist", collection);
+ assertEquals("Should have 1 replica", 1, collection.getReplicas().size());
+
+ // Verify replica on the node where we had the remnant is active
+ Replica recreatedReplica =
+ getReplicaOnNode(collectionName, "shard1", jettys.getFirst().getNodeName());
+ assertNotNull("Should have a replica on the primary node", recreatedReplica);
+ assertEquals("Replica should be active", Replica.State.ACTIVE, recreatedReplica.getState());
+ }
+
+ @Test
+ public void testAddReplicaWithRemnantFailsWithoutSetting() throws Exception {
+ assertNull(
+ "Property should not be set by default", System.getProperty(DELETE_UNKNOWN_CORES_PROP));
+
+ String collectionName = "coreRemnantAddReplicaNoSetting";
+ setupReplicaRemnant(collectionName);
+
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ // Try to add a new replica - this demonstrates the behavior without the setting
+ CollectionAdminRequest.AddReplica addReplicaRequest =
+ CollectionAdminRequest.addReplicaToShard(collectionName, "shard1");
+ addReplicaRequest.setNode(primaryNode);
+
+ Exception e =
+ assertThrows(
+ "This request to add a replica to the collection should have failed due to remnant files.",
+ Exception.class,
+ () -> addReplicaRequest.process(cluster.getSolrClient()));
+
+ assertTrue(
+ "Verify the exception was due to core creation failed.",
+ e.getMessage().contains("ADDREPLICA failed to create replica"));
+ }
+
+ @Test
+ public void testAddReplicaWithRemnantWithSetting() throws Exception {
+ System.setProperty(DELETE_UNKNOWN_CORES_PROP, "true");
+
+ String collectionName = "coreRemnantAddReplicaWithSetting";
+ setupReplicaRemnant(collectionName);
+
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ // With the setting enabled, replica addition should succeed despite remnants
+ CollectionAdminRequest.AddReplica addReplicaRequest =
+ CollectionAdminRequest.addReplicaToShard(collectionName, "shard1");
+ addReplicaRequest.setNode(primaryNode);
+ addReplicaRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected replica addition to finish",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 2));
+
+ // Verify collection now has 2 replicas
+ DocCollection collection = getCollectionState(collectionName);
+ assertNotNull("Collection should exist", collection);
+ assertEquals("Should have 2 replicas after adding", 2, collection.getReplicas().size());
+
+ // Verify the replica was added on the single node and is active
+ Replica addedReplica = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ assertNotNull("Should have added a replica on the primary node", addedReplica);
+ assertEquals("Added replica should be active", Replica.State.ACTIVE, addedReplica.getState());
+ }
+
+ /**
+ * This test demonstrates that you can't call the direct core unload admin operation to get rid of
+ * a remnant core, not because of the existence of the remnant, but because the core no longer has
+ * a CoreDescriptor record in ZooKeeper.
+ */
+ @Test
+ public void testDeleteCoreFailsWhenUnknown() throws Exception {
+
+ String collectionName = "coreRemnantDelete";
+ String coreName = setupCollectionRemnant(collectionName);
+
+ // Try to delete a core that only exists as a remnant - and has no record in ZooKeeper
+ CoreAdminRequest.Unload unloadRequest = new CoreAdminRequest.Unload(true);
+ unloadRequest.setDeleteIndex(true);
+ unloadRequest.setDeleteDataDir(true);
+ unloadRequest.setDeleteInstanceDir(true);
+ unloadRequest.setCoreName(coreName);
+ unloadRequest.setResponseParser(new JsonMapResponseParser());
+
+ Exception e =
+ assertThrows(
+ "Expected request to fail.",
+ Exception.class,
+ () -> cluster.getSolrClient().request(unloadRequest));
+
+ assertTrue(
+ "Verify the exception was due to ZK not knowing about the core existence.",
+ e.getMessage().contains("Cannot unload non-existent core [" + coreName + "]"));
+ }
+
+ private Replica getReplicaOnNode(String collectionName, String shard, String nodeName) {
+ DocCollection collectionState = getCollectionState(collectionName);
+ Slice slice = collectionState.getSlice(shard);
+ Optional<Replica> replica =
+ slice.getReplicas().stream().filter(r -> nodeName.equals(r.getNodeName())).findFirst();
+ return replica.orElseThrow(
+ () -> new AssertionError("No replica found on node " + nodeName + " for " + shard));
+ }
+}
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc
index 04ce53ad47e..f1d89ad6c38 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc
@@ -18,6 +18,8 @@
Core discovery means that creating a core is as simple as a `core.properties` file located on disk.
+TIP: If you are running in SolrCloud mode, you are shielded from this complexity.
+
== The core.properties File
In Solr, the term _core_ is used to refer to a single index and associated transaction log and configuration files (including the `solrconfig.xml` and schema files, among others).
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc
index 1c180319e8f..7cb304cd2f7 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc
@@ -46,6 +46,8 @@ NOTE: Properties marked with "!" indicate inverted meaning between pre Solr 10 a
|solr.cloud.startup.delete.unknown.cores.enabled|solr.deleteUnknownCores|false|Controls whether unknown cores are deleted at startup in SolrCloud mode.
+|solr.cloud.delete.unknown.cores.enabled| |false|Controls deletion of preexisting core data when a core is created in SolrCloud mode.
+
|solr.cloud.wait.for.zk.seconds|waitForZk|30|Specifies the number of seconds to wait for ZooKeeper connection in SolrCloud mode.
|solr.configset.bootstrap.confdir|bootstrap_confdir||Defines the directory of a configuration to be loaded as a configset when starting Solr. Often ends in `/conf`.
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
index 90006a03e94..7245c66714b 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
@@ -292,6 +292,15 @@ Request ID to track this action which will be xref:configuration-guide:collectio
Collections are first created in read-write mode but can be put in `readOnly`
mode using the xref:collection-management.adoc#modifycollection[MODIFYCOLLECTION] action.
+Solr can occasionally enter an inconsistent state where remnant core files remain on disk after previous collection deletion operations fail.
+
+The system property `solr.cloud.delete.unknown.cores.enabled` is an expert-level setting designed to handle this situation.
+When enabled, Solr automatically deletes any remnant core data on startup that lacks a record in ZooKeeper.
+During core creation (whether for new collections or replicas), Solr also deletes preexisting remnant files, allowing operations to complete.
+Without this setting, these files would cause new core creation to fail.
+
+Enable this feature with caution— it indicates an underlying problem in your Solr setup that should be investigated.
+
=== CREATE Response
The response will include the status of the request and the new core names.