Apache Solr open-source search software
SOLR-15706: Fix incorrect path in backup purging
Jason Gerlowskib532f4dc604d
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex aaa3db24059..c12d3483e39 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -440,13 +440,14 @@ Bug Fixes* SOLR-15628: The SolrException.log() helper method has been fixed to correctly passes the Throwable to the Logger w/o stringification (hossman)-* SOLR-15676: Fix PeerSync failure due to RealTimeGetComponent returning duplicates. (Ramsey Haddad, Christine Poerschke, David Smiley)* SOLR-15722: Delete Replica does not delete the Per replica state (noble)* SOLR-15766: MultiAuthPlugin should send non-AJAX anonymous requests to the plugin that allows anonymous requests (Timothy Potter, Eric Pugh)+* SOLR-15706: Incremental backup 'purges' no longer delete all index files(!) (Jason Gerlowski)+Build---------------------diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.javaindex b624e362fbb..fc8bb2ccc78 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java@@ -251,7 +251,8 @@ public class DeleteBackupCmd implements CollApiCmds.CollectionApiCommand {public void build(BackupRepository repository, URI backupPath) throws IOException {BackupFilePaths backupPaths = new BackupFilePaths(repository, backupPath);- buildLogicalGraph(repository, backupPath);+ buildLogicalGraph(repository, backupPaths);+findDeletableNodes(repository, backupPaths);}@@ -263,6 +264,22 @@ public class DeleteBackupCmd implements CollApiCmds.CollectionApiCommand {visitExistingNodes(repository.listAllOrEmpty(backupPaths.getIndexDir()),indexFileNodeMap, indexFileDeletes);+ // TODO Is this propagation logic really necessary?+ // The intention seems to be that if some index files are only referenced by a shard-metadata file that+ // is itself orphaned, then propagating the "orphaned" status down to each of these index files will allow+ // them to be deleted.+ //+ // But that doesn't seem to hold up under closer inspection.+ //+ // The graph is populated by following links out from the set of valid backup-id's. All files (shard-+ // metadata, or index) that trace back to a valid backup-ID will appear in the graph. If a shard-metadata+ // file becomes orphaned, it will be ignored during graph construction and any index files that it alone+ // references will not appear in the graph. Since those index files will be unrepresented in the graph, the+ // 'visitExistingNodes' calls above should be sufficient to detect them as orphaned.+ //+ // This all assumes though that propagation is intended to solve the scenario I think it does. If that+ // assumption is wrong, then this whole comment is wrong.+// for nodes which are not existing, propagate that information to other nodesshardBackupMetadataNodeMap.values().forEach(Node::propagateNotExisting);indexFileNodeMap.values().forEach(Node::propagateNotExisting);@@ -317,10 +334,12 @@ public class DeleteBackupCmd implements CollApiCmds.CollectionApiCommand {node2.addNeighbor(node1);}- private void buildLogicalGraph(BackupRepository repository, URI backupPath) throws IOException {- List<BackupId> backupIds = BackupFilePaths.findAllBackupIdsFromFileListing(repository.listAllOrEmpty(backupPath));+ private void buildLogicalGraph(BackupRepository repository, BackupFilePaths backupPaths) throws IOException {+ final URI baseBackupPath = backupPaths.getBackupLocation();++ List<BackupId> backupIds = BackupFilePaths.findAllBackupIdsFromFileListing(repository.listAllOrEmpty(baseBackupPath));for (BackupId backupId : backupIds) {- BackupProperties backupProps = BackupProperties.readFrom(repository, backupPath,+ BackupProperties backupProps = BackupProperties.readFrom(repository, baseBackupPath,BackupFilePaths.getBackupPropsName(backupId));Node backupIdNode = getBackupIdNode(BackupFilePaths.getBackupPropsName(backupId));@@ -328,7 +347,8 @@ public class DeleteBackupCmd implements CollApiCmds.CollectionApiCommand {Node shardBackupMetadataNode = getShardBackupIdNode(shardBackupMetadataFilename);addEdge(backupIdNode, shardBackupMetadataNode);- ShardBackupMetadata shardBackupMetadata = ShardBackupMetadata.from(repository, backupPath,++ ShardBackupMetadata shardBackupMetadata = ShardBackupMetadata.from(repository, backupPaths.getShardBackupMetadataDir(),ShardBackupId.fromShardMetadataFilename(shardBackupMetadataFilename));if (shardBackupMetadata == null)continue;diff --git a/solr/core/src/java/org/apache/solr/core/backup/ShardBackupMetadata.java b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupMetadata.javaindex ed3fdc24d61..dfac51a03aa 100644--- a/solr/core/src/java/org/apache/solr/core/backup/ShardBackupMetadata.java+++ b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupMetadata.java@@ -80,6 +80,14 @@ public class ShardBackupMetadata {return new ShardBackupMetadata();}+ /**+ * Reads a shard metadata file from a {@link BackupRepository} and parses the result into a {@link ShardBackupMetadata}+ *+ * @param repository the storage repository to read shard-metadata from+ * @param dir URI for the 'shard_backup_metadata' directory of the backup to read from+ * @param shardBackupId the ID of the shard metadata file to read+ * @return a ShardBackupMetadata object representing the provided 'shardBackupId' if it could be found in 'dir', null otherwise+ */public static ShardBackupMetadata from(BackupRepository repository, URI dir, ShardBackupId shardBackupId) throws IOException {final String shardBackupMetadataFilename = shardBackupId.getBackupMetadataFilename();if (!repository.exists(repository.resolve(dir, shardBackupMetadataFilename))) {diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.javaindex 0f1f2a09c63..cb4dcb335eb 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java@@ -17,171 +17,175 @@package org.apache.solr.cloud.api.collections;-import java.io.IOException;-import java.net.URI;-import java.net.URISyntaxException;-import java.util.Arrays;-import java.util.HashSet;-import java.util.Set;--import com.google.common.collect.ObjectArrays;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.cloud.api.collections.DeleteBackupCmd.PurgeGraph;+import org.apache.solr.core.backup.*;import org.apache.solr.core.backup.repository.BackupRepository;-import org.apache.solr.core.backup.BackupFilePaths;+import org.apache.solr.core.backup.repository.LocalFileSystemRepository;+import org.junit.Before;import org.junit.Test;-import org.mockito.stubbing.Answer;-import static org.mockito.ArgumentMatchers.same;-import static org.mockito.Mockito.mock;-import static org.mockito.Mockito.when;+import java.io.OutputStream;+import java.io.OutputStreamWriter;+import java.io.Writer;+import java.net.URI;+import java.nio.charset.StandardCharsets;+import java.util.Map;+import java.util.UUID;++import static org.hamcrest.Matchers.containsInAnyOrder;+/**+ * Unit tests for {@link PurgeGraph}+ */public class PurgeGraphTest extends SolrTestCaseJ4 {- private static final String[] shardBackupIds = new String[]{"b1_s1", "b1_s2", "b2_s1", "b2_s2", "b3_s1", "b3_s2", "b3_s3"};- @Test- public void test() throws URISyntaxException, IOException {- assumeWorkingMockito();- BackupRepository repository = mock(BackupRepository.class);- BackupFilePaths paths = mock(BackupFilePaths.class);- when(paths.getBackupLocation()).thenReturn(new URI("/temp"));- when(paths.getIndexDir()).thenReturn(new URI("/temp/index"));- when(paths.getShardBackupMetadataDir()).thenReturn(new URI("/temp/backup_point"));+ private BackupRepository repository;+ private URI baseLocationUri;+ private BackupFilePaths backupPaths;++ @Before+ public void setUpRepo() throws Exception {+ repository = new LocalFileSystemRepository();+ baseLocationUri = repository.createDirectoryURI(createTempDir("backup_files_" + UUID.randomUUID().toString()).toAbsolutePath().toString());+ backupPaths = new BackupFilePaths(repository, baseLocationUri);++ backupPaths.createIncrementalBackupFolders();+ }+ @Test+ public void testGraphBuildingOnNoBackups() throws Exception {PurgeGraph purgeGraph = new PurgeGraph();- buildCompleteGraph(repository, paths, purgeGraph);- purgeGraph.findDeletableNodes(repository, paths);+ purgeGraph.build(repository, backupPaths.getBackupLocation());+ purgeGraph.findDeletableNodes(repository, backupPaths);assertEquals(0, purgeGraph.backupIdDeletes.size());assertEquals(0, purgeGraph.shardBackupMetadataDeletes.size());assertEquals(0, purgeGraph.indexFileDeletes.size());-- testDeleteUnreferencedFiles(repository, paths, purgeGraph);- testMissingBackupPointFiles(repository, paths);- testMissingIndexFiles(repository, paths);}- private void testMissingIndexFiles(BackupRepository repository, BackupFilePaths paths) throws IOException {+ @Test+ public void testUnreferencedIndexFilesAreDeleted() throws Exception {+ // Backup 0 files+ createBackupIdFile(0, "shard1", "shard2");+ createShardMetadataFile(0, "shard1", Map.of("uniqName1", "localName1", "uniqName2", "localName2"));+ createShardMetadataFile(0, "shard2", Map.of("uniqName3", "localName3", "uniqName4", "localName4"));+ // Backup 1 files+ createBackupIdFile(1, "shard1", "shard2");+ createShardMetadataFile(1, "shard1", Map.of("uniqName5", "localName1", "uniqName6", "localName2"));+ createShardMetadataFile(1, "shard2", Map.of("uniqName7", "localName3", "uniqName8", "localName4"));+ // Valid, referenced index files+ createUniquelyNamedIndexFile("uniqName1", "uniqName2", "uniqName3", "uniqName4", "uniqName5", "uniqName6", "uniqName7", "uniqName8");+ // Single orphaned index file+ createUniquelyNamedIndexFile("someUnreferencedName");+PurgeGraph purgeGraph = new PurgeGraph();- buildCompleteGraph(repository, paths, purgeGraph);-- Set<String> indexFiles = purgeGraph.indexFileNodeMap.keySet();- when(repository.listAllOrEmpty(same(paths.getIndexDir()))).thenAnswer((Answer<String[]>) invocationOnMock -> {- Set<String> newFiles = new HashSet<>(indexFiles);- newFiles.remove("s1_102");- return newFiles.toArray(new String[0]);- });- purgeGraph.findDeletableNodes(repository, paths);-- assertEquals(3, purgeGraph.backupIdDeletes.size());- assertEquals(shardBackupIds.length, purgeGraph.shardBackupMetadataDeletes.size());- assertEquals(purgeGraph.indexFileNodeMap.size(), purgeGraph.indexFileDeletes.size() + 1);-- purgeGraph = new PurgeGraph();- buildCompleteGraph(repository, paths, purgeGraph);-- Set<String> indexFiles2 = purgeGraph.indexFileNodeMap.keySet();- when(repository.listAllOrEmpty(same(paths.getIndexDir()))).thenAnswer((Answer<String[]>) invocationOnMock -> {- Set<String> newFiles = new HashSet<>(indexFiles2);- newFiles.remove("s1_101");- return newFiles.toArray(new String[0]);- });- purgeGraph.findDeletableNodes(repository, paths);-- assertEquals(2, purgeGraph.backupIdDeletes.size());- assertEquals(4, purgeGraph.shardBackupMetadataDeletes.size());- assertTrue(purgeGraph.indexFileDeletes.contains("s1_100"));- assertFalse(purgeGraph.indexFileDeletes.contains("s1_101"));+ purgeGraph.build(repository, backupPaths.getBackupLocation());++ assertEquals(0, purgeGraph.backupIdDeletes.size());+ assertEquals(0, purgeGraph.shardBackupMetadataDeletes.size());+ assertEquals(1, purgeGraph.indexFileDeletes.size());+ assertEquals("someUnreferencedName", purgeGraph.indexFileDeletes.get(0));}- private void testMissingBackupPointFiles(BackupRepository repository, BackupFilePaths paths) throws IOException {+ // TODO - this seems a bit extreme - should this really occur by default?+ @Test+ public void testEntireBackupPointFlaggedForDeletionIfAnyIndexFilesMissing() throws Exception {+ // Backup 0 files+ createBackupIdFile(0, "shard1", "shard2");+ createShardMetadataFile(0, "shard1", Map.of("uniqName1", "localName1", "uniqName2", "localName2"));+ createShardMetadataFile(0, "shard2", Map.of("uniqName3", "localName3", "uniqName4", "localName4"));+ // Valid, referenced index files - 'uniqName3' is missing!+ createUniquelyNamedIndexFile("uniqName1", "uniqName2", "uniqName4");+PurgeGraph purgeGraph = new PurgeGraph();- buildCompleteGraph(repository, paths, purgeGraph);- when(repository.listAllOrEmpty(same(paths.getShardBackupMetadataDir()))).thenAnswer((Answer<String[]>)- invocationOnMock -> Arrays.copyOfRange(shardBackupIds, 1, shardBackupIds.length)- );- purgeGraph.findDeletableNodes(repository, paths);+ purgeGraph.build(repository, backupPaths.getBackupLocation());+ // All files associated with backup '0' should be flagged for deletion since the required file 'uniqName3' is missing.assertEquals(1, purgeGraph.backupIdDeletes.size());- assertEquals("b1", purgeGraph.backupIdDeletes.get(0));- assertEquals(1, purgeGraph.shardBackupMetadataDeletes.size());- assertEquals("b1_s2", purgeGraph.shardBackupMetadataDeletes.get(0));- assertTrue(purgeGraph.indexFileDeletes.contains("s1_100"));- assertFalse(purgeGraph.indexFileDeletes.contains("s1_101"));-- purgeGraph = new PurgeGraph();- buildCompleteGraph(repository, paths, purgeGraph);- when(repository.listAllOrEmpty(same(paths.getShardBackupMetadataDir()))).thenAnswer((Answer<String[]>)- invocationOnMock -> new String[]{"b1_s1", "b2_s1", "b3_s1", "b3_s2", "b3_s3"}- );- purgeGraph.findDeletableNodes(repository, paths);-- assertEquals(2, purgeGraph.backupIdDeletes.size());- assertTrue(purgeGraph.backupIdDeletes.containsAll(Arrays.asList("b1", "b2")));+ assertThat(purgeGraph.backupIdDeletes, containsInAnyOrder("backup_0.properties"));assertEquals(2, purgeGraph.shardBackupMetadataDeletes.size());- assertTrue(purgeGraph.shardBackupMetadataDeletes.containsAll(Arrays.asList("b2_s1", "b1_s1")));- assertTrue(purgeGraph.indexFileDeletes.containsAll(Arrays.asList("s1_100", "s1_101")));- assertFalse(purgeGraph.indexFileDeletes.contains("s1_102"));+ assertThat(purgeGraph.shardBackupMetadataDeletes, containsInAnyOrder("md_shard1_0.json", "md_shard2_0.json"));+ assertEquals(3, purgeGraph.indexFileDeletes.size());+ assertThat(purgeGraph.indexFileDeletes, containsInAnyOrder("uniqName1", "uniqName2", "uniqName4"));++ // If a subsequent backup relies on an index file (uniqName4) that was previously only used by the invalid backup '0', that file will not be flagged for deletion.+// createBackupIdFile(1, "shard1", "shard2");+// createShardMetadataFile(1, "shard1", Map.of("uniqName5", "localName1", "uniqName6", "localName2"));+// createShardMetadataFile(1, "shard2", Map.of("uniqName4", "localName4"));+// createUniquelyNamedIndexFile("uniqName5", "uniqName6");+//+// assertEquals(1, purgeGraph.backupIdDeletes.size());+// assertThat(purgeGraph.backupIdDeletes, containsInAnyOrder("backup_0.properties"));+// assertEquals(2, purgeGraph.shardBackupMetadataDeletes.size());+// assertThat(purgeGraph.shardBackupMetadataDeletes, containsInAnyOrder("md_shard1_0.json", "md_shard2_0.json"));+// // NOTE that 'uniqName4' is NOT marked for deletion+// assertEquals(2, purgeGraph.indexFileDeletes.size());+// assertThat(purgeGraph.indexFileDeletes, containsInAnyOrder("uniqName1", "uniqName2"));}- private void testDeleteUnreferencedFiles(BackupRepository repository, BackupFilePaths paths,- PurgeGraph purgeGraph) throws IOException {- buildCompleteGraph(repository, paths, purgeGraph);- String[] unRefBackupPoints = addUnRefFiles(repository, "b4_s", paths.getShardBackupMetadataDir());- String[] unRefIndexFiles = addUnRefFiles(repository, "s4_", paths.getIndexDir());+ @Test+ public void testUnreferencedShardMetadataFilesAreDeleted() throws Exception {+ // Backup 0 files+ createBackupIdFile(0, "shard1", "shard2");+ createShardMetadataFile(0, "shard1", Map.of("uniqName1", "localName1", "uniqName2", "localName2"));+ createShardMetadataFile(0, "shard2", Map.of("uniqName3", "localName3", "uniqName4", "localName4"));+ // Extra shard unreferenced by backup_0.properties+ createShardMetadataFile(0, "shard3", Map.of("uniqName5", "localName5", "uniqName6", "localName6"));+ createUniquelyNamedIndexFile("uniqName1", "uniqName2", "uniqName3", "uniqName4", "uniqName5", "uniqName6");- purgeGraph.findDeletableNodes(repository, paths);+ PurgeGraph purgeGraph = new PurgeGraph();+ purgeGraph.build(repository, backupPaths.getBackupLocation());assertEquals(0, purgeGraph.backupIdDeletes.size());- assertEquals(unRefBackupPoints.length, purgeGraph.shardBackupMetadataDeletes.size());- assertTrue(purgeGraph.shardBackupMetadataDeletes.containsAll(Arrays.asList(unRefBackupPoints)));- assertEquals(unRefIndexFiles.length, purgeGraph.indexFileDeletes.size());- assertTrue(purgeGraph.indexFileDeletes.containsAll(Arrays.asList(unRefIndexFiles)));+ assertEquals(1, purgeGraph.shardBackupMetadataDeletes.size());+ assertThat(purgeGraph.shardBackupMetadataDeletes, containsInAnyOrder("md_shard3_0.json"));+ assertEquals(2, purgeGraph.indexFileDeletes.size());+ assertThat(purgeGraph.indexFileDeletes, containsInAnyOrder("uniqName5", "uniqName6"));++ // If a subsequent backup relies on an index file (uniqName5) that was previously only used by the orphaned 'shard3' metadata file, that file should no longer be flagged for deletion+// createBackupIdFile(1, "shard1", "shard2");+// createShardMetadataFile(1, "shard1", Map.of("uniqName7", "localName7"));+// createShardMetadataFile(1, "shard2", Map.of("uniqName5", "localName5", "uniqName8", "localName8"));+//+// purgeGraph = new PurgeGraph();+// purgeGraph.build(repository, backupPaths.getBackupLocation());+//+// assertEquals(0, purgeGraph.backupIdDeletes.size());+// assertEquals(1, purgeGraph.shardBackupMetadataDeletes.size());+// assertThat(purgeGraph.shardBackupMetadataDeletes, containsInAnyOrder("md_shard3_0.json"));+// assertEquals(1, purgeGraph.indexFileDeletes.size());+// assertThat(purgeGraph.indexFileDeletes, containsInAnyOrder("uniqName6"));}- private String[] addUnRefFiles(BackupRepository repository, String prefix, URI dir) {- String[] unRefBackupPoints = new String[random().nextInt(10) + 1];- for (int i = 0; i < unRefBackupPoints.length; i++) {- unRefBackupPoints[i] = prefix + (100 + i);+ private void createBackupIdFile(int backupId, String... shardNames) throws Exception {+ final BackupProperties createdProps = BackupProperties.create("someBackupName", "someCollectionName",+ "someExtCollectionName", "someConfigName");+ for (String shardName : shardNames) {+ createdProps.putAndGetShardBackupIdFor(shardName, backupId);}- String[] shardBackupMetadataFiles = repository.listAllOrEmpty(dir);- when(repository.listAllOrEmpty(same(dir)))- .thenAnswer((Answer<String[]>) invocation- -> ObjectArrays.concat(shardBackupMetadataFiles, unRefBackupPoints, String.class));- return unRefBackupPoints;- }- private void buildCompleteGraph(BackupRepository repository, BackupFilePaths paths,- PurgeGraph purgeGraph) throws IOException {- when(repository.listAllOrEmpty(same(paths.getShardBackupMetadataDir()))).thenAnswer((Answer<String[]>) invocationOnMock -> shardBackupIds);- //logical-- for (String shardBackupId : shardBackupIds) {- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode(shardBackupId),- purgeGraph.getBackupIdNode(shardBackupId.substring(0, 2)));- for (int i = 0; i < random().nextInt(30); i++) {- String fileName = shardBackupId.substring(3) + "_" + random().nextInt(15);- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode(shardBackupId),- purgeGraph.getIndexFileNode(fileName));- }+ URI dest = repository.resolve(backupPaths.getBackupLocation(), BackupFilePaths.getBackupPropsName(new BackupId(backupId)));+ try (Writer propsWriter = new OutputStreamWriter(repository.createOutput(dest), StandardCharsets.UTF_8)) {+ createdProps.store(propsWriter);}+ }- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b1_s1"),- purgeGraph.getIndexFileNode("s1_100"));-- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b1_s1"),- purgeGraph.getIndexFileNode("s1_101"));- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b2_s1"),- purgeGraph.getIndexFileNode("s1_101"));-- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b1_s1"),- purgeGraph.getIndexFileNode("s1_102"));- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b2_s1"),- purgeGraph.getIndexFileNode("s1_102"));- purgeGraph.addEdge(purgeGraph.getShardBackupIdNode("b3_s1"),- purgeGraph.getIndexFileNode("s1_102"));+ private void createShardMetadataFile(int backupId, String shardName, Map<String,String> localIndexFilenamesByUniqueName) throws Exception {+ final ShardBackupMetadata createdShardMetadata = ShardBackupMetadata.empty();+ for (Map.Entry<String, String> entry : localIndexFilenamesByUniqueName.entrySet()) {+ createdShardMetadata.addBackedFile(entry.getKey(), entry.getValue(), new Checksum(1L, 1));+ }+ createdShardMetadata.store(repository, backupPaths.getShardBackupMetadataDir(), new ShardBackupId(shardName, new BackupId(backupId)));+ }- when(repository.listAllOrEmpty(same(paths.getIndexDir()))).thenAnswer((Answer<String[]>) invocationOnMock ->- purgeGraph.indexFileNodeMap.keySet().toArray(new String[0]));+ private void createUniquelyNamedIndexFile(String... uniqNames) throws Exception {+ for (String uniqName : uniqNames) {+ final String randomContent = "some value";+ final URI indexFileUri = repository.resolve(backupPaths.getIndexDir(), uniqName);+ try (OutputStream os = repository.createOutput(indexFileUri)) {+ os.write(randomContent.getBytes(StandardCharsets.UTF_8));+ }+ }}}