Light, fluffy, and always free - The AWS Local Emulator alternative
fix(timestream-influxdb): retry restore archive uploads (#6064)
Aniket Shuklafa4bacd0d158
diff --git a/docs/services/timestream-influxdb.md b/docs/services/timestream-influxdb.mdindex cf443c46f2..7c3cf1406a 100644--- a/docs/services/timestream-influxdb.md+++ b/docs/services/timestream-influxdb.md@@ -81,6 +81,8 @@ write and query data against it.- `RestoreFromDbBackup` with `NEW_RESOURCE` starts a new container and runs `influx restore --full`, so therestored resource has the users, tokens, organizations and buckets of the source. `REPLACE_EXISTING`restores into the existing container and requires `name` to match the existing resource.+ Data and CLI configuration archive uploads retry transient Docker I/O failures for up to six attempts,+ reopening the original archive for each attempt. The `influx restore` command itself is not retried.- `restoreToTime` is rejected with `ValidationException`. Point-in-time restore is only available forcontinuous backups, and Floci models on-demand backups only, so accepting the parameter would silentlyrestore the on-demand contents instead of the requested moment.diff --git a/src/main/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManager.javaindex 0ff8216a74..5472e03d6a 100644--- a/src/main/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManager.java@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.Cimport io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.PortAllocator;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -342,12 +343,16 @@ public class TimestreamInfluxDbContainerManager implements ContainerTeardown {}private void copyToContainer(String containerId, Path archive, String remotePath) throws IOException {- try (InputStream stream = Files.newInputStream(archive)) {- lifecycleManager.getDockerClient().copyArchiveToContainerCmd(containerId)- .withRemotePath(remotePath)- .withTarInputStream(stream)- .exec();- }+ copyToContainer(containerId, archive, remotePath, RetryingTarCopier.DEFAULT_BACKOFF_MS);+ }++ void copyToContainer(String containerId, Path archive, String remotePath, long backoffMillis) throws IOException {+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, remotePath,+ archive.getFileName().toString(), out -> {+ try (InputStream stream = Files.newInputStream(archive)) {+ stream.transferTo(out);+ }+ }, RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);}private void execOrThrow(String containerId, String[] command, String description) {diff --git a/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManagerTest.javaindex 9704e5136b..1316ce0694 100644--- a/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/timestreaminfluxdb/container/TimestreamInfluxDbContainerManagerTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.timestreaminfluxdb.container;import com.fasterxml.jackson.databind.ObjectMapper;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.exception.NotFoundException;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;@@ -11,22 +12,38 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.Cimport io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.PortAllocator;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;import io.github.hectorvent.floci.services.timestreaminfluxdb.container.TimestreamInfluxDbContainerManager.InfluxDbEndpoint;import io.github.hectorvent.floci.services.timestreaminfluxdb.container.TimestreamInfluxDbContainerManager.InfluxDbSetup;+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import org.mockito.Mockito;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;import java.util.List;import java.util.Map;import java.util.Optional;+import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyLong;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;+import static org.mockito.Mockito.spy;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -181,4 +198,126 @@ class TimestreamInfluxDbContainerManagerTest {assertEquals("influx backup failed with exit code 1: Error: failed to connect", error.getMessage());assertEquals(List.of(List.of("influx", "backup", "/tmp/floci-backup-backup-1")), commands);}++ @ParameterizedTest+ @ValueSource(ints = {0, 73, 8192})+ void restoreRetriesConsumedArchivesWithoutRetarringThem(int consumedBytes, @TempDir Path temporary) throws Exception {+ List<byte[]> archives = backupArchives(temporary);+ DockerClient docker = mock(DockerClient.class);+ when(lifecycleManager.getDockerClient()).thenReturn(docker);+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(docker, "influx-id", 0, "", "");+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "influx-id");+ uploads.failures = 1;+ uploads.consumedBytes = consumedBytes;+ uploads.failure = new RuntimeException(new IOException("Connection reset"));++ restoreWithoutBackoff().restore("influx-id", "backup-1");++ assertEquals(3, uploads.commands.size());+ assertEquals(2, uploads.completed.size());+ assertNotSame(uploads.streams.get(0), uploads.streams.get(1));+ assertArrayEquals(archives.get(0), uploads.completed.get(0));+ assertArrayEquals(archives.get(1), uploads.completed.get(1));+ verify(uploads.commands.get(0)).withRemotePath("/tmp");+ verify(uploads.commands.get(1)).withRemotePath("/tmp");+ verify(uploads.commands.get(2)).withRemotePath("/etc/influxdb2");+ assertEquals(List.of(List.of("influx", "restore", "--full", "/tmp/floci-backup-backup-1"),+ List.of("rm", "-rf", "/tmp/floci-backup-backup-1")), commands);+ }++ @Test+ void exhaustedDataUploadDoesNotRunRestoreButStillCleansUp(@TempDir Path temporary) throws Exception {+ backupArchives(temporary);+ DockerClient docker = mock(DockerClient.class);+ when(lifecycleManager.getDockerClient()).thenReturn(docker);+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(docker, "influx-id", 0, "", "");+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "influx-id");+ uploads.failures = 6;+ uploads.consumedBytes = 73;+ uploads.failure = new RuntimeException(new IOException("Broken pipe"));++ assertThrows(RuntimeException.class, () -> restoreWithoutBackoff().restore("influx-id", "backup-1"));++ assertEquals(6, uploads.commands.size());+ assertTrue(uploads.completed.isEmpty());+ assertEquals(List.of(List.of("rm", "-rf", "/tmp/floci-backup-backup-1")), commands);+ }++ @Test+ void exhaustedCliUploadDoesNotRepeatTheRestoreCommand(@TempDir Path temporary) throws Exception {+ List<byte[]> archives = backupArchives(temporary);+ DockerClient docker = mock(DockerClient.class);+ when(lifecycleManager.getDockerClient()).thenReturn(docker);+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(docker, "influx-id", 0, "", "");+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "influx-id");+ uploads.consumedBytes = 73;+ uploads.failure = new RuntimeException(new IOException("Broken pipe"));+ TimestreamInfluxDbContainerManager restoring = restoreWithoutBackoff();+ doAnswer(invocation -> {+ uploads.failures = uploads.commands.size() + 6;+ manager.copyToContainer(invocation.getArgument(0), invocation.getArgument(1),+ invocation.getArgument(2), 0);+ return null;+ }).when(restoring).copyToContainer(anyString(), any(Path.class), eq("/etc/influxdb2"), anyLong());++ assertThrows(RuntimeException.class, () -> restoring.restore("influx-id", "backup-1"));++ assertEquals(7, uploads.commands.size());+ assertEquals(1, uploads.completed.size());+ assertArrayEquals(archives.getFirst(), uploads.completed.getFirst());+ assertEquals(List.of(List.of("influx", "restore", "--full", "/tmp/floci-backup-backup-1"),+ List.of("rm", "-rf", "/tmp/floci-backup-backup-1")), commands);+ }++ @Test+ void daemonRejectionIsNotRetriedAndStillCleansUp(@TempDir Path temporary) throws Exception {+ backupArchives(temporary);+ DockerClient docker = mock(DockerClient.class);+ when(lifecycleManager.getDockerClient()).thenReturn(docker);+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(docker, "influx-id", 0, "", "");+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "influx-id");+ uploads.failures = 6;+ uploads.failure = new NotFoundException("container missing");++ assertThrows(RuntimeException.class, () -> restoreWithoutBackoff().restore("influx-id", "backup-1"));++ assertEquals(1, uploads.commands.size());+ assertEquals(List.of(List.of("rm", "-rf", "/tmp/floci-backup-backup-1")), commands);+ }++ @Test+ void missingLocalArchiveFailsWithoutRestoreOrRepeatedCopies(@TempDir Path temporary) {+ when(storage.persistentPath()).thenReturn(temporary.toString());+ DockerClient docker = mock(DockerClient.class);+ when(lifecycleManager.getDockerClient()).thenReturn(docker);+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(docker, "influx-id", 0, "", "");+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "influx-id");++ assertThrows(RuntimeException.class, () -> restoreWithoutBackoff().restore("influx-id", "backup-1"));++ assertEquals(1, uploads.commands.size());+ assertEquals(List.of(List.of("rm", "-rf", "/tmp/floci-backup-backup-1")), commands);+ }++ private TimestreamInfluxDbContainerManager restoreWithoutBackoff() throws IOException {+ TimestreamInfluxDbContainerManager restoring = spy(manager);+ doAnswer(invocation -> {+ manager.copyToContainer(invocation.getArgument(0), invocation.getArgument(1),+ invocation.getArgument(2), 0);+ return null;+ }).when(restoring).copyToContainer(anyString(), any(Path.class), anyString(), anyLong());+ return restoring;+ }++ private List<byte[]> backupArchives(Path temporary) throws IOException {+ when(storage.persistentPath()).thenReturn(temporary.toString());+ Path directory = temporary.resolve("timestream-influxdb/backups");+ Files.createDirectories(directory);+ List<byte[]> archives = List.of(+ RetryingTarCopier.singleFileTar("floci-backup-backup-1/manifest", new byte[16384], 0644),+ RetryingTarCopier.singleFileTar("influx-configs", "test config".getBytes(StandardCharsets.UTF_8), 0600));+ Files.write(directory.resolve("backup-1.tar"), archives.get(0));+ Files.write(directory.resolve("backup-1-cli.tar"), archives.get(1));+ return archives;+ }}