Light, fluffy, and always free - The AWS Local Emulator alternative
test(docker): share failing archive upload fixture (#6009)
Aniket Shuklae1d102f06b7e
diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioConfigRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioConfigRetryTest.javaindex ce35a8e490..f8ba9d6aaf 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioConfigRetryTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioConfigRetryTest.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.junit.jupiter.api.BeforeEach;@@ -17,13 +18,9 @@ import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;import java.io.ByteArrayInputStream;-import java.io.InputStream;import java.net.SocketException;import java.nio.charset.StandardCharsets;-import java.util.ArrayList;-import java.util.List;import java.util.Optional;-import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotSame;@@ -33,8 +30,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;-import static org.mockito.Mockito.RETURNS_SELF;-import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;@@ -44,55 +39,30 @@ class VerdaccioConfigRetryTest {private final DockerClient docker = mock(DockerClient.class);private final ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);- private final List<InputStream> streams = new ArrayList<>();- private final List<CopyArchiveToContainerCmd> commands = new ArrayList<>();- private final List<byte[]> completed = new ArrayList<>();+ private final DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "container");private VerdaccioSidecarManager manager;- private int failures;- private int consumedBytes;- private RuntimeException failure;@BeforeEachvoid setUp() {manager = new VerdaccioSidecarManager(mock(ContainerBuilder.class), lifecycle, mock(EmulatorConfig.class));when(lifecycle.getDockerClient()).thenReturn(docker);- when(docker.copyArchiveToContainerCmd("container")).thenAnswer(ignored -> {- CopyArchiveToContainerCmd command = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);- commands.add(command);- AtomicReference<InputStream> input = new AtomicReference<>();- when(command.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {- InputStream stream = invocation.getArgument(0);- streams.add(stream);- input.set(stream);- return command;- });- doAnswer(invocation -> {- if (commands.size() <= failures) {- input.get().readNBytes(consumedBytes);- throw failure;- }- completed.add(input.get().readAllBytes());- return null;- }).when(command).exec();- return command;- });}@ParameterizedTest@ValueSource(ints = {0, 73, 8192})void transientFailureRebuildsTheCompleteConfigArchive(int consumed) throws Exception {- failures = 1;- consumedBytes = consumed;- failure = new RuntimeException(new SocketException("Connection reset"));+ uploads.failures = 1;+ uploads.consumedBytes = consumed;+ uploads.failure = new RuntimeException(new SocketException("Connection reset"));manager.copyConfig("container", 0);- assertEquals(2, commands.size());- assertNotSame(streams.get(0), streams.get(1));- for (CopyArchiveToContainerCmd command : commands) {+ assertEquals(2, uploads.commands.size());+ assertNotSame(uploads.streams.get(0), uploads.streams.get(1));+ for (CopyArchiveToContainerCmd command : uploads.commands) {verify(command).withRemotePath("/verdaccio/conf");}- try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(completed.getFirst()))) {+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(uploads.completed.getFirst()))) {TarArchiveEntry entry = tar.getNextEntry();assertEquals("config.yaml", entry.getName());assertEquals(0644, entry.getMode());@@ -106,19 +76,19 @@ class VerdaccioConfigRetryTest {@Testvoid exhaustedTransportFailurePropagatesAfterTheRetryBudget() {- failures = 10;- failure = new RuntimeException(new SocketException("Connection reset"));+ uploads.failures = 10;+ uploads.failure = new RuntimeException(new SocketException("Connection reset"));RuntimeException thrown = assertThrows(RuntimeException.class, () -> manager.copyConfig("container", 0));- assertEquals(6, commands.size());- assertSame(failure, thrown.getCause());+ assertEquals(6, uploads.commands.size());+ assertSame(uploads.failure, thrown.getCause());}@Testvoid daemonRejectionDoesNotRetry() {- failures = 10;- failure = new NotFoundException("container missing");+ uploads.failures = 10;+ uploads.failure = new NotFoundException("container missing");RuntimeException thrown = assertThrows(RuntimeException.class, () -> manager.copyConfig("container", 0));- assertEquals(1, commands.size());+ assertEquals(1, uploads.commands.size());assertTrue(thrown.getMessage().contains("container missing"));}@@ -134,13 +104,13 @@ class VerdaccioConfigRetryTest {mock(DockerHostResolver.class), mock(EmbeddedDnsServer.class));manager = new VerdaccioSidecarManager(builder, lifecycle, config);when(lifecycle.create(any(ContainerSpec.class))).thenReturn("container");- failures = 1;- failure = new NotFoundException("container missing");+ uploads.failures = 1;+ uploads.failure = new NotFoundException("container missing");assertThrows(RuntimeException.class, () -> manager.ensureReady("npm-repo", "http://localhost:4566"));verify(lifecycle).stopAndRemove("container", null);verify(lifecycle, never()).startCreated(any(), any());- assertEquals(1, commands.size());+ assertEquals(1, uploads.commands.size());}}diff --git a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.javaindex 1a217547a1..a4b20713c5 100644--- a/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/sagemaker/SageMakerArchiveRetryTest.java@@ -17,6 +17,7 @@ import io.github.hectorvent.floci.services.s3.model.S3Object;import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.EndpointResource;import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.ModelResource;import io.github.hectorvent.floci.services.sagemaker.model.SageMakerEntities.TrainingJobResource;+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;@@ -29,17 +30,14 @@ import org.junit.jupiter.params.provider.ValueSource;import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;-import java.io.InputStream;import java.net.SocketException;import java.nio.charset.StandardCharsets;-import java.util.ArrayList;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Optional;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;-import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;@@ -49,7 +47,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.Mockito.RETURNS_DEEP_STUBS;-import static org.mockito.Mockito.RETURNS_SELF;import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;@@ -61,16 +58,11 @@ class SageMakerArchiveRetryTest {private final ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);private final S3Service s3 = mock(S3Service.class);private final ObjectMapper mapper = new ObjectMapper();- private final List<InputStream> streams = new ArrayList<>();- private final List<CopyArchiveToContainerCmd> commands = new ArrayList<>();- private final List<byte[]> completed = new ArrayList<>();+ private final DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "container");private final byte[] payload = "model-data".repeat(2000).getBytes(StandardCharsets.UTF_8);private SageMakerEndpointManager manager;private SageMakerTrainingRunner runner;private TrainingJobResource job;- private int failures;- private int consumedBytes;- private RuntimeException failure;@BeforeEachvoid setUp() {@@ -99,26 +91,6 @@ class SageMakerArchiveRetryTest {when(s3.getObject("bucket", "input/nested/data.csv")).thenReturn(input);when(lifecycle.create(any(ContainerSpec.class))).thenReturn("container");when(lifecycle.getDockerClient()).thenReturn(docker);- when(docker.copyArchiveToContainerCmd("container")).thenAnswer(ignored -> {- CopyArchiveToContainerCmd command = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);- commands.add(command);- AtomicReference<InputStream> inputStream = new AtomicReference<>();- when(command.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {- InputStream stream = invocation.getArgument(0);- streams.add(stream);- inputStream.set(stream);- return command;- });- doAnswer(invocation -> {- if (commands.size() <= failures) {- inputStream.get().readNBytes(consumedBytes);- throw failure;- }- completed.add(inputStream.get().readAllBytes());- return null;- }).when(command).exec();- return command;- });}@AfterEach@@ -181,23 +153,23 @@ class SageMakerArchiveRetryTest {@ValueSource(booleans = {false, true})void exhaustedTransportFailureStopsAtSixAttempts(boolean training) {failTransport(73);- failures = 10;+ uploads.failures = 10;RuntimeException thrown = assertThrows(RuntimeException.class, () -> copy(training));- assertEquals(6, commands.size());- assertSame(failure, thrown.getCause());+ assertEquals(6, uploads.commands.size());+ assertSame(uploads.failure, thrown.getCause());}@ParameterizedTest@ValueSource(booleans = {false, true})void daemonRejectionDoesNotRetry(boolean training) {- failures = 10;- failure = new NotFoundException("container missing");+ uploads.failures = 10;+ uploads.failure = new NotFoundException("container missing");RuntimeException thrown = assertThrows(RuntimeException.class, () -> copy(training));- assertEquals(1, commands.size());+ assertEquals(1, uploads.commands.size());assertTrue(thrown.getMessage().contains("container missing"));}@@ -211,14 +183,14 @@ class SageMakerArchiveRetryTest {IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> copy(training));assertEquals("source unavailable", thrown.getMessage());- assertTrue(commands.isEmpty());+ assertTrue(uploads.commands.isEmpty());}@ParameterizedTest@ValueSource(booleans = {false, true})void failedUploadRemovesTheUnstartedContainer(boolean training) throws InterruptedException {- failures = 1;- failure = new NotFoundException("container missing");+ uploads.failures = 1;+ uploads.failure = new NotFoundException("container missing");assertFailedStart(training, 1, "container missing");}@@ -226,7 +198,7 @@ class SageMakerArchiveRetryTest {@ValueSource(booleans = {false, true})void exhaustedUploadRetriesRemoveTheUnstartedContainer(boolean training) throws InterruptedException {failTransport(73);- failures = 10;+ uploads.failures = 10;assertFailedStart(training, 6, "Connection reset");}@@ -257,7 +229,7 @@ class SageMakerArchiveRetryTest {assertTrue(reported.await(5, TimeUnit.SECONDS));assertEquals("Failed", training ? job.trainingJobStatus : endpoint.endpointStatus);assertTrue((training ? job.failureReason : endpoint.failureReason).contains(reason));- assertEquals(attempts, commands.size());+ assertEquals(attempts, uploads.commands.size());verify(lifecycle).create(any(ContainerSpec.class));verify(lifecycle).stopAndRemove("container", null);verify(lifecycle, never()).startCreated(any(), any());@@ -272,24 +244,24 @@ class SageMakerArchiveRetryTest {}private void failTransport(int consumed) {- failures = 1;- consumedBytes = consumed;- failure = new RuntimeException(new SocketException("Connection reset"));+ uploads.failures = 1;+ uploads.consumedBytes = consumed;+ uploads.failure = new RuntimeException(new SocketException("Connection reset"));}private void assertRetried() {- assertEquals(2, commands.size());- assertNotSame(commands.get(0), commands.get(1));- assertNotSame(streams.get(0), streams.get(1));- for (CopyArchiveToContainerCmd command : commands) {+ assertEquals(2, uploads.commands.size());+ assertNotSame(uploads.commands.get(0), uploads.commands.get(1));+ assertNotSame(uploads.streams.get(0), uploads.streams.get(1));+ for (CopyArchiveToContainerCmd command : uploads.commands) {verify(command).withRemotePath("/");}- assertEquals(1, completed.size());+ assertEquals(1, uploads.completed.size());}private void assertArchive(Map<String, byte[]> expected) throws Exception {Map<String, byte[]> actual = new LinkedHashMap<>();- try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(completed.getFirst()))) {+ try (TarArchiveInputStream tar = new TarArchiveInputStream(new ByteArrayInputStream(uploads.completed.getFirst()))) {TarArchiveEntry entry;while ((entry = tar.getNextEntry()) != null) {assertEquals(0644, entry.getMode());diff --git a/src/test/java/io/github/hectorvent/floci/testing/DockerArchiveUploadFixture.java b/src/test/java/io/github/hectorvent/floci/testing/DockerArchiveUploadFixture.javanew file mode 100644index 0000000000..6735ffd11c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/testing/DockerArchiveUploadFixture.java@@ -0,0 +1,48 @@+package io.github.hectorvent.floci.testing;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;++import java.io.InputStream;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.atomic.AtomicReference;++import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/** Consumes part of each failed upload and captures the complete successful retry. */+public final class DockerArchiveUploadFixture {+ public final List<InputStream> streams = new ArrayList<>();+ public final List<CopyArchiveToContainerCmd> commands = new ArrayList<>();+ public final List<byte[]> completed = new ArrayList<>();+ public int failures;+ public int consumedBytes;+ public RuntimeException failure;++ public DockerArchiveUploadFixture(DockerClient docker, String containerId) {+ when(docker.copyArchiveToContainerCmd(containerId)).thenAnswer(ignored -> {+ CopyArchiveToContainerCmd command = mock(CopyArchiveToContainerCmd.class, RETURNS_SELF);+ commands.add(command);+ AtomicReference<InputStream> input = new AtomicReference<>();+ when(command.withTarInputStream(any(InputStream.class))).thenAnswer(invocation -> {+ InputStream stream = invocation.getArgument(0);+ streams.add(stream);+ input.set(stream);+ return command;+ });+ doAnswer(invocation -> {+ if (commands.size() <= failures) {+ input.get().readNBytes(consumedBytes);+ throw failure;+ }+ completed.add(input.get().readAllBytes());+ return null;+ }).when(command).exec();+ return command;+ });+ }+}