Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codebuild): retry source archive uploads (#6060)
Jimmy Liu Jiacbdeb0c8dcb9
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.javaindex cdc90cfc55..5e7a008a03 100644--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java@@ -16,6 +16,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedArtifacts;import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedBuildspec;import io.github.hectorvent.floci.services.codebuild.model.Build;@@ -39,6 +40,7 @@ import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.IOException;import java.io.InputStream;+import java.io.OutputStream;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;@@ -257,7 +259,13 @@ public class CodeBuildRunner implements ContainerTeardown {}// Copy downloaded source files into the container (no-op for NO_SOURCE builds)- copySourceToContainer(containerId, workspace, containerSrcDir);+ try {+ copySourceToContainer(containerId, workspace, containerSrcDir);+ } catch (RuntimeException e) {+ if (stopFlag.get()) { finishStopped(build); return; }+ completePhaseWithError(build, "DOWNLOAD_SOURCE", "FAILED", e.getMessage());+ throw e;+ }// INSTALLif (stopFlag.get()) { finishStopped(build); return; }@@ -500,24 +508,21 @@ public class CodeBuildRunner implements ContainerTeardown {// Copies files from the local workspace into the container's working directory.// Skips silently when the workspace is empty (e.g. NO_SOURCE builds).- private void copySourceToContainer(String containerId, Path sourceDir, String remotePath) {- try {- if (!Files.exists(sourceDir)) return;- boolean hasFiles;- try (Stream<Path> ls = Files.list(sourceDir)) {- hasFiles = ls.findAny().isPresent();- }- if (!hasFiles) return;-- ByteArrayOutputStream bos = new ByteArrayOutputStream();- createTarFromDir(sourceDir, bos);- dockerClient.copyArchiveToContainerCmd(containerId)- .withRemotePath(remotePath)- .withTarInputStream(new ByteArrayInputStream(bos.toByteArray()))- .exec();- } catch (Exception e) {- LOG.warnv("Could not copy source to container {0}: {1}", containerId, e.getMessage());+ void copySourceToContainer(String containerId, Path sourceDir, String remotePath) {+ copySourceToContainer(containerId, sourceDir, remotePath, RetryingTarCopier.DEFAULT_BACKOFF_MS);+ }++ void copySourceToContainer(String containerId, Path sourceDir, String remotePath, long backoffMillis) {+ if (!Files.exists(sourceDir)) return;+ try (Stream<Path> ls = Files.list(sourceDir)) {+ if (ls.findAny().isEmpty()) return;+ } catch (IOException e) {+ throw new IllegalStateException("Could not inspect CodeBuild source directory " + sourceDir, e);}++ RetryingTarCopier.copyStreamed(dockerClient, containerId, remotePath,+ "codebuild-source", out -> createTarFromDir(sourceDir, out),+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);}// Pulls the container's working directory back into the local workspace so@@ -564,7 +569,7 @@ public class CodeBuildRunner implements ContainerTeardown {}}- private void createTarFromDir(Path dir, ByteArrayOutputStream out) throws IOException {+ private void createTarFromDir(Path dir, OutputStream out) throws IOException {try (TarArchiveOutputStream tar = newTarStream(out);Stream<Path> stream = Files.walk(dir)) {for (Path path : (Iterable<Path>) stream::iterator) {@@ -588,7 +593,7 @@ public class CodeBuildRunner implements ContainerTeardown {}}- private static TarArchiveOutputStream newTarStream(ByteArrayOutputStream out) {+ private static TarArchiveOutputStream newTarStream(OutputStream out) {TarArchiveOutputStream tar = new TarArchiveOutputStream(out);tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerSourceFailureTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerSourceFailureTest.javanew file mode 100644index 0000000000..1e3f9ab3b5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerSourceFailureTest.java@@ -0,0 +1,150 @@+package io.github.hectorvent.floci.services.codebuild;++import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;+import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.services.codebuild.model.Build;+import io.github.hectorvent.floci.services.codebuild.model.BuildPhase;+import io.github.hectorvent.floci.services.codebuild.model.Project;+import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;+import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayOutputStream;+import java.io.Closeable;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Map;+import java.util.Optional;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.zip.ZipEntry;+import java.util.zip.ZipOutputStream;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.ArgumentMatchers.nullable;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.RETURNS_SELF;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.doReturn;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.when;++class CodeBuildRunnerSourceFailureTest {++ @Test+ void stoppingDuringSourceUploadKeepsStoppedStatus() throws Exception {+ Fixture fixture = fixture(true);++ fixture.run();++ assertEquals("STOPPED", fixture.build().getBuildStatus());+ assertTrue(fixture.build().getBuildComplete());+ }++ @Test+ void sourceUploadFailureIsRecordedOnDownloadSource() throws Exception {+ Fixture fixture = fixture(false);++ fixture.run();++ assertEquals("FAULT", fixture.build().getBuildStatus());+ BuildPhase sourcePhase = fixture.build().getPhases().stream()+ .filter(phase -> "DOWNLOAD_SOURCE".equals(phase.getPhaseType()))+ .findFirst().orElseThrow();+ assertEquals("FAILED", sourcePhase.getPhaseStatus());+ assertEquals("COMMAND_EXECUTION_ERROR", sourcePhase.getContexts().getFirst().get("statusCode"));+ assertEquals("source upload failed", sourcePhase.getContexts().getFirst().get("message"));+ }++ private Fixture fixture(boolean stopDuringCopy) throws IOException {+ DockerClient docker = mock(DockerClient.class);+ ContainerBuilder containerBuilder = mock(ContainerBuilder.class);+ ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);+ when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);+ when(specBuilder.build()).thenReturn(new ContainerSpec("alpine:latest"));++ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);+ when(lifecycle.createAndStart(any())).thenReturn(new ContainerLifecycleManager.ContainerInfo("container", Map.of()));+ CountDownLatch finished = new CountDownLatch(1);+ doAnswer(invocation -> { finished.countDown(); return null; })+ .when(lifecycle).stopAndRemove(eq("container"), nullable(Closeable.class));++ ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);+ when(logStreamer.attach(eq("container"), anyString(), anyString(), anyString(), anyString()))+ .thenReturn(mock(Closeable.class));++ S3Service s3 = mock(S3Service.class);+ S3Object source = mock(S3Object.class);+ when(source.getData()).thenReturn(sourceArchive());+ when(s3.getObject("source-bucket", "source.zip")).thenReturn(source);++ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().codebuild().dockerNetwork()).thenReturn(Optional.empty());+ RegionResolver regionResolver = mock(RegionResolver.class);+ when(regionResolver.getAccountId()).thenReturn("000000000000");+ ContainerReachableEndpoint endpoint = mock(ContainerReachableEndpoint.class);+ when(endpoint.port()).thenReturn(4566);++ CodeBuildRunner runner = spy(new CodeBuildRunner(docker, containerBuilder, lifecycle, logStreamer,+ s3, null, null, config, mock(ContainerDetector.class), regionResolver, endpoint));+ doReturn(CodeBuildRunner.PhaseResult.ofSuccess()).when(runner)+ .runPhase(anyString(), anyString(), anyList(), anyList(), anyInt(), any(AtomicBoolean.class));++ Build build = new Build();+ build.setId("project:build");+ build.setArn("arn:aws:codebuild:us-east-1:000000000000:build/project:build");+ build.setBuildNumber(1L);+ build.setPhases(new ArrayList<>());+ doAnswer(invocation -> {+ if (stopDuringCopy) runner.stopBuild(build.getArn());+ throw new IllegalStateException("source upload failed");+ }).when(runner).copySourceToContainer(eq("container"), any(), anyString());++ Project project = new Project();+ project.setName("project");+ ProjectSource projectSource = new ProjectSource();+ projectSource.setType("S3");+ projectSource.setLocation("source-bucket/source.zip");+ project.setSource(projectSource);+ ProjectEnvironment environment = new ProjectEnvironment();+ environment.setImage("alpine:latest");+ project.setEnvironment(environment);++ return new Fixture(runner, build, project, finished);+ }++ private byte[] sourceArchive() throws IOException {+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();+ try (ZipOutputStream zip = new ZipOutputStream(bytes)) {+ zip.putNextEntry(new ZipEntry("buildspec.yml"));+ zip.write("version: 0.2\n".getBytes(StandardCharsets.UTF_8));+ zip.closeEntry();+ }+ return bytes.toByteArray();+ }++ private record Fixture(CodeBuildRunner runner, Build build, Project project, CountDownLatch finished) {+ void run() throws InterruptedException {+ runner.startBuild("us-east-1", build, project, null);+ assertTrue(finished.await(10, TimeUnit.SECONDS), "build did not finish");+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildSourceCopyTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildSourceCopyTest.javanew file mode 100644index 0000000000..fee4445682--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildSourceCopyTest.java@@ -0,0 +1,93 @@+package io.github.hectorvent.floci.services.codebuild;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.ByteArrayInputStream;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotSame;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;++class CodeBuildSourceCopyTest {++ @Test+ void retriesSourceUploadWithFreshTarAfterBrokenPipe(@TempDir Path workspace) throws IOException {+ Files.writeString(workspace.resolve("buildspec.yml"), "version: 0.2\n");+ DockerClient docker = mock(DockerClient.class);+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "build-container");+ uploads.failures = 1;+ uploads.consumedBytes = 8;+ uploads.failure = new RuntimeException(new IOException("Broken pipe"));++ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src", 0);++ assertEquals(2, uploads.commands.size());+ assertNotSame(uploads.streams.get(0), uploads.streams.get(1));+ for (CopyArchiveToContainerCmd command : uploads.commands) {+ verify(command).withRemotePath("/codebuild/output/src/src");+ }+ try (TarArchiveInputStream tar = new TarArchiveInputStream(+ new ByteArrayInputStream(uploads.completed.getFirst()))) {+ TarArchiveEntry entry = tar.getNextEntry();+ assertEquals("buildspec.yml", entry.getName());+ assertEquals("version: 0.2\n", new String(tar.readAllBytes(), StandardCharsets.UTF_8));+ }+ }++ @Test+ void propagatesFailureAfterTransientRetryBudgetIsExhausted(@TempDir Path workspace) throws IOException {+ Files.writeString(workspace.resolve("buildspec.yml"), "version: 0.2\n");+ DockerClient docker = mock(DockerClient.class);+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "build-container");+ uploads.failures = 10;+ uploads.failure = new RuntimeException(new IOException("Broken pipe"));++ RuntimeException failure = assertThrows(RuntimeException.class, () ->+ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src", 0));++ assertEquals(6, uploads.commands.size());+ assertTrue(failure.getMessage().contains("codebuild-source"));+ }++ @Test+ void propagatesSourceUploadFailureInsteadOfRunningWithoutSource(@TempDir Path workspace) throws IOException {+ Files.writeString(workspace.resolve("buildspec.yml"), "version: 0.2\n");+ DockerClient docker = mock(DockerClient.class);+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "build-container");+ uploads.failures = 10;+ uploads.failure = new IllegalStateException("No such container");++ RuntimeException failure = assertThrows(RuntimeException.class, () ->+ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src", 0));++ assertTrue(failure.getMessage().contains("No such container"));+ assertEquals(1, uploads.commands.size());+ }++ @Test+ void skipsEmptyWorkspace(@TempDir Path workspace) {+ DockerClient docker = mock(DockerClient.class);++ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src");++ verifyNoInteractions(docker);+ }+}