Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codebuild): retry source archive uploads (#6060)

Jimmy Liu Jiacbdeb0c8dcb9
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
index cdc90cfc55..5e7a008a03 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
@@ -16,6 +16,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;
import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedArtifacts;
import io.github.hectorvent.floci.services.codebuild.BuildspecParser.ParsedBuildspec;
import io.github.hectorvent.floci.services.codebuild.model.Build;
@@ -39,6 +40,7 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -257,7 +259,13 @@ public class CodeBuildRunner implements ContainerTeardown {
}
// Copy downloaded source files into the container (no-op for NO_SOURCE builds)
- copySourceToContainer(containerId, workspace, containerSrcDir);
+ try {
+ copySourceToContainer(containerId, workspace, containerSrcDir);
+ } catch (RuntimeException e) {
+ if (stopFlag.get()) { finishStopped(build); return; }
+ completePhaseWithError(build, "DOWNLOAD_SOURCE", "FAILED", e.getMessage());
+ throw e;
+ }
// INSTALL
if (stopFlag.get()) { finishStopped(build); return; }
@@ -500,24 +508,21 @@ public class CodeBuildRunner implements ContainerTeardown {
// Copies files from the local workspace into the container's working directory.
// Skips silently when the workspace is empty (e.g. NO_SOURCE builds).
- private void copySourceToContainer(String containerId, Path sourceDir, String remotePath) {
- try {
- if (!Files.exists(sourceDir)) return;
- boolean hasFiles;
- try (Stream<Path> ls = Files.list(sourceDir)) {
- hasFiles = ls.findAny().isPresent();
- }
- if (!hasFiles) return;
-
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- createTarFromDir(sourceDir, bos);
- dockerClient.copyArchiveToContainerCmd(containerId)
- .withRemotePath(remotePath)
- .withTarInputStream(new ByteArrayInputStream(bos.toByteArray()))
- .exec();
- } catch (Exception e) {
- LOG.warnv("Could not copy source to container {0}: {1}", containerId, e.getMessage());
+ void copySourceToContainer(String containerId, Path sourceDir, String remotePath) {
+ copySourceToContainer(containerId, sourceDir, remotePath, RetryingTarCopier.DEFAULT_BACKOFF_MS);
+ }
+
+ void copySourceToContainer(String containerId, Path sourceDir, String remotePath, long backoffMillis) {
+ if (!Files.exists(sourceDir)) return;
+ try (Stream<Path> ls = Files.list(sourceDir)) {
+ if (ls.findAny().isEmpty()) return;
+ } catch (IOException e) {
+ throw new IllegalStateException("Could not inspect CodeBuild source directory " + sourceDir, e);
}
+
+ RetryingTarCopier.copyStreamed(dockerClient, containerId, remotePath,
+ "codebuild-source", out -> createTarFromDir(sourceDir, out),
+ RetryingTarCopier.DEFAULT_MAX_ATTEMPTS, backoffMillis);
}
// Pulls the container's working directory back into the local workspace so
@@ -564,7 +569,7 @@ public class CodeBuildRunner implements ContainerTeardown {
}
}
- private void createTarFromDir(Path dir, ByteArrayOutputStream out) throws IOException {
+ private void createTarFromDir(Path dir, OutputStream out) throws IOException {
try (TarArchiveOutputStream tar = newTarStream(out);
Stream<Path> stream = Files.walk(dir)) {
for (Path path : (Iterable<Path>) stream::iterator) {
@@ -588,7 +593,7 @@ public class CodeBuildRunner implements ContainerTeardown {
}
}
- private static TarArchiveOutputStream newTarStream(ByteArrayOutputStream out) {
+ private static TarArchiveOutputStream newTarStream(OutputStream out) {
TarArchiveOutputStream tar = new TarArchiveOutputStream(out);
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerSourceFailureTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerSourceFailureTest.java
new file mode 100644
index 0000000000..1e3f9ab3b5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerSourceFailureTest.java
@@ -0,0 +1,150 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import com.github.dockerjava.api.DockerClient;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLogStreamer;
+import io.github.hectorvent.floci.core.common.docker.ContainerReachableEndpoint;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.services.codebuild.model.Build;
+import io.github.hectorvent.floci.services.codebuild.model.BuildPhase;
+import io.github.hectorvent.floci.services.codebuild.model.Project;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;
+import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+class CodeBuildRunnerSourceFailureTest {
+
+ @Test
+ void stoppingDuringSourceUploadKeepsStoppedStatus() throws Exception {
+ Fixture fixture = fixture(true);
+
+ fixture.run();
+
+ assertEquals("STOPPED", fixture.build().getBuildStatus());
+ assertTrue(fixture.build().getBuildComplete());
+ }
+
+ @Test
+ void sourceUploadFailureIsRecordedOnDownloadSource() throws Exception {
+ Fixture fixture = fixture(false);
+
+ fixture.run();
+
+ assertEquals("FAULT", fixture.build().getBuildStatus());
+ BuildPhase sourcePhase = fixture.build().getPhases().stream()
+ .filter(phase -> "DOWNLOAD_SOURCE".equals(phase.getPhaseType()))
+ .findFirst().orElseThrow();
+ assertEquals("FAILED", sourcePhase.getPhaseStatus());
+ assertEquals("COMMAND_EXECUTION_ERROR", sourcePhase.getContexts().getFirst().get("statusCode"));
+ assertEquals("source upload failed", sourcePhase.getContexts().getFirst().get("message"));
+ }
+
+ private Fixture fixture(boolean stopDuringCopy) throws IOException {
+ DockerClient docker = mock(DockerClient.class);
+ ContainerBuilder containerBuilder = mock(ContainerBuilder.class);
+ ContainerBuilder.Builder specBuilder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(specBuilder);
+ when(specBuilder.build()).thenReturn(new ContainerSpec("alpine:latest"));
+
+ ContainerLifecycleManager lifecycle = mock(ContainerLifecycleManager.class);
+ when(lifecycle.createAndStart(any())).thenReturn(new ContainerLifecycleManager.ContainerInfo("container", Map.of()));
+ CountDownLatch finished = new CountDownLatch(1);
+ doAnswer(invocation -> { finished.countDown(); return null; })
+ .when(lifecycle).stopAndRemove(eq("container"), nullable(Closeable.class));
+
+ ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
+ when(logStreamer.attach(eq("container"), anyString(), anyString(), anyString(), anyString()))
+ .thenReturn(mock(Closeable.class));
+
+ S3Service s3 = mock(S3Service.class);
+ S3Object source = mock(S3Object.class);
+ when(source.getData()).thenReturn(sourceArchive());
+ when(s3.getObject("source-bucket", "source.zip")).thenReturn(source);
+
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().codebuild().dockerNetwork()).thenReturn(Optional.empty());
+ RegionResolver regionResolver = mock(RegionResolver.class);
+ when(regionResolver.getAccountId()).thenReturn("000000000000");
+ ContainerReachableEndpoint endpoint = mock(ContainerReachableEndpoint.class);
+ when(endpoint.port()).thenReturn(4566);
+
+ CodeBuildRunner runner = spy(new CodeBuildRunner(docker, containerBuilder, lifecycle, logStreamer,
+ s3, null, null, config, mock(ContainerDetector.class), regionResolver, endpoint));
+ doReturn(CodeBuildRunner.PhaseResult.ofSuccess()).when(runner)
+ .runPhase(anyString(), anyString(), anyList(), anyList(), anyInt(), any(AtomicBoolean.class));
+
+ Build build = new Build();
+ build.setId("project:build");
+ build.setArn("arn:aws:codebuild:us-east-1:000000000000:build/project:build");
+ build.setBuildNumber(1L);
+ build.setPhases(new ArrayList<>());
+ doAnswer(invocation -> {
+ if (stopDuringCopy) runner.stopBuild(build.getArn());
+ throw new IllegalStateException("source upload failed");
+ }).when(runner).copySourceToContainer(eq("container"), any(), anyString());
+
+ Project project = new Project();
+ project.setName("project");
+ ProjectSource projectSource = new ProjectSource();
+ projectSource.setType("S3");
+ projectSource.setLocation("source-bucket/source.zip");
+ project.setSource(projectSource);
+ ProjectEnvironment environment = new ProjectEnvironment();
+ environment.setImage("alpine:latest");
+ project.setEnvironment(environment);
+
+ return new Fixture(runner, build, project, finished);
+ }
+
+ private byte[] sourceArchive() throws IOException {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(bytes)) {
+ zip.putNextEntry(new ZipEntry("buildspec.yml"));
+ zip.write("version: 0.2\n".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ return bytes.toByteArray();
+ }
+
+ private record Fixture(CodeBuildRunner runner, Build build, Project project, CountDownLatch finished) {
+ void run() throws InterruptedException {
+ runner.startBuild("us-east-1", build, project, null);
+ assertTrue(finished.await(10, TimeUnit.SECONDS), "build did not finish");
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildSourceCopyTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildSourceCopyTest.java
new file mode 100644
index 0000000000..fee4445682
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildSourceCopyTest.java
@@ -0,0 +1,93 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import io.github.hectorvent.floci.testing.DockerArchiveUploadFixture;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+
+class CodeBuildSourceCopyTest {
+
+ @Test
+ void retriesSourceUploadWithFreshTarAfterBrokenPipe(@TempDir Path workspace) throws IOException {
+ Files.writeString(workspace.resolve("buildspec.yml"), "version: 0.2\n");
+ DockerClient docker = mock(DockerClient.class);
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "build-container");
+ uploads.failures = 1;
+ uploads.consumedBytes = 8;
+ uploads.failure = new RuntimeException(new IOException("Broken pipe"));
+
+ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)
+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src", 0);
+
+ assertEquals(2, uploads.commands.size());
+ assertNotSame(uploads.streams.get(0), uploads.streams.get(1));
+ for (CopyArchiveToContainerCmd command : uploads.commands) {
+ verify(command).withRemotePath("/codebuild/output/src/src");
+ }
+ try (TarArchiveInputStream tar = new TarArchiveInputStream(
+ new ByteArrayInputStream(uploads.completed.getFirst()))) {
+ TarArchiveEntry entry = tar.getNextEntry();
+ assertEquals("buildspec.yml", entry.getName());
+ assertEquals("version: 0.2\n", new String(tar.readAllBytes(), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ void propagatesFailureAfterTransientRetryBudgetIsExhausted(@TempDir Path workspace) throws IOException {
+ Files.writeString(workspace.resolve("buildspec.yml"), "version: 0.2\n");
+ DockerClient docker = mock(DockerClient.class);
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "build-container");
+ uploads.failures = 10;
+ uploads.failure = new RuntimeException(new IOException("Broken pipe"));
+
+ RuntimeException failure = assertThrows(RuntimeException.class, () ->
+ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)
+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src", 0));
+
+ assertEquals(6, uploads.commands.size());
+ assertTrue(failure.getMessage().contains("codebuild-source"));
+ }
+
+ @Test
+ void propagatesSourceUploadFailureInsteadOfRunningWithoutSource(@TempDir Path workspace) throws IOException {
+ Files.writeString(workspace.resolve("buildspec.yml"), "version: 0.2\n");
+ DockerClient docker = mock(DockerClient.class);
+ DockerArchiveUploadFixture uploads = new DockerArchiveUploadFixture(docker, "build-container");
+ uploads.failures = 10;
+ uploads.failure = new IllegalStateException("No such container");
+
+ RuntimeException failure = assertThrows(RuntimeException.class, () ->
+ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)
+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src", 0));
+
+ assertTrue(failure.getMessage().contains("No such container"));
+ assertEquals(1, uploads.commands.size());
+ }
+
+ @Test
+ void skipsEmptyWorkspace(@TempDir Path workspace) {
+ DockerClient docker = mock(DockerClient.class);
+
+ new CodeBuildRunner(docker, null, null, null, null, null, null, null, null, null, null)
+ .copySourceToContainer("build-container", workspace, "/codebuild/output/src/src");
+
+ verifyNoInteractions(docker);
+ }
+}