Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(codepipeline): execute ThirdParty GitHub version 1 source actions (#4615)

Sam Fakhreddinebcf104553630
diff --git a/docs/services/codepipeline.md b/docs/services/codepipeline.md
index 25e7edf581..1de95679a1 100644
--- a/docs/services/codepipeline.md
+++ b/docs/services/codepipeline.md
@@ -52,6 +52,7 @@ The following providers execute against local Floci services:
| Category | Provider | Behavior |
|---|---|---|
| Source | S3 | Reads the configured object and publishes the output artifact |
+| Source | GitHub (ThirdParty, v1) | Downloads the configured branch archive from github.com and publishes it with the repo contents at the artifact root |
| Build/Test | CodeBuild | Starts and monitors the configured local CodeBuild project |
| Deploy | S3 | Writes the input artifact to the configured bucket and key |
| Deploy | CodeDeploy | Starts and monitors a local CodeDeploy deployment |
diff --git a/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java b/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java
index fcaca77c63..0afe7d1c12 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java
@@ -29,15 +29,31 @@ import jakarta.annotation.PreDestroy;
import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.utils.SeekableInMemoryByteChannel;
import org.jboss.logging.Logger;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.Enumeration;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -50,6 +66,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.zip.CRC32;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+import java.util.zip.ZipEntry;
@ApplicationScoped
public class CodePipelineService {
@@ -60,6 +80,26 @@ public class CodePipelineService {
private static final long POLL_INTERVAL_MS = 100L;
private static final String SOURCE_POLL_TYPE = "source-poll";
private static final String MISSING_SOURCE_REVISION = "missing";
+ private static final long MAX_ARCHIVE_BYTES = 128L << 20;
+ private static final long MAX_UNCOMPRESSED_BYTES = 128L << 20;
+ private static final int MAX_ARCHIVE_ENTRIES = 100_000;
+
+ // Overridable accessors so tests can use tiny caps and a local server instead of 1 GiB archives.
+ protected long maxArchiveBytes() {
+ return MAX_ARCHIVE_BYTES;
+ }
+
+ protected long maxUncompressedBytes() {
+ return MAX_UNCOMPRESSED_BYTES;
+ }
+
+ protected int maxEntries() {
+ return MAX_ARCHIVE_ENTRIES;
+ }
+
+ protected String githubArchiveBaseUrl() {
+ return "https://codeload.github.com";
+ }
private final AccountAwareStorageBackend<CodePipelinePipeline> pipelineStore;
private final AccountAwareStorageBackend<CodePipelineExecution> executionStore;
@@ -1340,6 +1380,11 @@ public class CodePipelineService {
waitForApproval(execution, state);
return;
}
+ if ("Source".equals(category) && "GitHub".equals(provider) && "ThirdParty".equals(owner)
+ && "1".equals(action.path("actionTypeId").path("version").asText())) {
+ executeGitHubSource(execution, action, state);
+ return;
+ }
if (!"AWS".equals(owner)) {
waitForCustomJob(pipeline, execution, action, state);
return;
@@ -1394,6 +1439,294 @@ public class CodePipelineService {
state.setExternalExecutionId("s3://" + bucket + "/" + objectKey);
}
+ /**
+ * The ThirdParty GitHub (version 1) source action: fetches the branch archive from
+ * github.com and publishes it as the output artifact with the repo contents at the
+ * artifact root, the layout the real action produces.
+ */
+ private void executeGitHubSource(CodePipelineExecution execution, JsonNode action,
+ ActionExecution state) {
+ JsonNode config = action.path("configuration");
+ String repoOwner = config.path("Owner").asText(null);
+ String repo = config.path("Repo").asText(null);
+ String branch = config.path("Branch").asText("main");
+ if (repoOwner == null || repo == null) {
+ throw new AwsException("InvalidActionDeclarationException",
+ "GitHub source actions require Owner and Repo", 400);
+ }
+ if (!isValidGitHubPathSegment(repoOwner) || !isValidGitHubPathSegment(repo)
+ || !isValidGitHubRef(branch)) {
+ throw new AwsException("InvalidActionDeclarationException",
+ "GitHub source Owner and Repo must be non-empty and contain no '/' or '..', "
+ + "and Branch must be non-empty with no leading '/', '..' or whitespace", 400);
+ }
+ byte[] archive = fetchGitHubArchive(URI.create(
+ githubArchiveBaseUrl() + "/" + repoOwner + "/" + repo
+ + "/zip/refs/heads/" + encodeRefPath(branch)));
+ if (archive.length > maxArchiveBytes()) {
+ throw archiveDownloadTooLarge();
+ }
+ String commitSha = archiveCommitSha(archive);
+ byte[] artifact = stripTopLevelDirectory(archive);
+ for (JsonNode output : action.path("outputArtifacts")) {
+ runtimeArtifacts.put(artifactKey(execution, output.path("name").asText()), artifact);
+ }
+ Map<String, Object> revision = new LinkedHashMap<>();
+ revision.put("name", state.getActionName());
+ String summary = "github.com/" + repoOwner + "/" + repo + "@" + branch;
+ String commitUrl = "https://github.com/" + repoOwner + "/" + repo + "/commit/" + commitSha;
+ revision.put("revisionId", commitSha);
+ revision.put("revisionChangeIdentifier", commitSha);
+ revision.put("revisionSummary", summary);
+ revision.put("revisionUrl", commitUrl);
+ revision.put("created", now());
+ Map<String, Object> sourceRevision = new LinkedHashMap<>();
+ sourceRevision.put("actionName", state.getActionName());
+ sourceRevision.put("revisionId", commitSha);
+ sourceRevision.put("revisionSummary", summary);
+ sourceRevision.put("revisionUrl", commitUrl);
+ synchronized (execution) {
+ execution.getArtifactRevisions().add(revision);
+ execution.getSourceRevisions().removeIf(
+ existing -> state.getActionName().equals(existing.get("actionName")));
+ execution.getSourceRevisions().add(sourceRevision);
+ }
+ state.setExternalExecutionId(commitSha);
+ state.setExternalExecutionUrl("https://github.com/" + repoOwner + "/" + repo);
+ }
+
+ /** Percent-encodes each '/'-separated ref segment so '#', '%' and the like stay in the path. */
+ private static String encodeRefPath(String ref) {
+ List<String> segments = new ArrayList<>();
+ for (String segment : ref.split("/", -1)) {
+ segments.add(URLEncoder.encode(segment, StandardCharsets.UTF_8).replace("+", "%20"));
+ }
+ return String.join("/", segments);
+ }
+
+ /**
+ * codeload writes the commit SHA into the ZIP end-of-central-directory comment; that is the
+ * revision a GitHub source artifact reports. Anything else is not a codeload archive.
+ */
+ private static String archiveCommitSha(byte[] zip) {
+ int min = Math.max(0, zip.length - 22 - 0xFFFF);
+ for (int i = zip.length - 22; i >= min; i--) {
+ if (zip[i] == 0x50 && zip[i + 1] == 0x4b && zip[i + 2] == 0x05 && zip[i + 3] == 0x06) {
+ int len = (zip[i + 20] & 0xFF) | ((zip[i + 21] & 0xFF) << 8);
+ if (i + 22 + len <= zip.length) {
+ String comment = new String(zip, i + 22, len, StandardCharsets.US_ASCII).trim();
+ if (comment.matches("[0-9a-fA-F]{40}")) {
+ return comment.toLowerCase(Locale.ROOT);
+ }
+ }
+ break;
+ }
+ }
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source archive does not carry a commit SHA in its ZIP comment", 400);
+ }
+
+ /** repoOwner/repo path segments: no '/', no '..', non-empty. */
+ private static boolean isValidGitHubPathSegment(String segment) {
+ return !segment.isEmpty() && !segment.contains("/") && !segment.contains("..");
+ }
+
+ /** Branch ref: slashes are legal (e.g. {@code release/v1.16.0}), but no '..' traversal,
+ * leading slash, or whitespace/control characters that could reshape the request path. */
+ private static boolean isValidGitHubRef(String ref) {
+ return !ref.isEmpty() && !ref.startsWith("/") && !ref.contains("..")
+ && ref.chars().noneMatch(Character::isWhitespace);
+ }
+
+ /** Overridable seam for tests; production goes to github.com with the JVM's proxy settings. */
+ byte[] fetchGitHubArchive(URI uri) {
+ try {
+ HttpClient client = HttpClient.newBuilder()
+ .connectTimeout(Duration.ofSeconds(10))
+ .followRedirects(HttpClient.Redirect.NORMAL)
+ .build();
+ HttpResponse<InputStream> response = client.send(
+ HttpRequest.newBuilder(uri)
+ .timeout(Duration.ofSeconds(30))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofInputStream());
+ try (InputStream body = response.body()) {
+ if (response.statusCode() != 200) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source download returned HTTP " + response.statusCode()
+ + " for " + uri, 400);
+ }
+ if (response.headers().firstValueAsLong("Content-Length").orElse(0L) > maxArchiveBytes()) {
+ throw archiveDownloadTooLarge();
+ }
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ byte[] chunk = new byte[8192];
+ long total = 0;
+ int n;
+ while ((n = body.read(chunk)) >= 0) {
+ total += n;
+ if (total > maxArchiveBytes()) {
+ throw archiveDownloadTooLarge();
+ }
+ buffer.write(chunk, 0, n);
+ }
+ return buffer.toByteArray();
+ }
+ } catch (AwsException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source download failed: " + e.getMessage(), 400);
+ }
+ }
+
+ private AwsException archiveDownloadTooLarge() {
+ return new AwsException("ActionExecutionFailed",
+ "GitHub source archive exceeds the maximum archive download size of "
+ + maxArchiveBytes() + " bytes", 400);
+ }
+
+ /**
+ * GitHub's codeload archives wrap the repo in a {@code <repo>-<branch>/} directory;
+ * the real source artifact has the repo contents at the root.
+ */
+ private byte[] stripTopLevelDirectory(byte[] zip) {
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ZipFile zipFile = ZipFile.builder()
+ .setSeekableByteChannel(new SeekableInMemoryByteChannel(zip)).get();
+ ZipArchiveOutputStream zos = new ZipArchiveOutputStream(baos)) {
+ Enumeration<ZipArchiveEntry> entries = zipFile.getEntriesInPhysicalOrder();
+ int entryCount = 0;
+ long uncompressedTotal = 0;
+ while (entries.hasMoreElements()) {
+ ZipArchiveEntry entry = entries.nextElement();
+ if (++entryCount > maxEntries()) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source archive has more than the maximum of "
+ + maxEntries() + " entries", 400);
+ }
+ if (entry.getName().startsWith("/")) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source archive contains an escaping entry: " + entry.getName(), 400);
+ }
+ int slash = entry.getName().indexOf('/');
+ if (slash < 0 || slash == entry.getName().length() - 1) {
+ continue;
+ }
+ String stripped = entry.getName().substring(slash + 1);
+ if (isEscapingPath(stripped)) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source archive contains an escaping entry: " + entry.getName(), 400);
+ }
+ if (entry.isDirectory()) {
+ continue;
+ }
+ // Symlink entries (unix mode S_IFLNK, content = link target) are not
+ // directories, so they flow through this copy path: the unix mode and
+ // the content are kept as-is in the repackaged artifact. This only
+ // preserves the entries; the current CodeBuild extractZip writes every
+ // entry as a plain file and does not recreate symlinks.
+ ZipArchiveEntry copy = new ZipArchiveEntry(stripped);
+ if (entry.getUnixMode() != 0) {
+ copy.setUnixMode(entry.getUnixMode());
+ }
+ zos.putArchiveEntry(copy);
+ boolean symlink = (entry.getUnixMode() & 0xF000) == 0xA000;
+ ByteArrayOutputStream linkTarget = new ByteArrayOutputStream();
+ CRC32 crc = new CRC32();
+ try (InputStream in = openEntryStream(zipFile, entry)) {
+ byte[] chunk = new byte[8192];
+ int n;
+ while ((n = in.read(chunk)) >= 0) {
+ uncompressedTotal += n;
+ if (uncompressedTotal > maxUncompressedBytes()) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source archive exceeds the maximum uncompressed size of "
+ + maxUncompressedBytes() + " bytes", 400);
+ }
+ crc.update(chunk, 0, n);
+ if (symlink) {
+ linkTarget.write(chunk, 0, n);
+ }
+ zos.write(chunk, 0, n);
+ }
+ }
+ if (entry.getCrc() != -1 && crc.getValue() != entry.getCrc()) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source archive entry fails its checksum: " + entry.getName(), 400);
+ }
+ if (symlink && isEscapingSymlink(stripped, linkTarget.toString(StandardCharsets.UTF_8))) {
+ throw new AwsException("ActionExecutionFailed",
+ "GitHub source archive contains an escaping symlink: " + entry.getName(), 400);
+ }
+ zos.closeArchiveEntry();
+ }
+ }
+ return baos.toByteArray();
+ } catch (AwsException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AwsException("ActionExecutionFailed",
+ "Could not repackage GitHub archive: " + e.getMessage(), 400);
+ }
+ }
+
+ private static boolean isEscapingPath(String name) {
+ if (name.startsWith("/") || name.startsWith("\\")) {
+ return true;
+ }
+ for (String segment : name.split("[/\\\\]")) {
+ if (segment.equals("..")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** A symlink target is escaping if absolute or if, resolved against the link's directory, it leaves the root. */
+ private static boolean isEscapingSymlink(String linkName, String target) {
+ if (target.startsWith("/") || target.startsWith("\\")) {
+ return true;
+ }
+ Path root = Path.of("/root");
+ Path linkDir = root.resolve(linkName).getParent();
+ return !linkDir.resolve(target).normalize().startsWith(root);
+ }
+
+ /** Ends the Inflater when the stream closes; InflaterInputStream leaves a caller-supplied one open. */
+ private static final class EndingInflaterInputStream extends InflaterInputStream {
+ private EndingInflaterInputStream(InputStream in) {
+ super(in, new Inflater(true));
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ super.close();
+ } finally {
+ inf.end();
+ }
+ }
+ }
+
+ // ZipFile's decoding stream accessor dispatches on entry.getMethod() through a switch that also
+ // covers BZIP2, DEFLATE64, XZ and ZSTD, instantiating the matching CompressorInputStream
+ // inline; GraalVM's static analysis resolves every branch of that switch because the method
+ // itself is reachable, not just the branch a given entry happens to take, and XZ/ZSTD need
+ // org.tukaani:xz / com.github.luben:zstd-jni on the classpath at native link time to do so.
+ // GitHub archives are ordinary zips (STORED or DEFLATED only), so decoding just those two
+ // methods off the entry's raw bytes avoids the dispatch method and the optional codecs.
+ private static InputStream openEntryStream(ZipFile zipFile, ZipArchiveEntry entry) throws IOException {
+ InputStream raw = zipFile.getRawInputStream(entry);
+ return switch (entry.getMethod()) {
+ case ZipEntry.STORED -> raw;
+ case ZipEntry.DEFLATED -> new EndingInflaterInputStream(raw);
+ default -> throw new IOException(
+ "Unsupported zip compression method " + entry.getMethod() + " for entry " + entry.getName());
+ };
+ }
+
private void executeCodeBuild(CodePipelineExecution execution, JsonNode action,
ActionExecution state) throws InterruptedException {
String projectName = action.path("configuration").path("ProjectName").asText(null);
diff --git a/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineGitHubSourceServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineGitHubSourceServiceTest.java
new file mode 100644
index 0000000000..85fd11fbc5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineGitHubSourceServiceTest.java
@@ -0,0 +1,561 @@
+package io.github.hectorvent.floci.services.codepipeline;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sun.net.httpserver.HttpServer;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.codebuild.CodeBuildService;
+import io.github.hectorvent.floci.services.codedeploy.CodeDeployService;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.utils.SeekableInMemoryByteChannel;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.zip.CRC32;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+import java.util.zip.ZipEntry;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+
+/**
+ * ThirdParty GitHub (version 1) source actions: the branch archive is fetched, repackaged with
+ * the repo contents at the artifact root, and handed to the next stage with unix modes intact.
+ * The archive fetch is the only seam replaced; the artifact is observed through an S3 deploy
+ * action that writes the input artifact to the mocked S3 service.
+ */
+class CodePipelineGitHubSourceServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final S3Service s3Service = mock(S3Service.class);
+ private final List<String> fetched = new ArrayList<>();
+ private final List<String> requestedPaths = new CopyOnWriteArrayList<>();
+ private static final long DEFAULT_CAP = 128L << 20;
+ private static final int DEFAULT_ENTRIES = 100_000;
+
+ private CodePipelineService service;
+ private HttpServer server;
+ private static final String SHA = "0123456789abcdef0123456789abcdef01234567";
+
+ @AfterEach
+ void shutDown() {
+ if (server != null) {
+ server.stop(0);
+ }
+ if (service != null) {
+ service.shutdown();
+ }
+ }
+
+ @Test
+ void downloadsBranchArchiveAndPublishesRepoContentsAtTheArtifactRoot() throws Exception {
+ // Catches: the codeload wrapper directory left in the artifact, or the wrong URL/revision recorded
+ byte[] archive = zip(zos -> {
+ dir(zos, "repo-main/");
+ file(zos, "repo-main/README.md", "hello", 0, ZipEntry.DEFLATED);
+ file(zos, "repo-main/src/app.ts", "code", 0, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive);
+ createPipeline("github-sourced", "awslabs", "landing-zone-accelerator-on-aws", "release/v1.16.0");
+
+ String executionId = start("github-sourced");
+ JsonNode execution = awaitStatus("github-sourced", executionId, "Succeeded");
+
+ assertEquals(List.of("https://codeload.github.com/awslabs/landing-zone-accelerator-on-aws"
+ + "/zip/refs/heads/release/v1.16.0"), fetched);
+ assertEquals("github.com/awslabs/landing-zone-accelerator-on-aws@release/v1.16.0",
+ execution.path("artifactRevisions").get(0).path("revisionSummary").asText());
+ try (ZipFile artifact = deployedArtifact("github-sourced-out.zip")) {
+ assertEquals("hello", read(artifact, "README.md"));
+ assertEquals("code", read(artifact, "src/app.ts"));
+ assertNull(artifact.getEntry("repo-main/README.md"));
+ }
+ }
+
+ @Test
+ void rejectsPathTraversalInOwnerRepoOrBranch() throws Exception {
+ // Catches: a pipeline-declared Repo reshaping the codeload request path
+ service = serviceServing(new byte[0]);
+ createPipeline("traversal-repo", "awslabs", "../../etc", "main");
+
+ String executionId = start("traversal-repo");
+ awaitStatus("traversal-repo", executionId, "Failed");
+
+ assertEquals(List.of(), fetched);
+ JsonNode actions = service.handle("ListActionExecutions",
+ mapper.createObjectNode().put("pipelineName", "traversal-repo"), REGION, ACCOUNT)
+ .path("actionExecutionDetails");
+ assertTrue(actions.toString().contains("must be non-empty and contain no"), actions.toString());
+ }
+
+ @Test
+ void repackagingPreservesUnixFileModes() throws Exception {
+ // Catches: unix mode bits dropped from the repackaged artifact entries
+ byte[] archive = zip(zos -> {
+ dir(zos, "repo-main/");
+ file(zos, "repo-main/lib/bash/bootstrap.sh", "#!/bin/bash\n", 0755, ZipEntry.DEFLATED);
+ file(zos, "repo-main/README.md", "hello", 0, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive);
+ createPipeline("github-modes", "awslabs", "lza", "main");
+
+ awaitStatus("github-modes", start("github-modes"), "Succeeded");
+
+ try (ZipFile artifact = deployedArtifact("github-modes-out.zip")) {
+ assertEquals(0755, artifact.getEntry("lib/bash/bootstrap.sh").getUnixMode());
+ assertEquals(0, artifact.getEntry("README.md").getUnixMode());
+ }
+ }
+
+ @Test
+ void repackagingPreservesSymlinks() throws Exception {
+ // Catches: symlink entries flattened in the artifact (mode and link target lost)
+ byte[] archive = zip(zos -> {
+ dir(zos, "repo-main/");
+ file(zos, "repo-main/node_modules/ts-node/dist/bin.js", "#!/usr/bin/env node\n", 0755, ZipEntry.DEFLATED);
+ file(zos, "repo-main/node_modules/.bin/ts-node", "../ts-node/dist/bin.js", 0120777, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive);
+ createPipeline("github-symlinks", "awslabs", "lza", "main");
+
+ awaitStatus("github-symlinks", start("github-symlinks"), "Succeeded");
+
+ try (ZipFile artifact = deployedArtifact("github-symlinks-out.zip")) {
+ ZipArchiveEntry link = artifact.getEntry("node_modules/.bin/ts-node");
+ assertEquals(0xA000, link.getUnixMode() & 0xF000);
+ assertEquals("../ts-node/dist/bin.js", read(artifact, "node_modules/.bin/ts-node"));
+ assertEquals(0755, artifact.getEntry("node_modules/ts-node/dist/bin.js").getUnixMode());
+ }
+ }
+
+ @Test
+ void repackagesStoredAndDeflatedEntriesWithoutTheCodecDispatchPath() throws Exception {
+ // Catches: entry decoding that only handles one method, or that routes through
+ // ZipFile.getInputStream (ZSTD/XZ dispatch, which breaks the native image build)
+ byte[] archive = zip(zos -> {
+ dir(zos, "repo-main/");
+ file(zos, "repo-main/stored.txt", "stored payload", 0644, ZipEntry.STORED);
+ file(zos, "repo-main/deflated.txt", "deflated payload ".repeat(50), 0755, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive);
+ createPipeline("github-methods", "awslabs", "lza", "main");
+
+ awaitStatus("github-methods", start("github-methods"), "Succeeded");
+
+ try (ZipFile artifact = deployedArtifact("github-methods-out.zip")) {
+ assertEquals("stored payload", read(artifact, "stored.txt"));
+ assertEquals(0644, artifact.getEntry("stored.txt").getUnixMode());
+ assertEquals("deflated payload ".repeat(50), read(artifact, "deflated.txt"));
+ assertEquals(0755, artifact.getEntry("deflated.txt").getUnixMode());
+ }
+ }
+
+ @Test
+ void failsWhenTheDownloadedArchiveExceedsTheSizeCap() throws Exception {
+ // Catches: an oversized archive accepted and repackaged in memory instead of failing the action
+ byte[] archive = zip(zos -> file(zos, "repo-main/big.bin", "x".repeat(4096), 0, ZipEntry.STORED));
+ service = serviceServing(archive, 100, DEFAULT_CAP, DEFAULT_ENTRIES);
+ createPipeline("github-big-download", "awslabs", "lza", "main");
+
+ awaitStatus("github-big-download", start("github-big-download"), "Failed");
+
+ assertFailureMessage("github-big-download", "maximum archive download size of 100 bytes");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void failsWhenUncompressedContentExceedsTheSizeCap() throws Exception {
+ // Catches: a zip bomb (small archive, huge inflated content) repackaged without a total-bytes bound
+ byte[] archive = zip(zos -> {
+ file(zos, "repo-main/a.txt", "a".repeat(600), 0, ZipEntry.DEFLATED);
+ file(zos, "repo-main/b.txt", "b".repeat(600), 0, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive, DEFAULT_CAP, 1000, DEFAULT_ENTRIES);
+ createPipeline("github-bomb", "awslabs", "lza", "main");
+
+ awaitStatus("github-bomb", start("github-bomb"), "Failed");
+
+ assertFailureMessage("github-bomb", "maximum uncompressed size of 1000 bytes");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void failsWhenTheArchiveHasTooManyEntries() throws Exception {
+ // Catches: an archive with an unbounded entry count repackaged instead of failing the action
+ byte[] archive = zip(zos -> {
+ file(zos, "repo-main/1.txt", "1", 0, ZipEntry.DEFLATED);
+ file(zos, "repo-main/2.txt", "2", 0, ZipEntry.DEFLATED);
+ file(zos, "repo-main/3.txt", "3", 0, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive, DEFAULT_CAP, DEFAULT_CAP, 2);
+ createPipeline("github-many", "awslabs", "lza", "main");
+
+ awaitStatus("github-many", start("github-many"), "Failed");
+
+ assertFailureMessage("github-many", "maximum of 2 entries");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void rejectsAbsoluteEntryNames() throws Exception {
+ // Catches: a leading-slash entry whose slash is eaten as the top-level separator and so slips past validation
+ byte[] archive = zip(zos -> {
+ file(zos, "repo-main/ok.txt", "ok", 0, ZipEntry.DEFLATED);
+ file(zos, "/repo-main/evil.txt", "evil", 0, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive);
+ createPipeline("github-absolute", "awslabs", "lza", "main");
+
+ awaitStatus("github-absolute", start("github-absolute"), "Failed");
+
+ assertFailureMessage("github-absolute", "escaping entry");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void rejectsEntriesWithParentDirectorySegments() throws Exception {
+ // Catches: a ../ entry repackaged into the artifact so extraction writes outside the workspace
+ byte[] archive = zip(zos -> {
+ file(zos, "repo-main/ok.txt", "ok", 0, ZipEntry.DEFLATED);
+ file(zos, "repo-main/../../evil.txt", "evil", 0, ZipEntry.DEFLATED);
+ });
+ service = serviceServing(archive);
+ createPipeline("github-dotdot", "awslabs", "lza", "main");
+
+ awaitStatus("github-dotdot", start("github-dotdot"), "Failed");
+
+ assertFailureMessage("github-dotdot", "escaping entry");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void rejectsSymlinksResolvingOutsideTheRoot() throws Exception {
+ // Catches: a relative symlink climbing out of the root repackaged as-is
+ byte[] archive = zip(zos -> file(zos, "repo-main/a/link", "../../outside", 0120777, ZipEntry.DEFLATED));
+ service = serviceServing(archive);
+ createPipeline("github-link-out", "awslabs", "lza", "main");
+
+ awaitStatus("github-link-out", start("github-link-out"), "Failed");
+
+ assertFailureMessage("github-link-out", "escaping symlink");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void rejectsSymlinksWithAbsoluteTargets() throws Exception {
+ // Catches: an absolute symlink target (e.g. /etc/passwd) repackaged into the artifact
+ byte[] archive = zip(zos -> file(zos, "repo-main/link", "/etc/passwd", 0120777, ZipEntry.DEFLATED));
+ service = serviceServing(archive);
+ createPipeline("github-link-abs", "awslabs", "lza", "main");
+
+ awaitStatus("github-link-abs", start("github-link-abs"), "Failed");
+
+ assertFailureMessage("github-link-abs", "escaping symlink");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void doesNotInterceptCustomActionsNamedGitHub() throws Exception {
+ // Catches: a Custom-owner Source action with provider GitHub downloaded from codeload instead of reaching its worker
+ service = serviceServing(zip(zos -> file(zos, "repo-main/a.txt", "a", 0, ZipEntry.DEFLATED)));
+ createPipeline("github-custom", "Custom", "1", "awslabs", "lza", "main");
+
+ String executionId = start("github-custom");
+
+ // The action must stay with its worker for a sustained window: no codeload fetch, execution still InProgress.
+ await().during(Duration.ofMillis(400)).atMost(Duration.ofSeconds(5)).untilAsserted(() -> {
+ assertEquals(List.of(), fetched);
+ assertEquals("InProgress", service.handle("GetPipelineExecution", mapper.createObjectNode()
+ .put("pipelineName", "github-custom").put("pipelineExecutionId", executionId),
+ REGION, ACCOUNT).path("pipelineExecution").path("status").asText());
+ });
+ }
+
+ @Test
+ void recordsTheCommitFromTheArchiveCommentAsTheSourceRevision() throws Exception {
+ // Catches: the branch name reported as the revision, or no sourceRevisions entry for ListPipelineExecutions
+ service = serviceServing(zip(zos -> file(zos, "repo-main/a.txt", "a", 0, ZipEntry.DEFLATED)));
+ createPipeline("github-commit", "awslabs", "lza", "main");
+
+ JsonNode execution = awaitStatus("github-commit", start("github-commit"), "Succeeded");
+
+ JsonNode artifactRevision = execution.path("artifactRevisions").get(0);
+ assertEquals(SHA, artifactRevision.path("revisionId").asText());
+ assertEquals(SHA, artifactRevision.path("revisionChangeIdentifier").asText());
+ assertEquals("https://github.com/awslabs/lza/commit/" + SHA, artifactRevision.path("revisionUrl").asText());
+ JsonNode sourceRevision = service.handle("ListPipelineExecutions",
+ mapper.createObjectNode().put("pipelineName", "github-commit"), REGION, ACCOUNT)
+ .path("pipelineExecutionSummaries").get(0).path("sourceRevisions").get(0);
+ assertEquals("GitHubSource", sourceRevision.path("actionName").asText());
+ assertEquals(SHA, sourceRevision.path("revisionId").asText());
+ assertEquals("https://github.com/awslabs/lza/commit/" + SHA, sourceRevision.path("revisionUrl").asText());
+ }
+
+ @Test
+ void failsWhenTheArchiveCarriesNoCommitSha() throws Exception {
+ // Catches: a silent fallback to the branch name when the archive comment is not a commit SHA
+ service = serviceServing(zip(null, zos -> file(zos, "repo-main/a.txt", "a", 0, ZipEntry.DEFLATED)));
+ createPipeline("github-nosha", "awslabs", "lza", "main");
+
+ awaitStatus("github-nosha", start("github-nosha"), "Failed");
+
+ assertFailureMessage("github-nosha", "commit SHA");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void encodesBranchCharactersThatAreSpecialInAUri() throws Exception {
+ // Catches: a '#' branch turned into a URI fragment (downloading the wrong ref) or '%' breaking the URI
+ byte[] archive = zip(zos -> file(zos, "repo-main/a.txt", "a", 0, ZipEntry.DEFLATED));
+ startArchiveServer(archive, false, DEFAULT_CAP);
+ createPipeline("github-hash", "awslabs", "lza", "feat#1/x%y");
+
+ awaitStatus("github-hash", start("github-hash"), "Succeeded");
+
+ assertEquals(List.of("/awslabs/lza/zip/refs/heads/feat%231/x%25y"), requestedPaths);
+ }
+
+ @Test
+ void abortsAChunkedDownloadThatExceedsTheCapWhileItIsRead() throws Exception {
+ // Catches: a response without Content-Length buffered in full before the size cap is applied
+ byte[] archive = zip(zos -> file(zos, "repo-main/big.bin", "x".repeat(4096), 0, ZipEntry.STORED));
+ startArchiveServer(archive, true, 100);
+ createPipeline("github-chunked", "awslabs", "lza", "main");
+
+ awaitStatus("github-chunked", start("github-chunked"), "Failed");
+
+ assertFailureMessage("github-chunked", "maximum archive download size of 100 bytes");
+ verifyNoInteractions(s3Service);
+ }
+
+ @Test
+ void rejectsAnEntryWhoseContentDoesNotMatchItsRecordedChecksum() throws Exception {
+ // Catches: a corrupted entry re-checksummed by the repackaging and reported as a good source
+ byte[] archive = zip(zos -> file(zos, "repo-main/a.txt", "hello world", 0, ZipEntry.STORED));
+ String text = new String(archive, StandardCharsets.ISO_8859_1);
+ byte[] corrupt = text.replaceFirst("hello world", "jello world").getBytes(StandardCharsets.ISO_8859_1);
+ service = serviceServing(corrupt);
+ createPipeline("github-crc", "awslabs", "lza", "main");
+
+ awaitStatus("github-crc", start("github-crc"), "Failed");
+
+ assertFailureMessage("github-crc", "checksum");
+ verifyNoInteractions(s3Service);
+ }
+
+ private void startArchiveServer(byte[] archive, boolean chunked, long archiveCap) throws Exception {
+ server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ server.createContext("/", exchange -> {
+ requestedPaths.add(exchange.getRequestURI().getRawPath());
+ exchange.sendResponseHeaders(200, chunked ? 0 : archive.length);
+ try (OutputStream body = exchange.getResponseBody()) {
+ body.write(archive);
+ } catch (IOException ignored) {
+ // the client aborts an oversized chunked download mid-stream
+ }
+ });
+ server.start();
+ String baseUrl = "http://127.0.0.1:" + server.getAddress().getPort();
+ service = new CodePipelineService(new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class),
+ mock(CodeDeployService.class), mock(LambdaService.class), s3Service) {
+ @Override
+ protected String githubArchiveBaseUrl() {
+ return baseUrl;
+ }
+
+ @Override
+ protected long maxArchiveBytes() {
+ return archiveCap;
+ }
+ };
+ }
+
+ private void assertFailureMessage(String pipelineName, String expected) {
+ JsonNode actions = service.handle("ListActionExecutions",
+ mapper.createObjectNode().put("pipelineName", pipelineName), REGION, ACCOUNT)
+ .path("actionExecutionDetails");
+ assertTrue(actions.toString().contains(expected), actions.toString());
+ }
+
+ private CodePipelineService serviceServing(byte[] archive) {
+ return serviceServing(archive, DEFAULT_CAP, DEFAULT_CAP, DEFAULT_ENTRIES);
+ }
+
+ private CodePipelineService serviceServing(byte[] archive, long archiveCap, long uncompressedCap,
+ int entryCap) {
+ return new CodePipelineService(new InMemoryStorageFactory(), mapper, mock(CodeBuildService.class),
+ mock(CodeDeployService.class), mock(LambdaService.class), s3Service) {
+ @Override
+ byte[] fetchGitHubArchive(URI uri) {
+ fetched.add(uri.toString());
+ return archive;
+ }
+
+ @Override
+ protected long maxArchiveBytes() {
+ return archiveCap;
+ }
+
+ @Override
+ protected long maxUncompressedBytes() {
+ return uncompressedCap;
+ }
+
+ @Override
+ protected int maxEntries() {
+ return entryCap;
+ }
+ };
+ }
+
+ private void createPipeline(String name, String owner, String repo, String branch) throws Exception {
+ createPipeline(name, "ThirdParty", "1", owner, repo, branch);
+ }
+
+ private void createPipeline(String name, String actionOwner, String actionVersion, String owner,
+ String repo, String branch) throws Exception {
+ service.handle("CreatePipeline", mapper.readTree("""
+ {"pipeline": {
+ "name": "%s",
+ "roleArn": "arn:aws:iam::000000000000:role/cp",
+ "artifactStore": {"type": "S3", "location": "bucket"},
+ "stages": [{
+ "name": "Fetch",
+ "actions": [{
+ "name": "GitHubSource",
+ "actionTypeId": {"category": "Source", "owner": "%s", "provider": "GitHub", "version": "%s"},
+ "configuration": {"Owner": "%s", "Repo": "%s", "Branch": "%s"},
+ "outputArtifacts": [{"name": "Source"}],
+ "runOrder": 1
+ }]
+ }, {
+ "name": "Publish",
+ "actions": [{
+ "name": "WriteOut",
+ "actionTypeId": {"category": "Deploy", "owner": "AWS", "provider": "S3", "version": "1"},
+ "configuration": {"BucketName": "out", "ObjectKey": "%s-out.zip"},
+ "inputArtifacts": [{"name": "Source"}],
+ "runOrder": 1
+ }]
+ }]
+ }}
+ """.formatted(name, actionOwner, actionVersion, owner, repo, branch, name)), REGION, ACCOUNT);
+ }
+
+ private String start(String pipelineName) {
+ return service.handle("StartPipelineExecution",
+ mapper.createObjectNode().put("name", pipelineName), REGION, ACCOUNT)
+ .path("pipelineExecutionId").asText();
+ }
+
+ private JsonNode awaitStatus(String pipelineName, String executionId, String expected) {
+ return await().atMost(Duration.ofSeconds(5)).pollInterval(Duration.ofMillis(25))
+ .until(() -> service.handle("GetPipelineExecution", mapper.createObjectNode()
+ .put("pipelineName", pipelineName).put("pipelineExecutionId", executionId),
+ REGION, ACCOUNT).path("pipelineExecution"),
+ execution -> expected.equals(execution.path("status").asText()));
+ }
+
+ private ZipFile deployedArtifact(String objectKey) throws Exception {
+ ArgumentCaptor<byte[]> data = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service).putObject(eq("out"), eq(objectKey), data.capture(),
+ eq("application/zip"), eq(Map.of()));
+ return ZipFile.builder().setSeekableByteChannel(new SeekableInMemoryByteChannel(data.getValue())).get();
+ }
+
+ private static String read(ZipFile zip, String name) throws Exception {
+ // Raw bytes: the artifact is repackaged with DEFLATED or STORED entries only.
+ byte[] raw = zip.getRawInputStream(zip.getEntry(name)).readAllBytes();
+ if (zip.getEntry(name).getMethod() == ZipEntry.STORED) {
+ return new String(raw, StandardCharsets.UTF_8);
+ }
+ try (InflaterInputStream in = new InflaterInputStream(
+ new ByteArrayInputStream(raw), new Inflater(true))) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ private interface ZipWriter {
+ void write(ZipArchiveOutputStream zos) throws Exception;
+ }
+
+ private static byte[] zip(ZipWriter writer) throws Exception {
+ return zip(SHA, writer);
+ }
+
+ private static byte[] zip(String comment, ZipWriter writer) throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(out)) {
+ if (comment != null) {
+ zos.setComment(comment);
+ }
+ writer.write(zos);
+ }
+ return out.toByteArray();
+ }
+
+ private static void dir(ZipArchiveOutputStream zos, String name) throws Exception {
+ zos.putArchiveEntry(new ZipArchiveEntry(name));
+ zos.closeArchiveEntry();
+ }
+
+ private static void file(ZipArchiveOutputStream zos, String name, String content, int unixMode, int method)
+ throws Exception {
+ byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
+ ZipArchiveEntry entry = new ZipArchiveEntry(name);
+ entry.setMethod(method);
+ if (method == ZipEntry.STORED) {
+ CRC32 crc = new CRC32();
+ crc.update(bytes);
+ entry.setSize(bytes.length);
+ entry.setCrc(crc.getValue());
+ }
+ if (unixMode != 0) {
+ entry.setUnixMode(unixMode);
+ }
+ zos.putArchiveEntry(entry);
+ zos.write(bytes);
+ zos.closeArchiveEntry();
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return AccountAwareStorageBackend.inMemory(ACCOUNT);
+ }
+ }
+}