Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): read deployment packages via the zip central directory (#2649)
Davidcd3f3677f8e8
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.java b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.javaindex ca28933a21..40b566d7aa 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractor.java@@ -3,13 +3,15 @@ package io.github.hectorvent.floci.services.lambda.zip;import jakarta.enterprise.context.ApplicationScoped;import org.jboss.logging.Logger;-import java.io.ByteArrayInputStream;import java.io.IOException;+import java.io.InputStream;import java.io.OutputStream;import java.nio.file.Files;import java.nio.file.Path;+import java.util.zip.CRC32;import java.util.zip.ZipEntry;-import java.util.zip.ZipInputStream;+import java.util.zip.ZipException;+import java.util.zip.ZipFile;/*** Extracts ZIP bytes to a target directory.@@ -27,47 +29,91 @@ public class ZipExtractor {Path absTarget = targetDir.toAbsolutePath().normalize();Files.createDirectories(absTarget);- try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(zipBytes))) {- ZipEntry entry;- while ((entry = zis.getNextEntry()) != null) {- String entryName = entry.getName();+ // Read the archive through its central directory (ZipFile) rather than sequentially+ // over the local file headers (ZipInputStream). A streaming packager cannot seek back+ // to patch an entry's sizes, so it sets the data-descriptor flag (general-purpose bit+ // 3) and writes the real CRC and sizes after the data instead; anything that does not+ // compress is written STORED. ZipInputStream rejects that pairing outright with "only+ // DEFLATED entries can have EXT descriptor", because for a STORED entry it has no way+ // to find where the data ends. The central directory always carries the true method,+ // CRC and sizes regardless of the flag, so reading it accepts the same archives real+ // AWS Lambda does — e.g. a Serverless Framework package bundling node_modules+ // (issue #2593).+ //+ // ZipFile needs a seekable file, so the bytes are staged on disk. Stage inside the code+ // store rather than java.io.tmpdir: the shared temp filesystem must not become a new+ // unbounded consumer that concurrent deployments can exhaust and that unrelated+ // services would then fail on. The code store already has to hold this package's+ // *uncompressed* contents — always at least as large as the archive — so it is already+ // provisioned for the size, and the staged copy is released as soon as extraction ends.+ Path staged = Files.createTempFile(absTarget.getParent(), ".floci-staging-", ".zip");+ try {+ Files.write(staged, zipBytes);+ try (ZipFile zip = new ZipFile(staged.toFile())) {+ var entries = zip.entries();+ while (entries.hasMoreElements()) {+ ZipEntry entry = entries.nextElement();+ String entryName = entry.getName();- // PowerShell 5 Compress-Archive writes '\' as a literal filename byte on- // Linux. Real AWS Lambda does NOT normalize this (the archive extracts to- // a flat "wwwroot\app.css" file), so neither do we; masking it would let- // a broken package pass locally and then fail on deploy.- if (entryName.indexOf(BACKSLASH) >= 0) {- LOG.warnv("ZIP entry \"{0}\" uses backslash separators (PowerShell Compress-Archive). "- + "It extracts as a literal filename and will also fail on real AWS Lambda. "- + "Repackage with tar, PowerShell Core (pwsh), or the dotnet lambda CLI.", entryName);- }+ // PowerShell 5 Compress-Archive writes '\' as a literal filename byte on+ // Linux. Real AWS Lambda does NOT normalize this (the archive extracts to+ // a flat "wwwroot\app.css" file), so neither do we; masking it would let+ // a broken package pass locally and then fail on deploy.+ if (entryName.indexOf(BACKSLASH) >= 0) {+ LOG.warnv("ZIP entry \"{0}\" uses backslash separators (PowerShell Compress-Archive). "+ + "It extracts as a literal filename and will also fail on real AWS Lambda. "+ + "Repackage with tar, PowerShell Core (pwsh), or the dotnet lambda CLI.", entryName);+ }- // Security: prevent path traversal- if (entryName.contains("..") || entryName.startsWith("/")) {- LOG.warnv("Skipping suspicious ZIP entry: {0}", entryName);- zis.closeEntry();- continue;- }+ // Security: prevent path traversal+ if (entryName.contains("..") || entryName.startsWith("/")) {+ LOG.warnv("Skipping suspicious ZIP entry: {0}", entryName);+ continue;+ }- Path targetPath = absTarget.resolve(entryName).normalize();- if (!targetPath.startsWith(absTarget)) {- LOG.warnv("Skipping out-of-bounds ZIP entry: {0}", entryName);- zis.closeEntry();- continue;- }+ Path targetPath = absTarget.resolve(entryName).normalize();+ if (!targetPath.startsWith(absTarget)) {+ LOG.warnv("Skipping out-of-bounds ZIP entry: {0}", entryName);+ continue;+ }- if (entry.isDirectory()) {- Files.createDirectories(targetPath);- } else {- Files.createDirectories(targetPath.getParent());- try (OutputStream out = Files.newOutputStream(targetPath)) {- zis.transferTo(out);+ if (entry.isDirectory()) {+ Files.createDirectories(targetPath);+ } else {+ Files.createDirectories(targetPath.getParent());+ copyVerified(zip, entry, targetPath);}}- zis.closeEntry();}+ } finally {+ Files.deleteIfExists(staged);}LOG.debugv("Extracted ZIP to: {0}", absTarget);}++ /**+ * Copies one entry to disk, checking it against the CRC recorded for it in the archive.+ * ZipInputStream verified this on its own; ZipFile does not, so without an explicit check+ * a package corrupted in transit would be deployed silently as garbage code rather than+ * rejected. The message mirrors the JDK's own so existing reports stay recognisable.+ */+ private static void copyVerified(ZipFile zip, ZipEntry entry, Path targetPath) throws IOException {+ CRC32 checksum = new CRC32();+ try (InputStream in = zip.getInputStream(entry);+ OutputStream out = Files.newOutputStream(targetPath)) {+ byte[] buffer = new byte[8192];+ int read;+ while ((read = in.read(buffer)) >= 0) {+ checksum.update(buffer, 0, read);+ out.write(buffer, 0, read);+ }+ }+ long expected = entry.getCrc();+ if (expected != -1L && checksum.getValue() != expected) {+ throw new ZipException(String.format(+ "invalid entry CRC for \"%s\" (expected 0x%08x but got 0x%08x)",+ entry.getName(), expected, checksum.getValue()));+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.javaindex 14261346bb..ee962241a8 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/zip/ZipExtractorTest.java@@ -8,6 +8,11 @@ import java.io.IOException;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.zip.CRC32;+import java.util.zip.Deflater;+import java.util.zip.ZipException;import java.util.zip.ZipEntry;import java.util.zip.ZipOutputStream;@@ -70,4 +75,285 @@ class ZipExtractorTest {assertFalse(Files.exists(target.getParent().getParent().resolve("evil.sh")),"traversal entry must not escape the target dir");}++ /**+ * One entry for {@link #streamedZip}: {@code stored} selects STORED over DEFLATED,+ * as a packager does for content that does not compress.+ */+ private record StreamedEntry(String name, String content, boolean stored) {}++ /**+ * Builds a ZIP the way a streaming packager does — the Serverless Framework's+ * archiver, for one. Such a writer cannot seek back to patch an entry's header, so it+ * sets the data-descriptor flag (general-purpose bit 3), zeroes the CRC and sizes in+ * the local header, and writes the real values after the data. The central directory+ * still carries the true values. {@link ZipOutputStream} cannot emit this shape, so+ * the bytes are assembled by hand.+ */+ private static byte[] streamedZip(StreamedEntry... entries) throws IOException {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ List<int[]> localOffsets = new ArrayList<>();+ List<byte[]> names = new ArrayList<>();+ List<long[]> meta = new ArrayList<>();++ for (StreamedEntry entry : entries) {+ byte[] name = entry.name().getBytes(StandardCharsets.UTF_8);+ byte[] raw = entry.content().getBytes(StandardCharsets.UTF_8);+ byte[] payload = entry.stored() ? raw : deflate(raw);+ CRC32 crc = new CRC32();+ crc.update(raw);+ int method = entry.stored() ? 0 : 8;++ localOffsets.add(new int[]{out.size()});+ le32(out, 0x04034b50L); // local file header signature+ le16(out, 20); // version needed+ le16(out, 0x0008); // general-purpose bit 3: data descriptor follows+ le16(out, method);+ le16(out, 0); // mod time+ le16(out, 0); // mod date+ le32(out, 0); // crc-32 — deferred to the descriptor+ le32(out, 0); // compressed — deferred to the descriptor+ le32(out, 0); // uncompressed — deferred to the descriptor+ le16(out, name.length);+ le16(out, 0); // extra length+ out.write(name);+ out.write(payload);+ le32(out, 0x08074b50L); // data descriptor signature+ le32(out, crc.getValue());+ le32(out, payload.length);+ le32(out, raw.length);++ names.add(name);+ meta.add(new long[]{crc.getValue(), payload.length, raw.length, method});+ }++ int centralOffset = out.size();+ for (int i = 0; i < entries.length; i++) {+ byte[] name = names.get(i);+ long[] m = meta.get(i);+ le32(out, 0x02014b50L); // central directory header signature+ le16(out, 20); // version made by+ le16(out, 20); // version needed+ le16(out, 0x0008); // same descriptor flag as the local header+ le16(out, (int) m[3]);+ le16(out, 0);+ le16(out, 0);+ le32(out, m[0]); // crc-32 — the real value lives here+ le32(out, m[1]); // compressed — the real value lives here+ le32(out, m[2]); // uncompressed — the real value lives here+ le16(out, name.length);+ le16(out, 0); // extra length+ le16(out, 0); // comment length+ le16(out, 0); // disk number start+ le16(out, 0); // internal attributes+ le32(out, 0); // external attributes+ le32(out, localOffsets.get(i)[0]);+ out.write(name);+ }+ int centralSize = out.size() - centralOffset;++ le32(out, 0x06054b50L); // end of central directory signature+ le16(out, 0); // this disk+ le16(out, 0); // disk with central directory+ le16(out, entries.length);+ le16(out, entries.length);+ le32(out, centralSize);+ le32(out, centralOffset);+ le16(out, 0); // comment length+ return out.toByteArray();+ }++ private static byte[] deflate(byte[] raw) {+ Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true);+ deflater.setInput(raw);+ deflater.finish();+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ byte[] buffer = new byte[512];+ while (!deflater.finished()) {+ out.write(buffer, 0, deflater.deflate(buffer));+ }+ deflater.end();+ return out.toByteArray();+ }++ private static void le16(ByteArrayOutputStream out, int value) {+ out.write(value & 0xff);+ out.write((value >>> 8) & 0xff);+ }++ private static void le32(ByteArrayOutputStream out, long value) {+ out.write((int) (value & 0xff));+ out.write((int) ((value >>> 8) & 0xff));+ out.write((int) ((value >>> 16) & 0xff));+ out.write((int) ((value >>> 24) & 0xff));+ }++ @Test+ void extractsStoredEntryCarryingADataDescriptor(@TempDir Path target) throws IOException {+ // Issue #2593: a Serverless Framework package bundling node_modules failed with+ // "only DEFLATED entries can have EXT descriptor". ZipInputStream reads local file+ // headers in sequence, and for a STORED entry whose sizes were deferred to a trailing+ // descriptor it cannot tell where the data ends, so it refuses the entry outright.+ // Real AWS Lambda accepts the archive. Reading the central directory does too.+ byte[] zip = streamedZip(+ new StreamedEntry("index.js", "exports.handler = async () => ({statusCode: 200});", false),+ new StreamedEntry("node_modules/sharp/sharp.node", "\0\0BINARY-INCOMPRESSIBLE", true));++ extractor.extractTo(zip, target);++ Path handler = target.resolve("index.js");+ assertTrue(Files.isRegularFile(handler), "handler must extract from a streamed package");+ assertEquals("exports.handler = async () => ({statusCode: 200});", Files.readString(handler));++ // The STORED entry is the one ZipInputStream rejected; it must land intact.+ Path stored = target.resolve("node_modules").resolve("sharp").resolve("sharp.node");+ assertTrue(Files.isRegularFile(stored), "STORED entry with a data descriptor must extract");+ assertEquals("\0\0BINARY-INCOMPRESSIBLE", Files.readString(stored));+ }++ @Test+ void extractsDeflatedEntryCarryingADataDescriptor(@TempDir Path target) throws IOException {+ // The descriptor flag alone was always tolerated for DEFLATED entries. Pin it, so the+ // move to the central directory is not silently narrowing what already worked.+ byte[] zip = streamedZip(new StreamedEntry("app/main.py", "def handler(e, c): return 1", false));++ extractor.extractTo(zip, target);++ Path nested = target.resolve("app").resolve("main.py");+ assertTrue(Files.isRegularFile(nested));+ assertEquals("def handler(e, c): return 1", Files.readString(nested));+ }++ @Test+ void extractsExplicitDirectoryEntries(@TempDir Path target) throws IOException {+ // Directory entries are flagged by a trailing slash in the central directory just as+ // they are in a local header; make sure they still create a directory, not a file.+ byte[] zip = streamedZip(+ new StreamedEntry("lib/", "", true),+ new StreamedEntry("lib/util.js", "module.exports = {};", false));++ extractor.extractTo(zip, target);++ assertTrue(Files.isDirectory(target.resolve("lib")));+ assertEquals("module.exports = {};", Files.readString(target.resolve("lib").resolve("util.js")));+ }++ /**+ * Builds a structurally valid ZIP whose entry data has been corrupted after the fact, so+ * every header (including the CRC recorded for the entry) is intact but the bytes no+ * longer match it — what a package truncated or mangled in transit looks like.+ */+ private static byte[] corruptedPayloadZip(String entryName, String content) throws IOException {+ byte[] zip = zipWith(entryName, content);+ int nameLength = (zip[26] & 0xff) | ((zip[27] & 0xff) << 8);+ int extraLength = (zip[28] & 0xff) | ((zip[29] & 0xff) << 8);+ int dataStart = 30 + nameLength + extraLength;+ for (int i = dataStart + 1; i < dataStart + 6 && i < zip.length; i++) {+ zip[i] ^= 0x7F;+ }+ return zip;+ }++ @Test+ void rejectsAnEntryThatFailsItsChecksum(@TempDir Path target) throws IOException {+ // ZipInputStream verified each entry's CRC on its own; ZipFile does not. Without an+ // explicit check, a deployment package corrupted in transit would extract silently and+ // the function would run garbage instead of failing loudly at deploy time.+ byte[] zip = corruptedPayloadZip("payload.js", "REAL-CODE-REAL-CODE-REAL-CODE");++ ZipException thrown = assertThrows(ZipException.class, () -> extractor.extractTo(zip, target));+ assertTrue(thrown.getMessage().contains("invalid entry CRC"),+ "corrupt entry must be reported as a checksum failure, was: " + thrown.getMessage());+ }++ @Test+ void leavesNoStagedArchiveInTheCodeStore(@TempDir Path codeStore) throws IOException {+ // ZipFile needs a seekable file, so the archive is staged on disk next to the target+ // rather than in the shared java.io.tmpdir. That puts a stray file one level above the+ // function's code directory, so pin that none survives: a leaked staging archive would+ // sit in the code store indefinitely, and one leaked *inside* a function directory+ // would be tar-copied into that function's container at launch.+ Path target = codeStore.resolve("fn");+ extractor.extractTo(zipWith("index.js", "v1"), target);+ assertNoStagedFiles(codeStore);++ byte[] corrupt = corruptedPayloadZip("index.js", "v2-but-corrupted-in-transit");+ assertThrows(IOException.class, () -> extractor.extractTo(corrupt, target));+ assertNoStagedFiles(codeStore);+ }++ private static void assertNoStagedFiles(Path codeStore) throws IOException {+ try (var entries = Files.list(codeStore)) {+ assertEquals(List.of("fn"), entries.map(p -> p.getFileName().toString()).sorted().toList(),+ "no staged archive may survive extraction, successful or failed");+ }+ }++ @Test+ void rejectsAnEntryWhoseCompressionMethodCannotBeRead(@TempDir Path target) throws IOException {+ // A package must never be half-deployed because an entry used a codec the reader does+ // not implement — writing it out empty would leave a function whose code silently+ // vanished. Method 99 is WinZip AES: structurally valid, not decodable here.+ byte[] zip = zipWithCompressionMethod(99, "secret.js", "exports.handler = () => 1;");++ assertThrows(ZipException.class, () -> extractor.extractTo(zip, target));+ assertFalse(Files.exists(target.resolve("secret.js")),+ "an entry that cannot be decoded must not be written out");+ }++ /** A structurally valid archive declaring an arbitrary compression method. */+ private static byte[] zipWithCompressionMethod(int method, String entryName, String content)+ throws IOException {+ byte[] name = entryName.getBytes(StandardCharsets.UTF_8);+ byte[] data = content.getBytes(StandardCharsets.UTF_8);+ CRC32 crc = new CRC32();+ crc.update(data);+ ByteArrayOutputStream out = new ByteArrayOutputStream();++ int localOffset = out.size();+ le32(out, 0x04034b50L);+ le16(out, 20);+ le16(out, 0);+ le16(out, method);+ le16(out, 0);+ le16(out, 0);+ le32(out, crc.getValue());+ le32(out, data.length);+ le32(out, data.length);+ le16(out, name.length);+ le16(out, 0);+ out.write(name);+ out.write(data);++ int centralOffset = out.size();+ le32(out, 0x02014b50L);+ le16(out, 20);+ le16(out, 20);+ le16(out, 0);+ le16(out, method);+ le16(out, 0);+ le16(out, 0);+ le32(out, crc.getValue());+ le32(out, data.length);+ le32(out, data.length);+ le16(out, name.length);+ le16(out, 0);+ le16(out, 0);+ le16(out, 0);+ le16(out, 0);+ le32(out, 0);+ le32(out, localOffset);+ out.write(name);+ int centralSize = out.size() - centralOffset;++ le32(out, 0x06054b50L);+ le16(out, 0);+ le16(out, 0);+ le16(out, 1);+ le16(out, 1);+ le32(out, centralSize);+ le32(out, centralOffset);+ le16(out, 0);+ return out.toByteArray();+ }}