Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(artifacts): admit ZIP archives after unclaimed leading bytes (#5803)
Kaoru082235d6d5bb0aca
diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.tsindex 7b7f58afb..e03dc1e62 100644--- a/src/application/BinaryTargetResolver.ts+++ b/src/application/BinaryTargetResolver.ts@@ -41,12 +41,14 @@ import {import { hasGodotPckHeader } from "../domain/godotPckFormat.js";import {hasZipSignature,+ locateTrailingZipArchive,isZipPackageFormat,- zipPackageFormatForPath,+ zipRootPackageFormat,} from "../domain/zipPackageFormat.js";import {parseCodeViewData,+ claimsExecutableHeader,parseExecutableHeader,parsePeRecord,peDebugDirectoryEntry,@@ -219,12 +221,32 @@ export const parseBinaryTarget = async (hostArchitecture,signal,);- if (!detected.ok)+ if (!detected.ok) {+ // Only bytes no other format claims may lead a located ZIP, so an+ // executable with an appended archive keeps its executable route.+ if (+ !(await claimsLeadingExecutableHeader(+ handle,+ metadata.size,+ signal,+ )) &&+ (await locateTrailingZipArchive(metadata.size, (position, length) =>+ readHandleRange(handle, position, length, signal),+ )) !== undefined+ )+ return ok({+ path,+ sourcePath,+ sha256: await sha256Handle(handle, path, metadata, signal),+ kind: "archive",+ format: zipRootPackageFormat(path),+ });return err(new BinaryTargetError(path, detected.error, {constraint: "file_format_rejected",}),);+ }return ok({path,sourcePath,@@ -356,12 +378,8 @@ const detectArtifactFormat = async ("WebAssembly v1 artifacts are not supported by the native opening route",{ constraint: "wasm_requires_static_inspection" },);- if (hasZipSignature(magic.subarray(0, observed.bytesRead))) {- // The HarmonyOS App Pack suffix `.app` is a root-input concept; a nested- // ZIP member ending in `.app` keeps the generic zip classification.- if (lower.endsWith(".app")) return "app-pack";- return zipPackageFormatForPath(lower) ?? "zip";- }+ if (hasZipSignature(magic.subarray(0, observed.bytesRead)))+ return zipRootPackageFormat(lower);if (hasGodotPckHeader(magic.subarray(0, observed.bytesRead)))return "godot-pck";// Binary plists, including keyed archives, often use other extensions.@@ -446,6 +464,31 @@ const isArchiveFormat = (isZipPackageFormat(format) ||["asar", "godot-pck", "dmg", "pkg"].includes(format);+/** Whether the leading bytes begin any executable header family, valid or not. */+const claimsLeadingExecutableHeader = async (+ handle: FileHandle,+ size: number,+ signal?: AbortSignal,+): Promise<boolean> =>+ claimsExecutableHeader(+ Buffer.from(await readHandleRange(handle, 0, Math.min(size, 4096), signal)),+ size,+ );++/** Read one bounded range, honoring cancellation around the host read. */+const readHandleRange = async (+ handle: FileHandle,+ position: number,+ length: number,+ signal?: AbortSignal,+): Promise<Uint8Array> => {+ throwIfTargetResolutionCancelled(signal);+ const bytes = Buffer.alloc(length);+ const observed = await handle.read(bytes, 0, length, position);+ throwIfTargetResolutionCancelled(signal);+ return bytes.subarray(0, observed.bytesRead);+};+const sha256Handle = async (handle: FileHandle,path: string,diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.tsindex c36d6e43d..0e20fe67c 100644--- a/src/artifacts/inventory/classify.ts+++ b/src/artifacts/inventory/classify.ts@@ -8,13 +8,14 @@ import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js";import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";import { abortIfNeeded } from "../ArtifactHash.js";+import { claimsExecutableHeader } from "../../domain/binaryTarget.js";import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js";import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";import {hasZipSignature,+ locateTrailingZipArchive,isZipPackageFormat,- zipPackageFormatForPath,- type ZipPackageFormat,+ zipRootPackageFormat,} from "../../domain/zipPackageFormat.js";import { ArtifactReaderFailure } from "../ArtifactReader.js";import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js";@@ -188,6 +189,20 @@ const classifyRootFormat = async (},archiveBytes,).format;+ // Only bytes no other format claims may lead a located ZIP, so an+ // executable with an appended archive keeps its executable classification.+ if (+ (format === "file" || format === "unknown") &&+ !claimsExecutableHeader(evidence.prefix, archiveBytes) &&+ (await locateTrailingZipArchive(archiveBytes, async (position, length) => {+ abortIfNeeded(signal);+ const bytes = Buffer.alloc(length);+ const read = await handle.read(bytes, 0, length, position);+ abortIfNeeded(signal);+ return bytes.subarray(0, read.bytesRead);+ })) !== undefined+ )+ return zipRootPackageFormat(lower);if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) {const packaged = await containsPyInstallerArchive(handle,@@ -243,12 +258,6 @@ const readRootFatSliceHeaders = async (return { header, sliceHeadersByOffset };};-/** App Pack is a verified ZIP root role; nested .app members retain their observed format. */-const zipRootPackageFormat = (lower: string): ZipPackageFormat =>- lower.endsWith(".app")- ? "app-pack"- : (zipPackageFormatForPath(lower) ?? "zip");-const hasKolyTrailer = async (handle: FileHandle,size: number,diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.tsindex 9b265f278..406ff509e 100644--- a/src/domain/binaryTarget.ts+++ b/src/domain/binaryTarget.ts@@ -75,6 +75,22 @@ export type ExecutableMetadata =readonly codeView?: PeCodeViewRecord;};+/**+ * Whether leading bytes begin an executable header family that+ * `parseExecutableHeader` interprets, whether or not that header is valid.+ */+export const claimsExecutableHeader = (+ bytes: Buffer,+ fileSize = bytes.length,+): boolean =>+ (bytes.length >= 4 &&+ (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46])) ||+ [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(+ bytes.readUInt32BE(0),+ ))) ||+ (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) ||+ readFatContainerHeader(bytes, fileSize).state !== "not-fat";+/*** Parse supported executable headers without I/O. The caller supplies the host* architecture used for deterministic FAT Mach-O slice selection.diff --git a/src/domain/zipPackageFormat.ts b/src/domain/zipPackageFormat.tsindex 6d9fd4663..26f8af7c1 100644--- a/src/domain/zipPackageFormat.ts+++ b/src/domain/zipPackageFormat.ts@@ -26,6 +26,79 @@ export const hasZipSignature = (bytes: Uint8Array): boolean =>(bytes[2] === 0x05 && bytes[3] === 0x06) ||(bytes[2] === 0x07 && bytes[3] === 0x08));+const END_OF_CENTRAL_DIRECTORY = 0x06054b50;+const CENTRAL_DIRECTORY_HEADER = 0x02014b50;+const LOCAL_FILE_HEADER = 0x04034b50;+const END_RECORD_BYTES = 22;+const MAXIMUM_COMMENT_BYTES = 0xffff;+const CENTRAL_HEADER_BYTES = 46;++/** Positional read of at most `length` bytes; a short result means end of file. */+export type PositionalRead = (+ position: number,+ length: number,+) => Promise<Uint8Array>;++/**+ * Locate a single-disk ZIP whose end-of-central-directory record ends the+ * file after leading non-ZIP bytes, such as a self-extractor or script stub.+ * Its central directory and first local header must resolve, whether member+ * offsets include the leading bytes or were written before them. Zip64,+ * split and empty archives are not located. Returns the leading byte count.+ */+export const locateTrailingZipArchive = async (+ size: number,+ read: PositionalRead,+): Promise<{ readonly leadingBytes: number } | undefined> => {+ if (!Number.isSafeInteger(size) || size < END_RECORD_BYTES) return undefined;+ const tailStart = Math.max(+ 0,+ size - END_RECORD_BYTES - MAXIMUM_COMMENT_BYTES,+ );+ const tail = Buffer.from(await read(tailStart, size - tailStart));+ if (tail.length !== size - tailStart) return undefined;+ // The record's comment must end exactly at end of file.+ let record = -1;+ for (let at = tail.length - END_RECORD_BYTES; at >= 0; at -= 1)+ if (+ tail.readUInt32LE(at) === END_OF_CENTRAL_DIRECTORY &&+ at + END_RECORD_BYTES + tail.readUInt16LE(at + 20) === tail.length+ ) {+ record = at;+ break;+ }+ if (record < 0) return undefined;+ const entriesOnDisk = tail.readUInt16LE(record + 8);+ const directoryBytes = tail.readUInt32LE(record + 12);+ const declaredDirectory = tail.readUInt32LE(record + 16);+ if (+ tail.readUInt16LE(record + 4) !== 0 ||+ tail.readUInt16LE(record + 6) !== 0 ||+ entriesOnDisk === 0 ||+ entriesOnDisk === 0xffff ||+ entriesOnDisk !== tail.readUInt16LE(record + 10) ||+ directoryBytes === 0xffffffff ||+ declaredDirectory === 0xffffffff+ )+ return undefined;+ const directory = tailStart + record - directoryBytes;+ // Offsets written before leading bytes were prepended are shifted by them.+ const shift = directory - declaredDirectory;+ if (directory < 0 || shift < 0) return undefined;+ const header = Buffer.from(await read(directory, CENTRAL_HEADER_BYTES));+ if (+ header.length !== CENTRAL_HEADER_BYTES ||+ header.readUInt32LE(0) !== CENTRAL_DIRECTORY_HEADER+ )+ return undefined;+ const firstLocal = header.readUInt32LE(42) + shift;+ if (firstLocal === 0 || firstLocal >= directory) return undefined;+ const local = Buffer.from(await read(firstLocal, 4));+ return local.length === 4 && local.readUInt32LE(0) === LOCAL_FILE_HEADER+ ? { leadingBytes: firstLocal }+ : undefined;+};+/*** Classify a ZIP-backed package by its complete lower-cased path suffix.*@@ -52,6 +125,18 @@ export const zipPackageFormatForPath = (return undefined;};+/**+ * Classify a verified ZIP root input by its path. The HarmonyOS App Pack+ * suffix `.app` applies only to root inputs; a nested ZIP member ending in+ * `.app` keeps its observed format.+ */+export const zipRootPackageFormat = (path: string): ZipPackageFormat => {+ const lower = path.toLowerCase();+ return lower.endsWith(".app")+ ? "app-pack"+ : (zipPackageFormatForPath(lower) ?? "zip");+};+/** Inventory lists entries, not all bytes surrounding the central directory. */export const ZIP_NON_ENTRY_LIMITATION =- "ZIP inventory lists central-directory entries only. Non-entry bytes, including an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.";+ "ZIP inventory lists central-directory entries only. Non-entry bytes, including leading bytes before the first entry such as a self-extractor stub, an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.";diff --git a/tests/acceptance/artifactZipLeadingBytes.test.ts b/tests/acceptance/artifactZipLeadingBytes.test.tsnew file mode 100644index 000000000..4e3add055--- /dev/null+++ b/tests/acceptance/artifactZipLeadingBytes.test.ts@@ -0,0 +1,74 @@+import { readFile, rm, writeFile } from "node:fs/promises";+import { join } from "node:path";++import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js";+import { expect } from "vitest";++import {+ artifactExtractionResultSchema,+ artifactInventoryResultSchema,+} from "../../src/domain/artifactGraph.js";+import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js";+import { parseEvidence } from "../../src/domain/evidence.js";+import { ZIP_NON_ENTRY_LIMITATION } from "../../src/domain/zipPackageFormat.js";+import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";+import { cliTest } from "../support/cli/cliFixture.js";++cliTest(+ "inspects and extracts a ZIP after an inert script stub",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-zip-script-stub-");+ const archivePath = join(root, "installer.sh");+ const writer = new ZipWriter(new Uint8ArrayWriter());+ await writer.add("a.txt", new TextReader("hello world\n"));+ await writer.add("d/b.txt", new TextReader("x\n"));+ // A self-extractor stub prepended to an unchanged archive.+ await writeFile(+ archivePath,+ Buffer.concat([Buffer.from("#!/bin/sh\nexit 0\n"), await writer.close()]),+ );++ const inspected = await cli.run({+ arguments: ["inspect-artifact", archivePath, "--json"],+ });+ expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0);+ const inspection = artifactInspectionResultSchema.parse(+ parseEvidence(inspected.json).normalized_result,+ );+ expect(inspection.subject.root_format).toBe("zip");+ const inventoryEvidence = inspection.substeps[0]?.evidence;+ if (inventoryEvidence === undefined)+ throw new Error("CLI inspection did not retain ZIP inventory Evidence");+ const inventory = artifactInventoryResultSchema.parse(+ inventoryEvidence.normalized_result,+ );+ expect(+ inventory.occurrences+ .filter(({ entry_kind }) => entry_kind === "file")+ .map(({ logical_path, hash_status }) => [logical_path, hash_status]),+ ).toEqual([+ [".", "verified"],+ ["a.txt", "verified"],+ ["d/b.txt", "verified"],+ ]);+ expect(inventory.limitations).toContain(ZIP_NON_ENTRY_LIMITATION);++ const extracted = await cli.run({+ arguments: ["extract-artifact", archivePath, "--json"],+ });+ expect(extracted.exitCode, extracted.stdout + extracted.stderr).toBe(0);+ const extraction = artifactExtractionResultSchema.parse(+ parseEvidence(extracted.json).normalized_result,+ );+ try {+ expect(+ extraction.artifacts.map(({ relative_path }) => relative_path),+ ).toEqual(["a.txt", "d/b.txt"]);+ expect(+ await readFile(join(extraction.output_root, "a.txt"), "utf8"),+ ).toBe("hello world\n");+ } finally {+ await rm(extraction.output_root, { force: true, recursive: true });+ }+ },+);diff --git a/tests/boundary/filesystem/artifactComparison.test.ts b/tests/boundary/filesystem/artifactComparison.test.tsindex 1c3784efd..b29d4aef6 100644--- a/tests/boundary/filesystem/artifactComparison.test.ts+++ b/tests/boundary/filesystem/artifactComparison.test.ts@@ -896,7 +896,7 @@ describe("ZIP non-entry coverage", () => {left.normalized_result,);expect(inventory.limitations).toContain(- "ZIP inventory lists central-directory entries only. Non-entry bytes, including an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.",+ "ZIP inventory lists central-directory entries only. Non-entry bytes, including leading bytes before the first entry such as a self-extractor stub, an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.",);expect(inventory.occurrences.map(({ logical_path }) => logical_path).sort(),diff --git a/tests/boundary/filesystem/zipDetection.test.ts b/tests/boundary/filesystem/zipDetection.test.tsindex f7c5d6f1b..e5b7b0920 100644--- a/tests/boundary/filesystem/zipDetection.test.ts+++ b/tests/boundary/filesystem/zipDetection.test.ts@@ -184,6 +184,166 @@ describe("nested container family", () => {);});+const SCRIPT_STUB = Buffer.from("#!/bin/sh\nexit 0\n");++const memberZip = async (): Promise<Buffer> => {+ const writer = new ZipWriter(new Uint8ArrayWriter());+ await writer.add("a.txt", new TextReader("hello world\n"));+ await writer.add("d/b.txt", new TextReader("x\n"));+ return Buffer.from(await writer.close());+};++/** Prepend bytes; optionally shift stored offsets as `zip -A` does. */+const withLeadingBytes = (+ zip: Buffer,+ leading: Buffer,+ adjustOffsets: boolean,+): Buffer => {+ const bytes = Buffer.concat([leading, zip]);+ if (!adjustOffsets) return bytes;+ const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));+ const directory = bytes.readUInt32LE(end + 16) + leading.length;+ bytes.writeUInt32LE(directory, end + 16);+ for (let at = directory; at < end;) {+ bytes.writeUInt32LE(bytes.readUInt32LE(at + 42) + leading.length, at + 42);+ at +=+ 46 ++ bytes.readUInt16LE(at + 28) ++ bytes.readUInt16LE(at + 30) ++ bytes.readUInt16LE(at + 32);+ }+ return bytes;+};++/** Members by content-derived identity and verification state. */+const members = async (path: string) => {+ const inventory = await inventoryArtifact(path);+ return ["a.txt", "d/b.txt"].map((member) => {+ const occurrence = artifactOccurrenceAt(inventory, member);+ return [member, occurrence.hash_status, occurrence.artifact_id];+ });+};++describe("ZIP archives after leading bytes", () => {+ it.each([+ { style: "offsets that include the leading bytes", adjust: true },+ { style: "offsets written before the bytes were prepended", adjust: false },+ ])(+ "admits a ZIP after an unclaimed script stub with $style",+ async ({ adjust }) => {+ const root = await createTestTempDirectory("rea-zip-leading-");+ const plain = join(root, "plain.zip");+ const prefixed = join(root, "installer.bin");+ const zip = await memberZip();+ await writeFile(plain, zip);+ await writeFile(prefixed, withLeadingBytes(zip, SCRIPT_STUB, adjust));++ expect(+ (await classifyAndHashRoot(prefixed, false, await lstat(prefixed)))+ .format,+ ).toBe("zip");+ expect(await parseBinaryTarget(prefixed)).toMatchObject({+ ok: true,+ value: { kind: "archive", format: "zip" },+ });+ // Members are the same verified bytes as in the plain archive.+ const expected = await members(plain);+ expect(expected.map(([, status]) => status)).toEqual([+ "verified",+ "verified",+ ]);+ expect(await members(prefixed)).toEqual(expected);+ },+ );++ it("keeps the root App Pack family for a ZIP after a script stub", async () => {+ const root = await createTestTempDirectory("rea-zip-leading-app-pack-");+ const path = join(root, "Example.app");+ await writeFile(+ path,+ withLeadingBytes(await memberZip(), SCRIPT_STUB, false),+ );+ expect(+ (await classifyAndHashRoot(path, false, await lstat(path))).format,+ ).toBe("app-pack");+ expect(await parseBinaryTarget(path)).toMatchObject({+ ok: true,+ value: { kind: "archive", format: "app-pack" },+ });+ });++ it.each([+ { family: "ELF", leading: Buffer.from([0x7f, 0x45, 0x4c, 0x46, 1, 1, 1]) },+ {+ family: "MZ",+ leading: Buffer.concat([Buffer.from("MZ"), Buffer.alloc(62)]),+ },+ {+ family: "Mach-O",+ leading: Buffer.from([0xcf, 0xfa, 0xed, 0xfe, 0, 0, 0, 0]),+ },+ ])(+ "keeps a $family-led file with an appended ZIP out of the ZIP route",+ async ({ leading }) => {+ const root = await createTestTempDirectory("rea-zip-claimed-lead-");+ const path = join(root, "program");+ await writeFile(path, withLeadingBytes(await memberZip(), leading, true));+ expect(+ (await classifyAndHashRoot(path, false, await lstat(path))).format,+ ).not.toBe("zip");+ expect(await parseBinaryTarget(path)).not.toMatchObject({+ ok: true,+ value: { format: "zip" },+ });+ },+ );++ it.each([+ {+ shape: "bytes after the end record",+ mutate: (bytes: Buffer) => Buffer.concat([bytes, Buffer.from("junk")]),+ },+ {+ shape: "a corrupt central directory header",+ mutate: (bytes: Buffer) => {+ bytes.write("XXXX", bytes.indexOf(Buffer.from([0x50, 0x4b, 1, 2])));+ return bytes;+ },+ },+ {+ shape: "a Zip64 directory offset",+ mutate: (bytes: Buffer) => {+ const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 5, 6]));+ bytes.writeUInt32LE(0xffffffff, end + 16);+ return bytes;+ },+ },+ {+ shape: "an empty archive record",+ mutate: (bytes: Buffer) => {+ const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 5, 6]));+ bytes.writeUInt16LE(0, end + 8);+ bytes.writeUInt16LE(0, end + 10);+ return bytes;+ },+ },+ ])("does not locate a trailing ZIP with $shape", async ({ mutate }) => {+ const root = await createTestTempDirectory("rea-zip-leading-negative-");+ const path = join(root, "input.bin");+ await writeFile(+ path,+ mutate(withLeadingBytes(await memberZip(), SCRIPT_STUB, true)),+ );+ expect(+ (await classifyAndHashRoot(path, false, await lstat(path))).format,+ ).toBe("file");+ expect(await parseBinaryTarget(path)).toMatchObject({+ ok: false,+ error: { _tag: "BinaryTargetError" },+ });+ });+});+describe("package signature byte identity", () => {it.each([{ extension: "pkg", signature: "xar!", offset: 0, size: 4 },