Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(artifacts): admit ZIP archives after unclaimed leading bytes (#5803)

Kaoru082235d6d5bb0aca
diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts
index 7b7f58afb..e03dc1e62 100644
--- a/src/application/BinaryTargetResolver.ts
+++ b/src/application/BinaryTargetResolver.ts
@@ -41,12 +41,14 @@ import {
import { hasGodotPckHeader } from "../domain/godotPckFormat.js";
import {
hasZipSignature,
+ locateTrailingZipArchive,
isZipPackageFormat,
- zipPackageFormatForPath,
+ zipRootPackageFormat,
} from "../domain/zipPackageFormat.js";
import {
parseCodeViewData,
+ claimsExecutableHeader,
parseExecutableHeader,
parsePeRecord,
peDebugDirectoryEntry,
@@ -219,12 +221,32 @@ export const parseBinaryTarget = async (
hostArchitecture,
signal,
);
- if (!detected.ok)
+ if (!detected.ok) {
+ // Only bytes no other format claims may lead a located ZIP, so an
+ // executable with an appended archive keeps its executable route.
+ if (
+ !(await claimsLeadingExecutableHeader(
+ handle,
+ metadata.size,
+ signal,
+ )) &&
+ (await locateTrailingZipArchive(metadata.size, (position, length) =>
+ readHandleRange(handle, position, length, signal),
+ )) !== undefined
+ )
+ return ok({
+ path,
+ sourcePath,
+ sha256: await sha256Handle(handle, path, metadata, signal),
+ kind: "archive",
+ format: zipRootPackageFormat(path),
+ });
return err(
new BinaryTargetError(path, detected.error, {
constraint: "file_format_rejected",
}),
);
+ }
return ok({
path,
sourcePath,
@@ -356,12 +378,8 @@ const detectArtifactFormat = async (
"WebAssembly v1 artifacts are not supported by the native opening route",
{ constraint: "wasm_requires_static_inspection" },
);
- if (hasZipSignature(magic.subarray(0, observed.bytesRead))) {
- // The HarmonyOS App Pack suffix `.app` is a root-input concept; a nested
- // ZIP member ending in `.app` keeps the generic zip classification.
- if (lower.endsWith(".app")) return "app-pack";
- return zipPackageFormatForPath(lower) ?? "zip";
- }
+ if (hasZipSignature(magic.subarray(0, observed.bytesRead)))
+ return zipRootPackageFormat(lower);
if (hasGodotPckHeader(magic.subarray(0, observed.bytesRead)))
return "godot-pck";
// Binary plists, including keyed archives, often use other extensions.
@@ -446,6 +464,31 @@ const isArchiveFormat = (
isZipPackageFormat(format) ||
["asar", "godot-pck", "dmg", "pkg"].includes(format);
+/** Whether the leading bytes begin any executable header family, valid or not. */
+const claimsLeadingExecutableHeader = async (
+ handle: FileHandle,
+ size: number,
+ signal?: AbortSignal,
+): Promise<boolean> =>
+ claimsExecutableHeader(
+ Buffer.from(await readHandleRange(handle, 0, Math.min(size, 4096), signal)),
+ size,
+ );
+
+/** Read one bounded range, honoring cancellation around the host read. */
+const readHandleRange = async (
+ handle: FileHandle,
+ position: number,
+ length: number,
+ signal?: AbortSignal,
+): Promise<Uint8Array> => {
+ throwIfTargetResolutionCancelled(signal);
+ const bytes = Buffer.alloc(length);
+ const observed = await handle.read(bytes, 0, length, position);
+ throwIfTargetResolutionCancelled(signal);
+ return bytes.subarray(0, observed.bytesRead);
+};
+
const sha256Handle = async (
handle: FileHandle,
path: string,
diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts
index c36d6e43d..0e20fe67c 100644
--- a/src/artifacts/inventory/classify.ts
+++ b/src/artifacts/inventory/classify.ts
@@ -8,13 +8,14 @@ import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
import { abortIfNeeded } from "../ArtifactHash.js";
+import { claimsExecutableHeader } from "../../domain/binaryTarget.js";
import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
import {
hasZipSignature,
+ locateTrailingZipArchive,
isZipPackageFormat,
- zipPackageFormatForPath,
- type ZipPackageFormat,
+ zipRootPackageFormat,
} from "../../domain/zipPackageFormat.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js";
@@ -188,6 +189,20 @@ const classifyRootFormat = async (
},
archiveBytes,
).format;
+ // Only bytes no other format claims may lead a located ZIP, so an
+ // executable with an appended archive keeps its executable classification.
+ if (
+ (format === "file" || format === "unknown") &&
+ !claimsExecutableHeader(evidence.prefix, archiveBytes) &&
+ (await locateTrailingZipArchive(archiveBytes, async (position, length) => {
+ abortIfNeeded(signal);
+ const bytes = Buffer.alloc(length);
+ const read = await handle.read(bytes, 0, length, position);
+ abortIfNeeded(signal);
+ return bytes.subarray(0, read.bytesRead);
+ })) !== undefined
+ )
+ return zipRootPackageFormat(lower);
if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) {
const packaged = await containsPyInstallerArchive(
handle,
@@ -243,12 +258,6 @@ const readRootFatSliceHeaders = async (
return { header, sliceHeadersByOffset };
};
-/** App Pack is a verified ZIP root role; nested .app members retain their observed format. */
-const zipRootPackageFormat = (lower: string): ZipPackageFormat =>
- lower.endsWith(".app")
- ? "app-pack"
- : (zipPackageFormatForPath(lower) ?? "zip");
-
const hasKolyTrailer = async (
handle: FileHandle,
size: number,
diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts
index 9b265f278..406ff509e 100644
--- a/src/domain/binaryTarget.ts
+++ b/src/domain/binaryTarget.ts
@@ -75,6 +75,22 @@ export type ExecutableMetadata =
readonly codeView?: PeCodeViewRecord;
};
+/**
+ * Whether leading bytes begin an executable header family that
+ * `parseExecutableHeader` interprets, whether or not that header is valid.
+ */
+export const claimsExecutableHeader = (
+ bytes: Buffer,
+ fileSize = bytes.length,
+): boolean =>
+ (bytes.length >= 4 &&
+ (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46])) ||
+ [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(
+ bytes.readUInt32BE(0),
+ ))) ||
+ (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) ||
+ readFatContainerHeader(bytes, fileSize).state !== "not-fat";
+
/**
* Parse supported executable headers without I/O. The caller supplies the host
* architecture used for deterministic FAT Mach-O slice selection.
diff --git a/src/domain/zipPackageFormat.ts b/src/domain/zipPackageFormat.ts
index 6d9fd4663..26f8af7c1 100644
--- a/src/domain/zipPackageFormat.ts
+++ b/src/domain/zipPackageFormat.ts
@@ -26,6 +26,79 @@ export const hasZipSignature = (bytes: Uint8Array): boolean =>
(bytes[2] === 0x05 && bytes[3] === 0x06) ||
(bytes[2] === 0x07 && bytes[3] === 0x08));
+const END_OF_CENTRAL_DIRECTORY = 0x06054b50;
+const CENTRAL_DIRECTORY_HEADER = 0x02014b50;
+const LOCAL_FILE_HEADER = 0x04034b50;
+const END_RECORD_BYTES = 22;
+const MAXIMUM_COMMENT_BYTES = 0xffff;
+const CENTRAL_HEADER_BYTES = 46;
+
+/** Positional read of at most `length` bytes; a short result means end of file. */
+export type PositionalRead = (
+ position: number,
+ length: number,
+) => Promise<Uint8Array>;
+
+/**
+ * Locate a single-disk ZIP whose end-of-central-directory record ends the
+ * file after leading non-ZIP bytes, such as a self-extractor or script stub.
+ * Its central directory and first local header must resolve, whether member
+ * offsets include the leading bytes or were written before them. Zip64,
+ * split and empty archives are not located. Returns the leading byte count.
+ */
+export const locateTrailingZipArchive = async (
+ size: number,
+ read: PositionalRead,
+): Promise<{ readonly leadingBytes: number } | undefined> => {
+ if (!Number.isSafeInteger(size) || size < END_RECORD_BYTES) return undefined;
+ const tailStart = Math.max(
+ 0,
+ size - END_RECORD_BYTES - MAXIMUM_COMMENT_BYTES,
+ );
+ const tail = Buffer.from(await read(tailStart, size - tailStart));
+ if (tail.length !== size - tailStart) return undefined;
+ // The record's comment must end exactly at end of file.
+ let record = -1;
+ for (let at = tail.length - END_RECORD_BYTES; at >= 0; at -= 1)
+ if (
+ tail.readUInt32LE(at) === END_OF_CENTRAL_DIRECTORY &&
+ at + END_RECORD_BYTES + tail.readUInt16LE(at + 20) === tail.length
+ ) {
+ record = at;
+ break;
+ }
+ if (record < 0) return undefined;
+ const entriesOnDisk = tail.readUInt16LE(record + 8);
+ const directoryBytes = tail.readUInt32LE(record + 12);
+ const declaredDirectory = tail.readUInt32LE(record + 16);
+ if (
+ tail.readUInt16LE(record + 4) !== 0 ||
+ tail.readUInt16LE(record + 6) !== 0 ||
+ entriesOnDisk === 0 ||
+ entriesOnDisk === 0xffff ||
+ entriesOnDisk !== tail.readUInt16LE(record + 10) ||
+ directoryBytes === 0xffffffff ||
+ declaredDirectory === 0xffffffff
+ )
+ return undefined;
+ const directory = tailStart + record - directoryBytes;
+ // Offsets written before leading bytes were prepended are shifted by them.
+ const shift = directory - declaredDirectory;
+ if (directory < 0 || shift < 0) return undefined;
+ const header = Buffer.from(await read(directory, CENTRAL_HEADER_BYTES));
+ if (
+ header.length !== CENTRAL_HEADER_BYTES ||
+ header.readUInt32LE(0) !== CENTRAL_DIRECTORY_HEADER
+ )
+ return undefined;
+ const firstLocal = header.readUInt32LE(42) + shift;
+ if (firstLocal === 0 || firstLocal >= directory) return undefined;
+ const local = Buffer.from(await read(firstLocal, 4));
+ return local.length === 4 && local.readUInt32LE(0) === LOCAL_FILE_HEADER
+ ? { leadingBytes: firstLocal }
+ : undefined;
+};
+
/**
* Classify a ZIP-backed package by its complete lower-cased path suffix.
*
@@ -52,6 +125,18 @@ export const zipPackageFormatForPath = (
return undefined;
};
+/**
+ * Classify a verified ZIP root input by its path. The HarmonyOS App Pack
+ * suffix `.app` applies only to root inputs; a nested ZIP member ending in
+ * `.app` keeps its observed format.
+ */
+export const zipRootPackageFormat = (path: string): ZipPackageFormat => {
+ const lower = path.toLowerCase();
+ return lower.endsWith(".app")
+ ? "app-pack"
+ : (zipPackageFormatForPath(lower) ?? "zip");
+};
+
/** Inventory lists entries, not all bytes surrounding the central directory. */
export const ZIP_NON_ENTRY_LIMITATION =
- "ZIP inventory lists central-directory entries only. Non-entry bytes, including an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.";
+ "ZIP inventory lists central-directory entries only. Non-entry bytes, including leading bytes before the first entry such as a self-extractor stub, an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.";
diff --git a/tests/acceptance/artifactZipLeadingBytes.test.ts b/tests/acceptance/artifactZipLeadingBytes.test.ts
new file mode 100644
index 000000000..4e3add055
--- /dev/null
+++ b/tests/acceptance/artifactZipLeadingBytes.test.ts
@@ -0,0 +1,74 @@
+import { readFile, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js";
+import { expect } from "vitest";
+
+import {
+ artifactExtractionResultSchema,
+ artifactInventoryResultSchema,
+} from "../../src/domain/artifactGraph.js";
+import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js";
+import { parseEvidence } from "../../src/domain/evidence.js";
+import { ZIP_NON_ENTRY_LIMITATION } from "../../src/domain/zipPackageFormat.js";
+import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
+import { cliTest } from "../support/cli/cliFixture.js";
+
+cliTest(
+ "inspects and extracts a ZIP after an inert script stub",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-zip-script-stub-");
+ const archivePath = join(root, "installer.sh");
+ const writer = new ZipWriter(new Uint8ArrayWriter());
+ await writer.add("a.txt", new TextReader("hello world\n"));
+ await writer.add("d/b.txt", new TextReader("x\n"));
+ // A self-extractor stub prepended to an unchanged archive.
+ await writeFile(
+ archivePath,
+ Buffer.concat([Buffer.from("#!/bin/sh\nexit 0\n"), await writer.close()]),
+ );
+
+ const inspected = await cli.run({
+ arguments: ["inspect-artifact", archivePath, "--json"],
+ });
+ expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0);
+ const inspection = artifactInspectionResultSchema.parse(
+ parseEvidence(inspected.json).normalized_result,
+ );
+ expect(inspection.subject.root_format).toBe("zip");
+ const inventoryEvidence = inspection.substeps[0]?.evidence;
+ if (inventoryEvidence === undefined)
+ throw new Error("CLI inspection did not retain ZIP inventory Evidence");
+ const inventory = artifactInventoryResultSchema.parse(
+ inventoryEvidence.normalized_result,
+ );
+ expect(
+ inventory.occurrences
+ .filter(({ entry_kind }) => entry_kind === "file")
+ .map(({ logical_path, hash_status }) => [logical_path, hash_status]),
+ ).toEqual([
+ [".", "verified"],
+ ["a.txt", "verified"],
+ ["d/b.txt", "verified"],
+ ]);
+ expect(inventory.limitations).toContain(ZIP_NON_ENTRY_LIMITATION);
+
+ const extracted = await cli.run({
+ arguments: ["extract-artifact", archivePath, "--json"],
+ });
+ expect(extracted.exitCode, extracted.stdout + extracted.stderr).toBe(0);
+ const extraction = artifactExtractionResultSchema.parse(
+ parseEvidence(extracted.json).normalized_result,
+ );
+ try {
+ expect(
+ extraction.artifacts.map(({ relative_path }) => relative_path),
+ ).toEqual(["a.txt", "d/b.txt"]);
+ expect(
+ await readFile(join(extraction.output_root, "a.txt"), "utf8"),
+ ).toBe("hello world\n");
+ } finally {
+ await rm(extraction.output_root, { force: true, recursive: true });
+ }
+ },
+);
diff --git a/tests/boundary/filesystem/artifactComparison.test.ts b/tests/boundary/filesystem/artifactComparison.test.ts
index 1c3784efd..b29d4aef6 100644
--- a/tests/boundary/filesystem/artifactComparison.test.ts
+++ b/tests/boundary/filesystem/artifactComparison.test.ts
@@ -896,7 +896,7 @@ describe("ZIP non-entry coverage", () => {
left.normalized_result,
);
expect(inventory.limitations).toContain(
- "ZIP inventory lists central-directory entries only. Non-entry bytes, including an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.",
+ "ZIP inventory lists central-directory entries only. Non-entry bytes, including leading bytes before the first entry such as a self-extractor stub, an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.",
);
expect(
inventory.occurrences.map(({ logical_path }) => logical_path).sort(),
diff --git a/tests/boundary/filesystem/zipDetection.test.ts b/tests/boundary/filesystem/zipDetection.test.ts
index f7c5d6f1b..e5b7b0920 100644
--- a/tests/boundary/filesystem/zipDetection.test.ts
+++ b/tests/boundary/filesystem/zipDetection.test.ts
@@ -184,6 +184,166 @@ describe("nested container family", () => {
);
});
+const SCRIPT_STUB = Buffer.from("#!/bin/sh\nexit 0\n");
+
+const memberZip = async (): Promise<Buffer> => {
+ const writer = new ZipWriter(new Uint8ArrayWriter());
+ await writer.add("a.txt", new TextReader("hello world\n"));
+ await writer.add("d/b.txt", new TextReader("x\n"));
+ return Buffer.from(await writer.close());
+};
+
+/** Prepend bytes; optionally shift stored offsets as `zip -A` does. */
+const withLeadingBytes = (
+ zip: Buffer,
+ leading: Buffer,
+ adjustOffsets: boolean,
+): Buffer => {
+ const bytes = Buffer.concat([leading, zip]);
+ if (!adjustOffsets) return bytes;
+ const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
+ const directory = bytes.readUInt32LE(end + 16) + leading.length;
+ bytes.writeUInt32LE(directory, end + 16);
+ for (let at = directory; at < end;) {
+ bytes.writeUInt32LE(bytes.readUInt32LE(at + 42) + leading.length, at + 42);
+ at +=
+ 46 +
+ bytes.readUInt16LE(at + 28) +
+ bytes.readUInt16LE(at + 30) +
+ bytes.readUInt16LE(at + 32);
+ }
+ return bytes;
+};
+
+/** Members by content-derived identity and verification state. */
+const members = async (path: string) => {
+ const inventory = await inventoryArtifact(path);
+ return ["a.txt", "d/b.txt"].map((member) => {
+ const occurrence = artifactOccurrenceAt(inventory, member);
+ return [member, occurrence.hash_status, occurrence.artifact_id];
+ });
+};
+
+describe("ZIP archives after leading bytes", () => {
+ it.each([
+ { style: "offsets that include the leading bytes", adjust: true },
+ { style: "offsets written before the bytes were prepended", adjust: false },
+ ])(
+ "admits a ZIP after an unclaimed script stub with $style",
+ async ({ adjust }) => {
+ const root = await createTestTempDirectory("rea-zip-leading-");
+ const plain = join(root, "plain.zip");
+ const prefixed = join(root, "installer.bin");
+ const zip = await memberZip();
+ await writeFile(plain, zip);
+ await writeFile(prefixed, withLeadingBytes(zip, SCRIPT_STUB, adjust));
+
+ expect(
+ (await classifyAndHashRoot(prefixed, false, await lstat(prefixed)))
+ .format,
+ ).toBe("zip");
+ expect(await parseBinaryTarget(prefixed)).toMatchObject({
+ ok: true,
+ value: { kind: "archive", format: "zip" },
+ });
+ // Members are the same verified bytes as in the plain archive.
+ const expected = await members(plain);
+ expect(expected.map(([, status]) => status)).toEqual([
+ "verified",
+ "verified",
+ ]);
+ expect(await members(prefixed)).toEqual(expected);
+ },
+ );
+
+ it("keeps the root App Pack family for a ZIP after a script stub", async () => {
+ const root = await createTestTempDirectory("rea-zip-leading-app-pack-");
+ const path = join(root, "Example.app");
+ await writeFile(
+ path,
+ withLeadingBytes(await memberZip(), SCRIPT_STUB, false),
+ );
+ expect(
+ (await classifyAndHashRoot(path, false, await lstat(path))).format,
+ ).toBe("app-pack");
+ expect(await parseBinaryTarget(path)).toMatchObject({
+ ok: true,
+ value: { kind: "archive", format: "app-pack" },
+ });
+ });
+
+ it.each([
+ { family: "ELF", leading: Buffer.from([0x7f, 0x45, 0x4c, 0x46, 1, 1, 1]) },
+ {
+ family: "MZ",
+ leading: Buffer.concat([Buffer.from("MZ"), Buffer.alloc(62)]),
+ },
+ {
+ family: "Mach-O",
+ leading: Buffer.from([0xcf, 0xfa, 0xed, 0xfe, 0, 0, 0, 0]),
+ },
+ ])(
+ "keeps a $family-led file with an appended ZIP out of the ZIP route",
+ async ({ leading }) => {
+ const root = await createTestTempDirectory("rea-zip-claimed-lead-");
+ const path = join(root, "program");
+ await writeFile(path, withLeadingBytes(await memberZip(), leading, true));
+ expect(
+ (await classifyAndHashRoot(path, false, await lstat(path))).format,
+ ).not.toBe("zip");
+ expect(await parseBinaryTarget(path)).not.toMatchObject({
+ ok: true,
+ value: { format: "zip" },
+ });
+ },
+ );
+
+ it.each([
+ {
+ shape: "bytes after the end record",
+ mutate: (bytes: Buffer) => Buffer.concat([bytes, Buffer.from("junk")]),
+ },
+ {
+ shape: "a corrupt central directory header",
+ mutate: (bytes: Buffer) => {
+ bytes.write("XXXX", bytes.indexOf(Buffer.from([0x50, 0x4b, 1, 2])));
+ return bytes;
+ },
+ },
+ {
+ shape: "a Zip64 directory offset",
+ mutate: (bytes: Buffer) => {
+ const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 5, 6]));
+ bytes.writeUInt32LE(0xffffffff, end + 16);
+ return bytes;
+ },
+ },
+ {
+ shape: "an empty archive record",
+ mutate: (bytes: Buffer) => {
+ const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 5, 6]));
+ bytes.writeUInt16LE(0, end + 8);
+ bytes.writeUInt16LE(0, end + 10);
+ return bytes;
+ },
+ },
+ ])("does not locate a trailing ZIP with $shape", async ({ mutate }) => {
+ const root = await createTestTempDirectory("rea-zip-leading-negative-");
+ const path = join(root, "input.bin");
+ await writeFile(
+ path,
+ mutate(withLeadingBytes(await memberZip(), SCRIPT_STUB, true)),
+ );
+ expect(
+ (await classifyAndHashRoot(path, false, await lstat(path))).format,
+ ).toBe("file");
+ expect(await parseBinaryTarget(path)).toMatchObject({
+ ok: false,
+ error: { _tag: "BinaryTargetError" },
+ });
+ });
+});
+
describe("package signature byte identity", () => {
it.each([
{ extension: "pkg", signature: "xar!", offset: 0, size: 4 },