Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
import { createHash } from "node:crypto";
import { open, readFile, stat, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buffer } from "node:stream/consumers";
import { expect, it, vi } from "vitest";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { materializeArtifactInventory } from "../../../tests/fixtures/artifactInventory.js";
import { scanCanonicalArtifactInventory } from "../../../tests/fixtures/artifactInventory.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
import { reconstructedPycHeader } from "./PythonBytecode.js";
import {
PYTHON_39_MAGIC,
cArchiveExecutable,
cookieRangeOutsideFile,
pyzArchive,
type CArchiveFixtureMember,
} from "./CArchive.fixture.js";
const sha256 = (bytes: Buffer): string =>
createHash("sha256").update(bytes).digest("hex");
const derived = (payload: Buffer): Buffer =>
Buffer.concat([reconstructedPycHeader(PYTHON_39_MAGIC), payload]);
const writeArchive = async (name: string, bytes: Buffer): Promise<string> => {
const directory = await createTestTempDirectory("rea-pyinstaller-");
const path = join(directory, name);
await writeFile(path, bytes);
return path;
};
const sampleMembers = (
script: Buffer,
stored: Buffer,
pyz: Buffer,
): CArchiveFixtureMember[] => [
{ name: "hello", typeCode: "s", payload: script, compress: true },
{ name: "data.bin", typeCode: "x", payload: stored, compress: false },
{ name: "PYZ.pyz", typeCode: "z", payload: pyz, compress: false },
{
name: "pyi-contents-directory _internal",
typeCode: "o",
payload: Buffer.alloc(0),
compress: false,
},
];
it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async () => {
const script = Buffer.from("print('hi')");
const stored = Buffer.from("stored-data");
const moduleBody = Buffer.from("module-body");
const path = await writeArchive(
"hello",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 314,
pythonLibrary: "libpython3.14.dylib",
trailer: Buffer.from("signature"),
members: sampleMembers(
script,
stored,
pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: "json.decoder", typeCode: 0, body: moduleBody },
{ name: "pkg", typeCode: 1, body: moduleBody },
{ name: "ns", typeCode: 3, body: Buffer.alloc(0) },
],
}),
),
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
expect(snapshot.manifest.root_format).toBe("pyinstaller");
expect(snapshot.coverage.status).toBe("complete");
const root = snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === ".",
);
expect(root).toMatchObject({ artifact_kind: "container" });
expect(root?.format_metadata).toMatchObject({
kind: "pyinstaller",
layout_version: "2.1",
python_version_raw: 314,
python_major: 3,
python_minor: 14,
python_library: "libpython3.14.dylib",
runtime_option_count: 1,
});
const hello = snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "hello",
);
expect(hello?.format_metadata).toMatchObject({
kind: "pyinstaller-entry",
container: "carchive",
type_code: "s",
compression_flag: 1,
bytecode_header: "unknown",
bytecode_magic: null,
});
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(script)),
).toMatchObject({ size: script.length });
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(stored)),
).toBeDefined();
expect(
snapshot.occurrences.map((occurrence) => occurrence.logical_path),
).toEqual(
expect.arrayContaining([
"hello",
"data.bin",
"PYZ.pyz",
"PYZ.pyz/json/decoder.pyc",
"PYZ.pyz/pkg/__init__.pyc",
"PYZ.pyz/ns",
"__pyinstaller__/runtime-options/3",
]),
);
expect(
snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "PYZ.pyz",
)?.entry_kind,
).toBe("directory");
const scriptEdge = snapshot.edges.find(
(edge) => edge.occurrence_id === hello?.occurrence_id,
);
expect(scriptEdge?.relation).toBe("contains");
});
it("records a 2.0 cookie without a Python library name", async () => {
const path = await writeArchive(
"legacy",
cArchiveExecutable({
layout: "2.0",
pythonVersion: 307,
members: [
{
name: "hello",
typeCode: "s",
payload: Buffer.from("print"),
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
const root = snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === ".",
);
expect(root?.format_metadata).toMatchObject({
kind: "pyinstaller",
layout_version: "2.0",
python_version_raw: 307,
python_major: 3,
python_minor: 7,
python_library: null,
});
});
it("extracts derived members into the safe output tree", async () => {
const script = Buffer.from("print('hi')");
const path = await writeArchive(
"hello",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: sampleMembers(
script,
Buffer.from("stored"),
pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: "app", typeCode: 0, body: Buffer.from("app-body") },
],
}),
),
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
const directory = await createTestTempDirectory("rea-pyinstaller-out-");
const output = join(directory, "extracted");
const extracted = await materializeArtifactInventory(
{
inputPath: path,
inputFormat: "elf",
outputRoot: output,
environment: process.env,
integrityPolicy: "fail",
},
path,
snapshot,
);
expect(await readFile(join(output, "hello"))).toEqual(derived(script));
expect(
extracted.artifacts.find((artifact) => artifact.relative_path === "hello")
?.sha256,
).toBe(sha256(derived(script)));
expect(await readFile(join(output, "PYZ.pyz", "app.pyc"))).toEqual(
derived(Buffer.from("app-body")),
);
});
it("reports encrypted PYZ entries and refuses to extract them", async () => {
const path = await writeArchive(
"encrypted",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: "pyimod00_crypto_key",
typeCode: "m",
payload: Buffer.from("key"),
compress: false,
},
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: "secret", typeCode: 0, body: Buffer.from("hidden") },
],
}),
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
expect(
snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "PYZ.pyz/secret.pyc",
),
).toMatchObject({
encrypted: true,
hash_status: "unavailable",
});
const directory = await createTestTempDirectory("rea-pyinstaller-out-");
await expect(
materializeArtifactInventory(
{
inputPath: path,
inputFormat: "elf",
outputRoot: join(directory, "extracted"),
environment: process.env,
integrityPolicy: "fail",
},
path,
snapshot,
),
).rejects.toThrow(/encrypted/u);
});
it.each([false, true])(
"leaves an unsupported PYZ magic as one opaque member (marker: %s)",
async (marker) => {
const payload = pyzArchive({
magic: Buffer.alloc(4),
modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }],
});
const path = await writeArchive(
"opaque",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 314,
pythonLibrary: "libpython3.14.dylib",
members: [
...(marker
? [
{
name: "pyimod00_crypto_key",
typeCode: "m",
payload: Buffer.from("key"),
compress: false,
},
]
: []),
{
name: "PYZ.pyz",
typeCode: "z",
payload,
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
const pyz = snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "PYZ.pyz",
);
expect(pyz).toMatchObject({
entry_kind: "file",
encrypted: false,
hash_status: "verified",
});
expect(snapshot.coverage.status).toBe("unknown");
expect(
snapshot.nodes.find((node) => node.artifact_id === pyz?.artifact_id),
).toMatchObject({ sha256: sha256(payload), size: payload.length });
expect(pyz?.format_metadata).toMatchObject({
pyz_python_magic: "00000000",
pyz_python_version: null,
pyz_status: "opaque",
encryption_source: marker ? "carchive-marker" : null,
});
expect(pyz?.limitations.join(" ")).toContain("left opaque");
expect(
snapshot.occurrences.some((occurrence) =>
occurrence.logical_path.endsWith(".pyc"),
),
).toBe(false);
const outputRoot = join(
await createTestTempDirectory("rea-opaque-out-"),
"extracted",
);
await materializeArtifactInventory(
{
inputPath: path,
inputFormat: "elf",
outputRoot,
environment: process.env,
integrityPolicy: "fail",
},
path,
snapshot,
);
expect(await readFile(join(outputRoot, "PYZ.pyz"))).toEqual(payload);
},
);
it("reads children of a zlib-compressed PYZ", async () => {
const body = Buffer.from("compressed-child");
const path = await writeArchive(
"compressed-pyz",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "child", typeCode: 0, body }],
}),
compress: true,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(derived(body))),
).toMatchObject({ size: derived(body).length });
});
it("rejects a truncated table of contents", async () => {
const path = await writeArchive(
"truncated",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 314,
pythonLibrary: "libpython3.14.dylib",
members: [],
tocBytes: Buffer.concat([Buffer.from([0, 0, 0, 64]), Buffer.alloc(16)]),
}),
);
await expect(scanCanonicalArtifactInventory(path)).rejects.toBeInstanceOf(
ArtifactReaderFailure,
);
await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
reason: "format",
});
});
it("rejects a cookie whose package range is outside the file", async () => {
const path = await writeArchive("outside", cookieRangeOutsideFile());
const reader = new PyInstallerArtifactReader(path);
await expect(async () => {
for await (const entry of reader.entries()) void entry;
}).rejects.toMatchObject({
reason: "format",
message: expect.stringContaining("outside the file"),
});
await reader.close();
});
it("rejects a member whose declared size exceeds the decompression budget", async () => {
const path = await writeArchive(
"bomb",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 314,
pythonLibrary: "libpython3.14.dylib",
members: [
{
name: "bomb.bin",
typeCode: "b",
payload: Buffer.alloc(0),
compress: true,
declaredUncompressedSize: 300 * 1024 * 1024,
},
],
}),
);
await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
reason: "limit",
});
});
it("rejects a zlib member that expands past its declared size", async () => {
const path = await writeArchive(
"expand",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 314,
pythonLibrary: "libpython3.14.dylib",
members: [
{
name: "expand.bin",
typeCode: "b",
payload: Buffer.from("hello world hello world"),
compress: true,
declaredUncompressedSize: 4,
},
],
}),
);
await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
reason: "limit",
});
});
it("reads a compressed member whose uncompressed size is zero", async () => {
const path = await writeArchive(
"empty-compressed",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: "empty.bin",
typeCode: "x",
payload: Buffer.alloc(0),
compress: true,
},
{
name: "empty.py",
typeCode: "s",
payload: Buffer.alloc(0),
compress: true,
},
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }],
}),
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(Buffer.alloc(0))),
).toMatchObject({ size: 0 });
expect(
snapshot.nodes.find(
(node) => node.sha256 === sha256(derived(Buffer.alloc(0))),
),
).toMatchObject({ size: derived(Buffer.alloc(0)).length });
const expanded = await writeArchive(
"zero-declared",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: "expand.bin",
typeCode: "x",
payload: Buffer.from("x"),
compress: true,
declaredUncompressedSize: 0,
},
],
}),
);
await expect(scanCanonicalArtifactInventory(expanded)).rejects.toMatchObject({
reason: "limit",
});
});
it("preserves an uppercase Z ZIP member as a readable file", async () => {
const zip = Buffer.from("PK\u0003\u0004zip-member");
const path = await writeArchive(
"zip-member",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: "payload.zip",
typeCode: "Z",
payload: zip,
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
expect(
snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(),
).toEqual([".", "payload.zip"]);
expect(
snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "payload.zip",
),
).toMatchObject({
entry_kind: "file",
format_metadata: { kind: "pyinstaller-entry", type_code: "Z" },
});
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(zip)),
).toMatchObject({ size: zip.length });
});
it("keeps a PYZ data entry resource path", async () => {
const body = Buffer.from('{"enabled":true}');
const path = await writeArchive(
"pyz-data",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "pkg/config.json", typeCode: 2, body }],
}),
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
expect(
snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(),
).toEqual([".", "PYZ.pyz", "PYZ.pyz/pkg/config.json"]);
expect(
snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "PYZ.pyz/pkg/config.json",
),
).toMatchObject({ entry_kind: "file" });
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(body)),
).toMatchObject({ size: body.length });
});
it("projects CArchive type n entries as symlinks", async () => {
const target = Buffer.from("libfoo.so.1\0");
const payload = Buffer.from("stored-data");
const path = await writeArchive(
"symlink",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: "libfoo.so",
typeCode: "n",
payload: target,
compress: false,
},
{ name: "data.bin", typeCode: "x", payload, compress: false },
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
expect(
snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "libfoo.so",
),
).toMatchObject({
entry_kind: "symlink",
artifact_id: null,
hash_status: "not-hashed",
format_metadata: {
kind: "pyinstaller-entry",
type_code: "n",
compression_flag: 0,
},
limitations: expect.arrayContaining([
"PyInstaller symlink target was not read.",
]),
});
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(target)),
).toBeUndefined();
const directory = await createTestTempDirectory("rea-pyinstaller-out-");
const output = join(directory, "extracted");
await materializeArtifactInventory(
{
inputPath: path,
inputFormat: "elf",
outputRoot: output,
environment: process.env,
integrityPolicy: "fail",
},
path,
snapshot,
);
await expect(readFile(join(output, "libfoo.so"))).rejects.toMatchObject({
code: "ENOENT",
});
expect(await readFile(join(output, "data.bin"))).toEqual(payload);
});
it.each([false, true])(
"retains the file owner after a failed close (temporary PYZ: %s)",
async (temporaryPyz) => {
const path = await writeArchive(
"close-owner",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "libpython3.9.so",
members: [
{
name: temporaryPyz ? "PYZ.pyz" : "data.bin",
typeCode: temporaryPyz ? "z" : "x",
payload: temporaryPyz
? pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] })
: Buffer.from("data"),
compress: temporaryPyz,
},
],
}),
);
const reader = new PyInstallerArtifactReader(path);
for await (const entry of reader.entries()) void entry;
const originalClose = OwnedFileHandle.prototype.close;
const owners: OwnedFileHandle[] = [];
const closeFailure = new Error("close failed");
let attempts = 0;
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
owners.push(this);
attempts += 1;
if (attempts === 1) throw closeFailure;
await originalClose.call(this);
});
try {
await expect(reader.close()).rejects.toMatchObject({
cleanup: { reason: closeFailure.message, resources: [path] },
});
await reader.close();
// Spooled PYZ images close after listing, so only the source owner remains.
expect(attempts).toBe(2);
expect(owners[0]).toBe(owners[1]);
} finally {
close.mockRestore();
await reader.close();
}
},
);
it("retains the descriptor owner when admission cleanup fails", async () => {
const directory = await createTestTempDirectory("rea-pyinstaller-admission-");
const reader = new PyInstallerArtifactReader(directory);
const originalClose = OwnedFileHandle.prototype.close;
const owners: OwnedFileHandle[] = [];
const closeFailure = new Error("admission close failed");
let attempts = 0;
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
owners.push(this);
attempts += 1;
if (attempts === 1) throw closeFailure;
await originalClose.call(this);
});
try {
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({
reason: "format",
message: `PyInstaller source is not a regular file: ${directory}`,
cleanup: {
reason: closeFailure.message,
resources: [directory],
},
});
expect(owners).toHaveLength(1);
await reader.close();
expect(attempts).toBe(2);
expect(owners[0]).toBe(owners[1]);
} finally {
close.mockRestore();
await reader.close().catch(() => undefined);
await owners[0]?.close().catch(() => undefined);
}
});
it("inventories and extracts a Windows member name under its canonical path", async () => {
const storedName = String.raw`pkg\data.bin`;
const pyzName = String.raw`res\icon.bin`;
const payload = Buffer.from("nested-bytes");
const icon = Buffer.from("icon-bytes");
const path = await writeArchive(
"windows",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "python39.dll",
members: [
{ name: storedName, typeCode: "x", payload, compress: false },
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: pyzName, typeCode: 2, body: icon }],
}),
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
const member = snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "pkg/data.bin",
);
const resource = snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "PYZ.pyz/res/icon.bin",
);
expect(member?.format_metadata).toMatchObject({ source_name: storedName });
expect(resource?.format_metadata).toMatchObject({ source_name: pyzName });
const directory = await createTestTempDirectory("rea-pyinstaller-out-");
const output = join(directory, "extracted");
await materializeArtifactInventory(
{
inputPath: path,
inputFormat: "elf",
outputRoot: output,
environment: process.env,
integrityPolicy: "fail",
},
path,
snapshot,
);
expect(await readFile(join(output, "pkg", "data.bin"))).toEqual(payload);
expect(await readFile(join(output, "PYZ.pyz", "res", "icon.bin"))).toEqual(
icon,
);
});
it("rejects a stored member whose declared size disagrees with its bytes", async () => {
const path = await writeArchive(
"stored-size",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "python39.dll",
members: [
{
name: "data.bin",
typeCode: "x",
payload: Buffer.from("abcd"),
compress: false,
declaredUncompressedSize: 0,
},
],
}),
);
await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
reason: "integrity",
message: expect.stringContaining(
"Stored CArchive member size disagrees with its table entry",
),
});
});
it("keeps compressed member spans on the physical extent or unknown", async () => {
const payload = Buffer.alloc(10_000, 0x61);
const path = await writeArchive(
"compressed-span",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
pythonLibrary: "python39.dll",
members: [
{ name: "payload.bin", typeCode: "x", payload, compress: true },
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "app", typeCode: 0, body: payload }],
}),
compress: false,
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
const fileSize = (await stat(path)).size;
expect(fileSize).toBeLessThan(payload.length);
for (const logicalPath of ["payload.bin", "PYZ.pyz/app.pyc"]) {
const occurrence = snapshot.occurrences.find(
(candidate) => candidate.logical_path === logicalPath,
);
if (logicalPath === "payload.bin") {
expect(occurrence?.declared_size).toBeGreaterThan(fileSize);
expect(occurrence?.compressed_size ?? fileSize).toBeLessThan(
occurrence?.declared_size ?? 0,
);
} else {
expect(occurrence?.declared_size).toBeNull();
expect(
snapshot.nodes.find(
(node) => node.artifact_id === occurrence?.artifact_id,
)?.size,
).toBeGreaterThan(fileSize);
}
const span = occurrence?.source_location;
if (span === null || span === undefined) {
expect(span).toBeNull();
continue;
}
expect(span.length).toBe(occurrence?.compressed_size);
expect(span.offset + span.length).toBeLessThanOrEqual(fileSize);
}
for (const occurrence of snapshot.occurrences) {
const span = occurrence.source_location;
if (span === null) continue;
expect(span.offset + span.length).toBeLessThanOrEqual(fileSize);
}
});
it("reconstructs each PYZ image's bytecode with that image's own magic", async () => {
const python314Magic = Buffer.from("2b0e0d0a", "hex");
const first = Buffer.from("first-module");
const second = Buffer.from("second-module");
const path = await writeArchive(
"two-pyz",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 314,
pythonLibrary: "libpython3.14.dylib",
members: [
{
name: "PYZ.pyz",
typeCode: "z",
compress: false,
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "first", typeCode: 0, body: first }],
}),
},
{
name: "second.pyz",
typeCode: "z",
compress: false,
payload: pyzArchive({
magic: python314Magic,
modules: [{ name: "second", typeCode: 0, body: second }],
}),
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
const withHeader = (magic: Buffer, body: Buffer): Buffer =>
Buffer.concat([reconstructedPycHeader(magic), body]);
const firstBytes = withHeader(PYTHON_39_MAGIC, first);
const secondBytes = withHeader(python314Magic, second);
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(firstBytes)),
).toMatchObject({ size: firstBytes.length });
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(secondBytes)),
).toMatchObject({ size: secondBytes.length });
expect(
snapshot.nodes.find(
(node) => node.sha256 === sha256(withHeader(PYTHON_39_MAGIC, second)),
),
).toBeUndefined();
});
it("opens selected PYZ modules lazily and permits repeat reads without retaining their expanded bytes", async () => {
const good = Buffer.from("selected module");
const image = pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: "damaged", typeCode: 0, body: Buffer.from("damaged module") },
{ name: "selected", typeCode: 0, body: good },
],
});
image[12] = 0; // Damage the first zlib header without changing the valid PYZ table.
const path = await writeArchive(
"lazy-pyz",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{ name: "PYZ.pyz", typeCode: "z", payload: image, compress: false },
],
}),
);
const reader = new PyInstallerArtifactReader(path);
try {
const entries = [];
for await (const entry of reader.entries()) entries.push(entry);
const selected = entries.find(
(entry) => entry.path === "PYZ.pyz/selected.pyc",
);
const damaged = entries.find(
(entry) => entry.path === "PYZ.pyz/damaged.pyc",
);
if (selected === undefined || damaged === undefined)
throw new Error("Missing PYZ entries");
expect(selected.declaredSize).toBeNull();
for (let attempt = 0; attempt < 2; attempt += 1)
expect(await buffer(await reader.open(selected))).toEqual(derived(good));
await expect(buffer(await reader.open(damaged))).rejects.toMatchObject({
reason: "format",
});
} finally {
await reader.close();
}
});
it.each([false, true])(
"lists a large file-backed PYZ and reads a selected module (outer compression: %s)",
async (compress) => {
const original = pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: "selected", typeCode: 0, body: Buffer.from("selected module") },
],
});
const toc = original.readUInt32BE(8);
// Unreferenced image padding is legal and must not become retained module bytes.
const image = Buffer.concat([
original.subarray(0, toc),
Buffer.alloc(64 * 1024 * 1024),
original.subarray(toc),
]);
image.writeUInt32BE(toc + 64 * 1024 * 1024, 8);
const path = await writeArchive(
"large-pyz",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [{ name: "PYZ.pyz", typeCode: "z", payload: image, compress }],
}),
);
const reader = new PyInstallerArtifactReader(path);
try {
const entries = [];
for await (const entry of reader.entries()) entries.push(entry);
const selected = entries.find(
(entry) => entry.path === "PYZ.pyz/selected.pyc",
);
if (selected === undefined) throw new Error("Missing selected module");
expect(await buffer(await reader.open(selected))).toEqual(
derived(Buffer.from("selected module")),
);
} finally {
await reader.close();
}
},
);
it("uses known PYZ magic for separate CArchive scripts when the marshal table stays opaque", async () => {
const script = Buffer.from("entry script");
const path = await writeArchive(
"opaque-table",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{ name: "hello", typeCode: "s", payload: script, compress: true },
{
name: "PYZ.pyz",
typeCode: "z",
compress: false,
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [],
toc: Buffer.from([0x4e]),
}),
},
],
}),
);
const snapshot = await scanCanonicalArtifactInventory(path);
const opaque = snapshot.occurrences.find(
(entry) => entry.logical_path === "PYZ.pyz",
);
expect(opaque).toMatchObject({ entry_kind: "file" });
expect(opaque?.limitations.join(" ")).toContain("left opaque");
expect(
snapshot.nodes.find((node) => node.sha256 === sha256(derived(script))),
).toBeDefined();
const output = join(
await createTestTempDirectory("rea-opaque-script-"),
"extracted",
);
await materializeArtifactInventory(
{
inputPath: path,
inputFormat: "elf",
outputRoot: output,
environment: process.env,
integrityPolicy: "fail",
},
path,
snapshot,
);
expect(await readFile(join(output, "hello"))).toEqual(derived(script));
});
it("rejects duplicate CArchive member names before projecting PYZ children", async () => {
const image = pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "child", typeCode: 0, body: Buffer.from("child") }],
});
const path = await writeArchive(
"duplicate-names",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{ name: "PYZ.pyz", typeCode: "z", payload: image, compress: false },
{
name: "PYZ.pyz",
typeCode: "x",
payload: Buffer.alloc(0),
compress: false,
},
],
}),
);
await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
reason: "integrity",
message: "Duplicate CArchive member name at table index 1",
});
});
it("closes each spooled PYZ image after listing and reopens it for a requested module", async () => {
const bodies = [0, 1, 2].map((index) => Buffer.from(`module ${index}`));
const path = await writeArchive(
"spooled-pyz",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: bodies.map((body, index) => ({
name: `PYZ-${index}.pyz`,
typeCode: "z" as const,
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [{ name: "child", typeCode: 0, body }],
}),
compress: true,
})),
}),
);
const closed = new Set<OwnedFileHandle>();
const originalClose = OwnedFileHandle.prototype.close;
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
await originalClose.call(this);
closed.add(this);
});
const rootHandle = await open(path, "r");
const reader = new PyInstallerArtifactReader(path, {
handle: rootHandle,
initial: await rootHandle.stat(),
});
try {
const iterator = reader.entries()[Symbol.asyncIterator]();
const first = await iterator.next();
// Every spooled image is closed once its table is read, before any entry is yielded.
expect(closed.size).toBeGreaterThanOrEqual(bodies.length);
for (const owner of closed) expect(owner.handle.fd).toBe(-1);
const listedHandles = closed.size;
const entries = first.done === true ? [] : [first.value];
for (
let next = await iterator.next();
next.done !== true;
next = await iterator.next()
)
entries.push(next.value);
const selected = entries.find(
(entry) => entry.path === "PYZ-1.pyz/child.pyc",
);
if (selected === undefined) throw new Error("Missing PYZ child");
expect(await buffer(await reader.open(selected))).toEqual(
derived(bodies[1] as Buffer),
);
await reader.close();
expect(closed.size).toBe(listedHandles + 1);
for (const owner of closed) expect(owner.handle.fd).toBe(-1);
expect(rootHandle.fd).toBeGreaterThanOrEqual(0);
} finally {
try {
await reader.close();
} finally {
close.mockRestore();
await rootHandle.close();
}
}
});
it.each([1, 2])(
"retains a private PYZ owner whose close fails at stage %s",
async (failedStage) => {
const path = await writeArchive(
"spool-close-owner",
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }),
compress: true,
},
],
}),
);
const originalClose = OwnedFileHandle.prototype.close;
const owners: OwnedFileHandle[] = [];
const closeFailure = new Error("spool close failed");
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
owners.push(this);
if (owners.length === failedStage) throw closeFailure;
await originalClose.call(this);
});
const reader = new PyInstallerArtifactReader(path);
try {
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).rejects.toBe(closeFailure);
await reader.close();
expect(
owners.filter((owner) => owner === owners[failedStage - 1]),
).toHaveLength(2);
} finally {
close.mockRestore();
await reader.close().catch(() => undefined);
}
},
);