Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

35.2 KB
Raw Blame
import { createHash } from "node:crypto";
import { open, readFile, stat, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buffer } from "node:stream/consumers";

import { expect, it, vi } from "vitest";

import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { materializeArtifactInventory } from "../../../tests/fixtures/artifactInventory.js";
import { scanCanonicalArtifactInventory } from "../../../tests/fixtures/artifactInventory.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
import { reconstructedPycHeader } from "./PythonBytecode.js";
import {
  PYTHON_39_MAGIC,
  cArchiveExecutable,
  cookieRangeOutsideFile,
  pyzArchive,
  type CArchiveFixtureMember,
} from "./CArchive.fixture.js";

const sha256 = (bytes: Buffer): string =>
  createHash("sha256").update(bytes).digest("hex");

const derived = (payload: Buffer): Buffer =>
  Buffer.concat([reconstructedPycHeader(PYTHON_39_MAGIC), payload]);

const writeArchive = async (name: string, bytes: Buffer): Promise<string> => {
  const directory = await createTestTempDirectory("rea-pyinstaller-");
  const path = join(directory, name);
  await writeFile(path, bytes);
  return path;
};

const sampleMembers = (
  script: Buffer,
  stored: Buffer,
  pyz: Buffer,
): CArchiveFixtureMember[] => [
  { name: "hello", typeCode: "s", payload: script, compress: true },
  { name: "data.bin", typeCode: "x", payload: stored, compress: false },
  { name: "PYZ.pyz", typeCode: "z", payload: pyz, compress: false },
  {
    name: "pyi-contents-directory _internal",
    typeCode: "o",
    payload: Buffer.alloc(0),
    compress: false,
  },
];

it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async () => {
  const script = Buffer.from("print('hi')");
  const stored = Buffer.from("stored-data");
  const moduleBody = Buffer.from("module-body");
  const path = await writeArchive(
    "hello",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 314,
      pythonLibrary: "libpython3.14.dylib",
      trailer: Buffer.from("signature"),
      members: sampleMembers(
        script,
        stored,
        pyzArchive({
          magic: PYTHON_39_MAGIC,
          modules: [
            { name: "json.decoder", typeCode: 0, body: moduleBody },
            { name: "pkg", typeCode: 1, body: moduleBody },
            { name: "ns", typeCode: 3, body: Buffer.alloc(0) },
          ],
        }),
      ),
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  expect(snapshot.manifest.root_format).toBe("pyinstaller");
  expect(snapshot.coverage.status).toBe("complete");
  const root = snapshot.occurrences.find(
    (occurrence) => occurrence.logical_path === ".",
  );
  expect(root).toMatchObject({ artifact_kind: "container" });
  expect(root?.format_metadata).toMatchObject({
    kind: "pyinstaller",
    layout_version: "2.1",
    python_version_raw: 314,
    python_major: 3,
    python_minor: 14,
    python_library: "libpython3.14.dylib",
    runtime_option_count: 1,
  });
  const hello = snapshot.occurrences.find(
    (occurrence) => occurrence.logical_path === "hello",
  );
  expect(hello?.format_metadata).toMatchObject({
    kind: "pyinstaller-entry",
    container: "carchive",
    type_code: "s",
    compression_flag: 1,
    bytecode_header: "unknown",
    bytecode_magic: null,
  });
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(script)),
  ).toMatchObject({ size: script.length });
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(stored)),
  ).toBeDefined();
  expect(
    snapshot.occurrences.map((occurrence) => occurrence.logical_path),
  ).toEqual(
    expect.arrayContaining([
      "hello",
      "data.bin",
      "PYZ.pyz",
      "PYZ.pyz/json/decoder.pyc",
      "PYZ.pyz/pkg/__init__.pyc",
      "PYZ.pyz/ns",
      "__pyinstaller__/runtime-options/3",
    ]),
  );
  expect(
    snapshot.occurrences.find(
      (occurrence) => occurrence.logical_path === "PYZ.pyz",
    )?.entry_kind,
  ).toBe("directory");
  const scriptEdge = snapshot.edges.find(
    (edge) => edge.occurrence_id === hello?.occurrence_id,
  );
  expect(scriptEdge?.relation).toBe("contains");
});

it("records a 2.0 cookie without a Python library name", async () => {
  const path = await writeArchive(
    "legacy",
    cArchiveExecutable({
      layout: "2.0",
      pythonVersion: 307,
      members: [
        {
          name: "hello",
          typeCode: "s",
          payload: Buffer.from("print"),
          compress: false,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  const root = snapshot.occurrences.find(
    (occurrence) => occurrence.logical_path === ".",
  );
  expect(root?.format_metadata).toMatchObject({
    kind: "pyinstaller",
    layout_version: "2.0",
    python_version_raw: 307,
    python_major: 3,
    python_minor: 7,
    python_library: null,
  });
});

it("extracts derived members into the safe output tree", async () => {
  const script = Buffer.from("print('hi')");
  const path = await writeArchive(
    "hello",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: sampleMembers(
        script,
        Buffer.from("stored"),
        pyzArchive({
          magic: PYTHON_39_MAGIC,
          modules: [
            { name: "app", typeCode: 0, body: Buffer.from("app-body") },
          ],
        }),
      ),
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  const directory = await createTestTempDirectory("rea-pyinstaller-out-");
  const output = join(directory, "extracted");
  const extracted = await materializeArtifactInventory(
    {
      inputPath: path,
      inputFormat: "elf",
      outputRoot: output,
      environment: process.env,
      integrityPolicy: "fail",
    },
    path,
    snapshot,
  );
  expect(await readFile(join(output, "hello"))).toEqual(derived(script));
  expect(
    extracted.artifacts.find((artifact) => artifact.relative_path === "hello")
      ?.sha256,
  ).toBe(sha256(derived(script)));
  expect(await readFile(join(output, "PYZ.pyz", "app.pyc"))).toEqual(
    derived(Buffer.from("app-body")),
  );
});

it("reports encrypted PYZ entries and refuses to extract them", async () => {
  const path = await writeArchive(
    "encrypted",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: [
        {
          name: "pyimod00_crypto_key",
          typeCode: "m",
          payload: Buffer.from("key"),
          compress: false,
        },
        {
          name: "PYZ.pyz",
          typeCode: "z",
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [
              { name: "secret", typeCode: 0, body: Buffer.from("hidden") },
            ],
          }),
          compress: false,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  expect(
    snapshot.occurrences.find(
      (occurrence) => occurrence.logical_path === "PYZ.pyz/secret.pyc",
    ),
  ).toMatchObject({
    encrypted: true,
    hash_status: "unavailable",
  });
  const directory = await createTestTempDirectory("rea-pyinstaller-out-");
  await expect(
    materializeArtifactInventory(
      {
        inputPath: path,
        inputFormat: "elf",
        outputRoot: join(directory, "extracted"),
        environment: process.env,
        integrityPolicy: "fail",
      },
      path,
      snapshot,
    ),
  ).rejects.toThrow(/encrypted/u);
});

it.each([false, true])(
  "leaves an unsupported PYZ magic as one opaque member (marker: %s)",
  async (marker) => {
    const payload = pyzArchive({
      magic: Buffer.alloc(4),
      modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }],
    });
    const path = await writeArchive(
      "opaque",
      cArchiveExecutable({
        layout: "2.1",
        pythonVersion: 314,
        pythonLibrary: "libpython3.14.dylib",
        members: [
          ...(marker
            ? [
                {
                  name: "pyimod00_crypto_key",
                  typeCode: "m",
                  payload: Buffer.from("key"),
                  compress: false,
                },
              ]
            : []),
          {
            name: "PYZ.pyz",
            typeCode: "z",
            payload,
            compress: false,
          },
        ],
      }),
    );
    const snapshot = await scanCanonicalArtifactInventory(path);
    const pyz = snapshot.occurrences.find(
      (occurrence) => occurrence.logical_path === "PYZ.pyz",
    );
    expect(pyz).toMatchObject({
      entry_kind: "file",
      encrypted: false,
      hash_status: "verified",
    });
    expect(snapshot.coverage.status).toBe("unknown");
    expect(
      snapshot.nodes.find((node) => node.artifact_id === pyz?.artifact_id),
    ).toMatchObject({ sha256: sha256(payload), size: payload.length });
    expect(pyz?.format_metadata).toMatchObject({
      pyz_python_magic: "00000000",
      pyz_python_version: null,
      pyz_status: "opaque",
      encryption_source: marker ? "carchive-marker" : null,
    });
    expect(pyz?.limitations.join(" ")).toContain("left opaque");
    expect(
      snapshot.occurrences.some((occurrence) =>
        occurrence.logical_path.endsWith(".pyc"),
      ),
    ).toBe(false);
    const outputRoot = join(
      await createTestTempDirectory("rea-opaque-out-"),
      "extracted",
    );
    await materializeArtifactInventory(
      {
        inputPath: path,
        inputFormat: "elf",
        outputRoot,
        environment: process.env,
        integrityPolicy: "fail",
      },
      path,
      snapshot,
    );
    expect(await readFile(join(outputRoot, "PYZ.pyz"))).toEqual(payload);
  },
);

it("reads children of a zlib-compressed PYZ", async () => {
  const body = Buffer.from("compressed-child");
  const path = await writeArchive(
    "compressed-pyz",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: [
        {
          name: "PYZ.pyz",
          typeCode: "z",
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [{ name: "child", typeCode: 0, body }],
          }),
          compress: true,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(derived(body))),
  ).toMatchObject({ size: derived(body).length });
});

it("rejects a truncated table of contents", async () => {
  const path = await writeArchive(
    "truncated",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 314,
      pythonLibrary: "libpython3.14.dylib",
      members: [],
      tocBytes: Buffer.concat([Buffer.from([0, 0, 0, 64]), Buffer.alloc(16)]),
    }),
  );
  await expect(scanCanonicalArtifactInventory(path)).rejects.toBeInstanceOf(
    ArtifactReaderFailure,
  );
  await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
    reason: "format",
  });
});

it("rejects a cookie whose package range is outside the file", async () => {
  const path = await writeArchive("outside", cookieRangeOutsideFile());
  const reader = new PyInstallerArtifactReader(path);
  await expect(async () => {
    for await (const entry of reader.entries()) void entry;
  }).rejects.toMatchObject({
    reason: "format",
    message: expect.stringContaining("outside the file"),
  });
  await reader.close();
});

it("rejects a member whose declared size exceeds the decompression budget", async () => {
  const path = await writeArchive(
    "bomb",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 314,
      pythonLibrary: "libpython3.14.dylib",
      members: [
        {
          name: "bomb.bin",
          typeCode: "b",
          payload: Buffer.alloc(0),
          compress: true,
          declaredUncompressedSize: 300 * 1024 * 1024,
        },
      ],
    }),
  );
  await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
    reason: "limit",
  });
});

it("rejects a zlib member that expands past its declared size", async () => {
  const path = await writeArchive(
    "expand",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 314,
      pythonLibrary: "libpython3.14.dylib",
      members: [
        {
          name: "expand.bin",
          typeCode: "b",
          payload: Buffer.from("hello world hello world"),
          compress: true,
          declaredUncompressedSize: 4,
        },
      ],
    }),
  );
  await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
    reason: "limit",
  });
});

it("reads a compressed member whose uncompressed size is zero", async () => {
  const path = await writeArchive(
    "empty-compressed",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: [
        {
          name: "empty.bin",
          typeCode: "x",
          payload: Buffer.alloc(0),
          compress: true,
        },
        {
          name: "empty.py",
          typeCode: "s",
          payload: Buffer.alloc(0),
          compress: true,
        },
        {
          name: "PYZ.pyz",
          typeCode: "z",
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }],
          }),
          compress: false,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(Buffer.alloc(0))),
  ).toMatchObject({ size: 0 });
  expect(
    snapshot.nodes.find(
      (node) => node.sha256 === sha256(derived(Buffer.alloc(0))),
    ),
  ).toMatchObject({ size: derived(Buffer.alloc(0)).length });

  const expanded = await writeArchive(
    "zero-declared",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: [
        {
          name: "expand.bin",
          typeCode: "x",
          payload: Buffer.from("x"),
          compress: true,
          declaredUncompressedSize: 0,
        },
      ],
    }),
  );
  await expect(scanCanonicalArtifactInventory(expanded)).rejects.toMatchObject({
    reason: "limit",
  });
});

it("preserves an uppercase Z ZIP member as a readable file", async () => {
  const zip = Buffer.from("PK\u0003\u0004zip-member");
  const path = await writeArchive(
    "zip-member",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: [
        {
          name: "payload.zip",
          typeCode: "Z",
          payload: zip,
          compress: false,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  expect(
    snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(),
  ).toEqual([".", "payload.zip"]);
  expect(
    snapshot.occurrences.find(
      (occurrence) => occurrence.logical_path === "payload.zip",
    ),
  ).toMatchObject({
    entry_kind: "file",
    format_metadata: { kind: "pyinstaller-entry", type_code: "Z" },
  });
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(zip)),
  ).toMatchObject({ size: zip.length });
});

it("keeps a PYZ data entry resource path", async () => {
  const body = Buffer.from('{"enabled":true}');
  const path = await writeArchive(
    "pyz-data",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: [
        {
          name: "PYZ.pyz",
          typeCode: "z",
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [{ name: "pkg/config.json", typeCode: 2, body }],
          }),
          compress: false,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  expect(
    snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(),
  ).toEqual([".", "PYZ.pyz", "PYZ.pyz/pkg/config.json"]);
  expect(
    snapshot.occurrences.find(
      (occurrence) => occurrence.logical_path === "PYZ.pyz/pkg/config.json",
    ),
  ).toMatchObject({ entry_kind: "file" });
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(body)),
  ).toMatchObject({ size: body.length });
});

it("projects CArchive type n entries as symlinks", async () => {
  const target = Buffer.from("libfoo.so.1\0");
  const payload = Buffer.from("stored-data");
  const path = await writeArchive(
    "symlink",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "libpython3.9.so",
      members: [
        {
          name: "libfoo.so",
          typeCode: "n",
          payload: target,
          compress: false,
        },
        { name: "data.bin", typeCode: "x", payload, compress: false },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  expect(
    snapshot.occurrences.find(
      (occurrence) => occurrence.logical_path === "libfoo.so",
    ),
  ).toMatchObject({
    entry_kind: "symlink",
    artifact_id: null,
    hash_status: "not-hashed",
    format_metadata: {
      kind: "pyinstaller-entry",
      type_code: "n",
      compression_flag: 0,
    },
    limitations: expect.arrayContaining([
      "PyInstaller symlink target was not read.",
    ]),
  });
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(target)),
  ).toBeUndefined();
  const directory = await createTestTempDirectory("rea-pyinstaller-out-");
  const output = join(directory, "extracted");
  await materializeArtifactInventory(
    {
      inputPath: path,
      inputFormat: "elf",
      outputRoot: output,
      environment: process.env,
      integrityPolicy: "fail",
    },
    path,
    snapshot,
  );
  await expect(readFile(join(output, "libfoo.so"))).rejects.toMatchObject({
    code: "ENOENT",
  });
  expect(await readFile(join(output, "data.bin"))).toEqual(payload);
});

it.each([false, true])(
  "retains the file owner after a failed close (temporary PYZ: %s)",
  async (temporaryPyz) => {
    const path = await writeArchive(
      "close-owner",
      cArchiveExecutable({
        layout: "2.1",
        pythonVersion: 309,
        pythonLibrary: "libpython3.9.so",
        members: [
          {
            name: temporaryPyz ? "PYZ.pyz" : "data.bin",
            typeCode: temporaryPyz ? "z" : "x",
            payload: temporaryPyz
              ? pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] })
              : Buffer.from("data"),
            compress: temporaryPyz,
          },
        ],
      }),
    );
    const reader = new PyInstallerArtifactReader(path);
    for await (const entry of reader.entries()) void entry;
    const originalClose = OwnedFileHandle.prototype.close;
    const owners: OwnedFileHandle[] = [];
    const closeFailure = new Error("close failed");
    let attempts = 0;
    const close = vi
      .spyOn(OwnedFileHandle.prototype, "close")
      .mockImplementation(async function (this: OwnedFileHandle) {
        owners.push(this);
        attempts += 1;
        if (attempts === 1) throw closeFailure;
        await originalClose.call(this);
      });
    try {
      await expect(reader.close()).rejects.toMatchObject({
        cleanup: { reason: closeFailure.message, resources: [path] },
      });
      await reader.close();
      // Spooled PYZ images close after listing, so only the source owner remains.
      expect(attempts).toBe(2);
      expect(owners[0]).toBe(owners[1]);
    } finally {
      close.mockRestore();
      await reader.close();
    }
  },
);

it("retains the descriptor owner when admission cleanup fails", async () => {
  const directory = await createTestTempDirectory("rea-pyinstaller-admission-");
  const reader = new PyInstallerArtifactReader(directory);
  const originalClose = OwnedFileHandle.prototype.close;
  const owners: OwnedFileHandle[] = [];
  const closeFailure = new Error("admission close failed");
  let attempts = 0;
  const close = vi
    .spyOn(OwnedFileHandle.prototype, "close")
    .mockImplementation(async function (this: OwnedFileHandle) {
      owners.push(this);
      attempts += 1;
      if (attempts === 1) throw closeFailure;
      await originalClose.call(this);
    });
  try {
    await expect(
      reader.entries()[Symbol.asyncIterator]().next(),
    ).rejects.toMatchObject({
      reason: "format",
      message: `PyInstaller source is not a regular file: ${directory}`,
      cleanup: {
        reason: closeFailure.message,
        resources: [directory],
      },
    });
    expect(owners).toHaveLength(1);
    await reader.close();
    expect(attempts).toBe(2);
    expect(owners[0]).toBe(owners[1]);
  } finally {
    close.mockRestore();
    await reader.close().catch(() => undefined);
    await owners[0]?.close().catch(() => undefined);
  }
});

it("inventories and extracts a Windows member name under its canonical path", async () => {
  const storedName = String.raw`pkg\data.bin`;
  const pyzName = String.raw`res\icon.bin`;
  const payload = Buffer.from("nested-bytes");
  const icon = Buffer.from("icon-bytes");
  const path = await writeArchive(
    "windows",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "python39.dll",
      members: [
        { name: storedName, typeCode: "x", payload, compress: false },
        {
          name: "PYZ.pyz",
          typeCode: "z",
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [{ name: pyzName, typeCode: 2, body: icon }],
          }),
          compress: false,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  const member = snapshot.occurrences.find(
    (occurrence) => occurrence.logical_path === "pkg/data.bin",
  );
  const resource = snapshot.occurrences.find(
    (occurrence) => occurrence.logical_path === "PYZ.pyz/res/icon.bin",
  );
  expect(member?.format_metadata).toMatchObject({ source_name: storedName });
  expect(resource?.format_metadata).toMatchObject({ source_name: pyzName });
  const directory = await createTestTempDirectory("rea-pyinstaller-out-");
  const output = join(directory, "extracted");
  await materializeArtifactInventory(
    {
      inputPath: path,
      inputFormat: "elf",
      outputRoot: output,
      environment: process.env,
      integrityPolicy: "fail",
    },
    path,
    snapshot,
  );
  expect(await readFile(join(output, "pkg", "data.bin"))).toEqual(payload);
  expect(await readFile(join(output, "PYZ.pyz", "res", "icon.bin"))).toEqual(
    icon,
  );
});

it("rejects a stored member whose declared size disagrees with its bytes", async () => {
  const path = await writeArchive(
    "stored-size",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "python39.dll",
      members: [
        {
          name: "data.bin",
          typeCode: "x",
          payload: Buffer.from("abcd"),
          compress: false,
          declaredUncompressedSize: 0,
        },
      ],
    }),
  );
  await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
    reason: "integrity",
    message: expect.stringContaining(
      "Stored CArchive member size disagrees with its table entry",
    ),
  });
});

it("keeps compressed member spans on the physical extent or unknown", async () => {
  const payload = Buffer.alloc(10_000, 0x61);
  const path = await writeArchive(
    "compressed-span",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      pythonLibrary: "python39.dll",
      members: [
        { name: "payload.bin", typeCode: "x", payload, compress: true },
        {
          name: "PYZ.pyz",
          typeCode: "z",
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [{ name: "app", typeCode: 0, body: payload }],
          }),
          compress: false,
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  const fileSize = (await stat(path)).size;
  expect(fileSize).toBeLessThan(payload.length);
  for (const logicalPath of ["payload.bin", "PYZ.pyz/app.pyc"]) {
    const occurrence = snapshot.occurrences.find(
      (candidate) => candidate.logical_path === logicalPath,
    );
    if (logicalPath === "payload.bin") {
      expect(occurrence?.declared_size).toBeGreaterThan(fileSize);
      expect(occurrence?.compressed_size ?? fileSize).toBeLessThan(
        occurrence?.declared_size ?? 0,
      );
    } else {
      expect(occurrence?.declared_size).toBeNull();
      expect(
        snapshot.nodes.find(
          (node) => node.artifact_id === occurrence?.artifact_id,
        )?.size,
      ).toBeGreaterThan(fileSize);
    }
    const span = occurrence?.source_location;
    if (span === null || span === undefined) {
      expect(span).toBeNull();
      continue;
    }
    expect(span.length).toBe(occurrence?.compressed_size);
    expect(span.offset + span.length).toBeLessThanOrEqual(fileSize);
  }
  for (const occurrence of snapshot.occurrences) {
    const span = occurrence.source_location;
    if (span === null) continue;
    expect(span.offset + span.length).toBeLessThanOrEqual(fileSize);
  }
});

it("reconstructs each PYZ image's bytecode with that image's own magic", async () => {
  const python314Magic = Buffer.from("2b0e0d0a", "hex");
  const first = Buffer.from("first-module");
  const second = Buffer.from("second-module");
  const path = await writeArchive(
    "two-pyz",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 314,
      pythonLibrary: "libpython3.14.dylib",
      members: [
        {
          name: "PYZ.pyz",
          typeCode: "z",
          compress: false,
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [{ name: "first", typeCode: 0, body: first }],
          }),
        },
        {
          name: "second.pyz",
          typeCode: "z",
          compress: false,
          payload: pyzArchive({
            magic: python314Magic,
            modules: [{ name: "second", typeCode: 0, body: second }],
          }),
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  const withHeader = (magic: Buffer, body: Buffer): Buffer =>
    Buffer.concat([reconstructedPycHeader(magic), body]);
  const firstBytes = withHeader(PYTHON_39_MAGIC, first);
  const secondBytes = withHeader(python314Magic, second);
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(firstBytes)),
  ).toMatchObject({ size: firstBytes.length });
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(secondBytes)),
  ).toMatchObject({ size: secondBytes.length });
  expect(
    snapshot.nodes.find(
      (node) => node.sha256 === sha256(withHeader(PYTHON_39_MAGIC, second)),
    ),
  ).toBeUndefined();
});

it("opens selected PYZ modules lazily and permits repeat reads without retaining their expanded bytes", async () => {
  const good = Buffer.from("selected module");
  const image = pyzArchive({
    magic: PYTHON_39_MAGIC,
    modules: [
      { name: "damaged", typeCode: 0, body: Buffer.from("damaged module") },
      { name: "selected", typeCode: 0, body: good },
    ],
  });
  image[12] = 0; // Damage the first zlib header without changing the valid PYZ table.
  const path = await writeArchive(
    "lazy-pyz",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      members: [
        { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false },
      ],
    }),
  );
  const reader = new PyInstallerArtifactReader(path);
  try {
    const entries = [];
    for await (const entry of reader.entries()) entries.push(entry);
    const selected = entries.find(
      (entry) => entry.path === "PYZ.pyz/selected.pyc",
    );
    const damaged = entries.find(
      (entry) => entry.path === "PYZ.pyz/damaged.pyc",
    );
    if (selected === undefined || damaged === undefined)
      throw new Error("Missing PYZ entries");
    expect(selected.declaredSize).toBeNull();
    for (let attempt = 0; attempt < 2; attempt += 1)
      expect(await buffer(await reader.open(selected))).toEqual(derived(good));
    await expect(buffer(await reader.open(damaged))).rejects.toMatchObject({
      reason: "format",
    });
  } finally {
    await reader.close();
  }
});

it.each([false, true])(
  "lists a large file-backed PYZ and reads a selected module (outer compression: %s)",
  async (compress) => {
    const original = pyzArchive({
      magic: PYTHON_39_MAGIC,
      modules: [
        { name: "selected", typeCode: 0, body: Buffer.from("selected module") },
      ],
    });
    const toc = original.readUInt32BE(8);
    // Unreferenced image padding is legal and must not become retained module bytes.
    const image = Buffer.concat([
      original.subarray(0, toc),
      Buffer.alloc(64 * 1024 * 1024),
      original.subarray(toc),
    ]);
    image.writeUInt32BE(toc + 64 * 1024 * 1024, 8);
    const path = await writeArchive(
      "large-pyz",
      cArchiveExecutable({
        layout: "2.1",
        pythonVersion: 309,
        members: [{ name: "PYZ.pyz", typeCode: "z", payload: image, compress }],
      }),
    );
    const reader = new PyInstallerArtifactReader(path);
    try {
      const entries = [];
      for await (const entry of reader.entries()) entries.push(entry);
      const selected = entries.find(
        (entry) => entry.path === "PYZ.pyz/selected.pyc",
      );
      if (selected === undefined) throw new Error("Missing selected module");
      expect(await buffer(await reader.open(selected))).toEqual(
        derived(Buffer.from("selected module")),
      );
    } finally {
      await reader.close();
    }
  },
);

it("uses known PYZ magic for separate CArchive scripts when the marshal table stays opaque", async () => {
  const script = Buffer.from("entry script");
  const path = await writeArchive(
    "opaque-table",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      members: [
        { name: "hello", typeCode: "s", payload: script, compress: true },
        {
          name: "PYZ.pyz",
          typeCode: "z",
          compress: false,
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [],
            toc: Buffer.from([0x4e]),
          }),
        },
      ],
    }),
  );
  const snapshot = await scanCanonicalArtifactInventory(path);
  const opaque = snapshot.occurrences.find(
    (entry) => entry.logical_path === "PYZ.pyz",
  );
  expect(opaque).toMatchObject({ entry_kind: "file" });
  expect(opaque?.limitations.join(" ")).toContain("left opaque");
  expect(
    snapshot.nodes.find((node) => node.sha256 === sha256(derived(script))),
  ).toBeDefined();
  const output = join(
    await createTestTempDirectory("rea-opaque-script-"),
    "extracted",
  );
  await materializeArtifactInventory(
    {
      inputPath: path,
      inputFormat: "elf",
      outputRoot: output,
      environment: process.env,
      integrityPolicy: "fail",
    },
    path,
    snapshot,
  );
  expect(await readFile(join(output, "hello"))).toEqual(derived(script));
});

it("rejects duplicate CArchive member names before projecting PYZ children", async () => {
  const image = pyzArchive({
    magic: PYTHON_39_MAGIC,
    modules: [{ name: "child", typeCode: 0, body: Buffer.from("child") }],
  });
  const path = await writeArchive(
    "duplicate-names",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      members: [
        { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false },
        {
          name: "PYZ.pyz",
          typeCode: "x",
          payload: Buffer.alloc(0),
          compress: false,
        },
      ],
    }),
  );
  await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({
    reason: "integrity",
    message: "Duplicate CArchive member name at table index 1",
  });
});

it("closes each spooled PYZ image after listing and reopens it for a requested module", async () => {
  const bodies = [0, 1, 2].map((index) => Buffer.from(`module ${index}`));
  const path = await writeArchive(
    "spooled-pyz",
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      members: bodies.map((body, index) => ({
        name: `PYZ-${index}.pyz`,
        typeCode: "z" as const,
        payload: pyzArchive({
          magic: PYTHON_39_MAGIC,
          modules: [{ name: "child", typeCode: 0, body }],
        }),
        compress: true,
      })),
    }),
  );
  const closed = new Set<OwnedFileHandle>();
  const originalClose = OwnedFileHandle.prototype.close;
  const close = vi
    .spyOn(OwnedFileHandle.prototype, "close")
    .mockImplementation(async function (this: OwnedFileHandle) {
      await originalClose.call(this);
      closed.add(this);
    });
  const rootHandle = await open(path, "r");
  const reader = new PyInstallerArtifactReader(path, {
    handle: rootHandle,
    initial: await rootHandle.stat(),
  });
  try {
    const iterator = reader.entries()[Symbol.asyncIterator]();
    const first = await iterator.next();
    // Every spooled image is closed once its table is read, before any entry is yielded.
    expect(closed.size).toBeGreaterThanOrEqual(bodies.length);
    for (const owner of closed) expect(owner.handle.fd).toBe(-1);
    const listedHandles = closed.size;
    const entries = first.done === true ? [] : [first.value];
    for (
      let next = await iterator.next();
      next.done !== true;
      next = await iterator.next()
    )
      entries.push(next.value);
    const selected = entries.find(
      (entry) => entry.path === "PYZ-1.pyz/child.pyc",
    );
    if (selected === undefined) throw new Error("Missing PYZ child");
    expect(await buffer(await reader.open(selected))).toEqual(
      derived(bodies[1] as Buffer),
    );
    await reader.close();
    expect(closed.size).toBe(listedHandles + 1);
    for (const owner of closed) expect(owner.handle.fd).toBe(-1);
    expect(rootHandle.fd).toBeGreaterThanOrEqual(0);
  } finally {
    try {
      await reader.close();
    } finally {
      close.mockRestore();
      await rootHandle.close();
    }
  }
});

it.each([1, 2])(
  "retains a private PYZ owner whose close fails at stage %s",
  async (failedStage) => {
    const path = await writeArchive(
      "spool-close-owner",
      cArchiveExecutable({
        layout: "2.1",
        pythonVersion: 309,
        members: [
          {
            name: "PYZ.pyz",
            typeCode: "z",
            payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }),
            compress: true,
          },
        ],
      }),
    );
    const originalClose = OwnedFileHandle.prototype.close;
    const owners: OwnedFileHandle[] = [];
    const closeFailure = new Error("spool close failed");
    const close = vi
      .spyOn(OwnedFileHandle.prototype, "close")
      .mockImplementation(async function (this: OwnedFileHandle) {
        owners.push(this);
        if (owners.length === failedStage) throw closeFailure;
        await originalClose.call(this);
      });
    const reader = new PyInstallerArtifactReader(path);
    try {
      await expect(
        reader.entries()[Symbol.asyncIterator]().next(),
      ).rejects.toBe(closeFailure);
      await reader.close();
      expect(
        owners.filter((owner) => owner === owners[failedStage - 1]),
      ).toHaveLength(2);
    } finally {
      close.mockRestore();
      await reader.close().catch(() => undefined);
    }
  },
);