Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

21.8 KB
Raw Blame
import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js";
import { join } from "node:path";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { createHash } from "node:crypto";
import { writeFile, type FileHandle } from "node:fs/promises";
import { buffer } from "node:stream/consumers";
import { deflateSync } from "node:zlib";
import { setImmediate } from "node:timers/promises";
import { expect, it, vi } from "vitest";
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { hashReadable } from "../ArtifactHash.js";
import {
  ArtifactReaderFailure,
  type ArtifactEntry,
} from "../ArtifactReader.js";
import {
  cArchiveExecutable,
  pyzArchive,
  PYTHON_39_MAGIC,
} from "./CArchive.fixture.js";

const hash = (bytes: Buffer): string =>
  createHash("sha256").update(bytes).digest("hex");
const archive = (body: Buffer) =>
  cArchiveExecutable({
    layout: "2.1",
    pythonVersion: 309,
    members: [{ name: "data", typeCode: "x", payload: body, compress: false }],
  });
const members = async (reader: PyInstallerArtifactReader) => {
  const entries: ArtifactEntry[] = [];
  for await (const entry of reader.entries()) entries.push(entry);
  return entries;
};
const deferred = () => {
  let resolve: () => void = () => undefined;
  const promise = new Promise<void>((done) => {
    resolve = done;
  });
  return { promise, resolve };
};

it.each([false, true])(
  "settles independent private and carrier closes and retries only failed owners (carrier failure: %s)",
  async (failCarrier) => {
    const path = join(await createTestTempDirectory("rea-pyi-cleanup-"), "app");
    await writeFile(
      path,
      cArchiveExecutable({
        layout: "2.1",
        pythonVersion: 309,
        members: ["one", "two"].map((name) => ({
          name: `${name}.pyz`,
          typeCode: "z",
          compress: true,
          payload: pyzArchive({
            magic: PYTHON_39_MAGIC,
            modules: [{ name: "app", typeCode: 0, body: Buffer.from(name) }],
          }),
        })),
      }),
    );
    const reader = new PyInstallerArtifactReader(path);
    const entries = (await members(reader)).filter((entry) =>
      entry.path.endsWith("app.pyc"),
    );
    expect(entries).toHaveLength(2);
    const nativeClose = OwnedFileHandle.prototype.close;
    const privateOwners = new Set<OwnedFileHandle>();
    const attempts: OwnedFileHandle[] = [];
    let collecting = true;
    let failing = true;
    const close = vi
      .spyOn(OwnedFileHandle.prototype, "close")
      .mockImplementation(async function (this: OwnedFileHandle) {
        attempts.push(this);
        if (collecting) privateOwners.add(this);
        if (failing && (privateOwners.has(this) || failCarrier))
          throw new Error(
            privateOwners.has(this) ? "private close EIO" : "carrier close EIO",
          );
        await nativeClose.call(this);
      });
    try {
      for (const entry of entries)
        await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
          cleanup: { reason: "private close EIO" },
        });
      expect(privateOwners.size).toBe(2);
      collecting = false;
      attempts.length = 0;
      const failure = await reader.close().then(
        () => undefined,
        (cause: unknown) => cause,
      );
      expect(attempts).toHaveLength(3);
      expect(new Set(attempts).size).toBe(3);
      expect(failure).toMatchObject({
        cleanup: {
          resources: expect.arrayContaining(failCarrier ? [path] : []),
        },
      });
      expect(
        (failure as ArtifactReaderFailure).cleanup?.resources,
      ).toHaveLength(failCarrier ? 3 : 2);
      expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain(
        "private close EIO",
      );
      if (failCarrier)
        expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain(
          "carrier close EIO",
        );
      const carrier = attempts.find((owner) => !privateOwners.has(owner))!;
      expect(carrier.handle.fd >= 0).toBe(failCarrier);
      for (const owner of privateOwners)
        expect(owner.handle.fd).toBeGreaterThanOrEqual(0);
      failing = false;
      attempts.length = 0;
      await reader.close();
      expect(attempts).toHaveLength(failCarrier ? 3 : 2);
      for (const owner of [...privateOwners, carrier])
        expect(owner.handle.fd).toBe(-1);
      attempts.length = 0;
      await reader.close();
      expect(attempts).toEqual([]);
    } finally {
      close.mockRestore();
      await reader.close();
    }
  },
);

it("binds materialization to the inventoried digest and checks cached carrier freshness", async () => {
  const bytes = archive(Buffer.from("first"));
  await withPyInstallerReader(bytes, async (reader, _handle, path) => {
    await expect(reader.prepareContainer("0".repeat(64))).rejects.toMatchObject(
      { reason: "integrity" },
    );
    await reader.prepareContainer(hash(bytes));
    await reader.prepareContainer(hash(bytes));
    await writeFile(path, archive(Buffer.from("other")));
    await expect(reader.prepareContainer(hash(bytes))).rejects.toMatchObject({
      reason: "integrity",
    });
  });
});

it("bounds descriptor reads while hashing a selected stored member", async () => {
  const payload = Buffer.alloc(2 * 1024 * 1024, 0x61);
  await withPyInstallerReader(archive(payload), async (reader, handle) => {
    const entry = (await members(reader))[0]!;
    const read = handle.read.bind(handle);
    let maximum = 0;
    const spy = vi
      .spyOn(handle, "read")
      .mockImplementation((...args: unknown[]) => {
        if (typeof args[2] === "number") maximum = Math.max(maximum, args[2]);
        return Reflect.apply(read, handle, args);
      });
    try {
      const observed = await hashReadable(await reader.open(entry));
      expect(observed.sha256).toBe(hash(payload));
      expect(observed.bytes).toBe(payload.length);
      expect(maximum).toBeGreaterThan(0);
      expect(maximum).toBeLessThanOrEqual(64 * 1024);
    } finally {
      spy.mockRestore();
    }
  });
});

it.each([false, true])(
  "joins pending native reads during cancellation/close (I/O failure: %s)",
  async (fail) => {
    await withPyInstallerReader(
      archive(Buffer.alloc(256 * 1024, 0x61)),
      async (reader, handle) => {
        const entry = (await members(reader))[0]!;
        const entered = deferred(),
          release = deferred();
        const read = handle.read.bind(handle);
        const io = Object.assign(new Error("native read failed"), {
          code: "EIO",
        });
        const caller = new Error("caller stopped reading");
        const controller = new AbortController();
        let held = false;
        const spy = vi
          .spyOn(handle, "read")
          .mockImplementation(async (...args: unknown[]) => {
            if (!held) {
              held = true;
              entered.resolve();
              await release.promise;
              if (fail) throw io;
            }
            return Reflect.apply(read, handle, args);
          });
        const consumed = reader
          .open(entry, controller.signal)
          .then((stream) => buffer(stream));
        void consumed.catch(() => undefined);
        let closing: Promise<void> | undefined;
        try {
          await entered.promise;
          controller.abort(caller);
          let closed = false;
          closing = reader.close().then(() => {
            closed = true;
          });
          await setImmediate();
          expect(closed).toBe(false);
          expect(handle.fd).toBeGreaterThanOrEqual(0);
          release.resolve();
          await expect(consumed).rejects.toMatchObject({
            reason: fail ? "io" : "cancelled",
            cause: fail ? io : caller,
          });
          await closing;
          expect(handle.fd).toBeGreaterThanOrEqual(0);
        } finally {
          release.resolve();
          await Promise.allSettled([
            consumed,
            ...(closing === undefined ? [] : [closing]),
          ]);
          spy.mockRestore();
        }
      },
    );
  },
);

it("preserves BOM characters in length-delimited UTF-8 fields", async () => {
  const bytes = cArchiveExecutable({
    layout: "2.1",
    pythonVersion: 309,
    pythonLibrary: "\ufefflibpython.so",
    members: [
      {
        name: "\ufeffasset",
        typeCode: "x",
        payload: Buffer.from("data"),
        compress: false,
      },
      {
        name: "PYZ.pyz",
        typeCode: "z",
        compress: false,
        payload: pyzArchive({
          magic: PYTHON_39_MAGIC,
          modules: [
            { name: "\ufeffmodule", typeCode: 0, body: Buffer.from("module") },
          ],
        }),
      },
    ],
  });
  await withPyInstallerReader(bytes, async (reader) => {
    const entries = await members(reader);
    expect(reader.formatMetadata()).toMatchObject({
      python_library: "\ufefflibpython.so",
    });
    expect(entries.map((entry) => entry.path)).toContain("\ufeffasset");
    expect(
      entries.find((entry) => entry.path === "\ufeffasset")?.formatMetadata,
    ).toMatchObject({ source_name: "\ufeffasset" });
    expect(
      entries.find((entry) => entry.path === "PYZ.pyz/\ufeffmodule.pyc")
        ?.formatMetadata,
    ).toMatchObject({ source_name: "\ufeffmodule" });
  });
});

it("keeps runtime options as declarations with literal names and distinct source indices", async () => {
  const option = "pyi-runtime-tmpdir /tmp/../chosen";
  await withPyInstallerReader(
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      members: [
        {
          name: option,
          typeCode: "o",
          payload: Buffer.alloc(0),
          compress: false,
        },
        {
          name: option,
          typeCode: "o",
          payload: Buffer.alloc(0),
          compress: false,
        },
        {
          name: "data",
          typeCode: "x",
          payload: Buffer.from("data"),
          compress: false,
        },
      ],
    }),
    async (reader) => {
      const entries = await members(reader);
      const options = entries.filter((entry) => entry.kind === "metadata");
      expect(options.map((entry) => entry.path)).toEqual([
        "__pyinstaller__/runtime-options/0",
        "__pyinstaller__/runtime-options/1",
      ]);
      expect(options.map((entry) => entry.formatMetadata)).toMatchObject([
        { source_name: option, index: 0 },
        { source_name: option, index: 1 },
      ]);
      expect(reader.formatMetadata()).toMatchObject({
        runtime_option_count: 2,
        entry_count: 3,
      });
      await expect(reader.open(options[0]!)).rejects.toMatchObject({
        reason: "format",
      });
    },
  );
});

it("rejects metadata expansion before retaining every repeated long container path", async () => {
  const image = pyzArchive({
    magic: PYTHON_39_MAGIC,
    modules: Array.from({ length: 3000 }, (_, index) => ({
      name: `module${String(index)}`,
      typeCode: 0,
      body: Buffer.from("x"),
    })),
  });
  await withPyInstallerReader(
    cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      members: [
        {
          name: "a".repeat(40000),
          typeCode: "z",
          payload: image,
          compress: false,
        },
      ],
    }),
    async (reader) => {
      let failure: unknown;
      try {
        for await (const entry of reader.entries()) {
          void entry;
        }
      } catch (cause: unknown) {
        failure = cause;
      }
      expect(failure).toBeInstanceOf(ArtifactReaderFailure);
      expect(failure).toMatchObject({ reason: "limit", boundary: "memory" });
    },
  );
});

it.each([false, true])(
  "validates compressed member bounds through the stream (truncated: %s)",
  async (truncated) => {
    const payload = Buffer.from("complete module payload");
    const encoded = deflateSync(payload);
    const bytes = cArchiveExecutable({
      layout: "2.1",
      pythonVersion: 309,
      members: [
        {
          name: "data",
          typeCode: "x",
          payload,
          compress: true,
          declaredUncompressedSize: truncated ? payload.length : 4,
        },
      ],
    });
    if (truncated)
      bytes.writeUInt32BE(encoded.length - 2, 64 + encoded.length + 8);
    await withPyInstallerReader(bytes, async (reader) => {
      const entry = (await members(reader))[0]!;
      await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
        reason: truncated ? "format" : "limit",
      });
    });
  },
);

it("joins a late private-file close and retries its failed owner after cancellation", async () => {
  const bytes = cArchiveExecutable({
    layout: "2.1",
    pythonVersion: 309,
    members: [
      {
        name: "PYZ.pyz",
        typeCode: "z",
        payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }),
        compress: true,
      },
    ],
  });
  await withPyInstallerReader(bytes, async (reader, borrowed) => {
    const entered = deferred(),
      release = deferred();
    const nativeClose = OwnedFileHandle.prototype.close;
    const failure = new Error("private final close failed");
    let owner: OwnedFileHandle | undefined;
    let attempts = 0;
    const spy = vi
      .spyOn(OwnedFileHandle.prototype, "close")
      .mockImplementation(async function (this: OwnedFileHandle) {
        if (owner === undefined) {
          owner = this;
          entered.resolve();
          await release.promise;
          throw failure;
        }
        if (this === owner) attempts += 1;
        await nativeClose.call(this);
      });
    const signal = new AbortController();
    const listing = reader
      .entries(signal.signal)
      [Symbol.asyncIterator]()
      .next()
      .catch((cause: unknown) => cause);
    try {
      await entered.promise;
      let settled = false;
      const closing = reader.close().then(() => {
        settled = true;
      });
      signal.abort(new Error("cancel during final cleanup"));
      await setImmediate();
      expect(settled).toBe(false);
      expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
      release.resolve();
      expect(await listing).toBe(failure);
      await closing;
      expect(attempts).toBe(1);
      expect(owner?.handle.fd).toBe(-1);
      expect(borrowed.fd).toBeGreaterThanOrEqual(0);
    } finally {
      release.resolve();
      spy.mockRestore();
      await listing;
      await reader.close();
    }
  });
});

for (const cancel of [false, true]) {
  it.each([false, true])(
    `awaits private member close before completion (cancel: ${cancel}, close failure: %s)`,
    async (failClose) => {
      const bytes = cArchiveExecutable({
        layout: "2.1",
        pythonVersion: 309,
        members: [
          {
            name: "PYZ.pyz",
            typeCode: "z",
            compress: true,
            payload: pyzArchive({
              magic: PYTHON_39_MAGIC,
              modules: [
                { name: "app", typeCode: 0, body: Buffer.from("body") },
              ],
            }),
          },
        ],
      });
      await withPyInstallerReader(bytes, async (reader, borrowed) => {
        const entry = (await members(reader)).find((item) =>
          item.path.endsWith("app.pyc"),
        )!;
        const entered = deferred(),
          release = deferred();
        const signal = new AbortController();
        const failure = new Error("private member final close failed");
        const nativeClose = OwnedFileHandle.prototype.close;
        let owner: OwnedFileHandle | undefined;
        let attempts = 0;
        const spy = vi
          .spyOn(OwnedFileHandle.prototype, "close")
          .mockImplementation(async function (this: OwnedFileHandle) {
            if (owner === undefined) {
              owner = this;
              attempts += 1;
              entered.resolve();
              await release.promise;
              if (failClose) throw failure;
            } else if (this === owner) attempts += 1;
            await nativeClose.call(this);
          });
        const output = await reader.open(entry, signal.signal);
        const reading = buffer(output).then(
          (value) => ({ ok: true as const, value }),
          (cause: unknown) => ({ ok: false as const, cause }),
        );
        try {
          await entered.promise;
          expect(output.readableEnded).toBe(false);
          expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
          if (cancel) {
            signal.abort(new Error("cancel during member cleanup"));
            await setImmediate();
            expect(output.destroyed).toBe(false);
          }
          release.resolve();
          const result = await reading;
          if (failClose || cancel) {
            expect(result.ok).toBe(false);
            if (result.ok) throw new Error("Expected read failure");
            expect(result.cause).toMatchObject(
              failClose
                ? { reason: "io", message: failure.message }
                : { reason: "cancelled" },
            );
            if (failClose) expect(owner?.handle.fd).toBeGreaterThanOrEqual(0);
          } else {
            expect(result.ok).toBe(true);
            if (!result.ok) throw result.cause;
            expect(result.value.subarray(16)).toEqual(Buffer.from("body"));
          }
          await reader.close();
          expect(attempts).toBe(failClose ? 2 : 1);
          expect(owner?.handle.fd).toBe(-1);
          expect(borrowed.fd).toBeGreaterThanOrEqual(0);
        } finally {
          release.resolve();
          await reading;
          spy.mockRestore();
          await reader.close();
        }
      });
    },
  );
}

it("preserves a private-image read failure when its descriptor close also fails", async () => {
  const bytes = cArchiveExecutable({
    layout: "2.1",
    pythonVersion: 309,
    members: [
      {
        name: "PYZ.pyz",
        typeCode: "z",
        payload: pyzArchive({
          magic: PYTHON_39_MAGIC,
          modules: [{ name: "app", typeCode: 0, body: Buffer.from("body") }],
        }),
        compress: true,
      },
    ],
  });
  await withPyInstallerReader(bytes, async (reader, borrowed) => {
    const entry = (await members(reader)).find((item) =>
      item.path.endsWith("app.pyc"),
    )!;
    const nativeClose = OwnedFileHandle.prototype.close;
    const nativeRead = borrowed.read;
    const io = Object.assign(new Error("private read EIO"), { code: "EIO" });
    let failedOwner: OwnedFileHandle | undefined;
    const read = vi
      .spyOn(Object.getPrototypeOf(borrowed), "read")
      .mockImplementation(function (this: FileHandle, ...args: unknown[]) {
        return this === borrowed
          ? Reflect.apply(nativeRead, this, args)
          : Promise.reject(io);
      });
    const close = vi
      .spyOn(OwnedFileHandle.prototype, "close")
      .mockImplementation(async function (this: OwnedFileHandle) {
        if (this.handle !== borrowed) {
          failedOwner = this;
          throw new Error("private close EIO");
        }
        await nativeClose.call(this);
      });
    try {
      await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
        reason: "io",
        cause: io,
      });
      await expect(reader.close()).rejects.toMatchObject({
        cleanup: { reason: "private close EIO" },
      });
      expect(failedOwner?.handle.fd).toBeGreaterThanOrEqual(0);
    } finally {
      read.mockRestore();
      close.mockRestore();
      await reader.close();
    }
    expect(failedOwner?.handle.fd).toBe(-1);
    expect(borrowed.fd).toBeGreaterThanOrEqual(0);
  });
});

it.each([false, true])(
  "shares owned carrier admission across callers and joins late admission (owner closes: %s)",
  async (ownerCloses) => {
    const bytes = archive(Buffer.from("shared admission"));
    await withPyInstallerReader(
      bytes,
      async (_borrowedReader, borrowed, path) => {
        const reader = new PyInstallerArtifactReader(path);
        const entered = deferred(),
          release = deferred();
        const handles = new Set<FileHandle>();
        const stat = borrowed.stat;
        const spy = vi
          .spyOn(Object.getPrototypeOf(borrowed), "stat")
          .mockImplementation(async function (
            this: FileHandle,
            ...args: unknown[]
          ) {
            if (this !== borrowed) {
              handles.add(this);
              entered.resolve();
              await release.promise;
            }
            return Reflect.apply(stat, this, args);
          });
        const caller = new AbortController(),
          reason = new Error("one admission waiter leaves");
        const first = reader
          .prepareContainer(hash(bytes), caller.signal)
          .catch((cause: unknown) => cause);
        const peer = reader
          .prepareContainer(hash(bytes))
          .catch((cause: unknown) => cause);
        let closing: Promise<void> | undefined;
        try {
          await entered.promise;
          await setImmediate();
          expect(handles.size).toBe(1);
          caller.abort(reason);
          if (ownerCloses) {
            let closed = false;
            closing = reader.close().then(() => {
              closed = true;
            });
            await setImmediate();
            expect(closed).toBe(false);
          }
          release.resolve();
          expect(await first).toMatchObject({
            reason: "cancelled",
            cause: reason,
          });
          if (ownerCloses) {
            expect(await peer).toMatchObject({ reason: "cancelled" });
            await closing;
          } else expect(await peer).toBeUndefined();
          await reader.close();
          for (const handle of handles) expect(handle.fd).toBe(-1);
        } finally {
          release.resolve();
          await Promise.allSettled([first, peer]);
          spy.mockRestore();
          await reader.close();
          await closing;
          for (const handle of handles) {
            await handle.close();
          }
        }
      },
    );
  },
);