Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(python): compare code objects without inline caches

morluto3784a8e547f0
diff --git a/docs/python-pyc.md b/docs/python-pyc.md
index 614c87aa0..a33f20073 100644
--- a/docs/python-pyc.md
+++ b/docs/python-pyc.md
@@ -49,12 +49,16 @@ filename, the signed 32-bit `co_firstlineno` value, argument counts, flags,
names, locals (or `localsplusnames` on 3.11+), const
summaries, and bytecode length plus SHA-256.
-Code-object equality used by sets and dictionaries follows the selected layout:
+Code-object equality used by sets and dictionaries follows the selected CPython format:
3.7–3.10 derive `CO_NOFREE` from the free/cell tables for equality, while
3.11+ compare raw flags; raw `co_flags` remain in reported facts. The older
-layouts ignore line tables, while 3.11+ include line and exception tables. The
-artifact SHA-256 identifies the exact selected `.pyc` bytes independently of
-this decoded object graph. Reference slots follow each tag's CPython behavior:
+layouts ignore line tables, while 3.11+ include line and exception tables and
+skip inline-cache code units according to the cache table for the exact magic.
+Reported bytecode length and SHA-256 still describe the original code bytes.
+This does not emulate CPython's deoptimization of specialized or instrumented
+opcode bytes in handcrafted pyc files.
+The artifact SHA-256 identifies the exact selected `.pyc` bytes independently
+of this decoded object graph. Reference slots follow each tag's CPython behavior:
singleton values and `TYPE_REF` do not reserve new slots even when their tag
has `FLAG_REF` set.
diff --git a/src/python/PythonPycBytecodeEquality.ts b/src/python/PythonPycBytecodeEquality.ts
new file mode 100644
index 000000000..9936d501b
--- /dev/null
+++ b/src/python/PythonPycBytecodeEquality.ts
@@ -0,0 +1,468 @@
+import { createHash } from "node:crypto";
+
+type InlineCacheEntry = readonly [opcode: number, codeUnits: number];
+
+const inlineCacheProfiles = (
+ magics: readonly number[],
+ entries: readonly InlineCacheEntry[],
+): ReadonlyArray<readonly [number, ReadonlyMap<number, number>]> => {
+ const profile = new Map(entries);
+ return magics.map((magic) => [magic, profile]);
+};
+
+/**
+ * CPython equality skips cache code units after comparing original code size.
+ * These tables are pinned to CPython bytecode snapshots, not the shared marshal
+ * layout. The 3.11 profile is from v3.11.9; commit
+ * 616622cab7200bac476bf753d6cc98f4ee48808c adds magic 3495 with the same
+ * opcode/cache table. 3.12 magic/profile transitions
+ * follow CPython's v3.12.12 magic history and these cache-changing source
+ * commits: f012df706cf3a9e63625b8166fe3f419855e9a94 (3503),
+ * b083450f8896bb4a29ac522e4474d91c056b9f32 (3504),
+ * 5fcfdd87c9b5066a581d3ccb4b2fede938f343ec (3505),
+ * 6e4e14d98fe0868981f29701496d57a8223c5407 (3515),
+ * 7b14c2ef194b6eed79670aa9d7e29ab8e2256a56 (3516),
+ * 160f2fe2b90ed5ec7838cb4141dd35768422891f (3519),
+ * 08b67fb34f4519be1b0bb4673643a2c761c7ae92 (3521),
+ * 0444ae24875489dc290d8efdb9ee6440ee60dac8 (3523),
+ * 121057aa3600c4d4d392539aeb79e1b09fd5659d (3524),
+ * b4978ff872be5102117b4e25d93dbbb4e04c8292 (3525),
+ * ef25febcf2ede92a03c5ea00a13e167e0b5cb274 (3528), and
+ * 77262458fe3fe9f3f0266bbf578675a906b1e353 (3530). The 3.13 magic history
+ * is recorded in CPython's Lib/importlib/_bootstrap_external.py; opcode/cache sources include
+ * 1d857da7f0e4858e561223f319ae5afe737d5657 (3553/3554),
+ * 665a4391e10167dad1c854fb604c86f336fcd331 (3559),
+ * 3b2ecbc1275bd05534885cee9ac1389987238561 (3560),
+ * bcce5e271815c0bdbe894964e853210d2c75949b (3561),
+ * 8b55adfa8ff05477b4be7def36db7b66c73f181d (3562),
+ * 22e65eecaad3f5337862319687047afe9861e1ef (3563), and
+ * 52cc4af6ae9002f11605f91b672746c127494efd (3564). Adjacent magics share a
+ * profile only where their source changes do not alter cache widths or IDs.
+ */
+const INLINE_CACHE_ENTRIES_BY_MAGIC: ReadonlyMap<
+ number,
+ ReadonlyMap<number, number>
+> = new Map([
+ ...inlineCacheProfiles(
+ [3494, 3495],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [95, 4],
+ [106, 4],
+ [107, 2],
+ [116, 5],
+ [122, 1],
+ [160, 10],
+ [166, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3500, 3501, 3502],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [95, 4],
+ [106, 4],
+ [107, 2],
+ [116, 5],
+ [122, 1],
+ [160, 10],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3503],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [95, 4],
+ [106, 4],
+ [107, 2],
+ [116, 5],
+ [122, 1],
+ [160, 9],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3504],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [95, 4],
+ [106, 9],
+ [107, 2],
+ [116, 5],
+ [122, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 2],
+ [116, 5],
+ [122, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3515],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 5],
+ [122, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3516, 3517, 3518],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 5],
+ [122, 1],
+ [141, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3519, 3520],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 5],
+ [122, 1],
+ [123, 1],
+ [141, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3521, 3522],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 4],
+ [122, 1],
+ [123, 1],
+ [141, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3523],
+ [
+ [25, 4],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 4],
+ [122, 1],
+ [123, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3524],
+ [
+ [25, 1],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 4],
+ [122, 1],
+ [123, 1],
+ [171, 4],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3525, 3526, 3527],
+ [
+ [25, 1],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 4],
+ [122, 1],
+ [123, 1],
+ [171, 3],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3528, 3529],
+ [
+ [25, 1],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 4],
+ [122, 1],
+ [123, 1],
+ [141, 9],
+ [171, 3],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3530, 3531],
+ [
+ [25, 1],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 4],
+ [122, 1],
+ [123, 1],
+ [141, 1],
+ [171, 3],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558],
+ [
+ [25, 1],
+ [60, 1],
+ [92, 1],
+ [93, 1],
+ [95, 4],
+ [106, 9],
+ [107, 1],
+ [116, 4],
+ [122, 1],
+ [123, 1],
+ [140, 1],
+ [141, 1],
+ [171, 3],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3559],
+ [
+ [12, 1],
+ [53, 1],
+ [56, 3],
+ [67, 1],
+ [75, 3],
+ [97, 1],
+ [114, 1],
+ [123, 1],
+ [129, 9],
+ [150, 4],
+ [154, 1],
+ [168, 1],
+ [173, 4],
+ [183, 1],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3560],
+ [
+ [12, 1],
+ [54, 1],
+ [57, 3],
+ [68, 1],
+ [76, 3],
+ [98, 1],
+ [115, 1],
+ [124, 1],
+ [130, 9],
+ [151, 4],
+ [155, 1],
+ [169, 1],
+ [174, 4],
+ [184, 1],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3561],
+ [
+ [12, 1],
+ [54, 1],
+ [57, 3],
+ [68, 1],
+ [76, 3],
+ [98, 1],
+ [115, 1],
+ [124, 1],
+ [130, 9],
+ [151, 4],
+ [155, 1],
+ [161, 1],
+ [162, 1],
+ [163, 1],
+ [164, 1],
+ [169, 1],
+ [174, 4],
+ [184, 1],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3562],
+ [
+ [5, 1],
+ [39, 1],
+ [40, 3],
+ [45, 1],
+ [53, 3],
+ [57, 1],
+ [71, 1],
+ [76, 1],
+ [82, 9],
+ [91, 4],
+ [93, 1],
+ [97, 1],
+ [98, 1],
+ [99, 1],
+ [100, 1],
+ [104, 1],
+ [108, 4],
+ [117, 1],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3563, 3564, 3565, 3566, 3567, 3568],
+ [
+ [5, 1],
+ [39, 1],
+ [40, 3],
+ [45, 1],
+ [53, 3],
+ [58, 1],
+ [72, 1],
+ [77, 1],
+ [82, 9],
+ [91, 4],
+ [93, 1],
+ [97, 1],
+ [98, 1],
+ [99, 1],
+ [100, 1],
+ [104, 1],
+ [108, 4],
+ [117, 1],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3569, 3570],
+ [
+ [5, 1],
+ [39, 1],
+ [40, 3],
+ [45, 1],
+ [53, 3],
+ [58, 1],
+ [59, 1],
+ [72, 1],
+ [77, 1],
+ [82, 9],
+ [91, 4],
+ [93, 1],
+ [97, 1],
+ [98, 1],
+ [99, 1],
+ [100, 1],
+ [104, 1],
+ [108, 4],
+ [117, 1],
+ ],
+ ),
+ ...inlineCacheProfiles(
+ [3571],
+ [
+ [5, 1],
+ [38, 1],
+ [39, 3],
+ [44, 1],
+ [52, 3],
+ [57, 1],
+ [58, 1],
+ [71, 1],
+ [76, 1],
+ [81, 9],
+ [91, 4],
+ [93, 1],
+ [97, 1],
+ [98, 1],
+ [99, 1],
+ [100, 1],
+ [104, 1],
+ [108, 4],
+ [117, 1],
+ ],
+ ),
+]);
+
+/** Preserve CPython's bytecode-size check and omit only cache code units. */
+export const pythonPycBytecodeIdentity = (
+ bytecode: Buffer,
+ magic: number,
+ rawSha256: string,
+): string => {
+ const caches = INLINE_CACHE_ENTRIES_BY_MAGIC.get(magic);
+ if (caches === undefined) return `${bytecode.length}:${rawSha256}`;
+
+ const digest = createHash("sha256");
+ let activeStart = 0;
+ for (let offset = 0; offset < bytecode.length;) {
+ const opcode = bytecode[offset] ?? 0;
+ const afterOpcode = offset + 2;
+ const cacheBytes = (caches.get(opcode) ?? 0) * 2;
+ offset = afterOpcode + cacheBytes;
+ if (cacheBytes > 0) {
+ digest.update(bytecode.subarray(activeStart, afterOpcode));
+ activeStart = offset;
+ }
+ }
+ if (activeStart < bytecode.length) {
+ digest.update(bytecode.subarray(activeStart));
+ }
+ return `${bytecode.length}:${digest.digest("hex")}`;
+};
diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts
index 2c245241e..1ab4ab1d5 100644
--- a/src/python/PythonPycParser.ts
+++ b/src/python/PythonPycParser.ts
@@ -16,6 +16,7 @@ import {
PythonPycFormatFailure,
PythonPycResourceFailure,
} from "./PythonPyc.errors.js";
+import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js";
const FLAG_REF = 0x80;
const TYPE_NULL = 0x30;
@@ -154,6 +155,7 @@ type CodeEquality = {
/** The co_consts tuple itself, which may be a reference to an unfinished tuple. */
readonly constsTuple: MarshalValue;
readonly effectiveFlags: number;
+ readonly bytecode: Buffer;
readonly name: Buffer;
readonly nameTables: readonly (readonly Buffer[])[];
readonly linetable: string;
@@ -234,6 +236,7 @@ export const parsePythonPyc = (
const reader = new MarshalReader(
marshal,
mapped.code_layout,
+ magic,
input,
mapped.negative_code_fields === "rejected",
);
@@ -328,6 +331,7 @@ class MarshalReader {
constructor(
readonly bytes: Buffer,
readonly layout: PythonPycCodeLayout,
+ readonly magic: number,
readonly input: InspectPythonPycInput,
readonly rejectNegativeCodeFields: boolean,
) {}
@@ -558,6 +562,7 @@ class MarshalReader {
consts,
constsTuple,
effectiveFlags,
+ bytecode,
name: codeName.raw,
nameTables:
localsplusnames !== null
@@ -1148,7 +1153,7 @@ class MarshalReader {
":",
);
case "str":
- return this.#stringKey(value.raw);
+ return this.#marshalStringKey(value.raw);
case "bytes": {
this.#reserveKey(value.value.length * 2 + 8);
const prefix = mode === "hash" ? "b" : "y";
@@ -1227,7 +1232,11 @@ class MarshalReader {
node.nlocals === null ? "" : String(node.nlocals),
String(material.effectiveFlags),
String(node.firstlineno),
- `${node.bytecode.bytes}:${node.bytecode.sha256}`,
+ pythonPycBytecodeIdentity(
+ material.bytecode,
+ this.magic,
+ node.bytecode.sha256,
+ ),
this.#constKeyItems(material.consts),
...material.nameTables.map((items) => this.#nameTableKey(items)),
// CPython started including line and exception tables in code equality
@@ -1307,10 +1316,6 @@ class MarshalReader {
}
/** Python string identity is the marshal bytes, not the JavaScript string. */
- #stringKey(raw: Buffer): string {
- return this.#marshalStringKey(raw);
- }
-
#marshalStringKey(raw: Buffer): string {
this.#reserveKey(this.#marshalStringKeyLength(raw.length));
return `s${raw.length}:${raw.toString("hex")}`;
diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts
index d2ab1c758..404cfdb56 100644
--- a/tests/acceptance/pythonPycCli.test.ts
+++ b/tests/acceptance/pythonPycCli.test.ts
@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
import { mkdir, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
@@ -16,6 +17,70 @@ const marshalUtf8 = (bytes: Buffer): Buffer => {
return Buffer.concat([Buffer.from([0x75]), length, bytes]);
};
+cliTest(
+ "uses magic-specific cache tables for synthetic pyc bytecode equality",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-python-pyc-inline-cache-");
+ const cases = [
+ { magic: 3495, opcode: 166, cacheUnits: 1 },
+ { magic: 3500, opcode: 25, cacheUnits: 4 },
+ { magic: 3524, opcode: 25, cacheUnits: 1 },
+ { magic: 3531, opcode: 106, cacheUnits: 9 },
+ { magic: 3550, opcode: 25, cacheUnits: 1 },
+ { magic: 3553, opcode: 25, cacheUnits: 1 },
+ { magic: 3564, opcode: 58, cacheUnits: 1 },
+ { magic: 3571, opcode: 81, cacheUnits: 9 },
+ ];
+ for (const { magic, opcode, cacheUnits } of cases) {
+ const cache = Buffer.alloc(cacheUnits * 2);
+ const bytecode = (arg: number, cachePayload: Buffer) =>
+ Buffer.concat([
+ Buffer.from([opcode, arg]),
+ cachePayload,
+ Buffer.from([83, 0]),
+ ]);
+ const changedCache = Buffer.from(cache);
+ changedCache[0] = 1;
+ const frozenSet = Buffer.concat([
+ Buffer.from([0x3e, 4, 0, 0, 0]),
+ ...[
+ bytecode(0, cache),
+ bytecode(0, changedCache),
+ bytecode(1, cache),
+ Buffer.concat([bytecode(0, cache), Buffer.from([0, 0])]),
+ ].map((code) => marshalCode("py311", { bytecode: code })),
+ ]);
+ const pyc = Buffer.concat([
+ pythonPycHeader(magic),
+ marshalCode("py311", { consts: [frozenSet] }),
+ ]);
+ const path = join(root, `inline-caches-${magic}.pyc`);
+ await writeFile(path, pyc);
+
+ const result = await cli.run({
+ arguments: ["inspect-python-pyc", path, "--json"],
+ });
+ expect(
+ result.exitCode,
+ `${result.stderr}\n${JSON.stringify(result.json)}`,
+ ).toBe(0);
+ const evidence = toolContract("inspect_python_pyc").outputSchema.parse(
+ result.json,
+ );
+ expect(evidence.normalized_result.code_objects[0]?.consts).toEqual([
+ expect.objectContaining({ kind: "frozenset", length: 3 }),
+ ]);
+ expect(evidence.normalized_result.code_objects[1]?.bytecode).toEqual({
+ bytes: bytecode(0, cache).length,
+ sha256: createHash("sha256").update(bytecode(0, cache)).digest("hex"),
+ });
+ expect(evidence.normalized_result.artifact.sha256).toBe(
+ createHash("sha256").update(pyc).digest("hex"),
+ );
+ }
+ },
+);
+
cliTest(
"keeps distinct Python code names distinct when JavaScript display strings collide",
async ({ cli }) => {