Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(python): normalize legacy code flags for equality

morlutoaee249b1ddb3
diff --git a/docs/python-pyc.md b/docs/python-pyc.md
index c2674a8cc..33942187f 100644
--- a/docs/python-pyc.md
+++ b/docs/python-pyc.md
@@ -50,7 +50,9 @@ names, locals (or `localsplusnames` on 3.11+), const
summaries, and bytecode length plus SHA-256.
Code-object equality used by sets and dictionaries follows the selected layout:
-3.7–3.10 ignore line tables, while 3.11+ include line and exception tables. The
+3.7–3.10 derive `CO_NOFREE` from the free/cell tables for equality, while
+3.11+ compare raw flags; raw `co_flags` remain in reported facts. The older
+layouts ignore line tables, while 3.11+ include line and exception tables. The
artifact SHA-256 identifies the exact selected `.pyc` bytes independently of
this decoded object graph. Reference slots follow each tag's CPython behavior:
singleton values and `TYPE_REF` do not reserve new slots even when their tag
diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts
index 6698323ac..f27a9d181 100644
--- a/src/python/PythonPycParser.ts
+++ b/src/python/PythonPycParser.ts
@@ -56,6 +56,8 @@ const TYPES_WITHOUT_REFERENCE_SLOTS = new Set([
/** CPython `CO_VARARGS` / `CO_VARKEYWORDS`. Each consumes one local name. */
const CO_VARARGS = 0x04;
const CO_VARKEYWORDS = 0x08;
+/** CPython 3.7–3.10 derives this bit from the free/cell variable tables. */
+const CO_NOFREE = 0x40;
/** CPython `_PyLocals_Kind` bit counted by `_PyCode_Validate` as a local. */
const CO_FAST_LOCAL = 0x20;
@@ -149,6 +151,7 @@ const containerChildren = (value: MarshalValue): readonly MarshalValue[] => {
*/
type CodeEquality = {
readonly consts: readonly MarshalValue[];
+ readonly effectiveFlags: number;
readonly linetable: string;
readonly exceptiontable: string;
};
@@ -520,8 +523,15 @@ class MarshalReader {
consts: [],
nested: [],
};
+ const effectiveFlags =
+ this.layout === "py311"
+ ? flags
+ : (freevars?.length ?? 0) === 0 && (cellvars?.length ?? 0) === 0
+ ? flags | CO_NOFREE
+ : flags & ~CO_NOFREE;
this.#codeEquality.set(node, {
consts,
+ effectiveFlags,
linetable: `${linetable.length}:${sha256(linetable)}`,
exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`,
});
@@ -1176,7 +1186,7 @@ class MarshalReader {
String(node.posonlyargcount ?? 0),
String(node.kwonlyargcount),
node.nlocals === null ? "" : String(node.nlocals),
- String(node.flags),
+ String(material.effectiveFlags),
String(node.firstlineno),
`${node.bytecode.bytes}:${node.bytecode.sha256}`,
this.#constKeyItems(material.consts),
diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts
index 352d1a9fb..ef55f2b98 100644
--- a/tests/acceptance/pythonPycCli.test.ts
+++ b/tests/acceptance/pythonPycCli.test.ts
@@ -298,6 +298,116 @@ cliTest(
},
);
+cliTest(
+ "uses constructor-normalized legacy flags for code keys and preserves raw flags",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory(
+ "rea-python-pyc-effective-flags-",
+ );
+ const cases = [
+ {
+ name: "py37-empty-tables",
+ magic: 3394,
+ layout: "py37" as const,
+ freevars: [],
+ cellvars: [],
+ lengths: [1, 1],
+ },
+ {
+ name: "py310-nonempty-tables",
+ magic: 3439,
+ layout: "py38" as const,
+ freevars: ["capture"],
+ cellvars: [],
+ lengths: [1, 1],
+ },
+ {
+ name: "py311-raw-flags",
+ magic: 3531,
+ layout: "py311" as const,
+ freevars: [],
+ cellvars: [],
+ lengths: [2, 2],
+ },
+ ];
+ for (const testCase of cases) {
+ const code = (flags: number) =>
+ marshalCode(testCase.layout, {
+ name: "f",
+ filename: "fixture.py",
+ firstlineno: 1,
+ flags,
+ freevars: testCase.freevars,
+ cellvars: testCase.cellvars,
+ });
+ const first = code(0);
+ const second = code(0x40);
+ const frozenSet = Buffer.concat([
+ Buffer.from([0x3e]),
+ Buffer.from([2, 0, 0, 0]),
+ first,
+ second,
+ ]);
+ const dict = Buffer.concat([
+ Buffer.from([0x7b]),
+ first,
+ Buffer.from([0x4e]),
+ second,
+ Buffer.from([0x54, 0x30]),
+ ]);
+ const path = join(root, `${testCase.name}.pyc`);
+ await writeFile(
+ path,
+ Buffer.concat([
+ pythonPycHeader(testCase.magic),
+ marshalCode(testCase.layout, { consts: [frozenSet, dict] }),
+ ]),
+ );
+ const result = await cli.run({
+ arguments: ["inspect-python-pyc", path, "--json"],
+ });
+ expect(result.exitCode, result.stderr).toBe(0);
+ const evidence = toolContract("inspect_python_pyc").outputSchema.parse(
+ result.json,
+ );
+ expect(
+ evidence.normalized_result.code_objects[0]?.consts.map((value) =>
+ value.kind === "frozenset" || value.kind === "dict"
+ ? value.length
+ : null,
+ ),
+ ).toEqual(testCase.lengths);
+ expect(
+ evidence.normalized_result.code_objects
+ .filter((object) => object.name === "f")
+ .map((object) => object.flags),
+ ).toContain(0);
+
+ const rawPath = join(root, `raw-${testCase.name}.pyc`);
+ await writeFile(
+ rawPath,
+ Buffer.concat([
+ pythonPycHeader(testCase.magic),
+ marshalCode(testCase.layout, {
+ name: "f",
+ flags: 0x40,
+ freevars: testCase.freevars,
+ cellvars: testCase.cellvars,
+ }),
+ ]),
+ );
+ const rawResult = await cli.run({
+ arguments: ["inspect-python-pyc", rawPath, "--json"],
+ });
+ expect(rawResult.exitCode, rawResult.stderr).toBe(0);
+ const rawEvidence = toolContract("inspect_python_pyc").outputSchema.parse(
+ rawResult.json,
+ );
+ expect(rawEvidence.normalized_result.code_objects[0]?.flags).toBe(0x40);
+ }
+ },
+);
+
cliTest(
"resolves a relative pyc path against the working directory",
async ({ cli }) => {