Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(python): classify historical headers and signed flags
morluto81de82a1a4dd
diff --git a/docs/python-pyc.md b/docs/python-pyc.mdindex 4998b426b..c2674a8cc 100644--- a/docs/python-pyc.md+++ b/docs/python-pyc.md@@ -28,7 +28,8 @@ The inline Evidence result contains the artifact path, SHA-256, and byte length.- `magic`, the 16-bit little-endian field;- `python_version` and `code_layout`, mapped from the pinned CPython 3.7–3.13table (`py37`, `py38`, or `py311`); 3.11 alpha magics before 3494 used- other code layouts and are unsupported;+ other code layouts and are unsupported. The 3.7.0a1/a2 magics 3390/3391+ used 12-byte headers and are outside this reader's supported format;- `flags`, `hash_based`, and `check_source` from PEP 552;- `source_hash_hex` for hash-based pycs, or `source_mtime` and `source_size`for timestamp pycs.@@ -71,7 +72,9 @@ The selected file is limited to 16 MiB by default (64 MiB maximum). Marshalobject count, nesting depth, and per-string size are bounded; exceeding abudget fails without partial success. Argument counts, positional-only counts,keyword-only counts, local counts, and stack size are signed CPython fields; a-negative encoding is malformed. On the 3.11+ layout, `co_code` must cover a+negative encoding is malformed. Code `flags` preserve signed `co_flags` values;+CPython 3.8+ rejects negative flags, while 3.7 accepts them. On the 3.11++layout, `co_code` must cover awhole number of code units. Every layout rejects argument counts that do notfit the local table CPython checks (`co_varnames`, including `*args` /`**kwargs`, or fast locals on 3.11+). Those files are malformed, so thediff --git a/src/cli/pythonPycCommands.ts b/src/cli/pythonPycCommands.tsindex cf79aaee4..f408eee3a 100644--- a/src/cli/pythonPycCommands.ts+++ b/src/cli/pythonPycCommands.ts@@ -16,7 +16,7 @@ export const registerPythonPycCommands = (): void => {cli.command(CLI_COMMANDS.inspectPythonPyc, {description:- "Inspect a CPython 3.7–3.13 pyc header and marshal code objects without executing bytecode",+ "Inspect a supported CPython 3.7–3.13 pyc header and marshal code objects without executing bytecode",args: z.object({path: z.string().describe("Local CPython .pyc file"),}),diff --git a/src/contracts/python/pythonPycToolContracts.ts b/src/contracts/python/pythonPycToolContracts.tsindex 7db11087d..fe312eee0 100644--- a/src/contracts/python/pythonPycToolContracts.ts+++ b/src/contracts/python/pythonPycToolContracts.ts@@ -13,7 +13,7 @@ export const PYTHON_PYC_TOOL_CONTRACTS = [...toolContractMetadata("inspect_python_pyc"),kind: "artifact-provider",description:- "Inspect one explicit local CPython 3.7–3.13 .pyc file without executing bytecode, invoking CPython or recovering source. Returns the 16-byte header (magic, pinned python_version, PEP 552 flags, source hash or mtime/size), the top-level code object with nested functions, names, const summaries, and bytecode length/SHA-256. Marshal is walked with object, depth and string budgets; unknown magics, truncated streams and exhausted budgets return no partial success. A decoded header is not proof the matching .py is present. Portable on supported Node hosts; no Python installation required.",+ "Inspect one explicit local CPython 3.7–3.13 .pyc file with a supported 16-byte header without executing bytecode, invoking CPython or recovering source. Returns the header (magic, pinned python_version, PEP 552 flags, source hash or mtime/size), the top-level code object with nested functions, names, const summaries, and bytecode length/SHA-256. Marshal is walked with object, depth and string budgets; unsupported magics/layouts, truncated streams and exhausted budgets return no partial success. A decoded header is not proof the matching .py is present. Portable on supported Node hosts; no Python installation required.",inputSchema: inspectPythonPycInputSchema,outputSchema: evidenceResultOf(pythonPycSchema),examples: [diff --git a/src/domain/python/pythonPyc.test.ts b/src/domain/python/pythonPyc.test.tsindex ea98b5d25..1c7bc34c2 100644--- a/src/domain/python/pythonPyc.test.ts+++ b/src/domain/python/pythonPyc.test.ts@@ -9,18 +9,30 @@ it("maps pinned CPython 3.7–3.13 magics and leaves gaps unknown", () => {expect(pythonPycMagicEntry(3394)).toEqual({python_version: "3.7",code_layout: "py37",+ header_bytes: 16,+ });+ expect(pythonPycMagicEntry(3390)).toMatchObject({+ python_version: "3.7",+ header_bytes: 12,+ });+ expect(pythonPycMagicEntry(3391)).toMatchObject({+ python_version: "3.7",+ header_bytes: 12,});expect(pythonPycMagicEntry(3413)).toEqual({python_version: "3.8",code_layout: "py38",+ header_bytes: 16,});expect(pythonPycMagicEntry(3531)).toEqual({python_version: "3.12",code_layout: "py311",+ header_bytes: 16,});expect(pythonPycMagicEntry(3571)).toEqual({python_version: "3.13",code_layout: "py311",+ header_bytes: 16,});expect(pythonPycMagicEntry(3450)).toBeNull();expect(pythonPycMagicEntry(62171)).toBeNull();diff --git a/src/domain/python/pythonPyc.ts b/src/domain/python/pythonPyc.tsindex eed477c99..d55167cfe 100644--- a/src/domain/python/pythonPyc.ts+++ b/src/domain/python/pythonPyc.ts@@ -9,8 +9,8 @@ export const PYTHON_PYC_CHECK_SOURCE = 0x2;const uint = z.number().int().min(0).max(0xffffffff);const index = z.number().int().nonnegative();const signedInt32 = z.number().int().min(-0x80000000).max(0x7fffffff);-/** Non-negative CPython `int` counts. `0xffffffff` is -1 and is not evidence. */-const signedCount = z.number().int().min(0).max(0x7fffffff);+/** Non-negative CPython `int` counts. */+const nonNegativeInt32 = z.number().int().min(0).max(0x7fffffff);/** Marshal TYPE_CODE field order for CPython 3.7, 3.8–3.10, and 3.11+. */export const PYTHON_PYC_CODE_LAYOUTS = ["py37", "py38", "py311"] as const;@@ -19,14 +19,16 @@ export type PythonPycCodeLayout = (typeof PYTHON_PYC_CODE_LAYOUTS)[number];type MagicEntry = {readonly python_version: string;readonly code_layout: PythonPycCodeLayout;+ readonly header_bytes: 12 | 16;};const magics = (values: readonly number[],python_version: string,code_layout: PythonPycCodeLayout,+ header_bytes: 12 | 16 = PYTHON_PYC_HEADER_BYTES,): ReadonlyArray<readonly [number, MagicEntry]> =>- values.map((magic) => [magic, { python_version, code_layout }]);+ values.map((magic) => [magic, { python_version, code_layout, header_bytes }]);const consecutive = (start: number,@@ -46,7 +48,8 @@ const consecutive = (* bytecode opcode tables.*/export const PYTHON_PYC_MAGIC_TABLE: ReadonlyMap<number, MagicEntry> = new Map([- ...magics([3390, 3391, 3392, 3393, 3394], "3.7", "py37"),+ ...magics([3390, 3391], "3.7", "py37", 12),+ ...magics([3392, 3393, 3394], "3.7", "py37"),...magics([3400, 3401], "3.8", "py37"),...magics([3410, 3411, 3412, 3413], "3.8", "py38"),...magics([3420, 3421, 3422, 3423, 3424, 3425], "3.9", "py38"),@@ -159,12 +162,12 @@ const pythonCodeFields = {qualname: z.string().nullable(),filename: z.string(),firstlineno: signedInt32,- argcount: signedCount,- posonlyargcount: signedCount.nullable(),- kwonlyargcount: signedCount,- nlocals: signedCount.nullable(),- stacksize: signedCount,- flags: uint,+ argcount: nonNegativeInt32,+ posonlyargcount: nonNegativeInt32.nullable(),+ kwonlyargcount: nonNegativeInt32,+ nlocals: nonNegativeInt32.nullable(),+ stacksize: nonNegativeInt32,+ flags: signedInt32,names: z.array(z.string()),varnames: z.array(z.string()).nullable(),freevars: z.array(z.string()).nullable(),diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.tsindex d3bc5ad83..d2cf9d594 100644--- a/src/python/PythonPycParser.test.ts+++ b/src/python/PythonPycParser.test.ts@@ -139,6 +139,21 @@ it("fails unknown magics and truncated marshal without partial success", () => {);});+it("classifies the 12-byte 3.7 alpha header as unsupported", () => {+ for (const magic of [3390, 3391]) {+ const header = Buffer.alloc(12);+ header.writeUInt16LE(magic);+ header[2] = 0x0d;+ header[3] = 0x0a;+ expect(() => parse(Buffer.concat([header, marshalCode("py37")]))).toThrow(+ expect.objectContaining({+ kind: "unsupported",+ message: expect.stringContaining("unsupported 12-byte CPython header"),+ }),+ );+ }+});+const u32 = (value: number): Buffer => {const bytes = Buffer.alloc(4);bytes.writeUInt32LE(value);@@ -1405,6 +1420,12 @@ it("rejects code CPython treats as corrupt instead of complete coverage", () =>expect(() => parse(file("py37", 3394, { argcount: 1 }))).toThrow(/co_varnames is too small/,);+ expect(() => parse(file("py311", 3531, { flags: 0x80000000 }))).toThrow(+ /co_flags is negative/,+ );+ expect(() => parse(file("py37", 3401, { flags: 0x80000000 }))).toThrow(+ /co_flags is negative/,+ );const valid = parse(file("py311", 3531, {@@ -1418,6 +1439,14 @@ it("rejects code CPython treats as corrupt instead of complete coverage", () =>);expect(root(valid).argcount).toBe(1);expect(valid.coverage.status).toBe("complete");+ const unknownPositiveFlags = parse(+ file("py311", 3531, { flags: 0x40000000 }),+ );+ expect(root(unknownPositiveFlags).flags).toBe(0x40000000);+ expect(unknownPositiveFlags.coverage.status).toBe("complete");+ const negativePy37Flags = parse(file("py37", 3394, { flags: 0x80000000 }));+ expect(root(negativePy37Flags).flags).toBe(-0x80000000);+ expect(pythonPycSchema.safeParse(negativePy37Flags).success).toBe(true);const py37 = parse(file("py37", 3394, {argcount: 1,diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.tsindex 83d53ac8a..6698323ac 100644--- a/src/python/PythonPycParser.ts+++ b/src/python/PythonPycParser.ts@@ -188,14 +188,19 @@ export const parsePythonPyc = (): PythonPyc => {if (bytes.length !== artifact.bytes)malformed("Selected artifact length disagrees with the retained bytes.");- if (bytes.length < PYTHON_PYC_HEADER_BYTES)- malformed(- "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.",- );+ if (bytes.length < 4) malformed("Selected file is shorter than a pyc magic.");if (bytes[2] !== 0x0d || bytes[3] !== 0x0a)malformed("Pyc magic is not a little-endian integer followed by CR LF.");const magic = bytes.readUInt16LE(0);const mapped = pythonPycMagicEntry(magic);+ if (mapped !== null && mapped.header_bytes !== PYTHON_PYC_HEADER_BYTES)+ return unsupported(+ `Pyc magic ${magic} uses an unsupported ${mapped.header_bytes}-byte CPython header.`,+ );+ if (bytes.length < PYTHON_PYC_HEADER_BYTES)+ malformed(+ "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.",+ );if (mapped === null)return unsupported(`Pyc magic ${magic} is outside the pinned CPython 3.7–3.13 table.`,@@ -214,7 +219,12 @@ export const parsePythonPyc = (source_size: hashBased ? null : bytes.readUInt32LE(12),};const marshal = bytes.subarray(PYTHON_PYC_HEADER_BYTES);- const reader = new MarshalReader(marshal, mapped.code_layout, input);+ const reader = new MarshalReader(+ marshal,+ mapped.code_layout,+ input,+ mapped.python_version !== "3.7",+ );const root = reader.readObject(0);if (reader.offset !== marshal.length)malformed("Marshal stream has trailing bytes after the top-level object.");@@ -297,6 +307,7 @@ class MarshalReader {readonly bytes: Buffer,readonly layout: PythonPycCodeLayout,readonly input: InspectPythonPycInput,+ readonly rejectNegativeCodeFlags: boolean,) {}readObject(depth: number): MarshalValue {@@ -400,16 +411,20 @@ class MarshalReader {}code(depth: number): CodeNode {- const argcount = this.signedCount("co_argcount");+ const argcount = this.nonNegativeInt32("co_argcount");const posonlyargcount =- this.layout === "py37" ? null : this.signedCount("co_posonlyargcount");+ this.layout === "py37"+ ? null+ : this.nonNegativeInt32("co_posonlyargcount");if (posonlyargcount !== null && argcount < posonlyargcount)malformed("co_argcount is smaller than co_posonlyargcount.");- const kwonlyargcount = this.signedCount("co_kwonlyargcount");+ const kwonlyargcount = this.nonNegativeInt32("co_kwonlyargcount");const nlocals =- this.layout === "py311" ? null : this.signedCount("co_nlocals");- const stacksize = this.signedCount("co_stacksize");- const flags = this.u32();+ this.layout === "py311" ? null : this.nonNegativeInt32("co_nlocals");+ const stacksize = this.nonNegativeInt32("co_stacksize");+ const flags = this.i32();+ if (this.rejectNegativeCodeFlags && flags < 0)+ malformed("co_flags is negative.");const bytecode = asBytes(this.readObject(depth + 1), "co_code");// 3.11+ `_PyCode_Validate` rejects a length that is not a whole code unit.// 3.7–3.10 constructors do not, so an odd `co_code` stays readable there.@@ -1406,8 +1421,8 @@ class MarshalReader {return value;}- /** CPython `r_long` count. Negative values are rejected before Evidence. */- signedCount(field: string): number {+ /** CPython `r_long` int fields must not be negative in a code object. */+ nonNegativeInt32(field: string): number {const value = this.i32();if (value < 0) malformed(`${field} is negative.`);return value;diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.tsindex 98ff65855..352d1a9fb 100644--- a/tests/acceptance/pythonPycCli.test.ts+++ b/tests/acceptance/pythonPycCli.test.ts@@ -28,6 +28,137 @@ cliTest(},);+cliTest(+ "classifies unsupported 3.7 alpha headers without labeling their marshal data malformed",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-python-pyc-alpha-header-");+ const marshal = Buffer.from(+ "e3000000000000000000000000010000004000000073040000006400530029014ea900720100000072010000007201000000fa066d6f642e7079da083c6d6f64756c653e010000007300000000",+ "hex",+ );+ expect(marshal).toHaveLength(77);+ for (const magic of [3390, 3391]) {+ const path = join(root, `${magic}.pyc`);+ const header = Buffer.alloc(12);+ header.writeUInt16LE(magic);+ header[2] = 0x0d;+ header[3] = 0x0a;+ await writeFile(path, Buffer.concat([header, marshal]));+ const result = await cli.run({+ arguments: ["inspect-python-pyc", path, "--json"],+ });+ expect(result.exitCode).toBe(1);+ expect(result.json).toMatchObject({+ code: "unsupported_target",+ category: "unsupported_target",+ });+ expect(JSON.stringify(result.json)).toContain(+ "unsupported 12-byte CPython header",+ );+ expect(result.json).not.toHaveProperty("normalized_result");+ }++ for (const magic of [3392, 3394]) {+ const path = join(root, `${magic}.pyc`);+ await writeFile(path, Buffer.concat([pythonPycHeader(magic), marshal]));+ const result = await cli.run({+ arguments: ["inspect-python-pyc", path, "--json"],+ });+ expect(result.exitCode, result.stderr).toBe(0);+ const evidence = toolContract("inspect_python_pyc").outputSchema.parse(+ result.json,+ );+ expect(evidence.normalized_result.header.magic).toBe(magic);+ expect(evidence.normalized_result.code_objects[0]?.bytecode.bytes).toBe(+ 4,+ );+ }+ },+);++cliTest(+ "rejects constructor-invalid signed code flags and preserves unknown positive flags",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-python-pyc-flags-");+ for (const { name, magic, layout, flags, expected, valid } of [+ {+ name: "zero",+ magic: 3531,+ layout: "py311",+ flags: 0,+ expected: 0,+ valid: true,+ },+ {+ name: "unknown-positive",+ magic: 3531,+ layout: "py311",+ flags: 0x40000000,+ expected: 0x40000000,+ valid: true,+ },+ {+ name: "negative-py311",+ magic: 3531,+ layout: "py311",+ flags: 0x80000000,+ expected: -0x80000000,+ valid: false,+ },+ {+ name: "negative-plus-one-py311",+ magic: 3531,+ layout: "py311",+ flags: 0x80000001,+ expected: -0x7fffffff,+ valid: false,+ },+ {+ name: "negative-py37",+ magic: 3394,+ layout: "py37",+ flags: 0x80000000,+ expected: -0x80000000,+ valid: true,+ },+ ] as const) {+ const path = join(root, `${name}.pyc`);+ await writeFile(+ path,+ Buffer.concat([pythonPycHeader(magic), marshalCode(layout, { flags })]),+ );+ const result = await cli.run({+ arguments: ["inspect-python-pyc", path, "--json"],+ });+ if (valid) {+ expect(result.exitCode, result.stderr).toBe(0);+ const evidence = toolContract("inspect_python_pyc").outputSchema.parse(+ result.json,+ );+ expect(evidence.normalized_result.code_objects[0]?.flags).toBe(+ expected,+ );+ } else {+ expect(result.exitCode).toBe(1);+ expect(result.json).toMatchObject({+ code: "invalid_request",+ category: "invalid_input",+ details: {+ operation: "inspect_python_pyc",+ issues: [+ {+ reason: "invalid_format",+ message: expect.stringContaining("co_flags is negative"),+ },+ ],+ },+ });+ expect(result.json).not.toHaveProperty("normalized_result");+ }+ }+ },+);+cliTest("preserves negative CPython first line numbers through the public CLI",async ({ cli }) => {diff --git a/tests/boundary/mcp/pythonPycMcp.test.ts b/tests/boundary/mcp/pythonPycMcp.test.tsindex c8a70cb31..4f41dae43 100644--- a/tests/boundary/mcp/pythonPycMcp.test.ts+++ b/tests/boundary/mcp/pythonPycMcp.test.ts@@ -100,6 +100,67 @@ mcpTest("requires an absolute path for inspect_python_pyc", async ({ mcp }) => {}});+mcpTest(+ "advertised pyc output schema accepts signed legacy flags and positive unknown flags",+ async ({ mcp }) => {+ const { client, session } = await connect(mcp);+ try {+ const advertised = (await client.listTools()).tools.find(+ (tool) => tool.name === "inspect_python_pyc",+ );+ if (advertised?.outputSchema === undefined)+ throw new Error("Missing inspect_python_pyc output schema");+ const outputSchema: Record<string, unknown> = advertised.outputSchema;+ const ajv = new Ajv2020({ strict: false, validateFormats: false });+ const root = await createTestTempDirectory("rea-python-pyc-mcp-flags-");+ for (const testCase of [+ {+ name: "legacy-negative",+ magic: 3394,+ layout: "py37" as const,+ rawFlags: 0x80000000,+ expectedFlags: -0x80000000,+ },+ {+ name: "unknown-positive",+ magic: 3531,+ layout: "py311" as const,+ rawFlags: 0x40000000,+ expectedFlags: 0x40000000,+ },+ ]) {+ const path = join(root, `${testCase.name}.pyc`);+ await writeFile(+ path,+ Buffer.concat([+ pythonPycHeader(testCase.magic),+ marshalCode(testCase.layout, { flags: testCase.rawFlags }),+ ]),+ );+ const response = await client.callTool({+ name: "inspect_python_pyc",+ arguments: { path },+ });+ expect(response.isError, JSON.stringify(response.content)).not.toBe(+ true,+ );+ const evidence = toolContract("inspect_python_pyc").outputSchema.parse(+ response.structuredContent,+ );+ expect(+ ajv.validate(outputSchema, response.structuredContent),+ JSON.stringify(ajv.errors),+ ).toBe(true);+ expect(evidence.normalized_result.code_objects[0]?.flags).toBe(+ testCase.expectedFlags,+ );+ }+ } finally {+ await session.close();+ }+ },+);+mcpTest("reports an unknown pyc magic as unsupported", async ({ mcp }) => {const { client, session } = await connect(mcp);try {