Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(python): reject code keys and set members with unhashable consts (#6424)
Wadecc99d896bea2
diff --git a/docs/python-pyc.md b/docs/python-pyc.mdindex 33942187f..e422cd9c1 100644--- a/docs/python-pyc.md+++ b/docs/python-pyc.md@@ -65,8 +65,8 @@ is that marshal length. Integer consts report `bits`; their decimal `value` is`-inf`, and `nan` spellings. Dictionary and set lengths follow CPython:unhashable keys and members fail the walk, equal set members count once, equaldictionary keys keep the first key object and replace only its value, code-objects are valid keys and members, and a null dictionary value ends the dict-without that entry.+objects are valid keys and members only when all of their consts are hashable,+and a null dictionary value ends the dict without that entry.## Limits and failuresdiff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.tsindex d2cf9d594..e7fe76707 100644--- a/src/python/PythonPycParser.test.ts+++ b/src/python/PythonPycParser.test.ts@@ -933,6 +933,79 @@ it("accepts code objects as set members and dictionary keys", () => {expect(pythonPycSchema.safeParse(report).success).toBe(true);});+it("rejects code keys and members whose consts are unhashable", () => {+ const list = Buffer.concat([Buffer.from([0x5b]), u32(0)]);+ const int = Buffer.concat([Buffer.from([0x69]), i32(1)]);+ const tuple = (...items: Buffer[]): Buffer =>+ Buffer.concat([Buffer.from([0x29, items.length]), ...items]);+ const code = (...consts: Buffer[]): Buffer =>+ marshalCode("py311", { name: "u", qualname: "u", consts });+ const setOf = (kind: number, member: Buffer): Buffer =>+ Buffer.concat([Buffer.from([kind]), u32(1), member]);+ const dictKey = (key: Buffer): Buffer =>+ Buffer.concat([Buffer.from([0x7b]), key, int, Buffer.from([0x30])]);+ for (const key of [code(list), code(tuple(int, list)), code(code(list))]) {+ expect(() => parse(withConsts([setOf(0x3c, key)]))).toThrow(+ PythonPycFormatFailure,+ );+ expect(() => parse(withConsts([setOf(0x3e, key)]))).toThrow(+ PythonPycFormatFailure,+ );+ expect(() => parse(withConsts([dictKey(key)]))).toThrow(+ PythonPycFormatFailure,+ );+ }+ const cyclic = Buffer.concat([+ Buffer.from([0x29 | 0x80, 1]),+ marshalCode("py311", {+ name: "u",+ qualname: "u",+ flagged: true,+ constsObject: tuple(marshalRef(0)),+ }),+ ]);+ expect(() => parse(withConsts([cyclic, setOf(0x3c, marshalRef(1))]))).toThrow(+ PythonPycFormatFailure,+ );++ // A set inside a tuple that reaches the unfinished tuple, directly, inside+ // a code object's consts, or as those consts, before the list is read.+ for (const member of [+ marshalRef(0),+ marshalCode("py311", {+ name: "u",+ qualname: "u",+ constsObject: tuple(marshalRef(0)),+ }),+ marshalCode("py311", {+ name: "u",+ qualname: "u",+ constsObject: marshalRef(0),+ }),+ ])+ expect(() =>+ parse(+ withConsts([+ Buffer.concat([+ Buffer.from([0x29 | 0x80, 2]),+ setOf(0x3c, member),+ list,+ ]),+ ]),+ ),+ ).toThrow(PythonPycFormatFailure);++ const unhashed = parse(withConsts([tuple(code(list))]));+ expect(unhashed.coverage.status).toBe("complete");+ expect(unhashed.code_objects[1]?.consts[0]).toEqual({+ kind: "list",+ length: 0,+ });+ const hashable = parse(withConsts([setOf(0x3c, code(tuple(int)))]));+ expect(root(hashable).consts[0]).toEqual({ kind: "set", length: 1 });+ expect(hashable.coverage.status).toBe("complete");+});+it("uses the code equality fields for each CPython layout", () => {const codeCollections = (layout: "py38" | "py311",diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.tsindex f27a9d181..c943885d2 100644--- a/src/python/PythonPycParser.ts+++ b/src/python/PythonPycParser.ts@@ -151,6 +151,8 @@ const containerChildren = (value: MarshalValue): readonly MarshalValue[] => {*/type CodeEquality = {readonly consts: readonly MarshalValue[];+ /** The co_consts tuple itself, which may be a reference to an unfinished tuple. */+ readonly constsTuple: MarshalValue;readonly effectiveFlags: number;readonly linetable: string;readonly exceptiontable: string;@@ -295,6 +297,10 @@ class MarshalReader {readonly #codeKeys = new WeakMap<CodeNode, string>();readonly #codeIdentities = new WeakMap<CodeNode, number>();readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();+ /** Python hashability per code object. False while that check is in progress. */+ readonly #codeHashable = new WeakMap<CodeNode, boolean>();+ /** Tuples whose items are still being read; CPython cannot hash them yet. */+ readonly #readingTuples = new WeakSet<MarshalValue>();/** Code objects whose const summaries wait until the graph is fully decoded. */readonly #codes: CodeNode[] = [];/**@@ -433,7 +439,8 @@ class MarshalReader {// 3.7–3.10 constructors do not, so an odd `co_code` stays readable there.if (this.layout === "py311" && bytecode.length % 2 !== 0)malformed("co_code is malformed.");- const consts = asTuple(this.readObject(depth + 1), "co_consts");+ const constsTuple = this.readObject(depth + 1);+ const consts = asTuple(constsTuple, "co_consts");this.#chargeConsts(consts.length);const names = this.#nameTable(asTuple(this.readObject(depth + 1), "co_names"),@@ -531,6 +538,7 @@ class MarshalReader {: flags & ~CO_NOFREE;this.#codeEquality.set(node, {consts,+ constsTuple,effectiveFlags,linetable: `${linetable.length}:${sha256(linetable)}`,exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`,@@ -834,12 +842,14 @@ class MarshalReader {const value: MarshalValue =kind === "tuple" ? { kind: "tuple", items } : { kind: "list", items };this.#publish(slot, value);+ if (kind === "tuple") this.#readingTuples.add(value);for (let index = 0; index < count; index += 1) {const item = this.readObject(depth + 1);if (item.kind === "null")malformed("NULL object in marshal data for sequence.");items.push(item);}+ this.#readingTuples.delete(value);return value;}@@ -953,6 +963,9 @@ class MarshalReader {this.#takeKeyStep();const frame = frames[frames.length - 1];if (frame === undefined) break;+ // A tuple still being read has missing items; CPython cannot hash it.+ if (mode === "hash" && this.#readingTuples.has(frame.value))+ frame.failed = true;const items =frame.value.kind === "tuple" || frame.value.kind === "frozenset"? frame.value.items@@ -1107,6 +1120,7 @@ class MarshalReader {return `${prefix}${value.value.length}:${value.value.toString("hex")}`;}case "code":+ if (mode === "hash" && !this.#isCodeHashable(value.value)) return null;return this.#codeKey(value.value);default:return mode === "const" ? this.#identity(value) : null;@@ -1207,6 +1221,23 @@ class MarshalReader {return key;}+ /**+ * CPython hashes the consts tuple itself, so one unhashable const makes the+ * code unhashable. A const that leads back to the same code counts as+ * unhashable instead of recursing.+ */+ #isCodeHashable(node: CodeNode): boolean {+ const known = this.#codeHashable.get(node);+ if (known !== undefined) return known;+ this.#codeHashable.set(node, false);+ const material =+ this.#codeEquality.get(node) ??+ malformed("Code object is missing equality material.");+ const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;+ this.#codeHashable.set(node, hashable);+ return hashable;+ }+#codeIdentity(node: CodeNode): number {const existing = this.#codeIdentities.get(node);if (existing !== undefined) return existing;