Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(python): reject code keys and set members with unhashable consts (#6424)

Wadecc99d896bea2
diff --git a/docs/python-pyc.md b/docs/python-pyc.md
index 33942187f..e422cd9c1 100644
--- a/docs/python-pyc.md
+++ b/docs/python-pyc.md
@@ -65,8 +65,8 @@ is that marshal length. Integer consts report `bits`; their decimal `value` is
`-inf`, and `nan` spellings. Dictionary and set lengths follow CPython:
unhashable keys and members fail the walk, equal set members count once, equal
dictionary keys keep the first key object and replace only its value, code
-objects are valid keys and members, and a null dictionary value ends the dict
-without that entry.
+objects are valid keys and members only when all of their consts are hashable,
+and a null dictionary value ends the dict without that entry.
## Limits and failures
diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts
index d2cf9d594..e7fe76707 100644
--- a/src/python/PythonPycParser.test.ts
+++ b/src/python/PythonPycParser.test.ts
@@ -933,6 +933,79 @@ it("accepts code objects as set members and dictionary keys", () => {
expect(pythonPycSchema.safeParse(report).success).toBe(true);
});
+it("rejects code keys and members whose consts are unhashable", () => {
+ const list = Buffer.concat([Buffer.from([0x5b]), u32(0)]);
+ const int = Buffer.concat([Buffer.from([0x69]), i32(1)]);
+ const tuple = (...items: Buffer[]): Buffer =>
+ Buffer.concat([Buffer.from([0x29, items.length]), ...items]);
+ const code = (...consts: Buffer[]): Buffer =>
+ marshalCode("py311", { name: "u", qualname: "u", consts });
+ const setOf = (kind: number, member: Buffer): Buffer =>
+ Buffer.concat([Buffer.from([kind]), u32(1), member]);
+ const dictKey = (key: Buffer): Buffer =>
+ Buffer.concat([Buffer.from([0x7b]), key, int, Buffer.from([0x30])]);
+ for (const key of [code(list), code(tuple(int, list)), code(code(list))]) {
+ expect(() => parse(withConsts([setOf(0x3c, key)]))).toThrow(
+ PythonPycFormatFailure,
+ );
+ expect(() => parse(withConsts([setOf(0x3e, key)]))).toThrow(
+ PythonPycFormatFailure,
+ );
+ expect(() => parse(withConsts([dictKey(key)]))).toThrow(
+ PythonPycFormatFailure,
+ );
+ }
+ const cyclic = Buffer.concat([
+ Buffer.from([0x29 | 0x80, 1]),
+ marshalCode("py311", {
+ name: "u",
+ qualname: "u",
+ flagged: true,
+ constsObject: tuple(marshalRef(0)),
+ }),
+ ]);
+ expect(() => parse(withConsts([cyclic, setOf(0x3c, marshalRef(1))]))).toThrow(
+ PythonPycFormatFailure,
+ );
+
+ // A set inside a tuple that reaches the unfinished tuple, directly, inside
+ // a code object's consts, or as those consts, before the list is read.
+ for (const member of [
+ marshalRef(0),
+ marshalCode("py311", {
+ name: "u",
+ qualname: "u",
+ constsObject: tuple(marshalRef(0)),
+ }),
+ marshalCode("py311", {
+ name: "u",
+ qualname: "u",
+ constsObject: marshalRef(0),
+ }),
+ ])
+ expect(() =>
+ parse(
+ withConsts([
+ Buffer.concat([
+ Buffer.from([0x29 | 0x80, 2]),
+ setOf(0x3c, member),
+ list,
+ ]),
+ ]),
+ ),
+ ).toThrow(PythonPycFormatFailure);
+
+ const unhashed = parse(withConsts([tuple(code(list))]));
+ expect(unhashed.coverage.status).toBe("complete");
+ expect(unhashed.code_objects[1]?.consts[0]).toEqual({
+ kind: "list",
+ length: 0,
+ });
+ const hashable = parse(withConsts([setOf(0x3c, code(tuple(int)))]));
+ expect(root(hashable).consts[0]).toEqual({ kind: "set", length: 1 });
+ expect(hashable.coverage.status).toBe("complete");
+});
+
it("uses the code equality fields for each CPython layout", () => {
const codeCollections = (
layout: "py38" | "py311",
diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts
index f27a9d181..c943885d2 100644
--- a/src/python/PythonPycParser.ts
+++ b/src/python/PythonPycParser.ts
@@ -151,6 +151,8 @@ const containerChildren = (value: MarshalValue): readonly MarshalValue[] => {
*/
type CodeEquality = {
readonly consts: readonly MarshalValue[];
+ /** The co_consts tuple itself, which may be a reference to an unfinished tuple. */
+ readonly constsTuple: MarshalValue;
readonly effectiveFlags: number;
readonly linetable: string;
readonly exceptiontable: string;
@@ -295,6 +297,10 @@ class MarshalReader {
readonly #codeKeys = new WeakMap<CodeNode, string>();
readonly #codeIdentities = new WeakMap<CodeNode, number>();
readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
+ /** Python hashability per code object. False while that check is in progress. */
+ readonly #codeHashable = new WeakMap<CodeNode, boolean>();
+ /** Tuples whose items are still being read; CPython cannot hash them yet. */
+ readonly #readingTuples = new WeakSet<MarshalValue>();
/** Code objects whose const summaries wait until the graph is fully decoded. */
readonly #codes: CodeNode[] = [];
/**
@@ -433,7 +439,8 @@ class MarshalReader {
// 3.7–3.10 constructors do not, so an odd `co_code` stays readable there.
if (this.layout === "py311" && bytecode.length % 2 !== 0)
malformed("co_code is malformed.");
- const consts = asTuple(this.readObject(depth + 1), "co_consts");
+ const constsTuple = this.readObject(depth + 1);
+ const consts = asTuple(constsTuple, "co_consts");
this.#chargeConsts(consts.length);
const names = this.#nameTable(
asTuple(this.readObject(depth + 1), "co_names"),
@@ -531,6 +538,7 @@ class MarshalReader {
: flags & ~CO_NOFREE;
this.#codeEquality.set(node, {
consts,
+ constsTuple,
effectiveFlags,
linetable: `${linetable.length}:${sha256(linetable)}`,
exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`,
@@ -834,12 +842,14 @@ class MarshalReader {
const value: MarshalValue =
kind === "tuple" ? { kind: "tuple", items } : { kind: "list", items };
this.#publish(slot, value);
+ if (kind === "tuple") this.#readingTuples.add(value);
for (let index = 0; index < count; index += 1) {
const item = this.readObject(depth + 1);
if (item.kind === "null")
malformed("NULL object in marshal data for sequence.");
items.push(item);
}
+ this.#readingTuples.delete(value);
return value;
}
@@ -953,6 +963,9 @@ class MarshalReader {
this.#takeKeyStep();
const frame = frames[frames.length - 1];
if (frame === undefined) break;
+ // A tuple still being read has missing items; CPython cannot hash it.
+ if (mode === "hash" && this.#readingTuples.has(frame.value))
+ frame.failed = true;
const items =
frame.value.kind === "tuple" || frame.value.kind === "frozenset"
? frame.value.items
@@ -1107,6 +1120,7 @@ class MarshalReader {
return `${prefix}${value.value.length}:${value.value.toString("hex")}`;
}
case "code":
+ if (mode === "hash" && !this.#isCodeHashable(value.value)) return null;
return this.#codeKey(value.value);
default:
return mode === "const" ? this.#identity(value) : null;
@@ -1207,6 +1221,23 @@ class MarshalReader {
return key;
}
+ /**
+ * CPython hashes the consts tuple itself, so one unhashable const makes the
+ * code unhashable. A const that leads back to the same code counts as
+ * unhashable instead of recursing.
+ */
+ #isCodeHashable(node: CodeNode): boolean {
+ const known = this.#codeHashable.get(node);
+ if (known !== undefined) return known;
+ this.#codeHashable.set(node, false);
+ const material =
+ this.#codeEquality.get(node) ??
+ malformed("Code object is missing equality material.");
+ const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
+ this.#codeHashable.set(node, hashable);
+ return hashable;
+ }
+
#codeIdentity(node: CodeNode): number {
const existing = this.#codeIdentities.get(node);
if (existing !== undefined) return existing;