Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(plist): preserve complete XML dictionary key identities

morluto78cf6100f33a
diff --git a/docs/native-investigation.md b/docs/native-investigation.md
index 766182713..b9b010ed8 100644
--- a/docs/native-investigation.md
+++ b/docs/native-investigation.md
@@ -331,8 +331,9 @@ complete observation.
64 MiB of input, 200,000 objects/references and 128 nesting levels.
XML dictionary entries decoded as `__proto__` are omitted with an explicit
count because REA results cannot represent that key. XML tag whitespace and
- comments do not change that handling. Other values and the original archive
- digest are preserved; key interpretation follows the pinned plist decoder.
+ comments do not change that handling. XML key names combine their text and
+ CDATA segments without trimming or expanding literal CDATA entities. Other
+ values and the original archive digest are preserved.
- `decode_interface_builder` / `rea decode-interface-builder <app>` reads
compiled storyboard and nib resources into objects, outlets, actions,
connections and controller/class names. A full app graph is often several
diff --git a/src/domain/propertyListKeys.test.ts b/src/domain/propertyListKeys.test.ts
index ff3ca7bf3..259204a22 100644
--- a/src/domain/propertyListKeys.test.ts
+++ b/src/domain/propertyListKeys.test.ts
@@ -53,9 +53,9 @@ it("keeps __proto__ text in strings, comments, and similar keys", () => {
});
});
-it("keeps an entity-encoded key that decodes to the placeholder", () => {
+it("keeps a segmented entity-encoded key that decodes to the placeholder", () => {
const text = plist(
- "<dict><key>&#95;&#95;rea_prototype_key__</key><string>kept</string><key>__proto__</key><string>x</string></dict>",
+ "<dict><key>&#95;&#95;rea<![CDATA[_prototype_key__]]></key><string>kept</string><key>__proto__</key><string>x</string></dict>",
);
expect(parseXmlPropertyList(text)).toEqual({
value: { __rea_prototype_key__: "kept" },
@@ -83,16 +83,75 @@ it("omits own __proto__ entries that JSON decoding creates", () => {
});
});
-it("preserves the pinned decoder's first-child key semantics", () => {
+it("omits complete prototype keys split across character-data nodes", () => {
const text = plist(
"<dict><key>__pro<![CDATA[to__]]></key><string>kept</string><key>__proto__</key><string>omitted</string></dict>",
);
expect(parseXmlPropertyList(text)).toEqual({
- value: { __pro: "kept" },
+ value: {},
omittedPrototypeKeys: 1,
});
});
+it("preserves literal key text, XML characters and encoded line endings across segments", () => {
+ const text = plist(
+ '<dict><key note="&gt;">pre<![CDATA[&amp;</key>]]><!-- ignored -->&#13;&#x85;&#x2028;]]&gt;post</key><string>a&#13;b</string></dict>',
+ );
+ expect(parseXmlPropertyList(text)).toEqual({
+ value: { "pre&amp;</key>\r\u0085\u2028]]>post": "a\rb" },
+ omittedPrototypeKeys: 0,
+ });
+});
+
+it("does not infer key identity from nested elements", () => {
+ expect(
+ parseXmlPropertyList(
+ plist(
+ "<dict><key>pre<string>nested</string></key><string>kept</string><key>__proto__<string>nested</string></key><string>omitted</string></dict>",
+ ),
+ ),
+ ).toEqual({ value: { pre: "kept" }, omittedPrototypeKeys: 1 });
+});
+
+it.each([
+ ["ab", "a<![CDATA[b]]>", "ab"],
+ ["a<![CDATA[b]]>", "ab", "a<![CDATA[b]]>"],
+ ["ab", "a<!-- comment -->b", "ab"],
+])("retains last-value-wins for mixed spellings %j", (...keys) => {
+ const entries = keys
+ .map(
+ (key, index) =>
+ `<key>${key}</key><integer>${String(index + 1)}</integer>`,
+ )
+ .join("");
+ expect(parseXmlPropertyList(plist(`<dict>${entries}</dict>`))).toEqual({
+ value: { ab: 3 },
+ omittedPrototypeKeys: 0,
+ });
+});
+
+it("keeps empty keys and alias-looking source keys distinct", () => {
+ const entries =
+ "<key/><integer>1</integer><key><!-- ignored --></key><integer>2</integer>" +
+ "<key></key><integer>3</integer><key>__rea_plist_key_0__</key><string>literal</string>" +
+ "<key>ma<?ignored value?><![CDATA[in]]></key><integer>42</integer>";
+ expect(parseXmlPropertyList(plist(`<dict>${entries}</dict>`))).toEqual({
+ value: { "": 3, __rea_plist_key_0__: "literal", main: 42 },
+ omittedPrototypeKeys: 0,
+ });
+});
+
+it.each([
+ "<key>ma<![CDATA[in]]></key>",
+ "<array><key>ma<![CDATA[in]]></key><string>__rea_plist_key_0__</string></array>",
+])("does not alias key elements used as scalar values in %s", (body) => {
+ const text = plist(body);
+ expect(parseXmlPropertyList(text)).toEqual({
+ value: parse(text),
+ omittedPrototypeKeys: 0,
+ });
+});
+
it("preserves OpenStep dictionary and array autodetection", () => {
expect(
parseXmlPropertyList(
diff --git a/src/domain/propertyListKeys.ts b/src/domain/propertyListKeys.ts
index 7ed9157f8..12e3e7dba 100644
--- a/src/domain/propertyListKeys.ts
+++ b/src/domain/propertyListKeys.ts
@@ -19,9 +19,9 @@ export const omittedPrototypeKeysLimitation = (count: number | null): string =>
? "Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key."
: `${String(count)} dictionary ${count === 1 ? "entry" : "entries"} keyed __proto__ ${count === 1 ? "was" : "were"} omitted because REA results cannot represent that key.`;
-const withoutKey = (
+const projectDictionaryKeys = (
value: unknown,
- key: string,
+ projectName: (name: string) => string | undefined,
): PropertyListWithoutPrototypeKeys => {
let omitted = 0;
const strip = (item: unknown): unknown => {
@@ -35,7 +35,8 @@ const withoutKey = (
return item;
return Object.fromEntries(
Object.entries(item).flatMap(([name, entry]) => {
- if (name !== key) return [[name, strip(entry)]];
+ const projected = projectName(name);
+ if (projected !== undefined) return [[projected, strip(entry)]];
omitted += 1;
return [];
}),
@@ -48,7 +49,10 @@ const withoutKey = (
/** Remove own `__proto__` entries, such as JSON.parse creates, and count them. */
export const omitPrototypeKeys = (
value: unknown,
-): PropertyListWithoutPrototypeKeys => withoutKey(value, PROTOTYPE_KEY);
+): PropertyListWithoutPrototypeKeys =>
+ projectDictionaryKeys(value, (name) =>
+ name === PROTOTYPE_KEY ? undefined : name,
+ );
const XML_ENTITIES: Readonly<Record<string, string>> = {
amp: "&",
@@ -82,15 +86,14 @@ export const decodeXmlText = (text: string): string =>
);
/**
- * Decode an XML plist whose dictionaries may use the legal key `__proto__`,
- * which the `plist` decoder rejects outright. Those entries are omitted and
- * counted so callers can report them instead of failing the whole document.
+ * Decode complete XML character-data keys before the pinned value decoder.
+ * Entries keyed `__proto__` are omitted and counted at the same boundary.
*/
export const parseXmlPropertyList = (
text: string,
): PropertyListWithoutPrototypeKeys => {
// Preserve the pinned decoder's existing OpenStep autodetection, also used
- // by app Info.plist resolution. Only XML needs the key-element substitution.
+ // by app Info.plist resolution. Only XML needs key normalization.
const first = text.trimStart()[0];
if (first === "{" || first === "(") {
// The pinned decoder assigns a `__proto__` key to the object prototype, so
@@ -115,11 +118,11 @@ export const parseXmlPropertyList = (
if (!parsed.ok) throw parsed.error;
const document = parsed.value;
assertXmlPlistDepthWithinLimit(document);
- const { substituted, placeholder } = substitutePrototypeKeys(
- normalized,
- document,
- );
- return withoutKey(parse(substituted), placeholder);
+ const { substituted, names } = normalizeXmlKeys(normalized, document);
+ return projectDictionaryKeys(parse(substituted), (name) => {
+ const decoded = names.get(name) ?? name;
+ return decoded === PROTOTYPE_KEY ? undefined : decoded;
+ });
};
interface OpenStepContainer {
@@ -323,17 +326,48 @@ const assertXmlPlistDepthWithinLimit = (document: Document): void => {
}
};
-/** Locate keys with the same XML parser and first-child semantics as plist. */
-const substitutePrototypeKeys = (text: string, document: Document) => {
- const keys = Array.from(document.getElementsByTagName("key"));
- const decodedKeys = new Set(keys.map((key) => key.firstChild?.nodeValue));
- let placeholder = "__rea_prototype_key__";
- while (decodedKeys.has(placeholder)) placeholder = `_${placeholder}`;
-
- const omitted = keys
- .map((key) => key.firstChild)
- .filter((node) => node?.nodeValue === PROTOTYPE_KEY);
- if (omitted.length === 0) return { substituted: text, placeholder };
+/** The validated DOM owns key identities; aliases bypass first-child-only decoding. */
+const normalizeXmlKeys = (text: string, document: Document) => {
+ const keys = Array.from(document.getElementsByTagName("key"))
+ // The decoder also accepts <key> as a scalar outside dictionaries. Such
+ // values have no dictionary-name restoration boundary and must not alias.
+ .filter((key) => key.parentNode?.nodeName === "dict")
+ .map((key) => {
+ const first = key.firstChild;
+ // Nested elements are unsupported key markup. Retain the pinned decoder's
+ // interpretation instead of treating descendant element text as key data.
+ const characterDataOnly = !Array.from(key.childNodes).some(
+ (child) => child.nodeType === child.ELEMENT_NODE,
+ );
+ return {
+ node: first ?? key,
+ empty: first === null,
+ name: characterDataOnly
+ ? (key.textContent ?? "")
+ : first === null
+ ? ""
+ : String(first.nodeValue),
+ firstName: first?.nodeValue ?? "",
+ };
+ });
+ const occupied = new Set(
+ keys.flatMap(({ name, firstName }) => [name, firstName]),
+ );
+ const aliases = new Map<string, string>();
+ const names = new Map<string, string>();
+ for (const { name, firstName } of keys) {
+ if ((name !== PROTOTYPE_KEY && name === firstName) || aliases.has(name))
+ continue;
+ let alias = `__rea_plist_key_${String(aliases.size)}__`;
+ while (occupied.has(alias)) alias = `_${alias}`;
+ occupied.add(alias);
+ aliases.set(name, alias);
+ names.set(alias, name);
+ }
+ // Alias every occurrence of a corrected name, including plain spellings,
+ // so the decoder retains its last-value-wins behavior for duplicate keys.
+ const replacements = keys.filter(({ name }) => aliases.has(name));
+ if (replacements.length === 0) return { substituted: text, names };
// xmldom uses one-based locators in document order. Scan forward only to
// needed lines instead of retaining an offset for every source newline.
@@ -341,7 +375,7 @@ const substitutePrototypeKeys = (text: string, document: Document) => {
let lineStart = 0;
const parts: string[] = [];
let cursor = 0;
- for (const node of omitted) {
+ for (const { node, name, empty } of replacements) {
const nodeLine = node?.lineNumber;
const column = node?.columnNumber;
if (nodeLine === undefined || column === undefined)
@@ -354,9 +388,27 @@ const substitutePrototypeKeys = (text: string, document: Document) => {
line += 1;
}
const offset = lineStart + column - 1;
- parts.push(text.slice(cursor, offset), `<![CDATA[${placeholder}]]>`);
- cursor = offset;
+ const marker = `<![CDATA[${aliases.get(name)}]]>`;
+ parts.push(text.slice(cursor, offset));
+ if (empty) {
+ // Empty keys have no child locator. Replace only their opening tag,
+ // consuming quoted attributes whole, and expand self-closing spellings.
+ const opening = /^<(?:[^>"']|"[^"]*"|'[^']*')*>/u.exec(
+ text.slice(offset),
+ )?.[0];
+ if (opening === undefined)
+ throw new TypeError("XML plist key has an invalid opening tag");
+ parts.push(
+ /\/\s*>$/u.test(opening)
+ ? `${opening.replace(/\/\s*>$/u, ">")}${marker}</key>`
+ : `${opening}${marker}`,
+ );
+ cursor = offset + opening.length;
+ } else {
+ parts.push(marker);
+ cursor = offset;
+ }
}
parts.push(text.slice(cursor));
- return { substituted: parts.join(""), placeholder };
+ return { substituted: parts.join(""), names };
};
diff --git a/src/native/parsers/plist.test.ts b/src/native/parsers/plist.test.ts
index db1cbf135..8d6be612d 100644
--- a/src/native/parsers/plist.test.ts
+++ b/src/native/parsers/plist.test.ts
@@ -6,6 +6,20 @@ import {
plistJsonNeedsNumberTypes,
} from "./plist.js";
+it("retains complete segmented XML keys in native plist fallback evidence", () => {
+ const parsed = parsePlistXml(
+ "<plist><dict><key>user<![CDATA[ID]]></key><integer>42</integer><key>__pro<![CDATA[to__]]></key><string>omitted</string><key>Blob</key><data>AA==</data></dict></plist>",
+ );
+ if (!parsed.ok) throw parsed.error;
+ expect(parsed.value.value).toEqual({
+ userID: 42,
+ Blob: { $plist_type: "data", base64: "AA==" },
+ });
+ expect(parsed.value.limitations).toContain(
+ "1 dictionary entry keyed __proto__ was omitted because REA results cannot represent that key.",
+ );
+});
+
it("keeps exact decimal text for JSON integers that XML shows are integers", () => {
const parsed = parsePlistJson(
'{"Max":18446744073709551615,"Min":-9223372036854775808,"Near":9007199254740993,"Edge":9007199254740992,"Safe":9007199254740991,"Real":1e21,"Ratio":0.5}',
diff --git a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts
index a70f3e1aa..3c7049e3b 100644
--- a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts
+++ b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts
@@ -18,6 +18,7 @@ const scenarios = [
"integer-real collision",
"real",
"mixed XML",
+ "segmented keys",
] as const;
cliTest.for(scenarios)(
@@ -35,7 +36,9 @@ cliTest.for(scenarios)(
);
await writeFile(
join(contents, "Info.plist"),
- build({ CFBundleExecutable: "Fixture" }),
+ scenario === "segmented keys"
+ ? "<plist><dict><key>CFBundle<![CDATA[Executable]]></key><string>Fixture</string></dict></plist>"
+ : build({ CFBundleExecutable: "Fixture" }),
);
const source = {
$archiver: "NSKeyedArchiver",
@@ -54,7 +57,7 @@ cliTest.for(scenarios)(
{ $classname: "UIView", $classes: ["UIView", "NSObject"] },
],
};
- const bytes =
+ let bytes =
scenario === "binary"
? Buffer.from(buildBinary(source))
: Buffer.from(
@@ -86,6 +89,14 @@ cliTest.for(scenarios)(
bytes[offset] = 0x13;
bytes.writeBigInt64BE(9007199254740993n, offset + 1);
}
+ if (scenario === "segmented keys")
+ bytes = Buffer.from(
+ bytes
+ .toString("utf8")
+ .replaceAll("<key>$class</key>", "<key>$cl<![CDATA[ass]]></key>")
+ .replaceAll("<key>children</key>", "<key>chil<![CDATA[dren]]></key>")
+ .replaceAll("<key>safe</key>", "<key>sa<![CDATA[fe]]></key>"),
+ );
await writeFile(join(resources, "Main.nib"), bytes);
const output = await cli.run({
arguments: ["decode-interface-builder", app, "--json"],
@@ -109,7 +120,9 @@ cliTest.for(scenarios)(
id: 'ib:"Contents/Resources/Main.nib":object:["1"]',
attributes: {
large:
- scenario === "XML" || scenario === "binary"
+ scenario === "XML" ||
+ scenario === "binary" ||
+ scenario === "segmented keys"
? { $plist_type: "integer", decimal: "9007199254740993" }
: 9007199254740992,
safe: 42,
@@ -126,7 +139,11 @@ cliTest.for(scenarios)(
relation: "contains",
}),
);
- if (scenario === "XML" || scenario === "binary")
+ if (
+ scenario === "XML" ||
+ scenario === "binary" ||
+ scenario === "segmented keys"
+ )
expect(result.limitations).toContainEqual(
expect.stringContaining("1 unambiguous integer value"),
);
diff --git a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts
index 79716b6cd..a686d012f 100644
--- a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts
+++ b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts
@@ -1,3 +1,7 @@
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { buildBinary } from "plist";
import { expect } from "vitest";
import { parseEvidence } from "../../../src/domain/evidence.js";
@@ -6,6 +10,9 @@ import {
keyedArchivePrototypeKeyFixture,
} from "../../fixtures/keyedArchivePrototypeKeys.js";
import { cliTest } from "../../support/cli/cliFixture.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js";
+import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js";
cliTest(
"compiled CLI decodes formatted XML prototype keys",
@@ -22,3 +29,92 @@ cliTest(
);
},
);
+
+cliTest(
+ "preserves segmented XML root and field identities through inspection and root selection",
+ async ({ cli }) => {
+ const directory = await createTestTempDirectory(
+ "rea-keyed-character-data-",
+ );
+ const { call } = await connectLocalToolsMcp({
+ HOME: directory,
+ USERPROFILE: directory,
+ });
+ const mixed = Buffer.from(`<?xml version="1.0" encoding="UTF-8"?>
+<plist><dict>
+<key>$archiver</key><string>NSKeyedArchiver</string>
+<key>$version</key><integer>100000</integer>
+<key>$objects</key><array><string>$null</string><dict>
+<key>user<![CDATA[ID]]></key><integer>42</integer>
+</dict></array>
+<key>$top</key><dict><key>ma<![CDATA[in]]></key><dict>
+<key>CF$UID</key><integer>1</integer>
+</dict></dict></dict></plist>`);
+ const plain = Buffer.from(
+ mixed
+ .toString("utf8")
+ .replace("user<![CDATA[ID]]>", "userID")
+ .replace("ma<![CDATA[in]]>", "main"),
+ );
+ // Binary encoding is an independent control, not a decode of the XML fixture.
+ const binary = Buffer.from(
+ buildBinary({
+ $archiver: "NSKeyedArchiver",
+ $version: 100000,
+ $objects: ["$null", { userID: 42 }],
+ $top: { main: { CF$UID: 1 } },
+ }),
+ );
+ for (const [name, bytes] of [
+ ["mixed", mixed],
+ ["plain", plain],
+ ["binary", binary],
+ ] as const) {
+ const path = join(directory, `${name}.plist`);
+ await writeFile(path, bytes);
+ for (const rootArguments of [[], ["--root", "main"]]) {
+ const output = await cli.run({
+ arguments: [
+ "inspect-keyed-archive",
+ path,
+ ...rootArguments,
+ "--json",
+ ],
+ environment: {
+ REA_LOG_LEVEL: "silent",
+ HOME: directory,
+ USERPROFILE: directory,
+ },
+ });
+ expect(output.exitCode, output.stdout).toBe(0);
+ const result = keyedArchiveResultSchema.parse(
+ parseEvidence(output.json).normalized_result,
+ );
+ expect(result.archive_sha256).toBe(
+ createHash("sha256").update(bytes).digest("hex"),
+ );
+ expect(Object.keys(result.roots)).toEqual(["main"]);
+ expect(result.objects[1]).toMatchObject({
+ id: 1,
+ kind: "dictionary",
+ status: "observed",
+ value: { userID: 42 },
+ });
+ expect(result.references).toMatchObject([
+ { source: null, path: ["main"], target: 1, status: "resolved" },
+ ]);
+ expect(result.truncated).toBe(false);
+ const opened = await call("open_binary", { path });
+ expect(opened.isError, JSON.stringify(opened)).not.toBe(true);
+ const mcp = await call("inspect_keyed_archive", (rootArguments.length === 0 ? {} : { root: "main" }));
+ expect(mcp.isError, JSON.stringify(mcp)).not.toBe(true);
+ expect(
+ keyedArchiveResultSchema.parse(
+ parseEvidence(mcp.structuredContent).normalized_result,
+ ),
+ ).toEqual(result);
+ }
+ expect(await readFile(path)).toEqual(bytes);
+ }
+ },
+);