Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(environment): compare captured bodies by representation

morluto7af6fa01f56c
diff --git a/docs/specs/agent-environments/fidelity-compare/spec.md b/docs/specs/agent-environments/fidelity-compare/spec.md
index a2cec6126..9b76998a4 100644
--- a/docs/specs/agent-environments/fidelity-compare/spec.md
+++ b/docs/specs/agent-environments/fidelity-compare/spec.md
@@ -62,8 +62,10 @@ credentials and resets it between sequences.
reported `refused`. A browser probe keeps its request bodies; one captured without response bodies is refused, since
only statuses would compare, and so is one that makes POST, PUT or PATCH calls without recorded request bodies, since
those calls would be made empty.
-- Answers compare after normalisation: empty response text equals no body (a JSON empty string stays one); mapped ids
- compare equal, in scalar sets as well, an entity's own identity within the call's collection and any other id by the
+- Answers compare after normalisation: empty response text and JSON `null` equal no body (a JSON empty string stays one);
+ decoded JSON values compare structurally, plain text compares as text, and retained base64 bodies compare as bytes
+ (including against UTF-8 text with the same bytes), without treating a base64 spelling as text. Mapped ids compare
+ equal, in scalar sets as well, an entity's own identity within the call's collection and any other id by the
probe's mapping, and an entity's own identity both sides hold verbatim differs when its collection's mapping gives it
another environment value (a reference keeps raw equality, as the probe's mapping cannot tell which collection it
names); an object's members compare under the environment's mapped keys only when it is keyed by learned ids as
diff --git a/src/application/environment/environmentCompare.ts b/src/application/environment/environmentCompare.ts
index 53cd1bbcf..b50360623 100644
--- a/src/application/environment/environmentCompare.ts
+++ b/src/application/environment/environmentCompare.ts
@@ -38,6 +38,7 @@ import {
replayedBody,
type EnvironmentManifestTool,
type EnvironmentSource,
+ type ReplayedBody,
} from "./environmentSource.js";
import {
selectsRecordedLine,
@@ -159,13 +160,42 @@ const entityOf = (
const fromState = (answer: ReplayResult): boolean =>
(answer as Partial<StateAnswer>).state?.source === "state";
+/** Identity facts come from decoded JSON or text scalars, never binary presentations or truncated bodies. */
+const identityValue = (body: ReplayedBody): JsonValue | undefined => {
+ if (body.kind === "json") return body.value;
+ if (body.kind === "text") return body.text;
+ return undefined;
+};
+
/** The environment's body as the application's is read: a replayed recording of empty text, unlike a JSON empty string, is no body. */
-const envBody = (run: ProbeRun, answer: ReplayResult): JsonValue => {
- if (answer.body !== "" || fromState(answer)) return answer.body;
+const envBody = (run: ProbeRun, answer: ReplayResult): ReplayedBody => {
+ if (fromState(answer)) return { kind: "json", value: answer.body };
const exchange = run.source.exchanges.get(answer.replay.record_ordinal);
- return exchange !== undefined && replayedBody(exchange).kind === "json"
- ? answer.body
- : null;
+ if (exchange === undefined)
+ return answer.body === ""
+ ? { kind: "none" }
+ : { kind: "json", value: answer.body };
+ const body = replayedBody(exchange);
+ // Empty replayed text is no body; a JSON empty string remains a JSON value.
+ if (body.kind === "text" && body.text === "") return { kind: "none" };
+ if (body.kind === "json") return { kind: "json", value: answer.body };
+ if (body.kind === "text")
+ return {
+ kind: "text",
+ text: typeof answer.body === "string" ? answer.body : body.text,
+ };
+ if (body.kind === "base64")
+ return {
+ ...body,
+ encoded: typeof answer.body === "string" ? answer.body : body.encoded,
+ whole: body.whole && !answer.replay.truncated,
+ };
+ if (body.kind === "cut")
+ return {
+ kind: "cut",
+ text: typeof answer.body === "string" ? answer.body : body.text,
+ };
+ return body;
};
/**
@@ -323,7 +353,7 @@ const compareStep = (
const entity = entityOf(run, call.tool, liveSegments);
// Another call's recording holds other entities, whose ids would map wrongly and, the first pairing winning, for good.
if (answersCall(run, call, args, answered.value))
- learnIds(live.body, env.body, run.map, {
+ learnIds(identityValue(live.body), identityValue(env.body), run.map, {
heldIds: heldIds(run.store),
scope: entity.scope,
entityAt: entity.at,
diff --git a/src/application/environment/environmentCompareBody.test.ts b/src/application/environment/environmentCompareBody.test.ts
new file mode 100644
index 000000000..a6e934705
--- /dev/null
+++ b/src/application/environment/environmentCompareBody.test.ts
@@ -0,0 +1,89 @@
+import { createHash } from "node:crypto";
+import { describe, expect, it } from "vitest";
+
+import {
+ analysisViewCaptureEvidence,
+ environmentManifestCaptureFixture,
+} from "../../../tests/fixtures/analysisView.js";
+import {
+ environmentServeSource,
+ environmentStateServeFixture,
+} from "../../../tests/fixtures/environmentServe.js";
+import { compareEnvironment } from "./environmentCompare.js";
+
+const responseCapture = (
+ text: string,
+ base64?: string,
+ mimeType = "application/octet-stream",
+): ReturnType<typeof analysisViewCaptureEvidence> => {
+ const capture = environmentManifestCaptureFixture([
+ {
+ startedDateTime: "2026-10-17T18:00:00.000Z",
+ request: { method: "GET", url: "http://app.example/api/blob" },
+ response: {
+ status: 200,
+ content: {
+ mimeType,
+ text,
+ ...(base64 === undefined ? {} : { encoding: "base64" }),
+ },
+ },
+ },
+ ]);
+ if (base64 !== undefined) {
+ const bytes = Buffer.from(base64, "base64");
+ capture.records[0]!.binary_fields = [
+ {
+ pointer: "/response/content/text",
+ representation: "har-base64-content",
+ state: "retained",
+ content_base64: base64,
+ bytes: bytes.length,
+ sha256: createHash("sha256").update(bytes).digest("hex"),
+ },
+ ];
+ }
+ return analysisViewCaptureEvidence(capture);
+};
+
+const compare = (
+ recorded: ReturnType<typeof analysisViewCaptureEvidence>,
+ probe: ReturnType<typeof analysisViewCaptureEvidence>,
+) => {
+ const fixture = environmentStateServeFixture(recorded);
+ const report = compareEnvironment({
+ source: environmentServeSource(fixture),
+ captureValue: recorded,
+ probes: [{ label: "probe", value: probe }],
+ operation: "environment-compare",
+ });
+ if (!report.ok) throw report.error;
+ return report.value;
+};
+
+describe("environment comparison response representation", () => {
+ it("distinguishes binary byte ff from its base64 spelling as literal text", () => {
+ const binary = responseCapture("/w==", "/w==");
+ const literal = responseCapture("/w==");
+ const result = compare(binary, literal);
+ expect(result.probes[0]?.results[0]?.equal).toBe(false);
+ expect(result.probes[0]?.results[0]?.categories).toEqual(["value"]);
+ });
+
+ it("compares equal retained binary bytes and valid UTF-8 base64 as text", () => {
+ expect(
+ compare(responseCapture("/w==", "/w=="), responseCapture("/w==", "/w=="))
+ .probes[0]?.results[0]?.equal,
+ ).toBe(true);
+ expect(
+ compare(responseCapture("aGVsbG8=", "aGVsbG8="), responseCapture("hello"))
+ .probes[0]?.results[0]?.equal,
+ ).toBe(true);
+ });
+
+ it("does not conflate a JSON string value with identical plain text", () => {
+ const json = responseCapture('"hello"', undefined, "application/json");
+ const text = responseCapture("hello");
+ expect(compare(json, text).probes[0]?.results[0]?.equal).toBe(false);
+ });
+});
diff --git a/src/application/environment/environmentCompareDiff.test.ts b/src/application/environment/environmentCompareDiff.test.ts
index 7037a4744..7a33c970e 100644
--- a/src/application/environment/environmentCompareDiff.test.ts
+++ b/src/application/environment/environmentCompareDiff.test.ts
@@ -11,8 +11,8 @@ const compare = (
status: [number, number] = [200, 200],
) =>
compareAnswers(
- { status: status[0], body: live as never },
- { status: status[1], body: env as never },
+ { status: status[0], body: { kind: "json", value: live as JsonValue } },
+ { status: status[1], body: { kind: "json", value: env as JsonValue } },
map,
);
@@ -145,11 +145,13 @@ describe("compareAnswers", () => {
.differences,
).toEqual([]);
});
+});
+describe("comparison body coverage", () => {
it("compares a live body over the replay limit by status only and counts it", () => {
const result = compareAnswers(
- { status: 200, body: '{"data":[', cut: true },
- { status: 200, body: null },
+ { status: 200, body: { kind: "cut", text: '{"data":[' } },
+ { status: 200, body: { kind: "none" } },
new IdMap(),
);
expect(result.differences).toEqual([]);
@@ -157,17 +159,24 @@ describe("compareAnswers", () => {
});
it("compares a live body the probe did not record by status only and counts it", () => {
- const unrecorded = { status: 200, body: null, unobserved: true };
+ const unrecorded = {
+ status: 200,
+ body: { kind: "none" as const },
+ unobserved: true,
+ };
const result = compareAnswers(
unrecorded,
- { status: 200, body: { id: 1 } },
+ { status: 200, body: { kind: "json", value: { id: 1 } } },
new IdMap(),
);
expect(result.differences).toEqual([]);
expect(result.normalised.unobserved_bodies).toBe(1);
expect(
- compareAnswers(unrecorded, { status: 404, body: null }, new IdMap())
- .differences,
+ compareAnswers(
+ unrecorded,
+ { status: 404, body: { kind: "none" } },
+ new IdMap(),
+ ).differences,
).toEqual([{ category: "status", pointer: "", live: "200", env: "404" }]);
});
});
@@ -178,8 +187,8 @@ describe("scalar sets of mapped ids", () => {
map.learn(101, 1);
map.learn(102, 2);
const compared = compareAnswers(
- { status: 200, body: { ids: [101, 102] } },
- { status: 200, body: { ids: [2, 1] } },
+ { status: 200, body: { kind: "json", value: { ids: [101, 102] } } },
+ { status: 200, body: { kind: "json", value: { ids: [2, 1] } } },
map,
);
expect(compared.differences).toEqual([
@@ -194,7 +203,10 @@ describe("ids two collections reuse", () => {
const map = new IdMap();
map.learn(101, 1, "/api/books");
map.learn(101, 2, "/api/authors");
- const answer = (body: JsonValue) => ({ status: 200, body });
+ const answer = (body: JsonValue) => ({
+ status: 200,
+ body: { kind: "json" as const, value: body },
+ });
expect(
compareAnswers(answer({ id: 101 }), answer({ id: 2 }), map, {
scope: "/api/authors",
@@ -226,7 +238,10 @@ describe("identity responses two collections reuse", () => {
entityAt: "/created",
identity: true,
});
- const answer = (body: JsonValue) => ({ status: 201, body });
+ const answer = (body: JsonValue) => ({
+ status: 201,
+ body: { kind: "json" as const, value: body },
+ });
expect(books.env(103)).toBe(1);
expect(books.env(103, "/api/authors")).toBe(3);
expect(created.env(103, "/api/authors")).toBe(3);
@@ -264,8 +279,14 @@ describe("wrapped answers", () => {
expect(map.env(101, "/api/authors")).toBe(2);
expect(
compareAnswers(
- { status: 200, body: { data: { id: 101, name: "Bob" } } },
- { status: 200, body: { data: { id: 2, name: "Bob" } } },
+ {
+ status: 200,
+ body: { kind: "json", value: { data: { id: 101, name: "Bob" } } },
+ },
+ {
+ status: 200,
+ body: { kind: "json", value: { data: { id: 2, name: "Bob" } } },
+ },
map,
{ scope: "/api/authors", at: "/data" },
).differences,
@@ -274,7 +295,10 @@ describe("wrapped answers", () => {
});
describe("raw ids a mapping gives to other entities", () => {
- const answer = (body: JsonValue) => ({ status: 200, body });
+ const answer = (body: JsonValue) => ({
+ status: 200,
+ body: { kind: "json" as const, value: body },
+ });
it("reports an entity's own identity equal on both sides that its collection's mapping gives to another entity", () => {
const map = new IdMap();
diff --git a/src/application/environment/environmentCompareDiff.ts b/src/application/environment/environmentCompareDiff.ts
index 893fa2447..feb38c981 100644
--- a/src/application/environment/environmentCompareDiff.ts
+++ b/src/application/environment/environmentCompareDiff.ts
@@ -5,6 +5,7 @@ import {
type ClockFormat,
} from "../../domain/analysisView/environmentStateWriteValues.js";
import type { JsonValue } from "../../domain/jsonValue.js";
+import type { ReplayedBody } from "./environmentSource.js";
import {
IDENTITY_FIELD,
keyedByLearnedIds,
@@ -318,9 +319,7 @@ class Differ {
/** One answer: its status, and its JSON body, its text, or null without one. */
export interface Answer {
readonly status: number | null;
- readonly body: JsonValue;
- /** Set when the body was over the replay limit, so only a prefix is known. */
- readonly cut?: boolean;
+ readonly body: ReplayedBody;
/** Set when the probe did not record the body, so only the status is known. */
readonly unobserved?: boolean;
}
@@ -348,8 +347,50 @@ export const compareAnswers = (
live: String(live.status),
env: String(env.status),
});
- if (live.cut === true) differ.normalised.large_bodies += 1;
+ if (
+ live.body.kind === "cut" ||
+ (live.body.kind === "base64" && !live.body.whole)
+ )
+ differ.normalised.large_bodies += 1;
else if (live.unobserved === true) differ.normalised.unobserved_bodies += 1;
- else differ.value(live.body, env.body, "");
+ else if (
+ (live.body.kind === "none" &&
+ env.body.kind === "json" &&
+ env.body.value === null) ||
+ (env.body.kind === "none" &&
+ live.body.kind === "json" &&
+ live.body.value === null)
+ ) {
+ // The existing answer contract treats an absent body and JSON null alike.
+ } else if (live.body.kind === "base64" || env.body.kind === "base64") {
+ const bytes = (body: ReplayedBody): Buffer | undefined => {
+ if (body.kind === "base64") return Buffer.from(body.encoded, "base64");
+ if (body.kind === "text") return Buffer.from(body.text, "utf8");
+ return undefined;
+ };
+ const liveBytes = bytes(live.body);
+ const envBytes = bytes(env.body);
+ if (
+ liveBytes === undefined ||
+ envBytes === undefined ||
+ !liveBytes.equals(envBytes)
+ )
+ differ.add({
+ category: "value",
+ pointer: "",
+ live: shown(live.body),
+ env: shown(env.body),
+ });
+ } else if (live.body.kind !== env.body.kind)
+ differ.add({
+ category: "value",
+ pointer: "",
+ live: shown(live.body),
+ env: shown(env.body),
+ });
+ else if (live.body.kind === "json" && env.body.kind === "json")
+ differ.value(live.body.value, env.body.value, "");
+ else if (live.body.kind === "text" && env.body.kind === "text")
+ differ.value(live.body.text, env.body.text, "");
return { differences: differ.differences, normalised: differ.normalised };
};
diff --git a/src/application/environment/environmentCompareProbe.ts b/src/application/environment/environmentCompareProbe.ts
index 83364470b..e35d9af93 100644
--- a/src/application/environment/environmentCompareProbe.ts
+++ b/src/application/environment/environmentCompareProbe.ts
@@ -76,7 +76,7 @@ export const requestBody = (
: text;
};
-/** A body over the replay limit, text or bytes: only a prefix is known, so only the status compares. */
+/** A replay body the server cut at its size limit. */
export const isCut = (body: ReplayedBody): boolean =>
body.kind === "cut" || (body.kind === "base64" && !body.whole);
@@ -88,16 +88,11 @@ const liveAnswer = (
): Answer => {
if (body.kind === "none")
return method === "HEAD" || BODYLESS_STATUSES.has(status)
- ? { status, body: null }
- : { status, body: null, unobserved: true };
- if (body.kind === "json") return { status, body: body.value };
- if (body.kind === "base64")
- return { status, body: body.encoded, cut: isCut(body) };
- return {
- status,
- body: body.text === "" ? null : body.text,
- cut: isCut(body),
- };
+ ? { status, body }
+ : { status, body, unobserved: true };
+ if (body.kind === "text" && body.text === "")
+ return { status, body: { kind: "none" } };
+ return { status, body };
};
/** The origin a probe calls the application at, chosen as the manifest chooses its own. */