Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(process): preserve numeric output and original PTY evidence

N0zoM1z000d8e44cbf82
diff --git a/docs/process-capture.md b/docs/process-capture.md
index 2b4438139..f7a117b4a 100644
--- a/docs/process-capture.md
+++ b/docs/process-capture.md
@@ -94,6 +94,23 @@ deleted, modified, or unchanged. These are bounded snapshots, not a syscall
trace; short-lived changes between snapshots may be missed. With no selected
paths, filesystem effects remain unknown.
+Terminal frame `data` is the comparison text after the selected normalization.
+New captures also preserve `raw_data` when normalization changes a PTY chunk;
+otherwise `data` is already the original text. Older captures may lack the
+original text for changed chunks. Both values participate in Evidence identity,
+while comparisons and trace assertions use the normalized text. Retaining the
+original adds at most `limits.output_bytes` of text, because it comes from the
+same admitted PTY chunks rather than a second unbounded stream.
+
+Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and
+`listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with
+ports from 0 through 65535. It preserves ordinary counters, dimensions, file
+line numbers, and ambiguous bare host labels. Use explicit literal `patterns`
+for a different application-specific spelling. Set `ports: false` to preserve
+all endpoint numbers in comparison text. Captures commit the port-normalization
+version so older broad numeric normalization cannot silently compare as the
+same contract.
+
Output, file count, file size, process sampling, filesystem depth, total
runtime, idle time, and post-exit settlement are bounded by the scenario's
limits. The result marks truncated observations and residual unknowns rather
diff --git a/src/domain/process/processCapture.comparison.test.ts b/src/domain/process/processCapture.comparison.test.ts
index 93d724627..85fd98369 100644
--- a/src/domain/process/processCapture.comparison.test.ts
+++ b/src/domain/process/processCapture.comparison.test.ts
@@ -5,6 +5,28 @@ import {
emptyProcessCapture as emptyCapture,
} from "./processCapture.fixture.js";
+it("compares normalized terminal facts while retaining different source text for audit", () => {
+ const capture = emptyCapture();
+ const frame = { sequence: 0, at_ms: 0, data: "listen:<port> score=20" };
+ const left = {
+ ...capture,
+ frames: [{ ...frame, raw_data: "listen:8080 score=20" }],
+ };
+ const right = {
+ ...capture,
+ frames: [{ ...frame, raw_data: "listen:9090 score=20" }],
+ };
+ expect(compareProcessCaptures(left, right)).toMatchObject({
+ terminal: "unchanged",
+ });
+ expect(
+ compareProcessCaptures(left, {
+ ...right,
+ frames: [{ ...frame, data: "listen:<port> score=21" }],
+ }),
+ ).toMatchObject({ terminal: "changed" });
+});
+
it("distinguishes added evidence from unknown observations", () => {
const capture = emptyCapture();
const added = compareProcessCaptures(capture, {
diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.ts
index 9739cba66..45803af00 100644
--- a/src/domain/process/processCapture.ts
+++ b/src/domain/process/processCapture.ts
@@ -11,6 +11,8 @@ export interface TerminalFrame {
readonly sequence: number;
readonly at_ms: number;
readonly data: string;
+ /** Original PTY text when normalization changed it; older captures may omit it. */
+ readonly raw_data?: string | undefined;
}
/** Serialized terminal state after interpreting control and resize sequences. */
@@ -402,6 +404,7 @@ const processCaptureShapeSchema = z.strictObject({
sequence: z.number().int().nonnegative(),
at_ms: z.number().int().nonnegative(),
data: z.string(),
+ raw_data: z.string().optional(),
}),
),
rendered_frames: z.array(
diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts
index 1aa052bc4..04542631a 100644
--- a/src/domain/process/processComparison.ts
+++ b/src/domain/process/processComparison.ts
@@ -2,6 +2,7 @@ import canonicalize from "canonicalize";
import { z } from "zod";
import type { ProcessCapture } from "./processCapture.js";
+import { comparableTerminalFrame } from "./processObservation.js";
import { AnalysisInputError } from "../analysisErrorCore.js";
import { jsonValueSchema } from "../jsonValue.js";
import {
@@ -100,7 +101,10 @@ export type ProcessCaptureComparison = z.infer<
const terminalObservations = (capture: ProcessCapture): readonly unknown[] =>
[
- ...capture.frames.map((frame) => ({ kind: "raw" as const, ...frame })),
+ ...capture.frames.map((frame) => ({
+ kind: "raw" as const,
+ ...comparableTerminalFrame(frame),
+ })),
...capture.rendered_frames.map((frame) => ({
kind: "rendered" as const,
...frame,
diff --git a/src/domain/process/processObservation.ts b/src/domain/process/processObservation.ts
index 167d36ec5..f8ad26130 100644
--- a/src/domain/process/processObservation.ts
+++ b/src/domain/process/processObservation.ts
@@ -1,7 +1,17 @@
import { z } from "zod";
import { jsonValueSchema, type JsonValue } from "../jsonValue.js";
-import type { UnverifiedProcessCapture } from "./processCapture.js";
+import type {
+ TerminalFrame,
+ UnverifiedProcessCapture,
+} from "./processCapture.js";
+
+/** Project normalized PTY facts without source text used only for audit. */
+export const comparableTerminalFrame = (frame: TerminalFrame) => ({
+ sequence: frame.sequence,
+ at_ms: frame.at_ms,
+ data: frame.data,
+});
/** Process-capture observation families shared by live drivers and trace assertions. */
export const processObservationSourceSchema = z.enum([
@@ -126,11 +136,14 @@ const projectedRecord = (
): ProjectedRecord | null => {
const projected = (() => {
switch (location.collection) {
- case "frames":
+ case "frames": {
+ const frame = capture.frames[location.index];
return {
source: "terminal_raw" as const,
- value: capture.frames[location.index],
+ value:
+ frame === undefined ? undefined : comparableTerminalFrame(frame),
};
+ }
case "rendered_frames":
return {
source: "terminal_rendered" as const,
diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts
index 586775032..08672cdaf 100644
--- a/src/domain/process/processScenario.ts
+++ b/src/domain/process/processScenario.ts
@@ -84,6 +84,9 @@ export const processComparisonContract = (
settle_ms: scenario.settle_ms,
limits: scenario.limits,
normalization: scenario.normalization,
+ ...(scenario.normalization.ports
+ ? { port_normalization_version: "contextual-endpoints-v1" }
+ : {}),
});
/**
diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts
index d0b22abc9..0e413beb3 100644
--- a/src/process/capture/ProcessCaptureLifecycle.ts
+++ b/src/process/capture/ProcessCaptureLifecycle.ts
@@ -828,6 +828,7 @@ export const captureTerminalFrames = (options: {
sequence,
at_ms: atMs,
data: normalized,
+ ...(normalized === data ? {} : { raw_data: data }),
});
options.renderer.write(data, atMs);
options.recordEvent("frames", sequence);
diff --git a/src/process/capture/ProcessNormalization.test.ts b/src/process/capture/ProcessNormalization.test.ts
index 7a15b8b96..c28d1e42d 100644
--- a/src/process/capture/ProcessNormalization.test.ts
+++ b/src/process/capture/ProcessNormalization.test.ts
@@ -1,6 +1,9 @@
import { expect, it } from "vitest";
-import { normalizeProcessSamples } from "./ProcessNormalization.js";
+import {
+ normalizeProcessSamples,
+ normalizeProcessText,
+} from "./ProcessNormalization.js";
import { parseProcessScenario } from "../../domain/process/processCapture.js";
const rootPid = 41_010;
@@ -68,6 +71,34 @@ const scenario = (normalization: {
},
});
+it("preserves compact JSON, counters, line numbers, and ambiguous endpoint spellings", () => {
+ const input = [
+ '{"pid":32,"size":108,"height":480,"width":640,"score":20}',
+ "score=20 size:108 elapsed=3600 file.js:480 server:8080",
+ "port=65536 port:8080ms localhost:70000 999.1.2.3:8080",
+ "https://user:[email protected]/path http://user:20@localhost/path",
+ ].join("\n");
+ expect(normalizeProcessText(input, scenario({}), "/temporary", rootPid)).toBe(
+ input,
+ );
+});
+
+it("normalizes only contextual ports before overlapping PID values and honors disabled rules", () => {
+ const input =
+ 'http://example.test:8080/x https://user:[email protected]:443/x 127.0.0.1:9 [::1]:65535 localhost:41010 "port":80 tcp_port=123 udp_port:0 listen:8080';
+ expect(normalizeProcessText(input, scenario({}), "/temporary", rootPid)).toBe(
+ 'http://example.test:<port>/x https://user:[email protected]:<port>/x 127.0.0.1:<port> [::1]:<port> localhost:<port> "port":<port> tcp_port=<port> udp_port:<port> listen:<port>',
+ );
+ expect(
+ normalizeProcessText(
+ input,
+ scenario({ ports: false, pids: false }),
+ "/temporary",
+ rootPid,
+ ),
+ ).toBe(input);
+});
+
it("normalizes representative process identities and preserves null, zero, and unrelated numbers", () => {
const before = JSON.stringify(samples);
const normalized = normalizeProcessSamples(samples, scenario({}), rootPid);
diff --git a/src/process/capture/ProcessNormalization.ts b/src/process/capture/ProcessNormalization.ts
index 347c0e5cf..d9dd68385 100644
--- a/src/process/capture/ProcessNormalization.ts
+++ b/src/process/capture/ProcessNormalization.ts
@@ -1,3 +1,5 @@
+import { isIP } from "node:net";
+
import type {
ProcessSample,
ProcessScenario,
@@ -30,15 +32,34 @@ export const normalizeProcessText = (
`<filesystem-root-${String(index)}>`,
);
}
+ if (scenario.normalization.ports)
+ normalized = normalizePortTokens(normalized);
if (scenario.normalization.pids)
normalized = normalizePidTokens(normalized, [pid]);
- if (scenario.normalization.ports)
- normalized = normalized.replaceAll(/(?<=[:=])\d{2,5}\b/g, "<port>");
for (const pattern of scenario.normalization.patterns)
normalized = normalized.replaceAll(pattern.pattern, pattern.replacement);
return normalized;
};
+const normalizePortTokens = (value: string): string => {
+ const endpointNormalized = value.replaceAll(
+ /(\b[a-z][a-z0-9+.-]*:\/\/[^\s/?#"'<>]+|\blocalhost|\b(?:\d{1,3}\.){3}\d{1,3}|\[[\da-f:.%]+\]):(\d{1,5})(?![\w.@:])/giu,
+ (match: string, endpoint: string, port: string) => {
+ if (Number(port) > 65_535) return match;
+ const isEndpoint = endpoint.includes("://")
+ ? URL.canParse(`${endpoint}:${port}`)
+ : endpoint.toLowerCase() === "localhost" ||
+ isIP(endpoint.replace(/^\[|\]$/gu, "")) !== 0;
+ return isEndpoint ? `${endpoint}:<port>` : match;
+ },
+ );
+ return endpointNormalized.replaceAll(
+ /(\b(?:port|tcp_port|udp_port|listen)\b["']?\s*[:=]\s*)(\d{1,5})(?![\w.])/giu,
+ (match: string, prefix: string, port: string) =>
+ Number(port) <= 65_535 ? `${prefix}<port>` : match,
+ );
+};
+
/** Project sampled observations under the caller-selected normalization rules. */
export const normalizeProcessSamples = (
samples: readonly ProcessSample[],
diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts
index dbaef7c28..468339c86 100644
--- a/tests/boundary/process/processCaptureNormalization.test.ts
+++ b/tests/boundary/process/processCaptureNormalization.test.ts
@@ -27,6 +27,52 @@ const identitySchema = z.object({
const itWithLinuxCaptureCapability =
process.platform === "linux" ? itWithCaptureCapability : it.skip;
+itWithCaptureCapability(
+ "preserves numeric reports and original normalized PTY text in Evidence",
+ async () => {
+ const root = await createTestTempDirectory("rea-process-numeric-report-");
+ const report = JSON.stringify({
+ pid: 32,
+ size: 108,
+ height: 480,
+ width: 640,
+ score: 20,
+ });
+ const original = `${report}\nlisten:8080\n`;
+ const scenario = parseProcessScenario({
+ executable: process.execPath,
+ working_directory: root,
+ arguments: ["-e", `process.stdout.write(${JSON.stringify(original)})`],
+ settle_ms: 100,
+ });
+ const result = await captureProcessScenario(scenario);
+ if (!result.ok) throw result.error;
+ const capture = parseProcessCapture(
+ createProcessCaptureEvidence(scenario, result.value).normalized_result,
+ );
+ expect(capture.frames.map(({ data }) => data).join("")).toContain(report);
+ expect(capture.frames.map(({ data }) => data).join("")).toContain(
+ "listen:<port>",
+ );
+ expect(
+ capture.frames.map(({ data, raw_data }) => raw_data ?? data).join(""),
+ ).toBe(original.replaceAll("\n", "\r\n"));
+ expect(
+ capture.rendered_frames.some(({ serialized_state }) =>
+ serialized_state.includes(report),
+ ),
+ ).toBe(true);
+ expect(capture.manifest.comparison_contract).toMatchObject({
+ port_normalization_version: "contextual-endpoints-v1",
+ });
+ expect(capture.exit).toMatchObject({ code: 0, reason: "exited" });
+ expect(capture.cleanup).toMatchObject({
+ owned_process_group: "verified",
+ temporary_root: "removed",
+ });
+ },
+);
+
itWithLinuxCaptureCapability.each([false, true])(
"honors pids=%s in owned Linux capture samples and committed Evidence",
async (pids) => {