Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(process): preserve numeric output and original PTY evidence
N0zoM1z000d8e44cbf82
diff --git a/docs/process-capture.md b/docs/process-capture.mdindex 2b4438139..f7a117b4a 100644--- a/docs/process-capture.md+++ b/docs/process-capture.md@@ -94,6 +94,23 @@ deleted, modified, or unchanged. These are bounded snapshots, not a syscalltrace; short-lived changes between snapshots may be missed. With no selectedpaths, filesystem effects remain unknown.+Terminal frame `data` is the comparison text after the selected normalization.+New captures also preserve `raw_data` when normalization changes a PTY chunk;+otherwise `data` is already the original text. Older captures may lack the+original text for changed chunks. Both values participate in Evidence identity,+while comparisons and trace assertions use the normalized text. Retaining the+original adds at most `limits.output_bytes` of text, because it comes from the+same admitted PTY chunks rather than a second unbounded stream.++Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and+`listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with+ports from 0 through 65535. It preserves ordinary counters, dimensions, file+line numbers, and ambiguous bare host labels. Use explicit literal `patterns`+for a different application-specific spelling. Set `ports: false` to preserve+all endpoint numbers in comparison text. Captures commit the port-normalization+version so older broad numeric normalization cannot silently compare as the+same contract.+Output, file count, file size, process sampling, filesystem depth, totalruntime, idle time, and post-exit settlement are bounded by the scenario'slimits. The result marks truncated observations and residual unknowns ratherdiff --git a/src/domain/process/processCapture.comparison.test.ts b/src/domain/process/processCapture.comparison.test.tsindex 93d724627..85fd98369 100644--- a/src/domain/process/processCapture.comparison.test.ts+++ b/src/domain/process/processCapture.comparison.test.ts@@ -5,6 +5,28 @@ import {emptyProcessCapture as emptyCapture,} from "./processCapture.fixture.js";+it("compares normalized terminal facts while retaining different source text for audit", () => {+ const capture = emptyCapture();+ const frame = { sequence: 0, at_ms: 0, data: "listen:<port> score=20" };+ const left = {+ ...capture,+ frames: [{ ...frame, raw_data: "listen:8080 score=20" }],+ };+ const right = {+ ...capture,+ frames: [{ ...frame, raw_data: "listen:9090 score=20" }],+ };+ expect(compareProcessCaptures(left, right)).toMatchObject({+ terminal: "unchanged",+ });+ expect(+ compareProcessCaptures(left, {+ ...right,+ frames: [{ ...frame, data: "listen:<port> score=21" }],+ }),+ ).toMatchObject({ terminal: "changed" });+});+it("distinguishes added evidence from unknown observations", () => {const capture = emptyCapture();const added = compareProcessCaptures(capture, {diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.tsindex 9739cba66..45803af00 100644--- a/src/domain/process/processCapture.ts+++ b/src/domain/process/processCapture.ts@@ -11,6 +11,8 @@ export interface TerminalFrame {readonly sequence: number;readonly at_ms: number;readonly data: string;+ /** Original PTY text when normalization changed it; older captures may omit it. */+ readonly raw_data?: string | undefined;}/** Serialized terminal state after interpreting control and resize sequences. */@@ -402,6 +404,7 @@ const processCaptureShapeSchema = z.strictObject({sequence: z.number().int().nonnegative(),at_ms: z.number().int().nonnegative(),data: z.string(),+ raw_data: z.string().optional(),}),),rendered_frames: z.array(diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.tsindex 1aa052bc4..04542631a 100644--- a/src/domain/process/processComparison.ts+++ b/src/domain/process/processComparison.ts@@ -2,6 +2,7 @@ import canonicalize from "canonicalize";import { z } from "zod";import type { ProcessCapture } from "./processCapture.js";+import { comparableTerminalFrame } from "./processObservation.js";import { AnalysisInputError } from "../analysisErrorCore.js";import { jsonValueSchema } from "../jsonValue.js";import {@@ -100,7 +101,10 @@ export type ProcessCaptureComparison = z.infer<const terminalObservations = (capture: ProcessCapture): readonly unknown[] =>[- ...capture.frames.map((frame) => ({ kind: "raw" as const, ...frame })),+ ...capture.frames.map((frame) => ({+ kind: "raw" as const,+ ...comparableTerminalFrame(frame),+ })),...capture.rendered_frames.map((frame) => ({kind: "rendered" as const,...frame,diff --git a/src/domain/process/processObservation.ts b/src/domain/process/processObservation.tsindex 167d36ec5..f8ad26130 100644--- a/src/domain/process/processObservation.ts+++ b/src/domain/process/processObservation.ts@@ -1,7 +1,17 @@import { z } from "zod";import { jsonValueSchema, type JsonValue } from "../jsonValue.js";-import type { UnverifiedProcessCapture } from "./processCapture.js";+import type {+ TerminalFrame,+ UnverifiedProcessCapture,+} from "./processCapture.js";++/** Project normalized PTY facts without source text used only for audit. */+export const comparableTerminalFrame = (frame: TerminalFrame) => ({+ sequence: frame.sequence,+ at_ms: frame.at_ms,+ data: frame.data,+});/** Process-capture observation families shared by live drivers and trace assertions. */export const processObservationSourceSchema = z.enum([@@ -126,11 +136,14 @@ const projectedRecord = (): ProjectedRecord | null => {const projected = (() => {switch (location.collection) {- case "frames":+ case "frames": {+ const frame = capture.frames[location.index];return {source: "terminal_raw" as const,- value: capture.frames[location.index],+ value:+ frame === undefined ? undefined : comparableTerminalFrame(frame),};+ }case "rendered_frames":return {source: "terminal_rendered" as const,diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.tsindex 586775032..08672cdaf 100644--- a/src/domain/process/processScenario.ts+++ b/src/domain/process/processScenario.ts@@ -84,6 +84,9 @@ export const processComparisonContract = (settle_ms: scenario.settle_ms,limits: scenario.limits,normalization: scenario.normalization,+ ...(scenario.normalization.ports+ ? { port_normalization_version: "contextual-endpoints-v1" }+ : {}),});/**diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.tsindex d0b22abc9..0e413beb3 100644--- a/src/process/capture/ProcessCaptureLifecycle.ts+++ b/src/process/capture/ProcessCaptureLifecycle.ts@@ -828,6 +828,7 @@ export const captureTerminalFrames = (options: {sequence,at_ms: atMs,data: normalized,+ ...(normalized === data ? {} : { raw_data: data }),});options.renderer.write(data, atMs);options.recordEvent("frames", sequence);diff --git a/src/process/capture/ProcessNormalization.test.ts b/src/process/capture/ProcessNormalization.test.tsindex 7a15b8b96..c28d1e42d 100644--- a/src/process/capture/ProcessNormalization.test.ts+++ b/src/process/capture/ProcessNormalization.test.ts@@ -1,6 +1,9 @@import { expect, it } from "vitest";-import { normalizeProcessSamples } from "./ProcessNormalization.js";+import {+ normalizeProcessSamples,+ normalizeProcessText,+} from "./ProcessNormalization.js";import { parseProcessScenario } from "../../domain/process/processCapture.js";const rootPid = 41_010;@@ -68,6 +71,34 @@ const scenario = (normalization: {},});+it("preserves compact JSON, counters, line numbers, and ambiguous endpoint spellings", () => {+ const input = [+ '{"pid":32,"size":108,"height":480,"width":640,"score":20}',+ "score=20 size:108 elapsed=3600 file.js:480 server:8080",+ "port=65536 port:8080ms localhost:70000 999.1.2.3:8080",+ "https://user:[email protected]/path http://user:20@localhost/path",+ ].join("\n");+ expect(normalizeProcessText(input, scenario({}), "/temporary", rootPid)).toBe(+ input,+ );+});++it("normalizes only contextual ports before overlapping PID values and honors disabled rules", () => {+ const input =+ 'http://example.test:8080/x https://user:[email protected]:443/x 127.0.0.1:9 [::1]:65535 localhost:41010 "port":80 tcp_port=123 udp_port:0 listen:8080';+ expect(normalizeProcessText(input, scenario({}), "/temporary", rootPid)).toBe(+ 'http://example.test:<port>/x https://user:[email protected]:<port>/x 127.0.0.1:<port> [::1]:<port> localhost:<port> "port":<port> tcp_port=<port> udp_port:<port> listen:<port>',+ );+ expect(+ normalizeProcessText(+ input,+ scenario({ ports: false, pids: false }),+ "/temporary",+ rootPid,+ ),+ ).toBe(input);+});+it("normalizes representative process identities and preserves null, zero, and unrelated numbers", () => {const before = JSON.stringify(samples);const normalized = normalizeProcessSamples(samples, scenario({}), rootPid);diff --git a/src/process/capture/ProcessNormalization.ts b/src/process/capture/ProcessNormalization.tsindex 347c0e5cf..d9dd68385 100644--- a/src/process/capture/ProcessNormalization.ts+++ b/src/process/capture/ProcessNormalization.ts@@ -1,3 +1,5 @@+import { isIP } from "node:net";+import type {ProcessSample,ProcessScenario,@@ -30,15 +32,34 @@ export const normalizeProcessText = (`<filesystem-root-${String(index)}>`,);}+ if (scenario.normalization.ports)+ normalized = normalizePortTokens(normalized);if (scenario.normalization.pids)normalized = normalizePidTokens(normalized, [pid]);- if (scenario.normalization.ports)- normalized = normalized.replaceAll(/(?<=[:=])\d{2,5}\b/g, "<port>");for (const pattern of scenario.normalization.patterns)normalized = normalized.replaceAll(pattern.pattern, pattern.replacement);return normalized;};+const normalizePortTokens = (value: string): string => {+ const endpointNormalized = value.replaceAll(+ /(\b[a-z][a-z0-9+.-]*:\/\/[^\s/?#"'<>]+|\blocalhost|\b(?:\d{1,3}\.){3}\d{1,3}|\[[\da-f:.%]+\]):(\d{1,5})(?![\w.@:])/giu,+ (match: string, endpoint: string, port: string) => {+ if (Number(port) > 65_535) return match;+ const isEndpoint = endpoint.includes("://")+ ? URL.canParse(`${endpoint}:${port}`)+ : endpoint.toLowerCase() === "localhost" ||+ isIP(endpoint.replace(/^\[|\]$/gu, "")) !== 0;+ return isEndpoint ? `${endpoint}:<port>` : match;+ },+ );+ return endpointNormalized.replaceAll(+ /(\b(?:port|tcp_port|udp_port|listen)\b["']?\s*[:=]\s*)(\d{1,5})(?![\w.])/giu,+ (match: string, prefix: string, port: string) =>+ Number(port) <= 65_535 ? `${prefix}<port>` : match,+ );+};+/** Project sampled observations under the caller-selected normalization rules. */export const normalizeProcessSamples = (samples: readonly ProcessSample[],diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.tsindex dbaef7c28..468339c86 100644--- a/tests/boundary/process/processCaptureNormalization.test.ts+++ b/tests/boundary/process/processCaptureNormalization.test.ts@@ -27,6 +27,52 @@ const identitySchema = z.object({const itWithLinuxCaptureCapability =process.platform === "linux" ? itWithCaptureCapability : it.skip;+itWithCaptureCapability(+ "preserves numeric reports and original normalized PTY text in Evidence",+ async () => {+ const root = await createTestTempDirectory("rea-process-numeric-report-");+ const report = JSON.stringify({+ pid: 32,+ size: 108,+ height: 480,+ width: 640,+ score: 20,+ });+ const original = `${report}\nlisten:8080\n`;+ const scenario = parseProcessScenario({+ executable: process.execPath,+ working_directory: root,+ arguments: ["-e", `process.stdout.write(${JSON.stringify(original)})`],+ settle_ms: 100,+ });+ const result = await captureProcessScenario(scenario);+ if (!result.ok) throw result.error;+ const capture = parseProcessCapture(+ createProcessCaptureEvidence(scenario, result.value).normalized_result,+ );+ expect(capture.frames.map(({ data }) => data).join("")).toContain(report);+ expect(capture.frames.map(({ data }) => data).join("")).toContain(+ "listen:<port>",+ );+ expect(+ capture.frames.map(({ data, raw_data }) => raw_data ?? data).join(""),+ ).toBe(original.replaceAll("\n", "\r\n"));+ expect(+ capture.rendered_frames.some(({ serialized_state }) =>+ serialized_state.includes(report),+ ),+ ).toBe(true);+ expect(capture.manifest.comparison_contract).toMatchObject({+ port_normalization_version: "contextual-endpoints-v1",+ });+ expect(capture.exit).toMatchObject({ code: 0, reason: "exited" });+ expect(capture.cleanup).toMatchObject({+ owned_process_group: "verified",+ temporary_root: "removed",+ });+ },+);+itWithLinuxCaptureCapability.each([false, true])("honors pids=%s in owned Linux capture samples and committed Evidence",async (pids) => {