Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(pcap): retain parsed observations when process cleanup fails

morluto23c70d674c53
diff --git a/src/pcap/PcapCommand.ts b/src/pcap/PcapCommand.ts
index 0d28a3d75..c5d61fef1 100644
--- a/src/pcap/PcapCommand.ts
+++ b/src/pcap/PcapCommand.ts
@@ -9,6 +9,7 @@ import {
} from "../domain/analysisErrorCore.js";
import {
OwnedCommandFailure,
+ ownedCommandResources,
runOwnedCommand,
} from "../process/OwnedCommand.js";
import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js";
@@ -20,6 +21,26 @@ import {
} from "../process/WindowsBatchCommand.js";
import { pcapToolUnavailable } from "./PcapConfiguration.js";
+export type PcapCommandLauncher = NonNullable<
+ Parameters<typeof runOwnedCommand>[2]
+>["launcher"];
+
+export interface PcapCommandCleanup {
+ readonly reason: string;
+ readonly resources: readonly string[];
+ readonly capturedOutput: {
+ readonly stdout: string;
+ readonly stderr: string;
+ readonly truncated: boolean;
+ };
+}
+
+export interface PcapCommandOutput {
+ readonly stdout: string;
+ readonly stderr: string;
+ readonly cleanup?: PcapCommandCleanup;
+}
+
export interface PcapCommandLimits {
readonly timeoutMs: number;
readonly maxOutputBytes: number;
@@ -38,11 +59,14 @@ export const runPcapCommand = async (options: {
readonly limiter?: { command: string; arguments: readonly string[] };
readonly memoryLimitBytes?: number;
};
+ readonly launcher?: PcapCommandLauncher;
readonly retain: (
owner: ProviderProcessSupervisor,
resources: readonly string[],
+ reason: string,
+ capturedOutput: PcapCommandCleanup["capturedOutput"],
) => void;
-}): Promise<{ stdout: string; stderr: string }> => {
+}): Promise<PcapCommandOutput> => {
const { operation, command, limits, signal } = options;
if (signal?.aborted)
throw new AnalysisCancelledError(operation, { cause: signal.reason });
@@ -52,6 +76,7 @@ export const runPcapCommand = async (options: {
LC_ALL: "C",
LANG: "C",
};
+ let cleanup: PcapCommandCleanup | undefined;
try {
const batch = process.platform === "win32" && isWindowsBatchFile(command);
const invocation = batch
@@ -87,26 +112,64 @@ export const runPcapCommand = async (options: {
: { memoryLimitBytes: options.nativeBound.memoryLimitBytes }),
},
{ timeoutMs: limits.timeoutMs, diagnosticBytes: limits.maxOutputBytes },
- signal === undefined ? {} : { signal },
+ {
+ ...(signal === undefined ? {} : { signal }),
+ ...(options.launcher === undefined
+ ? {}
+ : { launcher: options.launcher }),
+ retainCleanup: (owner, stopped) => {
+ const resources =
+ owner.launch.ownership === undefined
+ ? ["owned-process"]
+ : ownedCommandResources(owner.launch.ownership);
+ const snapshot = owner.snapshot();
+ const capturedOutput = {
+ stdout: snapshot.stdout.text,
+ stderr: snapshot.stderr.text,
+ truncated:
+ snapshot.diagnosticTruncated === true ||
+ snapshot.outputIncomplete === true,
+ };
+ options.retain(owner, resources, stopped.reason, capturedOutput);
+ cleanup = { reason: stopped.reason, resources, capturedOutput };
+ },
+ },
);
- return { stdout: output.stdout.text, stderr: output.stderr.text };
+ return {
+ stdout: output.stdout.text,
+ stderr: output.stderr.text,
+ ...(cleanup === undefined ? {} : { cleanup }),
+ };
} catch (cause) {
if (cause instanceof AnalysisError) throw cause;
if (
cause instanceof OwnedCommandFailure &&
cause.cleanupOwner !== undefined
)
- options.retain(cause.cleanupOwner, cause.resources);
+ options.retain(
+ cause.cleanupOwner,
+ cause.resources,
+ cause.cleanupFailure ?? cause.message,
+ cause.snapshot === null
+ ? { stdout: "", stderr: "", truncated: true }
+ : {
+ stdout: cause.snapshot.stdout.text,
+ stderr: cause.snapshot.stderr.text,
+ truncated:
+ cause.snapshot.diagnosticTruncated === true ||
+ cause.snapshot.outputIncomplete === true,
+ },
+ );
const details = {
cause,
...(cause instanceof OwnedCommandFailure && cause.snapshot !== null
? {
capturedOutput: {
- stdout: "",
+ stdout: cause.snapshot.stdout.text,
stderr: cause.snapshot.stderr.text,
truncated:
- cause.snapshot.stdout.observedBytes > 0 ||
- (cause.snapshot.diagnosticTruncated ?? false),
+ cause.snapshot.diagnosticTruncated === true ||
+ cause.snapshot.outputIncomplete === true,
exit_code: cause.snapshot.exitCode ?? null,
signal: cause.snapshot.signal ?? null,
},
diff --git a/src/pcap/PcapProvider.ts b/src/pcap/PcapProvider.ts
index be2c12ed2..5f1e4e322 100644
--- a/src/pcap/PcapProvider.ts
+++ b/src/pcap/PcapProvider.ts
@@ -17,6 +17,7 @@ import {
AnalysisProtocolError,
} from "../domain/analysisErrorCore.js";
import { AnalysisError } from "../domain/analysisErrorBase.js";
+import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
import { err, ok, type Result } from "../domain/result.js";
import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js";
@@ -33,8 +34,14 @@ import {
parseTsharkJsonOutput,
TSHARK_JSON_MAX_BYTES,
} from "./PcapOutput.js";
-import { runPcapCommand, type PcapCommandLimits } from "./PcapCommand.js";
+import {
+ runPcapCommand,
+ type PcapCommandCleanup,
+ type PcapCommandLimits,
+ type PcapCommandLauncher,
+} from "./PcapCommand.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
+import { jsonValueSchema } from "../domain/jsonValue.js";
import { createPcapEvidence } from "../domain/pcap/pcapEvidence.js";
import { preparePcapNativeBound } from "./PcapNativeBound.js";
import { PcapSnapshot } from "./PcapSnapshot.js";
@@ -62,8 +69,62 @@ const FIELDS = [
"http.request.full_uri",
];
+const pcapPartialEvidence = (
+ request: PcapRequest,
+ provider:
+ | typeof PCAP_PROVIDER_IDENTITY
+ | {
+ readonly id: string;
+ readonly name: string;
+ readonly version: string | null;
+ },
+ path: string,
+ identity: { readonly bytes: number; readonly sha256: string },
+ result: unknown,
+ limitations: readonly string[],
+) =>
+ createPcapEvidence(request, {
+ subject: { path, format: "file", sha256: identity.sha256 },
+ provider,
+ result: jsonValueSchema.parse(result),
+ rawResult: null,
+ limitations,
+ locations: [{ kind: "artifact-path", path }],
+ });
+
+const pcapCommandCleanupError = (
+ operation: PcapOperation,
+ snapshotPath: string | undefined,
+ cleanup: PcapCommandCleanup,
+ capturedOutput: { readonly stdout: string; readonly stderr: string },
+ partialObservation?: ReturnType<typeof pcapPartialEvidence>,
+) =>
+ new ProviderCleanupError(
+ "pcap",
+ [
+ ...new Set([
+ ...(snapshotPath === undefined ? [] : [snapshotPath]),
+ ...cleanup.resources,
+ ]),
+ ],
+ {
+ reason: cleanup.reason,
+ previous_error: null,
+ captured_output: {
+ stdout: capturedOutput.stdout,
+ stderr: capturedOutput.stderr,
+ truncated: cleanup.capturedOutput.truncated,
+ },
+ },
+ {
+ operation,
+ ...(partialObservation === undefined ? {} : { partialObservation }),
+ },
+ );
+
export interface PcapProviderOptions {
readonly environment?: Readonly<Record<string, string | undefined>>;
+ readonly launcher?: PcapCommandLauncher;
}
/** Inspect saved captures through bounded, owned tool processes and one stable input copy. */
@@ -79,17 +140,24 @@ export class PcapProvider {
},
);
private readonly environment: Readonly<Record<string, string | undefined>>;
+ private readonly launcher: PcapCommandLauncher | undefined;
private readonly snapshots = new Set<PcapSnapshot>();
private readonly commands = new Map<
ProviderProcessSupervisor,
- { snapshot: PcapSnapshot | undefined; resources: readonly string[] }
+ {
+ snapshot: PcapSnapshot | undefined;
+ resources: readonly string[];
+ reason: string;
+ capturedOutput: PcapCommandCleanup["capturedOutput"];
+ }
>();
constructor(options: PcapProviderOptions = {}) {
this.environment = options.environment ?? process.env;
+ this.launcher = options.launcher;
}
- private run(
+ private async run(
operation: PcapOperation,
command: string,
args: readonly string[],
@@ -98,7 +166,8 @@ export class PcapProvider {
snapshot?: PcapSnapshot,
inputPath?: string,
) {
- return runPcapCommand({
+ let cleanup: PcapCommandCleanup | undefined;
+ const output = await runPcapCommand({
operation,
command,
arguments: args,
@@ -115,10 +184,33 @@ export class PcapProvider {
...(inputPath === undefined || this.nativeBound === undefined
? {}
: { nativeBound: this.nativeBound }),
- retain: (owner, resources) => {
- this.commands.set(owner, { snapshot, resources });
+ ...(this.launcher === undefined ? {} : { launcher: this.launcher }),
+ retain: (owner, resources, reason, capturedOutput) => {
+ this.commands.set(owner, {
+ snapshot,
+ resources,
+ reason,
+ capturedOutput,
+ });
+ cleanup = { reason, resources, capturedOutput };
},
});
+ if (inputPath === undefined && cleanup !== undefined)
+ throw new ProviderCleanupError(
+ "pcap",
+ cleanup.resources,
+ {
+ reason: cleanup.reason,
+ previous_error: null,
+ captured_output: {
+ stdout: output.stdout,
+ stderr: output.stderr,
+ truncated: cleanup.capturedOutput.truncated,
+ },
+ },
+ { operation },
+ );
+ return output;
}
private nativeBound: Parameters<typeof runPcapCommand>[0]["nativeBound"];
@@ -203,6 +295,7 @@ export class PcapProvider {
diagnostics: client,
};
} catch (cause) {
+ if (cause instanceof ProviderCleanupError) throw cause;
if (signal?.aborted || cause instanceof AnalysisCancelledError)
throw new AnalysisCancelledError("inspect_pcap_client", { cause });
if (cause instanceof PcapConfigurationFailure)
@@ -324,6 +417,9 @@ export class PcapProvider {
request.input.path,
identity,
signal,
+ request,
+ provider,
+ client,
);
const data = pcapResultSchemas.inspect_pcap_capture.parse({
client,
@@ -363,7 +459,13 @@ export class PcapProvider {
partialObservation: createPcapEvidence(request, result.value),
}),
);
- if (snapshot !== undefined) {
+ const pendingCommand =
+ snapshot === undefined
+ ? undefined
+ : [...this.commands.values()].find(
+ (command) => command.snapshot === snapshot,
+ );
+ if (snapshot !== undefined && pendingCommand === undefined) {
const cleanup = await this.discard(snapshot);
if (signal?.aborted && result.ok)
result = err(
@@ -387,6 +489,43 @@ export class PcapProvider {
),
);
}
+ if (
+ snapshot !== undefined &&
+ pendingCommand !== undefined &&
+ (result.ok || !(result.error instanceof ProviderCleanupError))
+ ) {
+ const partialObservation = result.ok
+ ? createPcapEvidence(request, result.value)
+ : result.error.partialObservation;
+ const capturedOutput = result.ok
+ ? undefined
+ : result.error.capturedOutput;
+ result = err(
+ new ProviderCleanupError(
+ "pcap",
+ [...new Set([snapshot.path, ...pendingCommand.resources])],
+ {
+ reason: pendingCommand.reason,
+ previous_error: result.ok
+ ? null
+ : projectAnalysisError(result.error),
+ ...(capturedOutput === undefined &&
+ pendingCommand.capturedOutput.stdout === "" &&
+ pendingCommand.capturedOutput.stderr === ""
+ ? {}
+ : {
+ captured_output: {
+ ...(capturedOutput ?? pendingCommand.capturedOutput),
+ },
+ }),
+ },
+ {
+ operation: request.operation,
+ ...(partialObservation === undefined ? {} : { partialObservation }),
+ },
+ ),
+ );
+ }
return result;
}
@@ -396,6 +535,13 @@ export class PcapProvider {
path: string,
identity: { bytes: number; sha256: string },
signal: AbortSignal | undefined,
+ request: PcapRequest,
+ provider: {
+ readonly id: string;
+ readonly name: string;
+ readonly version: string | null;
+ },
+ client: unknown,
) {
const info = await this.run(
"inspect_pcap_capture",
@@ -411,6 +557,34 @@ export class PcapProvider {
throw new AnalysisProtocolError(
"capinfos SHA256 disagrees with the owned capture snapshot",
);
+ if (info.cleanup !== undefined)
+ throw pcapCommandCleanupError(
+ request.operation,
+ snapshot.path,
+ info.cleanup,
+ info,
+ pcapPartialEvidence(
+ request,
+ provider,
+ path,
+ identity,
+ {
+ client,
+ target: { path, ...identity },
+ stage: "capinfos-complete-tshark-not-run",
+ file_type: metadata.fileType,
+ encapsulation: metadata.encapsulation,
+ packet_count: metadata.packetCount,
+ capture_start: metadata.captureStart,
+ capture_end: metadata.captureEnd,
+ capture_duration_seconds: metadata.captureDurationSeconds,
+ snapshot_length: metadata.snapshotLength,
+ },
+ [
+ "capinfos output was parsed from a complete bounded stream, but process-tree cleanup remains unconfirmed; tshark was not run",
+ ],
+ ),
+ );
const dissected = await this.run(
"inspect_pcap_capture",
selection.tshark.command,
@@ -435,7 +609,7 @@ export class PcapProvider {
throw new AnalysisProtocolError(
`capinfos counted ${String(metadata.packetCount)} packets but tshark returned ${String(fields.packetCount)}`,
);
- return {
+ const captured = {
result: {
target: { path, ...identity },
file_type: metadata.fileType,
@@ -487,5 +661,26 @@ export class PcapProvider {
: []),
],
};
+ if (dissected.cleanup !== undefined) {
+ const full = pcapResultSchemas.inspect_pcap_capture.parse({
+ client,
+ ...captured.result,
+ });
+ throw pcapCommandCleanupError(
+ request.operation,
+ snapshot.path,
+ dissected.cleanup,
+ dissected,
+ pcapPartialEvidence(
+ request,
+ provider,
+ path,
+ identity,
+ full,
+ captured.limitations,
+ ),
+ );
+ }
+ return captured;
}
}
diff --git a/tests/boundary/pcap/pcapCommand.test.ts b/tests/boundary/pcap/pcapCommand.test.ts
index cda8dbf77..d8d52b6bb 100644
--- a/tests/boundary/pcap/pcapCommand.test.ts
+++ b/tests/boundary/pcap/pcapCommand.test.ts
@@ -2,6 +2,9 @@ import { setTimeout as delay } from "node:timers/promises";
import { expect, it, onTestFinished, vi } from "vitest";
import { runPcapCommand } from "../../../src/pcap/PcapCommand.js";
import * as ownership from "../../../src/process/ProcessOwnershipObservation.js";
+import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js";
+import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js";
+import type { ProviderProcessSupervisor } from "../../../src/process/ProviderProcess.js";
it("does not spend the tool execution deadline preparing native ownership", async () => {
const original = ownership.prepareProcessOwnershipInspection;
@@ -51,3 +54,55 @@ it("does not spend the tool execution deadline preparing native ownership", asyn
if (!result.ok) throw result.error;
expect(result.value.stdout.trim()).toBe(process.version);
});
+
+it("returns complete stdout while retaining unresolved process cleanup", async () => {
+ let retained:
+ | {
+ readonly resources: readonly string[];
+ readonly reason: string;
+ readonly owner: ProviderProcessSupervisor;
+ }
+ | undefined;
+ let mayClean = false;
+ const output = await runPcapCommand({
+ operation: "inspect_pcap_capture",
+ command: process.execPath,
+ arguments: ["-e", 'process.stdout.write("complete producer output")'],
+ limits: { timeoutMs: 5000, maxOutputBytes: 1024 },
+ environment: {},
+ signal: undefined,
+ launcher: async (spawn) => {
+ const launched = await spawnOwnedProviderProcess(spawn);
+ const cleanup =
+ launched.cleanup ??
+ (() => cleanupOwnedProcessGroup(launched.ownership));
+ return {
+ ...launched,
+ cleanup: async () => {
+ const stopped = await cleanup();
+ if (!mayClean) throw new Error("injected cleanup uncertainty");
+ return stopped;
+ },
+ };
+ },
+ retain: (owner, resources, reason) => {
+ retained = { owner, resources, reason };
+ },
+ });
+ expect(output.stdout).toBe("complete producer output");
+ expect(output.cleanup).toMatchObject({
+ reason: "injected cleanup uncertainty",
+ capturedOutput: {
+ stdout: "complete producer output",
+ stderr: "",
+ truncated: false,
+ },
+ });
+ expect(retained?.resources).toContain(
+ `process-group:${retained?.owner.launch.ownership?.processGroupId}`,
+ );
+ expect(retained?.reason).toBe("injected cleanup uncertainty");
+ if (retained === undefined) throw new Error("Cleanup owner was not retained");
+ mayClean = true;
+ expect((await retained.owner.stop()).status).not.toBe("incomplete");
+});
diff --git a/tests/boundary/pcap/pcapProvider.test.ts b/tests/boundary/pcap/pcapProvider.test.ts
index fcb299ade..0f5f5fbd2 100644
--- a/tests/boundary/pcap/pcapProvider.test.ts
+++ b/tests/boundary/pcap/pcapProvider.test.ts
@@ -13,6 +13,9 @@ import {
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { holdFirstFileRead } from "../../support/holdFirstFileRead.js";
import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js";
+import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js";
+import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js";
+import type { PcapCommandLauncher } from "../../../src/pcap/PcapCommand.js";
const it = test.skipIf(process.platform !== "linux");
@@ -97,11 +100,14 @@ exit 0
return { tshark, capinfos };
};
-const providerWith = async (options: { failCapinfos?: boolean } = {}) => {
+const providerWith = async (
+ options: { failCapinfos?: boolean; launcher?: PcapCommandLauncher } = {},
+) => {
const root = await createTestTempDirectory("rea-pcap-");
const { tshark, capinfos } = await writeStubTools(root, options);
const provider = new PcapProvider({
environment: { REA_TSHARK_COMMAND: tshark, REA_CAPINFOS_COMMAND: capinfos },
+ ...(options.launcher === undefined ? {} : { launcher: options.launcher }),
});
onTestFinished(() => provider.close());
return {
@@ -171,6 +177,78 @@ it("inspects a capture with endpoints, DNS, TLS SNI, and HTTP", async () => {
expect(result.coverage).toBe("complete");
});
+it.each(["capinfos", "tshark"] as const)(
+ "retains parsed output and cleanup ownership when %s cleanup is unresolved",
+ async (stage) => {
+ let mayClean = false;
+ let producerLaunches = 0;
+ const launcher: PcapCommandLauncher = async (spawn) => {
+ const selectedStage = spawn.arguments.includes("-M")
+ ? "capinfos"
+ : spawn.arguments.includes("-T")
+ ? "tshark"
+ : undefined;
+ const launched = await spawnOwnedProviderProcess(spawn);
+ if (selectedStage !== stage) return launched;
+ producerLaunches += 1;
+ const cleanupOwned =
+ launched.cleanup ??
+ (() => cleanupOwnedProcessGroup(launched.ownership));
+ return {
+ ...launched,
+ cleanup: async () => {
+ const cleaned = await cleanupOwned();
+ if (!mayClean) throw new Error("injected retained cleanup failure");
+ return cleaned;
+ },
+ };
+ };
+ const { provider, root } = await providerWith({ launcher });
+ const path = await captureFixture(root);
+ const result = await provider.execute({
+ operation: "inspect_pcap_capture",
+ input: { path },
+ });
+ if (result.ok) throw new Error("Expected cleanup to remain unresolved");
+ expect(result.error).toMatchObject({ cleanupIncomplete: true });
+ expect(result.error.diagnostics).toMatchObject({
+ reason: "injected retained cleanup failure",
+ previous_error: null,
+ captured_output: {
+ stdout: stage === "capinfos" ? CAPINFOS_OUTPUT : TSHARK_JSON,
+ truncated: false,
+ },
+ });
+ expect(result.error.partialObservation).toMatchObject({
+ normalized_result:
+ stage === "capinfos"
+ ? {
+ stage: "capinfos-complete-tshark-not-run",
+ packet_count: 3,
+ }
+ : {
+ coverage: "complete",
+ packet_count: 3,
+ endpoints: expect.any(Array),
+ },
+ });
+ expect(producerLaunches).toBe(1);
+ expect(await readFile(path)).toEqual(Buffer.from("pcap-fixture-bytes"));
+ expect(
+ result.error.cleanupResources.some((resource) =>
+ resource.endsWith("/capture.pcap"),
+ ),
+ ).toBe(true);
+ expect(
+ result.error.cleanupResources.some((resource) =>
+ resource.startsWith("process-group:"),
+ ),
+ ).toBe(true);
+ mayClean = true;
+ await provider.close();
+ },
+);
+
it("reports a non-capture file as unsupported", async () => {
const { provider, root } = await providerWith({ failCapinfos: true });
const failPath = join(root, "fail.pcap");