Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(lldb): preserve observed failures through interrupted cleanup

morlutocc73587ec499
diff --git a/src/native/LldbCallTracer.ts b/src/native/LldbCallTracer.ts
index 9afce7cba..97054ce25 100644
--- a/src/native/LldbCallTracer.ts
+++ b/src/native/LldbCallTracer.ts
@@ -735,13 +735,28 @@ export class LldbCallTracer implements NativeCallTracer {
}),
);
}
- return ok({
+ const trace: NativeCallTrace = {
debugger: { ...lldb, version: output.version },
run: output,
stdout: stdout.value,
stderr: stderr.value,
terminated,
- });
+ };
+ if (exited.lateTimeout === true)
+ return err(
+ new AnalysisTimeoutError(
+ OPERATION,
+ input.duration_ms + DEADLINE_GRACE_MS,
+ {
+ partialObservation: partialObservationFromTrace(
+ request,
+ trace,
+ "timeout",
+ ),
+ },
+ ),
+ );
+ return ok(trace);
}
}
@@ -1056,6 +1071,8 @@ type LldbExit = LldbOutcome & {
readonly targetLaunched: boolean;
readonly targetIdentity: string | undefined;
readonly cleanupFailure: string | undefined;
+ /** Internal deadline elapsed during cleanup after a physical LLDB exit. */
+ readonly lateTimeout?: true;
readonly cleanupOwner?: ProviderProcessSupervisor;
};
@@ -1095,17 +1112,34 @@ const runLldb = async (
let targetIdentity: string | undefined;
let supervisor: ProviderProcessSupervisor | undefined;
let retained = false;
+ let processFailureBeforeInterruption:
+ | {
+ readonly exitCode: number | null;
+ readonly output: string;
+ }
+ | undefined;
const finish = async (outcome: LldbOutcome): Promise<LldbExit> => {
+ // Select the primary result once at the owner boundary, before cleanup.
+ // An exit observed before interruption outranks an interrupted wait, while
+ // an already selected exit keeps its fuller snapshot and diagnostics.
+ const selectedOutcome =
+ outcome.kind !== "exited" &&
+ processFailureBeforeInterruption !== undefined
+ ? { kind: "exited" as const, ...processFailureBeforeInterruption }
+ : outcome;
const stopped = await supervisor?.stop();
retained = stopped?.status === "incomplete";
+ const lateTimeout =
+ selectedOutcome.kind === "exited" && deadline.interruption === "timeout"
+ ? true
+ : undefined;
return {
- ...(deadline.interruption === undefined
- ? outcome
- : { kind: deadline.interruption }),
+ ...selectedOutcome,
targetLaunched: supervisor !== undefined,
targetIdentity,
cleanupFailure:
stopped?.status === "incomplete" ? stopped.reason : undefined,
+ ...(lateTimeout === undefined ? {} : { lateTimeout }),
...(retained && supervisor !== undefined
? { cleanupOwner: supervisor }
: {}),
@@ -1126,7 +1160,24 @@ const runLldb = async (
ownsProcessLifetime: true,
cleanup: () => cleanupOwnedProcessGroup(launched.ownership),
},
- { maxDiagnosticBytes: DIAGNOSTIC_BYTES },
+ {
+ maxDiagnosticBytes: DIAGNOSTIC_BYTES,
+ onDiagnostic: (event) => {
+ // Latch failure at the process event; a later snapshot cannot order it against abort.
+ if (
+ event.type === "exit" &&
+ deadline.interruption === undefined &&
+ (event.signal !== null || (event.code !== null && event.code !== 0))
+ )
+ processFailureBeforeInterruption ??= {
+ exitCode: event.code,
+ output:
+ `${event.snapshot.stdout.text}${event.snapshot.stderr.text}`.slice(
+ -DIAGNOSTIC_BYTES,
+ ),
+ };
+ },
+ },
);
let identityAckWritten = false;
while (!(await supervisor.waitForOutputClose(20))) {
@@ -1148,8 +1199,9 @@ const runLldb = async (
identityAckWritten = true;
}
const interruption = deadline.interruption;
- if (interruption !== undefined)
+ if (interruption !== undefined) {
return await finish({ kind: interruption });
+ }
await waitForAbortableDelay(20, deadline.signal);
}
const snapshot = supervisor.snapshot();
diff --git a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts
index 8e124f509..a2b653cd6 100644
--- a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts
+++ b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts
@@ -1,5 +1,6 @@
import {
access,
+ mkdtemp,
readFile,
rm,
writeFile,
@@ -8,11 +9,17 @@ import {
type FileHandle,
} from "node:fs/promises";
import { dirname } from "node:path";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
import { describe, expect, it, onTestFinished, vi } from "vitest";
import { z } from "zod";
-import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js";
+import {
+ AnalysisCancelledError,
+ AnalysisCapabilityUnavailableError,
+ AnalysisTimeoutError,
+} from "../../../../src/domain/analysisErrorCore.js";
import { EvidenceIntegrityError } from "../../../../src/domain/evidenceErrors.js";
import { ArtifactFileCleanupFailure } from "../../../../src/artifacts/ArtifactFileOperation.js";
import { resolvedAnalysisError } from "../../../../src/domain/analysisErrorBase.js";
@@ -26,6 +33,12 @@ import { NativeCommandFailure } from "../../../../src/native/CommandRunner.js";
import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js";
import { createDeferred } from "../../../fixtures/binarySession.js";
import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js";
+import {
+ ProviderProcessSupervisor,
+ type ProviderProcessStopOptions,
+ type ProviderProcessStopResult,
+} from "../../../../src/process/ProviderProcess.js";
+import { ProviderStartupDeadline } from "../../../../src/process/ProviderDeadline.js";
import {
NativeFixtureRunner,
nativeMachoTarget,
@@ -420,6 +433,309 @@ describe("LLDB failure retention through the production tracer", () => {
});
});
+describe("LLDB primary outcome during owned process cleanup", () => {
+ it
+ .skipIf(process.platform === "win32")
+ .each(["caller cancellation", "deadline timeout"] as const)(
+ "preserves a physical bridge failure and no-target proof when %s arrives during successful process cleanup",
+ async (interruption) => {
+ const fixtureRoot = await mkdtemp(
+ join(tmpdir(), "rea-lldb-primary-outcome-"),
+ );
+ const fakeLldb = join(fixtureRoot, "fake-lldb");
+ await writeFile(
+ fakeLldb,
+ `#!${process.execPath}\nconst fs = require("node:fs");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nfs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture physical bridge failure", target_process_created: false }));\nprocess.exit(17);\n`,
+ { mode: 0o700 },
+ );
+
+ const controller = new AbortController();
+ const stopEntered = createDeferred<void>();
+ const stopReleased = createDeferred<void>();
+ let stopping = false;
+ const deadlineSpy =
+ interruption === "deadline timeout"
+ ? vi
+ .spyOn(ProviderStartupDeadline.prototype, "interruption", "get")
+ .mockImplementation(() => (stopping ? "timeout" : undefined))
+ : undefined;
+ let observedExit:
+ | ReturnType<ProviderProcessSupervisor["observedExit"]>
+ | undefined;
+ let observedStop: ProviderProcessStopResult | undefined;
+ const originalStop = ProviderProcessSupervisor.prototype.stop;
+ const stopSpy = vi
+ .spyOn(ProviderProcessSupervisor.prototype, "stop")
+ .mockImplementation(async function (
+ this: ProviderProcessSupervisor,
+ options?: ProviderProcessStopOptions,
+ ): Promise<ProviderProcessStopResult> {
+ observedExit = this.observedExit();
+ stopping = true;
+ stopEntered.resolve(undefined);
+ await stopReleased.promise;
+ observedStop = await originalStop.call(this, options);
+ return observedStop;
+ });
+ const tracer = new LldbCallTracer({}, undefined, async () =>
+ ok({ path: fakeLldb, sha256: "c".repeat(64) }),
+ );
+
+ try {
+ const pending = tracer.trace(
+ request,
+ interruption === "caller cancellation"
+ ? controller.signal
+ : undefined,
+ );
+ await stopEntered.promise;
+ expect(observedExit).toEqual({ code: 17, signal: null });
+ if (interruption === "caller cancellation")
+ controller.abort(new Error("abort during final cleanup"));
+ stopReleased.resolve(undefined);
+ const result = await pending;
+
+ expect(observedStop?.status).toMatch(
+ /^(already-exited|verified-cleanup|terminated|killed)$/u,
+ );
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Expected the bridge target error");
+ expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError);
+ if (!(result.error instanceof AnalysisCapabilityUnavailableError))
+ return;
+ expect(result.error.reason).toBe(
+ "lldb-target-unloadable: fixture physical bridge failure",
+ );
+ expect(result.error.cleanup).toBeUndefined();
+ expect(result.error.cleanupIncomplete).toBe(false);
+ expect(result.error.cleanupResources).toEqual([]);
+ expect(result.error.partialObservation).toMatchObject({
+ coverage: { status: "partial", reason: "tracer-failure" },
+ process: { pid: null },
+ });
+ } finally {
+ stopReleased.resolve(undefined);
+ stopSpy.mockRestore();
+ deadlineSpy?.mockRestore();
+ await tracer.close();
+ await rm(fixtureRoot, { recursive: true, force: true });
+ }
+ },
+ );
+});
+
+describe("LLDB process exit while output is still open", () => {
+ it.skipIf(process.platform === "win32")(
+ "keeps a nonzero leader exit observed before cancellation while inherited output is still open",
+ async () => {
+ const fixtureRoot = await mkdtemp(
+ join(tmpdir(), "rea-lldb-exit-before-cancel-"),
+ );
+ const fakeLldb = join(fixtureRoot, "fake-lldb");
+ await writeFile(
+ fakeLldb,
+ `#!${process.execPath}\nconst fs = require("node:fs");\nconst { spawn } = require("node:child_process");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nconst child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: ["ignore", "inherit", "inherit"] });\nchild.once("error", () => process.exit(92));\nchild.once("spawn", () => { fs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture leader failure before drain", target_process_created: false })); process.exit(17); });\n`,
+ { mode: 0o700 },
+ );
+
+ const controller = new AbortController();
+ const drainObserved = createDeferred<void>();
+ const drainReleased = createDeferred<void>();
+ const originalWait =
+ ProviderProcessSupervisor.prototype.waitForOutputClose;
+ const drainSpy = vi
+ .spyOn(ProviderProcessSupervisor.prototype, "waitForOutputClose")
+ .mockImplementation(async function (
+ this: ProviderProcessSupervisor,
+ timeoutMs?: number,
+ signal?: AbortSignal,
+ ): Promise<boolean> {
+ const closed = await originalWait.call(this, timeoutMs, signal);
+ if (!closed && !stopping && this.observedExit()?.code === 17) {
+ drainObserved.resolve(undefined);
+ await drainReleased.promise;
+ }
+ return closed;
+ });
+ let stopping = false;
+ let stopReceipt: ProviderProcessStopResult | undefined;
+ const originalStop = ProviderProcessSupervisor.prototype.stop;
+ const stopSpy = vi
+ .spyOn(ProviderProcessSupervisor.prototype, "stop")
+ .mockImplementation(async function (
+ this: ProviderProcessSupervisor,
+ options?: ProviderProcessStopOptions,
+ ): Promise<ProviderProcessStopResult> {
+ stopping = true;
+ stopReceipt = await originalStop.call(this, options);
+ return stopReceipt;
+ });
+ const tracer = new LldbCallTracer({}, undefined, async () =>
+ ok({ path: fakeLldb, sha256: "d".repeat(64) }),
+ );
+
+ try {
+ const pending = tracer.trace(request, controller.signal);
+ await drainObserved.promise;
+ controller.abort(new Error("abort after leader exit"));
+ drainReleased.resolve(undefined);
+ const result = await pending;
+
+ expect(stopReceipt?.status).toMatch(
+ /^(already-exited|verified-cleanup|terminated|killed)$/u,
+ );
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Expected the bridge target error");
+ expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError);
+ if (!(result.error instanceof AnalysisCapabilityUnavailableError))
+ return;
+ expect(result.error.reason).toBe(
+ "lldb-target-unloadable: fixture leader failure before drain",
+ );
+ expect(result.error.cleanupIncomplete).toBe(false);
+ expect(result.error.cleanupResources).toEqual([]);
+ } finally {
+ drainReleased.resolve(undefined);
+ drainSpy.mockRestore();
+ stopSpy.mockRestore();
+ await tracer.close();
+ await rm(fixtureRoot, { recursive: true, force: true });
+ }
+ },
+ );
+});
+
+describe("LLDB process exit at output-close loop completion", () => {
+ it.skipIf(process.platform === "win32")(
+ "keeps a nonzero exit when output closure and cancellation meet at loop exit",
+ async () => {
+ const fixtureRoot = await mkdtemp(
+ join(tmpdir(), "rea-lldb-closed-output-cancel-"),
+ );
+ const fakeLldb = join(fixtureRoot, "fake-lldb");
+ await writeFile(
+ fakeLldb,
+ `#!${process.execPath}\nconst fs = require("node:fs");\nconst command = process.argv.at(-1);\nif (!command?.startsWith("rea_trace ")) process.exit(91);\nconst config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));\nfs.writeFileSync(config.result_path, JSON.stringify({ status: "target-error", error: "fixture exit before closed-output cancellation", target_process_created: false }));\nprocess.exit(17);\n`,
+ { mode: 0o700 },
+ );
+
+ const controller = new AbortController();
+ let interruptionApplied = false;
+ let stopReceipt: ProviderProcessStopResult | undefined;
+ const originalWait =
+ ProviderProcessSupervisor.prototype.waitForOutputClose;
+ const waitSpy = vi
+ .spyOn(ProviderProcessSupervisor.prototype, "waitForOutputClose")
+ .mockImplementation(async function (
+ this: ProviderProcessSupervisor,
+ timeoutMs?: number,
+ signal?: AbortSignal,
+ ): Promise<boolean> {
+ const closed = await originalWait.call(this, timeoutMs, signal);
+ if (
+ !interruptionApplied &&
+ closed &&
+ this.observedExit()?.code === 17
+ ) {
+ interruptionApplied = true;
+ controller.abort(new Error("abort at output-close loop exit"));
+ }
+ return closed;
+ });
+ const originalStop = ProviderProcessSupervisor.prototype.stop;
+ const stopSpy = vi
+ .spyOn(ProviderProcessSupervisor.prototype, "stop")
+ .mockImplementation(async function (
+ this: ProviderProcessSupervisor,
+ options?: ProviderProcessStopOptions,
+ ): Promise<ProviderProcessStopResult> {
+ stopReceipt = await originalStop.call(this, options);
+ return stopReceipt;
+ });
+ const tracer = new LldbCallTracer({}, undefined, async () =>
+ ok({ path: fakeLldb, sha256: "f".repeat(64) }),
+ );
+
+ try {
+ const result = await tracer.trace(request, controller.signal);
+
+ expect(interruptionApplied).toBe(true);
+ expect(stopReceipt?.status).toMatch(
+ /^(already-exited|verified-cleanup|terminated|killed)$/u,
+ );
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Expected the bridge target error");
+ expect(result.error).toBeInstanceOf(AnalysisCapabilityUnavailableError);
+ if (!(result.error instanceof AnalysisCapabilityUnavailableError))
+ return;
+ expect(result.error.reason).toBe(
+ "lldb-target-unloadable: fixture exit before closed-output cancellation",
+ );
+ expect(result.error.cleanupIncomplete).toBe(false);
+ expect(result.error.cleanupResources).toEqual([]);
+ } finally {
+ waitSpy.mockRestore();
+ stopSpy.mockRestore();
+ await tracer.close();
+ await rm(fixtureRoot, { recursive: true, force: true });
+ }
+ },
+ );
+});
+
+describe("LLDB late timeout publication", () => {
+ it("converts a late deadline to timeout at successful-result publication", async () => {
+ let runtime: string | undefined;
+ const tracer = new LldbCallTracer(
+ {},
+ async (_executable, arguments_) => {
+ const { config } = await configFromArguments(arguments_);
+ runtime = dirname(config.result_path);
+ await writeFile(
+ config.observation_path,
+ `${JSON.stringify({ kind: "event", event: EVENT })}\n`,
+ );
+ await writeFile(config.stdout_capture_path, "");
+ await writeFile(config.stderr_capture_path, "");
+ await writeFile(
+ config.result_path,
+ JSON.stringify(successfulTracerOutput),
+ );
+ return {
+ kind: "exited",
+ targetLaunched: false,
+ exitCode: 0,
+ output: "",
+ targetIdentity: undefined,
+ cleanupFailure: undefined,
+ lateTimeout: true,
+ };
+ },
+ async () => ok(fakeTool),
+ );
+
+ const result = await tracer.trace(request);
+ if (runtime === undefined) throw new Error("LLDB fixture did not run");
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Expected the late deadline to be retained");
+ expect(result.error).toBeInstanceOf(AnalysisTimeoutError);
+ if (!(result.error instanceof AnalysisTimeoutError)) return;
+ expect(result.error.timeoutMs).toBeGreaterThan(request.input.duration_ms);
+ expect(result.error.cleanup).toBeUndefined();
+ expect(result.error.cleanupIncomplete).toBe(false);
+ expect(result.error.partialObservation).toMatchObject({
+ coverage: { status: "partial", reason: "timeout" },
+ process: {
+ outcome: "exited",
+ exit_status: 0,
+ terminated: true,
+ },
+ events: [EVENT],
+ });
+ await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" });
+ });
+});
+
describe("LLDB pre-target phase proof", () => {
it("releases a proven pre-target integrity rejection and allows tracer reuse", async () => {
let launches = 0;