Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(process): retain child scope and bound native exit waits (#5280)

gensokyo-dev71ae02c6efce
diff --git a/src/process/ProcessLifecycle.ts b/src/process/ProcessLifecycle.ts
index 9c0d0f52e..df6619016 100644
--- a/src/process/ProcessLifecycle.ts
+++ b/src/process/ProcessLifecycle.ts
@@ -10,6 +10,11 @@ export type ProcessLifecycleObservation =
export const PROCESS_CLEANUP_VERIFICATION_GRACE_MS = 1_000;
export const PROCESS_CLEANUP_VERIFICATION_INTERVAL_MS = 25;
+/** One operation owns a lazily started native-exit waiting window. */
+export interface ProcessExitSettlementBudget {
+ deadline: number | undefined;
+}
+
export const processIdentityHasExited = (
identity: ProcessIdentityObservation | undefined,
): boolean =>
diff --git a/src/process/ProcessOwnership.exitScope.test.ts b/src/process/ProcessOwnership.exitScope.test.ts
new file mode 100644
index 000000000..64df933d6
--- /dev/null
+++ b/src/process/ProcessOwnership.exitScope.test.ts
@@ -0,0 +1,190 @@
+import { performance } from "node:perf_hooks";
+import { expect, it, vi } from "vitest";
+import {
+ cleanupOwnedProcessGroup,
+ verifyNoTokenOwnedProcesses,
+ type ProcessIdentityObservation,
+ type ProcessOwnershipHost,
+ type ProcessRunTokenObservation,
+ type ProcessTableEntry,
+} from "./ProcessOwnership.js";
+import { ProcessOwnershipEvidenceOwner } from "./ProcessOwnershipEvidence.js";
+
+const row = (pid: number, parentPid: number): ProcessTableEntry => ({
+ pid,
+ parentPid,
+ processGroupId: pid,
+ state: "R",
+ command: "/fixture/node",
+});
+const identity = (
+ process: ProcessTableEntry,
+ exiting: boolean,
+): ProcessIdentityObservation => ({
+ state: "readable",
+ identity: `start:${process.pid}`,
+ lifecycle: {
+ state: "observed",
+ lifecycle: exiting ? "exiting" : "non-zombie",
+ },
+ relation: {
+ state: "observed",
+ parentPid: process.parentPid,
+ processGroupId: process.processGroupId,
+ },
+});
+
+it.each(["readable", "retried", "reused", "retained-parent"] as const)(
+ "keeps child scope tied to its native generation after parent exit (%s)",
+ async (mode) => {
+ const parent = row(200, 1);
+ const child = row(300, parent.pid);
+ const owner = new ProcessOwnershipEvidenceOwner();
+ if (mode === "retained-parent")
+ owner.reconcile(
+ [
+ {
+ process: parent,
+ state: "same-generation",
+ identityBefore: identity(parent, false),
+ token: { state: "readable", runId: "run" },
+ identityAfter: identity(parent, false),
+ },
+ ],
+ {
+ runId: "run",
+ listedRows: 1,
+ listedPids: new Set([parent.pid]),
+ leaderPid: 100,
+ processGroupIds: new Set([100]),
+ },
+ );
+ const priorSnapshots = owner.coverage.snapshotCount;
+ let parentReads = 0;
+ let childTokenReads = 0;
+ let reparented = false;
+ const host: ProcessOwnershipHost = {
+ listProcesses: async () => [parent, child],
+ environment: async (pid) =>
+ pid === parent.pid ? { REA_PROCESS_RUN_ID: "run" } : {},
+ runTokens: async (selected) =>
+ new Map<number, ProcessRunTokenObservation>(
+ selected.map((process) => [
+ process.pid,
+ process.pid === parent.pid
+ ? mode === "retained-parent"
+ ? { state: "unavailable", reason: "EACCES" }
+ : { state: "readable", runId: "run" }
+ : ++childTokenReads === 1 && mode !== "readable"
+ ? { state: "unavailable", reason: "EACCES" }
+ : { state: "readable", runId: undefined },
+ ]),
+ ),
+ processIdentities: async (selected) =>
+ new Map<number, ProcessIdentityObservation>(
+ selected.map((process) => {
+ if (process.pid === parent.pid && ++parentReads > 2) {
+ reparented = true;
+ return [
+ process.pid,
+ { state: "absent", reason: "not-in-snapshot" },
+ ];
+ }
+ const observed =
+ process.pid === child.pid && reparented
+ ? { ...process, parentPid: 1 }
+ : process;
+ const current = identity(
+ observed,
+ process.pid === parent.pid && parentReads === 2,
+ );
+ return [
+ process.pid,
+ process.pid === child.pid &&
+ reparented &&
+ mode === "reused" &&
+ current.state === "readable"
+ ? { ...current, identity: "start:replacement" }
+ : current,
+ ];
+ }),
+ ),
+ signalGroup: vi.fn(),
+ };
+ const result = await verifyNoTokenOwnedProcesses("run", host, undefined, {
+ leaderPid: 100,
+ processGroupId: 100,
+ evidenceOwner: owner,
+ });
+ if (mode === "reused") {
+ // A replacement process cannot inherit the original child's history.
+ expect(result.cleaned).toBe(true);
+ expect(owner.candidates).not.toContainEqual(
+ expect.objectContaining({ pid: child.pid }),
+ );
+ } else {
+ // The original child stays live even though its parent has exited.
+ expect(result.cleaned).toBe(false);
+ expect(owner.candidates).toContainEqual(
+ expect.objectContaining({
+ pid: child.pid,
+ identity: "start:300",
+ applicability: "related-ownership-unresolved",
+ }),
+ );
+ }
+ expect(host.signalGroup).not.toHaveBeenCalled();
+ expect(owner.coverage.snapshotCount).toBe(priorSnapshots + 1);
+ },
+);
+
+it("shares native exit waiting before signaling healthy groups in one cleanup attempt", async () => {
+ const root = row(100, 1);
+ const rows = [
+ root,
+ row(200, root.pid),
+ row(300, root.pid),
+ row(400, root.pid),
+ ];
+ let clock = 0;
+ let firstSignalAt: number | undefined;
+ const nativeClock = vi
+ .spyOn(performance, "now")
+ .mockImplementation(() => clock);
+ const host: ProcessOwnershipHost = {
+ listProcesses: async () => rows,
+ environment: async () => ({ REA_PROCESS_RUN_ID: "run" }),
+ processIdentities: async (selected) => {
+ clock += 25;
+ return new Map<number, ProcessIdentityObservation>(
+ selected.map((process) => [
+ process.pid,
+ identity(process, process.pid !== root.pid),
+ ]),
+ );
+ },
+ signalGroup: vi.fn((group) => {
+ expect(group).toBe(root.pid);
+ firstSignalAt ??= clock;
+ }),
+ };
+ try {
+ const result = await cleanupOwnedProcessGroup(
+ {
+ runId: "run",
+ leaderPid: root.pid,
+ processGroupId: root.pid,
+ sweepTokenOwnedProcesses: true,
+ evidenceOwner: new ProcessOwnershipEvidenceOwner(),
+ },
+ host,
+ );
+ expect(result.cleaned).toBe(false);
+ expect(firstSignalAt).toBeDefined();
+ // Controlled native-observation time, not a wall-clock timeout: three
+ // stuck helpers must not each add another full second before root signal.
+ expect(firstSignalAt).toBeLessThan(2_000);
+ } finally {
+ nativeClock.mockRestore();
+ }
+});
diff --git a/src/process/ProcessOwnership.ts b/src/process/ProcessOwnership.ts
index 95a80b22d..c8a74a83a 100644
--- a/src/process/ProcessOwnership.ts
+++ b/src/process/ProcessOwnership.ts
@@ -22,6 +22,7 @@ import {
processIdentityHasExited,
processIdentityIsExiting,
type ProcessLifecycleObservation,
+ type ProcessExitSettlementBudget,
} from "./ProcessLifecycle.js";
const execFileAsync = promisify(execFile);
@@ -117,7 +118,7 @@ export type ProcessNativeRelationObservation =
| { readonly state: "unavailable"; readonly reason: string };
/** One caller-supplied process row validated across identity/token/identity reads. */
-export interface ProcessOwnershipFact {
+interface ProcessOwnershipFactSnapshot {
/** Caller-supplied candidate row; its parent/group may predate this read. */
readonly process: ProcessTableEntry;
/** Identity relation only; token ownership still requires matching readable runId. */
@@ -129,6 +130,11 @@ export interface ProcessOwnershipFact {
readonly identityAfter: ProcessIdentityObservation;
}
+export interface ProcessOwnershipFact extends ProcessOwnershipFactSnapshot {
+ /** Bracketed ownership proof before a later native exit observation. */
+ readonly beforeExitSettlement?: ProcessOwnershipFactSnapshot;
+}
+
/** Token-verified process lineage retained for one owned provider run. */
export interface OwnedProcessLineage {
readonly runId: string;
@@ -327,6 +333,7 @@ export const cleanupOwnedProcessGroup = async (
ownership: OwnedProcessGroup,
host: ProcessOwnershipHost = systemHost,
): Promise<ProcessCleanupResult> => {
+ const exitSettlement: ProcessExitSettlementBudget = { deadline: undefined };
const processTable = await readLiveProcessTable(host);
if (!processTable.available)
return {
@@ -337,6 +344,7 @@ export const cleanupOwnedProcessGroup = async (
ownership,
processTable.processes,
host,
+ exitSettlement,
);
if ("cleaned" in plan) return plan;
let signaled = false;
@@ -346,6 +354,7 @@ export const cleanupOwnedProcessGroup = async (
ownership,
host,
plan.tokenOwnedIdentities,
+ exitSettlement,
);
if ("cleaned" in revalidation)
return mergeUnverifiedObservations(revalidation, plan.unverified);
@@ -386,6 +395,7 @@ export const cleanupOwnedProcessGroup = async (
host,
ownership.captureBaseline,
relation,
+ exitSettlement,
);
while (!verification.cleaned) {
const remaining = deadline - performance.now();
@@ -398,6 +408,7 @@ export const cleanupOwnedProcessGroup = async (
host,
ownership.captureBaseline,
relation,
+ exitSettlement,
);
}
} catch (cause: unknown) {
@@ -429,6 +440,7 @@ export const verifyNoTokenOwnedProcesses = async (
host: ProcessOwnershipHost = systemHost,
captureBaseline?: ProcessOwnershipBaseline,
relation?: OwnershipSweepRelation,
+ exitSettlement: ProcessExitSettlementBudget = { deadline: undefined },
): Promise<ProcessCleanupResult> => {
const processTable = await readLiveProcessTable(host);
if (!processTable.available)
@@ -442,6 +454,7 @@ export const verifyNoTokenOwnedProcesses = async (
host,
captureBaseline,
relation,
+ exitSettlement,
);
if (scan.failures.length > 0)
return cleanupValidationFailure(scan.failures, scan.unverified);
@@ -555,6 +568,7 @@ const settleLauncherExit = async (
ownership: OwnedProcessGroup,
processes: readonly ProcessTableEntry[],
host: ProcessOwnershipHost,
+ exitSettlement: ProcessExitSettlementBudget,
): Promise<
| readonly ProcessTableEntry[]
| Extract<ProcessCleanupResult, { readonly cleaned: false }>
@@ -571,6 +585,8 @@ const settleLauncherExit = async (
[launcher],
host,
initialIdentities,
+ undefined,
+ exitSettlement,
);
const identity = identities.get(launcher.pid);
const initialIdentity = initialIdentities.get(launcher.pid);
@@ -647,8 +663,14 @@ const createOwnedCleanupPlan = async (
ownership: OwnedProcessGroup,
initialProcesses: readonly ProcessTableEntry[],
host: ProcessOwnershipHost,
+ exitSettlement: ProcessExitSettlementBudget,
): Promise<OwnedCleanupPlan | ProcessCleanupResult> => {
- const processes = await settleLauncherExit(ownership, initialProcesses, host);
+ const processes = await settleLauncherExit(
+ ownership,
+ initialProcesses,
+ host,
+ exitSettlement,
+ );
if ("cleaned" in processes) return processes;
const relationSnapshot = await readProcessRelationSnapshot(
ownership,
@@ -676,6 +698,7 @@ const createOwnedCleanupPlan = async (
? {}
: { sampledProcessGroupIds: [...sampledProcessGroupIds] }),
},
+ exitSettlement,
)
: { owned: [], failures: [], unverified: [], identities: new Map() };
const tokenOwnedGroupIds = new Set(
@@ -764,6 +787,7 @@ const createOwnedCleanupPlan = async (
ownership.runId,
host,
tokenOwned.identities,
+ exitSettlement,
);
if (failures.length === 0 && unverifiableSampledGroups.has(processGroupId))
unresolved.push(
@@ -839,6 +863,7 @@ const revalidateOwnedProcessGroup = async (
ownership: OwnedProcessGroup,
host: ProcessOwnershipHost,
expectedIdentities: ReadonlyMap<number, string | null>,
+ exitSettlement: ProcessExitSettlementBudget,
): Promise<{ readonly empty: boolean } | ProcessCleanupResult> => {
let members: readonly ProcessTableEntry[];
try {
@@ -846,6 +871,7 @@ const revalidateOwnedProcessGroup = async (
ownership,
liveProcesses(await host.listProcesses()),
host,
+ exitSettlement,
);
if ("cleaned" in processes) return processes;
const relationSnapshot = await readProcessRelationSnapshot(
@@ -870,6 +896,7 @@ const revalidateOwnedProcessGroup = async (
ownership.runId,
host,
expectedIdentities,
+ exitSettlement,
);
if (failures.length > 0) return cleanupValidationFailure(failures);
if (
@@ -982,9 +1009,12 @@ const processOwnershipFailures = async (
runId: string,
host: ProcessOwnershipHost,
expectedIdentities: ReadonlyMap<number, string | null> = new Map(),
+ exitSettlement: ProcessExitSettlementBudget = { deadline: undefined },
): Promise<readonly ProcessOwnershipValidationFailure[]> => {
if (host.processIdentities !== undefined) {
- const facts = await readProcessOwnershipFacts(members, host);
+ const facts = await readProcessOwnershipFacts(members, host, undefined, {
+ exitSettlement,
+ });
const failures: ProcessOwnershipValidationFailure[] = [];
for (const fact of facts) {
if (processIdentityHasExited(fact.identityAfter)) continue;
@@ -1173,6 +1203,7 @@ const scanTokenOwnedProcesses = async (
host: ProcessOwnershipHost,
captureBaseline?: ProcessOwnershipBaseline,
relation?: OwnershipSweepRelation,
+ exitSettlement: ProcessExitSettlementBudget = { deadline: undefined },
): Promise<TokenOwnedProcessScan> => {
if (host.processIdentities !== undefined)
return scanWithOwnershipEvidence(
@@ -1181,6 +1212,7 @@ const scanTokenOwnedProcesses = async (
host,
captureBaseline,
relation,
+ exitSettlement,
);
const candidates = processes;
@@ -1273,6 +1305,7 @@ const scanWithOwnershipEvidence = async (
host: ProcessOwnershipHost,
captureBaseline: ProcessOwnershipBaseline | undefined,
relation: OwnershipSweepRelation | undefined,
+ exitSettlement: ProcessExitSettlementBudget,
): Promise<TokenOwnedProcessScan> => {
const evidenceOwner =
relation === undefined
@@ -1323,6 +1356,7 @@ const scanWithOwnershipEvidence = async (
undefined,
{
identityBefore: firstIdentities,
+ exitSettlement,
},
);
let facts = initialFacts;
@@ -1337,7 +1371,12 @@ const scanWithOwnershipEvidence = async (
.map(({ process }) => process);
if (retryRows.length === 0) break;
await delay(TOKEN_READ_RETRY_DELAY_MS);
- const retries = await readProcessOwnershipFacts(retryRows, host);
+ const retries = await readProcessOwnershipFacts(
+ retryRows,
+ host,
+ undefined,
+ { exitSettlement },
+ );
const retryByPid = new Map(retries.map((fact) => [fact.process.pid, fact]));
facts = facts.map((fact) => retryByPid.get(fact.process.pid) ?? fact);
}
diff --git a/src/process/ProcessOwnershipEvidence.ts b/src/process/ProcessOwnershipEvidence.ts
index fd93b7804..10752d6b3 100644
--- a/src/process/ProcessOwnershipEvidence.ts
+++ b/src/process/ProcessOwnershipEvidence.ts
@@ -165,32 +165,50 @@ export class ProcessOwnershipEvidenceOwner {
number,
Extract<ProcessIdentityObservation, { readonly state: "readable" }>
>();
- const scopeTokenMatchedPids = new Set<number>();
- for (const fact of options.scopeFacts ?? []) {
- const current = sameGeneration.get(fact.process.pid);
+ const scopeOwnedPids = new Set<number>();
+ // Keep the original batch together: retries can reparent a child after its
+ // verified parent exits. Historical parent proof is scope evidence only;
+ // it neither establishes current liveness nor authorizes a signal.
+ for (const observed of options.scopeFacts ?? facts) {
+ const fact = observed.beforeExitSettlement ?? observed;
if (
fact.state !== "same-generation" ||
- fact.identityAfter.state !== "readable" ||
- current?.identity !== fact.identityAfter.identity
+ fact.identityAfter.state !== "readable"
)
continue;
scopeGenerations.set(fact.process.pid, fact.identityAfter);
if (
fact.identityBefore.state === "readable" &&
- current?.identity === fact.identityBefore.identity
+ fact.identityAfter.identity === fact.identityBefore.identity
)
scopeBeforeGenerations.set(fact.process.pid, fact.identityBefore);
- if (fact.token.state === "readable" && fact.token.runId === options.runId)
- scopeTokenMatchedPids.add(fact.process.pid);
+ // Capture earlier proof before applying terminal observations below.
+ // Its generation must match this historical native observation exactly.
+ const previous = this.#candidates.get(fact.process.pid);
+ if (
+ (fact.token.state === "readable" &&
+ fact.token.runId === options.runId) ||
+ (previous?.applicability === "token-verified" &&
+ previous.identity === fact.identityAfter.identity)
+ )
+ scopeOwnedPids.add(fact.process.pid);
}
const related = (pid: number) => {
const current =
findScopeEvidence(pid, sameGeneration, tokenMatchedPids) ??
findScopeEvidence(pid, sameGenerationBefore, tokenMatchedPids);
if (current !== undefined) return current;
+ // The child must still be the generation observed in that original
+ // batch. A reused PID cannot inherit its predecessor's historical scope.
+ const original = scopeGenerations.get(pid);
+ if (
+ original === undefined ||
+ sameGeneration.get(pid)?.identity !== original.identity
+ )
+ return undefined;
const initial =
- findScopeEvidence(pid, scopeGenerations, scopeTokenMatchedPids) ??
- findScopeEvidence(pid, scopeBeforeGenerations, scopeTokenMatchedPids);
+ findScopeEvidence(pid, scopeGenerations, scopeOwnedPids) ??
+ findScopeEvidence(pid, scopeBeforeGenerations, scopeOwnedPids);
return initial === undefined
? undefined
: "previously-related-generation";
diff --git a/src/process/ProcessOwnershipObservation.ts b/src/process/ProcessOwnershipObservation.ts
index ec1ae9ddb..fd287a031 100644
--- a/src/process/ProcessOwnershipObservation.ts
+++ b/src/process/ProcessOwnershipObservation.ts
@@ -6,6 +6,7 @@ import {
processIdentityIsExiting,
PROCESS_CLEANUP_VERIFICATION_GRACE_MS,
PROCESS_CLEANUP_VERIFICATION_INTERVAL_MS,
+ type ProcessExitSettlementBudget,
} from "./ProcessLifecycle.js";
import { parseLinuxProcessStat } from "./LinuxProcessStat.js";
import { parseLinuxProcessEffectiveUid } from "./LinuxProcessStatus.js";
@@ -539,6 +540,7 @@ export const settleExitingProcessIdentities = async (
host: ProcessOwnershipHost,
initial: ReadonlyMap<number, ProcessIdentityObservation>,
signal?: AbortSignal,
+ budget: ProcessExitSettlementBudget = { deadline: undefined },
): Promise<ReadonlyMap<number, ProcessIdentityObservation>> => {
signal?.throwIfAborted();
let pending = processes.filter(({ pid }) =>
@@ -546,7 +548,8 @@ export const settleExitingProcessIdentities = async (
);
if (pending.length === 0) return initial;
const identities = new Map(initial);
- const deadline = performance.now() + PROCESS_CLEANUP_VERIFICATION_GRACE_MS;
+ const deadline = (budget.deadline ??=
+ performance.now() + PROCESS_CLEANUP_VERIFICATION_GRACE_MS);
while (pending.length > 0) {
const remaining = deadline - performance.now();
if (remaining <= 0) break;
@@ -589,6 +592,7 @@ export const readProcessOwnershipFacts = async (
signal?: AbortSignal,
options: {
readonly identityBefore?: ReadonlyMap<number, ProcessIdentityObservation>;
+ readonly exitSettlement?: ProcessExitSettlementBudget;
} = {},
): Promise<readonly ProcessOwnershipFact[]> => {
signal?.throwIfAborted();
@@ -627,19 +631,30 @@ export const readProcessOwnershipFacts = async (
.filter(({ pid }) => tokens.get(pid)?.state !== "readable")
.map(({ pid }) => pid),
);
+ const observedAfter = await readIdentityFacts(
+ processes,
+ host,
+ "after",
+ signal,
+ effectiveUidPids,
+ );
const identityAfter = await settleExitingProcessIdentities(
processes,
host,
- await readIdentityFacts(processes, host, "after", signal, effectiveUidPids),
+ observedAfter,
signal,
+ options.exitSettlement,
);
signal?.throwIfAborted();
- const facts = processes.map((process): ProcessOwnershipFact => {
+ const deriveFact = (
+ process: ProcessTableEntry,
+ observation: ProcessIdentityObservation | undefined,
+ ): ProcessOwnershipFact => {
const before = identityBefore.get(process.pid) ?? {
state: "unavailable" as const,
reason: "process identity was not returned during before observation",
};
- const after = identityAfter.get(process.pid) ?? {
+ const after = observation ?? {
state: "unavailable" as const,
reason: "process identity was not returned during after observation",
};
@@ -664,6 +679,14 @@ export const readProcessOwnershipFacts = async (
token,
identityAfter: after,
};
+ };
+ const facts = processes.map((process): ProcessOwnershipFact => {
+ const after = identityAfter.get(process.pid);
+ const original = observedAfter.get(process.pid);
+ const fact = deriveFact(process, after);
+ return after === original
+ ? fact
+ : { ...fact, beforeExitSettlement: deriveFact(process, original) };
});
signal?.throwIfAborted();
return facts;