Multiplayer agent harness for work.
import { test } from "node:test";
import assert from "node:assert/strict";
import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
import { createDeliveryStore } from "../src/delivery/delivery-store.ts";
import {
runResultDelivery,
wireRunResultDeliveries,
recordRunFailureEntry,
type TurnFailureSessions,
} from "../src/delivery/run-result-delivery.ts";
import type { Run } from "../src/runs/run-store.ts";
import type { OrchestratorInput } from "../src/core/orchestrator.ts";
import type { Principal, TurnResult } from "../src/types.ts";
import { SECURITY_QUARANTINE_REFUSAL_TEXT } from "../plugins/chassis/src/security-quarantine.ts";
const actor: Principal = { id: "internal:U1", type: "internal" };
const turn = (text: string, deliveryTarget?: string): OrchestratorInput => ({
surface: "slack",
...(deliveryTarget ? { deliveryTarget } : {}),
actor,
conversation: { kind: "dm", threadRef: "t", audience: [actor] },
origin: { kind: "direct" },
text,
});
function run(over: Partial<Run>): Run {
return {
id: "r-1",
sessionId: "s-1",
status: "done",
request: turn("hi", "C9:171.001"),
result: { status: "ok", reply: "the reply" },
deliveryState: null,
turnUserSeq: null,
dedupKey: null,
attempts: 1,
errorAttempts: 0,
maxAttempts: 3,
priority: 2,
preemptions: 0,
leaseToken: null,
leaseExpiresAt: null,
workerId: null,
createdAt: 1,
startedAt: 1,
finishedAt: 2,
...over,
};
}
test("runResultDelivery maps ok-with-reply to a recovery delivery keyed by run", () => {
const d = runResultDelivery(run({}));
assert.deepEqual(d, {
destination: { type: "slack", target: "C9:171.001" },
text: "the reply",
provenance: {
trigger: "conversation",
surface: "slack",
fireKey: "run:r-1",
sourceScopeId: "personal:internal:U1",
sourceThreadRef: "t",
},
idempotencyKey: "run:r-1",
});
});
test("runResultDelivery carries the reply's attachments so recovery can replay the files", () => {
const atts = [{ name: "report.csv", mimetype: "text/csv", sizeBytes: 42, blobId: "blob-1" }];
const d = runResultDelivery(run({ result: { status: "ok", reply: "here's the file", attachments: atts } }));
assert.deepEqual(d?.attachments, atts);
assert.equal(d?.text, "here's the file");
});
test("runResultDelivery retains the Slack policy used for private admission", () => {
const request = {
...turn("hi", "DPRIVATE"),
slackSource: {
accountId: "partner",
teamId: "TPARTNER",
userId: "U1",
externalPolicyNamespace: "external-slack:TPARTNER:policy",
},
};
assert.deepEqual(runResultDelivery(run({ request }))?.destination, {
type: "slack",
target: "DPRIVATE",
slackAccountId: "partner",
slackTeamId: "TPARTNER",
slackPolicyNamespace: request.slackSource.externalPolicyNamespace,
});
});
test("runResultDelivery recovers an attachments-only reply (empty text, files still land)", () => {
const atts = [{ name: "report.csv", mimetype: "text/csv", sizeBytes: 42, blobId: "blob-1" }];
const d = runResultDelivery(run({ result: { status: "ok", attachments: atts } }));
assert.equal(d?.text, "");
assert.deepEqual(d?.attachments, atts);
assert.equal(d?.idempotencyKey, "run:r-1");
});
test("runResultDelivery does not recover a turn whose only output was file problems", () => {
const d = runResultDelivery(run({ result: { status: "ok", reply: "" } }));
assert.equal(d, null);
});
test("runResultDelivery sheds a surface-spine turn's text reply (the agent posted it via `post` — no double-post)", () => {
const spine = run({ result: { status: "silent" } });
spine.request = { ...spine.request, surfaceTools: true };
assert.equal(runResultDelivery(spine), null, "no recovery copy — post already delivered the text");
const okReply = run({ result: { status: "ok", reply: "leaked reply" } });
okReply.request = { ...okReply.request, surfaceTools: true };
assert.equal(runResultDelivery(okReply), null, "post owns the text — the turn reply is never re-posted");
});
test("runResultDelivery STILL recovers a surface-spine turn's file attachments (post is text-only)", () => {
const atts = [{ name: "report.csv", mimetype: "text/csv", sizeBytes: 42, blobId: "blob-1" }];
const spine = run({ result: { status: "ok", reply: "", attachments: atts } });
spine.request = { ...spine.request, surfaceTools: true };
const d = runResultDelivery(spine);
assert.deepEqual(d?.attachments, atts, "files ride the turn result, so they must not be shed");
assert.equal(d?.text, "");
});
test("runResultDelivery still posts a surface-spine turn's FAILURE note", () => {
const spine = run({ status: "failed", result: { status: "failed", reason: "boom" } });
spine.request = { ...spine.request, surfaceTools: true };
assert.equal(runResultDelivery(spine)?.text, "⚠️ I couldn't finish that turn: something went wrong on my end");
});
test("runResultDelivery recovers an addressed model-budget refusal with its explanation", () => {
const reason = "This workspace has used up its model budget, so I can't run until it resets.";
const spine = run({ result: { status: "refused", refusalKind: "model_budget", reason } });
spine.request = { ...spine.request, surfaceTools: true, addressed: true };
assert.equal(runResultDelivery(spine)?.text, reason);
spine.request = { ...spine.request, addressed: false, origin: { kind: "ambient" } };
assert.equal(runResultDelivery(spine), null);
});
test("runResultDelivery recovers a security quarantine without exposing its internal reason", () => {
const d = runResultDelivery(
run({
request: { ...turn("hi", "C9:171.001"), addressed: true },
result: {
status: "refused",
refusalKind: "security_quarantine",
reason: "internal screening details",
},
}),
);
assert.equal(d?.text, SECURITY_QUARANTINE_REFUSAL_TEXT);
assert.doesNotMatch(d?.text ?? "", /internal screening details/);
});
test("runResultDelivery keeps an unprompted quarantine silent — a replay has no live handler to suppress it", () => {
const spine = run({ result: { status: "refused", refusalKind: "security_quarantine" } });
spine.request = { ...spine.request, surfaceTools: true, origin: { kind: "ambient" } };
assert.equal(runResultDelivery(spine), null);
});
test("runResultDelivery recovers security quarantine for an addressed surface-spine turn", () => {
const spine = run({ result: { status: "refused", refusalKind: "security_quarantine" } });
spine.request = { ...spine.request, surfaceTools: true, addressed: true };
assert.equal(runResultDelivery(spine)?.text, SECURITY_QUARANTINE_REFUSAL_TEXT);
});
test("runResultDelivery keeps proactive ambient quarantine silent", () => {
const ambient = run({ result: { status: "refused", refusalKind: "security_quarantine" } });
ambient.request = { ...ambient.request, origin: { kind: "automation" }, surfaceTools: true };
assert.equal(runResultDelivery(ambient), null);
});
test("runResultDelivery carries the surface's edit checkpoint into the destination", () => {
const d = runResultDelivery(run({ deliveryState: { editRef: "171.002" } }));
assert.equal(d?.destination.editRef, "171.002");
});
test("runResultDelivery links the admin error page when a resolver is wired", () => {
const failed = run({
status: "failed",
result: { status: "failed", sessionId: "b6f3f9e2-0000-4000-8000-000000000001", reason: "boom" },
});
const d = runResultDelivery(failed, (sessionId) => `https://portal.example.com/admin/?session=${sessionId}`);
assert.equal(
d?.text,
"⚠️ I couldn't finish that turn: something went wrong on my end — full error: https://portal.example.com/admin/?session=b6f3f9e2-0000-4000-8000-000000000001",
);
});
test("runResultDelivery turns a parked run into a visible failure note", () => {
const d = runResultDelivery(
run({ status: "failed", result: { status: "failed", reason: "lease expired (reaped)" } }),
);
assert.equal(d?.text, "⚠️ I couldn't finish that turn: something went wrong on my end");
assert.doesNotMatch(d?.text ?? "", /lease expired/, "the internal failure reason never reaches the user");
assert.equal(d?.idempotencyKey, "run:r-1");
});
test("runResultDelivery keeps unprompted failures quiet, like the live path", () => {
const failed = run({ status: "failed", result: { status: "failed", reason: "boom" } });
failed.request = { ...failed.request, origin: { kind: "ambient" } };
assert.equal(runResultDelivery(failed), null, "no failure note where nobody addressed the agent");
assert.ok(runResultDelivery({ ...failed, request: { ...failed.request, sessionSenderId: "child" } }));
const ok = run({});
ok.request = { ...ok.request, origin: { kind: "ambient" } };
assert.equal(runResultDelivery(ok)?.text, "the reply", "an unprompted reply the agent chose to send still recovers");
});
test("runResultDelivery recognizes legacy queued ambient turns", () => {
const failed = run({ status: "failed", result: { status: "failed", reason: "boom" } });
failed.request = { ...failed.request, origin: undefined, unprompted: true } as unknown as OrchestratorInput;
assert.equal(runResultDelivery(failed), null);
});
test("runResultDelivery skips terminal results that cannot be safely replayed", () => {
assert.equal(runResultDelivery(run({ request: turn("hi") })), null);
assert.equal(runResultDelivery(run({ result: { status: "refused", reason: "not allowed" } })), null);
for (const result of [
{ status: "pending_approval" } as TurnResult,
{ status: "react", reactions: ["thumbsup"] } as TurnResult,
{ status: "silent" } as TurnResult,
{ status: "ok" } as TurnResult,
]) {
assert.equal(runResultDelivery(run({ result })), null, `skips ${result.status}`);
}
});
test("wired stores: a completed turn lands in the outbox unless the live path acked it", async () => {
const { runs } = createMemoryRunStore();
const deliveries = createDeliveryStore();
wireRunResultDeliveries(runs, deliveries);
const crashed = (await runs.enqueue({ sessionId: "sA", request: turn("a", "C9:171.001") })).run;
const c1 = await runs.claim("w1", 5_000);
await runs.setDeliveryState(crashed.id, null, { editRef: "171.002" });
await runs.complete(crashed.id, c1?.leaseToken ?? "", { status: "ok", reply: "recovered reply" });
const pending = await deliveries.pending("slack");
assert.equal(pending.length, 1);
assert.equal(pending[0]!.text, "recovered reply");
assert.equal(pending[0]!.destination.editRef, "171.002");
assert.equal(pending[0]!.idempotencyKey, `run:${crashed.id}`);
const live = (await runs.enqueue({ sessionId: "sB", request: turn("b", "C9") })).run;
const c2 = await runs.claim("w2", 5_000);
await deliveries.ackByKey(`run:${live.id}`, 99);
await runs.complete(live.id, c2?.leaseToken ?? "", { status: "ok", reply: "delivered live" });
await new Promise((r) => setTimeout(r, 0));
const after = await deliveries.pending("slack");
assert.deepEqual(
after.map((d) => d.idempotencyKey),
[`run:${crashed.id}`],
"live-acked copy stays suppressed",
);
});
test("sweep enqueues the recovery delivery for a terminal run whose outbox row never landed", async () => {
const { runs } = createMemoryRunStore();
const deliveries = createDeliveryStore();
const bypassed = (await runs.enqueue({ sessionId: "sA", request: turn("a", "C9:171.001") })).run;
const claimed = await runs.claim("w1", 5_000);
await runs.complete(bypassed.id, claimed?.leaseToken ?? "", { status: "ok", reply: "lost on the way out" });
const recovery = wireRunResultDeliveries(runs, deliveries);
assert.deepEqual(await deliveries.pending("slack"), [], "wired after the fact: nothing enqueued");
assert.equal(await recovery.sweep(Date.now() + 120_000), 1);
const pending = await deliveries.pending("slack");
assert.deepEqual(
pending.map((d) => [d.idempotencyKey, d.text]),
[[`run:${bypassed.id}`, "lost on the way out"]],
);
assert.equal(await recovery.sweep(Date.now() + 120_000), 0, "a second sweep finds the row and leaves it alone");
});
test("sweep leaves runs alone when their delivery already exists, was acked live, or is still inside the grace window", async () => {
const { runs } = createMemoryRunStore();
const deliveries = createDeliveryStore();
const recovery = wireRunResultDeliveries(runs, deliveries);
const finish = async (threadRef: string, reply: string): Promise<Run> => {
const run = (await runs.enqueue({ sessionId: threadRef, request: turn(reply, "C9:171.001") })).run;
const claimed = await runs.claimForSession(threadRef, "w1", 5_000);
await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply });
return run;
};
const enqueued = await finish("sA", "already enqueued");
const live = await finish("sB", "delivered live");
await deliveries.ackByKey(`run:${live.id}`, Date.now());
const silent = (await runs.enqueue({ sessionId: "sC", request: turn("c", "C9:171.001") })).run;
await runs.complete(silent.id, (await runs.claimForSession("sC", "w1", 5_000))?.leaseToken ?? "", {
status: "silent",
});
await new Promise((r) => setTimeout(r, 0));
assert.deepEqual(
(await deliveries.pending("slack")).map((d) => d.idempotencyKey),
[`run:${enqueued.id}`],
);
assert.equal(await recovery.sweep(Date.now() + 120_000), 0);
assert.deepEqual(
(await deliveries.pending("slack")).map((d) => d.idempotencyKey),
[`run:${enqueued.id}`],
);
const fresh = await finish("sD", "just finished");
await deliveries.ackByKey(`run:${fresh.id}`, Date.now());
const bare = createDeliveryStore();
const bareRecovery = wireRunResultDeliveries(runs, bare);
assert.equal(await bareRecovery.sweep(Date.now()), 0, "runs finished inside the grace window are not swept yet");
assert.equal(await bareRecovery.sweep(Date.now() + 120_000), 3, "the same runs sweep once the grace window passes");
});
test("sweep pages through terminal runs by (finishedAt, id) without skipping ties", async () => {
const { runs } = createMemoryRunStore();
const deliveries = createDeliveryStore();
const recovery = wireRunResultDeliveries(runs, deliveries);
const finished = new Set<string>();
for (let i = 0; i < 205; i++) {
const run = (await runs.enqueue({ sessionId: `s${i}`, request: turn(`r${i}`, "C9:171.001") })).run;
const claimed = await runs.claimForSession(`s${i}`, "w1", 5_000);
await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply: `r${i}` });
finished.add(`run:${run.id}`);
}
await new Promise((r) => setTimeout(r, 0));
const swept = createDeliveryStore();
const sweptRecovery = wireRunResultDeliveries(runs, swept);
assert.equal(await sweptRecovery.sweep(Date.now() + 120_000), 205);
assert.deepEqual(new Set((await swept.pending("slack")).map((d) => d.idempotencyKey)), finished);
assert.equal(await recovery.sweep(Date.now() + 120_000), 0);
});
test("wired stores: a parked run lands a durable, non-ackable failure note", async () => {
const { runs } = createMemoryRunStore();
const deliveries = createDeliveryStore();
wireRunResultDeliveries(runs, deliveries);
const parked = (await runs.enqueue({ sessionId: "sP", request: turn("p", "C9:171.001"), maxAttempts: 1 })).run;
const claimed = await runs.claim("w1", 5_000);
await runs.fail(parked.id, claimed?.leaseToken ?? "", "boom", { retry: true });
await new Promise((r) => setTimeout(r, 0));
const stored = await runs.get(parked.id);
assert.equal(stored?.status, "failed");
assert.equal(stored?.result?.status, "failed", "park stores a distinct terminal status, not refused");
const pending = await deliveries.pending("slack");
assert.equal(pending.length, 1, "the park enqueues a durable recovery copy");
assert.equal(pending[0]!.text, "⚠️ I couldn't finish that turn: something went wrong on my end");
assert.equal(pending[0]!.idempotencyKey, `run:${parked.id}`);
});
const failureSessions = async () => {
const { createMemorySessionStore } = await import("../src/sessions/memory-session-store.ts");
const sessions = createMemorySessionStore();
const scope = "personal:[email protected]" as import("../src/types.ts").ScopeId;
const session = await sessions.getOrCreateByThread("slack:D1", "dm", scope, undefined, "slack");
return { sessions, session };
};
const failedRun = (over: Partial<Run> = {}): Run =>
run({
sessionId: "slack:D1",
status: "failed",
result: { status: "failed", sessionId: "slack:D1", reason: "lease expired (reaped)" },
...over,
});
test("a parked run's failure lands as a turn_failure entry in the run's own session, once", async () => {
const { recordRunFailureEntry } = await import("../src/delivery/run-result-delivery.ts");
const { sessions, session } = await failureSessions();
assert.equal(await recordRunFailureEntry(sessions, failedRun()), true);
const entries = await sessions.getEntries(session.id);
assert.equal(entries.length, 1);
assert.equal(entries[0]!.type, "system");
assert.deepEqual(entries[0]!.payload, {
kind: "turn_failure",
message: "I couldn't finish that turn: something went wrong on my end",
runId: "r-1",
});
assert.equal(await recordRunFailureEntry(sessions, failedRun()), false, "recording is idempotent");
assert.equal((await sessions.getEntries(session.id)).length, 1);
});
test("an orchestrator-recorded in-turn failure suppresses the onTerminal entry", async () => {
const { recordRunFailureEntry } = await import("../src/delivery/run-result-delivery.ts");
const { sessions, session } = await failureSessions();
const { lease } = await sessions.acquireLease(session.id);
await sessions.append(lease!, {
type: "system",
payload: { kind: "turn_failure", message: "already recorded by the turn", runId: "r-1" },
scopeLabel: session.scopeId,
});
await sessions.releaseLease(lease!);
assert.equal(await recordRunFailureEntry(sessions, failedRun()), false);
assert.equal((await sessions.getEntries(session.id)).length, 1, "no duplicate entry");
});
test("a web-drain-recorded failure delivery suppresses the onTerminal entry by its key", async () => {
const { recordRunFailureEntry } = await import("../src/delivery/run-result-delivery.ts");
const { sessions, session } = await failureSessions();
const { lease } = await sessions.acquireLease(session.id);
await sessions.append(lease!, {
type: "system",
payload: { kind: "turn_failure", message: "recorded at the web drain", deliveryKey: "run:r-1" },
scopeLabel: session.scopeId,
});
await sessions.releaseLease(lease!);
assert.equal(await recordRunFailureEntry(sessions, failedRun()), false);
assert.equal((await sessions.getEntries(session.id)).length, 1);
});
test("a done run records nothing", async () => {
const { recordRunFailureEntry } = await import("../src/delivery/run-result-delivery.ts");
const { sessions, session } = await failureSessions();
assert.equal(await recordRunFailureEntry(sessions, run({ sessionId: "slack:D1" })), false);
assert.equal((await sessions.getEntries(session.id)).length, 0);
});
test("wired stores: a parked Slack run gets both the durable session entry and the recovery note", async () => {
const { sessions, session } = await failureSessions();
const { runs } = createMemoryRunStore();
const deliveries = createDeliveryStore();
wireRunResultDeliveries(runs, deliveries, undefined, sessions);
const parked = (await runs.enqueue({ sessionId: "slack:D1", request: turn("p", "D1:171.001"), maxAttempts: 1 })).run;
const claimed = await runs.claim("w1", 5_000);
await runs.fail(parked.id, claimed?.leaseToken ?? "", "lease expired (reaped)", { retry: true });
for (let i = 0; i < 50 && (await sessions.getEntries(session.id)).length === 0; i++) {
await new Promise((r) => setTimeout(r, 10));
}
const entries = await sessions.getEntries(session.id);
assert.equal(entries.length, 1, "the run's own session carries the failure durably");
assert.deepEqual(entries[0]!.payload, {
kind: "turn_failure",
message: "I couldn't finish that turn: something went wrong on my end",
runId: parked.id,
});
const pending = await deliveries.pending("slack");
assert.equal(pending.length, 1, "the Slack note still goes out alongside the entry");
assert.match(pending[0]!.text, /couldn't finish/);
});
test("run result delivery identifies the exact source session for shared attachments", () => {
const delivery = runResultDelivery(
run({ result: { status: "ok", sessionId: "source-session", reply: "File ready", sourceAssistantEntrySeq: 7 } }),
);
assert.equal(delivery?.provenance.sourceSessionId, "source-session");
assert.equal(delivery?.provenance.sourceAssistantEntrySeq, 7);
});
test("private session turns cannot deliver even with a stale destination", () => {
assert.equal(
runResultDelivery(run({ request: { ...turn("private", "C9:171.001"), privateSessionMessage: true } })),
null,
);
});
test("failed internal swarm notifications never append user-facing failure entries", async () => {
const failed = run({ status: "failed", result: { status: "failed", reason: "swarm service unavailable" } });
failed.request.swarm = { swarmId: "root", recipientId: "root", messageId: "message" };
const sessions = {
getByThread: async () => {
throw new Error("must not access the transcript");
},
} as unknown as TurnFailureSessions;
assert.equal(await recordRunFailureEntry(sessions, failed), false);
});
for (const surface of ["slack", "web"] as const) {
for (const result of [
{ status: "ok", reply: "INTERNAL_CHILD_REPORT" },
{ status: "failed", reason: "internal child failure" },
{ status: "refused", refusalKind: "security_quarantine", reason: "internal screening" },
{ status: "pending_approval", pendingApprovals: [{ requestId: "a", command: "cmd", reason: "approval" }] },
] satisfies TurnResult[]) {
test(`${surface}: child ${result.status} never becomes an automatic public delivery`, () => {
const threadRef = "agent:main:subagent:child";
assert.equal(
runResultDelivery(
run({
sessionId: threadRef,
status: result.status === "failed" ? "failed" : "done",
request: {
...turn("child", "D1"),
surface,
addressed: true,
conversation: { kind: "dm", threadRef, audience: [actor] },
},
result,
}),
),
null,
);
});
}
}