Multiplayer agent harness for work.
Sanitize persisted run and signal JSON (#1595)
Joshua France23bed461d44e
diff --git a/src/api/app-turn.ts b/src/api/app-turn.tsindex 8f75b7229..69aba1ca6 100644--- a/src/api/app-turn.ts+++ b/src/api/app-turn.ts@@ -27,6 +27,7 @@ import { selectableCatalogForHarness, selectableModelCatalog } from "../model/moimport { resolveRuntimeChoiceDurable } from "../harness/harness-router.ts";import { swallow, swallowAs } from "../util/errors.ts";import { sleep } from "../util/async.ts";+import { pgTextSafe } from "../util/text.ts";import { GENERIC_FAILURE_CLAUSE } from "../../plugins/chassis/src/failure-copy.ts";import type { App, AppDeps } from "./app-types.ts";@@ -511,7 +512,10 @@ export function createTurnMethods(const after = await deps.runs.get(targetRun.id);if (!after || isTerminal(after.status)) {const own = (await replayOrphanedRunSignals(targetRun.id)).find(- (d) => d.signal.text === route.text && d.signal.ts === steerTs,+ (d) =>+ (d.signal.text === route.text ||+ (route.text !== undefined && d.signal.text === pgTextSafe(route.text))) &&+ d.signal.ts === steerTs,);if (own?.replayRunId)return req.async ? { status: "queued", runId: own.replayRunId } : drive(own.replayRunId);@@ -562,7 +566,9 @@ export function createTurnMethods(const after = await deps.runs.get(liveAmbient.id);if (!after || isTerminal(after.status)) {const own = (await replayOrphanedRunSignals(liveAmbient.id)).find(- (d) => d.signal.text === req.text && d.signal.ts === origin.messageTs,+ (d) =>+ (d.signal.text === req.text || d.signal.text === pgTextSafe(req.text)) &&+ d.signal.ts === origin.messageTs,);if (own?.replayRunId)return req.async ? { status: "queued", runId: own.replayRunId } : drive(own.replayRunId);diff --git a/src/persistence/durable-map.ts b/src/persistence/durable-map.tsindex de84b1191..94390817b 100644--- a/src/persistence/durable-map.ts+++ b/src/persistence/durable-map.ts@@ -36,7 +36,7 @@ function jsonbSafe(value: unknown): unknown {if (typeof value === "string") return pgTextSafe(value);if (Array.isArray(value)) return value.map(jsonbSafe);if (value && typeof value === "object") {- const out: Record<string, unknown> = {};+ const out: Record<string, unknown> = Object.create(null);for (const [k, v] of Object.entries(value)) out[pgTextSafe(k)] = jsonbSafe(v);return out;}diff --git a/src/runs/postgres-run-signal-store.ts b/src/runs/postgres-run-signal-store.tsindex 066899b87..f4ef9d4a3 100644--- a/src/runs/postgres-run-signal-store.ts+++ b/src/runs/postgres-run-signal-store.ts@@ -1,4 +1,6 @@import { createPgPool, type Rows } from "../persistence/pg-pool.ts";+import { jsonbStringify } from "../persistence/durable-map.ts";+import { pgTextSafe } from "../util/text.ts";import { subscribePostgresChannel } from "../persistence/postgres-listener.ts";import type { RunSignal, RunSignalKind, RunSignalStore } from "./run-signal-store.ts";@@ -76,7 +78,14 @@ export function createPostgresRunSignalStore(connectionString: string): RunSignaRETURNING id)SELECT pg_notify('${CHANNEL}', $1) FROM ins`,- [runId, signal.kind, signal.text ?? null, JSON.stringify(signal), Date.now(), signal.dedupeKey ?? null],+ [+ runId,+ signal.kind,+ signal.text === undefined ? null : pgTextSafe(signal.text),+ jsonbStringify(signal),+ Date.now(),+ signal.dedupeKey ?? null,+ ],);return rows.length > 0;},diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex afab775d2..08e07b90f 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -2,6 +2,8 @@ import { createPostgresNotifyBus } from "../persistence/postgres-notify-bus.ts";import { randomUUID } from "node:crypto";import { EventEmitter } from "node:events";import { createPgPool } from "../persistence/pg-pool.ts";+import { jsonbStringify } from "../persistence/durable-map.ts";+import { pgTextSafe } from "../util/text.ts";import { isObj } from "../util/objects.ts";import type { TurnResult } from "../types.ts";import type { OrchestratorInput } from "../core/orchestrator.ts";@@ -285,7 +287,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaVALUES ($1,$2,'pending',$3,$4,0,$5,$6)ON CONFLICT (idempotency_key) DO NOTHINGRETURNING *, pg_notify('qm_run_available', 'null')`,- [id, sessionId, JSON.stringify(request), dedupKey ?? null, maxAttempts, Date.now()],+ [id, sessionId, jsonbStringify(request), dedupKey ?? null, maxAttempts, Date.now()],);if (rows[0]) return { run: rowToRun(rows[0]), deduped: false };const existing = await runs.getByDedupKey(dedupKey!);@@ -417,7 +419,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla`UPDATE runs SET request = (request::jsonb || jsonb_build_object('text', $2::text, 'displayText', $2::text))::textWHERE id = $1 AND status = 'pending' AND attempts = 0 AND turn_user_seq IS NULLAND COALESCE(request::jsonb ->> 'displayText', request::jsonb ->> 'text') = $3`,- [runId, text, expectedText],+ [runId, pgTextSafe(text), pgTextSafe(expectedText)],);return (rowCount ?? 0) > 0;},@@ -447,8 +449,8 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaqueuedRunId,targetRunId,signal.kind,- signal.text ?? null,- JSON.stringify(signal),+ signal.text === undefined ? null : pgTextSafe(signal.text),+ jsonbStringify(signal),Date.now(),signal.dedupeKey ?? null,],diff --git a/src/sessions/session-syscalls.ts b/src/sessions/session-syscalls.tsindex 5099b1483..28a0308b6 100644--- a/src/sessions/session-syscalls.ts+++ b/src/sessions/session-syscalls.ts@@ -4,6 +4,7 @@ import { deliveryCandidatesFor } from "../core/orchestrator/turn-helpers.ts";import type { SessionMailbox, SessionMessage } from "./session-mailbox.ts";import { conversationScope } from "../resolution/resolution-service.ts";import { sleep } from "../util/async.ts";+import { pgTextSafe } from "../util/text.ts";import { createMemoryAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";import { errMessage } from "../util/errors.ts";import { filterHistoryForAudience } from "../resolution/context-filter.ts";@@ -721,7 +722,10 @@ export function createSessionSyscalls(deps: SessionSyscallDeps): SessionSyscallsif (dedupKey) {const existing = await deps.runs.getByDedupKey(dedupKey);if (existing) {- if (existing.sessionId !== target.threadRef || existing.request.text !== stamped)+ if (+ existing.sessionId !== target.threadRef ||+ (existing.request.text !== stamped && existing.request.text !== pgTextSafe(stamped))+ )return { ok: false, message: "the requestId was already used for different work" };return { ok: true, sessionId: target.id, title, delivered: "queued_turn" };}diff --git a/test/durable-map-jsonb.test.ts b/test/durable-map-jsonb.test.tsindex dac230c80..03cec7472 100644--- a/test/durable-map-jsonb.test.ts+++ b/test/durable-map-jsonb.test.ts@@ -15,7 +15,13 @@ test("a lone high surrogate (emoji cut in half by slice) becomes U+FFFD", () =>});test("well-formed strings, keys, arrays, and nesting pass through untouched", () => {- const value = { a: "hello 😀", list: ["x", { deep: "ok" }], n: 3, b: true, z: null };+ const value = {+ a: "hello 😀",+ list: ["x", { deep: "ok", ["__proto__"]: { retained: true } }],+ n: 3,+ b: true,+ z: null,+ };assert.equal(jsonbStringify(value), JSON.stringify(value));});diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.tsindex 888dc3af7..0752af44b 100644--- a/test/postgres-store.test.ts+++ b/test/postgres-store.test.ts@@ -13,7 +13,8 @@ import {} from "../src/sessions/postgres-session-store.ts";import { SECURITY_SCREEN_STEP } from "../src/security/security-posture.ts";import { createPostgresRunStore } from "../src/runs/postgres-run-store.ts";-import { scopeId, type Principal, type TurnResult } from "../src/types.ts";+import { createPostgresRunSignalStore } from "../src/runs/postgres-run-signal-store.ts";+import { scopeId, type Principal, type TurnRequest, type TurnResult } from "../src/types.ts";import type { OrchestratorInput } from "../src/core/orchestrator.ts";import { assertParticipantSessionParity } from "./support/participant-session-parity.ts";import { byScopeId, rollupsFromSummaries } from "./support/scope-rollup-oracle.ts";@@ -1275,6 +1276,71 @@ test("pg session counters: boot backfill fills pre-column rows", { skip }, asyncassert.equal(stats.turns, 3 + 7, "stats sum the stored turns counters, never the entries");});+test("pg run store: Unicode stays jsonb-safe through enqueue, edit and both steering paths", { skip }, async () => {+ const { runs, close } = createPostgresRunStore(URL!);+ const signals = createPostgresRunSignalStore(URL!);+ const thread = `unicode-${randomUUID()}`;+ const unsafe = "nul\u0000 lone\ud800 low\udfff emoji😀 literal\\u0000";+ const safe = "nul lone� low� emoji😀 literal\\u0000";+ const inbound: TurnRequest = {+ surface: "web",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: thread },+ text: unsafe,+ };+ const request = {+ ...turn(unsafe),+ attachments: [{ name: unsafe, mimetype: "text/plain", sizeBytes: 5, blobId: "notes-blob" }],+ };+ try {+ const first = (await runs.enqueue({ sessionId: thread, request })).run;+ assert.equal(first.request.text, safe);+ assert.equal(first.request.attachments?.[0]?.name, safe);+ assert.equal(request.text, unsafe);+ const privateRun = (+ await runs.enqueue({ sessionId: thread, request: { ...turn(unsafe), privateSessionMessage: true } })+ ).run;+ assert.equal((await runs.latestForThread(thread))?.id, privateRun.id);+ assert.equal((await runs.latestForThread(thread, { excludePrivateMessages: true }))?.id, first.id);+ assert.equal(await runs.editPendingText(first.id, `edit ${unsafe}`, unsafe), true);+ assert.equal(await runs.editPendingText(first.id, "stale", unsafe), false);+ assert.equal((await runs.get(first.id))?.request.displayText, `edit ${safe}`);+ assert.equal(+ await runs.steerQueued(+ first.id,+ privateRun.id,+ {+ kind: "steer",+ text: `edit ${unsafe}`,+ request: { ...inbound, text: `edit ${unsafe}` },+ dedupeKey: `${thread}-queued`,+ },+ signals,+ ),+ true,+ );+ assert.equal(await runs.get(first.id), null);+ const queued = await signals.takePending(privateRun.id);+ assert.equal(queued[0]?.text, `edit ${safe}`);+ assert.equal(queued[0]?.request?.text, `edit ${safe}`);+ assert.equal(+ await signals.send(privateRun.id, {+ kind: "steer",+ text: unsafe,+ request: inbound,+ }),+ true,+ );+ const direct = await signals.takePending(privateRun.id);+ assert.equal(direct[0]?.text, safe);+ assert.equal(direct[0]?.request?.text, safe);+ } finally {+ for (const run of await runs.inFlightForThread(thread)) await runs.withdraw(run.id);+ await signals.close?.();+ await close();+ }+});+test("pg run store: a session_busy completion frees the dedup key so the same key runs again", { skip }, async () => {const { runs, close } = createPostgresRunStore(URL!);try {diff --git a/test/session-syscalls.test.ts b/test/session-syscalls.test.tsindex e5da2e53a..234e3be21 100644--- a/test/session-syscalls.test.ts+++ b/test/session-syscalls.test.ts@@ -2,7 +2,7 @@ import { wireRunResultDeliveries, runResultDelivery } from "../src/delivery/run-import { createDeliveryStore } from "../src/delivery/delivery-store.ts";import { createControlService } from "../src/api/control-service.ts";import { deliveryCandidatesFor } from "../src/core/orchestrator/turn-helpers.ts";-import { createMemoryMap } from "../src/persistence/durable-map.ts";+import { createMemoryMap, jsonbStringify } from "../src/persistence/durable-map.ts";import { createSessionMailbox, type SessionMailbox, type SessionMessage } from "../src/sessions/session-mailbox.ts";import { test } from "node:test";import assert from "node:assert/strict";@@ -753,14 +753,14 @@ test("follow-up retries recover the accepted run before checking full capacity",const enqueue = r.runs.enqueue.bind(r.runs);let first = true;t.mock.method(r.runs, "enqueue", async (input: Parameters<RunStore["enqueue"]>[0]) => {- const receipt = await enqueue(input);+ const receipt = await enqueue({ ...input, request: JSON.parse(jsonbStringify(input.request)) });if (first) {first = false;throw new Error("lost receipt");}return receipt;});- const request = { target: opened.sessionId, text: "more work", followup: true, requestId: "stable" };+ const request = { target: opened.sessionId, text: "more\u0000 work\ud800", followup: true, requestId: "stable" };assert.equal((await api.write(request)).ok, false);assert.equal((await api.write(request)).ok, true);assert.equal((await api.write({ ...request, text: "changed" })).ok, false);diff --git a/test/wake-steering.test.ts b/test/wake-steering.test.tsindex fa15520f3..b1c7e4d13 100644--- a/test/wake-steering.test.ts+++ b/test/wake-steering.test.ts@@ -6,6 +6,7 @@ import { mkdtempSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { buildApp } from "../src/wiring.ts";+import { jsonbStringify } from "../src/persistence/durable-map.ts";import type { TurnRequest } from "../src/types.ts";import { testConfig } from "./support/test-config.ts";import type { SecurityScreener } from "../src/security/security-screener.ts";@@ -762,10 +763,10 @@ test("signalRun: a steer already terminal at send is refused up front", async ()assert.equal((await built.signals.takePending(liveRunId)).length, 0, "nothing left rotting in the queue");});-function completeOnSend(built: ReturnType<typeof freshApp>): void {+function completeOnSend(built: ReturnType<typeof freshApp>, sanitize = false): void {const origSend = built.signals.send.bind(built.signals);built.signals.send = async (runId, signal) => {- const sent = await origSend(runId, signal);+ const sent = await origSend(runId, sanitize ? JSON.parse(jsonbStringify(signal)) : signal);const claimed = await built.runs.claim("w1", 30_000);if (claimed) await built.runs.complete(claimed.id, claimed.leaseToken!, { status: "silent" });return sent;@@ -786,28 +787,51 @@ test("signalRun: a steer whose run goes terminal mid-send is REFUSED, never a faassert.equal((await built.signals.takePending(liveRunId)).length, 0, "nothing left rotting in the queue");});-test("steer path: a message whose run goes terminal mid-send is replayed and the caller gets the FRESH run", async () => {+for (const sanitize of [false, true])+ test(`steer path: a message whose run goes terminal mid-send returns the fresh run (sanitize=${sanitize})`, async () => {+ const built = freshApp();+ const channel = "C13";+ const root = "1300.1";+ const threadRef = `ch:${channel}:${root}`;+ const first = await built.app.turn(mention("@bot go", channel, root));+ const liveRunId = first.runId!;+ completeOnSend(built, sanitize);+ built.app.replayOrphanedRunSignals = async () => {};++ const second = await built.app.turn({+ ...mention("@bot and another thing\u0000\ud800", channel, root),+ triggerTs: "1300.010",+ });+ assert.equal(second.status, "queued");+ assert.notEqual(second.runId, liveRunId, "the caller follows the replayed run, not the dead one");+ const replayed = (await built.runs.list()).find((r) => r.id === second.runId);+ assert.equal(replayed?.sessionId, threadRef);+ const text = `${replayed?.request.text ?? ""} ${replayed?.request.displayText ?? ""}`;+ assert.ok(+ text.includes("and another thing"),+ `the fresh run carries the raced message (got: ${text.slice(0, 120)})`,+ );+ assert.equal(+ (await built.signals.takePending(liveRunId)).length,+ 0,+ "the raced signal was consumed by the inline drain",+ );+ });++test("reverse race: a sanitized mention follows its fresh run after the ambient run ends", async () => {const built = freshApp();- const channel = "C13";- const root = "1300.1";- const threadRef = `ch:${channel}:${root}`;- const first = await built.app.turn(mention("@bot go", channel, root));- const liveRunId = first.runId!;- completeOnSend(built);+ const channel = "C-ambient-unicode";+ const askTs = "1600.2";+ const ambientRef = `slack:${channel}:ambient:${askTs}`;+ await built.sessions.getOrCreateByThread(ambientRef, "channel", `channel:${channel}`);+ const ambient = await built.app.turn(spawnedWorker(channel, askTs));+ completeOnSend(built, true);built.app.replayOrphanedRunSignals = async () => {};-- const second = await built.app.turn({ ...mention("@bot and another thing", channel, root), triggerTs: "1300.010" });+ const second = await built.app.turn({ ...mention("@bot more\u0000 work\ud800", channel, askTs), triggerTs: askTs });assert.equal(second.status, "queued");- assert.notEqual(second.runId, liveRunId, "the caller follows the replayed run, not the dead one");- const replayed = (await built.runs.list()).find((r) => r.id === second.runId);- assert.equal(replayed?.sessionId, threadRef);- const text = `${replayed?.request.text ?? ""} ${replayed?.request.displayText ?? ""}`;- assert.ok(text.includes("and another thing"), `the fresh run carries the raced message (got: ${text.slice(0, 120)})`);- assert.equal(- (await built.signals.takePending(liveRunId)).length,- 0,- "the raced signal was consumed by the inline drain",- );+ assert.notEqual(second.runId, ambient.runId);+ const replayed = await built.runs.get(second.runId!);+ assert.ok(`${replayed?.request.text} ${replayed?.request.displayText}`.includes("@bot more work�"));});test("screening off delivers ambient updates to the existing run without a classifier", async () => {