Multiplayer agent harness for work.
Run people's turns first and park background work for them (#2356)
Regan Bellf23e6da51bf2
diff --git a/package.json b/package.jsonindex 5f876208e..9c96b7f84 100644--- a/package.json+++ b/package.json@@ -48,7 +48,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",+ "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.tsindex a1b7e4e90..301020796 100644--- a/src/harness/agent-tools.ts+++ b/src/harness/agent-tools.ts@@ -1,4 +1,5 @@import { adminOperations } from "../tools/admin.ts";+import { handbackStop } from "../runs/run-store.ts";import { MAX_OUTBOUND_FILES, type OutboundFile } from "../core/attachments.ts";import { MaskedExecutionError } from "../security/secret-masking.ts";import type { DocumentInput } from "../core/document-inputs.ts";@@ -388,7 +389,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):sourceScopeId,});if (type === "tool_result" && isObj(payload)) {- if (ref.shutdown?.aborted) payload = { ...payload, interrupted: true, stop: { kind: "shutdown" } };+ if (ref.shutdown?.aborted) payload = { ...payload, interrupted: true, stop: handbackStop(ref.shutdown) };else if (ref.abortSignal?.aborted) payload = { ...payload, stopped: true };}await ref.emit({ type, payload, scopeLabel });diff --git a/src/harness/context-compaction.ts b/src/harness/context-compaction.tsindex 073076dee..d26613e34 100644--- a/src/harness/context-compaction.ts+++ b/src/harness/context-compaction.ts@@ -22,6 +22,7 @@ export const INTERRUPTED_TOOL_RESULT ="[system] This call was interrupted; its outcome wasn't recorded. Check what happened before retrying.";const STOP_CAUSES = {+ parked: "A pause to let a person's turn run first",shutdown: "A routine platform deploy",lease_expired: "A worker crash",error: "An internal error",diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.tsindex 2e6af8408..20139d706 100644--- a/src/runs/memory-run-store.ts+++ b/src/runs/memory-run-store.ts@@ -1,7 +1,7 @@import { randomUUID } from "node:crypto";import { EventEmitter } from "node:events";import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";-import { isTerminal, leaseLapsed, releasesDedupKey } from "./run-store.ts";+import { countedClaims, isTerminal, leaseLapsed, releasesDedupKey, runPriority } from "./run-store.ts";import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";export interface MemoryRuntime {@@ -34,9 +34,12 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti}function pendingRuns(now: number): Run[] {- return [...runs.values()]- .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now))- .sort((a, b) => a.createdAt - b.createdAt);+ const heads = new Map<string, Run>();+ for (const r of [...runs.values()].sort((a, b) => a.createdAt - b.createdAt)) {+ if (r.status === "pending" && !heads.has(r.sessionId) && !sessionUnavailable(r.sessionId, now))+ heads.set(r.sessionId, r);+ }+ return [...heads.values()].sort((a, b) => a.priority - b.priority || a.createdAt - b.createdAt);}function settle(run: Run): void {@@ -68,6 +71,11 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntiattempts: 0,errorAttempts: 0,maxAttempts,+ priority: runPriority(+ request,+ request.delegatingRunId ? (runs.get(request.delegatingRunId)?.priority ?? null) : null,+ ),+ preemptions: 0,leaseToken: null,leaseExpiresAt: null,workerId: null,@@ -87,6 +95,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntireturn lease(run, workerId, ttlMs);},+ async personWaiting(createdBefore) {+ return pendingRuns(Date.now()).some((run) => run.priority === 0 && run.createdAt <= createdBefore);+ },+async claimById(runId, workerId, ttlMs) {const run = runs.get(runId);if (!run || pendingRuns(Date.now()).find((pending) => pending.sessionId === run.sessionId)?.id !== runId)@@ -109,6 +121,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntiasync releaseLease(runId, leaseToken, stop) {const run = runs.get(runId);if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;+ if (stop?.kind === "parked") run.preemptions += 1;run.status = "pending";run.leaseToken = null;run.leaseExpiresAt = null;@@ -377,7 +390,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntirun.leaseExpiresAt = null;run.workerId = null;if (opts?.countsAsError) run.errorAttempts += 1;- const overClaimed = run.attempts >= maxClaims;+ const overClaimed = countedClaims(run) >= maxClaims;if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {run.status = "pending";run.lastStop = opts?.stop ?? (opts?.countsAsError ? { kind: "error" } : { kind: "lease_expired" });@@ -387,7 +400,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntirun.status = "failed";const reason =!opts?.countsAsError && overClaimed && retry && run.errorAttempts < run.maxAttempts- ? `run parked after ${run.attempts} claims without completing (suspected crash loop)`+ ? `run parked after ${countedClaims(run)} claims without completing (suspected crash loop)`: error;run.result = { status: "failed", sessionId: run.sessionId, reason };run.finishedAt = Date.now();diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex afac277e6..829e2730c 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -9,7 +9,8 @@ import type { TurnResult } from "../types.ts";import type { OrchestratorInput } from "../core/orchestrator.ts";import { resolveTurnOrigin } from "../core/turn-origin.ts";import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";-import { isTerminal, releasesDedupKey } from "./run-store.ts";+import type { RunPriority } from "./run-store.ts";+import { countedClaims, isTerminal, releasesDedupKey, runPriority } from "./run-store.ts";import { errMessage, swallow } from "../util/errors.ts";import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";@@ -37,6 +38,8 @@ function rowToRun(r: Record<string, unknown>): Run {attempts: Number(r.attempts),errorAttempts: Number(r.error_attempts),maxAttempts: Number(r.max_attempts),+ priority: Number(r.claim_priority) as RunPriority,+ preemptions: Number(r.preemptions),leaseToken: (r.lease_token as string | null) ?? null,leaseExpiresAt: r.lease_expires_at === null ? null : Number(r.lease_expires_at),workerId: (r.worker_id as string | null) ?? null,@@ -137,6 +140,20 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaid: "runs/store/0007-last-stop",statements: [`SET LOCAL lock_timeout = '3s'`, `ALTER TABLE runs ADD COLUMN IF NOT EXISTS last_stop TEXT`],},+ {+ id: "runs/store/0008-claim-priority",+ statements: [+ `SET LOCAL lock_timeout = '3s'`,+ `ALTER TABLE runs ADD COLUMN IF NOT EXISTS claim_priority SMALLINT NOT NULL DEFAULT 2`,+ `ALTER TABLE runs ADD COLUMN IF NOT EXISTS preemptions INT NOT NULL DEFAULT 0`,+ ],+ },+ {+ id: "runs/store/0009-claim-priority-index",+ statements: [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_pending_priority ON runs(claim_priority, created_at, seq) WHERE status='pending'`,+ ],+ },],[{@@ -229,7 +246,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaconst ifExpiredAt = opts?.ifExpiredAt ?? null;const countsAsError = opts?.countsAsError ?? false;const errorAttemptsAfter = run.errorAttempts + (countsAsError ? 1 : 0);- const overClaimed = run.attempts >= maxClaims;+ const overClaimed = countedClaims(run) >= maxClaims;if (retry && errorAttemptsAfter < run.maxAttempts && !overClaimed) {const { rowCount } = await q(`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,@@ -248,7 +265,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla}const reason =!countsAsError && overClaimed && retry && errorAttemptsAfter < run.maxAttempts- ? `run parked after ${run.attempts} claims without completing (suspected crash loop)`+ ? `run parked after ${countedClaims(run)} claims without completing (suspected crash loop)`: error;const result: TurnResult = { status: "failed", sessionId: run.sessionId, reason };const { rowCount } = await q(@@ -261,6 +278,13 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClareturn { requeued: false, applied: rowCount > 0 };}+ const claimable = (now: string) => `candidate.status='pending' AND candidate.retry_after <= ${now}+ AND NOT EXISTS (+ SELECT 1 FROM runs sibling WHERE sibling.session_id=candidate.session_id+ AND (sibling.status='running' OR (sibling.status='pending'+ AND (sibling.retry_after > ${now} OR (sibling.created_at, sibling.seq) < (candidate.created_at, candidate.seq))))+ )`;+async function claim(workerId: string, ttlMs: number, runId?: string, sessionId?: string): Promise<Run | null> {const token = randomUUID();const now = Date.now();@@ -269,16 +293,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla`UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,attempts=attempts+1, started_at=COALESCE(started_at,$4)WHERE id = (- SELECT candidate.id FROM runs candidate WHERE candidate.status='pending'- AND candidate.retry_after <= $4+ SELECT candidate.id FROM runs candidate WHERE ${claimable("$4")}AND ($5::text IS NULL OR candidate.id=$5)AND ($6::text IS NULL OR candidate.session_id=$6)- AND NOT EXISTS (- SELECT 1 FROM runs sibling WHERE sibling.session_id=candidate.session_id- AND (sibling.status='running' OR (sibling.status='pending'- AND (sibling.retry_after > $4 OR (sibling.created_at, sibling.seq) < (candidate.created_at, candidate.seq))))- )- ORDER BY candidate.created_at ASC, candidate.seq ASC FOR UPDATE SKIP LOCKED LIMIT 1+ ORDER BY candidate.claim_priority ASC, candidate.created_at ASC, candidate.seq ASC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`,[token, now + ttlMs, workerId, now, runId ?? null, sessionId ?? null],);@@ -309,13 +327,15 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaasync enqueue({ sessionId, request, dedupKey, maxAttempts = 3 }: EnqueueInput): Promise<EnqueueResult> {const id = randomUUID();+ const delegating = request.delegatingRunId ? await getRun(request.delegatingRunId) : null;+ const priority = runPriority(request, delegating?.priority ?? null);for (;;) {const { rows } = await q(- `INSERT INTO runs(id, session_id, status, request, idempotency_key, attempts, max_attempts, created_at)- VALUES ($1,$2,'pending',$3,$4,0,$5,$6)+ `INSERT INTO runs(id, session_id, status, request, idempotency_key, attempts, max_attempts, created_at, claim_priority)+ VALUES ($1,$2,'pending',$3,$4,0,$5,$6,$7)ON CONFLICT (idempotency_key) DO NOTHINGRETURNING *, pg_notify('qm_run_available', 'null')`,- [id, sessionId, jsonbStringify(request), dedupKey ?? null, maxAttempts, Date.now()],+ [id, sessionId, jsonbStringify(request), dedupKey ?? null, maxAttempts, Date.now(), priority],);if (rows[0]) return { run: rowToRun(rows[0]), deduped: false };const existing = await runs.getByDedupKey(dedupKey!);@@ -332,6 +352,16 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaclaimById: (runId, workerId, ttlMs) => claim(workerId, ttlMs, runId),claimForSession: (sessionId, workerId, ttlMs) => claim(workerId, ttlMs, undefined, sessionId),+ async personWaiting(createdBefore) {+ const { rows } = await q(+ `SELECT EXISTS (+ SELECT 1 FROM runs candidate WHERE ${claimable("$2")}+ AND candidate.claim_priority=0 AND candidate.created_at <= $1+ ) AS waiting`,+ [createdBefore, Date.now()],+ );+ return rows[0]?.waiting === true;+ },async heartbeat(runId, leaseToken, ttlMs): Promise<boolean> {const { rowCount } = await q(@@ -343,8 +373,8 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaasync releaseLease(runId, leaseToken, stop): Promise<boolean> {const { rowCount } = await q(- "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, last_stop=COALESCE($3, last_stop) WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",- [runId, leaseToken, stop ? JSON.stringify(stop) : null],+ "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, last_stop=COALESCE($3, last_stop), preemptions=preemptions+$4 WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",+ [runId, leaseToken, stop ? JSON.stringify(stop) : null, stop?.kind === "parked" ? 1 : 0],);return rowCount > 0;},diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 532d098bb..477a93d6b 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -2,9 +2,19 @@ import type { SubscribeOptions } from "../util/event-bus.ts";import type { RunSignal, RunSignalStore } from "./run-signal-store.ts";import type { TurnResult } from "../types.ts";import type { OrchestratorInput } from "../core/orchestrator.ts";+import { isPersonAuthored, resolveTurnOrigin } from "../core/turn-origin.ts";type RunStatus = "pending" | "running" | "done" | "failed";+export type RunPriority = 0 | 1 | 2;+export const MAX_PREEMPTIONS = 3;++export function runPriority(request: OrchestratorInput, delegating: RunPriority | null): RunPriority {+ if (request.swarm || (request.delegatingRunId && delegating === null)) return 2;+ if (request.delegatingRunId) return delegating === 2 ? 2 : 1;+ return isPersonAuthored(resolveTurnOrigin(request).kind) ? 0 : 2;+}+export interface ReapEvent {runId: string;sessionId: string;@@ -20,7 +30,16 @@ export interface RunDeliveryState {}export type RunStop =- { kind: "shutdown" } | { kind: "lease_expired" } | { kind: "error"; code?: string; status?: number };+ | { kind: "shutdown" }+ | { kind: "parked" }+ | { kind: "lease_expired" }+ | { kind: "error"; code?: string; status?: number };++export const PARKED: RunStop = { kind: "parked" };++export function handbackStop(shutdown: AbortSignal): RunStop {+ return shutdown.reason === PARKED ? PARKED : { kind: "shutdown" };+}export interface Run {id: string;@@ -34,6 +53,8 @@ export interface Run {attempts: number;errorAttempts: number;maxAttempts: number;+ priority: RunPriority;+ preemptions: number;leaseToken: string | null;leaseExpiresAt: number | null;workerId: string | null;@@ -74,6 +95,8 @@ export interface RunStore {claimForSession(sessionId: string, workerId: string, ttlMs: number): Promise<Run | null>;+ personWaiting(createdBefore: number): Promise<boolean>;+heartbeat(runId: string, leaseToken: string, ttlMs: number): Promise<boolean>;releaseLease(runId: string, leaseToken: string, stop?: RunStop): Promise<boolean>;@@ -137,8 +160,15 @@ export function releasesDedupKey(result: TurnResult): boolean {return result.refusalKind === "session_busy";}-export function errorParks(run: Pick<Run, "errorAttempts" | "maxAttempts" | "attempts">, maxClaims?: number): boolean {- return run.errorAttempts + 1 >= run.maxAttempts || (maxClaims !== undefined && run.attempts >= maxClaims);+export function countedClaims(run: Pick<Run, "attempts" | "preemptions">): number {+ return run.attempts - run.preemptions;+}++export function errorParks(+ run: Pick<Run, "errorAttempts" | "maxAttempts" | "attempts" | "preemptions">,+ maxClaims?: number,+): boolean {+ return run.errorAttempts + 1 >= run.maxAttempts || (maxClaims !== undefined && countedClaims(run) >= maxClaims);}export function leaseLapsed(run: Pick<Run, "status" | "leaseExpiresAt">, asOf: number): boolean {diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex a451ee774..366541c19 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -6,7 +6,16 @@ import type { TurnResult } from "../types.ts";import type { Orchestrator } from "../core/orchestrator.ts";import { isNonRetryable, NonRetryableTurnError, ProviderTurnError, turnFailureMessage } from "../core/turn-error.ts";import { resolveTurnOrigin } from "../core/turn-origin.ts";-import { errorParks, type Run, type RunStop, type RunStore } from "./run-store.ts";+import {+ countedClaims,+ errorParks,+ handbackStop,+ MAX_PREEMPTIONS,+ PARKED,+ type Run,+ type RunStop,+ type RunStore,+} from "./run-store.ts";import { errMessage, errorAlreadyReported, reportFailure, swallow } from "../util/errors.ts";import { sleep } from "../util/async.ts";import { retryDelay } from "./retry-delay.ts";@@ -75,7 +84,7 @@ export async function processRun(const { turnMs } = resolveSwarmSettings({ turnMs: run.request.turnWallClockMs });workDeadline = setTimeout(() => cancel.abort(), turnMs);}- if (run.request.swarm && run.attempts > 3) throw new NonRetryableTurnError("swarm claim budget exhausted");+ if (run.request.swarm && countedClaims(run) > 3) throw new NonRetryableTurnError("swarm claim budget exhausted");const queueMs = run.startedAt !== null ? Math.max(0, run.startedAt - run.createdAt) : undefined;started = true;const result = await deps.orchestrator.handleTurn({@@ -124,7 +133,7 @@ export async function processRun(opts?.shutdown?.removeEventListener("abort", onShutdown);if (opts?.shutdown?.aborted)await deps.runs- .releaseLease(run.id, token, started ? { kind: "shutdown" } : undefined)+ .releaseLease(run.id, token, started ? handbackStop(opts.shutdown) : undefined).catch((e) => swallow(`worker: shutdown handback failed run=${run.id}; lease will expire after exit`, e));}}@@ -139,6 +148,8 @@ export interface WorkerDeps extends ProcessDeps {export interface Worker {start(): void;+ held(): { run: Run; claimedAt: number; shutdown: AbortController } | null;+ preempt(runId: string): void;stopClaims(): Promise<void>;drained(): Promise<void>;stop(drainMs?: number): Promise<void>;@@ -174,7 +185,12 @@ export function createWorker(deps: WorkerDeps): Worker {let stopped = false;let loopDone: Promise<void> | null = null;let claimDone: Promise<void> | null = null;- let inFlight: { shutdown: AbortController; done: Promise<void> } | null = null;+ let inFlight: {+ run: Run;+ claimedAt: number;+ shutdown: AbortController;+ done: Promise<void>;+ } | null = null;async function loop(): Promise<void> {let claimFailures = 0;@@ -218,12 +234,15 @@ export function createWorker(deps: WorkerDeps): Worker {}const shutdown = new AbortController();let settled!: () => void;- inFlight = {+ const current: NonNullable<typeof inFlight> = {+ run,+ claimedAt: Date.now(),shutdown,done: new Promise<void>((resolve) => {settled = resolve;}),};+ inFlight = current;console.log(`[worker] claimed worker=${workerId} run=${run.id} thread=${run.sessionId}`);claimed();claimDone = null;@@ -251,6 +270,11 @@ export function createWorker(deps: WorkerDeps): Worker {}return {+ held: () => inFlight,+ preempt(runId) {+ if (inFlight?.run.id !== runId || inFlight.shutdown.signal.aborted) return;+ inFlight.shutdown.abort(PARKED);+ },start() {if (loopDone) return;stopped = false;@@ -283,3 +307,24 @@ function runStopFor(err: unknown): RunStop {return { kind: "error", code: err.code, ...(err.status !== undefined ? { status: err.status } : {}) };return { kind: "error" };}++export function createRunPreemption(deps: {+ runs: RunStore;+ workers: readonly Worker[];+ canClaim?: () => boolean;+ graceMs?: number;+}): { sweep(): Promise<boolean> } {+ return {+ async sweep() {+ if (deps.canClaim && !deps.canClaim()) return false;+ const held = deps.workers.map((worker) => ({ worker, held: worker.held() }));+ if (held.some(({ held }) => !held || held.shutdown.signal.aborted)) return false;+ const victim = held+ .filter(({ held }) => held!.run.priority === 2 && held!.run.preemptions < MAX_PREEMPTIONS)+ .sort((a, b) => b.held!.claimedAt - a.held!.claimedAt)[0];+ if (!victim || !(await deps.runs.personWaiting(Date.now() - (deps.graceMs ?? 1_000)))) return false;+ victim.worker.preempt(victim.held!.run.id);+ return true;+ },+ };+}diff --git a/src/wiring.ts b/src/wiring.tsindex e7fd78a13..6423641bd 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -351,7 +351,7 @@ import { createPostgresRunStore } from "./runs/postgres-run-store.ts";import { createMemoryRunSignalStore, type RunSignalStore } from "./runs/run-signal-store.ts";import { createPostgresRunSignalStore } from "./runs/postgres-run-signal-store.ts";import { isTerminal, type Run, type RunStore } from "./runs/run-store.ts";-import { createWorker, type Worker } from "./runs/worker.ts";+import { createRunPreemption, createWorker, type Worker } from "./runs/worker.ts";import {createNoopInstanceRegistry,createPostgresInstanceRegistry,@@ -2727,6 +2727,7 @@ export function buildApp(}): createNoopInstanceRegistry();const drain: DrainController = createDrainController({ registry: instanceRegistry });+ const canClaimRuns = () => backgroundAdmission() && drain.canClaim();const workers: Worker[] = Array.from({ length: Math.max(1, config.workers) }, () =>createWorker({admittedWork,@@ -2736,9 +2737,11 @@ export function buildApp(heartbeatIntervalMs: config.heartbeatIntervalMs,errors,pollMs: 250,- canClaim: () => backgroundAdmission() && drain.canClaim(),+ canClaim: canClaimRuns,}),);+ const preemption = createRunPreemption({ runs, workers, canClaim: canClaimRuns });+ const preemptionSweeper = createSweeper(() => preemption.sweep(), 1_000, { label: "run-preemption" });const processReaper: ProcessReaper | null = processes? createProcessReaper(processes, {intervalMs: config.processReaperIntervalMs,@@ -2818,6 +2821,7 @@ export function buildApp(}).catch(swallowAs("wiring: worker resume failed", undefined));}+ preemptionSweeper.start();const startPeriodic = () => {if (!backgroundRunning || generation !== backgroundGeneration) return;reaper.start();@@ -2875,6 +2879,7 @@ export function buildApp(sessionReturnSweeper.stop(),runResultSweeper.stop(),approvalDeliverySweeper.stop(),+ preemptionSweeper.stop(),...workers.map((worker) => worker.stopClaims()),];backgroundClaimsStopping = Promise.all(stopping).then(() => {});diff --git a/test/run-priority.test.ts b/test/run-priority.test.tsnew file mode 100644index 000000000..8f2a1b845--- /dev/null+++ b/test/run-priority.test.ts@@ -0,0 +1,211 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";+import { createPostgresRunStore } from "../src/runs/postgres-run-store.ts";+import { createRunPreemption, createWorker } from "../src/runs/worker.ts";+import type { Orchestrator, OrchestratorInput } from "../src/core/orchestrator.ts";+import { handbackStop, PARKED, type Run, type RunStore } from "../src/runs/run-store.ts";+import type { Principal, TurnResult } from "../src/types.ts";++const URL = process.env.DATABASE_URL;+const actor: Principal = { id: "internal:U1", type: "internal" };+const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));+const request = (text: string, over: Partial<OrchestratorInput> = {}): OrchestratorInput => ({+ actor,+ conversation: { kind: "dm", threadRef: "t", audience: [actor] },+ origin: { kind: "human" },+ text,+ ...over,+});+const cron = (text: string) => request(text, { origin: { kind: "automation" } });+const subagentOf = (parent: Run, text: string) =>+ request(text, { origin: { kind: "automation" }, delegatingRunId: parent.id });+const swarm = { swarmId: "s", messageId: "m", recipientId: "r" };++const backends: Array<{ name: string; skip: string | false; make: () => Promise<RunStore> }> = [+ { name: "memory", skip: false, make: async () => createMemoryRunStore().runs },+ {+ name: "postgres",+ skip: URL ? false : "set DATABASE_URL (a Postgres) to run the Postgres run-priority tests",+ make: async () => {+ const pool = new (await import("pg")).default.Pool({ connectionString: URL });+ await pool.query("DROP TABLE IF EXISTS runs, tool_calls CASCADE");+ await pool.query("DELETE FROM qm_schema_migrations WHERE id LIKE 'runs/%'").catch(() => {});+ await pool.end();+ return createPostgresRunStore(URL!).runs;+ },+ },+];++async function claimAll(runs: RunStore): Promise<string[]> {+ const order: string[] = [];+ for (let run = await runs.claim("w", 60_000); run; run = await runs.claim("w", 60_000)) {+ order.push(run.request.text);+ await runs.complete(run.id, run.leaseToken!, { status: "ok", sessionId: run.sessionId, reply: "" });+ }+ return order;+}++for (const backend of backends) {+ test(+ `[${backend.name}] people's turns run first, then their subagents, then background work`,+ { skip: backend.skip },+ async () => {+ const runs = await backend.make();+ try {+ const nightly = (await runs.enqueue({ sessionId: "cron", request: cron("cron") })).run;+ const human = (await runs.enqueue({ sessionId: "dm", request: request("person") })).run;+ const child = (await runs.enqueue({ sessionId: "c", request: subagentOf(human, "subagent") })).run;+ await runs.enqueue({ sessionId: "g", request: subagentOf(child, "sub-subagent") });+ await runs.enqueue({ sessionId: "nc", request: subagentOf(nightly, "cron subagent") });+ await runs.enqueue({ sessionId: "sw", request: { ...subagentOf(human, "swarm worker"), swarm } });+ await runs.enqueue({ sessionId: "shared", request: cron("background first in its conversation") });+ await runs.enqueue({ sessionId: "shared", request: request("person second in its conversation") });+ assert.equal(await runs.personWaiting(Date.now() - 60_000), false, "only turns past the grace period count");+ assert.equal(await runs.personWaiting(Date.now()), true);+ assert.deepEqual(await claimAll(runs), [+ "person",+ "subagent",+ "sub-subagent",+ "cron",+ "cron subagent",+ "swarm worker",+ "background first in its conversation",+ "person second in its conversation",+ ]);+ } finally {+ await runs.close?.();+ }+ },+ );+}++for (const backend of backends) {+ test(+ `[${backend.name}] a parked run records why it stopped and is not charged a claim`,+ { skip: backend.skip },+ async () => {+ const runs = await backend.make();+ try {+ const queued = (await runs.enqueue({ sessionId: "cron", request: cron("cron") })).run;+ const lease = (await runs.claim("w", 60_000))!;+ assert.equal(await runs.releaseLease(lease.id, lease.leaseToken!, { kind: "parked" }), true);+ const parked = (await runs.get(queued.id))!;+ assert.deepEqual([parked.status, parked.preemptions, parked.lastStop], ["pending", 1, { kind: "parked" }]);+ const again = (await runs.claim("w", 60_000))!;+ await runs.releaseLease(again.id, again.leaseToken!, { kind: "shutdown" });+ assert.equal((await runs.get(queued.id))!.preemptions, 1, "a deploy handback is not a park");+ } finally {+ await runs.close?.();+ }+ },+ );+}++function scripted(log: string[]): Orchestrator {+ return {+ async handleTurn(input: OrchestratorInput): Promise<TurnResult> {+ log.push(`start ${input.text} #${input.attempt}${input.resumeCause ? ` after ${input.resumeCause.kind}` : ""}`);+ const [mode] = input.text.split(" ");+ if (mode === "forever" || (input.attempt === 1 && mode !== "quick")) {+ await new Promise<void>((resolve) =>+ input.shutdown?.addEventListener("abort", () => resolve(), { once: true }),+ );+ if (mode === "slow") await sleep(150);+ return { status: "silent", sessionId: "x", stopped: true };+ }+ log.push(`finish ${input.text}`);+ return { status: "ok", sessionId: "x", reply: input.text };+ },+ } as unknown as Orchestrator;+}++async function until(check: () => Promise<boolean>): Promise<void> {+ for (let i = 0; i < 200 && !(await check()); i++) await sleep(10);+ assert.ok(await check(), "timed out");+}++async function withWorkers(+ count: number,+ body: (ctx: { runs: RunStore; log: string[]; sweep: () => Promise<boolean> }) => Promise<void>,+ maxClaims?: number,+): Promise<void> {+ const { runs } = createMemoryRunStore(maxClaims ? { maxClaims } : undefined);+ const log: string[] = [];+ const workers = Array.from({ length: count }, () =>+ createWorker({ runs, orchestrator: scripted(log), leaseTtlMs: 60_000, pollMs: 5 }),+ );+ const preemption = createRunPreemption({ runs, workers, graceMs: 0 });+ for (const worker of workers) worker.start();+ try {+ await body({ runs, log, sweep: () => preemption.sweep() });+ } finally {+ await Promise.all(workers.map((worker) => worker.releaseInFlight()));+ }+}++const status = (runs: RunStore, run: Run, want: string) => async () => (await runs.get(run.id))?.status === want;++test("a busy worker parks a cron turn for a person's turn, then resumes it", () =>+ withWorkers(+ 1,+ async ({ runs, log, sweep }) => {+ const background = (await runs.enqueue({ sessionId: "cron", request: cron("long cron") })).run;+ await until(status(runs, background, "running"));+ await runs.enqueue({ sessionId: "dm", request: request("quick person") });+ assert.equal(await sweep(), true);+ await until(status(runs, background, "done"));+ assert.equal((await runs.get(background.id))?.preemptions, 1);+ assert.deepEqual(log, [+ "start long cron #1",+ "start quick person #1",+ "finish quick person",+ "start long cron #2 after parked",+ "finish long cron",+ ]);+ },+ 1,+ ));++test("one waiting turn parks one run, even while that run is slow to stop", () =>+ withWorkers(3, async ({ runs, sweep }) => {+ const crons: Run[] = [];+ for (const name of ["a", "b", "c"])+ crons.push((await runs.enqueue({ sessionId: name, request: cron(`slow ${name}`) })).run);+ for (const run of crons) await until(status(runs, run, "running"));+ const person = (await runs.enqueue({ sessionId: "dm", request: request("quick person") })).run;+ assert.deepEqual([await sweep(), await sweep(), await sweep()], [true, false, false]);+ await until(status(runs, person, "done"));+ const parks = await Promise.all(crons.map(async (run) => (await runs.get(run.id))!.preemptions));+ assert.deepEqual(parks.sort(), [0, 0, 1]);+ }));++test("subagents of a person's turn and thrice-parked runs keep their worker", () =>+ withWorkers(2, async ({ runs, sweep }) => {+ const parent = (await runs.enqueue({ sessionId: "parent", request: request("quick parent") })).run;+ await until(status(runs, parent, "done"));+ const capped = (+ await runs.enqueue({ sessionId: "cron", request: { ...cron("forever cron"), swarm, turnWallClockMs: 60_000 } })+ ).run;+ for (let i = 0; i < 3; i++) {+ const lease = (await runs.claim("w", 60_000))!;+ await runs.releaseLease(lease.id, lease.leaseToken!, { kind: "parked" });+ }+ const child = (await runs.enqueue({ sessionId: "child", request: subagentOf(parent, "forever child") })).run;+ await until(status(runs, capped, "running"));+ await until(status(runs, child, "running"));+ await runs.enqueue({ sessionId: "dm", request: request("quick person") });+ assert.equal(await sweep(), false);+ assert.equal((await runs.get(capped.id))?.attempts, 4, "parks don't spend a swarm worker's claim budget");+ }));++test("a tool result written after a park records the park, not a deploy", () => {+ const parked = new AbortController();+ parked.abort(PARKED);+ const deploy = new AbortController();+ deploy.abort();+ assert.deepEqual(+ [handbackStop(parked.signal), handbackStop(deploy.signal)],+ [{ kind: "parked" }, { kind: "shutdown" }],+ );+});diff --git a/test/run-result-delivery.test.ts b/test/run-result-delivery.test.tsindex 8cab48d3c..1f9b8c7cc 100644--- a/test/run-result-delivery.test.ts+++ b/test/run-result-delivery.test.ts@@ -36,6 +36,8 @@ function run(over: Partial<Run>): Run {attempts: 1,errorAttempts: 0,maxAttempts: 3,+ priority: 2,+ preemptions: 0,leaseToken: null,leaseExpiresAt: null,workerId: null,