Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Run people's turns first and park background work for them (#2356)

Regan Bellf23e6da51bf2
diff --git a/package.json b/package.json
index 5f876208e..9c96b7f84 100644
--- a/package.json
+++ b/package.json
@@ -48,7 +48,7 @@
"sandbox:local:build": "bash scripts/local-sandbox-build.sh",
"sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"",
"deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",
- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
+ "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
"livetest": "node scripts/api-livetest.ts",
"bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts",
"prepare": "node .husky/install.mjs",
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.ts
index a1b7e4e90..301020796 100644
--- a/src/harness/agent-tools.ts
+++ b/src/harness/agent-tools.ts
@@ -1,4 +1,5 @@
import { adminOperations } from "../tools/admin.ts";
+import { handbackStop } from "../runs/run-store.ts";
import { MAX_OUTBOUND_FILES, type OutboundFile } from "../core/attachments.ts";
import { MaskedExecutionError } from "../security/secret-masking.ts";
import type { DocumentInput } from "../core/document-inputs.ts";
@@ -388,7 +389,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
sourceScopeId,
});
if (type === "tool_result" && isObj(payload)) {
- if (ref.shutdown?.aborted) payload = { ...payload, interrupted: true, stop: { kind: "shutdown" } };
+ if (ref.shutdown?.aborted) payload = { ...payload, interrupted: true, stop: handbackStop(ref.shutdown) };
else if (ref.abortSignal?.aborted) payload = { ...payload, stopped: true };
}
await ref.emit({ type, payload, scopeLabel });
diff --git a/src/harness/context-compaction.ts b/src/harness/context-compaction.ts
index 073076dee..d26613e34 100644
--- a/src/harness/context-compaction.ts
+++ b/src/harness/context-compaction.ts
@@ -22,6 +22,7 @@ export const INTERRUPTED_TOOL_RESULT =
"[system] This call was interrupted; its outcome wasn't recorded. Check what happened before retrying.";
const STOP_CAUSES = {
+ parked: "A pause to let a person's turn run first",
shutdown: "A routine platform deploy",
lease_expired: "A worker crash",
error: "An internal error",
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.ts
index 2e6af8408..20139d706 100644
--- a/src/runs/memory-run-store.ts
+++ b/src/runs/memory-run-store.ts
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
import { EventEmitter } from "node:events";
import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";
-import { isTerminal, leaseLapsed, releasesDedupKey } from "./run-store.ts";
+import { countedClaims, isTerminal, leaseLapsed, releasesDedupKey, runPriority } from "./run-store.ts";
import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";
export interface MemoryRuntime {
@@ -34,9 +34,12 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
}
function pendingRuns(now: number): Run[] {
- return [...runs.values()]
- .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now))
- .sort((a, b) => a.createdAt - b.createdAt);
+ const heads = new Map<string, Run>();
+ for (const r of [...runs.values()].sort((a, b) => a.createdAt - b.createdAt)) {
+ if (r.status === "pending" && !heads.has(r.sessionId) && !sessionUnavailable(r.sessionId, now))
+ heads.set(r.sessionId, r);
+ }
+ return [...heads.values()].sort((a, b) => a.priority - b.priority || a.createdAt - b.createdAt);
}
function settle(run: Run): void {
@@ -68,6 +71,11 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
attempts: 0,
errorAttempts: 0,
maxAttempts,
+ priority: runPriority(
+ request,
+ request.delegatingRunId ? (runs.get(request.delegatingRunId)?.priority ?? null) : null,
+ ),
+ preemptions: 0,
leaseToken: null,
leaseExpiresAt: null,
workerId: null,
@@ -87,6 +95,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
return lease(run, workerId, ttlMs);
},
+ async personWaiting(createdBefore) {
+ return pendingRuns(Date.now()).some((run) => run.priority === 0 && run.createdAt <= createdBefore);
+ },
+
async claimById(runId, workerId, ttlMs) {
const run = runs.get(runId);
if (!run || pendingRuns(Date.now()).find((pending) => pending.sessionId === run.sessionId)?.id !== runId)
@@ -109,6 +121,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
async releaseLease(runId, leaseToken, stop) {
const run = runs.get(runId);
if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;
+ if (stop?.kind === "parked") run.preemptions += 1;
run.status = "pending";
run.leaseToken = null;
run.leaseExpiresAt = null;
@@ -377,7 +390,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
run.leaseExpiresAt = null;
run.workerId = null;
if (opts?.countsAsError) run.errorAttempts += 1;
- const overClaimed = run.attempts >= maxClaims;
+ const overClaimed = countedClaims(run) >= maxClaims;
if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {
run.status = "pending";
run.lastStop = opts?.stop ?? (opts?.countsAsError ? { kind: "error" } : { kind: "lease_expired" });
@@ -387,7 +400,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
run.status = "failed";
const reason =
!opts?.countsAsError && overClaimed && retry && run.errorAttempts < run.maxAttempts
- ? `run parked after ${run.attempts} claims without completing (suspected crash loop)`
+ ? `run parked after ${countedClaims(run)} claims without completing (suspected crash loop)`
: error;
run.result = { status: "failed", sessionId: run.sessionId, reason };
run.finishedAt = Date.now();
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index afac277e6..829e2730c 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -9,7 +9,8 @@ import type { TurnResult } from "../types.ts";
import type { OrchestratorInput } from "../core/orchestrator.ts";
import { resolveTurnOrigin } from "../core/turn-origin.ts";
import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";
-import { isTerminal, releasesDedupKey } from "./run-store.ts";
+import type { RunPriority } from "./run-store.ts";
+import { countedClaims, isTerminal, releasesDedupKey, runPriority } from "./run-store.ts";
import { errMessage, swallow } from "../util/errors.ts";
import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";
@@ -37,6 +38,8 @@ function rowToRun(r: Record<string, unknown>): Run {
attempts: Number(r.attempts),
errorAttempts: Number(r.error_attempts),
maxAttempts: Number(r.max_attempts),
+ priority: Number(r.claim_priority) as RunPriority,
+ preemptions: Number(r.preemptions),
leaseToken: (r.lease_token as string | null) ?? null,
leaseExpiresAt: r.lease_expires_at === null ? null : Number(r.lease_expires_at),
workerId: (r.worker_id as string | null) ?? null,
@@ -137,6 +140,20 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
id: "runs/store/0007-last-stop",
statements: [`SET LOCAL lock_timeout = '3s'`, `ALTER TABLE runs ADD COLUMN IF NOT EXISTS last_stop TEXT`],
},
+ {
+ id: "runs/store/0008-claim-priority",
+ statements: [
+ `SET LOCAL lock_timeout = '3s'`,
+ `ALTER TABLE runs ADD COLUMN IF NOT EXISTS claim_priority SMALLINT NOT NULL DEFAULT 2`,
+ `ALTER TABLE runs ADD COLUMN IF NOT EXISTS preemptions INT NOT NULL DEFAULT 0`,
+ ],
+ },
+ {
+ id: "runs/store/0009-claim-priority-index",
+ statements: [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_pending_priority ON runs(claim_priority, created_at, seq) WHERE status='pending'`,
+ ],
+ },
],
[
{
@@ -229,7 +246,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
const ifExpiredAt = opts?.ifExpiredAt ?? null;
const countsAsError = opts?.countsAsError ?? false;
const errorAttemptsAfter = run.errorAttempts + (countsAsError ? 1 : 0);
- const overClaimed = run.attempts >= maxClaims;
+ const overClaimed = countedClaims(run) >= maxClaims;
if (retry && errorAttemptsAfter < run.maxAttempts && !overClaimed) {
const { rowCount } = await q(
`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,
@@ -248,7 +265,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
}
const reason =
!countsAsError && overClaimed && retry && errorAttemptsAfter < run.maxAttempts
- ? `run parked after ${run.attempts} claims without completing (suspected crash loop)`
+ ? `run parked after ${countedClaims(run)} claims without completing (suspected crash loop)`
: error;
const result: TurnResult = { status: "failed", sessionId: run.sessionId, reason };
const { rowCount } = await q(
@@ -261,6 +278,13 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
return { requeued: false, applied: rowCount > 0 };
}
+ const claimable = (now: string) => `candidate.status='pending' AND candidate.retry_after <= ${now}
+ AND NOT EXISTS (
+ SELECT 1 FROM runs sibling WHERE sibling.session_id=candidate.session_id
+ AND (sibling.status='running' OR (sibling.status='pending'
+ AND (sibling.retry_after > ${now} OR (sibling.created_at, sibling.seq) < (candidate.created_at, candidate.seq))))
+ )`;
+
async function claim(workerId: string, ttlMs: number, runId?: string, sessionId?: string): Promise<Run | null> {
const token = randomUUID();
const now = Date.now();
@@ -269,16 +293,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
`UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,
attempts=attempts+1, started_at=COALESCE(started_at,$4)
WHERE id = (
- SELECT candidate.id FROM runs candidate WHERE candidate.status='pending'
- AND candidate.retry_after <= $4
+ SELECT candidate.id FROM runs candidate WHERE ${claimable("$4")}
AND ($5::text IS NULL OR candidate.id=$5)
AND ($6::text IS NULL OR candidate.session_id=$6)
- AND NOT EXISTS (
- SELECT 1 FROM runs sibling WHERE sibling.session_id=candidate.session_id
- AND (sibling.status='running' OR (sibling.status='pending'
- AND (sibling.retry_after > $4 OR (sibling.created_at, sibling.seq) < (candidate.created_at, candidate.seq))))
- )
- ORDER BY candidate.created_at ASC, candidate.seq ASC FOR UPDATE SKIP LOCKED LIMIT 1
+ ORDER BY candidate.claim_priority ASC, candidate.created_at ASC, candidate.seq ASC FOR UPDATE SKIP LOCKED LIMIT 1
) RETURNING *`,
[token, now + ttlMs, workerId, now, runId ?? null, sessionId ?? null],
);
@@ -309,13 +327,15 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
async enqueue({ sessionId, request, dedupKey, maxAttempts = 3 }: EnqueueInput): Promise<EnqueueResult> {
const id = randomUUID();
+ const delegating = request.delegatingRunId ? await getRun(request.delegatingRunId) : null;
+ const priority = runPriority(request, delegating?.priority ?? null);
for (;;) {
const { rows } = await q(
- `INSERT INTO runs(id, session_id, status, request, idempotency_key, attempts, max_attempts, created_at)
- VALUES ($1,$2,'pending',$3,$4,0,$5,$6)
+ `INSERT INTO runs(id, session_id, status, request, idempotency_key, attempts, max_attempts, created_at, claim_priority)
+ VALUES ($1,$2,'pending',$3,$4,0,$5,$6,$7)
ON CONFLICT (idempotency_key) DO NOTHING
RETURNING *, pg_notify('qm_run_available', 'null')`,
- [id, sessionId, jsonbStringify(request), dedupKey ?? null, maxAttempts, Date.now()],
+ [id, sessionId, jsonbStringify(request), dedupKey ?? null, maxAttempts, Date.now(), priority],
);
if (rows[0]) return { run: rowToRun(rows[0]), deduped: false };
const existing = await runs.getByDedupKey(dedupKey!);
@@ -332,6 +352,16 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
claimById: (runId, workerId, ttlMs) => claim(workerId, ttlMs, runId),
claimForSession: (sessionId, workerId, ttlMs) => claim(workerId, ttlMs, undefined, sessionId),
+ async personWaiting(createdBefore) {
+ const { rows } = await q(
+ `SELECT EXISTS (
+ SELECT 1 FROM runs candidate WHERE ${claimable("$2")}
+ AND candidate.claim_priority=0 AND candidate.created_at <= $1
+ ) AS waiting`,
+ [createdBefore, Date.now()],
+ );
+ return rows[0]?.waiting === true;
+ },
async heartbeat(runId, leaseToken, ttlMs): Promise<boolean> {
const { rowCount } = await q(
@@ -343,8 +373,8 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
async releaseLease(runId, leaseToken, stop): Promise<boolean> {
const { rowCount } = await q(
- "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, last_stop=COALESCE($3, last_stop) WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",
- [runId, leaseToken, stop ? JSON.stringify(stop) : null],
+ "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, last_stop=COALESCE($3, last_stop), preemptions=preemptions+$4 WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",
+ [runId, leaseToken, stop ? JSON.stringify(stop) : null, stop?.kind === "parked" ? 1 : 0],
);
return rowCount > 0;
},
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 532d098bb..477a93d6b 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -2,9 +2,19 @@ import type { SubscribeOptions } from "../util/event-bus.ts";
import type { RunSignal, RunSignalStore } from "./run-signal-store.ts";
import type { TurnResult } from "../types.ts";
import type { OrchestratorInput } from "../core/orchestrator.ts";
+import { isPersonAuthored, resolveTurnOrigin } from "../core/turn-origin.ts";
type RunStatus = "pending" | "running" | "done" | "failed";
+export type RunPriority = 0 | 1 | 2;
+export const MAX_PREEMPTIONS = 3;
+
+export function runPriority(request: OrchestratorInput, delegating: RunPriority | null): RunPriority {
+ if (request.swarm || (request.delegatingRunId && delegating === null)) return 2;
+ if (request.delegatingRunId) return delegating === 2 ? 2 : 1;
+ return isPersonAuthored(resolveTurnOrigin(request).kind) ? 0 : 2;
+}
+
export interface ReapEvent {
runId: string;
sessionId: string;
@@ -20,7 +30,16 @@ export interface RunDeliveryState {
}
export type RunStop =
- { kind: "shutdown" } | { kind: "lease_expired" } | { kind: "error"; code?: string; status?: number };
+ | { kind: "shutdown" }
+ | { kind: "parked" }
+ | { kind: "lease_expired" }
+ | { kind: "error"; code?: string; status?: number };
+
+export const PARKED: RunStop = { kind: "parked" };
+
+export function handbackStop(shutdown: AbortSignal): RunStop {
+ return shutdown.reason === PARKED ? PARKED : { kind: "shutdown" };
+}
export interface Run {
id: string;
@@ -34,6 +53,8 @@ export interface Run {
attempts: number;
errorAttempts: number;
maxAttempts: number;
+ priority: RunPriority;
+ preemptions: number;
leaseToken: string | null;
leaseExpiresAt: number | null;
workerId: string | null;
@@ -74,6 +95,8 @@ export interface RunStore {
claimForSession(sessionId: string, workerId: string, ttlMs: number): Promise<Run | null>;
+ personWaiting(createdBefore: number): Promise<boolean>;
+
heartbeat(runId: string, leaseToken: string, ttlMs: number): Promise<boolean>;
releaseLease(runId: string, leaseToken: string, stop?: RunStop): Promise<boolean>;
@@ -137,8 +160,15 @@ export function releasesDedupKey(result: TurnResult): boolean {
return result.refusalKind === "session_busy";
}
-export function errorParks(run: Pick<Run, "errorAttempts" | "maxAttempts" | "attempts">, maxClaims?: number): boolean {
- return run.errorAttempts + 1 >= run.maxAttempts || (maxClaims !== undefined && run.attempts >= maxClaims);
+export function countedClaims(run: Pick<Run, "attempts" | "preemptions">): number {
+ return run.attempts - run.preemptions;
+}
+
+export function errorParks(
+ run: Pick<Run, "errorAttempts" | "maxAttempts" | "attempts" | "preemptions">,
+ maxClaims?: number,
+): boolean {
+ return run.errorAttempts + 1 >= run.maxAttempts || (maxClaims !== undefined && countedClaims(run) >= maxClaims);
}
export function leaseLapsed(run: Pick<Run, "status" | "leaseExpiresAt">, asOf: number): boolean {
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index a451ee774..366541c19 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -6,7 +6,16 @@ import type { TurnResult } from "../types.ts";
import type { Orchestrator } from "../core/orchestrator.ts";
import { isNonRetryable, NonRetryableTurnError, ProviderTurnError, turnFailureMessage } from "../core/turn-error.ts";
import { resolveTurnOrigin } from "../core/turn-origin.ts";
-import { errorParks, type Run, type RunStop, type RunStore } from "./run-store.ts";
+import {
+ countedClaims,
+ errorParks,
+ handbackStop,
+ MAX_PREEMPTIONS,
+ PARKED,
+ type Run,
+ type RunStop,
+ type RunStore,
+} from "./run-store.ts";
import { errMessage, errorAlreadyReported, reportFailure, swallow } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
import { retryDelay } from "./retry-delay.ts";
@@ -75,7 +84,7 @@ export async function processRun(
const { turnMs } = resolveSwarmSettings({ turnMs: run.request.turnWallClockMs });
workDeadline = setTimeout(() => cancel.abort(), turnMs);
}
- if (run.request.swarm && run.attempts > 3) throw new NonRetryableTurnError("swarm claim budget exhausted");
+ if (run.request.swarm && countedClaims(run) > 3) throw new NonRetryableTurnError("swarm claim budget exhausted");
const queueMs = run.startedAt !== null ? Math.max(0, run.startedAt - run.createdAt) : undefined;
started = true;
const result = await deps.orchestrator.handleTurn({
@@ -124,7 +133,7 @@ export async function processRun(
opts?.shutdown?.removeEventListener("abort", onShutdown);
if (opts?.shutdown?.aborted)
await deps.runs
- .releaseLease(run.id, token, started ? { kind: "shutdown" } : undefined)
+ .releaseLease(run.id, token, started ? handbackStop(opts.shutdown) : undefined)
.catch((e) => swallow(`worker: shutdown handback failed run=${run.id}; lease will expire after exit`, e));
}
}
@@ -139,6 +148,8 @@ export interface WorkerDeps extends ProcessDeps {
export interface Worker {
start(): void;
+ held(): { run: Run; claimedAt: number; shutdown: AbortController } | null;
+ preempt(runId: string): void;
stopClaims(): Promise<void>;
drained(): Promise<void>;
stop(drainMs?: number): Promise<void>;
@@ -174,7 +185,12 @@ export function createWorker(deps: WorkerDeps): Worker {
let stopped = false;
let loopDone: Promise<void> | null = null;
let claimDone: Promise<void> | null = null;
- let inFlight: { shutdown: AbortController; done: Promise<void> } | null = null;
+ let inFlight: {
+ run: Run;
+ claimedAt: number;
+ shutdown: AbortController;
+ done: Promise<void>;
+ } | null = null;
async function loop(): Promise<void> {
let claimFailures = 0;
@@ -218,12 +234,15 @@ export function createWorker(deps: WorkerDeps): Worker {
}
const shutdown = new AbortController();
let settled!: () => void;
- inFlight = {
+ const current: NonNullable<typeof inFlight> = {
+ run,
+ claimedAt: Date.now(),
shutdown,
done: new Promise<void>((resolve) => {
settled = resolve;
}),
};
+ inFlight = current;
console.log(`[worker] claimed worker=${workerId} run=${run.id} thread=${run.sessionId}`);
claimed();
claimDone = null;
@@ -251,6 +270,11 @@ export function createWorker(deps: WorkerDeps): Worker {
}
return {
+ held: () => inFlight,
+ preempt(runId) {
+ if (inFlight?.run.id !== runId || inFlight.shutdown.signal.aborted) return;
+ inFlight.shutdown.abort(PARKED);
+ },
start() {
if (loopDone) return;
stopped = false;
@@ -283,3 +307,24 @@ function runStopFor(err: unknown): RunStop {
return { kind: "error", code: err.code, ...(err.status !== undefined ? { status: err.status } : {}) };
return { kind: "error" };
}
+
+export function createRunPreemption(deps: {
+ runs: RunStore;
+ workers: readonly Worker[];
+ canClaim?: () => boolean;
+ graceMs?: number;
+}): { sweep(): Promise<boolean> } {
+ return {
+ async sweep() {
+ if (deps.canClaim && !deps.canClaim()) return false;
+ const held = deps.workers.map((worker) => ({ worker, held: worker.held() }));
+ if (held.some(({ held }) => !held || held.shutdown.signal.aborted)) return false;
+ const victim = held
+ .filter(({ held }) => held!.run.priority === 2 && held!.run.preemptions < MAX_PREEMPTIONS)
+ .sort((a, b) => b.held!.claimedAt - a.held!.claimedAt)[0];
+ if (!victim || !(await deps.runs.personWaiting(Date.now() - (deps.graceMs ?? 1_000)))) return false;
+ victim.worker.preempt(victim.held!.run.id);
+ return true;
+ },
+ };
+}
diff --git a/src/wiring.ts b/src/wiring.ts
index e7fd78a13..6423641bd 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -351,7 +351,7 @@ import { createPostgresRunStore } from "./runs/postgres-run-store.ts";
import { createMemoryRunSignalStore, type RunSignalStore } from "./runs/run-signal-store.ts";
import { createPostgresRunSignalStore } from "./runs/postgres-run-signal-store.ts";
import { isTerminal, type Run, type RunStore } from "./runs/run-store.ts";
-import { createWorker, type Worker } from "./runs/worker.ts";
+import { createRunPreemption, createWorker, type Worker } from "./runs/worker.ts";
import {
createNoopInstanceRegistry,
createPostgresInstanceRegistry,
@@ -2727,6 +2727,7 @@ export function buildApp(
})
: createNoopInstanceRegistry();
const drain: DrainController = createDrainController({ registry: instanceRegistry });
+ const canClaimRuns = () => backgroundAdmission() && drain.canClaim();
const workers: Worker[] = Array.from({ length: Math.max(1, config.workers) }, () =>
createWorker({
admittedWork,
@@ -2736,9 +2737,11 @@ export function buildApp(
heartbeatIntervalMs: config.heartbeatIntervalMs,
errors,
pollMs: 250,
- canClaim: () => backgroundAdmission() && drain.canClaim(),
+ canClaim: canClaimRuns,
}),
);
+ const preemption = createRunPreemption({ runs, workers, canClaim: canClaimRuns });
+ const preemptionSweeper = createSweeper(() => preemption.sweep(), 1_000, { label: "run-preemption" });
const processReaper: ProcessReaper | null = processes
? createProcessReaper(processes, {
intervalMs: config.processReaperIntervalMs,
@@ -2818,6 +2821,7 @@ export function buildApp(
})
.catch(swallowAs("wiring: worker resume failed", undefined));
}
+ preemptionSweeper.start();
const startPeriodic = () => {
if (!backgroundRunning || generation !== backgroundGeneration) return;
reaper.start();
@@ -2875,6 +2879,7 @@ export function buildApp(
sessionReturnSweeper.stop(),
runResultSweeper.stop(),
approvalDeliverySweeper.stop(),
+ preemptionSweeper.stop(),
...workers.map((worker) => worker.stopClaims()),
];
backgroundClaimsStopping = Promise.all(stopping).then(() => {});
diff --git a/test/run-priority.test.ts b/test/run-priority.test.ts
new file mode 100644
index 000000000..8f2a1b845
--- /dev/null
+++ b/test/run-priority.test.ts
@@ -0,0 +1,211 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
+import { createPostgresRunStore } from "../src/runs/postgres-run-store.ts";
+import { createRunPreemption, createWorker } from "../src/runs/worker.ts";
+import type { Orchestrator, OrchestratorInput } from "../src/core/orchestrator.ts";
+import { handbackStop, PARKED, type Run, type RunStore } from "../src/runs/run-store.ts";
+import type { Principal, TurnResult } from "../src/types.ts";
+
+const URL = process.env.DATABASE_URL;
+const actor: Principal = { id: "internal:U1", type: "internal" };
+const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+const request = (text: string, over: Partial<OrchestratorInput> = {}): OrchestratorInput => ({
+ actor,
+ conversation: { kind: "dm", threadRef: "t", audience: [actor] },
+ origin: { kind: "human" },
+ text,
+ ...over,
+});
+const cron = (text: string) => request(text, { origin: { kind: "automation" } });
+const subagentOf = (parent: Run, text: string) =>
+ request(text, { origin: { kind: "automation" }, delegatingRunId: parent.id });
+const swarm = { swarmId: "s", messageId: "m", recipientId: "r" };
+
+const backends: Array<{ name: string; skip: string | false; make: () => Promise<RunStore> }> = [
+ { name: "memory", skip: false, make: async () => createMemoryRunStore().runs },
+ {
+ name: "postgres",
+ skip: URL ? false : "set DATABASE_URL (a Postgres) to run the Postgres run-priority tests",
+ make: async () => {
+ const pool = new (await import("pg")).default.Pool({ connectionString: URL });
+ await pool.query("DROP TABLE IF EXISTS runs, tool_calls CASCADE");
+ await pool.query("DELETE FROM qm_schema_migrations WHERE id LIKE 'runs/%'").catch(() => {});
+ await pool.end();
+ return createPostgresRunStore(URL!).runs;
+ },
+ },
+];
+
+async function claimAll(runs: RunStore): Promise<string[]> {
+ const order: string[] = [];
+ for (let run = await runs.claim("w", 60_000); run; run = await runs.claim("w", 60_000)) {
+ order.push(run.request.text);
+ await runs.complete(run.id, run.leaseToken!, { status: "ok", sessionId: run.sessionId, reply: "" });
+ }
+ return order;
+}
+
+for (const backend of backends) {
+ test(
+ `[${backend.name}] people's turns run first, then their subagents, then background work`,
+ { skip: backend.skip },
+ async () => {
+ const runs = await backend.make();
+ try {
+ const nightly = (await runs.enqueue({ sessionId: "cron", request: cron("cron") })).run;
+ const human = (await runs.enqueue({ sessionId: "dm", request: request("person") })).run;
+ const child = (await runs.enqueue({ sessionId: "c", request: subagentOf(human, "subagent") })).run;
+ await runs.enqueue({ sessionId: "g", request: subagentOf(child, "sub-subagent") });
+ await runs.enqueue({ sessionId: "nc", request: subagentOf(nightly, "cron subagent") });
+ await runs.enqueue({ sessionId: "sw", request: { ...subagentOf(human, "swarm worker"), swarm } });
+ await runs.enqueue({ sessionId: "shared", request: cron("background first in its conversation") });
+ await runs.enqueue({ sessionId: "shared", request: request("person second in its conversation") });
+ assert.equal(await runs.personWaiting(Date.now() - 60_000), false, "only turns past the grace period count");
+ assert.equal(await runs.personWaiting(Date.now()), true);
+ assert.deepEqual(await claimAll(runs), [
+ "person",
+ "subagent",
+ "sub-subagent",
+ "cron",
+ "cron subagent",
+ "swarm worker",
+ "background first in its conversation",
+ "person second in its conversation",
+ ]);
+ } finally {
+ await runs.close?.();
+ }
+ },
+ );
+}
+
+for (const backend of backends) {
+ test(
+ `[${backend.name}] a parked run records why it stopped and is not charged a claim`,
+ { skip: backend.skip },
+ async () => {
+ const runs = await backend.make();
+ try {
+ const queued = (await runs.enqueue({ sessionId: "cron", request: cron("cron") })).run;
+ const lease = (await runs.claim("w", 60_000))!;
+ assert.equal(await runs.releaseLease(lease.id, lease.leaseToken!, { kind: "parked" }), true);
+ const parked = (await runs.get(queued.id))!;
+ assert.deepEqual([parked.status, parked.preemptions, parked.lastStop], ["pending", 1, { kind: "parked" }]);
+ const again = (await runs.claim("w", 60_000))!;
+ await runs.releaseLease(again.id, again.leaseToken!, { kind: "shutdown" });
+ assert.equal((await runs.get(queued.id))!.preemptions, 1, "a deploy handback is not a park");
+ } finally {
+ await runs.close?.();
+ }
+ },
+ );
+}
+
+function scripted(log: string[]): Orchestrator {
+ return {
+ async handleTurn(input: OrchestratorInput): Promise<TurnResult> {
+ log.push(`start ${input.text} #${input.attempt}${input.resumeCause ? ` after ${input.resumeCause.kind}` : ""}`);
+ const [mode] = input.text.split(" ");
+ if (mode === "forever" || (input.attempt === 1 && mode !== "quick")) {
+ await new Promise<void>((resolve) =>
+ input.shutdown?.addEventListener("abort", () => resolve(), { once: true }),
+ );
+ if (mode === "slow") await sleep(150);
+ return { status: "silent", sessionId: "x", stopped: true };
+ }
+ log.push(`finish ${input.text}`);
+ return { status: "ok", sessionId: "x", reply: input.text };
+ },
+ } as unknown as Orchestrator;
+}
+
+async function until(check: () => Promise<boolean>): Promise<void> {
+ for (let i = 0; i < 200 && !(await check()); i++) await sleep(10);
+ assert.ok(await check(), "timed out");
+}
+
+async function withWorkers(
+ count: number,
+ body: (ctx: { runs: RunStore; log: string[]; sweep: () => Promise<boolean> }) => Promise<void>,
+ maxClaims?: number,
+): Promise<void> {
+ const { runs } = createMemoryRunStore(maxClaims ? { maxClaims } : undefined);
+ const log: string[] = [];
+ const workers = Array.from({ length: count }, () =>
+ createWorker({ runs, orchestrator: scripted(log), leaseTtlMs: 60_000, pollMs: 5 }),
+ );
+ const preemption = createRunPreemption({ runs, workers, graceMs: 0 });
+ for (const worker of workers) worker.start();
+ try {
+ await body({ runs, log, sweep: () => preemption.sweep() });
+ } finally {
+ await Promise.all(workers.map((worker) => worker.releaseInFlight()));
+ }
+}
+
+const status = (runs: RunStore, run: Run, want: string) => async () => (await runs.get(run.id))?.status === want;
+
+test("a busy worker parks a cron turn for a person's turn, then resumes it", () =>
+ withWorkers(
+ 1,
+ async ({ runs, log, sweep }) => {
+ const background = (await runs.enqueue({ sessionId: "cron", request: cron("long cron") })).run;
+ await until(status(runs, background, "running"));
+ await runs.enqueue({ sessionId: "dm", request: request("quick person") });
+ assert.equal(await sweep(), true);
+ await until(status(runs, background, "done"));
+ assert.equal((await runs.get(background.id))?.preemptions, 1);
+ assert.deepEqual(log, [
+ "start long cron #1",
+ "start quick person #1",
+ "finish quick person",
+ "start long cron #2 after parked",
+ "finish long cron",
+ ]);
+ },
+ 1,
+ ));
+
+test("one waiting turn parks one run, even while that run is slow to stop", () =>
+ withWorkers(3, async ({ runs, sweep }) => {
+ const crons: Run[] = [];
+ for (const name of ["a", "b", "c"])
+ crons.push((await runs.enqueue({ sessionId: name, request: cron(`slow ${name}`) })).run);
+ for (const run of crons) await until(status(runs, run, "running"));
+ const person = (await runs.enqueue({ sessionId: "dm", request: request("quick person") })).run;
+ assert.deepEqual([await sweep(), await sweep(), await sweep()], [true, false, false]);
+ await until(status(runs, person, "done"));
+ const parks = await Promise.all(crons.map(async (run) => (await runs.get(run.id))!.preemptions));
+ assert.deepEqual(parks.sort(), [0, 0, 1]);
+ }));
+
+test("subagents of a person's turn and thrice-parked runs keep their worker", () =>
+ withWorkers(2, async ({ runs, sweep }) => {
+ const parent = (await runs.enqueue({ sessionId: "parent", request: request("quick parent") })).run;
+ await until(status(runs, parent, "done"));
+ const capped = (
+ await runs.enqueue({ sessionId: "cron", request: { ...cron("forever cron"), swarm, turnWallClockMs: 60_000 } })
+ ).run;
+ for (let i = 0; i < 3; i++) {
+ const lease = (await runs.claim("w", 60_000))!;
+ await runs.releaseLease(lease.id, lease.leaseToken!, { kind: "parked" });
+ }
+ const child = (await runs.enqueue({ sessionId: "child", request: subagentOf(parent, "forever child") })).run;
+ await until(status(runs, capped, "running"));
+ await until(status(runs, child, "running"));
+ await runs.enqueue({ sessionId: "dm", request: request("quick person") });
+ assert.equal(await sweep(), false);
+ assert.equal((await runs.get(capped.id))?.attempts, 4, "parks don't spend a swarm worker's claim budget");
+ }));
+
+test("a tool result written after a park records the park, not a deploy", () => {
+ const parked = new AbortController();
+ parked.abort(PARKED);
+ const deploy = new AbortController();
+ deploy.abort();
+ assert.deepEqual(
+ [handbackStop(parked.signal), handbackStop(deploy.signal)],
+ [{ kind: "parked" }, { kind: "shutdown" }],
+ );
+});
diff --git a/test/run-result-delivery.test.ts b/test/run-result-delivery.test.ts
index 8cab48d3c..1f9b8c7cc 100644
--- a/test/run-result-delivery.test.ts
+++ b/test/run-result-delivery.test.ts
@@ -36,6 +36,8 @@ function run(over: Partial<Run>): Run {
attempts: 1,
errorAttempts: 0,
maxAttempts: 3,
+ priority: 2,
+ preemptions: 0,
leaseToken: null,
leaseExpiresAt: null,
workerId: null,