Multiplayer agent harness for work.
import { cronTriggerAuthority } from "./authority.ts";
import { WorkAdmissionClosed, type AdmittedWork } from "../util/admitted-work.ts";
import { randomUUID } from "node:crypto";
import { type Cron, type CronFireLogEntry, type CronFireNote, type TurnRequest, type TurnResult } from "../types.ts";
import type { IdentityService } from "../identity/identity-service.ts";
import { isDeferred, type CronStore } from "./cron-store.ts";
import type { DeliveryStore } from "../delivery/delivery-store.ts";
import type { IdempotencyStore } from "../idempotency/idempotency-store.ts";
import { runTrigger, type TriggerDeps } from "../triggers/run-trigger.ts";
import type { CurrentScopeMembers } from "../resolution/scope-membership.ts";
import type { VisibilityDirectory } from "../directory/visibility.ts";
import type { DirectoryMember } from "../directory/directory-store.ts";
import type { LeaderLease } from "../persistence/leader-lease.ts";
import { createSweeper } from "../util/sweeper.ts";
import { recoverNextFireAt } from "./schedule.ts";
import type { CronFireJob, CronJobQueue } from "./job-queue.ts";
import { hashId } from "../util/crypto.ts";
import { utcMinute } from "../util/time.ts";
import { errMessage, reportFailure, reportFailureAs } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
const TICK_LEASE_KEY = "cron:scheduler:tick";
const CRON_FIRE_REPLY_MAX_CHARS = 2000;
const STRANDED_SWEEP_INTERVAL_MS = 10 * 60_000;
const FIRE_GC_INTERVAL_MS = 6 * 60 * 60_000;
const BUSY_DEFER_MS = 30_000;
const BUSY_DEFER_MAX_LATE_MS = 10 * 60_000;
const MAINTENANCE_INTERVAL_MS = 5000;
const REPAIR_INTERVAL_MS = 60_000;
type FireResult = { authzFailed: boolean; deferred?: boolean };
type RunNowResult =
| { started: true; fireKey: string; settled: Promise<void> }
| { started: false; reason: "unavailable" }
| { started: false; reason: "already_running"; running: CronFireLogEntry };
export function describeRunNowRefusal(
id: string,
result: RunNowResult,
now = Date.now(),
): { error: "already_running" | "bad_request"; message: string } | null {
if (result.started) return null;
if (result.reason === "already_running") {
const ageMin = Math.round((now - result.running.firedAt) / 60_000);
return {
error: "already_running",
message: `cron ${id} is already firing (fire ${result.running.fireKey}, started ${ageMin}m ago) — wait for it to finish instead of firing again`,
};
}
return { error: "bad_request", message: `cron ${id} can't be fired on demand right now` };
}
export interface Scheduler {
tick(now?: number): Promise<void>;
runNow(cronId: string): Promise<RunNowResult>;
notifyChanged(cronId: string): void;
start(intervalMs: number): void;
ready(): Promise<void>;
stopClaims(): Promise<void>;
drained(): Promise<void>;
stop(): Promise<void>;
}
export interface SchedulerDeps {
admittedWork?: AdmittedWork;
advisoryLock: import("../persistence/advisory-lock.ts").AdvisoryLock;
crons: CronStore;
deliveries: DeliveryStore;
idempotency: IdempotencyStore;
identity: IdentityService;
run: (req: TurnRequest) => Promise<TurnResult>;
currentScopeMembers?: CurrentScopeMembers;
now?: () => number;
maxFiresPerTick?: number;
leaderLease: LeaderLease;
directory?: VisibilityDirectory & {
get(principalId: string): Promise<{ displayName: string } | null>;
list(): Promise<DirectoryMember[]>;
};
sweepAsks?: (now: number) => Promise<void>;
jobQueue?: CronJobQueue;
requireQueueStart?: boolean;
sessions?: TriggerDeps["sessions"];
resolveRecipient?: TriggerDeps["resolveRecipient"];
skipFire?: (cron: Cron) => Promise<string | undefined>;
deleteCron?: (id: string) => Promise<void>;
fireLoop?: (
loopId: string,
fireKey: string,
cronId: string,
) => Promise<{ status?: TurnResult["status"]; note?: string }>;
}
function truncate(s: string, maxChars: number): string {
return s.length <= maxChars ? s : `${s.slice(0, maxChars - 3)}...`;
}
const CRON_CONTEXT_MARKERS = ["[Cron runtime context]", "[End cron runtime context]"] as const;
export function echoesCronContextMarkers(s: string): boolean {
return CRON_CONTEXT_MARKERS.some((marker) => s.includes(marker));
}
function cronFireLogReply(s: string): string {
if (echoesCronContextMarkers(s)) {
return "[reply echoed cron runtime context; omitted]";
}
return truncate(s, CRON_FIRE_REPLY_MAX_CHARS);
}
function isOneShotSchedule(schedule: Cron["schedule"]): boolean {
return schedule.everyMs == null && schedule.cron == null;
}
export function cronFireReadsNotes(cron: Pick<Cron, "schedule" | "action" | "loopId">): boolean {
const task = cron.action ?? "";
if (cron.loopId || !task.trim() || /^!(run|scratch)\s/.test(task.trimStart())) return false;
return !isOneShotSchedule(cron.schedule);
}
export function flattenFireNote(text: string): string {
return text.replace(/\s+/g, " ").trim();
}
function fireNoteLine(note: CronFireNote | undefined): string[] {
if (!note || !Number.isFinite(note.at)) return [];
const text = flattenFireNote(note.text);
if (!text || echoesCronContextMarkers(text)) return [];
return [
note.by
? `Note left for this fire by ${note.by} (${utcMinute(note.at)}): ${text}`
: `Notes from last fire agent (${utcMinute(note.at)}): ${text}`,
];
}
function cronFireThreadRef(cronId: string, fireKey: string): string {
return `cron:${cronId}:fire:${hashId([fireKey], 12)}`;
}
const CRON_MENTION_ROSTER_MAX = 40;
function mentionable(m: DirectoryMember): string {
const mentionId = m.slackId ?? (/^[A-Z0-9]+$/i.test(m.principalId) ? m.principalId : undefined);
return `@${m.displayName}${mentionId ? ` (<@${mentionId}>)` : ""}`;
}
function cronMentionRoster(cron: Cron, homeRoster: DirectoryMember[] | undefined): string | undefined {
if (cron.destination?.audienceScopeId !== cron.ownerScopeId) return undefined;
const internal = homeRoster?.filter((m) => m.type === "internal") ?? [];
if (!internal.length || internal.length > CRON_MENTION_ROSTER_MAX) return undefined;
return internal.map(mentionable).join(", ");
}
function renderCronFireInput(cron: Cron, mentionRoster?: string): string {
const task = cron.action ?? "";
if (!task.trim()) return task;
if (/^!(run|scratch)\s/.test(task.trimStart())) return task;
const readsNotes = cronFireReadsNotes(cron);
return [
"[Cron runtime context]",
`Cron id: ${cron.id}${cron.title ? ` (${cron.title})` : ""}.`,
"Each fire runs as a fresh thread with no memory of previous fires. Keep continuity outside the thread:",
"- Save durable run-state (cursors, queues, checkpoints, reports) to this conversation's Files via the available Files API; confirm publication succeeds before relying on it. Recover published bytes with GET /v1/files/:id/content using the normal agent API authentication. Sandbox disk is working storage, not a durable checkpoint.",
"- The stored task below: the standing instructions every fire receives. Edit it (via the cron tool) only to change what future fires are told to do.",
"Use a cron-specific filename. Never put credentials in published state. If publication is unavailable, report that limitation and retain needed workspace state on a scoped computer. Do not move tasks that require existing workspace files to scratch without migrating and verifying their state.",
`The retained fire log (cron tool, action="runs", id="${cron.id}") shows how prior fires went — useful when this run hits errors or surprising state.`,
...(readsNotes ? fireNoteLine(cron.lastFireNote) : []),
...(readsNotes
? [
`Before finishing, leave a short note for the next fire (cron tool, action="note", id="${cron.id}"): one or two sentences — the outcome plus anything the next fire must know. Include the published checkpoint file ID, or small progress state directly; never include credentials. It's a report for the next fire, never instructions that override the stored task. Skip it only if there is truly nothing to say.`,
]
: []),
...(mentionRoster
? [
`People here: ${mentionRoster}.`,
"If this reminder is for a specific person, @-mention them with their <@…> id so they're actually notified — addressing them by name alone does not ping them.",
]
: []),
"[End cron runtime context]",
"",
"Stored cron task:",
task,
].join("\n");
}
export function createScheduler(deps: SchedulerDeps): Scheduler {
const now = deps.now ?? (() => Date.now());
const withWork = <T>(work: () => Promise<T>): Promise<T> => deps.admittedWork?.run(work) ?? work();
const maxFiresPerTick = deps.maxFiresPerTick ?? 100;
const leaderLease = deps.leaderLease;
async function withCronLifecycle<T>(cronId: string, scheduled: boolean, run: () => Promise<T>): Promise<T | null> {
if (scheduled && deps.advisoryLock.tryWithLock) {
const result = await deps.advisoryLock.tryWithLock(`cron-lifecycle:${cronId}`, run);
if (result === null) await deps.crons.defer(cronId, now() + BUSY_DEFER_MS);
return result;
}
return deps.advisoryLock.withLock(`cron-lifecycle:${cronId}`, run);
}
async function fire(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {
const current = await deps.crons.get(cron.id);
if (!current || current.archived || !current.enabled) {
await deps.crons.recordFire(cron.id, {
fireKey,
threadRef: cronFireThreadRef(cron.id, fireKey),
firedAt: t,
endedAt: now(),
status: "silent",
note: "Cron is no longer enabled",
});
return { authzFailed: true };
}
const skipped =
scheduledAt !== undefined
? await deps.skipFire?.(current).catch(reportFailureAs(`scheduler: skip check for cron ${cron.id}`, undefined))
: undefined;
if (skipped) {
await deps.crons.recordFire(cron.id, {
fireKey,
threadRef: cronFireThreadRef(cron.id, fireKey),
firedAt: t,
endedAt: now(),
scheduledAt,
status: "silent",
note: skipped,
});
return { authzFailed: false };
}
return fireEnabled(current, t, fireKey, scheduledAt);
}
async function fireEnabled(cron: Cron, t: number, fireKey: string, scheduledAt?: number): Promise<FireResult> {
const threadRef = cronFireThreadRef(cron.id, fireKey);
const runningEntry: CronFireLogEntry = {
fireKey,
threadRef,
firedAt: t,
...(scheduledAt !== undefined ? { scheduledAt } : {}),
status: "running",
};
if (cron.loopId) {
await deps.crons.beginFire(cron.id, runningEntry);
let result: { status?: TurnResult["status"]; note?: string };
if (!deps.fireLoop) {
result = { status: "failed", note: "loop service unavailable" };
} else
try {
result = await deps.fireLoop(cron.loopId, fireKey, cron.id);
} catch (e) {
result = { status: "failed", note: errMessage(e) };
}
await deps.crons.recordFire(cron.id, {
fireKey,
threadRef,
firedAt: t,
endedAt: now(),
...(scheduledAt !== undefined ? { scheduledAt } : {}),
status: result.status ?? "ok",
...(result.note ? { note: truncate(result.note, CRON_FIRE_REPLY_MAX_CHARS) } : {}),
});
if (isOneShotSchedule(cron.schedule)) await deps.crons.setEnabled(cron.id, false);
return { authzFailed: false };
}
let outcome: Awaited<ReturnType<typeof runTrigger>>;
try {
outcome = await runTrigger(
{
deliveries: deps.deliveries,
idempotency: deps.idempotency,
identity: deps.identity,
run: deps.run,
...(deps.directory ? { directory: deps.directory } : {}),
...(deps.currentScopeMembers ? { currentScopeMembers: deps.currentScopeMembers } : {}),
...(deps.sessions ? { sessions: deps.sessions } : {}),
...(deps.resolveRecipient ? { resolveRecipient: deps.resolveRecipient } : {}),
},
{
...cronTriggerAuthority(cron),
input: (homeRoster) => renderCronFireInput(cron, cronMentionRoster(cron, homeRoster)),
fireKey,
threadRef,
surface: "cron",
...(cron.runtime ? { runtime: cron.runtime } : {}),
...(cron.title ? { title: cron.title } : {}),
onClaimed: async () => {
await deps.crons.beginFire(cron.id, runningEntry);
},
...(cron.message !== undefined ? { message: cron.message } : {}),
...(cron.destination ? { destination: cron.destination } : {}),
deferWhenBusy: scheduledAt !== undefined && t - scheduledAt <= BUSY_DEFER_MAX_LATE_MS,
},
);
} catch (e) {
await deps.crons.recordFire(cron.id, {
fireKey,
threadRef,
firedAt: t,
endedAt: now(),
...(scheduledAt !== undefined ? { scheduledAt } : {}),
status: "failed",
note: truncate(errMessage(e), CRON_FIRE_REPLY_MAX_CHARS),
});
throw e;
}
if (outcome.deferred) {
const deferUntil = now() + BUSY_DEFER_MS;
await deps.crons.recordFire(cron.id, {
fireKey,
threadRef,
firedAt: t,
endedAt: now(),
...(scheduledAt !== undefined ? { scheduledAt } : {}),
status: "deferred",
note: `session busy — retrying at ${utcMinute(deferUntil)}`,
});
await deps.crons.defer(cron.id, deferUntil);
return { authzFailed: false, deferred: true };
}
if (outcome.ran || outcome.authzFailed) {
await deps.crons.recordFire(cron.id, {
fireKey,
threadRef,
firedAt: t,
endedAt: now(),
...(scheduledAt !== undefined ? { scheduledAt } : {}),
status: outcome.status ?? (outcome.authzFailed ? "refused" : "ok"),
...(outcome.note ? { note: truncate(outcome.note, CRON_FIRE_REPLY_MAX_CHARS) } : {}),
...(outcome.reply !== undefined ? { reply: cronFireLogReply(outcome.reply) } : {}),
...(outcome.sessionId ? { sessionId: outcome.sessionId } : {}),
});
}
if (outcome.ownerLeft) await (deps.deleteCron ? deps.deleteCron(cron.id) : deps.crons.delete(cron.id));
else if (outcome.authzFailed) await deps.crons.setEnabled(cron.id, false);
if (outcome.authzFailed) return { authzFailed: true };
if (isOneShotSchedule(cron.schedule)) await deps.crons.setEnabled(cron.id, false);
return { authzFailed: false };
}
let lastStrandedSweep = 0;
const sweepStranded = async (t: number): Promise<void> => {
if (t - lastStrandedSweep < STRANDED_SWEEP_INTERVAL_MS) return;
lastStrandedSweep = t;
try {
const swept = await deps.crons.sweepStrandedFires(t);
if (swept > 0) console.warn(`[scheduler] closed ${swept} stranded running fire(s) as failed`);
} catch (e) {
reportFailure("scheduler: stranded-fire sweep", e);
}
};
let lastFireGc = 0;
const gcFires = async (t: number): Promise<void> => {
if (t - lastFireGc < FIRE_GC_INTERVAL_MS) return;
lastFireGc = t;
try {
const pruned = await deps.crons.pruneFires(t);
if (pruned > 0) console.log(`[scheduler] pruned ${pruned} old cron fire row(s)`);
} catch (e) {
reportFailure("scheduler: cron fire gc", e);
}
};
const fireDue = async (t: number, observed: number): Promise<void> => {
const due = await deps.crons.due(t);
let batch = due;
if (due.length > maxFiresPerTick) {
const ordered = [...due].sort((a, b) => (a.lastAttemptAt ?? 0) - (b.lastAttemptAt ?? 0));
batch = [];
for (const cron of ordered) {
if (batch.length >= maxFiresPerTick) break;
try {
await deps.crons.markAttempted(cron.id, t);
batch.push(cron);
} catch (e) {
reportFailure("scheduler: attempt mark (holding this cron back)", e);
}
}
console.warn(`[scheduler] fan-out capped: firing ${batch.length}/${due.length} due crons this tick`);
}
for (const cron of batch) {
if (stopped || observed !== epoch) break;
try {
await withCronLifecycle(cron.id, true, async () => {
if (stopped || observed !== epoch) return;
const { authzFailed, deferred } = await fire(
cron,
t,
`cron:${cron.id}:${cron.scheduledAt}`,
cron.scheduledAt,
);
if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);
});
} catch (e) {
reportFailure("scheduler: fire", e);
}
}
};
const tick = async (nowArg?: number): Promise<void> => {
const t = nowArg ?? now();
const observed = epoch;
await withWork(() =>
leaderLease.hold(TICK_LEASE_KEY, async () => {
await fireDue(t, observed);
await sweepStranded(t);
await gcFires(t);
await deps.sweepAsks?.(t).catch(reportFailureAs("scheduler: ask sweep", undefined));
}),
);
};
const makeSweeper = () =>
createSweeper(() => tick().catch(reportFailureAs("scheduler: tick", undefined)), 1000, {
label: "scheduler",
});
let sweeper = makeSweeper();
function nextSlot(cron: Cron): number | undefined {
return recoverNextFireAt(cron.schedule, cron.createdAt, cron.lastFiredAt, cron.nextFireAt);
}
function nextJob(cron: Cron): CronFireJob | undefined {
const slot = nextSlot(cron);
if (slot === undefined) return undefined;
return {
cronId: cron.id,
scheduledAt: slot,
...(cron.deferUntil !== undefined ? { notBefore: cron.deferUntil } : {}),
};
}
async function enqueueNext(cronId: string): Promise<void> {
const cron = await deps.crons.get(cronId);
if (!cron || cron.archived || !cron.enabled) return;
const job = nextJob(cron);
if (job) await deps.jobQueue!.enqueueFire(job);
}
async function fireJob(job: CronFireJob): Promise<void> {
const observed = epoch;
const cron = await deps.crons.get(job.cronId);
if (!cron || cron.archived || !cron.enabled) return;
const slot = nextSlot(cron);
if (slot !== job.scheduledAt) return;
const t = now();
if (t < slot) {
await deps.jobQueue!.enqueueFire(job);
return;
}
if (isDeferred(cron, t)) {
await deps.jobQueue!.enqueueFire({ ...job, notBefore: cron.deferUntil! });
return;
}
if (stopped || observed !== epoch) return;
const result = await withCronLifecycle(job.cronId, true, async () => {
if (stopped || observed !== epoch) return;
if (!(await deps.crons.claimSlot(job.cronId, slot, t))) return;
if (stopped || observed !== epoch) {
await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
return;
}
try {
const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${slot}`, slot);
if (authzFailed || deferred) {
await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
if (deferred) await enqueueNext(job.cronId);
return;
}
} catch (e) {
reportFailure("scheduler: fire", e);
await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
return;
}
await enqueueNext(job.cronId);
});
if (result === null) await enqueueNext(job.cronId);
}
async function reconcile(): Promise<void> {
try {
for (const cron of await deps.crons.list()) {
if (stopped) break;
if (cron.archived || !cron.enabled) continue;
const job = nextJob(cron);
if (job) await deps.jobQueue!.enqueueFire(job);
}
} catch (e) {
reportFailure("scheduler: reconcile", e);
}
}
async function maintain(): Promise<void> {
await sweepStranded(now());
await gcFires(now());
await deps.sweepAsks?.(now()).catch(reportFailureAs("scheduler: ask sweep", undefined));
}
let stopSignal = Promise.withResolvers<void>();
const leaseGuard = createSweeper(
() =>
deps.jobQueue!.healthy()
? leaderLease.hold(TICK_LEASE_KEY, async (lost) => {
let lockLost = false;
void lost.then(() => (lockLost = true));
let repairAt = 0;
while (!stopped && !lockLost && deps.jobQueue!.healthy()) {
if (Date.now() >= repairAt) {
repairAt = Date.now() + REPAIR_INTERVAL_MS;
await runQueueTask(reconcile);
}
await runQueueTask(maintain);
await Promise.race([sleep(MAINTENANCE_INTERVAL_MS, { unref: true }), lost, stopSignal.promise]);
}
})
: undefined,
1000,
{ label: "scheduler lease guard", immediate: true },
);
let stopped = false;
let epoch = 0;
let started = false;
let stopFailed = false;
let starting: Promise<void> | null = null;
let stopping: Promise<void> | null = null;
let pausing: Promise<void> | null = null;
const oldSweeps = new Set<Promise<void>>();
const retireSweep = (work: Promise<void>) => {
oldSweeps.add(work);
void work.finally(() => oldSweeps.delete(work));
};
const pending = new Set<Promise<unknown>>();
async function runQueueTask(work: () => Promise<void>): Promise<void> {
if (stopped) return;
const task = withWork(work);
pending.add(task);
try {
await task;
} finally {
pending.delete(task);
}
}
return {
tick,
async runNow(cronId) {
try {
const preparing = withWork(async (): Promise<RunNowResult> => {
const cron = await deps.crons.get(cronId);
if (!cron || cron.archived || !cron.enabled) return { started: false, reason: "unavailable" };
const t = now();
const fireKey = `cron:${cron.id}:manual:${randomUUID()}`;
const begin = await deps.crons.beginFire(
cronId,
{ fireKey, threadRef: cronFireThreadRef(cron.id, fireKey), firedAt: t, status: "running" },
{ exclusive: true },
);
if (!begin.begun) {
return begin.running
? { started: false, reason: "already_running", running: begin.running }
: { started: false, reason: "unavailable" };
}
const settled = withWork(() => withCronLifecycle(cron.id, false, () => fire(cron, t, fireKey))).then(
() => undefined,
reportFailureAs("scheduler: manual fire", undefined, `cron=${cronId}`),
);
pending.add(settled);
void settled.finally(() => pending.delete(settled));
return { started: true, fireKey, settled };
});
pending.add(preparing);
try {
return await preparing;
} finally {
pending.delete(preparing);
}
} catch (error) {
if (error instanceof WorkAdmissionClosed) return { started: false, reason: "unavailable" };
throw error;
}
},
notifyChanged(cronId) {
if (!deps.jobQueue) return;
void enqueueNext(cronId).catch(reportFailureAs("scheduler: cron enqueue", undefined, `cron=${cronId}`));
},
start(intervalMs) {
if (started || stopping || pausing || stopFailed) return;
started = true;
stopped = false;
stopSignal = Promise.withResolvers<void>();
if (!deps.jobQueue) {
sweeper.start(intervalMs);
return;
}
const observed = epoch;
starting = deps.jobQueue
.start({
onFire: (job) => (observed === epoch ? runQueueTask(() => fireJob(job)) : Promise.resolve()),
})
.then(
() => {
if (!stopped) leaseGuard.start();
},
(e: unknown) => {
if (deps.requireQueueStart) throw e;
reportFailure("scheduler: cron job queue start (falling back to interval ticks)", e);
if (!stopped) sweeper.start(intervalMs);
},
);
},
async ready() {
await starting;
},
stopClaims() {
if (pausing) return pausing;
stopped = true;
started = false;
epoch++;
stopSignal.resolve();
retireSweep(sweeper.stop());
sweeper = makeSweeper();
const guard = leaseGuard.stop();
pausing = (async () => {
await starting?.catch(() => {});
if (deps.jobQueue?.stopClaims) await deps.jobQueue.stopClaims();
else await deps.jobQueue?.stop();
await guard;
stopFailed = false;
})()
.catch((error) => {
stopFailed = true;
throw error;
})
.finally(() => {
pausing = null;
});
return pausing;
},
async drained() {
while (oldSweeps.size || pending.size) await Promise.all([...oldSweeps, ...pending]);
},
stop() {
if (stopping) return stopping;
stopped = true;
epoch++;
stopSignal.resolve();
started = false;
const sweeps = [sweeper.stop(), leaseGuard.stop()];
stopping = (async () => {
await starting?.catch(() => {});
if (deps.jobQueue?.stopClaims) await deps.jobQueue.stopClaims();
await pausing;
await Promise.all(sweeps);
await Promise.all(oldSweeps);
while (pending.size) await Promise.allSettled(pending);
await deps.jobQueue?.stop();
stopFailed = false;
})()
.catch((error: unknown) => {
stopFailed = true;
throw error;
})
.finally(() => {
stopping = null;
});
return stopping;
},
};
}