Multiplayer agent harness for work.
Report caught backend failures at shared boundaries to Sentry (#1408)
Regan Bellee37609c4f7f
diff --git a/src/admin/error-log.ts b/src/admin/error-log.tsindex 10f7986b7..a0e52343c 100644--- a/src/admin/error-log.ts+++ b/src/admin/error-log.ts@@ -1,4 +1,5 @@import { reportBackendError } from "../../plugins/chassis/src/error-reporting.ts";+import { errorAlreadyReported, markErrorReported } from "../util/errors.ts";import type { ScopeId } from "../types.ts";import { createTimestampedEventSink } from "./scoped-event-sink.ts";@@ -30,21 +31,14 @@ export function createErrorLog(): ErrorLog {};}-const recordedErrors = new WeakSet<object>();--export function markErrorRecorded(err: unknown): void {- if (typeof err === "object" && err !== null) recordedErrors.add(err);-}--export function errorAlreadyRecorded(err: unknown): boolean {- return typeof err === "object" && err !== null && recordedErrors.has(err);-}-export function withErrorReporting(store: ErrorLog): ErrorLog {return {...store,record(event, error) {- reportBackendError(error ?? new Error("Recorded backend failure"), `${event.category}:${event.code}`);+ if (!errorAlreadyReported(error)) {+ reportBackendError(error ?? new Error("Recorded backend failure"), `${event.category}:${event.code}`);+ markErrorReported(error);+ }store.record(event);},};diff --git a/src/admin/postgres-audit-log.ts b/src/admin/postgres-audit-log.tsindex 2a36cfad9..7a5e0cdfa 100644--- a/src/admin/postgres-audit-log.ts+++ b/src/admin/postgres-audit-log.ts@@ -1,7 +1,7 @@import { createPgPool } from "../persistence/pg-pool.ts";import type { ScopeId } from "../types.ts";import type { AuditEvent, AuditLog } from "../audit/audit-log.ts";-import { errMessage } from "../util/errors.ts";+import { reportFailureAs } from "../util/errors.ts";function rowToEvent(r: Record<string, unknown>): AuditEvent {return {@@ -72,7 +72,7 @@ export function createPostgresAuditLog(connectionString: string): AuditLog {e.detail ?? null,]).then(() => undefined)- .catch((err) => console.error("[audit] failed to persist event to durable store:", errMessage(err)));+ .catch(reportFailureAs("audit: persist event", undefined));pendingWrites.add(write);void write.finally(() => pendingWrites.delete(write));},diff --git a/src/admin/postgres-metrics-sink.ts b/src/admin/postgres-metrics-sink.tsindex 944f1baea..77144c322 100644--- a/src/admin/postgres-metrics-sink.ts+++ b/src/admin/postgres-metrics-sink.ts@@ -1,6 +1,6 @@import { createPostgresEventSink, scopedEventMigrationId, type EventColumn } from "./scoped-event-sink.ts";import type { MetricsSink, TurnMetricSample } from "./metrics-sink.ts";-import { errMessage } from "../util/errors.ts";+import { reportFailureAs } from "../util/errors.ts";type MetricColumn = EventColumn<keyof TurnMetricSample & string>;@@ -96,7 +96,7 @@ export function createPostgresMetricsSink(connectionString: string): MetricsSinkparams.push(runId);await sink.q(`UPDATE turn_metrics SET ${sets.join(", ")} WHERE run_id = $${params.length}`, params)- .catch((err) => console.error("[metrics] failed to patch turn metric:", errMessage(err)));+ .catch(reportFailureAs("metrics: patch turn metric", undefined));},list: (opts = {}) => sink.list(opts),};diff --git a/src/admin/scoped-event-sink.ts b/src/admin/scoped-event-sink.tsindex 598f419cf..ba9180a4e 100644--- a/src/admin/scoped-event-sink.ts+++ b/src/admin/scoped-event-sink.ts@@ -1,6 +1,6 @@import type { ScopeId } from "../types.ts";import { createPgPool, type PgMigrationDefinition, type PgPool } from "../persistence/pg-pool.ts";-import { errMessage } from "../util/errors.ts";+import { reportFailureAs } from "../util/errors.ts";export interface ScopedEvent {scopeLabel: ScopeId;@@ -193,7 +193,7 @@ export function createPostgresEventSink<E>(cfg: PostgresEventSinkConfig<E>): Posconst s = input as Record<string, unknown>;const values = cfg.columns.map(([, js]) => (js === "ts" ? Date.now() : (s[js] ?? null)));const write = q(insertSql, values)- .catch((err) => console.error(cfg.persistErrorMessage, errMessage(err)))+ .catch(reportFailureAs(cfg.persistErrorMessage, undefined)).finally(() => pendingWrites.delete(write));pendingWrites.add(write);},diff --git a/src/api/app-ambient.ts b/src/api/app-ambient.tsindex da19b54f6..c51002168 100644--- a/src/api/app-ambient.ts+++ b/src/api/app-ambient.ts@@ -12,7 +12,7 @@ import {type AmbientDecision,} from "../surface-cache/ambient-judge.ts";import type { AmbientJudgmentStore } from "../surface-cache/ambient-judgment-store.ts";-import { errMessage } from "../util/errors.ts";+import { reportFailureAs } from "../util/errors.ts";import { buildWakeEnvelope } from "../core/wake-envelope.ts";import { isTerminal } from "../runs/run-store.ts";import { unscreenedNotice } from "../security/security-posture.ts";@@ -36,9 +36,7 @@ export function createAmbientHelpers(deps: AppDeps, app: App) {const ambientSelf = new Map<string, { name?: string; mentionId?: string }>();const recordJudgment = (j: Parameters<AmbientJudgmentStore["record"]>[0]): void => {- void deps.ambientJudgments- ?.record(j)- .catch((e) => console.error("[ambient] record judgment failed:", errMessage(e)));+ void deps.ambientJudgments?.record(j).catch(reportFailureAs("ambient: record judgment", undefined));};async function judgeAmbientContainer(surface: string,@@ -292,7 +290,7 @@ export function createAmbientHelpers(deps: AppDeps, app: App) {idempotencyKey: `ambient:${orgIdOf()}:${batch.surface}:${batch.container}:${latestTs}`,};if (!solicited && (await steerIntoLiveAmbientRun(batch, latestTs, req))) return;- await app.turn(req).catch((e) => console.error("[ambient] spawn worker failed:", errMessage(e)));+ await app.turn(req).catch(reportFailureAs("ambient: spawn worker", undefined));}async function steerIntoLiveAmbientRun(batch: AmbientBatch, latestTs: string, req: TurnRequest): Promise<boolean> {diff --git a/src/api/app-messaging.ts b/src/api/app-messaging.tsindex a14ef6932..3a9d5d634 100644--- a/src/api/app-messaging.ts+++ b/src/api/app-messaging.ts@@ -3,7 +3,7 @@ import { orgId as orgIdOf } from "../config.ts";import { parseScopeId, scopeId } from "../types.ts";import { personKey, personKeys, samePersonInDirectory, samePersonMatcher } from "../directory/person.ts";import type { Destination, SurfaceContextRequest, SurfaceContextResult } from "../types.ts";-import { errMessage } from "../util/errors.ts";+import { reportFailureAs } from "../util/errors.ts";import { adminCronHistoryUrl } from "../util/admin-links.ts";import { createMemoryMap } from "../persistence/durable-map.ts";import { randomUUID } from "node:crypto";@@ -312,14 +312,12 @@ export function createMessagingMethods(if (self && (self.name || self.mentionId)) ambientSelf.set(`${orgIdOf()}:${surface}`, self);const out = await deps.surfaceCache.ingest(events);if (surface === "slack" && hasRevisionEvents(events)) {- void recordMessageRevisions(deps.sessions, events).catch((e) =>- console.error("[revisions] surface revision record failed:", errMessage(e)),+ void recordMessageRevisions(deps.sessions, events).catch(+ reportFailureAs("revisions: surface revision record", undefined),);}for (const container of new Set(events.filter((e) => !e.self).map((e) => e.container))) {- void judgeAmbientContainer(surface, container).catch((e) =>- console.error("[ambient] judge failed:", errMessage(e)),- );+ void judgeAmbientContainer(surface, container).catch(reportFailureAs("ambient: judge", undefined));}return out;},diff --git a/src/api/routes/admin-resources.ts b/src/api/routes/admin-resources.tsindex 1b9552a7b..ee89db7c0 100644--- a/src/api/routes/admin-resources.ts+++ b/src/api/routes/admin-resources.ts@@ -35,7 +35,7 @@ import { authorizeUrl, PROVIDERS, type ConsentMode } from "../../connectors/oautimport { resolverFor } from "./connectors.ts";import { encodeRef, serviceCredRef } from "../../acl/resource-ref.ts";import { audit } from "./shared.ts";-import { errMessage } from "../../util/errors.ts";+import { errMessage, reportFailure } from "../../util/errors.ts";import {DEFAULT_SECURITY_SCREEN_RUBRIC,parseSecurityPosture,@@ -1111,7 +1111,7 @@ export const ADMIN_RESOURCES: readonly AdminResource[] = [auditGrantChanges(grantChanges(replacement));return { ok: true };} catch (e) {- console.error("[admin] credential delete failed after conditional mutation:", errMessage(e));+ reportFailure("admin: credential delete after conditional mutation", e);return mutationFailed("delete");}}@@ -1219,7 +1219,7 @@ export const ADMIN_RESOURCES: readonly AdminResource[] = [auditGrantChanges(grantChanges(replacement));return { ok: true };} catch (e) {- console.error("[admin] credential update failed after conditional mutation:", errMessage(e));+ reportFailure("admin: credential update after conditional mutation", e);return mutationFailed(existing ? "update" : "create");}},diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex 4fa949505..c581225ca 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -11,7 +11,6 @@ import { evaluateCommandWithLayer } from "../policy/command-policy.ts";import { createSecretValueMasker } from "../security/secret-masking.ts";import { shq } from "../util/shell.ts";import { goalViewFromEntry } from "../runs/turn-stream.ts";-import { markErrorRecorded } from "../admin/error-log.ts";import type {CommandApprovalGrant,DeliveryProvenance,@@ -162,7 +161,7 @@ import {selectOverheardToImport,type OverheardEntryPayload,} from "../harness/replay.ts";-import { errMessage, swallow, swallowAs } from "../util/errors.ts";+import { errMessage, reportFailure, swallow, swallowAs } from "../util/errors.ts";import { isObj } from "../util/objects.ts";import { absoluteAppLinks, headSlice, jsonbSafeStringify } from "../util/text.ts";import { NonRetryableTurnError, TitleRejected, turnFailureMessage, type TurnFailurePayload } from "./turn-error.ts";@@ -738,7 +737,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {try {await deps.sessions.recordLlmRequest(screenSession.id, { ...rec, scopeLabel: scopeId }, signal);} catch (err) {- console.error("[orchestrator] failed to persist security screen request snapshot:", errMessage(err));+ reportFailure("orchestrator: persist security screen request snapshot", err);}};let screenedOverheard: OverheardEntryPayload[] = [];@@ -3412,7 +3411,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {try {await deps.sessions.recordLlmRequest(session.id, { ...rec, scopeLabel: scopeId }, signal);} catch (err) {- console.error("[orchestrator] failed to persist LLM request snapshot:", errMessage(err));+ reportFailure("orchestrator: persist LLM request snapshot", err);}},});@@ -3536,7 +3535,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {spine.surfaceOutboundCount += 1;if (input.runId) deps.turnStream?.markSurfacePosted(input.runId);} catch (e) {- console.error("%s", `[orchestrator] direct reply delivery failed session=${session.id}:`, errMessage(e));+ reportFailure("orchestrator: direct reply delivery", e, `session=${session.id}`);}}if (spine.surfaceOutboundCount === 0 && !silentPollNarration) {@@ -3969,7 +3968,6 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {},err,);- markErrorRecorded(err);if ((err instanceof NonRetryableTurnError || input.finalAttempt) && !input.cancel?.aborted) {const mirrorFailureEntry = async (entry: SessionEntry | undefined): Promise<void> => {if (!entry) return;diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.tsindex cc81b6748..58287143b 100644--- a/src/cron/scheduler.ts+++ b/src/cron/scheduler.ts@@ -22,7 +22,7 @@ import { recoverNextFireAt } from "./schedule.ts";import type { CronFireJob, CronJobQueue } from "./job-queue.ts";import { hashId } from "../util/crypto.ts";import { utcMinute } from "../util/time.ts";-import { errMessage } from "../util/errors.ts";+import { errMessage, reportFailure, reportFailureAs } from "../util/errors.ts";import { sleep } from "../util/async.ts";const TICK_LEASE_KEY = "cron:scheduler:tick";@@ -315,7 +315,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {const swept = await deps.crons.sweepStrandedFires(t);if (swept > 0) console.warn(`[scheduler] closed ${swept} stranded running fire(s) as failed`);} catch (e) {- console.error("[scheduler] stranded-fire sweep failed:", errMessage(e));+ reportFailure("scheduler: stranded-fire sweep", e);}};@@ -327,7 +327,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {const pruned = await deps.crons.pruneFires(t);if (pruned > 0) console.log(`[scheduler] pruned ${pruned} old cron fire row(s)`);} catch (e) {- console.error("[scheduler] cron fire GC failed:", errMessage(e));+ reportFailure("scheduler: cron fire gc", e);}};@@ -343,7 +343,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {await deps.crons.markAttempted(cron.id, t);batch.push(cron);} catch (e) {- console.error("[scheduler] attempt mark failed, holding this cron back:", errMessage(e));+ reportFailure("scheduler: attempt mark (holding this cron back)", e);}}console.warn(`[scheduler] fan-out capped: firing ${batch.length}/${due.length} due crons this tick`);@@ -354,7 +354,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt);if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);} catch (e) {- console.error("[scheduler] fire failed:", errMessage(e));+ reportFailure("scheduler: fire", e);}}};@@ -367,13 +367,13 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {await fireDue(t, observed);await sweepStranded(t);await gcFires(t);- await deps.sweepAsks?.(t).catch((e: unknown) => console.error("[scheduler] ask sweep failed:", errMessage(e)));+ await deps.sweepAsks?.(t).catch(reportFailureAs("scheduler: ask sweep", undefined));}),);};const makeSweeper = () =>- createSweeper(() => tick().catch((e: unknown) => console.error("[scheduler] tick failed:", errMessage(e))), 1000, {+ createSweeper(() => tick().catch(reportFailureAs("scheduler: tick", undefined)), 1000, {label: "scheduler",});@@ -429,7 +429,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {return;}} catch (e) {- console.error("[scheduler] fire failed:", errMessage(e));+ reportFailure("scheduler: fire", e);await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);return;}@@ -445,11 +445,11 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {if (job) await deps.jobQueue!.enqueueFire(job);}} catch (e) {- console.error("[scheduler] tick failed:", errMessage(e));+ reportFailure("scheduler: reconcile", e);}await sweepStranded(now());await gcFires(now());- await deps.sweepAsks?.(now()).catch((e: unknown) => console.error("[scheduler] ask sweep failed:", errMessage(e)));+ await deps.sweepAsks?.(now()).catch(reportFailureAs("scheduler: ask sweep", undefined));}let stopSignal = Promise.withResolvers<void>();@@ -511,7 +511,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {}const settled = withWork(() => fire(cron, t, fireKey)).then(() => undefined,- (e: unknown) => console.error("%s", `[scheduler] manual fire of cron ${cronId} failed:`, errMessage(e)),+ reportFailureAs("scheduler: manual fire", undefined, `cron=${cronId}`),);pending.add(settled);void settled.finally(() => pending.delete(settled));@@ -530,7 +530,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {},notifyChanged(cronId) {if (!deps.jobQueue) return;- void enqueueNext(cronId).catch((e: unknown) => console.error("[scheduler] cron enqueue failed:", errMessage(e)));+ void enqueueNext(cronId).catch(reportFailureAs("scheduler: cron enqueue", undefined, `cron=${cronId}`));},start(intervalMs) {if (started || stopping || pausing || stopFailed) return;@@ -556,7 +556,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {},(e: unknown) => {if (deps.requireQueueStart) throw e;- console.error("[scheduler] cron job queue failed to start; falling back to interval ticks:", errMessage(e));+ reportFailure("scheduler: cron job queue start (falling back to interval ticks)", e);if (!stopped) sweeper.start(intervalMs);},);diff --git a/src/delivery/run-result-delivery.ts b/src/delivery/run-result-delivery.tsindex 67eeeacc9..70bf1b15e 100644--- a/src/delivery/run-result-delivery.ts+++ b/src/delivery/run-result-delivery.ts@@ -14,7 +14,7 @@ import {type TranscriptAppendSessions,} from "../sessions/session-store.ts";import { turnRecordedFailure } from "./web-transcript-delivery.ts";-import { errMessage } from "../util/errors.ts";+import { reportFailureAs } from "../util/errors.ts";export interface RunResultDelivery {destination: Destination;@@ -140,8 +140,8 @@ export function wireRunResultDeliveries(): void {runs.onTerminal((run) => {if (sessions) {- void recordRunFailureEntry(sessions, run).catch((err) =>- console.error("%s", `[delivery] failed to record turn_failure entry for run ${run.id}:`, errMessage(err)),+ void recordRunFailureEntry(sessions, run).catch(+ reportFailureAs("delivery: record turn_failure entry", undefined, `run=${run.id}`),);}void (async () => {@@ -149,8 +149,6 @@ export function wireRunResultDeliveries(const delivery = runResultDelivery(run, taskList, adminUrlFor);if (!delivery) return;await deliveries.enqueue(delivery);- })().catch((err) =>- console.error("%s", `[delivery] failed to enqueue recovery delivery for run ${run.id}:`, errMessage(err)),- );+ })().catch(reportFailureAs("delivery: enqueue recovery delivery", undefined, `run=${run.id}`));});}diff --git a/src/delivery/web-transcript-delivery.ts b/src/delivery/web-transcript-delivery.tsindex cf42e01b4..ba0c8cc7e 100644--- a/src/delivery/web-transcript-delivery.ts+++ b/src/delivery/web-transcript-delivery.ts@@ -7,7 +7,7 @@ import {type TranscriptAppendSessions,} from "../sessions/session-store.ts";import { messageTag } from "../util/message-tag.ts";-import { errMessage, swallow } from "../util/errors.ts";+import { reportFailure, swallow } from "../util/errors.ts";const DEDUPE_SCAN_LIMIT = 200;const RECORDED_CACHE_CAP = 1000;@@ -128,9 +128,10 @@ export function withWebTranscriptDeliveries(store: DeliveryStore, sessions: WebTif (disposition === "deliver" || (await recordedForDelivery(d))) out.push(d);} catch (err) {if (overdue(d)) {- console.error(- `[delivery] web delivery ${d.idempotencyKey} could not be recorded in ${d.destination.target}'s transcript within ${WRITE_GIVEUP_MS}ms — delivering as a nudge only:`,- errMessage(err),+ reportFailure(+ "web delivery: transcript write gave up (delivering as a nudge only)",+ err,+ `delivery=${d.idempotencyKey} target=${d.destination.target} after=${WRITE_GIVEUP_MS}ms`,);remember(d.idempotencyKey);out.push(d);diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.tsindex 00653b00f..0327171c4 100644--- a/src/deploy/deploy-service.ts+++ b/src/deploy/deploy-service.ts@@ -20,7 +20,7 @@ import type { DeployProfile, DeployProvider } from "./deploy-provider.ts";import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";import { createKeyedQueue } from "../util/async.ts";-import { errMessage, swallow } from "../util/errors.ts";+import { errMessage, reportFailure, swallow } from "../util/errors.ts";export interface DeployFile {path: string;@@ -602,7 +602,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {scopeLabel: before.ownerScopeId,});} catch (e) {- console.error("%s", `[deploy] failed to register pushed version for ${id}:`, errMessage(e));+ reportFailure("deploy: register pushed version", e, `deployment=${id}`);deps.auditLog.record({at: Date.now(),principalId: "system",diff --git a/src/index.ts b/src/index.tsindex c254b99fb..91a7b4c03 100644--- a/src/index.ts+++ b/src/index.ts@@ -9,7 +9,7 @@ import { buildApp, serverDeps, stopWithBackstop } from "./wiring.ts";import { shutdownOnUncaught } from "./util/process-guard.ts";import { createServer } from "./api/server.ts";import { dockerDaemonFailure } from "./deploy/docker-deploy-provider.ts";-import { errMessage } from "./util/errors.ts";+import { errMessage, reportFailureAs } from "./util/errors.ts";import { slackAccountConfigsFromEnv, slackPluginConfigFromEnv, startSlackPlugin } from "./slack/index.ts";import { createSlackRuntimeReconciler } from "./surfaces/slack-runtime.ts";import { migrateRegisteredPgSchemas } from "./persistence/pg-pool.ts";@@ -113,7 +113,7 @@ const slackRuntime = createSlackRuntimeReconciler({return null;},startPlugin: (desired) => startSlackPlugin(desired, built.slackCore),- onError: (error) => console.error(`[qm] slack plugin reconciliation failed: ${errMessage(error)}`),+ onError: reportFailureAs("slack plugin reconciliation", undefined),});if (config.backgroundWorkEnabled && !config.backgroundDeploymentId) slackRuntime.start();@@ -122,8 +122,7 @@ const slackAccountRuntimes = slackAccountConfigsFromEnv(process.env).map((accounstartPaused: Boolean(config.backgroundDeploymentId),load: () => Promise.resolve({ version: `environment:${account.accountId}`, config: account }),startPlugin: (desired) => startSlackPlugin(desired, built.slackCore),- onError: (error) =>- console.error(`[qm] slack account "${account.accountId}" reconciliation failed: ${errMessage(error)}`),+ onError: reportFailureAs("slack account reconciliation", undefined, `account=${account.accountId}`),}),);if (config.backgroundWorkEnabled && !config.backgroundDeploymentId)@@ -180,7 +179,7 @@ if (built.backgroundOwnership) {async drained() {await Promise.all([built.runtime.backgroundDrained(), built.scheduler.drained(), periodicStop]);},- onError: (error) => console.error("[qm] background ownership failed:", errMessage(error)),+ onError: reportFailureAs("background ownership", undefined),});built.runtime.setBackgroundAdmission(backgroundController.canClaim);backgroundController.start();diff --git a/src/monitors/monitor-poller.ts b/src/monitors/monitor-poller.tsindex 9f36793a8..ad2b1c4d0 100644--- a/src/monitors/monitor-poller.ts+++ b/src/monitors/monitor-poller.ts@@ -10,7 +10,7 @@ import { processIsGone } from "../sandbox/process-poll.ts";import { runTrigger, type TriggerDeps, type TriggerOutcome } from "../triggers/run-trigger.ts";import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";import { createSweeper } from "../util/sweeper.ts";-import { errMessage } from "../util/errors.ts";+import { errMessage, reportFailureAs } from "../util/errors.ts";import type { CurrentScopeMembers } from "../resolution/scope-membership.ts";import { compileMonitorPattern } from "./monitor-broker.ts";import { buildEventWakeEnvelope, capForEscaping } from "../core/wake-envelope.ts";@@ -314,11 +314,9 @@ export function createMonitorPoller(deps: MonitorPollerDeps): MonitorPoller {else await work();};- const sweeper = createSweeper(- () => tick().catch((e: unknown) => console.error("[monitor] tick failed:", errMessage(e))),- 10_000,- { label: "monitor" },- );+ const sweeper = createSweeper(() => tick().catch(reportFailureAs("monitor: tick", undefined)), 10_000, {+ label: "monitor",+ });return {tick,start(intervalMs) {diff --git a/src/ratelimit/postgres-budget.ts b/src/ratelimit/postgres-budget.tsindex e3679ca55..5ecd03d20 100644--- a/src/ratelimit/postgres-budget.ts+++ b/src/ratelimit/postgres-budget.ts@@ -1,7 +1,7 @@import { createPgPool } from "../persistence/pg-pool.ts";import type { BudgetTracker } from "./budget.ts";import { DEFAULT_BUDGET_WINDOW_MS } from "./budget.ts";-import { errMessage } from "../util/errors.ts";+import { reportFailure } from "../util/errors.ts";export function createPostgresBudgetTracker(connectionString: string,@@ -46,7 +46,7 @@ export function createPostgresBudgetTracker();}} catch (err) {- console.error("[budget] failed to persist spend:", errMessage(err));+ reportFailure("budget: persist spend", err);}},};diff --git a/src/resolution/config-store.ts b/src/resolution/config-store.tsindex 446601713..655378c0a 100644--- a/src/resolution/config-store.ts+++ b/src/resolution/config-store.ts@@ -20,7 +20,7 @@ import {type PublicConnectorClient,type DecryptedConnectorClient,} from "../connectors/connector-client-store.ts";-import { errMessage } from "../util/errors.ts";+import { errMessage, reportFailureAs } from "../util/errors.ts";export interface PersistedSoul {scopeId: ScopeId;@@ -353,8 +353,7 @@ export function createMemoryConfigStore(const autoFlaggerStore = opts.autoFlaggerConfigs ?? createMemoryMap<PersistedAutoFlaggerConfig>();const turnWallClockStore = opts.turnWallClocks ?? createMemoryMap<PersistedTurnWallClock>();const deploymentIdentity = opts.deploymentIdentity ?? createMemoryMap<PersistedDeploymentIdentity>();- const persistWarn = (what: string) => (e: unknown) =>- console.error("%s", `[config] failed to persist ${what}:`, errMessage(e));+ const persistWarn = (what: string) => reportFailureAs(`config: persist ${what}`, undefined);const writeQueue = createKeyedQueue();const pendingWrites = new Map<string, Promise<void>>();const persist = (key: string, what: string, op: () => Promise<unknown>): void => {diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex 342ed481d..83fc4e63b 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -1,6 +1,6 @@import type { AdmittedWork } from "../util/admitted-work.ts";import { randomUUID } from "node:crypto";-import { errorAlreadyRecorded, type ErrorLog } from "../admin/error-log.ts";+import type { ErrorLog } from "../admin/error-log.ts";import { conversationScope } from "../resolution/resolution-service.ts";import type { TurnResult } from "../types.ts";import type { Orchestrator } from "../core/orchestrator.ts";@@ -8,7 +8,7 @@ import { NonRetryableTurnError, turnFailureMessage } from "../core/turn-error.tsimport { resolveTurnOrigin } from "../core/turn-origin.ts";import { errorParks, type Run, type RunStore } from "./run-store.ts";import type { SessionStore } from "../sessions/session-store.ts";-import { errMessage, swallow } from "../util/errors.ts";+import { errMessage, errorAlreadyReported, swallow } from "../util/errors.ts";import { sleep } from "../util/async.ts";import { retryDelay } from "./retry-delay.ts";import { resolveSwarmSettings } from "../swarms/swarm-settings.ts";@@ -89,7 +89,7 @@ export async function processRun(deps: ProcessDeps, run: Run, opts?: { backgroun} catch (err) {stopBeat();console.error(`[worker] run ${run.id} turn failed: ${errMessage(err)}`);- if (!errorAlreadyRecorded(err))+ if (!errorAlreadyReported(err))deps.errors?.record({category: "turn",diff --git a/src/skills/skill-sync-engine.ts b/src/skills/skill-sync-engine.tsindex 97bda5d7b..ba66d45bb 100644--- a/src/skills/skill-sync-engine.ts+++ b/src/skills/skill-sync-engine.ts@@ -2,7 +2,7 @@ import type { SkillPackStore } from "./skill-pack-store.ts";import type { SkillPackFetcher } from "./pack-fetcher.ts";import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";import { createSweeper } from "../util/sweeper.ts";-import { errMessage } from "../util/errors.ts";+import { errMessage, reportFailureAs } from "../util/errors.ts";const TICK_LEASE_KEY = "skills:sync:tick";const DEFAULT_INTERVAL_MS = 300_000;@@ -54,7 +54,7 @@ export function createSkillSyncEngine(deps: SkillSyncDeps): SkillSyncEngine {};const sweeper = createSweeper(- () => tick().catch((e: unknown) => console.error("[skill-sync] tick failed:", errMessage(e))),+ () => tick().catch(reportFailureAs("skill-sync: tick", undefined)),DEFAULT_INTERVAL_MS,{ label: "skill-sync" },);diff --git a/src/slack/index.ts b/src/slack/index.tsindex 180d86dba..87d01680f 100644--- a/src/slack/index.ts+++ b/src/slack/index.ts@@ -1,7 +1,7 @@import { SlackPluginStartCleanupError } from "../surfaces/slack-runtime.ts";import { createSlackRateLimitNotice } from "./rate-limit-notice.ts";import { createSlackHistoryReader } from "./history.ts";-import { errMessage, swallow, swallowAs } from "../util/errors.ts";+import { reportFailure, swallow, swallowAs } from "../util/errors.ts";import { createEnvelopeStaging } from "./envelope-staging.ts";import { createSweeper } from "../util/sweeper.ts";import bolt from "@slack/bolt";@@ -290,7 +290,7 @@ export async function startSlackPlugin(}}} catch (err) {- console.error("[slack] channel header default sweep failed:", errMessage(err));+ reportFailure("slack: channel header default sweep", err);}})();});diff --git a/src/slack/turn-handler.ts b/src/slack/turn-handler.tsindex f104cde74..1b76fd018 100644--- a/src/slack/turn-handler.ts+++ b/src/slack/turn-handler.ts@@ -2,7 +2,7 @@ import type { SlackRateLimitNotice } from "./rate-limit-notice.ts";import type { SlackHistoryReader } from "./history.ts";import { performance } from "node:perf_hooks";import { slackFailureText } from "./turn-flow.ts";-import { errMessage, swallowAs } from "../util/errors.ts";+import { errMessage, reportFailure, reportFailureAs, swallowAs } from "../util/errors.ts";import {type ActorAssertion,type ChannelMeta,@@ -840,7 +840,7 @@ export function createTurnHandler(deps: {() => handleIncoming(stamped, client),(err) => {stamped.ackGate?.failed(errMessage(err));- console.error("[slack-plugin] handler error:", errMessage(err));+ reportFailure("slack: incoming handler", err);},);if (!ran) gate?.failed("already in flight on this instance");@@ -962,7 +962,7 @@ export function createTurnHandler(deps: {);await handleIncoming(inc, client);},- (err) => console.error("[slack-plugin] handler error:", errMessage(err)),+ reportFailureAs("slack: reaction handler", undefined),);} finally {reactionsInFlight.delete(flightKey);diff --git a/src/util/errors.ts b/src/util/errors.tsindex 59bca5c9e..1eaec7bf2 100644--- a/src/util/errors.ts+++ b/src/util/errors.ts@@ -1,3 +1,6 @@+import { reportBackendError } from "../../plugins/chassis/src/error-reporting.ts";+import { WorkAdmissionClosed } from "./admitted-work.ts";+const CAUSE_DEPTH = 5;function errorText(value: unknown): string {@@ -44,3 +47,40 @@ export function swallowAs<T>(context: string, fallback: T): (e: unknown) => T {return fallback;};}++const reportedErrors = new WeakSet<object>();++export function markErrorReported(e: unknown): void {+ if (typeof e === "object" && e !== null) reportedErrors.add(e);+}++export function errorAlreadyReported(e: unknown): boolean {+ return typeof e === "object" && e !== null && reportedErrors.has(e);+}++function isExpectedInterruption(e: unknown): boolean {+ return (e instanceof Error && e.name === "AbortError") || e instanceof WorkAdmissionClosed;+}++export function failureCode(context: string): string {+ return context+ .toLowerCase()+ .replace(/\s*:\s*/g, ":")+ .replace(/[^a-z0-9_.:-]+/g, "_")+ .replace(/^_+|_+$/g, "")+ .slice(0, 120);+}++export function reportFailure(context: string, e: unknown, detail?: string): void {+ console.error(`[failed] ${context}${detail ? ` (${detail})` : ""}: ${errMessage(e)}`);+ if (isExpectedInterruption(e) || errorAlreadyReported(e)) return;+ markErrorReported(e);+ reportBackendError(asError(e), failureCode(context));+}++export function reportFailureAs<T>(context: string, fallback: T, detail?: string): (e: unknown) => T {+ return (e) => {+ reportFailure(context, e, detail);+ return fallback;+ };+}diff --git a/src/util/sweeper.ts b/src/util/sweeper.tsindex f626975df..450e95bac 100644--- a/src/util/sweeper.ts+++ b/src/util/sweeper.ts@@ -1,4 +1,4 @@-import { swallow, swallowAs } from "./errors.ts";+import { reportFailure, reportFailureAs } from "./errors.ts";export interface Sweeper {start(intervalMs?: number): void;@@ -16,11 +16,11 @@ export function createSweeper(let stopping: Promise<void> | null = null;const sweep = (): void => {try {- const work = Promise.resolve(fn()).then(() => {}, swallowAs(`${label}: sweep failed`, undefined));+ const work = Promise.resolve(fn()).then(() => {}, reportFailureAs(`${label}: sweep failed`, undefined));pending.add(work);void work.finally(() => pending.delete(work));} catch (e) {- swallow(`${label}: sweep failed`, e);+ reportFailure(`${label}: sweep failed`, e);}};return {diff --git a/src/webhooks/webhook-receiver.ts b/src/webhooks/webhook-receiver.tsindex d27625e6c..cc99cf469 100644--- a/src/webhooks/webhook-receiver.ts+++ b/src/webhooks/webhook-receiver.ts@@ -3,7 +3,7 @@ import type { WebhookStore } from "./webhook-store.ts";import { getVerifier, type VerifierInput } from "./verifiers.ts";import { runTrigger, type TriggerDeps } from "../triggers/run-trigger.ts";import { buildWebhookWakeEnvelope, capForEscaping } from "../core/wake-envelope.ts";-import { errMessage } from "../util/errors.ts";+import { errMessage, reportFailure } from "../util/errors.ts";export type DeliverResult = { status: 202 } | { status: 200; body: string } | { status: 401 } | { status: 404 };@@ -139,7 +139,7 @@ export function createWebhookReceiver(deps: WebhookReceiverDeps): WebhookReceive.catch((e: unknown) => {const msg = errMessage(e);void deps.webhooks.recordFire(wh.id, { at: Date.now(), error: msg });- console.error(`[webhook] ${wh.id} turn threw: ${msg}`);+ reportFailure("webhook: fire", e, `webhook=${wh.id}`);});return { status: 202 };diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.tsindex 6365d446e..3f7f73e40 100644--- a/test/cron-scheduler.test.ts+++ b/test/cron-scheduler.test.ts@@ -961,12 +961,12 @@ test("a failing cron fire is logged, not swallowed", async (t) => {});scheduler.start(1000);t.mock.timers.tick(1000);- for (let i = 0; i < 50 && !logged.some((l) => l.includes("[scheduler] fire failed")); i++) {+ for (let i = 0; i < 50 && !logged.some((l) => l.includes("[failed] scheduler: fire:")); i++) {await new Promise((r) => setImmediate(r));}scheduler.stop();assert.ok(- logged.some((l) => l.includes("[scheduler] fire failed") && l.includes("boom")),+ logged.some((l) => l.includes("[failed] scheduler: fire:") && l.includes("boom")),"the fire error must reach the log",);const { runs: after } = await crons.listFires(cron.id);diff --git a/test/error-log-dedup.test.ts b/test/error-log-dedup.test.tsindex d03b42212..f6e6ae4f6 100644--- a/test/error-log-dedup.test.ts+++ b/test/error-log-dedup.test.ts@@ -1,12 +1,16 @@import { test } from "node:test";import assert from "node:assert/strict";-import { errorAlreadyRecorded, markErrorRecorded } from "../src/admin/error-log.ts";+import { createErrorLog, withErrorReporting } from "../src/admin/error-log.ts";+import { errorAlreadyReported, markErrorReported } from "../src/util/errors.ts";test("an error the orchestrator recorded is recognized by the worker, and unmarked or non-object throws are not", () => {const recorded = new Error("boom");- markErrorRecorded(recorded);- assert.equal(errorAlreadyRecorded(recorded), true);- assert.equal(errorAlreadyRecorded(new Error("boom")), false, "identity, not message, decides");- markErrorRecorded("a string throw");- assert.equal(errorAlreadyRecorded("a string throw"), false);+ withErrorReporting(createErrorLog()).record(+ { category: "turn", code: "error", message: "boom", scopeLabel: "s" },+ recorded,+ );+ assert.equal(errorAlreadyReported(recorded), true);+ assert.equal(errorAlreadyReported(new Error("boom")), false, "identity, not message, decides");+ markErrorReported("a string throw");+ assert.equal(errorAlreadyReported("a string throw"), false);});diff --git a/test/error-reporting.test.ts b/test/error-reporting.test.tsindex 7b7b0c8e0..ca27e16cc 100644--- a/test/error-reporting.test.ts+++ b/test/error-reporting.test.ts@@ -173,3 +173,30 @@ test("operator error records retain local details and send one classified eventassert.equal(events[0]!.tags.error_code, "turn:failed");assert.doesNotMatch(JSON.stringify(events), /private-/);});++test("reportFailure sends one classified event per distinct failure and skips cancellations and recorded errors", async () => {+ const { events, code } = await runReporting(`+ const { reportFailure } = await import('./src/util/errors.ts');+ const { createErrorLog, withErrorReporting } = await import('./src/admin/error-log.ts');+ const errors = withErrorReporting(createErrorLog());+ const recorded = new Error('private-turn-failure');+ errors.record({category:'turn', code:'error', message:recorded.message, scopeLabel:'private-scope'}, recorded);+ reportFailure('scheduler: fire', recorded);+ reportFailure('scheduler: fire', new DOMException('private-cancel', 'AbortError'));+ const infra = new Error('private-db-down');+ reportFailure('scheduler: tick', infra);+ reportFailure('worker: background run crashed', infra);+ reportFailure('audit: persist event', 'private-string-throw');+ const reportedFirst = new Error('private-tool-failure');+ reportFailure('tools: persist artifact', reportedFirst);+ errors.record({category:'turn', code:'error', message:reportedFirst.message, scopeLabel:'private-scope'}, reportedFirst);+ await flushErrorReporting();+ `);+ assert.equal(code, 0);+ assert.deepEqual(+ events.map((event) => event.tags.error_code),+ ["turn:error", "scheduler:tick", "audit:persist_event", "tools:persist_artifact"],+ );+ assert.doesNotMatch(JSON.stringify(events), /private-/);+ assert.equal(events[1]!.exception.values[0].value, "scheduler:tick");+});diff --git a/test/errors.test.ts b/test/errors.test.tsindex a3bdf6b7c..dec636fc6 100644--- a/test/errors.test.ts+++ b/test/errors.test.ts@@ -1,6 +1,7 @@import { test } from "node:test";import assert from "node:assert/strict";-import { asError, errMessage } from "../src/util/errors.ts";+import { asError, errMessage, errorAlreadyReported, failureCode, reportFailure } from "../src/util/errors.ts";+import { WorkAdmissionClosed } from "../src/util/admitted-work.ts";import { errMessage as pluginErrMessage } from "../plugins/chassis/src/errors.ts";import { runInNewContext } from "node:vm";@@ -79,3 +80,36 @@ test("asError preserves cross-realm messages without invoking custom stringifier"Unknown error",);});++test("failureCode turns a static context into a Sentry-safe grouping code", () => {+ assert.equal(failureCode("scheduler: fire"), "scheduler:fire");+ assert.equal(+ failureCode("web delivery: transcript write gave up (delivering as a nudge only)"),+ "web_delivery:transcript_write_gave_up_delivering_as_a_nudge_only",+ );+ assert.equal(failureCode("x".repeat(200)).length, 120);+ assert.match(failureCode("Ünïcode / spaces"), /^[a-z0-9_.:-]+$/);+});++test("reportFailure logs every failure but marks only reportable ones as reported", (t) => {+ const logged: string[] = [];+ t.mock.method(console, "error", (...args: unknown[]) => {+ logged.push(args.map(String).join(" "));+ });+ const cancelled = new DOMException("stopped", "AbortError");+ reportFailure("scheduler: fire", cancelled);+ assert.equal(errorAlreadyReported(cancelled), false, "cancellations are logged, never reported");+ const closed = new WorkAdmissionClosed();+ reportFailure("scheduler: tick", closed);+ assert.equal(errorAlreadyReported(closed), false, "admission closed during handoff is logged, never reported");+ const boom = new Error("boom");+ reportFailure("scheduler: fire", boom);+ assert.equal(errorAlreadyReported(boom), true);+ reportFailure("worker: retry", "a string throw", "run=r1");+ assert.deepEqual(logged, [+ "[failed] scheduler: fire: stopped",+ "[failed] scheduler: tick: This deployment is not accepting synchronous work",+ "[failed] scheduler: fire: boom",+ "[failed] worker: retry (run=r1): a string throw",+ ]);+});diff --git a/test/sweeper.test.ts b/test/sweeper.test.tsindex f0e493e71..d55481b48 100644--- a/test/sweeper.test.ts+++ b/test/sweeper.test.ts@@ -83,9 +83,9 @@ test("createSweeper start(intervalMs) overrides the construction-time interval",assert.ok(ticks >= 2, `expected ticks at the start-time interval, got ${ticks}`);});-test("createSweeper tags swallowed failures with its label", async (t) => {+test("createSweeper tags reported failures with its label", async (t) => {const logged: string[] = [];- t.mock.method(console, "warn", (...args: unknown[]) => {+ t.mock.method(console, "error", (...args: unknown[]) => {logged.push(args.map(String).join(" "));});const s = createSweeper(@@ -100,7 +100,7 @@ test("createSweeper tags swallowed failures with its label", async (t) => {s.stop();assert.ok(logged.some((l) => l.includes("test-loop: sweep failed") && l.includes("boom")),- `expected a labelled swallow line, got: ${logged.join(" | ")}`,+ `expected a labelled failure line, got: ${logged.join(" | ")}`,);});