Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Report caught backend failures at shared boundaries to Sentry (#1408)

Regan Bellee37609c4f7f
diff --git a/src/admin/error-log.ts b/src/admin/error-log.ts
index 10f7986b7..a0e52343c 100644
--- a/src/admin/error-log.ts
+++ b/src/admin/error-log.ts
@@ -1,4 +1,5 @@
import { reportBackendError } from "../../plugins/chassis/src/error-reporting.ts";
+import { errorAlreadyReported, markErrorReported } from "../util/errors.ts";
import type { ScopeId } from "../types.ts";
import { createTimestampedEventSink } from "./scoped-event-sink.ts";
@@ -30,21 +31,14 @@ export function createErrorLog(): ErrorLog {
};
}
-const recordedErrors = new WeakSet<object>();
-
-export function markErrorRecorded(err: unknown): void {
- if (typeof err === "object" && err !== null) recordedErrors.add(err);
-}
-
-export function errorAlreadyRecorded(err: unknown): boolean {
- return typeof err === "object" && err !== null && recordedErrors.has(err);
-}
-
export function withErrorReporting(store: ErrorLog): ErrorLog {
return {
...store,
record(event, error) {
- reportBackendError(error ?? new Error("Recorded backend failure"), `${event.category}:${event.code}`);
+ if (!errorAlreadyReported(error)) {
+ reportBackendError(error ?? new Error("Recorded backend failure"), `${event.category}:${event.code}`);
+ markErrorReported(error);
+ }
store.record(event);
},
};
diff --git a/src/admin/postgres-audit-log.ts b/src/admin/postgres-audit-log.ts
index 2a36cfad9..7a5e0cdfa 100644
--- a/src/admin/postgres-audit-log.ts
+++ b/src/admin/postgres-audit-log.ts
@@ -1,7 +1,7 @@
import { createPgPool } from "../persistence/pg-pool.ts";
import type { ScopeId } from "../types.ts";
import type { AuditEvent, AuditLog } from "../audit/audit-log.ts";
-import { errMessage } from "../util/errors.ts";
+import { reportFailureAs } from "../util/errors.ts";
function rowToEvent(r: Record<string, unknown>): AuditEvent {
return {
@@ -72,7 +72,7 @@ export function createPostgresAuditLog(connectionString: string): AuditLog {
e.detail ?? null,
])
.then(() => undefined)
- .catch((err) => console.error("[audit] failed to persist event to durable store:", errMessage(err)));
+ .catch(reportFailureAs("audit: persist event", undefined));
pendingWrites.add(write);
void write.finally(() => pendingWrites.delete(write));
},
diff --git a/src/admin/postgres-metrics-sink.ts b/src/admin/postgres-metrics-sink.ts
index 944f1baea..77144c322 100644
--- a/src/admin/postgres-metrics-sink.ts
+++ b/src/admin/postgres-metrics-sink.ts
@@ -1,6 +1,6 @@
import { createPostgresEventSink, scopedEventMigrationId, type EventColumn } from "./scoped-event-sink.ts";
import type { MetricsSink, TurnMetricSample } from "./metrics-sink.ts";
-import { errMessage } from "../util/errors.ts";
+import { reportFailureAs } from "../util/errors.ts";
type MetricColumn = EventColumn<keyof TurnMetricSample & string>;
@@ -96,7 +96,7 @@ export function createPostgresMetricsSink(connectionString: string): MetricsSink
params.push(runId);
await sink
.q(`UPDATE turn_metrics SET ${sets.join(", ")} WHERE run_id = $${params.length}`, params)
- .catch((err) => console.error("[metrics] failed to patch turn metric:", errMessage(err)));
+ .catch(reportFailureAs("metrics: patch turn metric", undefined));
},
list: (opts = {}) => sink.list(opts),
};
diff --git a/src/admin/scoped-event-sink.ts b/src/admin/scoped-event-sink.ts
index 598f419cf..ba9180a4e 100644
--- a/src/admin/scoped-event-sink.ts
+++ b/src/admin/scoped-event-sink.ts
@@ -1,6 +1,6 @@
import type { ScopeId } from "../types.ts";
import { createPgPool, type PgMigrationDefinition, type PgPool } from "../persistence/pg-pool.ts";
-import { errMessage } from "../util/errors.ts";
+import { reportFailureAs } from "../util/errors.ts";
export interface ScopedEvent {
scopeLabel: ScopeId;
@@ -193,7 +193,7 @@ export function createPostgresEventSink<E>(cfg: PostgresEventSinkConfig<E>): Pos
const s = input as Record<string, unknown>;
const values = cfg.columns.map(([, js]) => (js === "ts" ? Date.now() : (s[js] ?? null)));
const write = q(insertSql, values)
- .catch((err) => console.error(cfg.persistErrorMessage, errMessage(err)))
+ .catch(reportFailureAs(cfg.persistErrorMessage, undefined))
.finally(() => pendingWrites.delete(write));
pendingWrites.add(write);
},
diff --git a/src/api/app-ambient.ts b/src/api/app-ambient.ts
index da19b54f6..c51002168 100644
--- a/src/api/app-ambient.ts
+++ b/src/api/app-ambient.ts
@@ -12,7 +12,7 @@ import {
type AmbientDecision,
} from "../surface-cache/ambient-judge.ts";
import type { AmbientJudgmentStore } from "../surface-cache/ambient-judgment-store.ts";
-import { errMessage } from "../util/errors.ts";
+import { reportFailureAs } from "../util/errors.ts";
import { buildWakeEnvelope } from "../core/wake-envelope.ts";
import { isTerminal } from "../runs/run-store.ts";
import { unscreenedNotice } from "../security/security-posture.ts";
@@ -36,9 +36,7 @@ export function createAmbientHelpers(deps: AppDeps, app: App) {
const ambientSelf = new Map<string, { name?: string; mentionId?: string }>();
const recordJudgment = (j: Parameters<AmbientJudgmentStore["record"]>[0]): void => {
- void deps.ambientJudgments
- ?.record(j)
- .catch((e) => console.error("[ambient] record judgment failed:", errMessage(e)));
+ void deps.ambientJudgments?.record(j).catch(reportFailureAs("ambient: record judgment", undefined));
};
async function judgeAmbientContainer(
surface: string,
@@ -292,7 +290,7 @@ export function createAmbientHelpers(deps: AppDeps, app: App) {
idempotencyKey: `ambient:${orgIdOf()}:${batch.surface}:${batch.container}:${latestTs}`,
};
if (!solicited && (await steerIntoLiveAmbientRun(batch, latestTs, req))) return;
- await app.turn(req).catch((e) => console.error("[ambient] spawn worker failed:", errMessage(e)));
+ await app.turn(req).catch(reportFailureAs("ambient: spawn worker", undefined));
}
async function steerIntoLiveAmbientRun(batch: AmbientBatch, latestTs: string, req: TurnRequest): Promise<boolean> {
diff --git a/src/api/app-messaging.ts b/src/api/app-messaging.ts
index a14ef6932..3a9d5d634 100644
--- a/src/api/app-messaging.ts
+++ b/src/api/app-messaging.ts
@@ -3,7 +3,7 @@ import { orgId as orgIdOf } from "../config.ts";
import { parseScopeId, scopeId } from "../types.ts";
import { personKey, personKeys, samePersonInDirectory, samePersonMatcher } from "../directory/person.ts";
import type { Destination, SurfaceContextRequest, SurfaceContextResult } from "../types.ts";
-import { errMessage } from "../util/errors.ts";
+import { reportFailureAs } from "../util/errors.ts";
import { adminCronHistoryUrl } from "../util/admin-links.ts";
import { createMemoryMap } from "../persistence/durable-map.ts";
import { randomUUID } from "node:crypto";
@@ -312,14 +312,12 @@ export function createMessagingMethods(
if (self && (self.name || self.mentionId)) ambientSelf.set(`${orgIdOf()}:${surface}`, self);
const out = await deps.surfaceCache.ingest(events);
if (surface === "slack" && hasRevisionEvents(events)) {
- void recordMessageRevisions(deps.sessions, events).catch((e) =>
- console.error("[revisions] surface revision record failed:", errMessage(e)),
+ void recordMessageRevisions(deps.sessions, events).catch(
+ reportFailureAs("revisions: surface revision record", undefined),
);
}
for (const container of new Set(events.filter((e) => !e.self).map((e) => e.container))) {
- void judgeAmbientContainer(surface, container).catch((e) =>
- console.error("[ambient] judge failed:", errMessage(e)),
- );
+ void judgeAmbientContainer(surface, container).catch(reportFailureAs("ambient: judge", undefined));
}
return out;
},
diff --git a/src/api/routes/admin-resources.ts b/src/api/routes/admin-resources.ts
index 1b9552a7b..ee89db7c0 100644
--- a/src/api/routes/admin-resources.ts
+++ b/src/api/routes/admin-resources.ts
@@ -35,7 +35,7 @@ import { authorizeUrl, PROVIDERS, type ConsentMode } from "../../connectors/oaut
import { resolverFor } from "./connectors.ts";
import { encodeRef, serviceCredRef } from "../../acl/resource-ref.ts";
import { audit } from "./shared.ts";
-import { errMessage } from "../../util/errors.ts";
+import { errMessage, reportFailure } from "../../util/errors.ts";
import {
DEFAULT_SECURITY_SCREEN_RUBRIC,
parseSecurityPosture,
@@ -1111,7 +1111,7 @@ export const ADMIN_RESOURCES: readonly AdminResource[] = [
auditGrantChanges(grantChanges(replacement));
return { ok: true };
} catch (e) {
- console.error("[admin] credential delete failed after conditional mutation:", errMessage(e));
+ reportFailure("admin: credential delete after conditional mutation", e);
return mutationFailed("delete");
}
}
@@ -1219,7 +1219,7 @@ export const ADMIN_RESOURCES: readonly AdminResource[] = [
auditGrantChanges(grantChanges(replacement));
return { ok: true };
} catch (e) {
- console.error("[admin] credential update failed after conditional mutation:", errMessage(e));
+ reportFailure("admin: credential update after conditional mutation", e);
return mutationFailed(existing ? "update" : "create");
}
},
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index 4fa949505..c581225ca 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -11,7 +11,6 @@ import { evaluateCommandWithLayer } from "../policy/command-policy.ts";
import { createSecretValueMasker } from "../security/secret-masking.ts";
import { shq } from "../util/shell.ts";
import { goalViewFromEntry } from "../runs/turn-stream.ts";
-import { markErrorRecorded } from "../admin/error-log.ts";
import type {
CommandApprovalGrant,
DeliveryProvenance,
@@ -162,7 +161,7 @@ import {
selectOverheardToImport,
type OverheardEntryPayload,
} from "../harness/replay.ts";
-import { errMessage, swallow, swallowAs } from "../util/errors.ts";
+import { errMessage, reportFailure, swallow, swallowAs } from "../util/errors.ts";
import { isObj } from "../util/objects.ts";
import { absoluteAppLinks, headSlice, jsonbSafeStringify } from "../util/text.ts";
import { NonRetryableTurnError, TitleRejected, turnFailureMessage, type TurnFailurePayload } from "./turn-error.ts";
@@ -738,7 +737,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
try {
await deps.sessions.recordLlmRequest(screenSession.id, { ...rec, scopeLabel: scopeId }, signal);
} catch (err) {
- console.error("[orchestrator] failed to persist security screen request snapshot:", errMessage(err));
+ reportFailure("orchestrator: persist security screen request snapshot", err);
}
};
let screenedOverheard: OverheardEntryPayload[] = [];
@@ -3412,7 +3411,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
try {
await deps.sessions.recordLlmRequest(session.id, { ...rec, scopeLabel: scopeId }, signal);
} catch (err) {
- console.error("[orchestrator] failed to persist LLM request snapshot:", errMessage(err));
+ reportFailure("orchestrator: persist LLM request snapshot", err);
}
},
});
@@ -3536,7 +3535,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
spine.surfaceOutboundCount += 1;
if (input.runId) deps.turnStream?.markSurfacePosted(input.runId);
} catch (e) {
- console.error("%s", `[orchestrator] direct reply delivery failed session=${session.id}:`, errMessage(e));
+ reportFailure("orchestrator: direct reply delivery", e, `session=${session.id}`);
}
}
if (spine.surfaceOutboundCount === 0 && !silentPollNarration) {
@@ -3969,7 +3968,6 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
},
err,
);
- markErrorRecorded(err);
if ((err instanceof NonRetryableTurnError || input.finalAttempt) && !input.cancel?.aborted) {
const mirrorFailureEntry = async (entry: SessionEntry | undefined): Promise<void> => {
if (!entry) return;
diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.ts
index cc81b6748..58287143b 100644
--- a/src/cron/scheduler.ts
+++ b/src/cron/scheduler.ts
@@ -22,7 +22,7 @@ import { recoverNextFireAt } from "./schedule.ts";
import type { CronFireJob, CronJobQueue } from "./job-queue.ts";
import { hashId } from "../util/crypto.ts";
import { utcMinute } from "../util/time.ts";
-import { errMessage } from "../util/errors.ts";
+import { errMessage, reportFailure, reportFailureAs } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
const TICK_LEASE_KEY = "cron:scheduler:tick";
@@ -315,7 +315,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
const swept = await deps.crons.sweepStrandedFires(t);
if (swept > 0) console.warn(`[scheduler] closed ${swept} stranded running fire(s) as failed`);
} catch (e) {
- console.error("[scheduler] stranded-fire sweep failed:", errMessage(e));
+ reportFailure("scheduler: stranded-fire sweep", e);
}
};
@@ -327,7 +327,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
const pruned = await deps.crons.pruneFires(t);
if (pruned > 0) console.log(`[scheduler] pruned ${pruned} old cron fire row(s)`);
} catch (e) {
- console.error("[scheduler] cron fire GC failed:", errMessage(e));
+ reportFailure("scheduler: cron fire gc", e);
}
};
@@ -343,7 +343,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
await deps.crons.markAttempted(cron.id, t);
batch.push(cron);
} catch (e) {
- console.error("[scheduler] attempt mark failed, holding this cron back:", errMessage(e));
+ reportFailure("scheduler: attempt mark (holding this cron back)", e);
}
}
console.warn(`[scheduler] fan-out capped: firing ${batch.length}/${due.length} due crons this tick`);
@@ -354,7 +354,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
const { authzFailed, deferred } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt);
if (!authzFailed && !deferred) await deps.crons.markFired(cron.id, t, cron.scheduledAt);
} catch (e) {
- console.error("[scheduler] fire failed:", errMessage(e));
+ reportFailure("scheduler: fire", e);
}
}
};
@@ -367,13 +367,13 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
await fireDue(t, observed);
await sweepStranded(t);
await gcFires(t);
- await deps.sweepAsks?.(t).catch((e: unknown) => console.error("[scheduler] ask sweep failed:", errMessage(e)));
+ await deps.sweepAsks?.(t).catch(reportFailureAs("scheduler: ask sweep", undefined));
}),
);
};
const makeSweeper = () =>
- createSweeper(() => tick().catch((e: unknown) => console.error("[scheduler] tick failed:", errMessage(e))), 1000, {
+ createSweeper(() => tick().catch(reportFailureAs("scheduler: tick", undefined)), 1000, {
label: "scheduler",
});
@@ -429,7 +429,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
return;
}
} catch (e) {
- console.error("[scheduler] fire failed:", errMessage(e));
+ reportFailure("scheduler: fire", e);
await deps.crons.unclaimSlot(job.cronId, slot, t, cron.lastFiredAt);
return;
}
@@ -445,11 +445,11 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
if (job) await deps.jobQueue!.enqueueFire(job);
}
} catch (e) {
- console.error("[scheduler] tick failed:", errMessage(e));
+ reportFailure("scheduler: reconcile", e);
}
await sweepStranded(now());
await gcFires(now());
- await deps.sweepAsks?.(now()).catch((e: unknown) => console.error("[scheduler] ask sweep failed:", errMessage(e)));
+ await deps.sweepAsks?.(now()).catch(reportFailureAs("scheduler: ask sweep", undefined));
}
let stopSignal = Promise.withResolvers<void>();
@@ -511,7 +511,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
}
const settled = withWork(() => fire(cron, t, fireKey)).then(
() => undefined,
- (e: unknown) => console.error("%s", `[scheduler] manual fire of cron ${cronId} failed:`, errMessage(e)),
+ reportFailureAs("scheduler: manual fire", undefined, `cron=${cronId}`),
);
pending.add(settled);
void settled.finally(() => pending.delete(settled));
@@ -530,7 +530,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
},
notifyChanged(cronId) {
if (!deps.jobQueue) return;
- void enqueueNext(cronId).catch((e: unknown) => console.error("[scheduler] cron enqueue failed:", errMessage(e)));
+ void enqueueNext(cronId).catch(reportFailureAs("scheduler: cron enqueue", undefined, `cron=${cronId}`));
},
start(intervalMs) {
if (started || stopping || pausing || stopFailed) return;
@@ -556,7 +556,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
},
(e: unknown) => {
if (deps.requireQueueStart) throw e;
- console.error("[scheduler] cron job queue failed to start; falling back to interval ticks:", errMessage(e));
+ reportFailure("scheduler: cron job queue start (falling back to interval ticks)", e);
if (!stopped) sweeper.start(intervalMs);
},
);
diff --git a/src/delivery/run-result-delivery.ts b/src/delivery/run-result-delivery.ts
index 67eeeacc9..70bf1b15e 100644
--- a/src/delivery/run-result-delivery.ts
+++ b/src/delivery/run-result-delivery.ts
@@ -14,7 +14,7 @@ import {
type TranscriptAppendSessions,
} from "../sessions/session-store.ts";
import { turnRecordedFailure } from "./web-transcript-delivery.ts";
-import { errMessage } from "../util/errors.ts";
+import { reportFailureAs } from "../util/errors.ts";
export interface RunResultDelivery {
destination: Destination;
@@ -140,8 +140,8 @@ export function wireRunResultDeliveries(
): void {
runs.onTerminal((run) => {
if (sessions) {
- void recordRunFailureEntry(sessions, run).catch((err) =>
- console.error("%s", `[delivery] failed to record turn_failure entry for run ${run.id}:`, errMessage(err)),
+ void recordRunFailureEntry(sessions, run).catch(
+ reportFailureAs("delivery: record turn_failure entry", undefined, `run=${run.id}`),
);
}
void (async () => {
@@ -149,8 +149,6 @@ export function wireRunResultDeliveries(
const delivery = runResultDelivery(run, taskList, adminUrlFor);
if (!delivery) return;
await deliveries.enqueue(delivery);
- })().catch((err) =>
- console.error("%s", `[delivery] failed to enqueue recovery delivery for run ${run.id}:`, errMessage(err)),
- );
+ })().catch(reportFailureAs("delivery: enqueue recovery delivery", undefined, `run=${run.id}`));
});
}
diff --git a/src/delivery/web-transcript-delivery.ts b/src/delivery/web-transcript-delivery.ts
index cf42e01b4..ba0c8cc7e 100644
--- a/src/delivery/web-transcript-delivery.ts
+++ b/src/delivery/web-transcript-delivery.ts
@@ -7,7 +7,7 @@ import {
type TranscriptAppendSessions,
} from "../sessions/session-store.ts";
import { messageTag } from "../util/message-tag.ts";
-import { errMessage, swallow } from "../util/errors.ts";
+import { reportFailure, swallow } from "../util/errors.ts";
const DEDUPE_SCAN_LIMIT = 200;
const RECORDED_CACHE_CAP = 1000;
@@ -128,9 +128,10 @@ export function withWebTranscriptDeliveries(store: DeliveryStore, sessions: WebT
if (disposition === "deliver" || (await recordedForDelivery(d))) out.push(d);
} catch (err) {
if (overdue(d)) {
- console.error(
- `[delivery] web delivery ${d.idempotencyKey} could not be recorded in ${d.destination.target}'s transcript within ${WRITE_GIVEUP_MS}ms — delivering as a nudge only:`,
- errMessage(err),
+ reportFailure(
+ "web delivery: transcript write gave up (delivering as a nudge only)",
+ err,
+ `delivery=${d.idempotencyKey} target=${d.destination.target} after=${WRITE_GIVEUP_MS}ms`,
);
remember(d.idempotencyKey);
out.push(d);
diff --git a/src/deploy/deploy-service.ts b/src/deploy/deploy-service.ts
index 00653b00f..0327171c4 100644
--- a/src/deploy/deploy-service.ts
+++ b/src/deploy/deploy-service.ts
@@ -20,7 +20,7 @@ import type { DeployProfile, DeployProvider } from "./deploy-provider.ts";
import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";
import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";
import { createKeyedQueue } from "../util/async.ts";
-import { errMessage, swallow } from "../util/errors.ts";
+import { errMessage, reportFailure, swallow } from "../util/errors.ts";
export interface DeployFile {
path: string;
@@ -602,7 +602,7 @@ export function createDeployService(deps: DeployServiceDeps): DeployService {
scopeLabel: before.ownerScopeId,
});
} catch (e) {
- console.error("%s", `[deploy] failed to register pushed version for ${id}:`, errMessage(e));
+ reportFailure("deploy: register pushed version", e, `deployment=${id}`);
deps.auditLog.record({
at: Date.now(),
principalId: "system",
diff --git a/src/index.ts b/src/index.ts
index c254b99fb..91a7b4c03 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -9,7 +9,7 @@ import { buildApp, serverDeps, stopWithBackstop } from "./wiring.ts";
import { shutdownOnUncaught } from "./util/process-guard.ts";
import { createServer } from "./api/server.ts";
import { dockerDaemonFailure } from "./deploy/docker-deploy-provider.ts";
-import { errMessage } from "./util/errors.ts";
+import { errMessage, reportFailureAs } from "./util/errors.ts";
import { slackAccountConfigsFromEnv, slackPluginConfigFromEnv, startSlackPlugin } from "./slack/index.ts";
import { createSlackRuntimeReconciler } from "./surfaces/slack-runtime.ts";
import { migrateRegisteredPgSchemas } from "./persistence/pg-pool.ts";
@@ -113,7 +113,7 @@ const slackRuntime = createSlackRuntimeReconciler({
return null;
},
startPlugin: (desired) => startSlackPlugin(desired, built.slackCore),
- onError: (error) => console.error(`[qm] slack plugin reconciliation failed: ${errMessage(error)}`),
+ onError: reportFailureAs("slack plugin reconciliation", undefined),
});
if (config.backgroundWorkEnabled && !config.backgroundDeploymentId) slackRuntime.start();
@@ -122,8 +122,7 @@ const slackAccountRuntimes = slackAccountConfigsFromEnv(process.env).map((accoun
startPaused: Boolean(config.backgroundDeploymentId),
load: () => Promise.resolve({ version: `environment:${account.accountId}`, config: account }),
startPlugin: (desired) => startSlackPlugin(desired, built.slackCore),
- onError: (error) =>
- console.error(`[qm] slack account "${account.accountId}" reconciliation failed: ${errMessage(error)}`),
+ onError: reportFailureAs("slack account reconciliation", undefined, `account=${account.accountId}`),
}),
);
if (config.backgroundWorkEnabled && !config.backgroundDeploymentId)
@@ -180,7 +179,7 @@ if (built.backgroundOwnership) {
async drained() {
await Promise.all([built.runtime.backgroundDrained(), built.scheduler.drained(), periodicStop]);
},
- onError: (error) => console.error("[qm] background ownership failed:", errMessage(error)),
+ onError: reportFailureAs("background ownership", undefined),
});
built.runtime.setBackgroundAdmission(backgroundController.canClaim);
backgroundController.start();
diff --git a/src/monitors/monitor-poller.ts b/src/monitors/monitor-poller.ts
index 9f36793a8..ad2b1c4d0 100644
--- a/src/monitors/monitor-poller.ts
+++ b/src/monitors/monitor-poller.ts
@@ -10,7 +10,7 @@ import { processIsGone } from "../sandbox/process-poll.ts";
import { runTrigger, type TriggerDeps, type TriggerOutcome } from "../triggers/run-trigger.ts";
import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";
import { createSweeper } from "../util/sweeper.ts";
-import { errMessage } from "../util/errors.ts";
+import { errMessage, reportFailureAs } from "../util/errors.ts";
import type { CurrentScopeMembers } from "../resolution/scope-membership.ts";
import { compileMonitorPattern } from "./monitor-broker.ts";
import { buildEventWakeEnvelope, capForEscaping } from "../core/wake-envelope.ts";
@@ -314,11 +314,9 @@ export function createMonitorPoller(deps: MonitorPollerDeps): MonitorPoller {
else await work();
};
- const sweeper = createSweeper(
- () => tick().catch((e: unknown) => console.error("[monitor] tick failed:", errMessage(e))),
- 10_000,
- { label: "monitor" },
- );
+ const sweeper = createSweeper(() => tick().catch(reportFailureAs("monitor: tick", undefined)), 10_000, {
+ label: "monitor",
+ });
return {
tick,
start(intervalMs) {
diff --git a/src/ratelimit/postgres-budget.ts b/src/ratelimit/postgres-budget.ts
index e3679ca55..5ecd03d20 100644
--- a/src/ratelimit/postgres-budget.ts
+++ b/src/ratelimit/postgres-budget.ts
@@ -1,7 +1,7 @@
import { createPgPool } from "../persistence/pg-pool.ts";
import type { BudgetTracker } from "./budget.ts";
import { DEFAULT_BUDGET_WINDOW_MS } from "./budget.ts";
-import { errMessage } from "../util/errors.ts";
+import { reportFailure } from "../util/errors.ts";
export function createPostgresBudgetTracker(
connectionString: string,
@@ -46,7 +46,7 @@ export function createPostgresBudgetTracker(
);
}
} catch (err) {
- console.error("[budget] failed to persist spend:", errMessage(err));
+ reportFailure("budget: persist spend", err);
}
},
};
diff --git a/src/resolution/config-store.ts b/src/resolution/config-store.ts
index 446601713..655378c0a 100644
--- a/src/resolution/config-store.ts
+++ b/src/resolution/config-store.ts
@@ -20,7 +20,7 @@ import {
type PublicConnectorClient,
type DecryptedConnectorClient,
} from "../connectors/connector-client-store.ts";
-import { errMessage } from "../util/errors.ts";
+import { errMessage, reportFailureAs } from "../util/errors.ts";
export interface PersistedSoul {
scopeId: ScopeId;
@@ -353,8 +353,7 @@ export function createMemoryConfigStore(
const autoFlaggerStore = opts.autoFlaggerConfigs ?? createMemoryMap<PersistedAutoFlaggerConfig>();
const turnWallClockStore = opts.turnWallClocks ?? createMemoryMap<PersistedTurnWallClock>();
const deploymentIdentity = opts.deploymentIdentity ?? createMemoryMap<PersistedDeploymentIdentity>();
- const persistWarn = (what: string) => (e: unknown) =>
- console.error("%s", `[config] failed to persist ${what}:`, errMessage(e));
+ const persistWarn = (what: string) => reportFailureAs(`config: persist ${what}`, undefined);
const writeQueue = createKeyedQueue();
const pendingWrites = new Map<string, Promise<void>>();
const persist = (key: string, what: string, op: () => Promise<unknown>): void => {
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index 342ed481d..83fc4e63b 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -1,6 +1,6 @@
import type { AdmittedWork } from "../util/admitted-work.ts";
import { randomUUID } from "node:crypto";
-import { errorAlreadyRecorded, type ErrorLog } from "../admin/error-log.ts";
+import type { ErrorLog } from "../admin/error-log.ts";
import { conversationScope } from "../resolution/resolution-service.ts";
import type { TurnResult } from "../types.ts";
import type { Orchestrator } from "../core/orchestrator.ts";
@@ -8,7 +8,7 @@ import { NonRetryableTurnError, turnFailureMessage } from "../core/turn-error.ts
import { resolveTurnOrigin } from "../core/turn-origin.ts";
import { errorParks, type Run, type RunStore } from "./run-store.ts";
import type { SessionStore } from "../sessions/session-store.ts";
-import { errMessage, swallow } from "../util/errors.ts";
+import { errMessage, errorAlreadyReported, swallow } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
import { retryDelay } from "./retry-delay.ts";
import { resolveSwarmSettings } from "../swarms/swarm-settings.ts";
@@ -89,7 +89,7 @@ export async function processRun(deps: ProcessDeps, run: Run, opts?: { backgroun
} catch (err) {
stopBeat();
console.error(`[worker] run ${run.id} turn failed: ${errMessage(err)}`);
- if (!errorAlreadyRecorded(err))
+ if (!errorAlreadyReported(err))
deps.errors?.record(
{
category: "turn",
diff --git a/src/skills/skill-sync-engine.ts b/src/skills/skill-sync-engine.ts
index 97bda5d7b..ba66d45bb 100644
--- a/src/skills/skill-sync-engine.ts
+++ b/src/skills/skill-sync-engine.ts
@@ -2,7 +2,7 @@ import type { SkillPackStore } from "./skill-pack-store.ts";
import type { SkillPackFetcher } from "./pack-fetcher.ts";
import { createNoopLeaderLease, type LeaderLease } from "../persistence/leader-lease.ts";
import { createSweeper } from "../util/sweeper.ts";
-import { errMessage } from "../util/errors.ts";
+import { errMessage, reportFailureAs } from "../util/errors.ts";
const TICK_LEASE_KEY = "skills:sync:tick";
const DEFAULT_INTERVAL_MS = 300_000;
@@ -54,7 +54,7 @@ export function createSkillSyncEngine(deps: SkillSyncDeps): SkillSyncEngine {
};
const sweeper = createSweeper(
- () => tick().catch((e: unknown) => console.error("[skill-sync] tick failed:", errMessage(e))),
+ () => tick().catch(reportFailureAs("skill-sync: tick", undefined)),
DEFAULT_INTERVAL_MS,
{ label: "skill-sync" },
);
diff --git a/src/slack/index.ts b/src/slack/index.ts
index 180d86dba..87d01680f 100644
--- a/src/slack/index.ts
+++ b/src/slack/index.ts
@@ -1,7 +1,7 @@
import { SlackPluginStartCleanupError } from "../surfaces/slack-runtime.ts";
import { createSlackRateLimitNotice } from "./rate-limit-notice.ts";
import { createSlackHistoryReader } from "./history.ts";
-import { errMessage, swallow, swallowAs } from "../util/errors.ts";
+import { reportFailure, swallow, swallowAs } from "../util/errors.ts";
import { createEnvelopeStaging } from "./envelope-staging.ts";
import { createSweeper } from "../util/sweeper.ts";
import bolt from "@slack/bolt";
@@ -290,7 +290,7 @@ export async function startSlackPlugin(
}
}
} catch (err) {
- console.error("[slack] channel header default sweep failed:", errMessage(err));
+ reportFailure("slack: channel header default sweep", err);
}
})();
});
diff --git a/src/slack/turn-handler.ts b/src/slack/turn-handler.ts
index f104cde74..1b76fd018 100644
--- a/src/slack/turn-handler.ts
+++ b/src/slack/turn-handler.ts
@@ -2,7 +2,7 @@ import type { SlackRateLimitNotice } from "./rate-limit-notice.ts";
import type { SlackHistoryReader } from "./history.ts";
import { performance } from "node:perf_hooks";
import { slackFailureText } from "./turn-flow.ts";
-import { errMessage, swallowAs } from "../util/errors.ts";
+import { errMessage, reportFailure, reportFailureAs, swallowAs } from "../util/errors.ts";
import {
type ActorAssertion,
type ChannelMeta,
@@ -840,7 +840,7 @@ export function createTurnHandler(deps: {
() => handleIncoming(stamped, client),
(err) => {
stamped.ackGate?.failed(errMessage(err));
- console.error("[slack-plugin] handler error:", errMessage(err));
+ reportFailure("slack: incoming handler", err);
},
);
if (!ran) gate?.failed("already in flight on this instance");
@@ -962,7 +962,7 @@ export function createTurnHandler(deps: {
);
await handleIncoming(inc, client);
},
- (err) => console.error("[slack-plugin] handler error:", errMessage(err)),
+ reportFailureAs("slack: reaction handler", undefined),
);
} finally {
reactionsInFlight.delete(flightKey);
diff --git a/src/util/errors.ts b/src/util/errors.ts
index 59bca5c9e..1eaec7bf2 100644
--- a/src/util/errors.ts
+++ b/src/util/errors.ts
@@ -1,3 +1,6 @@
+import { reportBackendError } from "../../plugins/chassis/src/error-reporting.ts";
+import { WorkAdmissionClosed } from "./admitted-work.ts";
+
const CAUSE_DEPTH = 5;
function errorText(value: unknown): string {
@@ -44,3 +47,40 @@ export function swallowAs<T>(context: string, fallback: T): (e: unknown) => T {
return fallback;
};
}
+
+const reportedErrors = new WeakSet<object>();
+
+export function markErrorReported(e: unknown): void {
+ if (typeof e === "object" && e !== null) reportedErrors.add(e);
+}
+
+export function errorAlreadyReported(e: unknown): boolean {
+ return typeof e === "object" && e !== null && reportedErrors.has(e);
+}
+
+function isExpectedInterruption(e: unknown): boolean {
+ return (e instanceof Error && e.name === "AbortError") || e instanceof WorkAdmissionClosed;
+}
+
+export function failureCode(context: string): string {
+ return context
+ .toLowerCase()
+ .replace(/\s*:\s*/g, ":")
+ .replace(/[^a-z0-9_.:-]+/g, "_")
+ .replace(/^_+|_+$/g, "")
+ .slice(0, 120);
+}
+
+export function reportFailure(context: string, e: unknown, detail?: string): void {
+ console.error(`[failed] ${context}${detail ? ` (${detail})` : ""}: ${errMessage(e)}`);
+ if (isExpectedInterruption(e) || errorAlreadyReported(e)) return;
+ markErrorReported(e);
+ reportBackendError(asError(e), failureCode(context));
+}
+
+export function reportFailureAs<T>(context: string, fallback: T, detail?: string): (e: unknown) => T {
+ return (e) => {
+ reportFailure(context, e, detail);
+ return fallback;
+ };
+}
diff --git a/src/util/sweeper.ts b/src/util/sweeper.ts
index f626975df..450e95bac 100644
--- a/src/util/sweeper.ts
+++ b/src/util/sweeper.ts
@@ -1,4 +1,4 @@
-import { swallow, swallowAs } from "./errors.ts";
+import { reportFailure, reportFailureAs } from "./errors.ts";
export interface Sweeper {
start(intervalMs?: number): void;
@@ -16,11 +16,11 @@ export function createSweeper(
let stopping: Promise<void> | null = null;
const sweep = (): void => {
try {
- const work = Promise.resolve(fn()).then(() => {}, swallowAs(`${label}: sweep failed`, undefined));
+ const work = Promise.resolve(fn()).then(() => {}, reportFailureAs(`${label}: sweep failed`, undefined));
pending.add(work);
void work.finally(() => pending.delete(work));
} catch (e) {
- swallow(`${label}: sweep failed`, e);
+ reportFailure(`${label}: sweep failed`, e);
}
};
return {
diff --git a/src/webhooks/webhook-receiver.ts b/src/webhooks/webhook-receiver.ts
index d27625e6c..cc99cf469 100644
--- a/src/webhooks/webhook-receiver.ts
+++ b/src/webhooks/webhook-receiver.ts
@@ -3,7 +3,7 @@ import type { WebhookStore } from "./webhook-store.ts";
import { getVerifier, type VerifierInput } from "./verifiers.ts";
import { runTrigger, type TriggerDeps } from "../triggers/run-trigger.ts";
import { buildWebhookWakeEnvelope, capForEscaping } from "../core/wake-envelope.ts";
-import { errMessage } from "../util/errors.ts";
+import { errMessage, reportFailure } from "../util/errors.ts";
export type DeliverResult = { status: 202 } | { status: 200; body: string } | { status: 401 } | { status: 404 };
@@ -139,7 +139,7 @@ export function createWebhookReceiver(deps: WebhookReceiverDeps): WebhookReceive
.catch((e: unknown) => {
const msg = errMessage(e);
void deps.webhooks.recordFire(wh.id, { at: Date.now(), error: msg });
- console.error(`[webhook] ${wh.id} turn threw: ${msg}`);
+ reportFailure("webhook: fire", e, `webhook=${wh.id}`);
});
return { status: 202 };
diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.ts
index 6365d446e..3f7f73e40 100644
--- a/test/cron-scheduler.test.ts
+++ b/test/cron-scheduler.test.ts
@@ -961,12 +961,12 @@ test("a failing cron fire is logged, not swallowed", async (t) => {
});
scheduler.start(1000);
t.mock.timers.tick(1000);
- for (let i = 0; i < 50 && !logged.some((l) => l.includes("[scheduler] fire failed")); i++) {
+ for (let i = 0; i < 50 && !logged.some((l) => l.includes("[failed] scheduler: fire:")); i++) {
await new Promise((r) => setImmediate(r));
}
scheduler.stop();
assert.ok(
- logged.some((l) => l.includes("[scheduler] fire failed") && l.includes("boom")),
+ logged.some((l) => l.includes("[failed] scheduler: fire:") && l.includes("boom")),
"the fire error must reach the log",
);
const { runs: after } = await crons.listFires(cron.id);
diff --git a/test/error-log-dedup.test.ts b/test/error-log-dedup.test.ts
index d03b42212..f6e6ae4f6 100644
--- a/test/error-log-dedup.test.ts
+++ b/test/error-log-dedup.test.ts
@@ -1,12 +1,16 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { errorAlreadyRecorded, markErrorRecorded } from "../src/admin/error-log.ts";
+import { createErrorLog, withErrorReporting } from "../src/admin/error-log.ts";
+import { errorAlreadyReported, markErrorReported } from "../src/util/errors.ts";
test("an error the orchestrator recorded is recognized by the worker, and unmarked or non-object throws are not", () => {
const recorded = new Error("boom");
- markErrorRecorded(recorded);
- assert.equal(errorAlreadyRecorded(recorded), true);
- assert.equal(errorAlreadyRecorded(new Error("boom")), false, "identity, not message, decides");
- markErrorRecorded("a string throw");
- assert.equal(errorAlreadyRecorded("a string throw"), false);
+ withErrorReporting(createErrorLog()).record(
+ { category: "turn", code: "error", message: "boom", scopeLabel: "s" },
+ recorded,
+ );
+ assert.equal(errorAlreadyReported(recorded), true);
+ assert.equal(errorAlreadyReported(new Error("boom")), false, "identity, not message, decides");
+ markErrorReported("a string throw");
+ assert.equal(errorAlreadyReported("a string throw"), false);
});
diff --git a/test/error-reporting.test.ts b/test/error-reporting.test.ts
index 7b7b0c8e0..ca27e16cc 100644
--- a/test/error-reporting.test.ts
+++ b/test/error-reporting.test.ts
@@ -173,3 +173,30 @@ test("operator error records retain local details and send one classified event
assert.equal(events[0]!.tags.error_code, "turn:failed");
assert.doesNotMatch(JSON.stringify(events), /private-/);
});
+
+test("reportFailure sends one classified event per distinct failure and skips cancellations and recorded errors", async () => {
+ const { events, code } = await runReporting(`
+ const { reportFailure } = await import('./src/util/errors.ts');
+ const { createErrorLog, withErrorReporting } = await import('./src/admin/error-log.ts');
+ const errors = withErrorReporting(createErrorLog());
+ const recorded = new Error('private-turn-failure');
+ errors.record({category:'turn', code:'error', message:recorded.message, scopeLabel:'private-scope'}, recorded);
+ reportFailure('scheduler: fire', recorded);
+ reportFailure('scheduler: fire', new DOMException('private-cancel', 'AbortError'));
+ const infra = new Error('private-db-down');
+ reportFailure('scheduler: tick', infra);
+ reportFailure('worker: background run crashed', infra);
+ reportFailure('audit: persist event', 'private-string-throw');
+ const reportedFirst = new Error('private-tool-failure');
+ reportFailure('tools: persist artifact', reportedFirst);
+ errors.record({category:'turn', code:'error', message:reportedFirst.message, scopeLabel:'private-scope'}, reportedFirst);
+ await flushErrorReporting();
+ `);
+ assert.equal(code, 0);
+ assert.deepEqual(
+ events.map((event) => event.tags.error_code),
+ ["turn:error", "scheduler:tick", "audit:persist_event", "tools:persist_artifact"],
+ );
+ assert.doesNotMatch(JSON.stringify(events), /private-/);
+ assert.equal(events[1]!.exception.values[0].value, "scheduler:tick");
+});
diff --git a/test/errors.test.ts b/test/errors.test.ts
index a3bdf6b7c..dec636fc6 100644
--- a/test/errors.test.ts
+++ b/test/errors.test.ts
@@ -1,6 +1,7 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { asError, errMessage } from "../src/util/errors.ts";
+import { asError, errMessage, errorAlreadyReported, failureCode, reportFailure } from "../src/util/errors.ts";
+import { WorkAdmissionClosed } from "../src/util/admitted-work.ts";
import { errMessage as pluginErrMessage } from "../plugins/chassis/src/errors.ts";
import { runInNewContext } from "node:vm";
@@ -79,3 +80,36 @@ test("asError preserves cross-realm messages without invoking custom stringifier
"Unknown error",
);
});
+
+test("failureCode turns a static context into a Sentry-safe grouping code", () => {
+ assert.equal(failureCode("scheduler: fire"), "scheduler:fire");
+ assert.equal(
+ failureCode("web delivery: transcript write gave up (delivering as a nudge only)"),
+ "web_delivery:transcript_write_gave_up_delivering_as_a_nudge_only",
+ );
+ assert.equal(failureCode("x".repeat(200)).length, 120);
+ assert.match(failureCode("Ünïcode / spaces"), /^[a-z0-9_.:-]+$/);
+});
+
+test("reportFailure logs every failure but marks only reportable ones as reported", (t) => {
+ const logged: string[] = [];
+ t.mock.method(console, "error", (...args: unknown[]) => {
+ logged.push(args.map(String).join(" "));
+ });
+ const cancelled = new DOMException("stopped", "AbortError");
+ reportFailure("scheduler: fire", cancelled);
+ assert.equal(errorAlreadyReported(cancelled), false, "cancellations are logged, never reported");
+ const closed = new WorkAdmissionClosed();
+ reportFailure("scheduler: tick", closed);
+ assert.equal(errorAlreadyReported(closed), false, "admission closed during handoff is logged, never reported");
+ const boom = new Error("boom");
+ reportFailure("scheduler: fire", boom);
+ assert.equal(errorAlreadyReported(boom), true);
+ reportFailure("worker: retry", "a string throw", "run=r1");
+ assert.deepEqual(logged, [
+ "[failed] scheduler: fire: stopped",
+ "[failed] scheduler: tick: This deployment is not accepting synchronous work",
+ "[failed] scheduler: fire: boom",
+ "[failed] worker: retry (run=r1): a string throw",
+ ]);
+});
diff --git a/test/sweeper.test.ts b/test/sweeper.test.ts
index f0e493e71..d55481b48 100644
--- a/test/sweeper.test.ts
+++ b/test/sweeper.test.ts
@@ -83,9 +83,9 @@ test("createSweeper start(intervalMs) overrides the construction-time interval",
assert.ok(ticks >= 2, `expected ticks at the start-time interval, got ${ticks}`);
});
-test("createSweeper tags swallowed failures with its label", async (t) => {
+test("createSweeper tags reported failures with its label", async (t) => {
const logged: string[] = [];
- t.mock.method(console, "warn", (...args: unknown[]) => {
+ t.mock.method(console, "error", (...args: unknown[]) => {
logged.push(args.map(String).join(" "));
});
const s = createSweeper(
@@ -100,7 +100,7 @@ test("createSweeper tags swallowed failures with its label", async (t) => {
s.stop();
assert.ok(
logged.some((l) => l.includes("test-loop: sweep failed") && l.includes("boom")),
- `expected a labelled swallow line, got: ${logged.join(" | ")}`,
+ `expected a labelled failure line, got: ${logged.join(" | ")}`,
);
});