Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Swarm retries log: transaction attempts, retries by code, busy answers and reloads per route (DB 2.1) (#2386)

0xken15eec0a76c9e8e
diff --git a/backend/src/server.ts b/backend/src/server.ts
index 91286b6c1..5b4736627 100644
--- a/backend/src/server.ts
+++ b/backend/src/server.ts
@@ -32,6 +32,7 @@ import { redisWindowCounter } from './lib/redisRateCounter.js'
import { selfProfileFromEnv } from './lib/selfProfile.js'
import { startRelayUsageFlush } from './lib/productUsage.js'
import { registerWorkContext, startQueryStats } from './lib/queryStats.js'
+import { startRetryStats } from './superTabs/retryStats.js'
import { registerRouteLatency, startRouteLatency } from './lib/routeLatency.js'
import { drainAllSockets, openSocketCount, RELEASE_UPGRADE_SLOT, type SlotSocket } from './lib/wsServer.js'
import { endpointOf, refusedUpgrade, startWsCounterLog, watchUpgrade } from './lib/wsAttribution.js'
@@ -143,6 +144,8 @@ async function start(): Promise<void> {
selfProfileFromEnv(app)
// Always on, every minute (DB roadmap Phase 1): queries by model.action, filter shape and route, and each route's p50/p95.
const stopQueryStats = startQueryStats()
+ // Every minute, how often Swarm transactions ran again and commands loaded twice (superTabs/retryStats.ts, DB 2.1).
+ const stopRetryStats = startRetryStats()
const stopRouteLatency = startRouteLatency(registerRouteLatency(app))
app.setErrorHandler(errorHandler)
@@ -221,6 +224,7 @@ async function start(): Promise<void> {
await Promise.race([stopRelayUsage().catch(() => {}), new Promise((resolve) => setTimeout(resolve, 3_000))])
// The last partial minute of both logs, once the drain and the usage flush have run their queries.
stopQueryStats()
+ stopRetryStats()
stopRouteLatency()
stopMachineReportLog()
// The database stays connected through the drain, for the requests still being served (lib/prisma.ts).
diff --git a/backend/src/superTabs/retry.ts b/backend/src/superTabs/retry.ts
index 7f6273173..8ca5b5062 100644
--- a/backend/src/superTabs/retry.ts
+++ b/backend/src/superTabs/retry.ts
@@ -1,4 +1,5 @@
import { fail } from './model.js'
+import { recordAttempt, recordBusy, recordRetry } from './retryStats.js'
/** A Swarm's answers that say "try again" rather than "no": busy after its retries, or moving to its new store. */
export const RETRYABLE_SWARM_CODES: ReadonlySet<string> = new Set(['SWARM_BUSY', 'SWARM_MIGRATING'])
@@ -25,6 +26,7 @@ export async function retryTransient<T>(attempt: () => Promise<T>,
sleep: (ms: number) => Promise<void> = ms => new Promise(resolve => setTimeout(resolve, ms)),
alsoRetry: (error: unknown) => boolean = () => false): Promise<T> {
for (let tries = 1; ; tries++) {
+ recordAttempt()
try {
return await attempt()
} catch (error) {
@@ -33,8 +35,10 @@ export async function retryTransient<T>(attempt: () => Promise<T>,
if (tries >= (errorCode(error) === 'P2028' ? EXPIRED_TRANSACTION_ATTEMPTS : TRANSACTION_ATTEMPTS)) {
// A unique key still taken after every retry is a real duplicate, not contention.
if (errorCode(error) === 'P2002') throw error
+ recordBusy()
fail('SWARM_BUSY', 'This Swarm is busy. Try again in a moment.', 503)
}
+ recordRetry(typeof errorCode(error) === 'string' ? errorCode(error) as string : 'transient')
await sleep(Math.random() * Math.min(TRANSACTION_BACKOFF_CAP_MS, TRANSACTION_BACKOFF_MS * 2 ** (tries - 1)))
}
}
diff --git a/backend/src/superTabs/retryStats.test.ts b/backend/src/superTabs/retryStats.test.ts
new file mode 100644
index 000000000..a2ec57da4
--- /dev/null
+++ b/backend/src/superTabs/retryStats.test.ts
@@ -0,0 +1,38 @@
+import { afterEach, expect, it } from 'vitest'
+import { runWork } from '../lib/queryStats.js'
+import { retryTransient } from './retry.js'
+import { recordReload, startRetryStats } from './retryStats.js'
+
+let stop = () => {}
+afterEach(() => stop())
+
+const conflict = () => Object.assign(new Error('Transaction failed due to a write conflict'), { code: 'P2034' })
+const noWait = async () => {}
+const logged = async (logs: Array<Record<string, unknown>>) => { while (!logs.length) await new Promise(resolve => setTimeout(resolve, 10)) }
+
+it('counts each route\'s transaction attempts, retries by code, busy answers and reloads, and logs nothing in a quiet minute', async () => {
+ const logs: Array<Record<string, unknown>> = []
+ stop = startRetryStats({ everyMs: 50, log: (_m, c) => logs.push(c) })
+ await runWork('http POST /api/super-tabs/:tabId/commands', async () => {
+ let failures = 2
+ await retryTransient(async () => { if (failures-- > 0) throw conflict(); return 'ok' }, noWait)
+ recordReload()
+ await expect(retryTransient(async () => { throw conflict() }, noWait)).rejects.toMatchObject({ code: 'SWARM_BUSY' })
+ })
+ await logged(logs)
+ expect(logs[0]!.routes).toEqual({ 'http POST /api/super-tabs/:tabId/commands': { attempts: 11, retries: { P2034: 9 }, busy: 1, reloads: 1 } })
+ // A minute with nothing to count says nothing.
+ const count = logs.length
+ await new Promise(resolve => setTimeout(resolve, 120))
+ expect(logs).toHaveLength(count)
+})
+
+it('counts nothing for an error that is not retried', async () => {
+ const logs: Array<Record<string, unknown>> = []
+ stop = startRetryStats({ everyMs: 50, log: (_m, c) => logs.push(c) })
+ await runWork('background', async () => {
+ await expect(retryTransient(async () => { throw new Error('no') }, noWait)).rejects.toThrow('no')
+ })
+ await logged(logs)
+ expect(logs[0]!.routes).toEqual({ background: { attempts: 1, retries: {}, busy: 0, reloads: 0 } })
+})
diff --git a/backend/src/superTabs/retryStats.ts b/backend/src/superTabs/retryStats.ts
new file mode 100644
index 000000000..044187d94
--- /dev/null
+++ b/backend/src/superTabs/retryStats.ts
@@ -0,0 +1,45 @@
+/**
+ * How often Swarm transactions run again, per route, logged every minute (DB roadmap 2.1). The design keeps a retry of up
+ * to 8 attempts for write conflicts (retry.ts) and a second load when a command names what its first load lacks
+ * (store.ts ensure); prod receipts showed ~0.05% of commands could have collided (2026-10-31), but conflicts with
+ * deliveries, machine reports and sweeps leave no receipt. This says when that changes. Route patterns and error codes
+ * only, never an id.
+ */
+import { logger } from '../utils/logger.js'
+import { workKind } from '../lib/queryStats.js'
+
+interface Row { attempts: number; retries: Record<string, number>; busy: number; reloads: number }
+/** Routes one window keeps; any more count as `other`, so memory stays bounded. */
+const MAX_ROUTES = 50
+
+let rows = new Map<string, Row>()
+const row = (): Row => {
+ const kind = workKind()
+ const key = rows.has(kind) || rows.size < MAX_ROUTES ? kind : 'other'
+ let found = rows.get(key)
+ if (!found) rows.set(key, found = { attempts: 0, retries: {}, busy: 0, reloads: 0 })
+ return found
+}
+
+/** A transaction attempt started (retry.ts). */
+export function recordAttempt(): void { row().attempts++ }
+/** An attempt failed with `code` and runs again. */
+export function recordRetry(code: string): void { const r = row(); r.retries[code] = (r.retries[code] ?? 0) + 1 }
+/** The attempts ran out: the caller answers SWARM_BUSY. */
+export function recordBusy(): void { row().busy++ }
+/** A command loaded its Swarm again for parts the first load lacked (store.ts ensure). */
+export function recordReload(): void { row().reloads++ }
+
+/** Logs `swarm retries` every `everyMs` while anything was counted. Returns its stop. */
+export function startRetryStats({ everyMs = 60_000, log = (message: string, context: Record<string, unknown>) => logger.info(message, context) }:
+ { everyMs?: number; log?: (message: string, context: Record<string, unknown>) => void } = {}): () => void {
+ let since = Date.now()
+ const timer = setInterval(() => {
+ const now = Date.now(), window = rows
+ rows = new Map()
+ if (window.size) log('swarm retries', { windowMs: now - since, routes: Object.fromEntries(window) })
+ since = now
+ }, everyMs)
+ timer.unref()
+ return () => clearInterval(timer)
+}
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index f7d73aef3..936bc878f 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -21,6 +21,7 @@ import { randomBytes } from 'node:crypto'
import type { Readable } from 'node:stream'
import { logger } from '../utils/logger.js'
import { retryTransient } from './retry.js'
+import { recordReload } from './retryStats.js'
import { buddyChatId, type ChosenBuddyName } from './buddies.js'
import { canReadAgent, canReadGroup, canWatchAgent, groupProject, ledProject, onTeamAccount, projectOf, projectsOf, agentProjectId, readConversation, roleVersion, skillModeOf, type ProjectVisibility } from './projects.js'
import { bindSession, makeProjectPrivate } from './sessionScope.js'
@@ -700,11 +701,11 @@ export class SuperTabStore {
/** Loads again with more named groups, agents or starts when the first load lacks one a command needs. */
private async ensure(tx: Transaction, loaded: Loaded, named: Named): Promise<Loaded> {
- if (named.allGroups) return loadRow(tx, loaded.row, named)
const has = (ids: string[] | undefined, present: (id: string) => boolean) => (ids ?? []).every(present)
const { state } = loaded
- if (has(named.groupIds, id => state.conversations.some(c => c.id === id)) && has(named.startIds, id => !!state.starts?.some(s => s.id === id))
+ if (!named.allGroups && has(named.groupIds, id => state.conversations.some(c => c.id === id)) && has(named.startIds, id => !!state.starts?.some(s => s.id === id))
&& !named.agentIds?.length && !named.projectIds?.length && !named.startAgentIds?.length) return loaded
+ recordReload()
return loadRow(tx, loaded.row, named)
}
diff --git a/backend/src/worker.ts b/backend/src/worker.ts
index 3055d228d..a57b7a800 100644
--- a/backend/src/worker.ts
+++ b/backend/src/worker.ts
@@ -15,6 +15,7 @@ import {
import { ensurePlans } from './lib/db/seed.js'
import { prisma } from './lib/prisma.js'
import { startQueryStats } from './lib/queryStats.js'
+import { startRetryStats } from './superTabs/retryStats.js'
import { releaseSchema } from './lib/db/schemaRelease.js'
import { startSwarmSweeps } from './superTabs/sweeps.js'
import { startLegacyDrop } from './superTabs/migration/dropLegacyJob.js'
@@ -38,6 +39,8 @@ async function start(): Promise<void> {
// Every minute, this process's queries by model.action and filter shape (lib/queryStats.ts, DB roadmap Phase 1); its
// jobs and startup migrations count as background.
const stopQueryStats = startQueryStats()
+ // Every minute, how often Swarm transactions ran again and commands loaded twice (superTabs/retryStats.ts, DB 2.1).
+ const stopRetryStats = startRetryStats()
// Existing production data was created against staging upstreams. Stamp it before Prisma swaps
// the old global unique indexes for their per-environment replacements.
await migrateAutonomousEnvironment()
@@ -79,6 +82,7 @@ async function start(): Promise<void> {
stopReflection()
// The last partial minute, before the database closes.
stopQueryStats()
+ stopRetryStats()
void Promise.allSettled([prisma.$disconnect(), closeBus()]).finally(() => process.exit(0))
}
process.on('SIGINT', () => shutdown('SIGINT'))