The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Swarm retries log: transaction attempts, retries by code, busy answers and reloads per route (DB 2.1) (#2386)
0xken15eec0a76c9e8e
diff --git a/backend/src/server.ts b/backend/src/server.tsindex 91286b6c1..5b4736627 100644--- a/backend/src/server.ts+++ b/backend/src/server.ts@@ -32,6 +32,7 @@ import { redisWindowCounter } from './lib/redisRateCounter.js'import { selfProfileFromEnv } from './lib/selfProfile.js'import { startRelayUsageFlush } from './lib/productUsage.js'import { registerWorkContext, startQueryStats } from './lib/queryStats.js'+import { startRetryStats } from './superTabs/retryStats.js'import { registerRouteLatency, startRouteLatency } from './lib/routeLatency.js'import { drainAllSockets, openSocketCount, RELEASE_UPGRADE_SLOT, type SlotSocket } from './lib/wsServer.js'import { endpointOf, refusedUpgrade, startWsCounterLog, watchUpgrade } from './lib/wsAttribution.js'@@ -143,6 +144,8 @@ async function start(): Promise<void> {selfProfileFromEnv(app)// Always on, every minute (DB roadmap Phase 1): queries by model.action, filter shape and route, and each route's p50/p95.const stopQueryStats = startQueryStats()+ // Every minute, how often Swarm transactions ran again and commands loaded twice (superTabs/retryStats.ts, DB 2.1).+ const stopRetryStats = startRetryStats()const stopRouteLatency = startRouteLatency(registerRouteLatency(app))app.setErrorHandler(errorHandler)@@ -221,6 +224,7 @@ async function start(): Promise<void> {await Promise.race([stopRelayUsage().catch(() => {}), new Promise((resolve) => setTimeout(resolve, 3_000))])// The last partial minute of both logs, once the drain and the usage flush have run their queries.stopQueryStats()+ stopRetryStats()stopRouteLatency()stopMachineReportLog()// The database stays connected through the drain, for the requests still being served (lib/prisma.ts).diff --git a/backend/src/superTabs/retry.ts b/backend/src/superTabs/retry.tsindex 7f6273173..8ca5b5062 100644--- a/backend/src/superTabs/retry.ts+++ b/backend/src/superTabs/retry.ts@@ -1,4 +1,5 @@import { fail } from './model.js'+import { recordAttempt, recordBusy, recordRetry } from './retryStats.js'/** A Swarm's answers that say "try again" rather than "no": busy after its retries, or moving to its new store. */export const RETRYABLE_SWARM_CODES: ReadonlySet<string> = new Set(['SWARM_BUSY', 'SWARM_MIGRATING'])@@ -25,6 +26,7 @@ export async function retryTransient<T>(attempt: () => Promise<T>,sleep: (ms: number) => Promise<void> = ms => new Promise(resolve => setTimeout(resolve, ms)),alsoRetry: (error: unknown) => boolean = () => false): Promise<T> {for (let tries = 1; ; tries++) {+ recordAttempt()try {return await attempt()} catch (error) {@@ -33,8 +35,10 @@ export async function retryTransient<T>(attempt: () => Promise<T>,if (tries >= (errorCode(error) === 'P2028' ? EXPIRED_TRANSACTION_ATTEMPTS : TRANSACTION_ATTEMPTS)) {// A unique key still taken after every retry is a real duplicate, not contention.if (errorCode(error) === 'P2002') throw error+ recordBusy()fail('SWARM_BUSY', 'This Swarm is busy. Try again in a moment.', 503)}+ recordRetry(typeof errorCode(error) === 'string' ? errorCode(error) as string : 'transient')await sleep(Math.random() * Math.min(TRANSACTION_BACKOFF_CAP_MS, TRANSACTION_BACKOFF_MS * 2 ** (tries - 1)))}}diff --git a/backend/src/superTabs/retryStats.test.ts b/backend/src/superTabs/retryStats.test.tsnew file mode 100644index 000000000..a2ec57da4--- /dev/null+++ b/backend/src/superTabs/retryStats.test.ts@@ -0,0 +1,38 @@+import { afterEach, expect, it } from 'vitest'+import { runWork } from '../lib/queryStats.js'+import { retryTransient } from './retry.js'+import { recordReload, startRetryStats } from './retryStats.js'++let stop = () => {}+afterEach(() => stop())++const conflict = () => Object.assign(new Error('Transaction failed due to a write conflict'), { code: 'P2034' })+const noWait = async () => {}+const logged = async (logs: Array<Record<string, unknown>>) => { while (!logs.length) await new Promise(resolve => setTimeout(resolve, 10)) }++it('counts each route\'s transaction attempts, retries by code, busy answers and reloads, and logs nothing in a quiet minute', async () => {+ const logs: Array<Record<string, unknown>> = []+ stop = startRetryStats({ everyMs: 50, log: (_m, c) => logs.push(c) })+ await runWork('http POST /api/super-tabs/:tabId/commands', async () => {+ let failures = 2+ await retryTransient(async () => { if (failures-- > 0) throw conflict(); return 'ok' }, noWait)+ recordReload()+ await expect(retryTransient(async () => { throw conflict() }, noWait)).rejects.toMatchObject({ code: 'SWARM_BUSY' })+ })+ await logged(logs)+ expect(logs[0]!.routes).toEqual({ 'http POST /api/super-tabs/:tabId/commands': { attempts: 11, retries: { P2034: 9 }, busy: 1, reloads: 1 } })+ // A minute with nothing to count says nothing.+ const count = logs.length+ await new Promise(resolve => setTimeout(resolve, 120))+ expect(logs).toHaveLength(count)+})++it('counts nothing for an error that is not retried', async () => {+ const logs: Array<Record<string, unknown>> = []+ stop = startRetryStats({ everyMs: 50, log: (_m, c) => logs.push(c) })+ await runWork('background', async () => {+ await expect(retryTransient(async () => { throw new Error('no') }, noWait)).rejects.toThrow('no')+ })+ await logged(logs)+ expect(logs[0]!.routes).toEqual({ background: { attempts: 1, retries: {}, busy: 0, reloads: 0 } })+})diff --git a/backend/src/superTabs/retryStats.ts b/backend/src/superTabs/retryStats.tsnew file mode 100644index 000000000..044187d94--- /dev/null+++ b/backend/src/superTabs/retryStats.ts@@ -0,0 +1,45 @@+/**+ * How often Swarm transactions run again, per route, logged every minute (DB roadmap 2.1). The design keeps a retry of up+ * to 8 attempts for write conflicts (retry.ts) and a second load when a command names what its first load lacks+ * (store.ts ensure); prod receipts showed ~0.05% of commands could have collided (2026-10-31), but conflicts with+ * deliveries, machine reports and sweeps leave no receipt. This says when that changes. Route patterns and error codes+ * only, never an id.+ */+import { logger } from '../utils/logger.js'+import { workKind } from '../lib/queryStats.js'++interface Row { attempts: number; retries: Record<string, number>; busy: number; reloads: number }+/** Routes one window keeps; any more count as `other`, so memory stays bounded. */+const MAX_ROUTES = 50++let rows = new Map<string, Row>()+const row = (): Row => {+ const kind = workKind()+ const key = rows.has(kind) || rows.size < MAX_ROUTES ? kind : 'other'+ let found = rows.get(key)+ if (!found) rows.set(key, found = { attempts: 0, retries: {}, busy: 0, reloads: 0 })+ return found+}++/** A transaction attempt started (retry.ts). */+export function recordAttempt(): void { row().attempts++ }+/** An attempt failed with `code` and runs again. */+export function recordRetry(code: string): void { const r = row(); r.retries[code] = (r.retries[code] ?? 0) + 1 }+/** The attempts ran out: the caller answers SWARM_BUSY. */+export function recordBusy(): void { row().busy++ }+/** A command loaded its Swarm again for parts the first load lacked (store.ts ensure). */+export function recordReload(): void { row().reloads++ }++/** Logs `swarm retries` every `everyMs` while anything was counted. Returns its stop. */+export function startRetryStats({ everyMs = 60_000, log = (message: string, context: Record<string, unknown>) => logger.info(message, context) }:+ { everyMs?: number; log?: (message: string, context: Record<string, unknown>) => void } = {}): () => void {+ let since = Date.now()+ const timer = setInterval(() => {+ const now = Date.now(), window = rows+ rows = new Map()+ if (window.size) log('swarm retries', { windowMs: now - since, routes: Object.fromEntries(window) })+ since = now+ }, everyMs)+ timer.unref()+ return () => clearInterval(timer)+}diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex f7d73aef3..936bc878f 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -21,6 +21,7 @@ import { randomBytes } from 'node:crypto'import type { Readable } from 'node:stream'import { logger } from '../utils/logger.js'import { retryTransient } from './retry.js'+import { recordReload } from './retryStats.js'import { buddyChatId, type ChosenBuddyName } from './buddies.js'import { canReadAgent, canReadGroup, canWatchAgent, groupProject, ledProject, onTeamAccount, projectOf, projectsOf, agentProjectId, readConversation, roleVersion, skillModeOf, type ProjectVisibility } from './projects.js'import { bindSession, makeProjectPrivate } from './sessionScope.js'@@ -700,11 +701,11 @@ export class SuperTabStore {/** Loads again with more named groups, agents or starts when the first load lacks one a command needs. */private async ensure(tx: Transaction, loaded: Loaded, named: Named): Promise<Loaded> {- if (named.allGroups) return loadRow(tx, loaded.row, named)const has = (ids: string[] | undefined, present: (id: string) => boolean) => (ids ?? []).every(present)const { state } = loaded- if (has(named.groupIds, id => state.conversations.some(c => c.id === id)) && has(named.startIds, id => !!state.starts?.some(s => s.id === id))+ if (!named.allGroups && has(named.groupIds, id => state.conversations.some(c => c.id === id)) && has(named.startIds, id => !!state.starts?.some(s => s.id === id))&& !named.agentIds?.length && !named.projectIds?.length && !named.startAgentIds?.length) return loaded+ recordReload()return loadRow(tx, loaded.row, named)}diff --git a/backend/src/worker.ts b/backend/src/worker.tsindex 3055d228d..a57b7a800 100644--- a/backend/src/worker.ts+++ b/backend/src/worker.ts@@ -15,6 +15,7 @@ import {import { ensurePlans } from './lib/db/seed.js'import { prisma } from './lib/prisma.js'import { startQueryStats } from './lib/queryStats.js'+import { startRetryStats } from './superTabs/retryStats.js'import { releaseSchema } from './lib/db/schemaRelease.js'import { startSwarmSweeps } from './superTabs/sweeps.js'import { startLegacyDrop } from './superTabs/migration/dropLegacyJob.js'@@ -38,6 +39,8 @@ async function start(): Promise<void> {// Every minute, this process's queries by model.action and filter shape (lib/queryStats.ts, DB roadmap Phase 1); its// jobs and startup migrations count as background.const stopQueryStats = startQueryStats()+ // Every minute, how often Swarm transactions ran again and commands loaded twice (superTabs/retryStats.ts, DB 2.1).+ const stopRetryStats = startRetryStats()// Existing production data was created against staging upstreams. Stamp it before Prisma swaps// the old global unique indexes for their per-environment replacements.await migrateAutonomousEnvironment()@@ -79,6 +82,7 @@ async function start(): Promise<void> {stopReflection()// The last partial minute, before the database closes.stopQueryStats()+ stopRetryStats()void Promise.allSettled([prisma.$disconnect(), closeBus()]).finally(() => process.exit(0))}process.on('SIGINT', () => shutdown('SIGINT'))