The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Merge batch 11-05 i1300: #3087 #3070 (#3096)
0xken15b14dc059f093
diff --git a/backend/integration/prismaQueryTiming.test.ts b/backend/integration/prismaQueryTiming.test.tsindex e62928ef6..e5524c28e 100644--- a/backend/integration/prismaQueryTiming.test.ts+++ b/backend/integration/prismaQueryTiming.test.ts@@ -8,6 +8,7 @@ import { MongoMemoryReplSet } from 'mongodb-memory-server-core'import { PrismaClient } from '@prisma/client'import { createMetrics } from '../src/lib/metrics.js'import { withQueryStats } from '../src/lib/queryStats.js'+import { createEngineCpu } from '../src/lib/engineCpu.js'// A fresh loopback replica set, created here. This suite never reads DATABASE_URL or an existing database.let replica: MongoMemoryReplSet@@ -62,4 +63,20 @@ describe('Prisma query timing against a real database', () => {expect(await queryCounts(metrics)).toEqual(['prisma_query_duration_seconds_count{model="User",operation="count"} 1'])await prisma.$disconnect()})++ // Linux only, as in production: the sampler reads the engine's own threads from /proc (task #3041). If Prisma renamed+ // its tokio threads, nothing would be attributed and `prisma by query` would read zero.+ it.skipIf(process.platform !== 'linux')('measures the real engine\'s CPU for queries that run alone', async () => {+ const infos: Array<Record<string, unknown>> = []+ const engine = createEngineCpu({ gapMs: 0, log: (_message, context) => { infos.push(context) } })+ const db = withQueryStats(base, () => undefined, undefined, undefined, undefined, () => engine)+ for (let i = 0; i < 50; i++) await db.user.findMany({ take: 20 })+ engine.flush()+ const line = infos[0]!+ expect(line.threads).toBeUndefined()+ expect(line).toMatchObject({ queries: 50, discarded: 0 })+ expect(line.sampled).toBeGreaterThanOrEqual(45)+ expect(line.engineMs as number).toBeGreaterThan(0)+ expect((line.top as Array<{ op: string }>)[0]).toMatchObject({ op: 'User.findMany', n: 50 })+ })})diff --git a/backend/src/lib/engineCpu.test.ts b/backend/src/lib/engineCpu.test.tsnew file mode 100644index 000000000..4a2a4069f--- /dev/null+++ b/backend/src/lib/engineCpu.test.ts@@ -0,0 +1,169 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { afterEach, describe, expect, it } from 'vitest'+import { createEngineCpu, readEngineCpu, type EngineCpuOptions } from './engineCpu.js'+import { runWork } from './queryStats.js'++const roots: string[] = []+afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) })++/** A fake /proc/self/task: each thread's comm, and its schedstat (ns on CPU) or only its stat line (10 ms ticks). */+function procTree(threads: Array<{ tid: number; name: string; ns?: number; ticks?: number }>): string {+ const root = mkdtempSync(join(tmpdir(), 'engine-cpu-'))+ roots.push(root)+ for (const t of threads) {+ const dir = join(root, String(t.tid))+ mkdirSync(dir)+ writeFileSync(join(dir, 'comm'), `${t.name}\n`)+ if (t.ns !== undefined) writeFileSync(join(dir, 'schedstat'), `${t.ns} 5000 12\n`)+ const ticks = t.ticks ?? 0+ writeFileSync(join(dir, 'stat'), `${t.tid} (${t.name}) S 1 1 1 0 -1 0 0 0 0 0 ${ticks} 0 0 0 20 0 1 0 100 0 0`)+ }+ return root+}++describe('the engine threads\' CPU', () => {+ it('sums the Prisma engine threads only, in ns from schedstat', () => {+ const root = procTree([+ { tid: 10, name: 'node', ns: 9_000_000_000 },+ { tid: 11, name: 'tokio-runtime-w', ns: 1_500_000 },+ { tid: 12, name: 'tokio-runtime-w', ns: 2_500_000 },+ { tid: 13, name: 'opentelemetry-e', ns: 1_000 },+ { tid: 14, name: 'libuv-worker', ns: 7_000_000 },+ { tid: 15, name: 'node', ns: 8_000_000 },+ ])+ expect(readEngineCpu(root, 10)).toEqual({ ns: 4_001_000, clock: 'schedstat' })+ })++ it('falls back to stat\'s ticks where the kernel has no schedstat', () => {+ const root = procTree([{ tid: 10, name: 'node', ticks: 99 }, { tid: 11, name: 'tokio-runtime-w', ticks: 3 }])+ expect(readEngineCpu(root, 10)).toEqual({ ns: 30_000_000, clock: 'stat' })+ })++ it('fails loudly when the folder cannot be read', () => {+ expect(() => readEngineCpu('/nonexistent-engine-cpu-root', 1)).toThrow()+ })+})++/** A sampler on a fake clock and a fake engine CPU counter the test moves. */+function sampler(options: Partial<EngineCpuOptions> = {}) {+ let ms = 1_000, ns = 0, reads = 0+ const infos: Array<{ message: string; context: Record<string, unknown> }> = []+ const engine = createEngineCpu({+ pod: 'pod-1', now: () => ms, read: () => { reads++; return { ns, clock: 'schedstat' } },+ log: (message, context) => { infos.push({ message, context }) }, warn: () => {}, ...options,+ })+ return {+ engine, infos,+ advance(byMs: number) { ms += byMs },+ burn(engineMs: number) { ns += engineMs * 1e6 },+ reads: () => reads,+ }+}++type Row = { op: string; route: string; n: number; sampled: number; engineMs: number; usPerQuery: number; docs: number }+const top = (context: Record<string, unknown>) => context.top as Row[]++describe('prisma by query', () => {+ it('times a query that runs alone and scales its mean by how often it ran', () => {+ const t = sampler()+ runWork('http GET /api/super-tabs/:tabId/changes', () => {+ const sample = t.engine.start()+ t.burn(4)+ t.engine.end(sample, 'SwarmGroup', 'findMany', {}, [{ a: 1 }, { a: 2 }])+ // Inside the gap: counted, not sampled.+ for (let i = 0; i < 9; i++) t.engine.end(t.engine.start(), 'SwarmGroup', 'findMany', {}, [{ a: 1 }, { a: 2 }])+ })+ t.engine.flush()+ expect(t.infos).toHaveLength(1)+ expect(t.infos[0]!.message).toBe('prisma by query')+ expect(top(t.infos[0]!.context)[0]).toEqual({ op: 'SwarmGroup.findMany', route: 'http GET /api/super-tabs/:tabId/changes', n: 10, sampled: 1,+ engineMs: 40, usPerQuery: 4000, docs: 20 })+ expect(t.infos[0]!.context).toMatchObject({ pod: 'pod-1', clock: 'schedstat', queries: 10, sampled: 1, discarded: 0, engineMs: 40 })+ })++ it('drops a sample another query overlapped, at its start or before its end', () => {+ const t = sampler()+ // Another query starts before the sampled one ends.+ const first = t.engine.start()+ const second = t.engine.start()+ t.burn(10)+ t.engine.end(first, 'Machine', 'findUnique', {}, { id: 'm' })+ t.engine.end(second, 'User', 'findFirst', {}, { id: 'u' })+ t.advance(1_000)+ // One already in flight: the new one is not sampled at all.+ const running = t.engine.start()+ t.advance(1_000)+ const late = t.engine.start()+ t.burn(10)+ t.engine.end(late, 'Machine', 'findUnique', {}, { id: 'm' })+ t.engine.end(running, 'User', 'findFirst', {}, { id: 'u' })+ t.engine.flush()+ const context = t.infos[0]!.context+ // Both samples opened were overlapped: the first by a second start, the one in flight by the late one.+ expect(context).toMatchObject({ queries: 4, sampled: 0, discarded: 2, engineMs: 0 })+ })++ it('reads the engine CPU at most once per gap, so its cost stays bounded at any query rate', () => {+ const t = sampler({ gapMs: 100 })+ for (let i = 0; i < 1_000; i++) {+ t.advance(1)+ t.engine.end(t.engine.start(), 'SwarmHead', 'findUnique', {}, { id: 'h' })+ }+ t.engine.flush()+ // 1,000 solo queries over a second: about 10 samples, each a read before and after.+ expect(t.reads()).toBeLessThanOrEqual(22)+ expect(t.infos[0]!.context).toMatchObject({ queries: 1_000 })+ expect((t.infos[0]!.context.sampled as number)).toBeGreaterThanOrEqual(9)+ })++ it('brings the in-flight count back after every end, so sampling never stops', () => {+ const t = sampler()+ for (let i = 0; i < 5; i++) t.engine.start()+ for (let i = 0; i < 5; i++) t.engine.end(undefined, 'User', 'update', {}, undefined)+ t.advance(1_000)+ const sample = t.engine.start()+ t.burn(2)+ t.engine.end(sample, 'User', 'findFirst', {}, { id: 'u' })+ t.engine.flush()+ expect(t.infos[0]!.context).toMatchObject({ sampled: 1, engineMs: 2 })+ })++ it('ranks by engine ms, names raw commands by collection, sums each op across routes, and starts a new window', () => {+ const t = sampler({ gapMs: 0 })+ const run = (route: string, model: string | undefined, action: string, args: unknown, engineMs: number) => runWork(route, () => {+ const sample = t.engine.start()+ t.burn(engineMs)+ t.engine.end(sample, model, action, args, [])+ })+ run('http GET /api/a', 'SwarmGroup', 'findMany', {}, 1)+ run('http GET /api/b', 'SwarmGroup', 'findMany', {}, 2)+ run('http POST /api/hosts/:id/readings', undefined, '$runCommandRaw', { aggregate: 'SwarmMachine', pipeline: [] }, 5)+ t.engine.flush()+ const context = t.infos[0]!.context+ expect(top(context).map(r => [r.op, r.route, r.engineMs])).toEqual([+ ['SwarmMachine.$runCommandRaw aggregate', 'http POST /api/hosts/:id/readings', 5],+ ['SwarmGroup.findMany', 'http GET /api/b', 2],+ ['SwarmGroup.findMany', 'http GET /api/a', 1],+ ])+ expect(context.byOp).toEqual({ 'SwarmMachine.$runCommandRaw aggregate': 5, 'SwarmGroup.findMany': 3 })+ expect(context.costMs).toEqual(expect.any(Number))+ t.engine.flush()+ expect(t.infos[1]!.context).toMatchObject({ queries: 0, sampled: 0, engineMs: 0, top: [] })+ })++ it('logs unavailable, and still counts queries, where the engine threads cannot be read', () => {+ const t = sampler({ read: () => { throw new Error('ENOENT /proc/self/task') } })+ t.engine.end(t.engine.start(), 'User', 'findFirst', {}, { id: 'u' })+ t.engine.flush()+ expect(t.infos[0]!.context).toMatchObject({ threads: 'unavailable', reason: 'ENOENT /proc/self/task', queries: 1, sampled: 0 })+ })++ it('names no value: ops are model.action and routes are patterns', () => {+ const t = sampler()+ runWork('http GET /api/x', () => t.engine.end(t.engine.start(), 'User', 'findFirst', { where: { email: '[email protected]' } }, { email: '[email protected]' }))+ t.engine.flush()+ expect(JSON.stringify(t.infos)).not.toContain('[email protected]')+ })+})diff --git a/backend/src/lib/engineCpu.ts b/backend/src/lib/engineCpu.tsnew file mode 100644index 000000000..60aca846b--- /dev/null+++ b/backend/src/lib/engineCpu.ts@@ -0,0 +1,195 @@+import { readdirSync, readFileSync } from 'node:fs'+import { hostname } from 'node:os'+import { performance } from 'node:perf_hooks'+import { logger } from '../utils/logger.js'+import { parseThreadStat, threadBucket } from './cpuStats.js'+import { docsOf, opName, workKind } from './queryStats.js'++/**+ * Which queries the Prisma engine spends its CPU on (task #3041). On 11-05 the engine's threads took 45% of a pod's CPU+ * (`cpu by thread`, lib/cpuStats.ts), as much as all our JS on the main thread, and nothing said which queries.+ *+ * Prisma's `query` events cannot say it: for MongoDB their `duration` is the database call, not the engine's own work+ * (building the query, BSON to values, values to the JSON string the client parses), and the `query` log level formats+ * every query, so it would cost what it measures. Instead: Linux keeps each thread's time on CPU in ns+ * (`/proc/self/task/<tid>/schedstat`). While a query runs alone in the process, the engine threads' CPU between its start+ * and its end is its own. The query hook (lib/queryStats.ts withQueryStats) calls `start` and `end` for every query; a+ * query that starts with none in flight, at most once per `gapMs`, is read before and after, and kept only if no other+ * query started before it ended. Each key's mean is scaled by how often it ran in the minute.+ *+ * `prisma by query`, every minute: per op (model.action, a raw command by collection) and route, the count, the samples,+ * the estimated engine ms and µs per query, and the documents; `byOp` sums each op across routes; `engineMs` sums the+ * estimates, to read beside `cpu by thread`'s `prismaEngineMs`; `costMs` is the sampler's own time. Raw commands run in+ * the same engine and are measured the same way. Like `db by shape`, it names no value: routes are patterns.+ */++export type EngineClock = 'schedstat' | 'stat'++/** The Prisma engine threads' CPU so far, in ns, from `schedstat` where the kernel has it, else `stat`'s 10 ms ticks (each+ * sample is coarse, the mean over many is not). [names] caches each thread's name by tid. Throws when [root] cannot be+ * listed (not Linux, or /proc not mounted); a thread that exits while it is read is skipped. */+export function readEngineCpu(root = '/proc/self/task', pid = process.pid, names = new Map<number, string>()): { ns: number; clock: EngineClock } {+ let ns = 0, clock: EngineClock = 'schedstat'+ for (const entry of readdirSync(root)) {+ const tid = Number(entry)+ if (tid === pid) continue+ let name = names.get(tid)+ if (name === undefined) {+ try { name = readFileSync(`${root}/${entry}/comm`, 'utf8').trim() } catch { continue }+ names.set(tid, name)+ }+ if (threadBucket(name, tid, pid) !== 'prismaEngine') continue+ try {+ ns += Number(readFileSync(`${root}/${entry}/schedstat`, 'utf8').split(' ', 1)[0]) || 0+ } catch {+ try { ns += parseThreadStat(readFileSync(`${root}/${entry}/stat`, 'utf8')).cpuMs * 1e6; clock = 'stat' } catch { continue }+ }+ }+ return { ns, clock }+}++/** What `start` opened: the engine CPU before the query, and the start count it must still match at its end. */+export type EngineSample = { ns: number; seq: number }++export type EngineCpu = {+ /** Every query calls it as it starts. Answers the sample it opened, if it opened one. */+ start(): EngineSample | undefined+ /** Every query calls it once as it ends, failed ones too, with what its start answered. */+ end(sample: EngineSample | undefined, model: string | undefined, action: string, args: unknown, result: unknown): void+ /** Logs the window so far and starts a new one. */+ flush(): void+}++export type EngineCpuOptions = {+ pod?: string+ /** Least time between two samples' starts: bounds the sampler's cost at any query rate. */+ gapMs?: number+ /** How many keys the line lists, by engine ms. */+ top?: number+ /** Keys one window keeps; queries past them count under `other`. */+ maxKeys?: number+ now?: () => number+ read?: () => { ns: number; clock: EngineClock }+ log?: (message: string, context: Record<string, unknown>) => void+ warn?: (message: string, context: Record<string, unknown>) => void+}++type Entry = { op: string; route: string; n: number; docs: number; sampled: number; sampledNs: number }++const OTHER = 'other'++export function createEngineCpu({ pod = `${hostname()}:${process.pid}`, gapMs = 100, top = 25, maxKeys = 1000, now = () => performance.now(),+ read, log = (m, c) => logger.info(m, c), warn = (m, c) => logger.warn(m, c) }: EngineCpuOptions = {}): EngineCpu {+ const names = new Map<number, string>()+ const reader = read ?? (() => readEngineCpu(undefined, undefined, names))+ let entries = new Map<string, Entry>()+ let inFlight = 0, starts = 0, lastSample = -Infinity, queries = 0, sampled = 0, discarded = 0, costMs = 0, started = now()+ let clock: EngineClock | undefined, unavailable: string | undefined, warned = false++ /** The engine CPU now, timed into the sampler's own cost; undefined when it cannot be read. */+ const timed = (): number | undefined => {+ const t = performance.now()+ try {+ const reading = reader()+ clock = reading.clock+ unavailable = undefined+ return reading.ns+ } catch (error) {+ unavailable = error instanceof Error ? error.message : String(error)+ return undefined+ } finally {+ costMs += performance.now() - t+ }+ }++ const entryFor = (op: string, route: string): Entry => {+ let key = `${op}\0${route}`+ let entry = entries.get(key)+ if (entry) return entry+ if (entries.size >= maxKeys - 1) { key = OTHER; entry = entries.get(OTHER) }+ if (!entry) {+ entry = key === OTHER ? { op: OTHER, route: '', n: 0, docs: 0, sampled: 0, sampledNs: 0 } : { op, route, n: 0, docs: 0, sampled: 0, sampledNs: 0 }+ entries.set(key, entry)+ }+ return entry+ }++ return {+ start() {+ const alone = inFlight === 0+ inFlight+++ starts+++ if (!alone) return undefined+ const at = now()+ if (at - lastSample < gapMs) return undefined+ lastSample = at+ const ns = timed()+ return ns === undefined ? undefined : { ns, seq: starts }+ },++ end(sample, model, action, args, result) {+ inFlight = Math.max(0, inFlight - 1)+ try {+ queries+++ const entry = entryFor(opName(model, action, args), workKind())+ const docs = docsOf(result)+ entry.n+++ entry.docs += docs+ if (!sample) return+ // Another query started while this one ran: the engine's CPU since is not this query's alone.+ if (starts !== sample.seq) { discarded++; return }+ const ns = timed()+ if (ns === undefined) return+ entry.sampled+++ entry.sampledNs += Math.max(0, ns - sample.ns)+ sampled+++ } catch (error) {+ if (warned) return+ warned = true+ warn('prisma by query failed', { error: error instanceof Error ? error.message : String(error) })+ }+ },++ flush() {+ const rows = [...entries.values()].map(e => {+ const mean = e.sampled ? e.sampledNs / e.sampled : 0+ return { op: e.op, route: e.route, n: e.n, sampled: e.sampled, engineMs: Math.round(mean * e.n / 1e6), usPerQuery: Math.round(mean / 1e3), docs: e.docs }+ })+ const byOp = new Map<string, number>()+ for (const r of rows) byOp.set(r.op, (byOp.get(r.op) ?? 0) + r.engineMs)+ const context: Record<string, unknown> = { pod, windowMs: Math.round(now() - started) }+ if (unavailable !== undefined) Object.assign(context, { threads: 'unavailable', reason: unavailable })+ else Object.assign(context, { clock: clock ?? 'none' })+ Object.assign(context, {+ queries, sampled, discarded, engineMs: rows.reduce((total, r) => total + r.engineMs, 0),+ costMs: Math.round(costMs * 10) / 10, keys: rows.length,+ top: rows.filter(r => r.engineMs > 0).sort((a, b) => b.engineMs - a.engineMs || b.n - a.n).slice(0, top),+ byOp: Object.fromEntries([...byOp].filter(([, ms]) => ms > 0).sort((a, b) => b[1] - a[1]).slice(0, 15)),+ })+ log('prisma by query', context)+ entries = new Map(); queries = 0; sampled = 0; discarded = 0; costMs = 0; started = now()+ // A tid can be reused by a new thread: names are read again each window.+ names.clear()+ },+ }+}++let active: EngineCpu | undefined++/** This process's sampler, once started; the query hook (lib/prisma.ts) reads it on every query. */+export function activeEngineCpu(): EngineCpu | undefined {+ return active+}++/** Starts this process's `prisma by query` line, every [everyMs]. Returns a stop that logs the last partial window. */+export function startEngineCpu(options: EngineCpuOptions & { everyMs?: number } = {}): () => void {+ const engine = createEngineCpu(options)+ active = engine+ const timer = setInterval(() => engine.flush(), options.everyMs ?? 60_000)+ timer.unref()+ return () => {+ clearInterval(timer)+ if (active === engine) active = undefined+ engine.flush()+ }+}diff --git a/backend/src/lib/harnessAuthCache.test.ts b/backend/src/lib/harnessAuthCache.test.tsnew file mode 100644index 000000000..1faeb64dc--- /dev/null+++ b/backend/src/lib/harnessAuthCache.test.ts@@ -0,0 +1,60 @@+import { describe, expect, it, vi } from 'vitest'++const fakes = vi.hoisted(() => {+ const writes: Array<{ key: string; value: string; ttl: number }> = []+ let fail = false+ return {+ writes,+ setFail: (v: boolean) => { fail = v },+ pub: { set: vi.fn(async (key: string, value: string, _ex: string, ttl: number) => {+ if (fail) throw new Error('redis down')+ writes.push({ key, value, ttl }); return 'OK'+ }) },+ }+})+vi.mock('./bus.js', () => ({ pub: fakes.pub }))++import { AUTH_CACHE_TTL_MS, AUTH_EPOCH_TTL_SEC, authEpochKey, createAuthCache, signInsChanged } from './harnessAuthCache.js'++const user = { sub: 'u1', email: '[email protected]', role: 'user', autonomousEnv: 'prod' as const, harnessSessionId: 's1', harnessSessionKind: 'viewer' as const }+const entry = { sessionId: 's1', userId: 'u1', epoch: null, user }++describe('the per-pod sign-in cache', () => {+ it('keeps a sign-in for 30 seconds, then forgets it', () => {+ let now = 1_000+ const cache = createAuthCache({ now: () => now })+ cache.set('h1', entry)+ now += AUTH_CACHE_TTL_MS - 1+ expect(cache.get('h1')).toMatchObject({ sessionId: 's1', userId: 'u1' })+ now += 1+ expect(cache.get('h1')).toBeUndefined()+ expect(AUTH_CACHE_TTL_MS).toBe(30_000)+ })++ it('holds at most its bound, dropping the oldest first', () => {+ const cache = createAuthCache({ max: 2, now: () => 0 })+ cache.set('a', entry); cache.set('b', entry); cache.set('c', entry)+ expect(cache.get('a')).toBeUndefined()+ expect(cache.get('b')).toBeDefined()+ expect(cache.get('c')).toBeDefined()+ expect(cache.size()).toBe(2)+ })+})++describe('a change to a person\'s sign-ins', () => {+ it('writes a fresh epoch for each person, never one an earlier epoch could equal, outliving every cached sign-in', async () => {+ fakes.writes.length = 0+ await signInsChanged('u1', 'u2', 'u1')+ await signInsChanged('u1')+ expect(fakes.writes.map(w => w.key)).toEqual([authEpochKey('u1'), authEpochKey('u2'), authEpochKey('u1')])+ expect(new Set(fakes.writes.map(w => w.value)).size).toBe(3)+ for (const w of fakes.writes) expect(w.ttl).toBe(AUTH_EPOCH_TTL_SEC)+ expect(AUTH_EPOCH_TTL_SEC * 1000).toBeGreaterThan(AUTH_CACHE_TTL_MS * 10)+ })++ it('throws when the epoch cannot be written, so the caller cannot report a sign-out that other pods may still serve', async () => {+ fakes.setFail(true)+ await expect(signInsChanged('u1')).rejects.toThrow('redis down')+ fakes.setFail(false)+ })+})diff --git a/backend/src/lib/harnessAuthCache.ts b/backend/src/lib/harnessAuthCache.tsnew file mode 100644index 000000000..7e380928e--- /dev/null+++ b/backend/src/lib/harnessAuthCache.ts@@ -0,0 +1,72 @@+import { randomBytes } from 'node:crypto'+import { pub } from './bus.js'+import type { AuthUser } from './ssoAuth.js'++/**+ * A Harness sign-in each pod remembers for [AUTH_CACHE_TTL_MS], so a request signed in with a Harness session does+ * not read its session row and its user row from Mongo every time (task #3041: `harness_sessions` 47 reads/s and+ * `users` 53/s on the primary, nearly all of them this check).+ *+ * Revocation still takes effect at once, on every pod: each person has a sign-in EPOCH in Redis, read in the same+ * call that reads the access token (lib/harnessSession.ts). Every write that ends a session or changes what a sign-in+ * answers sets it to a new random value, after the write ([signInsChanged]); a remembered sign-in whose epoch moved is+ * read from Mongo again. The miss path reads the epoch before the session and the user, so a sign-out that lands+ * between those reads leaves the remembered epoch stale rather than being missed.+ */++export const AUTH_CACHE_TTL_MS = 30_000+/** Far past [AUTH_CACHE_TTL_MS]: an epoch that expired only costs a pod one read from Mongo. */+export const AUTH_EPOCH_TTL_SEC = 60 * 60+const AUTH_CACHE_MAX = 20_000++export const authEpochKey = (userId: string): string => `hnauth:epoch:${userId}`++export interface RememberedSignIn {+ sessionId: string+ userId: string+ /** The person's epoch as read before the session and the user were; null when Redis held none. */+ epoch: string | null+ user: AuthUser+}++export interface AuthCache {+ get(tokenHash: string): RememberedSignIn | undefined+ set(tokenHash: string, entry: RememberedSignIn): void+ delete(tokenHash: string): void+ clear(): void+ size(): number+}++/** A map of at most [max] sign-ins, each forgotten [AUTH_CACHE_TTL_MS] after it was remembered, oldest dropped first. */+export function createAuthCache({ max = AUTH_CACHE_MAX, now = () => Date.now() }: { max?: number; now?: () => number } = {}): AuthCache {+ const entries = new Map<string, { at: number; entry: RememberedSignIn }>()+ return {+ get(tokenHash) {+ const held = entries.get(tokenHash)+ if (!held) return undefined+ if (now() - held.at >= AUTH_CACHE_TTL_MS) { entries.delete(tokenHash); return undefined }+ return held.entry+ },+ set(tokenHash, entry) {+ entries.delete(tokenHash)+ entries.set(tokenHash, { at: now(), entry })+ // A Map keeps insertion order, so its first key is the oldest.+ while (entries.size > max) entries.delete(entries.keys().next().value!)+ },+ delete(tokenHash) { entries.delete(tokenHash) },+ clear() { entries.clear() },+ size() { return entries.size },+ }+}++export const harnessAuthCache = createAuthCache()++/**+ * The sign-ins of [userIds] changed: a session ended, or what their user row answers did. Call it AFTER the database+ * write. It throws when an epoch cannot be written, so a caller never reports a sign-out other pods may still serve.+ * A random value, never a counter: a key that expired and was written again cannot equal one a pod remembers.+ */+export async function signInsChanged(...userIds: string[]): Promise<void> {+ await Promise.all([...new Set(userIds)].map((userId) =>+ pub.set(authEpochKey(userId), randomBytes(12).toString('base64url'), 'EX', AUTH_EPOCH_TTL_SEC)))+}diff --git a/backend/src/lib/harnessSession.test.ts b/backend/src/lib/harnessSession.test.tsindex ef3eb331a..8fc0b4b3c 100644--- a/backend/src/lib/harnessSession.test.ts+++ b/backend/src/lib/harnessSession.test.ts@@ -21,6 +21,10 @@ const fakes = vi.hoisted(() => {if (failRedis) throw new Error('redis down')return redis.get(key) ?? null}),+ mget: vi.fn(async (...keys: string[]) => {+ if (failRedis) throw new Error('redis down')+ return keys.map((key) => redis.get(key) ?? null)+ }),// GET then DEL, in one transaction — how a redeem spends a code.multi: vi.fn(() => {const ops: Array<() => unknown> = []@@ -88,7 +92,8 @@ vi.mock('../services/UserService.js', () => ({vi.mock('../services/index.js', () => ({ userService: { get: fakes.get, toPublic: (u: unknown) => u } }))import { authenticateAccessToken, SsoAuthError } from './ssoAuth.js'-import { HANDOFF_TTL_SEC, redeemHandoff, refreshHarnessSession, revokeHarnessSession, startHandoff } from './harnessSession.js'+import { HANDOFF_TTL_SEC, redeemHandoff, refreshHarnessSession, revokeHarnessSession, revokeHarnessSessionById, startHandoff } from './harnessSession.js'+import { harnessAuthCache } from './harnessAuthCache.js'import { HARNESS_ACCESS_PREFIX, harnessTokenHash, newHarnessToken } from './harnessTokenFormat.js'import { teamConversionMessage } from './teamSession.js'import { ed25519 } from '@noble/curves/ed25519.js'@@ -102,6 +107,7 @@ beforeEach(() => {fakes.redis.clear(); fakes.ttls.clear(); fakes.sessions.length = 0; fakes.users.clear(); fakes.machines.length = 0fakes.users.set(USER.id, USER)fakes.setFailRedis(false)+ harnessAuthCache.clear()vi.clearAllMocks()})@@ -164,6 +170,99 @@ describe('scan to sign in — the handoff', () => {})})+describe('a sign-in remembered for 30 seconds per pod (#3041)', () => {+ const signIn = async () => (await redeemHandoff((await startHandoff(USER.id)).code, 'phone'))!++ it('a second request in the window reads neither the session nor the user from Mongo, and answers the same', async () => {+ const { token } = await signIn()+ const first = await authenticateAccessToken(token)+ fakes.harnessSession.findUnique.mockClear(); fakes.get.mockClear()+ const second = await authenticateAccessToken(token)+ expect(second).toEqual(first)+ expect(fakes.harnessSession.findUnique).not.toHaveBeenCalled()+ expect(fakes.get).not.toHaveBeenCalled()+ })++ it('every request still checks its token in Redis: an access token gone from Redis stops at once', async () => {+ const { token } = await signIn()+ await authenticateAccessToken(token)+ for (const key of [...fakes.redis.keys()]) if (key.startsWith('hnauth:access:')) fakes.redis.delete(key)+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })+ })++ it('signing out ends a remembered sign-in at once', async () => {+ const { token, refreshToken } = await signIn()+ await authenticateAccessToken(token)+ await revokeHarnessSession(refreshToken!)+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })+ })++ it('revoking a device by id ends its remembered sign-in at once', async () => {+ const { token } = await signIn()+ await authenticateAccessToken(token)+ await revokeHarnessSessionById('s1', USER.id)+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })+ })++ it('a pod that remembers nothing of the sign-out still learns it from Redis: the change is not kept in this pod alone', async () => {+ const { token, refreshToken } = await signIn()+ await authenticateAccessToken(token)+ // The sign-out ran on another pod: only Redis and Mongo saw it.+ const remembered = harnessAuthCache.get(harnessTokenHash(token))+ await revokeHarnessSession(refreshToken!)+ expect(harnessAuthCache.get(harnessTokenHash(token))).toBe(remembered)+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })+ })++ it('a sign-out that lands between reading the session and remembering it is not missed', async () => {+ const { token } = await signIn()+ const read = fakes.harnessSession.findUnique.getMockImplementation()!+ fakes.harnessSession.findUnique.mockImplementationOnce(async (args: { where: Record<string, unknown> }) => {+ const live = { ...(await read(args))! }+ await revokeHarnessSessionById('s1', USER.id)+ return live+ })+ // This request read the session while it was live.+ await expect(authenticateAccessToken(token)).resolves.toMatchObject({ harnessSessionId: 's1' })+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })+ })++ it('a sign-in is read again after 30 seconds', async () => {+ const { token } = await signIn()+ const now = vi.spyOn(Date, 'now')+ const start = Date.now()+ now.mockReturnValue(start)+ await authenticateAccessToken(token)+ fakes.harnessSession.findUnique.mockClear()+ now.mockReturnValue(start + 30_000)+ await authenticateAccessToken(token)+ expect(fakes.harnessSession.findUnique).toHaveBeenCalledTimes(1)+ now.mockRestore()+ })++ it('Redis down on a remembered sign-in is still an outage, not a sign-in', async () => {+ const { token } = await signIn()+ await authenticateAccessToken(token)+ fakes.setFailRedis(true)+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'AUTH_SERVICE_UNAVAILABLE' })+ })++ it('remembers no refusal: a token refused once is read again next time', async () => {+ const { token } = await signIn()+ fakes.users.delete(USER.id)+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })+ fakes.users.set(USER.id, USER)+ await expect(authenticateAccessToken(token)).resolves.toMatchObject({ sub: USER.id })+ })++ it('a caller changing what it was handed changes nothing for the next request', async () => {+ const { token } = await signIn()+ const first = await authenticateAccessToken(token) as unknown as Record<string, unknown>+ first.role = 'admin'+ await expect(authenticateAccessToken(token)).resolves.toMatchObject({ role: 'user' })+ })+})+describe('scan to sign in — the routes', () => {let app: FastifyInstanceconst signedIn = { sub: USER.id, email: USER.email, role: 'user', autonomousEnv: 'prod' as const }diff --git a/backend/src/lib/harnessSession.ts b/backend/src/lib/harnessSession.tsindex 7c3c0592d..632da0d0c 100644--- a/backend/src/lib/harnessSession.ts+++ b/backend/src/lib/harnessSession.ts@@ -7,6 +7,7 @@ import { TEAM_CONVERTED_FRAME } from './teamSession.js'import { userService } from '../services/UserService.js'import { storedAutonomousEnvironment, type AutonomousEnvironment } from './autonomousEnvironment.js'import { SsoAuthError, type AuthUser } from './ssoAuth.js'+import { authEpochKey, harnessAuthCache, signInsChanged } from './harnessAuthCache.js'import type { SsoClientId } from './sso.js'import {endedSessionData,@@ -182,6 +183,7 @@ async function convertComputer(userId: string, label: string, computerId: string} })return { session: created, machineIds: machines.map((m) => m.machineId) }})+ await signInsChanged(userId)for (const machineId of machineIds) {await publishDown(machineId, { connId: '', frame: { type: TEAM_CONVERTED_FRAME } }).catch(() => { /* best effort: the next connect decides */ })}@@ -289,10 +291,12 @@ export async function refreshHarnessSession(refreshToken: string): Promise<Harne* tokens it held are deleted. */export async function revokeHarnessSession(refreshToken: string): Promise<void> {if (!isHarnessRefreshToken(refreshToken)) return- await prisma.harnessSession.updateMany({- where: { refreshHash: harnessTokenHash(refreshToken), ...notRevoked },- data: endedSessionData(),- })+ const refreshHash = harnessTokenHash(refreshToken)+ const session = await prisma.harnessSession.findUnique({ where: { refreshHash }, select: { userId: true } })+ if (!session) return+ await prisma.harnessSession.updateMany({ where: { refreshHash, ...notRevoked }, data: endedSessionData() })+ // Even when it was already revoked: a retry after a failed epoch write must still reach every pod.+ await signInsChanged(session.userId)}async function issueAccess(sessionId: string, userId: string, kind: HarnessSessionKind): Promise<string> {@@ -306,36 +310,71 @@ async function issueAccess(sessionId: string, userId: string, kind: HarnessSessireturn token}+const invalid = (): SsoAuthError => new SsoAuthError('Invalid or expired access token', 'INVALID_TOKEN')+/*** The account a Harness access token speaks for. The session is read on every call, so a revoked- * one stops at once rather than when its access token runs out.+ * one stops at once rather than when its access token runs out — or, within [AUTH_CACHE_TTL_MS] of the+ * last read, remembered by this pod and checked against the person's sign-in epoch instead+ * (lib/harnessAuthCache.ts), which every sign-out moves. The access token is read from Redis either way.** Only a token that is not good is `INVALID_TOKEN`. Redis or Mongo failing is* `AUTH_SERVICE_UNAVAILABLE` — the caller's 503, not its 401 — because a 401 sends the phone to* refresh and, failing that, to sign in again, over what is only an outage.*/export async function authenticateHarnessAccessToken(token: string): Promise<AuthUser> {- const invalid = (): SsoAuthError => new SsoAuthError('Invalid or expired access token', 'INVALID_TOKEN')+ const hash = harnessTokenHash(token)+ const remembered = harnessAuthCache.get(hash)let user: Awaited<ReturnType<typeof userService.get>>let sessionId: string+ let userId: string+ let epoch: string | nulllet kind: string | null | undefinedlet computerId: string | null | undefinedtry {- const raw = await pub.get(accessKey(harnessTokenHash(token)))- if (!raw) throw invalid()+ let raw: string | null+ if (remembered) {+ [raw, epoch] = await pub.mget(accessKey(hash), authEpochKey(remembered.userId))+ } else {+ raw = await pub.get(accessKey(hash))+ epoch = null+ }+ if (!raw) {+ harnessAuthCache.delete(hash)+ throw invalid()+ }const held = JSON.parse(raw) as { sessionId?: unknown; userId?: unknown }if (typeof held.sessionId !== 'string' || typeof held.userId !== 'string') throw invalid()sessionId = held.sessionId+ userId = held.userId+ if (remembered && remembered.sessionId === sessionId && remembered.userId === userId && remembered.epoch === epoch) {+ return { ...remembered.user }+ }+ // The epoch before the session and the user: a sign-out landing between these reads leaves it stale.+ if (!remembered || remembered.userId !== userId) epoch = await pub.get(authEpochKey(userId))const session = await prisma.harnessSession.findUnique({ where: { id: sessionId } })- if (!session || session.revokedAt || session.userId !== held.userId) throw invalid()+ if (!session || session.revokedAt || session.userId !== userId) throw invalid()kind = session.kindcomputerId = session.computerId- user = await userService.get(held.userId)+ user = await userService.get(userId)} catch (err) {if (err instanceof SsoAuthError) throw errthrow new SsoAuthError('Authentication service unavailable', 'AUTH_SERVICE_UNAVAILABLE')}if (!user) throw invalid()+ const answer = identityOf(token, user, sessionId, kind, computerId)+ harnessAuthCache.set(hash, { sessionId, userId, epoch, user: answer })+ return { ...answer }+}++/** What a live session answers for [token]; throws INVALID_TOKEN for a session of the wrong kind or bound to nothing. */+function identityOf(+ token: string,+ user: NonNullable<Awaited<ReturnType<typeof userService.get>>>,+ sessionId: string,+ kind: string | null | undefined,+ computerId: string | null | undefined,+): AuthUser {const identity = {sub: user.id,email: user.email,@@ -357,6 +396,7 @@ export async function authenticateHarnessAccessToken(token: string): Promise<Aut/** Revoke the session with this id (a device removed from the account's device key log). */export async function revokeHarnessSessionById(sessionId: string, userId: string): Promise<void> {await prisma.harnessSession.updateMany({ where: { id: sessionId, userId, ...notRevoked }, data: endedSessionData() })+ await signInsChanged(userId)}// ── Sign in a computer by scanning its QR with a signed-in phone ──────────────────────────────────diff --git a/backend/src/lib/prisma.ts b/backend/src/lib/prisma.tsindex e58e28b05..8364be235 100644--- a/backend/src/lib/prisma.ts+++ b/backend/src/lib/prisma.ts@@ -2,16 +2,17 @@ import { PrismaClient, type Prisma } from '@prisma/client'import { env } from '../config/env.js'import { POOL_DEFAULTS, secondaryReadsUrl, withPoolDefaults } from './databaseUrl.js'import { withQueryStats } from './queryStats.js'+import { activeEngineCpu } from './engineCpu.js'import { metrics } from './metrics.js'// The Prisma CLI (db push) reads env("DATABASE_URL"); keep it in sync for any manual run.process.env.DATABASE_URL ??= env.DATABASE_URL// Every query is timed for the per-minute `db by shape` log (lib/queryStats.ts, DB roadmap Phase 1) and into-// prisma_query_duration_seconds (lib/metrics.ts).+// prisma_query_duration_seconds (lib/metrics.ts), and its engine CPU sampled for `prisma by query` (lib/engineCpu.ts).export const prisma = withQueryStats(new PrismaClient({datasources: { db: { url: withPoolDefaults(env.DATABASE_URL, POOL_DEFAULTS) } },-}), undefined, undefined, metrics.observeQuery)+}), undefined, undefined, metrics.observeQuery, undefined, activeEngineCpu)/** The client for reads that tolerate a few seconds of lag, on [url] with the secondary options (databaseUrl.ts* secondaryReadsUrl) and the pool defaults, its queries logged as `secondary:` in `db by shape`; [primary] itself when@@ -19,7 +20,7 @@ export const prisma = withQueryStats(new PrismaClient({export function readReplica(primary: PrismaClient, url: string, reads: 'on' | 'off',make: (url: string) => PrismaClient = url => new PrismaClient({ datasources: { db: { url } } })): PrismaClient {if (reads === 'off') return primary- return withQueryStats(make(withPoolDefaults(secondaryReadsUrl(url), POOL_DEFAULTS)), undefined, undefined, metrics.observeQuery, 'secondary')+ return withQueryStats(make(withPoolDefaults(secondaryReadsUrl(url), POOL_DEFAULTS)), undefined, undefined, metrics.observeQuery, 'secondary', activeEngineCpu)}/** Older history pages, costs, search, leaderboard names and usage totals (task #2970). It connects on its first query. On adiff --git a/backend/src/lib/queryStats.test.ts b/backend/src/lib/queryStats.test.tsindex 3d93b2df4..29d1710be 100644--- a/backend/src/lib/queryStats.test.ts+++ b/backend/src/lib/queryStats.test.ts@@ -1,6 +1,7 @@import Fastify from 'fastify'import { afterEach, describe, expect, it } from 'vitest'import type { PrismaClient } from '@prisma/client'+import type { EngineCpu } from './engineCpu.js'import { createQueryStats, enterWork, estimateBytes, filterShape, registerWorkContext, runWork, withQueryStats, workKind, type QueryStatsOptions } from './queryStats.js'type Logged = { message: string; context: Record<string, unknown> }@@ -281,6 +282,34 @@ describe('the query hook', () => {})})+describe('the query hook feeds the engine CPU sampler (#3041)', () => {+ type Call = { model?: string; operation: string; args: unknown; query: (args: unknown) => Promise<unknown> }+ function hookedEngine(engine: Partial<EngineCpu>) {+ let hook: ((call: Call) => Promise<unknown>) | undefined+ const client = { $extends: (extension: { query: { $allOperations: (call: Call) => Promise<unknown> } }) => { hook = extension.query.$allOperations; return client } }+ withQueryStats(client as unknown as PrismaClient, () => undefined, () => {}, undefined, undefined, () => engine as EngineCpu)+ return (call: Call) => hook!(call)+ }++ it('starts and ends one slot for every query, a failed one too, with what its start answered', async () => {+ const calls: unknown[][] = []+ const sample = { ns: 5, seq: 1 }+ const run = hookedEngine({ start: () => { calls.push(['start']); return sample }, end: (...args) => { calls.push(['end', ...args]) } })+ await run({ model: 'User', operation: 'findFirst', args: {}, query: async () => ({ id: 'u1' }) })+ const failure = new Error('write conflict')+ await expect(run({ model: 'User', operation: 'update', args: {}, query: async () => { throw failure } })).rejects.toBe(failure)+ expect(calls).toEqual([+ ['start'], ['end', sample, 'User', 'findFirst', {}, { id: 'u1' }],+ ['start'], ['end', sample, 'User', 'update', {}, undefined],+ ])+ })++ it('keeps a query\'s result when the sampler throws at its start or its end', async () => {+ const run = hookedEngine({ start: () => { throw new Error('start broke') }, end: () => { throw new Error('end broke') } })+ await expect(run({ model: 'User', operation: 'count', args: {}, query: async () => 3 })).resolves.toBe(3)+ })+})+describe('the minute handed on to the counters (#2957)', () => {it('hands every route of the minute to onWindow, not only the 40 the line lists', () => {const windows: Array<Map<string, { n: number; ms: number; bytes: number }>> = []diff --git a/backend/src/lib/queryStats.ts b/backend/src/lib/queryStats.tsindex 80b8fa047..df5ae875e 100644--- a/backend/src/lib/queryStats.ts+++ b/backend/src/lib/queryStats.ts@@ -4,6 +4,7 @@ import { performance } from 'node:perf_hooks'import type { PrismaClient } from '@prisma/client'import type { FastifyInstance, FastifyRequest } from 'fastify'import { logger } from '../utils/logger.js'+import type { EngineCpu, EngineSample } from './engineCpu.js'/*** Which queries this process sends, per kind of work (DB roadmap Phase 1). Atlas logs only operations over 100 ms; the@@ -114,7 +115,7 @@ export function filterShape(action: string, args: unknown): string {}/** `Model.action`; a raw command by its collection and command name (`SwarmGroup.$runCommandRaw aggregate`). */-function opName(model: string | undefined, action: string, args: unknown): string {+export function opName(model: string | undefined, action: string, args: unknown): string {if (model) return `${model}.${action}`if (action === '$runCommandRaw' && isPlainObject(args)) {const command = Object.keys(args)[0] ?? '?'@@ -125,7 +126,7 @@ function opName(model: string | undefined, action: string, args: unknown): strin}/** Documents a result holds: a list's length, one for a document, a raw cursor reply's first batch. */-function docsOf(result: unknown): number {+export function docsOf(result: unknown): number {if (Array.isArray(result)) return result.lengthif (isPlainObject(result)) {const cursor = result.cursor@@ -319,33 +320,44 @@ export function startQueryStats(options: QueryStatsOptions & { reportMs?: number/** The client with one query hook that times every operation, model calls and raw commands alike, for `db by shape` and* for `observe` (the Prometheus histogram, lib/metrics.ts; `raw` for a call without a model). Query extensions keep the* client's types, so it is still a PrismaClient to its callers. [label] names this client's queries in `db by shape`- * (`secondary:User.findMany`, lib/prisma.ts prismaSecondary). A failure of either never reaches the query: it is warned- * about once per client, and the query's own result or error goes back unchanged. */+ * (`secondary:User.findMany`, lib/prisma.ts prismaSecondary). [engine] is the engine CPU sampler (lib/engineCpu.ts),+ * told of every query's start and end. A failure of any of them never reaches the query: it is warned about once per+ * client, and the query's own result or error goes back unchanged. */export function withQueryStats(client: PrismaClient, stats: () => QueryStats | undefined = () => active,warn: (message: string, context: Record<string, unknown>) => void = (m, c) => logger.warn(m, c),- observe?: (model: string, operation: string, seconds: number) => void, label?: string): PrismaClient {+ observe?: (model: string, operation: string, seconds: number) => void, label?: string,+ engine: () => EngineCpu | undefined = () => undefined): PrismaClient {let warned = false+ const failedOnce = (error: unknown) => {+ if (warned) return+ warned = true+ warn('query stats failed', { error: error instanceof Error ? error.message : String(error) })+ }const note = (model: string | undefined, operation: string, args: unknown, ms: number, result: unknown, failed: boolean) => {try {observe?.(model ?? 'raw', operation, ms / 1000)stats()?.record(model, operation, args, ms, result, failed, label)- } catch (error) {- if (warned) return- warned = true- warn('query stats failed', { error: error instanceof Error ? error.message : String(error) })- }+ } catch (error) { failedOnce(error) }}return client.$extends({query: {async $allOperations({ model, operation, args, query }) {+ // The engine CPU sampler (lib/engineCpu.ts) sees every query's start and end, so it knows which one ran alone.+ let sampler: EngineCpu | undefined, sample: EngineSample | undefined+ try { sampler = engine(); sample = sampler?.start() } catch (error) { failedOnce(error) }+ const ended = (result: unknown) => {+ try { sampler?.end(sample, model, operation, args, result) } catch (error) { failedOnce(error) }+ }const started = performance.now()let result: unknowntry {result = await query(args)} catch (error) {+ ended(undefined)note(model, operation, args, performance.now() - started, undefined, true)throw error}+ ended(result)note(model, operation, args, performance.now() - started, result, false)return result},diff --git a/backend/src/lib/ssoAuth.userReads.test.ts b/backend/src/lib/ssoAuth.userReads.test.tsindex a53e70805..6f4a08b5d 100644--- a/backend/src/lib/ssoAuth.userReads.test.ts+++ b/backend/src/lib/ssoAuth.userReads.test.ts@@ -11,6 +11,11 @@ const create = vi.hoisted(() => vi.fn())vi.mock('./prisma.js', () => ({prisma: { user: { findUnique, update, create } },}))+// An update moves the person's sign-in epoch in Redis (lib/harnessAuthCache.ts), which these tests do not run.+vi.mock('./harnessAuthCache.js', async (importOriginal) => ({+ ...(await importOriginal<typeof import('./harnessAuthCache.js')>()),+ signInsChanged: vi.fn(async () => {}),+}))import { authenticateAccessToken, clearSsoProfileCache } from './ssoAuth.js'diff --git a/backend/src/lib/ssoSession.test.ts b/backend/src/lib/ssoSession.test.tsindex 5851ac9d8..75ddb4b20 100644--- a/backend/src/lib/ssoSession.test.ts+++ b/backend/src/lib/ssoSession.test.ts@@ -24,6 +24,7 @@ const fakes = vi.hoisted(() => {redis.set(key, value); return 'OK'}),get: vi.fn(async (key: string) => redis.get(key) ?? null),+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),del: vi.fn(async (key: string) => (redis.delete(key) ? 1 : 0)),// The lock's compare-and-delete (lib/ssoSession.ts RELEASE_SCRIPT).eval: vi.fn(async (_script: string, _n: number, key: string, owner: string) => (redis.get(key) === owner && redis.delete(key) ? 1 : 0)),@@ -141,6 +142,7 @@ vi.mock('./sso.js', async (importOriginal) => {import { clearSsoProfileCache, authenticateAccessToken, resetSsoBreakers, SsoAuthError } from './ssoAuth.js'import { refreshHarnessSession, revokeHarnessSession } from './harnessSession.js'+import { harnessAuthCache } from './harnessAuthCache.js'import { decryptMachineCredential } from './machineCredential.js'import { SSO_OUTAGE_GRACE_MS, SSO_RECHECK_MS, ensureHarnessSessionExpiry, ssoAccessTokenFor } from './ssoSession.js'import { authRoutes } from '../routes/auth.js'@@ -169,6 +171,7 @@ beforeEach(async () => {fakes.sso.live.clear(); fakes.sso.rotate = true; fakes.sso.down = false; fakes.sso.calls = 0; fakes.sso.counter = 0; fakes.sso.gate = nullfakes.setFailCreate(false)clearSsoProfileCache()+ harnessAuthCache.clear()// The profile service's circuit (lib/ssoAuth.ts) is shared: tests that fail it on purpose must not open it for the next.resetSsoBreakers()vi.clearAllMocks()@@ -358,6 +361,20 @@ describe('the hourly SSO re-check', () => {expect(Date.now() - start).toBeLessThan(2 * HOUR)})+ it('an SSO refusal ends a sign-in this pod remembers at once (#3041)', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ const start = Date.now()+ const { refreshToken } = await harnessSignIn()+ vi.setSystemTime(start + SSO_RECHECK_MS - 1)+ const last = String((await refresh(refreshToken)).body.data!.token)+ // Remembered by this pod a moment before the re-check.+ expect(await me(last)).toBe(200)+ fakes.sso.live.clear()+ vi.setSystemTime(start + SSO_RECHECK_MS + 1)+ expect((await refresh(refreshToken)).status).toBe(401)+ expect(await me(last)).toBe(401)+ })+it('SSO being down keeps people signed in, for seven days from the last good check', async () => {vi.useFakeTimers({ toFake: ['Date'] })const start = Date.now()diff --git a/backend/src/lib/ssoSession.ts b/backend/src/lib/ssoSession.tsindex 171f5edbe..147cf9058 100644--- a/backend/src/lib/ssoSession.ts+++ b/backend/src/lib/ssoSession.ts@@ -1,6 +1,7 @@import { randomBytes } from 'node:crypto'import type { HarnessSession, Prisma, PrismaClient } from '@prisma/client'import { pub } from './bus.js'+import { signInsChanged } from './harnessAuthCache.js'import { prisma } from './prisma.js'import { ensureIndex } from './db/indexes.js'import { decryptMachineCredential, encryptMachineCredential } from './machineCredential.js'@@ -149,6 +150,7 @@ async function renew(sessionId: string, fresh: (row: HarnessSession, now: number} catch (err) {if (err instanceof SsoTokenError && err.code === 'INVALID_GRANT' && !unfinished) {await writeLive(sessionId, endedSessionData())+ await signInsChanged(row.userId)logger.info('sso sign-in ended a harness session', { sessionId, userId: row.userId })return { outcome: 'ended' }}diff --git a/backend/src/routes/control.test.ts b/backend/src/routes/control.test.tsindex d9450acd3..d93b21084 100644--- a/backend/src/routes/control.test.ts+++ b/backend/src/routes/control.test.ts@@ -13,6 +13,7 @@ const fakes = vi.hoisted(() => {pub: {set: vi.fn(async (key: string, value: string) => { redis.set(key, value); return 'OK' }),get: vi.fn(async (key: string) => redis.get(key) ?? null),+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),multi: vi.fn(() => {const ops: Array<() => unknown> = []const chain = {diff --git a/backend/src/routes/qrSignIn.test.ts b/backend/src/routes/qrSignIn.test.tsindex 844a5c573..333434e3a 100644--- a/backend/src/routes/qrSignIn.test.ts+++ b/backend/src/routes/qrSignIn.test.ts@@ -31,6 +31,7 @@ const fakes = vi.hoisted(() => {}),set: vi.fn(async (key: string, value: string, ...opts: unknown[]) => set(key, value, ...opts)),get: vi.fn(async (key: string) => redis.get(key) ?? null),+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),del: vi.fn(async (key: string) => (redis.delete(key) ? 1 : 0)),multi: vi.fn(() => {const ops: Array<() => unknown> = []diff --git a/backend/src/routes/superTabs.test.ts b/backend/src/routes/superTabs.test.tsindex 7256a9809..3e09a637b 100644--- a/backend/src/routes/superTabs.test.ts+++ b/backend/src/routes/superTabs.test.ts@@ -12,6 +12,7 @@ const fakes = vi.hoisted(() => {pub: {set: vi.fn(async (key: string, value: string) => { redis.set(key, value); return 'OK' }),get: vi.fn(async (key: string) => redis.get(key) ?? null),+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),multi: vi.fn(() => {const ops: Array<() => unknown> = []const chain = {diff --git a/backend/src/server.ts b/backend/src/server.tsindex 68e50bd65..e5516ba76 100644--- a/backend/src/server.ts+++ b/backend/src/server.ts@@ -42,6 +42,7 @@ import { selfProfileFromEnv } from './lib/selfProfile.js'import { registerReplyJsonTiming, startCpuStats } from './lib/cpuStats.js'import { startRelayUsageFlush } from './lib/productUsage.js'import { registerWorkContext, startQueryStats } from './lib/queryStats.js'+import { startEngineCpu } from './lib/engineCpu.js'import { startRetryStats } from './superTabs/retryStats.js'import { startRelayStats } from './lib/relayStats.js'import { startPodRoutes } from './lib/hub.js'@@ -160,6 +161,9 @@ async function start(): Promise<void> {registerReplyJsonTiming(app, cpuStats)// Always on, every minute (DB roadmap Phase 1): queries by model.action, filter shape and route, and each route's p50/p95.const stopQueryStats = startQueryStats({ onWindow: routes => metrics.dbWindow(routes) })+ // Always on, every minute: the Prisma engine's CPU by query, from queries sampled while they run alone+ // (lib/engineCpu.ts, task #3041), to read beside `cpu by thread`'s prismaEngineMs.+ const stopEngineCpu = startEngineCpu()// Always on, every minute: `last hour`, the sums of the last 60 minutes the production scan reads from a log tail// (lib/lastHour.ts, task #2957).const stopLastHour = startLastHour(metrics.hour)@@ -266,6 +270,7 @@ async function start(): Promise<void> {await Promise.race([stopRelayUsage().catch(() => {}), new Promise((resolve) => setTimeout(resolve, 3_000))])// The last partial minute of these logs, once the drain and the usage flush have run their queries.stopQueryStats()+ stopEngineCpu()stopLastHour()stopRetryStats()stopDeliveryCalls()diff --git a/backend/src/services/MachineService.computer.test.ts b/backend/src/services/MachineService.computer.test.tsindex 862e0f94f..1c9c22ccc 100644--- a/backend/src/services/MachineService.computer.test.ts+++ b/backend/src/services/MachineService.computer.test.ts@@ -180,6 +180,17 @@ describe('destroying a team machine', () => {})})+ it("ends its team account's remembered sign-ins on every pod, after the session is revoked (#3041)", async () => {+ findUnique.mockResolvedValue({ machineId: 'm1', userId: 'u1', managerId: '', authMode: 'remote', billingStatus: 'not_required',+ autonomousEnv: 'prod', computerId: 'b'.repeat(32), sessionKind: 'team' })+ update.mockResolvedValue({})+ sessionUpdateMany.mockResolvedValue({ count: 1 })+ await machineService.destroy('a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4', { sub: 'u1', role: 'user', autonomousEnv: 'prod' })+ const epoch = redisSet.mock.calls.findIndex(([key]) => key === 'hnauth:epoch:u1')+ expect(epoch).toBeGreaterThanOrEqual(0)+ expect(redisSet.mock.invocationCallOrder[epoch]).toBeGreaterThan(sessionUpdateMany.mock.invocationCallOrder[0]!)+ })+it('touches no session for a machine with no computer', async () => {findUnique.mockResolvedValue({ machineId: 'm1', userId: 'u1', managerId: '', authMode: 'remote', billingStatus: 'not_required', autonomousEnv: 'prod' })update.mockResolvedValue({})diff --git a/backend/src/services/MachineService.ts b/backend/src/services/MachineService.tsindex 59195a81b..8277bef22 100644--- a/backend/src/services/MachineService.ts+++ b/backend/src/services/MachineService.ts@@ -16,6 +16,7 @@ import { machineBillingService } from './MachineBillingService.js'import { storedAutonomousEnvironment, type AutonomousEnvironment } from '../lib/autonomousEnvironment.js'import { TEAM_SESSION_LIMITED, TEAM_SESSION_MESSAGE } from '../lib/teamSession.js'import type { AuthUser } from '../lib/ssoAuth.js'+import { signInsChanged } from '../lib/harnessAuthCache.js'import { machineRenamed } from '../superTabs/machineRenames.js'/** The agent's tier / how it runs Claude, resolved through its plan.@@ -709,7 +710,8 @@ export const machineService = {await prisma.harnessSession.updateMany({where: { userId: binding.userId, kind: 'team', computerId: binding.computerId, OR: [{ revokedAt: null }, { revokedAt: { isSet: false } }] },data: { revokedAt: new Date() },- }).catch((err: unknown) => logger.warn('team session revoke failed', { machineId, error: String(err) }))+ }).then(() => signInsChanged(binding.userId))+ .catch((err: unknown) => logger.warn('team session revoke failed', { machineId, error: String(err) }))}} else {// Managed: reap the container + node row in the background (idempotent on the manager → retryable).diff --git a/backend/src/services/UserService.test.ts b/backend/src/services/UserService.test.tsindex cadea6189..e48324eab 100644--- a/backend/src/services/UserService.test.ts+++ b/backend/src/services/UserService.test.ts@@ -6,6 +6,7 @@ const update = vi.hoisted(() => vi.fn())const remove = vi.hoisted(() => vi.fn())const updateManyDevices = vi.hoisted(() => vi.fn())const transaction = vi.hoisted(() => vi.fn())+const signInsChanged = vi.hoisted(() => vi.fn(async () => {}))vi.mock('../lib/prisma.js', () => ({prisma: {@@ -16,6 +17,7 @@ vi.mock('../lib/prisma.js', () => ({}))vi.mock('../config/env.js', () => ({ ADMIN_EMAILS: new Set<string>(['[email protected]']) }))vi.mock('../lib/password.js', () => ({ hashPassword: (p: string) => `hashed:${p}` }))+vi.mock('../lib/harnessAuthCache.js', () => ({ signInsChanged }))import {isProvisionalUserEmail,@@ -321,3 +323,44 @@ describe('create — locally-created accounts', () => {expect(findUnique).toHaveBeenCalledWith({ where: { email: '[email protected]' } })})})++describe('a change to what a sign-in answers ends every pod\'s remembered sign-in (#3041)', () => {+ const after = (write: { mock: { invocationCallOrder: number[] } }): void => {+ expect(signInsChanged.mock.invocationCallOrder.at(-1)).toBeGreaterThan(write.mock.invocationCallOrder.at(-1)!)+ }++ it('an SSO sign-in that rewrites the email or the role, after the write', async () => {+ findUnique.mockResolvedValue(row({ email: '[email protected]' }))+ await userService.upsertFromSso({ externalId: 'prod-sub-1', email: '[email protected]', autonomousEnv: 'prod' })+ expect(signInsChanged).toHaveBeenCalledWith('u1')+ after(update)+ })++ it('an account already current writes nothing and moves nothing: SSO auth runs on every request', async () => {+ findUnique.mockResolvedValue(row())+ await userService.upsertFromSso({ externalId: 'prod-sub-1', email: '[email protected]', autonomousEnv: 'prod' })+ expect(signInsChanged).not.toHaveBeenCalled()+ })++ it('a provisional row merged away, after it is deleted', async () => {+ findUnique.mockResolvedValueOnce(row({ id: 'real', externalId: 'local-prod-real' }))+ .mockResolvedValueOnce(row({ id: 'pending', email: provisionalEmailForExternalId('prod-sub-1') }))+ await userService.upsertFromSso({ externalId: 'prod-sub-1', email: '[email protected]', autonomousEnv: 'prod' })+ expect(signInsChanged).toHaveBeenCalledWith('pending')+ expect(signInsChanged.mock.invocationCallOrder[0]).toBeGreaterThan(remove.mock.invocationCallOrder[0]!)+ })++ it('an update of the role or the plane, after the write', async () => {+ findUnique.mockResolvedValue(row())+ await userService.update('u1', { role: 'admin', autonomousEnv: 'stag' })+ expect(signInsChanged).toHaveBeenCalledWith('u1')+ after(update)+ })++ it('removing an account, after the delete', async () => {+ findUnique.mockResolvedValue(row())+ await userService.remove('u1')+ expect(signInsChanged).toHaveBeenCalledWith('u1')+ after(remove)+ })+})diff --git a/backend/src/services/UserService.ts b/backend/src/services/UserService.tsindex d81376f05..d1fb225a4 100644--- a/backend/src/services/UserService.ts+++ b/backend/src/services/UserService.ts@@ -3,6 +3,7 @@ import { randomUUID } from 'crypto'import { prisma } from '../lib/prisma.js'import { ADMIN_EMAILS } from '../config/env.js'import { hashPassword } from '../lib/password.js'+import { signInsChanged } from '../lib/harnessAuthCache.js'import { ConflictError, NotFoundError } from '../errors/index.js'import type { AutonomousEnvironment } from '../lib/autonomousEnvironment.js'import type { SignInAttribution } from '../lib/signInAttribution.js'@@ -56,6 +57,7 @@ async function mergeProdProvisionalUser(provisional: User, user: User): Promise<}),prisma.user.delete({ where: { id: provisional.id } }),])+ await signInsChanged(provisional.id)}export const userService = {@@ -125,10 +127,13 @@ export const userService = {// SSO profile auth runs on every control-plane request. Avoid turning that validation into a// write-per-request DB hot path when the mirrored user record is already current.if (Object.keys(data).length === 0) return existing- return prisma.user.update({+ const updated = await prisma.user.update({where: { id: existing.id },data,})+ // Its email or role may have changed: a Harness sign-in each pod remembers answers them (lib/harnessAuthCache.ts).+ await signInsChanged(existing.id)+ return updated}const data = {@@ -259,7 +264,7 @@ export const userService = {async update(id: string, input: { name?: string; password?: string; role?: string; autonomousEnv?: AutonomousEnvironment }): Promise<User> {if (!(await prisma.user.findUnique({ where: { id } }))) throw new NotFoundError('User')- return prisma.user.update({+ const updated = await prisma.user.update({where: { id },data: {...(input.name !== undefined ? { name: input.name } : {}),@@ -268,10 +273,13 @@ export const userService = {...(input.password ? { passwordHash: hashPassword(input.password) } : {}),},})+ await signInsChanged(id)+ return updated},async remove(id: string): Promise<void> {if (!(await prisma.user.findUnique({ where: { id } }))) throw new NotFoundError('User')await prisma.user.delete({ where: { id } })+ await signInsChanged(id)},}diff --git a/docs/plans/2026-11-05-harness-sign-in-cache-plan.md b/docs/plans/2026-11-05-harness-sign-in-cache-plan.mdnew file mode 100644index 000000000..88852d411--- /dev/null+++ b/docs/plans/2026-11-05-harness-sign-in-cache-plan.md@@ -0,0 +1,92 @@+# Harness sign-in cache per pod — plan (task #3041)++**Goal:** Stop reading the session row and the user row from Mongo on every request signed in with a Harness session.+Atlas's per-collection counters on 11-05 show `harness_sessions` at 47.2 reads/s and `users` at 52.8/s on the primary+(River, from Ken's table), about 16 ops per backend request. Signing out, revoking and removing a computer must still+take effect at once, on every pod.++**Architecture:** `authenticateHarnessAccessToken` (lib/harnessSession.ts) remembers a good sign-in for 30 seconds in a+bounded per-pod map keyed by the access token's hash. Every request still reads its access token from Redis, as today;+the cached path reads the person's **sign-in epoch** in the same Redis call (`MGET`). Every write that ends a session or+changes what a sign-in answers sets that person's epoch to a new random value after the database write. A cached sign-in+whose epoch moved is dropped and read from Mongo again.++**Tech Stack:** Node 22, Prisma 5.22 / MongoDB, ioredis, Vitest.++**Spec:** River's addition to task #3041 (11-05): a short in-memory cache per pod for the sign-in lookups, signing out and+revoking clearing it at once, two security reviews, Atlas ops before and after.++This plan is written and carried out by its Engineer, test first; there is no execution handoff.++## The rules it keeps++- **Revocation at once, on every pod.** The epoch is in Redis, not in the pod: a sign-out on pod A is seen by pod B's+ next request, in the Redis read that request already makes.+- **No lost sign-out in a race.** The miss path reads the epoch *before* it reads the session and the user, and writers+ set it *after* their write. A sign-out that lands between the two reads leaves the remembered epoch stale, so the next+ request reads Mongo again.+- **An epoch never repeats.** It is a random value, not a counter: a key that expired and was written again cannot equal+ a value a pod remembers. It lives an hour, far past the 30-second cache; an expired epoch only causes a miss.+- **A failed epoch write fails the sign-out.** `signInsChanged` throws, so a revoke route answers an error rather than+ "signed out" while other pods could still serve the session for up to 30 seconds. Retrying is safe: every revoke is+ idempotent.+- **Only success is cached**, and callers get a copy.+- **Redis down is still an outage** (`AUTH_SERVICE_UNAVAILABLE`), cached or not.++Writes that set the epoch: sign-out (`revokeHarnessSession`), revoking a device (`revokeHarnessSessionById`), a computer+turning itself into a team machine (`convertComputer`), SSO refusing a renewal (`ssoSession.ts renew`), removing a+computer that ends its team session (`MachineService`), and user rows changing what a sign-in answers: `UserService`+update (role, plane), remove, a provisional row merged away, and an SSO sign-in that rewrites the email, role or subject.++Also not changed: the one-time email migration at start (lib/db/migrate.ts) only lower-cases an email; a pod may+answer the old casing for up to 30 seconds.++Not changed: SSO-token requests (no Harness session) keep their path; they are about 5 of the 53 user reads a second.++## Design++- Growth: at most 20,000 entries per pod, oldest dropped first, each gone after 30 s; epoch keys expire after an hour.+- Concurrency: none in the pod (one thread); across pods the epoch orders a sign-out against a cached read (above).+- Read cost: a hit is one Redis `MGET` (today: one `GET`) and no Mongo read; a miss adds one Redis `GET`.+- Scale: at 10x sign-ins the map's bound holds memory under ~10 MB; the first warning is the hit rate falling, seen as+ `harness_sessions` reads per second on Atlas rising back.+- Compatibility: no client, API or schema change; tokens are unchanged.+- Migration + rollback: none; reverting removes the cache, and epoch keys expire by themselves.+- Audience: the server checks every request as before; the cache only remembers its own answer.+- Identity: the cached identity is the session's user, never the machine owner's login.+- Failure: Redis down answers 503 as today; an epoch that cannot be written fails the sign-out loudly.+- Platforms: backend only.++## Global Constraints++No behaviour change for a caller except fewer database reads. No new settings or switches. Two independent security+reviews before merge (sign-in and tokens).++## Review Focus++- Every write that ends a Harness session or changes a user's sign-in answer sets the epoch, after the write.+- The miss path's order: access record, epoch, then session and user.++## Tasks++### 1. The cache and the epoch (`backend/src/lib/harnessAuthCache.ts`)++- [x] Failing tests: 30 s life, bound, fresh epochs that never repeat with an hour's life, a failed write throws.+- [x] Code that passes them.++### 2. Authentication uses it (`backend/src/lib/harnessSession.ts`)++- [x] Failing tests: a second request reads no Mongo; a token gone from Redis stops; sign-out and revoke by id end a+ remembered sign-in at once, with nothing cleared in the pod; a sign-out between the reads is not missed; read again+ after 30 s; Redis down is an outage; no refusal remembered; callers get a copy.+- [x] Code: the cached path, the miss path's order, epochs set by sign-out, revoke and conversion.++### 3. Every other writer sets the epoch++- [x] Failing tests: SSO refusing a renewal, removing a computer, and `UserService` update, remove, merge and an+ SSO rewrite each set the person's epoch.+- [x] Code.++### 4. Before and after++- [ ] Atlas ops/s and `harness_sessions` / `users` reads per second before and after the release, in Ken's thread.diff --git a/docs/plans/2026-11-05-prisma-engine-cpu-plan.md b/docs/plans/2026-11-05-prisma-engine-cpu-plan.mdnew file mode 100644index 000000000..5fc5e12d7--- /dev/null+++ b/docs/plans/2026-11-05-prisma-engine-cpu-plan.md@@ -0,0 +1,100 @@+# Prisma engine CPU by query — plan (task #3041)++**Goal:** Name which queries burn the Prisma engine's CPU (45% of process CPU on v1.4.7, Ken's 11-05 `cpu by thread`+split), then move the top reads off the engine's heaviest path, so CPU per request reaches ≤ 38 ms and 3 pods hold peak.++**Architecture:** Step 1 measures. A sampler in the one query hook (`lib/queryStats.ts withQueryStats`) times the engine+threads' own CPU around queries that run alone, and a per-minute `prisma by query` line ranks operations by engine ms.+Step 2 moves the top reads, one group per pull request, chosen from that line. Step 3 reports before and after.++**Tech Stack:** Node 22, Prisma 5.22 (library engine, in-process tokio threads), MongoDB, Vitest.++**Spec:** the task's brief (#3041); Watch Production (#2773) thread of 11-05.++This plan is written and carried out by its Engineer, test first; there is no execution handoff.++## Why not Prisma's query events++The brief named Prisma's `query` events. They do not measure what we need, and turning them on costs what we measure:++- For MongoDB, an event's `duration` is the database call as the engine sees it (send, wait, reply). It does not include+ the engine's own work: building the query, converting BSON to its values and those to the JSON string the client+ parses. That is the CPU `cpu by thread` counts.+- The `query` log level makes the engine format every query as a string. It cannot be sampled: it is on for every query+ or for none.++What we measure instead: Linux keeps each thread's time on CPU in nanoseconds (`/proc/self/task/<tid>/schedstat`, first+field). When a query runs alone in the process, the engine threads' CPU between its start and its end is that query's+engine CPU. The sampler takes such a query at most every 100 ms, reads the engine threads before and after (two+directory reads and a few small file reads, tens of microseconds), and scales each operation's mean by how often it ran+in the minute. Where `schedstat` is missing it falls back to `stat`'s 10 ms ticks, unbiased over many samples; off Linux+the line says `unavailable`.++## Raw commands are still the engine++`$runCommandRaw` and `aggregateRaw` (resolveIds, the unfollow update, `parts/indexed.ts readBy`) run inside the same+engine: it still converts the command to BSON and every reply document to Extended JSON and then to a string. They skip+Prisma's query building and per-field result shaping, so they may cost less per document, but not nothing. The new line+measures raw commands too, so step 2 compares engine µs per document of a model read against a raw read on the same+collection before moving anything. If raw reads are about as costly, the fix is the plain `mongodb` driver for the top+reads; that is a second connection pool per pod against Atlas M20's connection limit and more memory, and it goes to the+PM with those costs before any code (River, 11-05).++## Design++- Growth: a log line per pod-minute, the top 25 keys; nothing stored. Logs rotate.+- Concurrency: none; one process, counters on the main thread.+- Read cost: at most 10 samples a second, each two `/proc` listings and the engine threads' `schedstat`; the line reports+ the sampler's own time (`costMs`) so its cost is measured, not assumed.+- Scale: at 10x queries the sample rate stays capped; keys stay capped (`maxKeys`, overflow into `other`).+- Compatibility: a new log line; no client, API or schema change.+- Migration + rollback: none; revert the commit.+- Audience: server logs only; ops and routes are patterns, never values or ids.+- Identity: N/A because nothing is read or written for a person.+- Failure: a sampler error never reaches a query; it is warned about once per process, and `/proc` missing says+ `unavailable` on every line.+- Platforms: Linux pods; macOS dev logs `unavailable`.++## Global Constraints++No behaviour change; no new settings or switches. Swarm code stays out of `cli/src/core`. Each moved query keeps its+results and its index, with an explain test (backend AGENTS.md rule 3). One pull request per group of queries.++## Review Focus++- The sampler's solo rule: a sample counts only if no other query was in flight at its start and none started before it+ ended; the in-flight count returns to zero after a failed query.+- Its cost bound (gap between samples) and that a failure inside it never changes a query's result.++## Tasks++### 1. Engine CPU reader and sampler (`backend/src/lib/engineCpu.ts`)++Files: `src/lib/engineCpu.ts`, `src/lib/engineCpu.test.ts`.+Interfaces: `readEngineCpuNs(root, pid)`, `createEngineCpu(options)` → `{ start(), end(sample, model, action, args,+result), flush() }`, `startEngineCpu(options)`, `activeEngineCpu()`.++- [ ] Failing tests: reads engine threads only, from `schedstat`, falling back to `stat`; a solo query is sampled and+ scaled by its count; an overlapped one is discarded; samples respect the gap; the line ranks by engine ms with op+ and route, reports totals and cost, and resets.+- [ ] Code that passes them.++### 2. The query hook feeds it (`backend/src/lib/queryStats.ts`)++Files: `src/lib/queryStats.ts`, `src/lib/queryStats.test.ts`, `src/lib/prisma.ts`, `src/server.ts`.++- [ ] Failing tests: every query, a failed one too, starts and ends one sample slot (in flight back to zero); a sampler+ that throws leaves the query's result alone.+- [ ] Code: `withQueryStats` calls the sampler; both clients pass it (lib/prisma.ts); the API server starts it beside+ `startQueryStats`, as it starts `cpu by thread` (the worker logs neither).++### 3. Read production, pick the first group (after the release)++- [ ] Read `prisma by query` beside `cpu by thread` at peak; post the top operations by engine ms, their engine µs per+ document, and raw against model reads.+- [ ] Open the next pull request for the top group behind GET changes and hosts readings/changes, with an explain test+ for each moved query, or bring the plain driver's costs to the PM.++### 4. Before and after++- [ ] After each move ships: Prisma engine ms/min and CPU per request from `cpu by thread` and `prisma by query`.