Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Merge batch 11-05 i1300: #3087 #3070 (#3096)

0xken15b14dc059f093
diff --git a/backend/integration/prismaQueryTiming.test.ts b/backend/integration/prismaQueryTiming.test.ts
index e62928ef6..e5524c28e 100644
--- a/backend/integration/prismaQueryTiming.test.ts
+++ b/backend/integration/prismaQueryTiming.test.ts
@@ -8,6 +8,7 @@ import { MongoMemoryReplSet } from 'mongodb-memory-server-core'
import { PrismaClient } from '@prisma/client'
import { createMetrics } from '../src/lib/metrics.js'
import { withQueryStats } from '../src/lib/queryStats.js'
+import { createEngineCpu } from '../src/lib/engineCpu.js'
// A fresh loopback replica set, created here. This suite never reads DATABASE_URL or an existing database.
let replica: MongoMemoryReplSet
@@ -62,4 +63,20 @@ describe('Prisma query timing against a real database', () => {
expect(await queryCounts(metrics)).toEqual(['prisma_query_duration_seconds_count{model="User",operation="count"} 1'])
await prisma.$disconnect()
})
+
+ // Linux only, as in production: the sampler reads the engine's own threads from /proc (task #3041). If Prisma renamed
+ // its tokio threads, nothing would be attributed and `prisma by query` would read zero.
+ it.skipIf(process.platform !== 'linux')('measures the real engine\'s CPU for queries that run alone', async () => {
+ const infos: Array<Record<string, unknown>> = []
+ const engine = createEngineCpu({ gapMs: 0, log: (_message, context) => { infos.push(context) } })
+ const db = withQueryStats(base, () => undefined, undefined, undefined, undefined, () => engine)
+ for (let i = 0; i < 50; i++) await db.user.findMany({ take: 20 })
+ engine.flush()
+ const line = infos[0]!
+ expect(line.threads).toBeUndefined()
+ expect(line).toMatchObject({ queries: 50, discarded: 0 })
+ expect(line.sampled).toBeGreaterThanOrEqual(45)
+ expect(line.engineMs as number).toBeGreaterThan(0)
+ expect((line.top as Array<{ op: string }>)[0]).toMatchObject({ op: 'User.findMany', n: 50 })
+ })
})
diff --git a/backend/src/lib/engineCpu.test.ts b/backend/src/lib/engineCpu.test.ts
new file mode 100644
index 000000000..4a2a4069f
--- /dev/null
+++ b/backend/src/lib/engineCpu.test.ts
@@ -0,0 +1,169 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { createEngineCpu, readEngineCpu, type EngineCpuOptions } from './engineCpu.js'
+import { runWork } from './queryStats.js'
+
+const roots: string[] = []
+afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) })
+
+/** A fake /proc/self/task: each thread's comm, and its schedstat (ns on CPU) or only its stat line (10 ms ticks). */
+function procTree(threads: Array<{ tid: number; name: string; ns?: number; ticks?: number }>): string {
+ const root = mkdtempSync(join(tmpdir(), 'engine-cpu-'))
+ roots.push(root)
+ for (const t of threads) {
+ const dir = join(root, String(t.tid))
+ mkdirSync(dir)
+ writeFileSync(join(dir, 'comm'), `${t.name}\n`)
+ if (t.ns !== undefined) writeFileSync(join(dir, 'schedstat'), `${t.ns} 5000 12\n`)
+ const ticks = t.ticks ?? 0
+ writeFileSync(join(dir, 'stat'), `${t.tid} (${t.name}) S 1 1 1 0 -1 0 0 0 0 0 ${ticks} 0 0 0 20 0 1 0 100 0 0`)
+ }
+ return root
+}
+
+describe('the engine threads\' CPU', () => {
+ it('sums the Prisma engine threads only, in ns from schedstat', () => {
+ const root = procTree([
+ { tid: 10, name: 'node', ns: 9_000_000_000 },
+ { tid: 11, name: 'tokio-runtime-w', ns: 1_500_000 },
+ { tid: 12, name: 'tokio-runtime-w', ns: 2_500_000 },
+ { tid: 13, name: 'opentelemetry-e', ns: 1_000 },
+ { tid: 14, name: 'libuv-worker', ns: 7_000_000 },
+ { tid: 15, name: 'node', ns: 8_000_000 },
+ ])
+ expect(readEngineCpu(root, 10)).toEqual({ ns: 4_001_000, clock: 'schedstat' })
+ })
+
+ it('falls back to stat\'s ticks where the kernel has no schedstat', () => {
+ const root = procTree([{ tid: 10, name: 'node', ticks: 99 }, { tid: 11, name: 'tokio-runtime-w', ticks: 3 }])
+ expect(readEngineCpu(root, 10)).toEqual({ ns: 30_000_000, clock: 'stat' })
+ })
+
+ it('fails loudly when the folder cannot be read', () => {
+ expect(() => readEngineCpu('/nonexistent-engine-cpu-root', 1)).toThrow()
+ })
+})
+
+/** A sampler on a fake clock and a fake engine CPU counter the test moves. */
+function sampler(options: Partial<EngineCpuOptions> = {}) {
+ let ms = 1_000, ns = 0, reads = 0
+ const infos: Array<{ message: string; context: Record<string, unknown> }> = []
+ const engine = createEngineCpu({
+ pod: 'pod-1', now: () => ms, read: () => { reads++; return { ns, clock: 'schedstat' } },
+ log: (message, context) => { infos.push({ message, context }) }, warn: () => {}, ...options,
+ })
+ return {
+ engine, infos,
+ advance(byMs: number) { ms += byMs },
+ burn(engineMs: number) { ns += engineMs * 1e6 },
+ reads: () => reads,
+ }
+}
+
+type Row = { op: string; route: string; n: number; sampled: number; engineMs: number; usPerQuery: number; docs: number }
+const top = (context: Record<string, unknown>) => context.top as Row[]
+
+describe('prisma by query', () => {
+ it('times a query that runs alone and scales its mean by how often it ran', () => {
+ const t = sampler()
+ runWork('http GET /api/super-tabs/:tabId/changes', () => {
+ const sample = t.engine.start()
+ t.burn(4)
+ t.engine.end(sample, 'SwarmGroup', 'findMany', {}, [{ a: 1 }, { a: 2 }])
+ // Inside the gap: counted, not sampled.
+ for (let i = 0; i < 9; i++) t.engine.end(t.engine.start(), 'SwarmGroup', 'findMany', {}, [{ a: 1 }, { a: 2 }])
+ })
+ t.engine.flush()
+ expect(t.infos).toHaveLength(1)
+ expect(t.infos[0]!.message).toBe('prisma by query')
+ expect(top(t.infos[0]!.context)[0]).toEqual({ op: 'SwarmGroup.findMany', route: 'http GET /api/super-tabs/:tabId/changes', n: 10, sampled: 1,
+ engineMs: 40, usPerQuery: 4000, docs: 20 })
+ expect(t.infos[0]!.context).toMatchObject({ pod: 'pod-1', clock: 'schedstat', queries: 10, sampled: 1, discarded: 0, engineMs: 40 })
+ })
+
+ it('drops a sample another query overlapped, at its start or before its end', () => {
+ const t = sampler()
+ // Another query starts before the sampled one ends.
+ const first = t.engine.start()
+ const second = t.engine.start()
+ t.burn(10)
+ t.engine.end(first, 'Machine', 'findUnique', {}, { id: 'm' })
+ t.engine.end(second, 'User', 'findFirst', {}, { id: 'u' })
+ t.advance(1_000)
+ // One already in flight: the new one is not sampled at all.
+ const running = t.engine.start()
+ t.advance(1_000)
+ const late = t.engine.start()
+ t.burn(10)
+ t.engine.end(late, 'Machine', 'findUnique', {}, { id: 'm' })
+ t.engine.end(running, 'User', 'findFirst', {}, { id: 'u' })
+ t.engine.flush()
+ const context = t.infos[0]!.context
+ // Both samples opened were overlapped: the first by a second start, the one in flight by the late one.
+ expect(context).toMatchObject({ queries: 4, sampled: 0, discarded: 2, engineMs: 0 })
+ })
+
+ it('reads the engine CPU at most once per gap, so its cost stays bounded at any query rate', () => {
+ const t = sampler({ gapMs: 100 })
+ for (let i = 0; i < 1_000; i++) {
+ t.advance(1)
+ t.engine.end(t.engine.start(), 'SwarmHead', 'findUnique', {}, { id: 'h' })
+ }
+ t.engine.flush()
+ // 1,000 solo queries over a second: about 10 samples, each a read before and after.
+ expect(t.reads()).toBeLessThanOrEqual(22)
+ expect(t.infos[0]!.context).toMatchObject({ queries: 1_000 })
+ expect((t.infos[0]!.context.sampled as number)).toBeGreaterThanOrEqual(9)
+ })
+
+ it('brings the in-flight count back after every end, so sampling never stops', () => {
+ const t = sampler()
+ for (let i = 0; i < 5; i++) t.engine.start()
+ for (let i = 0; i < 5; i++) t.engine.end(undefined, 'User', 'update', {}, undefined)
+ t.advance(1_000)
+ const sample = t.engine.start()
+ t.burn(2)
+ t.engine.end(sample, 'User', 'findFirst', {}, { id: 'u' })
+ t.engine.flush()
+ expect(t.infos[0]!.context).toMatchObject({ sampled: 1, engineMs: 2 })
+ })
+
+ it('ranks by engine ms, names raw commands by collection, sums each op across routes, and starts a new window', () => {
+ const t = sampler({ gapMs: 0 })
+ const run = (route: string, model: string | undefined, action: string, args: unknown, engineMs: number) => runWork(route, () => {
+ const sample = t.engine.start()
+ t.burn(engineMs)
+ t.engine.end(sample, model, action, args, [])
+ })
+ run('http GET /api/a', 'SwarmGroup', 'findMany', {}, 1)
+ run('http GET /api/b', 'SwarmGroup', 'findMany', {}, 2)
+ run('http POST /api/hosts/:id/readings', undefined, '$runCommandRaw', { aggregate: 'SwarmMachine', pipeline: [] }, 5)
+ t.engine.flush()
+ const context = t.infos[0]!.context
+ expect(top(context).map(r => [r.op, r.route, r.engineMs])).toEqual([
+ ['SwarmMachine.$runCommandRaw aggregate', 'http POST /api/hosts/:id/readings', 5],
+ ['SwarmGroup.findMany', 'http GET /api/b', 2],
+ ['SwarmGroup.findMany', 'http GET /api/a', 1],
+ ])
+ expect(context.byOp).toEqual({ 'SwarmMachine.$runCommandRaw aggregate': 5, 'SwarmGroup.findMany': 3 })
+ expect(context.costMs).toEqual(expect.any(Number))
+ t.engine.flush()
+ expect(t.infos[1]!.context).toMatchObject({ queries: 0, sampled: 0, engineMs: 0, top: [] })
+ })
+
+ it('logs unavailable, and still counts queries, where the engine threads cannot be read', () => {
+ const t = sampler({ read: () => { throw new Error('ENOENT /proc/self/task') } })
+ t.engine.end(t.engine.start(), 'User', 'findFirst', {}, { id: 'u' })
+ t.engine.flush()
+ expect(t.infos[0]!.context).toMatchObject({ threads: 'unavailable', reason: 'ENOENT /proc/self/task', queries: 1, sampled: 0 })
+ })
+
+ it('names no value: ops are model.action and routes are patterns', () => {
+ const t = sampler()
+ runWork('http GET /api/x', () => t.engine.end(t.engine.start(), 'User', 'findFirst', { where: { email: '[email protected]' } }, { email: '[email protected]' }))
+ t.engine.flush()
+ expect(JSON.stringify(t.infos)).not.toContain('[email protected]')
+ })
+})
diff --git a/backend/src/lib/engineCpu.ts b/backend/src/lib/engineCpu.ts
new file mode 100644
index 000000000..60aca846b
--- /dev/null
+++ b/backend/src/lib/engineCpu.ts
@@ -0,0 +1,195 @@
+import { readdirSync, readFileSync } from 'node:fs'
+import { hostname } from 'node:os'
+import { performance } from 'node:perf_hooks'
+import { logger } from '../utils/logger.js'
+import { parseThreadStat, threadBucket } from './cpuStats.js'
+import { docsOf, opName, workKind } from './queryStats.js'
+
+/**
+ * Which queries the Prisma engine spends its CPU on (task #3041). On 11-05 the engine's threads took 45% of a pod's CPU
+ * (`cpu by thread`, lib/cpuStats.ts), as much as all our JS on the main thread, and nothing said which queries.
+ *
+ * Prisma's `query` events cannot say it: for MongoDB their `duration` is the database call, not the engine's own work
+ * (building the query, BSON to values, values to the JSON string the client parses), and the `query` log level formats
+ * every query, so it would cost what it measures. Instead: Linux keeps each thread's time on CPU in ns
+ * (`/proc/self/task/<tid>/schedstat`). While a query runs alone in the process, the engine threads' CPU between its start
+ * and its end is its own. The query hook (lib/queryStats.ts withQueryStats) calls `start` and `end` for every query; a
+ * query that starts with none in flight, at most once per `gapMs`, is read before and after, and kept only if no other
+ * query started before it ended. Each key's mean is scaled by how often it ran in the minute.
+ *
+ * `prisma by query`, every minute: per op (model.action, a raw command by collection) and route, the count, the samples,
+ * the estimated engine ms and µs per query, and the documents; `byOp` sums each op across routes; `engineMs` sums the
+ * estimates, to read beside `cpu by thread`'s `prismaEngineMs`; `costMs` is the sampler's own time. Raw commands run in
+ * the same engine and are measured the same way. Like `db by shape`, it names no value: routes are patterns.
+ */
+
+export type EngineClock = 'schedstat' | 'stat'
+
+/** The Prisma engine threads' CPU so far, in ns, from `schedstat` where the kernel has it, else `stat`'s 10 ms ticks (each
+ * sample is coarse, the mean over many is not). [names] caches each thread's name by tid. Throws when [root] cannot be
+ * listed (not Linux, or /proc not mounted); a thread that exits while it is read is skipped. */
+export function readEngineCpu(root = '/proc/self/task', pid = process.pid, names = new Map<number, string>()): { ns: number; clock: EngineClock } {
+ let ns = 0, clock: EngineClock = 'schedstat'
+ for (const entry of readdirSync(root)) {
+ const tid = Number(entry)
+ if (tid === pid) continue
+ let name = names.get(tid)
+ if (name === undefined) {
+ try { name = readFileSync(`${root}/${entry}/comm`, 'utf8').trim() } catch { continue }
+ names.set(tid, name)
+ }
+ if (threadBucket(name, tid, pid) !== 'prismaEngine') continue
+ try {
+ ns += Number(readFileSync(`${root}/${entry}/schedstat`, 'utf8').split(' ', 1)[0]) || 0
+ } catch {
+ try { ns += parseThreadStat(readFileSync(`${root}/${entry}/stat`, 'utf8')).cpuMs * 1e6; clock = 'stat' } catch { continue }
+ }
+ }
+ return { ns, clock }
+}
+
+/** What `start` opened: the engine CPU before the query, and the start count it must still match at its end. */
+export type EngineSample = { ns: number; seq: number }
+
+export type EngineCpu = {
+ /** Every query calls it as it starts. Answers the sample it opened, if it opened one. */
+ start(): EngineSample | undefined
+ /** Every query calls it once as it ends, failed ones too, with what its start answered. */
+ end(sample: EngineSample | undefined, model: string | undefined, action: string, args: unknown, result: unknown): void
+ /** Logs the window so far and starts a new one. */
+ flush(): void
+}
+
+export type EngineCpuOptions = {
+ pod?: string
+ /** Least time between two samples' starts: bounds the sampler's cost at any query rate. */
+ gapMs?: number
+ /** How many keys the line lists, by engine ms. */
+ top?: number
+ /** Keys one window keeps; queries past them count under `other`. */
+ maxKeys?: number
+ now?: () => number
+ read?: () => { ns: number; clock: EngineClock }
+ log?: (message: string, context: Record<string, unknown>) => void
+ warn?: (message: string, context: Record<string, unknown>) => void
+}
+
+type Entry = { op: string; route: string; n: number; docs: number; sampled: number; sampledNs: number }
+
+const OTHER = 'other'
+
+export function createEngineCpu({ pod = `${hostname()}:${process.pid}`, gapMs = 100, top = 25, maxKeys = 1000, now = () => performance.now(),
+ read, log = (m, c) => logger.info(m, c), warn = (m, c) => logger.warn(m, c) }: EngineCpuOptions = {}): EngineCpu {
+ const names = new Map<number, string>()
+ const reader = read ?? (() => readEngineCpu(undefined, undefined, names))
+ let entries = new Map<string, Entry>()
+ let inFlight = 0, starts = 0, lastSample = -Infinity, queries = 0, sampled = 0, discarded = 0, costMs = 0, started = now()
+ let clock: EngineClock | undefined, unavailable: string | undefined, warned = false
+
+ /** The engine CPU now, timed into the sampler's own cost; undefined when it cannot be read. */
+ const timed = (): number | undefined => {
+ const t = performance.now()
+ try {
+ const reading = reader()
+ clock = reading.clock
+ unavailable = undefined
+ return reading.ns
+ } catch (error) {
+ unavailable = error instanceof Error ? error.message : String(error)
+ return undefined
+ } finally {
+ costMs += performance.now() - t
+ }
+ }
+
+ const entryFor = (op: string, route: string): Entry => {
+ let key = `${op}\0${route}`
+ let entry = entries.get(key)
+ if (entry) return entry
+ if (entries.size >= maxKeys - 1) { key = OTHER; entry = entries.get(OTHER) }
+ if (!entry) {
+ entry = key === OTHER ? { op: OTHER, route: '', n: 0, docs: 0, sampled: 0, sampledNs: 0 } : { op, route, n: 0, docs: 0, sampled: 0, sampledNs: 0 }
+ entries.set(key, entry)
+ }
+ return entry
+ }
+
+ return {
+ start() {
+ const alone = inFlight === 0
+ inFlight++
+ starts++
+ if (!alone) return undefined
+ const at = now()
+ if (at - lastSample < gapMs) return undefined
+ lastSample = at
+ const ns = timed()
+ return ns === undefined ? undefined : { ns, seq: starts }
+ },
+
+ end(sample, model, action, args, result) {
+ inFlight = Math.max(0, inFlight - 1)
+ try {
+ queries++
+ const entry = entryFor(opName(model, action, args), workKind())
+ const docs = docsOf(result)
+ entry.n++
+ entry.docs += docs
+ if (!sample) return
+ // Another query started while this one ran: the engine's CPU since is not this query's alone.
+ if (starts !== sample.seq) { discarded++; return }
+ const ns = timed()
+ if (ns === undefined) return
+ entry.sampled++
+ entry.sampledNs += Math.max(0, ns - sample.ns)
+ sampled++
+ } catch (error) {
+ if (warned) return
+ warned = true
+ warn('prisma by query failed', { error: error instanceof Error ? error.message : String(error) })
+ }
+ },
+
+ flush() {
+ const rows = [...entries.values()].map(e => {
+ const mean = e.sampled ? e.sampledNs / e.sampled : 0
+ return { op: e.op, route: e.route, n: e.n, sampled: e.sampled, engineMs: Math.round(mean * e.n / 1e6), usPerQuery: Math.round(mean / 1e3), docs: e.docs }
+ })
+ const byOp = new Map<string, number>()
+ for (const r of rows) byOp.set(r.op, (byOp.get(r.op) ?? 0) + r.engineMs)
+ const context: Record<string, unknown> = { pod, windowMs: Math.round(now() - started) }
+ if (unavailable !== undefined) Object.assign(context, { threads: 'unavailable', reason: unavailable })
+ else Object.assign(context, { clock: clock ?? 'none' })
+ Object.assign(context, {
+ queries, sampled, discarded, engineMs: rows.reduce((total, r) => total + r.engineMs, 0),
+ costMs: Math.round(costMs * 10) / 10, keys: rows.length,
+ top: rows.filter(r => r.engineMs > 0).sort((a, b) => b.engineMs - a.engineMs || b.n - a.n).slice(0, top),
+ byOp: Object.fromEntries([...byOp].filter(([, ms]) => ms > 0).sort((a, b) => b[1] - a[1]).slice(0, 15)),
+ })
+ log('prisma by query', context)
+ entries = new Map(); queries = 0; sampled = 0; discarded = 0; costMs = 0; started = now()
+ // A tid can be reused by a new thread: names are read again each window.
+ names.clear()
+ },
+ }
+}
+
+let active: EngineCpu | undefined
+
+/** This process's sampler, once started; the query hook (lib/prisma.ts) reads it on every query. */
+export function activeEngineCpu(): EngineCpu | undefined {
+ return active
+}
+
+/** Starts this process's `prisma by query` line, every [everyMs]. Returns a stop that logs the last partial window. */
+export function startEngineCpu(options: EngineCpuOptions & { everyMs?: number } = {}): () => void {
+ const engine = createEngineCpu(options)
+ active = engine
+ const timer = setInterval(() => engine.flush(), options.everyMs ?? 60_000)
+ timer.unref()
+ return () => {
+ clearInterval(timer)
+ if (active === engine) active = undefined
+ engine.flush()
+ }
+}
diff --git a/backend/src/lib/harnessAuthCache.test.ts b/backend/src/lib/harnessAuthCache.test.ts
new file mode 100644
index 000000000..1faeb64dc
--- /dev/null
+++ b/backend/src/lib/harnessAuthCache.test.ts
@@ -0,0 +1,60 @@
+import { describe, expect, it, vi } from 'vitest'
+
+const fakes = vi.hoisted(() => {
+ const writes: Array<{ key: string; value: string; ttl: number }> = []
+ let fail = false
+ return {
+ writes,
+ setFail: (v: boolean) => { fail = v },
+ pub: { set: vi.fn(async (key: string, value: string, _ex: string, ttl: number) => {
+ if (fail) throw new Error('redis down')
+ writes.push({ key, value, ttl }); return 'OK'
+ }) },
+ }
+})
+vi.mock('./bus.js', () => ({ pub: fakes.pub }))
+
+import { AUTH_CACHE_TTL_MS, AUTH_EPOCH_TTL_SEC, authEpochKey, createAuthCache, signInsChanged } from './harnessAuthCache.js'
+
+const user = { sub: 'u1', email: '[email protected]', role: 'user', autonomousEnv: 'prod' as const, harnessSessionId: 's1', harnessSessionKind: 'viewer' as const }
+const entry = { sessionId: 's1', userId: 'u1', epoch: null, user }
+
+describe('the per-pod sign-in cache', () => {
+ it('keeps a sign-in for 30 seconds, then forgets it', () => {
+ let now = 1_000
+ const cache = createAuthCache({ now: () => now })
+ cache.set('h1', entry)
+ now += AUTH_CACHE_TTL_MS - 1
+ expect(cache.get('h1')).toMatchObject({ sessionId: 's1', userId: 'u1' })
+ now += 1
+ expect(cache.get('h1')).toBeUndefined()
+ expect(AUTH_CACHE_TTL_MS).toBe(30_000)
+ })
+
+ it('holds at most its bound, dropping the oldest first', () => {
+ const cache = createAuthCache({ max: 2, now: () => 0 })
+ cache.set('a', entry); cache.set('b', entry); cache.set('c', entry)
+ expect(cache.get('a')).toBeUndefined()
+ expect(cache.get('b')).toBeDefined()
+ expect(cache.get('c')).toBeDefined()
+ expect(cache.size()).toBe(2)
+ })
+})
+
+describe('a change to a person\'s sign-ins', () => {
+ it('writes a fresh epoch for each person, never one an earlier epoch could equal, outliving every cached sign-in', async () => {
+ fakes.writes.length = 0
+ await signInsChanged('u1', 'u2', 'u1')
+ await signInsChanged('u1')
+ expect(fakes.writes.map(w => w.key)).toEqual([authEpochKey('u1'), authEpochKey('u2'), authEpochKey('u1')])
+ expect(new Set(fakes.writes.map(w => w.value)).size).toBe(3)
+ for (const w of fakes.writes) expect(w.ttl).toBe(AUTH_EPOCH_TTL_SEC)
+ expect(AUTH_EPOCH_TTL_SEC * 1000).toBeGreaterThan(AUTH_CACHE_TTL_MS * 10)
+ })
+
+ it('throws when the epoch cannot be written, so the caller cannot report a sign-out that other pods may still serve', async () => {
+ fakes.setFail(true)
+ await expect(signInsChanged('u1')).rejects.toThrow('redis down')
+ fakes.setFail(false)
+ })
+})
diff --git a/backend/src/lib/harnessAuthCache.ts b/backend/src/lib/harnessAuthCache.ts
new file mode 100644
index 000000000..7e380928e
--- /dev/null
+++ b/backend/src/lib/harnessAuthCache.ts
@@ -0,0 +1,72 @@
+import { randomBytes } from 'node:crypto'
+import { pub } from './bus.js'
+import type { AuthUser } from './ssoAuth.js'
+
+/**
+ * A Harness sign-in each pod remembers for [AUTH_CACHE_TTL_MS], so a request signed in with a Harness session does
+ * not read its session row and its user row from Mongo every time (task #3041: `harness_sessions` 47 reads/s and
+ * `users` 53/s on the primary, nearly all of them this check).
+ *
+ * Revocation still takes effect at once, on every pod: each person has a sign-in EPOCH in Redis, read in the same
+ * call that reads the access token (lib/harnessSession.ts). Every write that ends a session or changes what a sign-in
+ * answers sets it to a new random value, after the write ([signInsChanged]); a remembered sign-in whose epoch moved is
+ * read from Mongo again. The miss path reads the epoch before the session and the user, so a sign-out that lands
+ * between those reads leaves the remembered epoch stale rather than being missed.
+ */
+
+export const AUTH_CACHE_TTL_MS = 30_000
+/** Far past [AUTH_CACHE_TTL_MS]: an epoch that expired only costs a pod one read from Mongo. */
+export const AUTH_EPOCH_TTL_SEC = 60 * 60
+const AUTH_CACHE_MAX = 20_000
+
+export const authEpochKey = (userId: string): string => `hnauth:epoch:${userId}`
+
+export interface RememberedSignIn {
+ sessionId: string
+ userId: string
+ /** The person's epoch as read before the session and the user were; null when Redis held none. */
+ epoch: string | null
+ user: AuthUser
+}
+
+export interface AuthCache {
+ get(tokenHash: string): RememberedSignIn | undefined
+ set(tokenHash: string, entry: RememberedSignIn): void
+ delete(tokenHash: string): void
+ clear(): void
+ size(): number
+}
+
+/** A map of at most [max] sign-ins, each forgotten [AUTH_CACHE_TTL_MS] after it was remembered, oldest dropped first. */
+export function createAuthCache({ max = AUTH_CACHE_MAX, now = () => Date.now() }: { max?: number; now?: () => number } = {}): AuthCache {
+ const entries = new Map<string, { at: number; entry: RememberedSignIn }>()
+ return {
+ get(tokenHash) {
+ const held = entries.get(tokenHash)
+ if (!held) return undefined
+ if (now() - held.at >= AUTH_CACHE_TTL_MS) { entries.delete(tokenHash); return undefined }
+ return held.entry
+ },
+ set(tokenHash, entry) {
+ entries.delete(tokenHash)
+ entries.set(tokenHash, { at: now(), entry })
+ // A Map keeps insertion order, so its first key is the oldest.
+ while (entries.size > max) entries.delete(entries.keys().next().value!)
+ },
+ delete(tokenHash) { entries.delete(tokenHash) },
+ clear() { entries.clear() },
+ size() { return entries.size },
+ }
+}
+
+export const harnessAuthCache = createAuthCache()
+
+/**
+ * The sign-ins of [userIds] changed: a session ended, or what their user row answers did. Call it AFTER the database
+ * write. It throws when an epoch cannot be written, so a caller never reports a sign-out other pods may still serve.
+ * A random value, never a counter: a key that expired and was written again cannot equal one a pod remembers.
+ */
+export async function signInsChanged(...userIds: string[]): Promise<void> {
+ await Promise.all([...new Set(userIds)].map((userId) =>
+ pub.set(authEpochKey(userId), randomBytes(12).toString('base64url'), 'EX', AUTH_EPOCH_TTL_SEC)))
+}
diff --git a/backend/src/lib/harnessSession.test.ts b/backend/src/lib/harnessSession.test.ts
index ef3eb331a..8fc0b4b3c 100644
--- a/backend/src/lib/harnessSession.test.ts
+++ b/backend/src/lib/harnessSession.test.ts
@@ -21,6 +21,10 @@ const fakes = vi.hoisted(() => {
if (failRedis) throw new Error('redis down')
return redis.get(key) ?? null
}),
+ mget: vi.fn(async (...keys: string[]) => {
+ if (failRedis) throw new Error('redis down')
+ return keys.map((key) => redis.get(key) ?? null)
+ }),
// GET then DEL, in one transaction — how a redeem spends a code.
multi: vi.fn(() => {
const ops: Array<() => unknown> = []
@@ -88,7 +92,8 @@ vi.mock('../services/UserService.js', () => ({
vi.mock('../services/index.js', () => ({ userService: { get: fakes.get, toPublic: (u: unknown) => u } }))
import { authenticateAccessToken, SsoAuthError } from './ssoAuth.js'
-import { HANDOFF_TTL_SEC, redeemHandoff, refreshHarnessSession, revokeHarnessSession, startHandoff } from './harnessSession.js'
+import { HANDOFF_TTL_SEC, redeemHandoff, refreshHarnessSession, revokeHarnessSession, revokeHarnessSessionById, startHandoff } from './harnessSession.js'
+import { harnessAuthCache } from './harnessAuthCache.js'
import { HARNESS_ACCESS_PREFIX, harnessTokenHash, newHarnessToken } from './harnessTokenFormat.js'
import { teamConversionMessage } from './teamSession.js'
import { ed25519 } from '@noble/curves/ed25519.js'
@@ -102,6 +107,7 @@ beforeEach(() => {
fakes.redis.clear(); fakes.ttls.clear(); fakes.sessions.length = 0; fakes.users.clear(); fakes.machines.length = 0
fakes.users.set(USER.id, USER)
fakes.setFailRedis(false)
+ harnessAuthCache.clear()
vi.clearAllMocks()
})
@@ -164,6 +170,99 @@ describe('scan to sign in — the handoff', () => {
})
})
+describe('a sign-in remembered for 30 seconds per pod (#3041)', () => {
+ const signIn = async () => (await redeemHandoff((await startHandoff(USER.id)).code, 'phone'))!
+
+ it('a second request in the window reads neither the session nor the user from Mongo, and answers the same', async () => {
+ const { token } = await signIn()
+ const first = await authenticateAccessToken(token)
+ fakes.harnessSession.findUnique.mockClear(); fakes.get.mockClear()
+ const second = await authenticateAccessToken(token)
+ expect(second).toEqual(first)
+ expect(fakes.harnessSession.findUnique).not.toHaveBeenCalled()
+ expect(fakes.get).not.toHaveBeenCalled()
+ })
+
+ it('every request still checks its token in Redis: an access token gone from Redis stops at once', async () => {
+ const { token } = await signIn()
+ await authenticateAccessToken(token)
+ for (const key of [...fakes.redis.keys()]) if (key.startsWith('hnauth:access:')) fakes.redis.delete(key)
+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })
+ })
+
+ it('signing out ends a remembered sign-in at once', async () => {
+ const { token, refreshToken } = await signIn()
+ await authenticateAccessToken(token)
+ await revokeHarnessSession(refreshToken!)
+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })
+ })
+
+ it('revoking a device by id ends its remembered sign-in at once', async () => {
+ const { token } = await signIn()
+ await authenticateAccessToken(token)
+ await revokeHarnessSessionById('s1', USER.id)
+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })
+ })
+
+ it('a pod that remembers nothing of the sign-out still learns it from Redis: the change is not kept in this pod alone', async () => {
+ const { token, refreshToken } = await signIn()
+ await authenticateAccessToken(token)
+ // The sign-out ran on another pod: only Redis and Mongo saw it.
+ const remembered = harnessAuthCache.get(harnessTokenHash(token))
+ await revokeHarnessSession(refreshToken!)
+ expect(harnessAuthCache.get(harnessTokenHash(token))).toBe(remembered)
+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })
+ })
+
+ it('a sign-out that lands between reading the session and remembering it is not missed', async () => {
+ const { token } = await signIn()
+ const read = fakes.harnessSession.findUnique.getMockImplementation()!
+ fakes.harnessSession.findUnique.mockImplementationOnce(async (args: { where: Record<string, unknown> }) => {
+ const live = { ...(await read(args))! }
+ await revokeHarnessSessionById('s1', USER.id)
+ return live
+ })
+ // This request read the session while it was live.
+ await expect(authenticateAccessToken(token)).resolves.toMatchObject({ harnessSessionId: 's1' })
+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })
+ })
+
+ it('a sign-in is read again after 30 seconds', async () => {
+ const { token } = await signIn()
+ const now = vi.spyOn(Date, 'now')
+ const start = Date.now()
+ now.mockReturnValue(start)
+ await authenticateAccessToken(token)
+ fakes.harnessSession.findUnique.mockClear()
+ now.mockReturnValue(start + 30_000)
+ await authenticateAccessToken(token)
+ expect(fakes.harnessSession.findUnique).toHaveBeenCalledTimes(1)
+ now.mockRestore()
+ })
+
+ it('Redis down on a remembered sign-in is still an outage, not a sign-in', async () => {
+ const { token } = await signIn()
+ await authenticateAccessToken(token)
+ fakes.setFailRedis(true)
+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'AUTH_SERVICE_UNAVAILABLE' })
+ })
+
+ it('remembers no refusal: a token refused once is read again next time', async () => {
+ const { token } = await signIn()
+ fakes.users.delete(USER.id)
+ await expect(authenticateAccessToken(token)).rejects.toMatchObject({ code: 'INVALID_TOKEN' })
+ fakes.users.set(USER.id, USER)
+ await expect(authenticateAccessToken(token)).resolves.toMatchObject({ sub: USER.id })
+ })
+
+ it('a caller changing what it was handed changes nothing for the next request', async () => {
+ const { token } = await signIn()
+ const first = await authenticateAccessToken(token) as unknown as Record<string, unknown>
+ first.role = 'admin'
+ await expect(authenticateAccessToken(token)).resolves.toMatchObject({ role: 'user' })
+ })
+})
+
describe('scan to sign in — the routes', () => {
let app: FastifyInstance
const signedIn = { sub: USER.id, email: USER.email, role: 'user', autonomousEnv: 'prod' as const }
diff --git a/backend/src/lib/harnessSession.ts b/backend/src/lib/harnessSession.ts
index 7c3c0592d..632da0d0c 100644
--- a/backend/src/lib/harnessSession.ts
+++ b/backend/src/lib/harnessSession.ts
@@ -7,6 +7,7 @@ import { TEAM_CONVERTED_FRAME } from './teamSession.js'
import { userService } from '../services/UserService.js'
import { storedAutonomousEnvironment, type AutonomousEnvironment } from './autonomousEnvironment.js'
import { SsoAuthError, type AuthUser } from './ssoAuth.js'
+import { authEpochKey, harnessAuthCache, signInsChanged } from './harnessAuthCache.js'
import type { SsoClientId } from './sso.js'
import {
endedSessionData,
@@ -182,6 +183,7 @@ async function convertComputer(userId: string, label: string, computerId: string
} })
return { session: created, machineIds: machines.map((m) => m.machineId) }
})
+ await signInsChanged(userId)
for (const machineId of machineIds) {
await publishDown(machineId, { connId: '', frame: { type: TEAM_CONVERTED_FRAME } }).catch(() => { /* best effort: the next connect decides */ })
}
@@ -289,10 +291,12 @@ export async function refreshHarnessSession(refreshToken: string): Promise<Harne
* tokens it held are deleted. */
export async function revokeHarnessSession(refreshToken: string): Promise<void> {
if (!isHarnessRefreshToken(refreshToken)) return
- await prisma.harnessSession.updateMany({
- where: { refreshHash: harnessTokenHash(refreshToken), ...notRevoked },
- data: endedSessionData(),
- })
+ const refreshHash = harnessTokenHash(refreshToken)
+ const session = await prisma.harnessSession.findUnique({ where: { refreshHash }, select: { userId: true } })
+ if (!session) return
+ await prisma.harnessSession.updateMany({ where: { refreshHash, ...notRevoked }, data: endedSessionData() })
+ // Even when it was already revoked: a retry after a failed epoch write must still reach every pod.
+ await signInsChanged(session.userId)
}
async function issueAccess(sessionId: string, userId: string, kind: HarnessSessionKind): Promise<string> {
@@ -306,36 +310,71 @@ async function issueAccess(sessionId: string, userId: string, kind: HarnessSessi
return token
}
+const invalid = (): SsoAuthError => new SsoAuthError('Invalid or expired access token', 'INVALID_TOKEN')
+
/**
* The account a Harness access token speaks for. The session is read on every call, so a revoked
- * one stops at once rather than when its access token runs out.
+ * one stops at once rather than when its access token runs out — or, within [AUTH_CACHE_TTL_MS] of the
+ * last read, remembered by this pod and checked against the person's sign-in epoch instead
+ * (lib/harnessAuthCache.ts), which every sign-out moves. The access token is read from Redis either way.
*
* Only a token that is not good is `INVALID_TOKEN`. Redis or Mongo failing is
* `AUTH_SERVICE_UNAVAILABLE` — the caller's 503, not its 401 — because a 401 sends the phone to
* refresh and, failing that, to sign in again, over what is only an outage.
*/
export async function authenticateHarnessAccessToken(token: string): Promise<AuthUser> {
- const invalid = (): SsoAuthError => new SsoAuthError('Invalid or expired access token', 'INVALID_TOKEN')
+ const hash = harnessTokenHash(token)
+ const remembered = harnessAuthCache.get(hash)
let user: Awaited<ReturnType<typeof userService.get>>
let sessionId: string
+ let userId: string
+ let epoch: string | null
let kind: string | null | undefined
let computerId: string | null | undefined
try {
- const raw = await pub.get(accessKey(harnessTokenHash(token)))
- if (!raw) throw invalid()
+ let raw: string | null
+ if (remembered) {
+ [raw, epoch] = await pub.mget(accessKey(hash), authEpochKey(remembered.userId))
+ } else {
+ raw = await pub.get(accessKey(hash))
+ epoch = null
+ }
+ if (!raw) {
+ harnessAuthCache.delete(hash)
+ throw invalid()
+ }
const held = JSON.parse(raw) as { sessionId?: unknown; userId?: unknown }
if (typeof held.sessionId !== 'string' || typeof held.userId !== 'string') throw invalid()
sessionId = held.sessionId
+ userId = held.userId
+ if (remembered && remembered.sessionId === sessionId && remembered.userId === userId && remembered.epoch === epoch) {
+ return { ...remembered.user }
+ }
+ // The epoch before the session and the user: a sign-out landing between these reads leaves it stale.
+ if (!remembered || remembered.userId !== userId) epoch = await pub.get(authEpochKey(userId))
const session = await prisma.harnessSession.findUnique({ where: { id: sessionId } })
- if (!session || session.revokedAt || session.userId !== held.userId) throw invalid()
+ if (!session || session.revokedAt || session.userId !== userId) throw invalid()
kind = session.kind
computerId = session.computerId
- user = await userService.get(held.userId)
+ user = await userService.get(userId)
} catch (err) {
if (err instanceof SsoAuthError) throw err
throw new SsoAuthError('Authentication service unavailable', 'AUTH_SERVICE_UNAVAILABLE')
}
if (!user) throw invalid()
+ const answer = identityOf(token, user, sessionId, kind, computerId)
+ harnessAuthCache.set(hash, { sessionId, userId, epoch, user: answer })
+ return { ...answer }
+}
+
+/** What a live session answers for [token]; throws INVALID_TOKEN for a session of the wrong kind or bound to nothing. */
+function identityOf(
+ token: string,
+ user: NonNullable<Awaited<ReturnType<typeof userService.get>>>,
+ sessionId: string,
+ kind: string | null | undefined,
+ computerId: string | null | undefined,
+): AuthUser {
const identity = {
sub: user.id,
email: user.email,
@@ -357,6 +396,7 @@ export async function authenticateHarnessAccessToken(token: string): Promise<Aut
/** Revoke the session with this id (a device removed from the account's device key log). */
export async function revokeHarnessSessionById(sessionId: string, userId: string): Promise<void> {
await prisma.harnessSession.updateMany({ where: { id: sessionId, userId, ...notRevoked }, data: endedSessionData() })
+ await signInsChanged(userId)
}
// ── Sign in a computer by scanning its QR with a signed-in phone ──────────────────────────────────
diff --git a/backend/src/lib/prisma.ts b/backend/src/lib/prisma.ts
index e58e28b05..8364be235 100644
--- a/backend/src/lib/prisma.ts
+++ b/backend/src/lib/prisma.ts
@@ -2,16 +2,17 @@ import { PrismaClient, type Prisma } from '@prisma/client'
import { env } from '../config/env.js'
import { POOL_DEFAULTS, secondaryReadsUrl, withPoolDefaults } from './databaseUrl.js'
import { withQueryStats } from './queryStats.js'
+import { activeEngineCpu } from './engineCpu.js'
import { metrics } from './metrics.js'
// The Prisma CLI (db push) reads env("DATABASE_URL"); keep it in sync for any manual run.
process.env.DATABASE_URL ??= env.DATABASE_URL
// Every query is timed for the per-minute `db by shape` log (lib/queryStats.ts, DB roadmap Phase 1) and into
-// prisma_query_duration_seconds (lib/metrics.ts).
+// prisma_query_duration_seconds (lib/metrics.ts), and its engine CPU sampled for `prisma by query` (lib/engineCpu.ts).
export const prisma = withQueryStats(new PrismaClient({
datasources: { db: { url: withPoolDefaults(env.DATABASE_URL, POOL_DEFAULTS) } },
-}), undefined, undefined, metrics.observeQuery)
+}), undefined, undefined, metrics.observeQuery, undefined, activeEngineCpu)
/** The client for reads that tolerate a few seconds of lag, on [url] with the secondary options (databaseUrl.ts
* secondaryReadsUrl) and the pool defaults, its queries logged as `secondary:` in `db by shape`; [primary] itself when
@@ -19,7 +20,7 @@ export const prisma = withQueryStats(new PrismaClient({
export function readReplica(primary: PrismaClient, url: string, reads: 'on' | 'off',
make: (url: string) => PrismaClient = url => new PrismaClient({ datasources: { db: { url } } })): PrismaClient {
if (reads === 'off') return primary
- return withQueryStats(make(withPoolDefaults(secondaryReadsUrl(url), POOL_DEFAULTS)), undefined, undefined, metrics.observeQuery, 'secondary')
+ return withQueryStats(make(withPoolDefaults(secondaryReadsUrl(url), POOL_DEFAULTS)), undefined, undefined, metrics.observeQuery, 'secondary', activeEngineCpu)
}
/** Older history pages, costs, search, leaderboard names and usage totals (task #2970). It connects on its first query. On a
diff --git a/backend/src/lib/queryStats.test.ts b/backend/src/lib/queryStats.test.ts
index 3d93b2df4..29d1710be 100644
--- a/backend/src/lib/queryStats.test.ts
+++ b/backend/src/lib/queryStats.test.ts
@@ -1,6 +1,7 @@
import Fastify from 'fastify'
import { afterEach, describe, expect, it } from 'vitest'
import type { PrismaClient } from '@prisma/client'
+import type { EngineCpu } from './engineCpu.js'
import { createQueryStats, enterWork, estimateBytes, filterShape, registerWorkContext, runWork, withQueryStats, workKind, type QueryStatsOptions } from './queryStats.js'
type Logged = { message: string; context: Record<string, unknown> }
@@ -281,6 +282,34 @@ describe('the query hook', () => {
})
})
+describe('the query hook feeds the engine CPU sampler (#3041)', () => {
+ type Call = { model?: string; operation: string; args: unknown; query: (args: unknown) => Promise<unknown> }
+ function hookedEngine(engine: Partial<EngineCpu>) {
+ let hook: ((call: Call) => Promise<unknown>) | undefined
+ const client = { $extends: (extension: { query: { $allOperations: (call: Call) => Promise<unknown> } }) => { hook = extension.query.$allOperations; return client } }
+ withQueryStats(client as unknown as PrismaClient, () => undefined, () => {}, undefined, undefined, () => engine as EngineCpu)
+ return (call: Call) => hook!(call)
+ }
+
+ it('starts and ends one slot for every query, a failed one too, with what its start answered', async () => {
+ const calls: unknown[][] = []
+ const sample = { ns: 5, seq: 1 }
+ const run = hookedEngine({ start: () => { calls.push(['start']); return sample }, end: (...args) => { calls.push(['end', ...args]) } })
+ await run({ model: 'User', operation: 'findFirst', args: {}, query: async () => ({ id: 'u1' }) })
+ const failure = new Error('write conflict')
+ await expect(run({ model: 'User', operation: 'update', args: {}, query: async () => { throw failure } })).rejects.toBe(failure)
+ expect(calls).toEqual([
+ ['start'], ['end', sample, 'User', 'findFirst', {}, { id: 'u1' }],
+ ['start'], ['end', sample, 'User', 'update', {}, undefined],
+ ])
+ })
+
+ it('keeps a query\'s result when the sampler throws at its start or its end', async () => {
+ const run = hookedEngine({ start: () => { throw new Error('start broke') }, end: () => { throw new Error('end broke') } })
+ await expect(run({ model: 'User', operation: 'count', args: {}, query: async () => 3 })).resolves.toBe(3)
+ })
+})
+
describe('the minute handed on to the counters (#2957)', () => {
it('hands every route of the minute to onWindow, not only the 40 the line lists', () => {
const windows: Array<Map<string, { n: number; ms: number; bytes: number }>> = []
diff --git a/backend/src/lib/queryStats.ts b/backend/src/lib/queryStats.ts
index 80b8fa047..df5ae875e 100644
--- a/backend/src/lib/queryStats.ts
+++ b/backend/src/lib/queryStats.ts
@@ -4,6 +4,7 @@ import { performance } from 'node:perf_hooks'
import type { PrismaClient } from '@prisma/client'
import type { FastifyInstance, FastifyRequest } from 'fastify'
import { logger } from '../utils/logger.js'
+import type { EngineCpu, EngineSample } from './engineCpu.js'
/**
* Which queries this process sends, per kind of work (DB roadmap Phase 1). Atlas logs only operations over 100 ms; the
@@ -114,7 +115,7 @@ export function filterShape(action: string, args: unknown): string {
}
/** `Model.action`; a raw command by its collection and command name (`SwarmGroup.$runCommandRaw aggregate`). */
-function opName(model: string | undefined, action: string, args: unknown): string {
+export function opName(model: string | undefined, action: string, args: unknown): string {
if (model) return `${model}.${action}`
if (action === '$runCommandRaw' && isPlainObject(args)) {
const command = Object.keys(args)[0] ?? '?'
@@ -125,7 +126,7 @@ function opName(model: string | undefined, action: string, args: unknown): strin
}
/** Documents a result holds: a list's length, one for a document, a raw cursor reply's first batch. */
-function docsOf(result: unknown): number {
+export function docsOf(result: unknown): number {
if (Array.isArray(result)) return result.length
if (isPlainObject(result)) {
const cursor = result.cursor
@@ -319,33 +320,44 @@ export function startQueryStats(options: QueryStatsOptions & { reportMs?: number
/** The client with one query hook that times every operation, model calls and raw commands alike, for `db by shape` and
* for `observe` (the Prometheus histogram, lib/metrics.ts; `raw` for a call without a model). Query extensions keep the
* client's types, so it is still a PrismaClient to its callers. [label] names this client's queries in `db by shape`
- * (`secondary:User.findMany`, lib/prisma.ts prismaSecondary). A failure of either never reaches the query: it is warned
- * about once per client, and the query's own result or error goes back unchanged. */
+ * (`secondary:User.findMany`, lib/prisma.ts prismaSecondary). [engine] is the engine CPU sampler (lib/engineCpu.ts),
+ * told of every query's start and end. A failure of any of them never reaches the query: it is warned about once per
+ * client, and the query's own result or error goes back unchanged. */
export function withQueryStats(client: PrismaClient, stats: () => QueryStats | undefined = () => active,
warn: (message: string, context: Record<string, unknown>) => void = (m, c) => logger.warn(m, c),
- observe?: (model: string, operation: string, seconds: number) => void, label?: string): PrismaClient {
+ observe?: (model: string, operation: string, seconds: number) => void, label?: string,
+ engine: () => EngineCpu | undefined = () => undefined): PrismaClient {
let warned = false
+ const failedOnce = (error: unknown) => {
+ if (warned) return
+ warned = true
+ warn('query stats failed', { error: error instanceof Error ? error.message : String(error) })
+ }
const note = (model: string | undefined, operation: string, args: unknown, ms: number, result: unknown, failed: boolean) => {
try {
observe?.(model ?? 'raw', operation, ms / 1000)
stats()?.record(model, operation, args, ms, result, failed, label)
- } catch (error) {
- if (warned) return
- warned = true
- warn('query stats failed', { error: error instanceof Error ? error.message : String(error) })
- }
+ } catch (error) { failedOnce(error) }
}
return client.$extends({
query: {
async $allOperations({ model, operation, args, query }) {
+ // The engine CPU sampler (lib/engineCpu.ts) sees every query's start and end, so it knows which one ran alone.
+ let sampler: EngineCpu | undefined, sample: EngineSample | undefined
+ try { sampler = engine(); sample = sampler?.start() } catch (error) { failedOnce(error) }
+ const ended = (result: unknown) => {
+ try { sampler?.end(sample, model, operation, args, result) } catch (error) { failedOnce(error) }
+ }
const started = performance.now()
let result: unknown
try {
result = await query(args)
} catch (error) {
+ ended(undefined)
note(model, operation, args, performance.now() - started, undefined, true)
throw error
}
+ ended(result)
note(model, operation, args, performance.now() - started, result, false)
return result
},
diff --git a/backend/src/lib/ssoAuth.userReads.test.ts b/backend/src/lib/ssoAuth.userReads.test.ts
index a53e70805..6f4a08b5d 100644
--- a/backend/src/lib/ssoAuth.userReads.test.ts
+++ b/backend/src/lib/ssoAuth.userReads.test.ts
@@ -11,6 +11,11 @@ const create = vi.hoisted(() => vi.fn())
vi.mock('./prisma.js', () => ({
prisma: { user: { findUnique, update, create } },
}))
+// An update moves the person's sign-in epoch in Redis (lib/harnessAuthCache.ts), which these tests do not run.
+vi.mock('./harnessAuthCache.js', async (importOriginal) => ({
+ ...(await importOriginal<typeof import('./harnessAuthCache.js')>()),
+ signInsChanged: vi.fn(async () => {}),
+}))
import { authenticateAccessToken, clearSsoProfileCache } from './ssoAuth.js'
diff --git a/backend/src/lib/ssoSession.test.ts b/backend/src/lib/ssoSession.test.ts
index 5851ac9d8..75ddb4b20 100644
--- a/backend/src/lib/ssoSession.test.ts
+++ b/backend/src/lib/ssoSession.test.ts
@@ -24,6 +24,7 @@ const fakes = vi.hoisted(() => {
redis.set(key, value); return 'OK'
}),
get: vi.fn(async (key: string) => redis.get(key) ?? null),
+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),
del: vi.fn(async (key: string) => (redis.delete(key) ? 1 : 0)),
// The lock's compare-and-delete (lib/ssoSession.ts RELEASE_SCRIPT).
eval: vi.fn(async (_script: string, _n: number, key: string, owner: string) => (redis.get(key) === owner && redis.delete(key) ? 1 : 0)),
@@ -141,6 +142,7 @@ vi.mock('./sso.js', async (importOriginal) => {
import { clearSsoProfileCache, authenticateAccessToken, resetSsoBreakers, SsoAuthError } from './ssoAuth.js'
import { refreshHarnessSession, revokeHarnessSession } from './harnessSession.js'
+import { harnessAuthCache } from './harnessAuthCache.js'
import { decryptMachineCredential } from './machineCredential.js'
import { SSO_OUTAGE_GRACE_MS, SSO_RECHECK_MS, ensureHarnessSessionExpiry, ssoAccessTokenFor } from './ssoSession.js'
import { authRoutes } from '../routes/auth.js'
@@ -169,6 +171,7 @@ beforeEach(async () => {
fakes.sso.live.clear(); fakes.sso.rotate = true; fakes.sso.down = false; fakes.sso.calls = 0; fakes.sso.counter = 0; fakes.sso.gate = null
fakes.setFailCreate(false)
clearSsoProfileCache()
+ harnessAuthCache.clear()
// The profile service's circuit (lib/ssoAuth.ts) is shared: tests that fail it on purpose must not open it for the next.
resetSsoBreakers()
vi.clearAllMocks()
@@ -358,6 +361,20 @@ describe('the hourly SSO re-check', () => {
expect(Date.now() - start).toBeLessThan(2 * HOUR)
})
+ it('an SSO refusal ends a sign-in this pod remembers at once (#3041)', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ const start = Date.now()
+ const { refreshToken } = await harnessSignIn()
+ vi.setSystemTime(start + SSO_RECHECK_MS - 1)
+ const last = String((await refresh(refreshToken)).body.data!.token)
+ // Remembered by this pod a moment before the re-check.
+ expect(await me(last)).toBe(200)
+ fakes.sso.live.clear()
+ vi.setSystemTime(start + SSO_RECHECK_MS + 1)
+ expect((await refresh(refreshToken)).status).toBe(401)
+ expect(await me(last)).toBe(401)
+ })
+
it('SSO being down keeps people signed in, for seven days from the last good check', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
const start = Date.now()
diff --git a/backend/src/lib/ssoSession.ts b/backend/src/lib/ssoSession.ts
index 171f5edbe..147cf9058 100644
--- a/backend/src/lib/ssoSession.ts
+++ b/backend/src/lib/ssoSession.ts
@@ -1,6 +1,7 @@
import { randomBytes } from 'node:crypto'
import type { HarnessSession, Prisma, PrismaClient } from '@prisma/client'
import { pub } from './bus.js'
+import { signInsChanged } from './harnessAuthCache.js'
import { prisma } from './prisma.js'
import { ensureIndex } from './db/indexes.js'
import { decryptMachineCredential, encryptMachineCredential } from './machineCredential.js'
@@ -149,6 +150,7 @@ async function renew(sessionId: string, fresh: (row: HarnessSession, now: number
} catch (err) {
if (err instanceof SsoTokenError && err.code === 'INVALID_GRANT' && !unfinished) {
await writeLive(sessionId, endedSessionData())
+ await signInsChanged(row.userId)
logger.info('sso sign-in ended a harness session', { sessionId, userId: row.userId })
return { outcome: 'ended' }
}
diff --git a/backend/src/routes/control.test.ts b/backend/src/routes/control.test.ts
index d9450acd3..d93b21084 100644
--- a/backend/src/routes/control.test.ts
+++ b/backend/src/routes/control.test.ts
@@ -13,6 +13,7 @@ const fakes = vi.hoisted(() => {
pub: {
set: vi.fn(async (key: string, value: string) => { redis.set(key, value); return 'OK' }),
get: vi.fn(async (key: string) => redis.get(key) ?? null),
+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),
multi: vi.fn(() => {
const ops: Array<() => unknown> = []
const chain = {
diff --git a/backend/src/routes/qrSignIn.test.ts b/backend/src/routes/qrSignIn.test.ts
index 844a5c573..333434e3a 100644
--- a/backend/src/routes/qrSignIn.test.ts
+++ b/backend/src/routes/qrSignIn.test.ts
@@ -31,6 +31,7 @@ const fakes = vi.hoisted(() => {
}),
set: vi.fn(async (key: string, value: string, ...opts: unknown[]) => set(key, value, ...opts)),
get: vi.fn(async (key: string) => redis.get(key) ?? null),
+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),
del: vi.fn(async (key: string) => (redis.delete(key) ? 1 : 0)),
multi: vi.fn(() => {
const ops: Array<() => unknown> = []
diff --git a/backend/src/routes/superTabs.test.ts b/backend/src/routes/superTabs.test.ts
index 7256a9809..3e09a637b 100644
--- a/backend/src/routes/superTabs.test.ts
+++ b/backend/src/routes/superTabs.test.ts
@@ -12,6 +12,7 @@ const fakes = vi.hoisted(() => {
pub: {
set: vi.fn(async (key: string, value: string) => { redis.set(key, value); return 'OK' }),
get: vi.fn(async (key: string) => redis.get(key) ?? null),
+ mget: vi.fn(async (...keys: string[]) => keys.map((key) => redis.get(key) ?? null)),
multi: vi.fn(() => {
const ops: Array<() => unknown> = []
const chain = {
diff --git a/backend/src/server.ts b/backend/src/server.ts
index 68e50bd65..e5516ba76 100644
--- a/backend/src/server.ts
+++ b/backend/src/server.ts
@@ -42,6 +42,7 @@ import { selfProfileFromEnv } from './lib/selfProfile.js'
import { registerReplyJsonTiming, startCpuStats } from './lib/cpuStats.js'
import { startRelayUsageFlush } from './lib/productUsage.js'
import { registerWorkContext, startQueryStats } from './lib/queryStats.js'
+import { startEngineCpu } from './lib/engineCpu.js'
import { startRetryStats } from './superTabs/retryStats.js'
import { startRelayStats } from './lib/relayStats.js'
import { startPodRoutes } from './lib/hub.js'
@@ -160,6 +161,9 @@ async function start(): Promise<void> {
registerReplyJsonTiming(app, cpuStats)
// Always on, every minute (DB roadmap Phase 1): queries by model.action, filter shape and route, and each route's p50/p95.
const stopQueryStats = startQueryStats({ onWindow: routes => metrics.dbWindow(routes) })
+ // Always on, every minute: the Prisma engine's CPU by query, from queries sampled while they run alone
+ // (lib/engineCpu.ts, task #3041), to read beside `cpu by thread`'s prismaEngineMs.
+ const stopEngineCpu = startEngineCpu()
// Always on, every minute: `last hour`, the sums of the last 60 minutes the production scan reads from a log tail
// (lib/lastHour.ts, task #2957).
const stopLastHour = startLastHour(metrics.hour)
@@ -266,6 +270,7 @@ async function start(): Promise<void> {
await Promise.race([stopRelayUsage().catch(() => {}), new Promise((resolve) => setTimeout(resolve, 3_000))])
// The last partial minute of these logs, once the drain and the usage flush have run their queries.
stopQueryStats()
+ stopEngineCpu()
stopLastHour()
stopRetryStats()
stopDeliveryCalls()
diff --git a/backend/src/services/MachineService.computer.test.ts b/backend/src/services/MachineService.computer.test.ts
index 862e0f94f..1c9c22ccc 100644
--- a/backend/src/services/MachineService.computer.test.ts
+++ b/backend/src/services/MachineService.computer.test.ts
@@ -180,6 +180,17 @@ describe('destroying a team machine', () => {
})
})
+ it("ends its team account's remembered sign-ins on every pod, after the session is revoked (#3041)", async () => {
+ findUnique.mockResolvedValue({ machineId: 'm1', userId: 'u1', managerId: '', authMode: 'remote', billingStatus: 'not_required',
+ autonomousEnv: 'prod', computerId: 'b'.repeat(32), sessionKind: 'team' })
+ update.mockResolvedValue({})
+ sessionUpdateMany.mockResolvedValue({ count: 1 })
+ await machineService.destroy('a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4', { sub: 'u1', role: 'user', autonomousEnv: 'prod' })
+ const epoch = redisSet.mock.calls.findIndex(([key]) => key === 'hnauth:epoch:u1')
+ expect(epoch).toBeGreaterThanOrEqual(0)
+ expect(redisSet.mock.invocationCallOrder[epoch]).toBeGreaterThan(sessionUpdateMany.mock.invocationCallOrder[0]!)
+ })
+
it('touches no session for a machine with no computer', async () => {
findUnique.mockResolvedValue({ machineId: 'm1', userId: 'u1', managerId: '', authMode: 'remote', billingStatus: 'not_required', autonomousEnv: 'prod' })
update.mockResolvedValue({})
diff --git a/backend/src/services/MachineService.ts b/backend/src/services/MachineService.ts
index 59195a81b..8277bef22 100644
--- a/backend/src/services/MachineService.ts
+++ b/backend/src/services/MachineService.ts
@@ -16,6 +16,7 @@ import { machineBillingService } from './MachineBillingService.js'
import { storedAutonomousEnvironment, type AutonomousEnvironment } from '../lib/autonomousEnvironment.js'
import { TEAM_SESSION_LIMITED, TEAM_SESSION_MESSAGE } from '../lib/teamSession.js'
import type { AuthUser } from '../lib/ssoAuth.js'
+import { signInsChanged } from '../lib/harnessAuthCache.js'
import { machineRenamed } from '../superTabs/machineRenames.js'
/** The agent's tier / how it runs Claude, resolved through its plan.
@@ -709,7 +710,8 @@ export const machineService = {
await prisma.harnessSession.updateMany({
where: { userId: binding.userId, kind: 'team', computerId: binding.computerId, OR: [{ revokedAt: null }, { revokedAt: { isSet: false } }] },
data: { revokedAt: new Date() },
- }).catch((err: unknown) => logger.warn('team session revoke failed', { machineId, error: String(err) }))
+ }).then(() => signInsChanged(binding.userId))
+ .catch((err: unknown) => logger.warn('team session revoke failed', { machineId, error: String(err) }))
}
} else {
// Managed: reap the container + node row in the background (idempotent on the manager → retryable).
diff --git a/backend/src/services/UserService.test.ts b/backend/src/services/UserService.test.ts
index cadea6189..e48324eab 100644
--- a/backend/src/services/UserService.test.ts
+++ b/backend/src/services/UserService.test.ts
@@ -6,6 +6,7 @@ const update = vi.hoisted(() => vi.fn())
const remove = vi.hoisted(() => vi.fn())
const updateManyDevices = vi.hoisted(() => vi.fn())
const transaction = vi.hoisted(() => vi.fn())
+const signInsChanged = vi.hoisted(() => vi.fn(async () => {}))
vi.mock('../lib/prisma.js', () => ({
prisma: {
@@ -16,6 +17,7 @@ vi.mock('../lib/prisma.js', () => ({
}))
vi.mock('../config/env.js', () => ({ ADMIN_EMAILS: new Set<string>(['[email protected]']) }))
vi.mock('../lib/password.js', () => ({ hashPassword: (p: string) => `hashed:${p}` }))
+vi.mock('../lib/harnessAuthCache.js', () => ({ signInsChanged }))
import {
isProvisionalUserEmail,
@@ -321,3 +323,44 @@ describe('create — locally-created accounts', () => {
expect(findUnique).toHaveBeenCalledWith({ where: { email: '[email protected]' } })
})
})
+
+describe('a change to what a sign-in answers ends every pod\'s remembered sign-in (#3041)', () => {
+ const after = (write: { mock: { invocationCallOrder: number[] } }): void => {
+ expect(signInsChanged.mock.invocationCallOrder.at(-1)).toBeGreaterThan(write.mock.invocationCallOrder.at(-1)!)
+ }
+
+ it('an SSO sign-in that rewrites the email or the role, after the write', async () => {
+ findUnique.mockResolvedValue(row({ email: '[email protected]' }))
+ await userService.upsertFromSso({ externalId: 'prod-sub-1', email: '[email protected]', autonomousEnv: 'prod' })
+ expect(signInsChanged).toHaveBeenCalledWith('u1')
+ after(update)
+ })
+
+ it('an account already current writes nothing and moves nothing: SSO auth runs on every request', async () => {
+ findUnique.mockResolvedValue(row())
+ await userService.upsertFromSso({ externalId: 'prod-sub-1', email: '[email protected]', autonomousEnv: 'prod' })
+ expect(signInsChanged).not.toHaveBeenCalled()
+ })
+
+ it('a provisional row merged away, after it is deleted', async () => {
+ findUnique.mockResolvedValueOnce(row({ id: 'real', externalId: 'local-prod-real' }))
+ .mockResolvedValueOnce(row({ id: 'pending', email: provisionalEmailForExternalId('prod-sub-1') }))
+ await userService.upsertFromSso({ externalId: 'prod-sub-1', email: '[email protected]', autonomousEnv: 'prod' })
+ expect(signInsChanged).toHaveBeenCalledWith('pending')
+ expect(signInsChanged.mock.invocationCallOrder[0]).toBeGreaterThan(remove.mock.invocationCallOrder[0]!)
+ })
+
+ it('an update of the role or the plane, after the write', async () => {
+ findUnique.mockResolvedValue(row())
+ await userService.update('u1', { role: 'admin', autonomousEnv: 'stag' })
+ expect(signInsChanged).toHaveBeenCalledWith('u1')
+ after(update)
+ })
+
+ it('removing an account, after the delete', async () => {
+ findUnique.mockResolvedValue(row())
+ await userService.remove('u1')
+ expect(signInsChanged).toHaveBeenCalledWith('u1')
+ after(remove)
+ })
+})
diff --git a/backend/src/services/UserService.ts b/backend/src/services/UserService.ts
index d81376f05..d1fb225a4 100644
--- a/backend/src/services/UserService.ts
+++ b/backend/src/services/UserService.ts
@@ -3,6 +3,7 @@ import { randomUUID } from 'crypto'
import { prisma } from '../lib/prisma.js'
import { ADMIN_EMAILS } from '../config/env.js'
import { hashPassword } from '../lib/password.js'
+import { signInsChanged } from '../lib/harnessAuthCache.js'
import { ConflictError, NotFoundError } from '../errors/index.js'
import type { AutonomousEnvironment } from '../lib/autonomousEnvironment.js'
import type { SignInAttribution } from '../lib/signInAttribution.js'
@@ -56,6 +57,7 @@ async function mergeProdProvisionalUser(provisional: User, user: User): Promise<
}),
prisma.user.delete({ where: { id: provisional.id } }),
])
+ await signInsChanged(provisional.id)
}
export const userService = {
@@ -125,10 +127,13 @@ export const userService = {
// SSO profile auth runs on every control-plane request. Avoid turning that validation into a
// write-per-request DB hot path when the mirrored user record is already current.
if (Object.keys(data).length === 0) return existing
- return prisma.user.update({
+ const updated = await prisma.user.update({
where: { id: existing.id },
data,
})
+ // Its email or role may have changed: a Harness sign-in each pod remembers answers them (lib/harnessAuthCache.ts).
+ await signInsChanged(existing.id)
+ return updated
}
const data = {
@@ -259,7 +264,7 @@ export const userService = {
async update(id: string, input: { name?: string; password?: string; role?: string; autonomousEnv?: AutonomousEnvironment }): Promise<User> {
if (!(await prisma.user.findUnique({ where: { id } }))) throw new NotFoundError('User')
- return prisma.user.update({
+ const updated = await prisma.user.update({
where: { id },
data: {
...(input.name !== undefined ? { name: input.name } : {}),
@@ -268,10 +273,13 @@ export const userService = {
...(input.password ? { passwordHash: hashPassword(input.password) } : {}),
},
})
+ await signInsChanged(id)
+ return updated
},
async remove(id: string): Promise<void> {
if (!(await prisma.user.findUnique({ where: { id } }))) throw new NotFoundError('User')
await prisma.user.delete({ where: { id } })
+ await signInsChanged(id)
},
}
diff --git a/docs/plans/2026-11-05-harness-sign-in-cache-plan.md b/docs/plans/2026-11-05-harness-sign-in-cache-plan.md
new file mode 100644
index 000000000..88852d411
--- /dev/null
+++ b/docs/plans/2026-11-05-harness-sign-in-cache-plan.md
@@ -0,0 +1,92 @@
+# Harness sign-in cache per pod — plan (task #3041)
+
+**Goal:** Stop reading the session row and the user row from Mongo on every request signed in with a Harness session.
+Atlas's per-collection counters on 11-05 show `harness_sessions` at 47.2 reads/s and `users` at 52.8/s on the primary
+(River, from Ken's table), about 16 ops per backend request. Signing out, revoking and removing a computer must still
+take effect at once, on every pod.
+
+**Architecture:** `authenticateHarnessAccessToken` (lib/harnessSession.ts) remembers a good sign-in for 30 seconds in a
+bounded per-pod map keyed by the access token's hash. Every request still reads its access token from Redis, as today;
+the cached path reads the person's **sign-in epoch** in the same Redis call (`MGET`). Every write that ends a session or
+changes what a sign-in answers sets that person's epoch to a new random value after the database write. A cached sign-in
+whose epoch moved is dropped and read from Mongo again.
+
+**Tech Stack:** Node 22, Prisma 5.22 / MongoDB, ioredis, Vitest.
+
+**Spec:** River's addition to task #3041 (11-05): a short in-memory cache per pod for the sign-in lookups, signing out and
+revoking clearing it at once, two security reviews, Atlas ops before and after.
+
+This plan is written and carried out by its Engineer, test first; there is no execution handoff.
+
+## The rules it keeps
+
+- **Revocation at once, on every pod.** The epoch is in Redis, not in the pod: a sign-out on pod A is seen by pod B's
+ next request, in the Redis read that request already makes.
+- **No lost sign-out in a race.** The miss path reads the epoch *before* it reads the session and the user, and writers
+ set it *after* their write. A sign-out that lands between the two reads leaves the remembered epoch stale, so the next
+ request reads Mongo again.
+- **An epoch never repeats.** It is a random value, not a counter: a key that expired and was written again cannot equal
+ a value a pod remembers. It lives an hour, far past the 30-second cache; an expired epoch only causes a miss.
+- **A failed epoch write fails the sign-out.** `signInsChanged` throws, so a revoke route answers an error rather than
+ "signed out" while other pods could still serve the session for up to 30 seconds. Retrying is safe: every revoke is
+ idempotent.
+- **Only success is cached**, and callers get a copy.
+- **Redis down is still an outage** (`AUTH_SERVICE_UNAVAILABLE`), cached or not.
+
+Writes that set the epoch: sign-out (`revokeHarnessSession`), revoking a device (`revokeHarnessSessionById`), a computer
+turning itself into a team machine (`convertComputer`), SSO refusing a renewal (`ssoSession.ts renew`), removing a
+computer that ends its team session (`MachineService`), and user rows changing what a sign-in answers: `UserService`
+update (role, plane), remove, a provisional row merged away, and an SSO sign-in that rewrites the email, role or subject.
+
+Also not changed: the one-time email migration at start (lib/db/migrate.ts) only lower-cases an email; a pod may
+answer the old casing for up to 30 seconds.
+
+Not changed: SSO-token requests (no Harness session) keep their path; they are about 5 of the 53 user reads a second.
+
+## Design
+
+- Growth: at most 20,000 entries per pod, oldest dropped first, each gone after 30 s; epoch keys expire after an hour.
+- Concurrency: none in the pod (one thread); across pods the epoch orders a sign-out against a cached read (above).
+- Read cost: a hit is one Redis `MGET` (today: one `GET`) and no Mongo read; a miss adds one Redis `GET`.
+- Scale: at 10x sign-ins the map's bound holds memory under ~10 MB; the first warning is the hit rate falling, seen as
+ `harness_sessions` reads per second on Atlas rising back.
+- Compatibility: no client, API or schema change; tokens are unchanged.
+- Migration + rollback: none; reverting removes the cache, and epoch keys expire by themselves.
+- Audience: the server checks every request as before; the cache only remembers its own answer.
+- Identity: the cached identity is the session's user, never the machine owner's login.
+- Failure: Redis down answers 503 as today; an epoch that cannot be written fails the sign-out loudly.
+- Platforms: backend only.
+
+## Global Constraints
+
+No behaviour change for a caller except fewer database reads. No new settings or switches. Two independent security
+reviews before merge (sign-in and tokens).
+
+## Review Focus
+
+- Every write that ends a Harness session or changes a user's sign-in answer sets the epoch, after the write.
+- The miss path's order: access record, epoch, then session and user.
+
+## Tasks
+
+### 1. The cache and the epoch (`backend/src/lib/harnessAuthCache.ts`)
+
+- [x] Failing tests: 30 s life, bound, fresh epochs that never repeat with an hour's life, a failed write throws.
+- [x] Code that passes them.
+
+### 2. Authentication uses it (`backend/src/lib/harnessSession.ts`)
+
+- [x] Failing tests: a second request reads no Mongo; a token gone from Redis stops; sign-out and revoke by id end a
+ remembered sign-in at once, with nothing cleared in the pod; a sign-out between the reads is not missed; read again
+ after 30 s; Redis down is an outage; no refusal remembered; callers get a copy.
+- [x] Code: the cached path, the miss path's order, epochs set by sign-out, revoke and conversion.
+
+### 3. Every other writer sets the epoch
+
+- [x] Failing tests: SSO refusing a renewal, removing a computer, and `UserService` update, remove, merge and an
+ SSO rewrite each set the person's epoch.
+- [x] Code.
+
+### 4. Before and after
+
+- [ ] Atlas ops/s and `harness_sessions` / `users` reads per second before and after the release, in Ken's thread.
diff --git a/docs/plans/2026-11-05-prisma-engine-cpu-plan.md b/docs/plans/2026-11-05-prisma-engine-cpu-plan.md
new file mode 100644
index 000000000..5fc5e12d7
--- /dev/null
+++ b/docs/plans/2026-11-05-prisma-engine-cpu-plan.md
@@ -0,0 +1,100 @@
+# Prisma engine CPU by query — plan (task #3041)
+
+**Goal:** Name which queries burn the Prisma engine's CPU (45% of process CPU on v1.4.7, Ken's 11-05 `cpu by thread`
+split), then move the top reads off the engine's heaviest path, so CPU per request reaches ≤ 38 ms and 3 pods hold peak.
+
+**Architecture:** Step 1 measures. A sampler in the one query hook (`lib/queryStats.ts withQueryStats`) times the engine
+threads' own CPU around queries that run alone, and a per-minute `prisma by query` line ranks operations by engine ms.
+Step 2 moves the top reads, one group per pull request, chosen from that line. Step 3 reports before and after.
+
+**Tech Stack:** Node 22, Prisma 5.22 (library engine, in-process tokio threads), MongoDB, Vitest.
+
+**Spec:** the task's brief (#3041); Watch Production (#2773) thread of 11-05.
+
+This plan is written and carried out by its Engineer, test first; there is no execution handoff.
+
+## Why not Prisma's query events
+
+The brief named Prisma's `query` events. They do not measure what we need, and turning them on costs what we measure:
+
+- For MongoDB, an event's `duration` is the database call as the engine sees it (send, wait, reply). It does not include
+ the engine's own work: building the query, converting BSON to its values and those to the JSON string the client
+ parses. That is the CPU `cpu by thread` counts.
+- The `query` log level makes the engine format every query as a string. It cannot be sampled: it is on for every query
+ or for none.
+
+What we measure instead: Linux keeps each thread's time on CPU in nanoseconds (`/proc/self/task/<tid>/schedstat`, first
+field). When a query runs alone in the process, the engine threads' CPU between its start and its end is that query's
+engine CPU. The sampler takes such a query at most every 100 ms, reads the engine threads before and after (two
+directory reads and a few small file reads, tens of microseconds), and scales each operation's mean by how often it ran
+in the minute. Where `schedstat` is missing it falls back to `stat`'s 10 ms ticks, unbiased over many samples; off Linux
+the line says `unavailable`.
+
+## Raw commands are still the engine
+
+`$runCommandRaw` and `aggregateRaw` (resolveIds, the unfollow update, `parts/indexed.ts readBy`) run inside the same
+engine: it still converts the command to BSON and every reply document to Extended JSON and then to a string. They skip
+Prisma's query building and per-field result shaping, so they may cost less per document, but not nothing. The new line
+measures raw commands too, so step 2 compares engine µs per document of a model read against a raw read on the same
+collection before moving anything. If raw reads are about as costly, the fix is the plain `mongodb` driver for the top
+reads; that is a second connection pool per pod against Atlas M20's connection limit and more memory, and it goes to the
+PM with those costs before any code (River, 11-05).
+
+## Design
+
+- Growth: a log line per pod-minute, the top 25 keys; nothing stored. Logs rotate.
+- Concurrency: none; one process, counters on the main thread.
+- Read cost: at most 10 samples a second, each two `/proc` listings and the engine threads' `schedstat`; the line reports
+ the sampler's own time (`costMs`) so its cost is measured, not assumed.
+- Scale: at 10x queries the sample rate stays capped; keys stay capped (`maxKeys`, overflow into `other`).
+- Compatibility: a new log line; no client, API or schema change.
+- Migration + rollback: none; revert the commit.
+- Audience: server logs only; ops and routes are patterns, never values or ids.
+- Identity: N/A because nothing is read or written for a person.
+- Failure: a sampler error never reaches a query; it is warned about once per process, and `/proc` missing says
+ `unavailable` on every line.
+- Platforms: Linux pods; macOS dev logs `unavailable`.
+
+## Global Constraints
+
+No behaviour change; no new settings or switches. Swarm code stays out of `cli/src/core`. Each moved query keeps its
+results and its index, with an explain test (backend AGENTS.md rule 3). One pull request per group of queries.
+
+## Review Focus
+
+- The sampler's solo rule: a sample counts only if no other query was in flight at its start and none started before it
+ ended; the in-flight count returns to zero after a failed query.
+- Its cost bound (gap between samples) and that a failure inside it never changes a query's result.
+
+## Tasks
+
+### 1. Engine CPU reader and sampler (`backend/src/lib/engineCpu.ts`)
+
+Files: `src/lib/engineCpu.ts`, `src/lib/engineCpu.test.ts`.
+Interfaces: `readEngineCpuNs(root, pid)`, `createEngineCpu(options)` → `{ start(), end(sample, model, action, args,
+result), flush() }`, `startEngineCpu(options)`, `activeEngineCpu()`.
+
+- [ ] Failing tests: reads engine threads only, from `schedstat`, falling back to `stat`; a solo query is sampled and
+ scaled by its count; an overlapped one is discarded; samples respect the gap; the line ranks by engine ms with op
+ and route, reports totals and cost, and resets.
+- [ ] Code that passes them.
+
+### 2. The query hook feeds it (`backend/src/lib/queryStats.ts`)
+
+Files: `src/lib/queryStats.ts`, `src/lib/queryStats.test.ts`, `src/lib/prisma.ts`, `src/server.ts`.
+
+- [ ] Failing tests: every query, a failed one too, starts and ends one sample slot (in flight back to zero); a sampler
+ that throws leaves the query's result alone.
+- [ ] Code: `withQueryStats` calls the sampler; both clients pass it (lib/prisma.ts); the API server starts it beside
+ `startQueryStats`, as it starts `cpu by thread` (the worker logs neither).
+
+### 3. Read production, pick the first group (after the release)
+
+- [ ] Read `prisma by query` beside `cpu by thread` at peak; post the top operations by engine ms, their engine µs per
+ document, and raw against model reads.
+- [ ] Open the next pull request for the top group behind GET changes and hosts readings/changes, with an explain test
+ for each moved query, or bring the plain driver's costs to the PM.
+
+### 4. Before and after
+
+- [ ] After each move ships: Prisma engine ms/min and CPU per request from `cpu by thread` and `prisma by query`.