The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(backend): hot SwarmGroup reads name their index (no query planning); closed pages and mentions read only what they return (#1994)
0xken152f7ae79fcbe6
diff --git a/backend/integration/bench/run.ts b/backend/integration/bench/run.tsindex 27d52a14a..f02feda47 100644--- a/backend/integration/bench/run.ts+++ b/backend/integration/bench/run.ts@@ -7,6 +7,7 @@ import { localDatabase } from './mongo.js'import { readProfile, renderProfile, startProfiling } from './profile.js'import { renderMarkdown } from './report.js'import { benchStamps } from './stamps.js'+import { listGroupIndexes } from '../../src/superTabs/parts/indexed.js'/** `npm run bench:swarm-state -- [options] [--profile]` (README.md). Writes `<driver>-<time>.json` and `.md` to the results* folder; with `--profile`, every database operation of the run goes through Mongo's profiler and the report adds what each@@ -16,6 +17,8 @@ async function main() {const { config, out, driver, cache } = parseArgs(argv.filter(arg => arg !== '--profile'))const folder = out ?? join(process.cwd(), 'integration/bench/results')const database = await localDatabase()+ // As each server does at startup (server.ts): hot reads name the indexes the database has.+ await listGroupIndexes(database.db)const stamps = await benchStamps(cache)try {if (profile) await startProfiling(database.db)diff --git a/backend/integration/superTabGroupPlans.test.ts b/backend/integration/superTabGroupPlans.test.tsnew file mode 100644index 000000000..fefa2c69d--- /dev/null+++ b/backend/integration/superTabGroupPlans.test.ts@@ -0,0 +1,422 @@+import { randomUUID } from 'node:crypto'+import { execFile } from 'node:child_process'+import { promisify } from 'node:util'+import { join } from 'node:path'+import { tmpdir } from 'node:os'+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import { MongoMemoryReplSet } from 'mongodb-memory-server-core'+import { PrismaClient, type Prisma } from '@prisma/client'+import type { AuthUser } from '../src/lib/ssoAuth.js'+import { SuperTabStore, transaction } from '../src/superTabs/store.js'+import { migrateSwarm } from '../src/superTabs/migration/migrate.js'+import { HostReads } from '../src/superTabs/read/host.js'+import { DIRECT, readLight } from '../src/superTabs/cachedReads.js'+import * as parts from '../src/superTabs/parts/index.js'+import { recentDormantGroups } from '../src/superTabs/leaderboard/read.js'+import { decisionsWaiting, listDecisions } from '../src/superTabs/decisions.js'+import { GROUP_INDEX, groupsBy, listGroupIndexes, retryStaleIndex } from '../src/superTabs/parts/indexed.js'+import { logger } from '../src/utils/logger.js'+import type { SuperTabState } from '../src/superTabs/model.js'+import { writeLegacySwarm } from '../scripts/swarm-parts/legacy.js'+import { NOW, owner as ownerActor, swarm } from '../src/superTabs/read/fixture.test-support.js'+import { PROFILE_STAGES, queryShape } from './bench/profile.js'++// Prod's slow log (2026-10-28): SwarmGroup reads spent ~200 ms each planning. Prisma sends its filters as $expr, and a filter+// several indexes could serve is planned again whenever the plan cache misses, a race of every candidate index that a big+// Swarm's rows make slow, and that can pick an index which reads every quiet conversation to return none. Each hot read here+// runs under Mongo's profiler with the plan cache emptied first: it must name its index (never planned, no $expr), read only+// the documents it returns, and return the same rows as before. A fresh loopback replica set; never DATABASE_URL.+let replica: MongoMemoryReplSet+let db: PrismaClient+let store: SuperTabStore+let clock = NOW+const owner: AuthUser = { sub: ownerActor.userId, email: ownerActor.email, autonomousEnv: 'prod', role: 'user' }+const MIGRATED = NOW - 60_000+const DAY = 86_400_000+/** A busy Swarm's quiet conversations: prod's read 446 documents to return none. */+const QUIET_LEAD = 300, QUIET_OTHER = 60, OPEN = 140++beforeAll(async () => {+ replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },+ replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })+ const uri = replica.getUri(`swarm_group_plans_test_${randomUUID().replaceAll('-', '_')}`)+ await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {+ env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,+ })+ db = new PrismaClient({ datasources: { db: { url: uri } } })+ await db.user.create({ data: { id: owner.sub, externalId: `fixture-${owner.sub}`, email: owner.email, autonomousEnv: 'prod' } })+ store = new SuperTabStore(db, () => clock)+})+afterAll(async () => {+ await db?.$disconnect()+ await replica?.stop()+})++/** A Swarm on two machines of the owner's: a Lead with a digest due, a worker running work on the other, a finished+ * conversation, a second project and a third with no conversations; then QUIET_LEAD and QUIET_OTHER quiet conversations,+ * the n-th last active n days ago (mod 28), and OPEN open ones without work. */+async function scene() {+ const [machineId, leadMachine] = [`m-${randomUUID().slice(0, 8)}`, `l-${randomUUID().slice(0, 8)}`]+ for (const id of [machineId, leadMachine]) {+ await db.machine.create({ data: { machineId: id, userId: owner.sub, apiKey: 'fixture-only', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })+ }+ const s = swarm(MIGRATED)+ s.contribute(machineId, 'swarm')+ const lead = s.add('lead-harness', 'lead', true, ownerActor, leadMachine, owner.sub)+ const other = s.run({ type: 'project.create', name: 'Other', visibility: 'public', brief: '' }).projectId as string+ const empty = s.run({ type: 'project.create', name: 'Empty', visibility: 'public', brief: '' }).projectId as string+ const web = s.group('Web')+ const worker = s.add('worker-harness', web, false, ownerActor, machineId, owner.sub)+ s.run({ type: 'work.assign', conversationId: web, agentId: worker })+ const done = s.group('Done work')+ // A Reviewer kept for more work, in the conversations that go quiet below: the Lead's poll reads those (store.ts fullDeliveries).+ const reviewer = s.add('reviewer-harness', done, false, ownerActor, machineId, owner.sub)+ s.edit(state => {+ state.projects![0]!.digest = { time: '00:00', timeZone: 'UTC' }+ Object.assign(state.agents.find(a => a.id === reviewer)!, { role: 'reviewer', workDecision: 'keep' })+ })+ const tabId = `tab-${randomUUID()}`+ const state = { ...structuredClone(s.state), id: tabId } as SuperTabState+ await writeLegacySwarm(db, { state, revision: 700, messages: [] })+ const report = await migrateSwarm(db, tabId, { now: () => MIGRATED })+ if (!report.ok) throw new Error(report.error)+ const { id: _id, ...row } = (await db.swarmGroup.findFirst({ where: { tabId, key: done } }))!+ const bulk = (prefix: string, n: number, projectId: string, dormant: boolean) => Array.from({ length: n }, (_, i) => {+ const key = `${prefix}${String(i).padStart(3, '0')}`+ return { ...row, id: `${tabId}:${key}`, key, projectId, dormant, ordinal: 100 + i, changedAt: new Date(MIGRATED - (i + 1) * 60_000),+ lastActivityAt: new Date(MIGRATED - (i % 28) * DAY), data: { ...row.data as object, id: key, projectId, title: key, brief: 'b'.repeat(2000) } }+ })+ const quietLead = bulk('ql', QUIET_LEAD, 'lead', true), quietOther = bulk('qo', QUIET_OTHER, other, true), open = bulk('op', OPEN, 'lead', false)+ await db.swarmGroup.createMany({ data: [...quietLead, ...quietOther, ...open] })+ // Every project's Lead conversation and the Leads' one: a host's light load always reads them (parts/light.ts).+ const leadConversations = [...state.projects!.map(p => p.leadConversationId), ...state.leadsConversationId ? [state.leadsConversationId] : []]+ const active = [...leadConversations, web, done, ...open.map(r => r.key)]+ return { tabId, machineId, lead, worker, reviewer, web, done, other, empty, quietLead, quietOther, leadConversations, active }+}++/** One read as the profiler saw it: its command and every getMore (they share a query hash). */+interface Read { shape: string; plans: string[]; multiPlanned: boolean; planningMicros: number; docsExamined: number; returned: number }+interface Row { ns: string; op: string; match?: unknown; planSummary?: string; docsExamined?: number; nreturned?: number; queryHash?: string+ fromMultiPlanner?: boolean; planningTimeMicros?: number; command?: { filter?: unknown; q?: unknown } }++/** The SwarmGroup reads `run` sends, with the plan cache emptied first: a read the planner would serve from its cache in a+ * test is planned again in prod whenever the cache misses. */+async function profiled(run: () => Promise<unknown>): Promise<Read[]> {+ await db.$runCommandRaw({ planCacheClear: 'SwarmGroup' })+ await db.$runCommandRaw({ profile: 0 })+ await db.$runCommandRaw({ drop: 'system.profile' }).catch(() => {})+ await db.$runCommandRaw({ create: 'system.profile', capped: true, size: 64 * 1024 * 1024 })+ await db.$runCommandRaw({ profile: 2, slowms: 0 })+ try {+ await run()+ } finally {+ await db.$runCommandRaw({ profile: 0 })+ }+ const stages = [{ $match: { ns: { $regex: '\\.SwarmGroup$' }, op: { $in: ['command', 'getmore', 'query'] } } },+ { $project: { ...PROFILE_STAGES[0]!.$project, op: 1, queryHash: 1, fromMultiPlanner: 1, planningTimeMicros: 1 } }]+ const rows = (await db.$runCommandRaw({ aggregate: 'system.profile', pipeline: stages, cursor: { batchSize: 100000 } }) as unknown as+ { cursor: { firstBatch: Row[] } }).cursor.firstBatch+ const reads = new Map<string, Read>()+ for (const row of rows) {+ const id = row.queryHash ?? `${reads.size}`+ const filter = row.match ?? row.command?.filter ?? row.command?.q+ const read = reads.get(id) ?? { shape: '', plans: [], multiPlanned: false, planningMicros: 0, docsExamined: 0, returned: 0 }+ if (filter !== undefined && !read.shape) read.shape = queryShape(filter)+ if (row.planSummary && !read.plans.includes(row.planSummary)) read.plans.push(row.planSummary)+ read.multiPlanned ||= !!row.fromMultiPlanner+ read.planningMicros += Number(row.planningTimeMicros ?? 0)+ read.docsExamined += Number(row.docsExamined ?? 0)+ read.returned += Number(row.nreturned ?? 0)+ reads.set(id, read)+ }+ return [...reads.values()]+}++const mentions = (shape: string, field: string) => shape.includes(`"$${field}"`) || shape.includes(`"${field}":`)+/** The reads whose filter mentions every field of `fields`. */+const where = (reads: Read[], fields: string[]) => reads.filter(r => fields.every(f => mentions(r.shape, f)))++/** Each read examined only the documents it returned. */+function expectOnlyReturned(reads: Read[]) {+ for (const read of reads) expect(read.docsExamined, JSON.stringify(read)).toBeLessThanOrEqual(read.returned)+}++/** Named its index: never raced against the others, never through $expr. */+function expectIndexed(reads: Read[]) {+ for (const read of reads) {+ expect(read.multiPlanned, JSON.stringify(read)).toBe(false)+ expect(read.shape, JSON.stringify(read)).not.toContain('$expr')+ }+}++describe('hot SwarmGroup reads name their index and read only what they return (prod slow log, 2026-10-28)', () => {+ let s: Awaited<ReturnType<typeof scene>>+ beforeAll(async () => {+ // Another Swarm of the same size: a read not bounded by its Swarm reads that one's rows too.+ await scene()+ s = await scene()+ // As each server does at startup (server.ts): reads name only the indexes the database has.+ await listGroupIndexes(db)+ })++ it('the host\'s light load reads the Lead conversations, running work and the named conversations, nothing else', async () => {+ let groups: string[] = []+ const reads = await profiled(async () => {+ const loaded = await db.$transaction(tx => parts.loadLight(tx, s.tabId, { groups: 'working', agentIds: [s.worker], groupIds: [s.done], machines: false }))+ groups = loaded.groups.map(g => g.key)+ })+ expect(groups.sort()).toEqual([...s.leadConversations, s.web, s.done].sort())+ expect(reads.length).toBeGreaterThan(0)+ expectIndexed(reads)+ const docs = reads.reduce((n, r) => n + r.docsExamined, 0)+ // The worker's conversation is also running work: read once per branch at most.+ expect(docs, JSON.stringify(reads)).toBeLessThanOrEqual(groups.length + 1)+ })++ it('a command loads the Swarm\'s open conversations and those it names, each read through its index', async () => {+ const quiet = s.quietLead[5]!.key+ for (const [named, expected] of [+ [{ groupIds: [quiet] }, [...s.active, quiet]],+ // An agent's conversations and a project's, quiet ones too (store.ts moreNamed, namedFor).+ [{ agentIds: [s.worker], projectIds: [s.other] }, [...s.active, ...s.quietOther.map(r => r.key)]],+ ] as const) {+ let keys: string[] = []+ const reads = await profiled(async () => { keys = (await db.$transaction(tx => parts.loadActive(tx, s.tabId, named))).groups.map(g => g.key) })+ expect(keys.sort()).toEqual([...new Set(expected)].sort())+ expectIndexed(reads)+ expectOnlyReturned(reads)+ }+ })++ it('the read cache\'s base loads the Swarm\'s open conversations through the dormancy index', async () => {+ let keys: string[] = []+ const reads = await profiled(async () => { keys = (await db.$transaction(tx => parts.loadReadBase(tx, s.tabId))).groups.map(g => g.key) })+ expect(keys.sort()).toEqual([...s.active].sort())+ expect(reads).toHaveLength(1)+ expectIndexed(reads)+ expectOnlyReturned(reads)+ })++ it('the mentions panel reads a tagged quiet conversation by id, never every quiet conversation', async () => {+ const tagged = s.quietLead[3]!.key+ await db.superTabMention.create({ data: { id: `${s.tabId}-tag`, tabId: s.tabId, personId: owner.sub, projectId: 'lead', conversationId: tagged,+ messageId: `${s.tabId}-tagged`, authorAgentId: s.lead, createdAt: new Date(Date.now() - 60_000) } })+ let listed: Array<{ conversationId: string }> = []+ const reads = await profiled(async () => { listed = (await store.mentions(owner, s.tabId, 'waiting')).mentions })+ expect(listed.map(m => m.conversationId)).toEqual([tagged])+ expectIndexed(reads)+ expectOnlyReturned(reads)+ // The open conversations and the tagged one, at most: never the Swarm's QUIET_LEAD + QUIET_OTHER quiet ones.+ expect(reads.reduce((n, r) => n + r.docsExamined, 0), JSON.stringify(reads)).toBeLessThanOrEqual(s.active.length + 1)+ })++ it('a person\'s decisions read the quiet conversations they wait in by id', async () => {+ const quiet = s.quietLead[4]!.key+ await db.superTabDecision.create({ data: { id: `${s.tabId}-ask`, tabId: s.tabId, batchId: 'b', projectId: 'lead', conversationId: quiet, personId: owner.sub,+ askerAgentId: s.lead, question: 'Which?', options: ['A', 'B'], recommended: 0, unblocks: 'Work', createdAt: new Date(MIGRATED) } })+ const state = parts.reassemble(await db.$transaction(tx => parts.loadActive(tx, s.tabId)))+ const actor = { kind: 'person' as const, userId: owner.sub, email: owner.email }+ let count = 0, listed: Array<{ conversationId: string }> = []+ const reads = await profiled(async () => {+ count = (await db.$transaction(tx => decisionsWaiting(tx, s.tabId, state, actor))).count+ listed = (await db.$transaction(tx => listDecisions(tx, s.tabId, state, actor, 'open'))).decisions+ })+ expect(count).toBe(1)+ expect(listed.map(d => d.conversationId)).toEqual([quiet])+ expect(reads.length).toBeGreaterThan(0)+ expectIndexed(reads)+ expectOnlyReturned(reads)+ })++ // Folded from #1964: a later page started at no bound, so each page read every newer quiet conversation again.+ it('each page of a project\'s quiet conversations reads the 50 it returns, newest activity first', async () => {+ const order = [...s.quietLead].sort((a, b) => b.lastActivityAt.getTime() - a.lastActivityAt.getTime() || (a.key < b.key ? 1 : -1))+ const pages: string[][] = []+ const reads = await profiled(async () => {+ const first = await db.$transaction(tx => parts.loadDormant(tx, s.tabId, { projectId: 'lead' }))+ const last = order.find(r => r.key === first.at(-1)!.key)!+ const second = await db.$transaction(tx => parts.loadDormant(tx, s.tabId, { projectId: 'lead', before: { lastActivityAt: last.lastActivityAt.getTime(), key: last.key } }))+ pages.push(first.map(g => g.key), second.map(g => g.key))+ })+ expect(pages).toEqual([order.slice(0, 50).map(r => r.key), order.slice(50, 100).map(r => r.key)])+ expect(reads.length).toBeGreaterThan(0)+ expectIndexed(reads)+ expectOnlyReturned(reads)+ })++ it('the leaderboard reads a project\'s quiet conversations of the weeks it sums, and only those', async () => {+ const since = MIGRATED - 13 * DAY+ const recent = (rows: typeof s.quietLead) => rows.filter(r => r.lastActivityAt.getTime() >= since).map(r => r.key).sort()+ for (const [projectId, expected] of [['lead', recent(s.quietLead)], [s.other, recent(s.quietOther)], [s.empty, []]] as const) {+ let found: string[] = []+ const reads = await profiled(async () => { found = (await db.$transaction(tx => recentDormantGroups(tx, s.tabId, projectId, since))).map(g => g.id).sort() })+ expect(found).toEqual(expected)+ expect(reads).toHaveLength(1)+ expectIndexed(reads)+ expect(reads[0]!.docsExamined, `${projectId} ${JSON.stringify(reads)}`).toBeLessThanOrEqual(expected.length)+ }+ })++ it('a Lead\'s poll with a digest due reads the quiet conversations its project finished since the last one, and those its Reviewers are in, by index', async () => {+ clock = MIGRATED + 60_000+ const expected = s.quietLead.filter(r => r.lastActivityAt.getTime() >= clock - DAY).length+ const all = await profiled(() => store.deliveries(owner, s.tabId, s.lead))+ const reads = where(all, ['tabId', 'dormant', 'lastActivityAt'])+ expect(reads).toHaveLength(1)+ expectIndexed(reads)+ expect(reads[0]!.returned).toBe(expected)+ expect(reads[0]!.docsExamined, JSON.stringify(reads)).toBeLessThanOrEqual(expected)+ // The same poll reads the quiet conversations its project's kept Reviewer is in (every quiet one here), through the+ // Reviewer's conversations: no index has both.+ const asleep = where(all, ['tabId', 'dormant', 'agentIds'])+ expect(asleep).toHaveLength(1)+ expectIndexed(asleep)+ expect(asleep[0]!.returned).toBe(s.quietLead.length + s.quietOther.length)+ const its = await db.swarmGroup.count({ where: { tabId: s.tabId, agentIds: { has: s.reviewer } } })+ expect(asleep[0]!.docsExamined, JSON.stringify(asleep)).toBeLessThanOrEqual(its)+ })++ it('the host\'s wake check finds a change that concerns its machine, reading only the conversations changed since the cursor', async () => {+ const hosts = new HostReads(store, { now: () => clock })+ clock = Date.now()+ const first = await hosts.changes(owner, s.machineId, { full: true })+ // Ten conversations another machine's host was told about since: this host's wake check reads none of them (review of #1994).+ await db.swarmGroup.updateMany({ where: { tabId: s.tabId, key: { in: s.quietOther.slice(0, 10).map(r => r.key) } },+ data: { changedAt: new Date(Date.now() + 500), hostMachineIds: ['other-host'] } })+ clock = Date.now()+ let quiet: Awaited<ReturnType<HostReads['changes']>> | undefined+ const idle = where(await profiled(async () => { quiet = await hosts.changes(owner, s.machineId, { cursor: first.cursor }) }), ['tabId', 'changedAt', 'hostMachineIds'])+ expect(idle).toHaveLength(1)+ expectIndexed(idle)+ // None of its conversations changed for this machine since the cursor: it reads none, not even those changed for others.+ expect(idle[0]!.returned).toBe(0)+ expectOnlyReturned(idle)+ // A write that concerns this machine's host, as parts/commit.ts marks it.+ clock = Date.now() + 1000+ await db.swarmGroup.updateMany({ where: { tabId: s.tabId, key: s.web }, data: { changedAt: new Date(clock), hostMachineIds: [s.machineId] } })+ let woken: Awaited<ReturnType<HostReads['changes']>> | undefined+ const reads = where(await profiled(async () => { woken = await hosts.changes(owner, s.machineId, { cursor: quiet!.cursor }) }), ['tabId', 'changedAt', 'hostMachineIds'])+ expect(woken!.tabs.map(t => t.tabId)).toEqual([s.tabId])+ expect(reads).toHaveLength(1)+ expectIndexed(reads)+ expect(reads[0]!.returned).toBe(1)+ expectOnlyReturned(reads)+ })+})++describe('a hot read whose index the database lacks', () => {+ let s: Awaited<ReturnType<typeof scene>>+ beforeAll(async () => { s = await scene() })++ it('reads without it, in a transaction too, and says so; once the index is built it names it again', async () => {+ const read = () => db.$transaction(tx => groupsBy(tx, 'running', [{ $match: { tabId: s.tabId, executionMachineId: { $ne: null } } }, { $project: { _id: 0, key: 1 } }]))+ const said = vi.spyOn(logger, 'error')+ await db.$runCommandRaw({ dropIndexes: 'SwarmGroup', index: GROUP_INDEX.running })+ try {+ await listGroupIndexes(db)+ expect(said).toHaveBeenCalledWith(expect.stringContaining('lacks indexes'), { missing: [GROUP_INDEX.running] })+ expect(await read()).toEqual([{ key: s.web }])+ } finally {+ said.mockRestore()+ await db.$runCommandRaw({ createIndexes: 'SwarmGroup', indexes: [{ key: GROUP_INDEX.running, name: 'SwarmGroup_tabId_executionMachineId_idx' }] })+ }+ await listGroupIndexes(db)+ const reads = await profiled(async () => expect(await read()).toEqual([{ key: s.web }]))+ expect(reads).toHaveLength(1)+ expectIndexed(reads)+ })++ // Review of #1994: listed first, then removed (a rollback, an index replaced). The listing still names it until the next one.+ describe('removed after the last listing', () => {+ const match = () => [{ $match: { tabId: s.tabId, agentIds: s.worker } }, { $project: { _id: 0, key: 1 } }]+ /** Listed, then removed, then `run`; with `before`, that runs between the listing and the removal and hands `run` its result. */+ async function removed<T, B = undefined>(...steps: [run: () => Promise<T>] | [before: () => Promise<B>, run: (started: B) => Promise<T>]): Promise<T> {+ await listGroupIndexes(db)+ const started = steps.length === 2 ? await steps[0]() : undefined as B+ await db.$runCommandRaw({ dropIndexes: 'SwarmGroup', index: GROUP_INDEX.agents })+ try {+ return await (steps.length === 2 ? steps[1](started) : steps[0]())+ } finally {+ await db.$runCommandRaw({ createIndexes: 'SwarmGroup', indexes: [{ key: GROUP_INDEX.agents, name: 'SwarmGroup_tabId_agentIds_idx' }] })+ await listGroupIndexes(db)+ }+ }++ it('a read outside a transaction says so and answers without it at once', async () => {+ const said = vi.spyOn(logger, 'error')+ expect(await removed(() => groupsBy(db, 'agents', match()))).toEqual([{ key: s.web }])+ expect(said).toHaveBeenCalledWith(expect.stringContaining('lacks an index'), expect.anything())+ said.mockRestore()+ })++ it('a store transaction runs again without it and answers', async () => {+ expect(await removed(() => transaction(db, tx => groupsBy(tx, 'agents', match())))).toEqual([{ key: s.web }])+ })++ // Re-check of #1994: a listing whose reply was read before the removal lands after a read dropped the listing.+ it('a listing that started before the removal does not name it again once it lands', async () => {+ let land!: () => void+ const landed = new Promise<void>(resolve => { land = resolve })+ const slow = { $runCommandRaw: async (command: Prisma.InputJsonObject) => { const reply = await db.$runCommandRaw(command); await landed; return reply } } as unknown as PrismaClient+ const rows = await removed(async () => {+ const late = listGroupIndexes(slow)+ // The reply is read (index still there) before the removal below: give the command time to come back.+ await new Promise(resolve => setTimeout(resolve, 200))+ return { late }+ }, async ({ late }) => {+ expect(await groupsBy(db, 'agents', match())).toEqual([{ key: s.web }])+ land()+ await late+ // A transaction opened directly, no runner to run it again: it answers only if nothing names the removed index.+ return db.$transaction(tx => groupsBy(tx, 'agents', match()))+ })+ expect(rows).toEqual([{ key: s.web }])+ })++ it('a store transaction\'s rerun names no index, even if a listing names the removed one again meanwhile', async () => {+ let attempts = 0+ const rows = await removed(async () => transaction(db, async tx => { attempts++; return groupsBy(tx, 'agents', match()) }, async () => {+ // Between the attempts the index comes back, is listed, and goes again (an index rebuilt during a rollout).+ await db.$runCommandRaw({ createIndexes: 'SwarmGroup', indexes: [{ key: GROUP_INDEX.agents, name: 'SwarmGroup_tabId_agentIds_idx' }] })+ await listGroupIndexes(db)+ await db.$runCommandRaw({ dropIndexes: 'SwarmGroup', index: GROUP_INDEX.agents })+ }))+ expect(rows).toEqual([{ key: s.web }])+ expect(attempts).toBe(2)+ })++ // The reviewer's probe (re-check of #1994): an older listing reply, read before the removal, lands after the failed attempt.+ it('a directly opened transaction\'s rerun names no index when an older listing reply lands before it', async () => {+ let attempts = 0+ const rows = await removed(async () => ({ older: await db.$runCommandRaw({ listIndexes: 'SwarmGroup' }) }), ({ older }) => retryStaleIndex(async () => {+ attempts+++ try {+ return await db.$transaction(tx => groupsBy(tx, 'agents', match()))+ } finally {+ if (attempts === 1) await listGroupIndexes({ $runCommandRaw: async () => older } as unknown as PrismaClient)+ }+ }))+ expect(rows).toEqual([{ key: s.web }])+ expect(attempts).toBe(2)+ })++ it('a directly opened transaction\'s rerun names no index, even if a listing names the removed one again meanwhile', async () => {+ let attempts = 0+ const rows = await removed(() => retryStaleIndex(async () => {+ if (++attempts === 2) {+ await db.$runCommandRaw({ createIndexes: 'SwarmGroup', indexes: [{ key: GROUP_INDEX.agents, name: 'SwarmGroup_tabId_agentIds_idx' }] })+ await listGroupIndexes(db)+ await db.$runCommandRaw({ dropIndexes: 'SwarmGroup', index: GROUP_INDEX.agents })+ }+ return db.$transaction(tx => groupsBy(tx, 'agents', match()))+ }))+ expect(rows).toEqual([{ key: s.web }])+ expect(attempts).toBe(2)+ })++ it('the read cache\'s own transaction runs again without it and answers', async () => {+ const loaded = await removed(() => readLight(db, DIRECT, s.tabId, { groups: 'working', agentIds: [s.worker] }))+ expect(loaded.groups.map(g => g.key).sort()).toEqual([...s.leadConversations, s.web].sort())+ })+ })+})diff --git a/backend/integration/superTabLiveness.test.ts b/backend/integration/superTabLiveness.test.tsindex c142564fd..feadba38d 100644--- a/backend/integration/superTabLiveness.test.ts+++ b/backend/integration/superTabLiveness.test.ts@@ -596,7 +596,8 @@ describe('the delivery poll of an agent with nothing waiting', () => {counted.ops.length = 0f.clock.now += 1expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])- expect(counted.ops).toContain('SwarmGroup.findMany')+ // The Swarm's groups load through their indexes (parts/load.ts).+ expect(counted.ops).toContain('SwarmGroup.aggregateRaw')})it('still runs the Lead\'s time checks: the five-minute update request arrives once 30 seconds have passed', async () => {@@ -641,7 +642,8 @@ describe('the delivery poll of an agent with nothing waiting', () => {counted.ops.length = 0f.clock.now += 1_000expect(await f.timed.deliveries(host, f.tab, f.lead)).toEqual([])- expect(counted.ops).toContain('SwarmGroup.findMany')+ // The Swarm's groups load through their indexes (parts/load.ts).+ expect(counted.ops).toContain('SwarmGroup.aggregateRaw')// A stranger still meets the authority checks, never a remembered answer.await expect(f.timed.deliveries(owner, f.tab, f.lead)).rejects.toMatchObject({ code: 'FORBIDDEN' })})diff --git a/backend/integration/superTabRead.test.ts b/backend/integration/superTabRead.test.tsindex 436c7a825..c81507ee1 100644--- a/backend/integration/superTabRead.test.ts+++ b/backend/integration/superTabRead.test.ts@@ -818,6 +818,7 @@ describe('decisions asked of one person (v2)', () => {const waiting = await counted.$transaction(async tx => {const state = parts.reassemble(await parts.loadActive(tx, s.tabId))whole.length = 0+ quiet.length = 0return decisionsWaiting(tx, s.tabId, state, { kind: 'person', userId: peer.sub, email: peer.email })})expect(waiting.count).toBe(150)diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prismaindex 39227705f..afb277527 100644--- a/backend/prisma/schema.prisma+++ b/backend/prisma/schema.prisma@@ -1484,10 +1484,13 @@ model SwarmGroup {@@index([tabId, changedAt])@@index([tabId, dormant, changedAt])@@index([tabId, agentIds])- @@index([tabId, projectId, dormant, lastActivityAt])+ /// A project's quiet conversations by activity (leaderboard, digest), and their pages newest first, then by key (parts/load.ts loadDormant).+ @@index([tabId, projectId, dormant, lastActivityAt, key])@@index([tabId, projectId, dormant, reportSeq])@@index([dormant, lastActivityAt])@@index([workFinished, archived, lastActivityAt])+ /// A host's wake check (read/host.ts woken): the conversations its last writes concerned, by Swarm, since its cursor.+ @@index([hostMachineIds, tabId, changedAt])/// A Swarm's running work (parts/light.ts, the host's light load).@@index([tabId, executionMachineId])/// The host signature (read/host.ts inputs) reads every conversation's digest and dormancy from this index alone.diff --git a/backend/src/server.ts b/backend/src/server.tsindex 843023de0..41e78fc7f 100644--- a/backend/src/server.ts+++ b/backend/src/server.ts@@ -23,6 +23,7 @@ import { closeBus, pub, redisCount, redisSsoProfileStore, redisSwarmStamps } froimport { prisma } from './lib/prisma.js'import { backendReadinessChecks, createReadiness, drainBeforeClosing, processDrain } from './lib/readiness.js'import { configureSwarmStamps } from './superTabs/cache.js'+import { watchGroupIndexes } from './superTabs/parts/indexed.js'import { useSharedSsoProfileStore } from './lib/ssoAuth.js'import { useSharedRateCounter } from './lib/rateLimit.js'import { createRouteRateLimit } from './middlewares/routeRateLimit.js'@@ -150,6 +151,8 @@ async function start(): Promise<void> {// Rate limits count across every process and pod (lib/rateLimit.ts), not once per process.useSharedRateCounter(redisWindowCounter(pub))configureSwarmStamps(redisSwarmStamps(), { reads: env.SWARM_READ_CACHE === 'on', maxBytes: env.SWARM_CACHE_MB * 2 ** 20 })+ // Hot Swarm reads name their indexes once the database is seen to have them (superTabs/parts/indexed.ts).+ watchGroupIndexes(prisma)// SSO access-token gate for the control API (data-plane auth happens before Fastify), inside the per-route limit// (Stability 3.2): a request counts for the person it signed in as, and a token already refused this window isdiff --git a/backend/src/superTabs/cachedReads.ts b/backend/src/superTabs/cachedReads.tsindex 707853903..43454f0b8 100644--- a/backend/src/superTabs/cachedReads.ts+++ b/backend/src/superTabs/cachedReads.ts@@ -10,6 +10,7 @@ import { CacheView } from './cache.js'import * as store from './parts/index.js'import type { GroupData, Part, SwarmParts } from './parts/index.js'import { startFinished } from './parts/write.js'+import { retryStaleIndex } from './parts/indexed.js'type Db = Pick<PrismaClient, '$transaction'> & Prisma.TransactionClienttype LightOptions = NonNullable<Parameters<typeof store.loadLight>[2]>@@ -30,7 +31,7 @@ export const readMachines = (db: Prisma.TransactionClient, view: CacheView, tabI/** The active parts, then the machines: one read of the base in a transaction, so its parts are one snapshot. */const base = (db: Db, view: CacheView, tabId: string) => Promise.all([- view.get(tabId, 'parts', 'main', () => db.$transaction(tx => store.loadReadBase(tx, tabId))),+ view.get(tabId, 'parts', 'main', () => retryStaleIndex(() => db.$transaction(tx => store.loadReadBase(tx, tabId)))),readMachines(db, view, tabId),])@@ -41,13 +42,13 @@ export async function readParts(db: Db, view: CacheView, tabId: string, named: Rconst extra: Part<GroupData>[] = []const add = (groups: Part<GroupData>[]) => { for (const g of groups) if (!held.has(g.key)) { held.add(g.key); extra.push(g) } }if (named.allGroups || named.agentIds?.length || named.projectIds?.length) {- const dormant = await view.get(tabId, 'dormant', 'main', () => store.loadGroups(db, tabId, { dormant: true }))+ const dormant = await view.get(tabId, 'dormant', 'main', () => store.loadGroups(db, tabId, 'dormancy', { dormant: true }))const agents = new Set(named.agentIds ?? []), projects = new Set(named.projectIds ?? [])// A group stored with no project belongs to the default one (projects.ts groupProject).add(named.allGroups ? dormant : dormant.filter(g => g.data!.agentIds.some(id => agents.has(id)) || projects.has(g.data!.projectId ?? 'lead')))}for (const id of named.groupIds ?? []) {- if (!held.has(id)) add(await view.get(tabId, `group:${id}`, 'main', () => store.loadGroups(db, tabId, { key: id })))+ if (!held.has(id)) add(await view.get(tabId, `group:${id}`, 'main', () => store.loadGroups(db, tabId, 'key', { key: id })))}const startIds = new Set(named.startIds ?? []), startAgents = new Set(named.startAgentIds ?? [])const agentOf = (start: AgentStart) => start.harnessId ? stableId(start.request.machineId, start.harnessId) : null@@ -61,7 +62,7 @@ export async function readLight(db: Db, view: CacheView, tabId: string, opts: Omconst name = `light:${JSON.stringify([opts.groups ?? 'active', !!opts.includeRecentStarts, ...(['groupIds', 'agentIds', 'projectIds'] as const).map(key => [...opts[key] ?? []].sort())])}`const [light, machines] = await Promise.all([- view.get(tabId, name, 'main', () => db.$transaction(tx => store.loadLight(tx, tabId, { ...opts, machines: false }))),+ view.get(tabId, name, 'main', () => retryStaleIndex(() => db.$transaction(tx => store.loadLight(tx, tabId, { ...opts, machines: false })))),readMachines(db, view, tabId),])return { ...light, machines }diff --git a/backend/src/superTabs/decisions.ts b/backend/src/superTabs/decisions.tsindex 6d074f0e2..e288642ff 100644--- a/backend/src/superTabs/decisions.ts+++ b/backend/src/superTabs/decisions.ts@@ -11,6 +11,7 @@ import { AGENT_ONLY, fail, isLeads, person, stableId, type Actor, type Author, timport { canReadGroup, groupProject, ledProject, projectOf } from './projects.js'import { LEDGER_WEEKS } from './leaderboard/record.js'import { WEEK_MS } from './leaderboard/week.js'+import { groupsBy } from './parts/indexed.js'type Tx = Prisma.TransactionClientexport const MAX_QUESTIONS = 10@@ -200,10 +201,9 @@ async function conversationsOf(tx: Tx, tabId: string, state: SuperTabState, ids:const found = new Map(state.conversations.filter(c => named.has(c.id)).map(c => [c.id, c]))const missing = [...named].filter(id => !found.has(id))if (!missing.length) return found- const rows = only- ? await tx.swarmGroup.aggregateRaw({ pipeline: [{ $match: { tabId, deleted: false, key: { $in: missing } } },- { $project: { _id: 0, key: 1, ...Object.fromEntries(only.map(field => [`data.${field}`, 1])) } }] }) as unknown as Array<{ key: string; data: unknown }>- : await tx.swarmGroup.findMany({ where: { tabId, key: { in: missing }, deleted: false }, select: { key: true, data: true } })+ // By id through its index (parts/indexed.ts): a person's read counts these on each poll.+ const rows = await groupsBy(tx, 'key', [{ $match: { tabId, deleted: false, key: { $in: missing } } },+ { $project: { _id: 0, key: 1, ...only ? Object.fromEntries(only.map(field => [`data.${field}`, 1])) : { data: 1 } } }]) as unknown as Array<{ key: string; data: unknown }>for (const row of rows) found.set(row.key, row.data as Conversation)return found}diff --git a/backend/src/superTabs/leaderboard/read.test.ts b/backend/src/superTabs/leaderboard/read.test.tsindex 78e84ebe6..e5eec0582 100644--- a/backend/src/superTabs/leaderboard/read.test.ts+++ b/backend/src/superTabs/leaderboard/read.test.ts@@ -19,7 +19,7 @@ describe('the dormant groups the leaderboard sums', () => {const { tx, asked } = reading([{ data: { id: 'g1', requesterId: 'peer', ledger: { shippedAt: since + 1 }, assignment: { id: 'a1', status: 'done' } } }])expect(await recentDormantGroups(tx, 'tab', 'web', since)).toEqual([{ id: 'g1', requesterId: 'peer', ledger: { shippedAt: since + 1 }, assignment: { id: 'a1', status: 'done' } }])const [{ pipeline }] = asked as [{ pipeline: Array<Record<string, Record<string, unknown>>> }]- expect(pipeline[0]!.$match).toEqual({ tabId: 'tab', deleted: false, dormant: true, lastActivityAt: { $gte: { $date: new Date(since).toISOString() } }, projectId: 'web' })+ expect(pipeline[0]!.$match).toEqual({ tabId: 'tab', deleted: false, dormant: true, lastActivityAt: { $gte: { $date: new Date(since).toISOString() } }, projectId: { $in: ['web'] } })expect(Object.keys(pipeline[1]!.$project!).sort()).toEqual(['_id', ...['archived', 'assignment.id', 'assignment.status', 'audience', 'id', 'kind', 'ledger','projectId', 'pullRequestUrls', 'requesterId', 'reviewOf', 'title'].map(field => `data.${field}`)].sort())})@@ -27,7 +27,7 @@ describe('the dormant groups the leaderboard sums', () => {it('reads the default project\'s groups stored with no project too', async () => {const { tx, asked } = reading([])await recentDormantGroups(tx, 'tab', 'lead', since)- expect((asked[0] as { pipeline: Array<Record<string, Record<string, unknown>>> }).pipeline[0]!.$match).toMatchObject({ $or: [{ projectId: 'lead' }, { projectId: null }] })+ expect((asked[0] as { pipeline: Array<Record<string, Record<string, unknown>>> }).pipeline[0]!.$match).toMatchObject({ projectId: { $in: ['lead', null] } })})})diff --git a/backend/src/superTabs/leaderboard/read.ts b/backend/src/superTabs/leaderboard/read.tsindex 754dfed1d..f1f42aa47 100644--- a/backend/src/superTabs/leaderboard/read.ts+++ b/backend/src/superTabs/leaderboard/read.ts@@ -2,6 +2,7 @@import type { Prisma } from '@prisma/client'import type { Conversation } from '../model.js'import { fail } from '../model.js'+import { groupsBy, quietSince } from '../parts/indexed.js'import { LEDGER_WEEKS } from './record.js'import { WEEK_MS, weekStart } from './week.js'@@ -22,13 +23,11 @@ const BOARD_FIELDS = ['id', 'kind', 'projectId', 'title', 'audience', 'requester/** Dormant groups of one project active since [since]: a request shipped in a week the board sums wrote its Done row then.* Only the fields the board reads: every open board reads them once a minute (Diego's issue file, 2026-10-27). */export async function recentDormantGroups(tx: Prisma.TransactionClient, tabId: string, projectId: string, since: number): Promise<Conversation[]> {- // A group stored with no project belongs to the default one (projects.ts groupProject), as in a Swarm from before projects;- // a null match is a missing field too.- const project = projectId === 'lead' ? { $or: [{ projectId: 'lead' }, { projectId: null }] } : { projectId }- const rows = await tx.swarmGroup.aggregateRaw({ pipeline: [- { $match: { tabId, deleted: false, dormant: true, lastActivityAt: { $gte: { $date: new Date(since).toISOString() } }, ...project } },+ // Through the project index by name (parts/indexed.ts): planned, this read could take the dormancy one and read every quiet+ // conversation of the Swarm to return none.+ const rows = await groupsBy(tx, 'project', [{ $match: quietSince(tabId, projectId, since) },{ $project: { _id: 0, ...Object.fromEntries(BOARD_FIELDS.map(field => [`data.${field}`, 1])) } },- ] }) as unknown as Array<{ data: Conversation }>+ ]) as unknown as Array<{ data: Conversation }>return rows.map(row => row.data)}diff --git a/backend/src/superTabs/mentions.ts b/backend/src/superTabs/mentions.tsindex ae76b9959..f0877b130 100644--- a/backend/src/superTabs/mentions.ts+++ b/backend/src/superTabs/mentions.ts@@ -4,6 +4,7 @@import type { Prisma, PrismaClient } from '@prisma/client'import { fail, isLeads, person, stableId, type Actor, type Conversation, type MessageDraft, type SuperTabState } from './model.js'import { canReadGroup, groupProject, ledProject, projectOf } from './projects.js'+import { groupsBy } from './parts/indexed.js'type Tx = Prisma.TransactionClientexport const MAX_MENTIONS = 5@@ -79,8 +80,9 @@ async function readable(tx: Tx, tabId: string, state: SuperTabState, actor: Extrconst groups = new Map(state.conversations.map(c => [c.id, c]))const missing = [...new Set(rows.map(r => r.conversationId))].filter(id => !groups.has(id))if (missing.length) {- for (const row of await tx.swarmGroup.findMany({ where: { tabId, deleted: false, key: { in: missing } }, select: { key: true, data: true } })) {- groups.set(row.key, row.data as unknown as Conversation)+ // By id through its index (parts/indexed.ts): every person's read counts these on each poll.+ for (const row of await groupsBy(tx, 'key', [{ $match: { tabId, deleted: false, key: { $in: missing } } }, { $project: { _id: 0, key: 1, data: 1 } }])) {+ groups.set(row.key as string, row.data as unknown as Conversation)}}return rows.flatMap(row => {diff --git a/backend/src/superTabs/parts/indexed.ts b/backend/src/superTabs/parts/indexed.tsnew file mode 100644index 000000000..441a99e5c--- /dev/null+++ b/backend/src/superTabs/parts/indexed.ts@@ -0,0 +1,103 @@+/**+ * Hot SwarmGroup reads that name their index (prod slow log, 2026-10-28: ~200 ms of each read was query planning). Prisma+ * sends its filters as $expr, and a filter several indexes could serve is planned again whenever the plan cache misses: a+ * race of every candidate index over the Swarm's rows, which a bigger Swarm repeats by evicting the cached plan, and which+ * can pick an index that reads every quiet conversation to return none. A named index is never planned.+ * integration/superTabGroupPlans.test.ts holds each read to it under Mongo's profiler.+ */+import { AsyncLocalStorage } from 'node:async_hooks'+import type { Prisma, PrismaClient } from '@prisma/client'+import { logger } from '../../utils/logger.js'++type Transaction = Prisma.TransactionClient++/** The SwarmGroup indexes hot reads name, as prisma/schema.prisma declares them. */+export const GROUP_INDEX = {+ key: { tabId: 1, key: 1 },+ running: { tabId: 1, executionMachineId: 1 },+ agents: { tabId: 1, agentIds: 1 },+ project: { tabId: 1, projectId: 1, dormant: 1, lastActivityAt: 1, key: 1 },+ dormancy: { tabId: 1, dormant: 1, changedAt: 1 },+ changes: { tabId: 1, changedAt: 1 },+ /** A host's changed conversations: led by the host, so the wake check reads no other host's. */+ wake: { hostMachineIds: 1, tabId: 1, changedAt: 1 },+ /** Every live group: the host signature's index. */+ live: { tabId: 1, deleted: 1, key: 1, hostDigest: 1, dormant: 1 },+} as const+export type GroupIndex = keyof typeof GROUP_INDEX++/** The indexes SwarmGroup had when last listed; none until then. A read names only these: naming one the database lacks+ * fails the read, and in a transaction aborts it. */+let present = new Set<GroupIndex>()++/** Lists SwarmGroup's indexes, outside any transaction, and says which hot reads go without theirs. The worker's+ * `prisma db push` builds a new index at startup (Dockerfile.k8s), possibly after a server first lists them. */+export async function listGroupIndexes(db: PrismaClient): Promise<void> {+ const generation = drops+ const reply = await db.$runCommandRaw({ listIndexes: 'SwarmGroup' }) as unknown as { cursor: { firstBatch: Array<{ key: Record<string, unknown> }> } }+ // A read dropped the listing while this one was on its way: its reply may predate the removal (review of #1994).+ if (drops !== generation) return+ const fields = new Set(reply.cursor.firstBatch.map(index => Object.keys(index.key).join(',')))+ const names = Object.keys(GROUP_INDEX) as GroupIndex[]+ present = new Set(names.filter(name => fields.has(Object.keys(GROUP_INDEX[name]).join(','))))+ const missing = names.filter(name => !present.has(name))+ if (missing.length) logger.error('[swarm-reads] SwarmGroup lacks indexes hot reads name; they read without them', { missing: missing.map(name => GROUP_INDEX[name]) })+}++/** Lists the indexes now and every five minutes, until the process ends. */+export function watchGroupIndexes(db: PrismaClient): void {+ const list = () => listGroupIndexes(db).catch(error => logger.warn('[swarm-reads] listing SwarmGroup indexes failed', { error: error instanceof Error ? error.message : String(error) }))+ void list()+ setInterval(list, 5 * 60_000).unref()+}++/** How many times a read dropped the listing. A transaction a dropped index aborted fails in whichever of its reads Mongo+ * answers first, often not the one that named it: its runner reruns it when this moved during the attempt. */+let drops = 0+export const listingDrops = (): number => drops++/** Set while a transaction a dropped index aborted runs again: its reads name no index, whatever a listing since says. */+const rerun = new AsyncLocalStorage<true>()+export const withoutHints = <T>(run: () => Promise<T>): Promise<T> => rerun.run(true, run)++/** Runs a transaction opened directly (outside store.ts `transaction`) once more when a read in it dropped the listing:+ * the second attempt names no index (`withoutHints`). */+export async function retryStaleIndex<T>(run: () => Promise<T>): Promise<T> {+ const before = drops+ try {+ return await run()+ } catch (error) {+ if (drops === before) throw error+ return withoutHints(run)+ }+}++/**+ * The rows of an `aggregateRaw` pipeline on SwarmGroup, through `index` once the database is known to have it. An index+ * removed since the last listing (a rollback, an index replaced) fails the read: the listing is dropped, said, and the read+ * runs without it at once outside a transaction. In one, the failed command aborted it, and Prisma may report only that:+ * its runner runs it again (`listingDrops`).+ */+export async function groupsBy(tx: Transaction, index: GroupIndex, pipeline: Prisma.InputJsonValue[]): Promise<Array<Record<string, unknown>>> {+ const run = async (hint?: Record<string, number>) => await tx.swarmGroup.aggregateRaw({ pipeline, ...hint ? { options: { hint } } : {} }) as unknown as+ Array<Record<string, unknown>>+ if (rerun.getStore() || !present.has(index)) return run()+ try {+ return await run(GROUP_INDEX[index])+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error)+ if (!/hint provided does not correspond to an existing index|has been aborted/.test(message)) throw error+ present = new Set()+ drops+++ logger.error('[swarm-reads] SwarmGroup lacks an index a hot read names; reads name none until the next listing', { index: GROUP_INDEX[index] })+ if (typeof (tx as { $transaction?: unknown }).$transaction === 'function') return run()+ throw error+ }+}++/** A project's quiet conversations last active since `since`, as the project index serves them. A conversation stored with no+ * project belongs to the first one (projects.ts groupProject); a null match is a missing field too. */+export function quietSince(tabId: string, projectId: string, since: number): Record<string, Prisma.InputJsonValue> {+ return { tabId, deleted: false, dormant: true, lastActivityAt: { $gte: { $date: new Date(since).toISOString() } },+ projectId: { $in: projectId === 'lead' ? ['lead', null] : [projectId] } }+}diff --git a/backend/src/superTabs/parts/light.ts b/backend/src/superTabs/parts/light.tsindex f245a2da8..9e5e2e229 100644--- a/backend/src/superTabs/parts/light.ts+++ b/backend/src/superTabs/parts/light.ts@@ -9,6 +9,7 @@ import type { Prisma } from '@prisma/client'import { activeWork, type Agent, type AgentStart } from '../model.js'import type { GroupData, HeadData, Part, PartKind, SwarmParts } from './types.js'import { delegate, partId, type PartRow } from './write.js'+import { groupsBy, type GroupIndex } from './indexed.js'type Transaction = Prisma.TransactionClient@@ -60,11 +61,11 @@ function lightGroup(row: Record<string, unknown>): Part<GroupData> {* awaitingWorkers): a lost task leaves none stored (liveness.ts). The 'working' selection then also needs every active* conversation such a worker is in, works in or leads, as a whole first load sees them (review of #1745).*/-async function undecidedWork(agents: Part<Agent>[], groups: Part<GroupData>[], lightGroups: (match: Record<string, unknown>) => Promise<Part<GroupData>[]>) {+async function undecidedWork(agents: Part<Agent>[], groups: Part<GroupData>[], lightGroups: (index: GroupIndex, match: Record<string, unknown>) => Promise<Part<GroupData>[]>) {const busy = new Set(groups.flatMap(g => [...g.data!.leadId ? [g.data!.leadId] : [], ...activeWork(g.data!) ? [g.data!.assignment!.agentId, ...g.data!.agentIds] : []]))const undecided = agents.filter(a => !a.data!.workDecision && !busy.has(a.key)).map(a => a.key)if (!undecided.length) return []- return lightGroups({ dormant: false, key: { $nin: groups.map(g => g.key) },+ return lightGroups('dormancy', { dormant: false, key: { $nin: groups.map(g => g.key) },$or: [{ 'data.assignment.agentId': { $in: undecided } }, { 'data.leadId': { $in: undecided } }, { agentIds: { $in: undecided } }] })}@@ -90,18 +91,22 @@ export async function loadLight(tx: Transaction, tabId: string,// those only alongside their conversation (model.ts visibleState).const working = [...projects ? projects.map(p => p.leadConversationId) : ['lead'], leadsConversationId ?? '',...starts.map(s => s.data!.request.conversationId), ...invitations.map(i => i.conversationId)]- const which = opts.groups === 'working' ? [{ key: { $in: working } }, { executionMachineId: { $ne: null } }] : [{ dormant: false }]- const named = [...which, ...opts.groupIds?.length ? [{ key: { $in: opts.groupIds } }] : [], ...opts.agentIds?.length ? [{ agentIds: { $in: opts.agentIds } }] : [],- ...opts.projectIds?.length ? [{ projectId: { $in: opts.projectIds }, dormant: false }] : [],- // The Swarm's first project also holds conversations stored with no project (model.ts groupProject).- ...opts.projectIds?.includes('lead') ? [{ projectId: null, dormant: false }] : []]- const lightGroups = async (match: Record<string, unknown>) => (await tx.swarmGroup.aggregateRaw({ pipeline: [{ $match: { tabId, deleted: false, ...match } },- { $project: LIGHT_GROUP }] }) as unknown as Array<Record<string, unknown>>).map(lightGroup)+ // One read per index, never one $or over several: Mongo plans an $or again whenever its plan cache misses (parts/indexed.ts).+ const keys = [...opts.groups === 'working' ? working : [], ...opts.groupIds ?? []]+ // The Swarm's first project also holds conversations stored with no project (model.ts groupProject).+ const projectIds = [...opts.projectIds ?? [], ...opts.projectIds?.includes('lead') ? [null] : []]+ const named: Array<readonly [GroupIndex, Record<string, unknown>]> = [+ ...opts.groups === 'working' ? [['running', { executionMachineId: { $ne: null } }] as const] : [['dormancy', { dormant: false }] as const],+ ...keys.length ? [['key', { key: { $in: keys } }] as const] : [], ...opts.agentIds?.length ? [['agents', { agentIds: { $in: opts.agentIds } }] as const] : [],+ ...projectIds.length ? [['project', { projectId: { $in: projectIds }, dormant: false }] as const] : []]+ const lightGroups = async (index: GroupIndex, match: Record<string, unknown>) => (await groupsBy(tx, index, [{ $match: { tabId, deleted: false, ...match } },+ { $project: LIGHT_GROUP }])).map(lightGroup)+ const unique = (lists: Part<GroupData>[][]) => [...new Map(lists.flat().map(group => [group.key, group])).values()]const [members, agents, machines, progress, groups] = await Promise.all([...(['member', 'agent', 'machine'] as const).map(kind => kind === 'machine' && opts.machines === false ? Promise.resolve([]): delegate(tx, kind).findMany({ where: live }).then(rows => rows.map(row => toPart(kind, row)))),delegate(tx, 'progress').findMany({ where: live }).then(rows => rows.map(row => toPart('progress', row))),- lightGroups({ $or: named }),+ Promise.all(named.map(([index, match]) => lightGroups(index, match))).then(unique),])if (opts.groups === 'working') groups.push(...await undecidedWork(agents as Part<unknown>[] as Part<Agent>[], groups as Part<GroupData>[], lightGroups))return { tabId, head: toPart<HeadData>('head', head as unknown as PartRow), members, agents, groups, machines, starts, progress,diff --git a/backend/src/superTabs/parts/load.ts b/backend/src/superTabs/parts/load.tsindex 8c327d084..a06054d45 100644--- a/backend/src/superTabs/parts/load.ts+++ b/backend/src/superTabs/parts/load.ts@@ -4,19 +4,34 @@ import type { GroupData, HeadData, MachineData, Part, PartKind, SwarmParts } froimport { PART_KINDS } from './types.js'import { noteChange } from '../cache.js'import { ms } from './light.js'+import { groupsBy, type GroupIndex } from './indexed.js'type Transaction = Prisma.TransactionClienttype Where = Record<string, unknown>+/** Live groups as one read per index (parts/indexed.ts): Prisma's filters, and one $or over several indexes, are planned. */+type GroupReads = Array<readonly [GroupIndex, Where]>const toPart = <T>(kind: PartKind, row: PartRow): Part<T> => ({ kind, key: row.key, version: row.version, changedAt: row.changedAt.getTime(),ordinal: row.ordinal, data: row.data as T, ...(row.deleted ? { deleted: true as const } : {}), ...(row.dormant ? { dormant: true as const } : {}) })-/** `null` for a kind leaves it out: none are loaded. */-async function assemble(tx: Transaction, tabId: string, where: Record<Exclude<PartKind, 'head'>, Where | null>): Promise<SwarmParts> {+/** A group part as `aggregateRaw` returns it, as the PartRow Prisma would. */+const rawGroupRow = (row: Record<string, unknown>): PartRow => ({ ...row, changedAt: new Date(ms(row.changedAt)) }) as unknown as PartRow++/** The live groups these reads find, each once. */+async function liveGroups(tx: Transaction, tabId: string, reads: GroupReads): Promise<PartRow[]> {+ const found = await Promise.all(reads.map(([index, match]) => groupsBy(tx, index, [{ $match: { tabId, deleted: false, ...match } as Prisma.InputJsonValue }])))+ return [...new Map(found.flat().map(row => [row.key as string, rawGroupRow(row)])).values()]+}++/** `null` for a kind leaves it out: none are loaded. Groups go by `where` (every part, tombstones too) or as named reads. */+async function assemble(tx: Transaction, tabId: string, where: Record<Exclude<PartKind, 'head' | 'group'>, Where | null> & { group: Where | GroupReads }): Promise<SwarmParts> {const head = await tx.swarmHead.findUnique({ where: { id: partId(tabId, 'head') } })if (!head) throw Object.assign(new Error(`Swarm ${tabId} has no parts`), { code: 'PARTS_MISSING' })+ const group = where.groupconst [members, agents, groups, machines, starts, progress] = await Promise.all((['member', 'agent', 'group', 'machine', 'start', 'progress'] as const)- .map(kind => where[kind] === null ? Promise.resolve([]) : delegate(tx, kind).findMany({ where: { tabId, ...where[kind] } }).then(rows => rows.map(row => toPart(kind, row)))))+ .map(kind => kind === 'group' && Array.isArray(group) ? liveGroups(tx, tabId, group).then(rows => rows.map(row => toPart(kind, row)))+ : where[kind] === null ? Promise.resolve([])+ : delegate(tx, kind).findMany({ where: { tabId, ...where[kind] as Where } }).then(rows => rows.map(row => toPart(kind, row)))))return { tabId, head: toPart<HeadData>('head', head as unknown as PartRow), members, agents, groups, machines, starts, progress,sequenceBase: head.sequenceBase, visibilityEpoch: head.visibilityEpoch, groupCounter: head.groupCounter } as SwarmParts}@@ -34,12 +49,14 @@ export function loadActive(tx: Transaction, tabId: string,allGroups?: boolean } = {}): Promise<SwarmParts> {const live = { deleted: false }const or = (always: Where, ...named: Array<Where | false>) => ({ ...live, OR: [always, ...named.filter((w): w is Where => !!w)] })+ // A group stored with no project belongs to the default one (projects.ts groupProject), as in a Swarm from before projects;+ // a null match is a missing field too.+ const projectIds = [...opts.projectIds ?? [], ...opts.projectIds?.includes('lead') ? [null] : []]return assemble(tx, tabId, {member: live, agent: live, machine: live, progress: live,- group: opts.allGroups ? live : or({ dormant: false }, !!opts.groupIds?.length && { key: { in: opts.groupIds } },- !!opts.agentIds?.length && { agentIds: { hasSome: opts.agentIds } }, !!opts.projectIds?.length && { projectId: { in: opts.projectIds } },- // A group stored with no project belongs to the default one (projects.ts groupProject), as in a Swarm from before projects.- !!opts.projectIds?.includes('lead') && { projectId: null }, !!opts.projectIds?.includes('lead') && { projectId: { isSet: false } }),+ group: opts.allGroups ? [['live', {}]] : [['dormancy', { dormant: false }], ...opts.groupIds?.length ? [['key', { key: { $in: opts.groupIds } }] as const] : [],+ ...opts.agentIds?.length ? [['agents', { agentIds: { $in: opts.agentIds } }] as const] : [],+ ...projectIds.length ? [['project', { projectId: { $in: projectIds } }] as const] : []],start: opts.includeRecentStarts ? live : or({ finishedAt: null }, !!opts.startIds?.length && { key: { in: opts.startIds } },!!opts.startAgentIds?.length && { agentId: { in: opts.startAgentIds } }),})@@ -49,7 +66,7 @@ export function loadActive(tx: Transaction, tabId: string,* machines, which it keeps at their own stamp (`loadMachines`). Dormant groups and their like come by name (`loadGroups`). */export function loadReadBase(tx: Transaction, tabId: string): Promise<SwarmParts> {const live = { deleted: false }- return assemble(tx, tabId, { member: live, agent: live, group: { ...live, dormant: false }, machine: null, start: live, progress: live })+ return assemble(tx, tabId, { member: live, agent: live, group: [['dormancy', { dormant: false }]], machine: null, start: live, progress: live })}/** A Swarm's live machine parts. */@@ -57,9 +74,9 @@ export async function loadMachines(tx: Transaction, tabId: string): Promise<Partreturn (await delegate(tx, 'machine').findMany({ where: { tabId, deleted: false } })).map(row => toPart<MachineData>('machine', row))}-/** A Swarm's live groups that match `where` (by key, or the dormant ones). */-export async function loadGroups(tx: Transaction, tabId: string, where: Where): Promise<Part<GroupData>[]> {- return (await delegate(tx, 'group').findMany({ where: { tabId, deleted: false, ...where } })).map(row => toPart<GroupData>('group', row))+/** A Swarm's live groups that `match` finds through `index`. */+export async function loadGroups(tx: Transaction, tabId: string, index: GroupIndex, match: Where): Promise<Part<GroupData>[]> {+ return (await liveGroups(tx, tabId, [[index, match]])).map(row => toPart<GroupData>('group', row))}/** Every part of one Swarm, tombstones included with their last data (migration verification and rollback). */@@ -96,14 +113,21 @@ export async function loadChanged(tx: Transaction, tabIds: string[], since: Datereturn changed}-/** Dormant groups the caller pages through, newest activity first, 50 at a time. */+const DORMANT_PAGE = 50++/** Dormant groups the caller pages through, newest activity first, 50 at a time. A project's go through its index in that+ * order. A later page is two reads that index bounds: the rest of its cursor's moment by key, then what is older. The+ * cursor as one `$or` is no bound, and each page read every newer group again (#1964). */export async function loadDormant(tx: Transaction, tabId: string,page: { projectId?: string; before?: { lastActivityAt: number; key: string } }): Promise<Part<GroupData>[]> {- const before = page.before && { OR: [{ lastActivityAt: { lt: new Date(page.before.lastActivityAt) } },- { lastActivityAt: new Date(page.before.lastActivityAt), key: { lt: page.before.key } }] }- const rows = await tx.swarmGroup.findMany({ where: { tabId, deleted: false, dormant: true, ...(page.projectId ? { projectId: page.projectId } : {}),- ...before }, orderBy: [{ lastActivityAt: 'desc' }, { key: 'desc' }], take: 50 })- return rows.map(row => toPart<GroupData>('group', row as unknown as PartRow))+ const index = page.projectId ? 'project' : 'dormancy'+ const read = async (match: Where, limit: number) => limit <= 0 ? [] : groupsBy(tx, index, [+ { $match: { tabId, deleted: false, dormant: true, ...page.projectId ? { projectId: page.projectId } : {}, ...match } as Prisma.InputJsonValue },+ { $sort: { lastActivityAt: -1, key: -1 } }, { $limit: limit }])+ const at = page.before && { $date: new Date(page.before.lastActivityAt).toISOString() }+ const same = page.before ? await read({ lastActivityAt: at, key: { $lt: page.before.key } }, DORMANT_PAGE) : []+ const older = await read(at ? { lastActivityAt: { $lt: at } } : {}, DORMANT_PAGE - same.length)+ return [...same, ...older].map(row => toPart<GroupData>('group', rawGroupRow(row)))}/** The newest data change of each Swarm, over all its parts (design section 3.7). A touch does not count.diff --git a/backend/src/superTabs/read/host.ts b/backend/src/superTabs/read/host.tsindex 1ea820f70..e36683057 100644--- a/backend/src/superTabs/read/host.ts+++ b/backend/src/superTabs/read/host.ts@@ -15,6 +15,7 @@ import type { CacheView } from '../cache.js'import { readMachines } from '../cachedReads.js'import * as parts from '../parts/index.js'import type { MachineData, Part } from '../parts/index.js'+import { groupsBy, retryStaleIndex } from '../parts/indexed.js'import { READ_FORMAT, type HostChanges, type HostChangesRequest } from './contract.js'import { cursorFor, readCursor } from './cursor.js'import { hostTab, type HostLookups } from './hostRows.js'@@ -94,7 +95,9 @@ export class HostReads {const after = new Date(since)const [changed, groups, machines] = await Promise.all([parts.loadChanged(tx, ids, after, PLAIN_KINDS),- tx.swarmGroup.findMany({ where: { tabId: { in: ids }, changedAt: { gt: after }, hostMachineIds: { has: machineId } }, select: { tabId: true } }),+ // Through the host's own index by name (parts/indexed.ts): every host runs this every second, and reads no other host's.+ groupsBy(tx, 'wake', [{ $match: { tabId: { $in: ids }, changedAt: { $gt: { $date: after.toISOString() } }, hostMachineIds: machineId } },+ { $project: { _id: 0, tabId: 1 } }]) as Promise<unknown> as Promise<Array<{ tabId: string }>>,tx.swarmMachine.findMany({ where: { tabId: { in: ids }, hostChangedAt: { gt: after } }, select: { tabId: true, key: true, data: true } }),])const woken = new Set([...ids.filter(id => changed.get(id)?.some(p => plainlyRelevant(p, machineId))), ...groups.map(g => g.tabId)])@@ -114,7 +117,7 @@ export class HostReads {private async tabState(tx: Transaction, cached: CacheView, tabId: string, machineId: string): Promise<SuperTabState> {const [light, machines] = await Promise.all([cached.get(tabId, `host:state:${machineId}`, 'main',// On a miss, one snapshot of what the host's rows are built from.- () => this.db.$transaction(t => this.hostParts(t, tabId, machineId))),+ () => retryStaleIndex(() => this.db.$transaction(t => this.hostParts(t, tabId, machineId)))),readMachines(tx, cached, tabId)])return parts.reassemble({ ...light, machines }, { share: true })}diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 2e79bb1c6..9d28d1ee5 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -24,6 +24,7 @@ import { digestDue, digestMessageId, digestRequest, digestSince } from './digestimport { judgedMachines, loseTasks, sweepLiveness, SWEEP_EVERY_MS } from './liveness.js'import { archivable, archiveLine, stopIdleWorkers, stoppedMachines } from './tidy.js'import { clock, insertParts, loadActive, reassemble, split, type HeadData, type SwarmParts } from './parts/index.js'+import { groupsBy, listingDrops, quietSince, withoutHints } from './parts/indexed.js'import { commitParts, indexesOf, numberMessages, type Touch, type Written } from './parts/commit.js'import { touchesFor } from './parts/touches.js'import { freeze, swarmReadCache, swarmTransaction, type CacheView, type SwarmCache } from './cache.js'@@ -141,17 +142,22 @@ export async function loadRow(tx: Transaction, row: SwarmRow, named: Named = {})*/export async function transaction<T>(db: Pick<PrismaClient, '$transaction'>, body: (tx: Transaction) => Promise<T>,sleep: (ms: number) => Promise<void> = ms => new Promise(resolve => setTimeout(resolve, ms))): Promise<T> {+ let hintless = falsefor (let attempt = 1; ; attempt++) {let committed: Array<() => void> = []+ const drops = listingDrops()try {// The cache's stamps are bumped, and awaited, once the attempt commits (cache.ts); after-commit work runs after them.- const result = await swarmTransaction(db, tx => { committed = []; pendingCommits.set(tx, committed); return body(tx) }, TRANSACTION_OPTIONS)+ const run = () => swarmTransaction(db, tx => { committed = []; pendingCommits.set(tx, committed); return body(tx) }, TRANSACTION_OPTIONS)+ const result = await (hintless ? withoutHints(run) : run())for (const callback of committed) {try { callback() } catch (error) { logger.warn('[super-tab] after-commit callback failed', { error: error instanceof Error ? error.message : String(error) }) }}return result} catch (error) {- if (!retryable(error)) throw error+ // A read in this attempt named an index removed since the last listing (parts/indexed.ts): the next attempts name none.+ if (listingDrops() !== drops) hintless = true+ else if (!retryable(error)) throw error// An expired or unstartable transaction (P2028) is load: one more try, then the client retries after Retry-After.if (attempt >= (errorCode(error) === 'P2028' ? EXPIRED_TRANSACTION_ATTEMPTS : TRANSACTION_ATTEMPTS)) {// A unique key still taken after every retry is a real duplicate, not contention.@@ -193,6 +199,7 @@ const errorCode = (error: unknown): unknown => error && typeof error === 'objectfunction retryable(error: unknown): boolean {if (!error || typeof error !== 'object') return falsereturn ['P2034', 'P2002', 'P2028'].includes(errorCode(error) as string) || (error instanceof Error && error.message.includes('TransientTransactionError'))+}/** A Lead's start dispatches within seconds of being recorded; one still undispatched after this long was given up. */@@ -1671,7 +1678,8 @@ export class SuperTabStore {/** The caller's own tags in this Swarm, newest first: still waiting, or answered and marked done. */async mentions(user: AuthUser, tabId: string, state: 'waiting' | 'earlier') {this.personOnly(user)- const { state: tab } = await this.readLoad(user, tabId, { allGroups: true })+ // Not every quiet conversation: listMentions reads the quiet ones its tags are in by id (mentions.ts readable).+ const { state: tab } = await this.readLoad(user, tabId)await this.actor(this.db, user, tab)return { mentions: await listMentions(this.db, tabId, tab, { kind: 'person', userId: user.sub, email: user.email }, this.now(), state === 'waiting') }}@@ -1946,8 +1954,8 @@ export class SuperTabStore {// out: it only gets marked, or every poll would read for it again and fail on the duplicate (prod slow log, 2026-10-28).const sentAlready = !!due && !!await tx.superTabMessage.findUnique({ where: { id: digestMessageId(tabId, led!.id, due) }, select: { id: true } })if (led && due && !sentAlready) {- const finished = await tx.swarmGroup.findMany({ where: { tabId, deleted: false, dormant: true, lastActivityAt: { gte: new Date(digestSince(led, this.now())) },- OR: [{ projectId: led.id }, ...led.id === 'lead' ? [{ projectId: null }] : []] }, select: { key: true } })+ const finished = await groupsBy(tx, 'project', [{ $match: quietSince(tabId, led.id, digestSince(led, this.now())) },+ { $project: { _id: 0, key: 1 } }]) as unknown as Array<{ key: string }>// A decision waits in its conversation even once that conversation went quiet.asked = await waitingInProject(tx, tabId, led.id)const groupIds = [...new Set([...finished.map(g => g.key), ...asked.map(d => d.conversationId)])]@@ -1964,8 +1972,8 @@ export class SuperTabStore {const written = ledProject(state, agentId) ? sweepLiveness(state, this.now()) : []// The Lead's poll stops its project's idle workers too (tidy.ts): it is where what waits for each agent is known.const reviewers = led ? state.agents.filter(a => agentRole(a) === 'reviewer' && a.workDecision !== 'stop' && agentProjectId(state, a) === led.id).map(a => a.id) : []- const asleep = reviewers.length ? (await tx.swarmGroup.findMany({ where: { tabId, deleted: false, dormant: true, agentIds: { hasSome: reviewers } },- select: { data: true } })).map(row => row.data as unknown as Conversation) : []+ const asleep = reviewers.length ? (await groupsBy(tx, 'agents', [{ $match: { tabId, deleted: false, dormant: true, agentIds: { $in: reviewers } } },+ { $project: { _id: 0, data: 1 } }])).map(row => row.data as unknown as Conversation) : []const idle = led ? stopIdleWorkers(state, led.id, this.now(), new Set((await tx.superTabDelivery.findMany({where: { tabId, status: { in: [...OPEN_DELIVERY] } }, select: { agentId: true }, distinct: ['agentId'] })).map(d => d.agentId)), asleep) : []written.push(...idle)diff --git a/backend/vitest.super-tabs.config.ts b/backend/vitest.super-tabs.config.tsindex 0bac80ecf..e058983ee 100644--- a/backend/vitest.super-tabs.config.ts+++ b/backend/vitest.super-tabs.config.ts@@ -3,5 +3,5 @@ import { defineConfig } from 'vitest/config'export default defineConfig({ test: { include: ['integration/superTabs.test.ts', 'integration/superTabSharing.test.ts', 'integration/superTabGithub.test.ts', 'integration/superTabTeam.test.ts', 'integration/superTabWorkspaces.test.ts','integration/superTabRead.test.ts', 'integration/superTabLeaderboard.test.ts', 'integration/superTabMigration.test.ts', 'integration/superTabLiveness.test.ts', 'integration/superTabTidy.test.ts','integration/superTabProfile.test.ts', 'integration/superTabLegacyDrop.test.ts', 'integration/shareRaces.test.ts',- 'integration/superTabIndexes.test.ts', 'integration/superTabHotReads.test.ts', 'integration/superTabCache.test.ts', 'integration/superTabObserver.test.ts', 'integration/productUsageConflicts.test.ts'], hookTimeout: 120000, testTimeout: 30000, setupFiles: ['integration/cacheOn.test-support.ts'],+ 'integration/superTabIndexes.test.ts', 'integration/superTabHotReads.test.ts', 'integration/superTabGroupPlans.test.ts', 'integration/superTabCache.test.ts', 'integration/superTabObserver.test.ts', 'integration/productUsageConflicts.test.ts'], hookTimeout: 120000, testTimeout: 30000, setupFiles: ['integration/cacheOn.test-support.ts'],fileParallelism: false, env: { NODE_ENV: 'test', HARNESS_CREDENTIAL_ENCRYPTION_KEY: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' } } })