The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Merge batch 11-05 i1450: #3113 (#3123)
0xken159cc103db8764
diff --git a/backend/integration/superTabActivity.test.ts b/backend/integration/superTabActivity.test.tsindex 7ef9cf196..88a423ec1 100644--- a/backend/integration/superTabActivity.test.ts+++ b/backend/integration/superTabActivity.test.ts@@ -1,4 +1,4 @@-import { randomUUID } from 'node:crypto'+import { createHash, randomUUID } from 'node:crypto'import { execFile } from 'node:child_process'import { promisify } from 'node:util'import { join } from 'node:path'@@ -15,7 +15,9 @@ import { migrateSwarm } from '../src/superTabs/migration/migrate.js'import { SwarmReads } from '../src/superTabs/read/service.js'import { HostReads } from '../src/superTabs/read/host.js'import { buddyChatId, openSwarmBuddyChat } from '../src/superTabs/buddies.js'-import { ACTIVITY_EXPIRY_MS, ACTIVITY_READS, resetActivityReads, sweepActivity, writeReplyActivity } from '../src/superTabs/activity.js'+import { ACTIVITY_EXPIRY_MS, ACTIVITY_READS, activityDigest, resetActivityReads, sweepActivity, writeReplyActivity } from '../src/superTabs/activity.js'+import { listNamedIndexes } from '../src/superTabs/parts/indexed.js'+import { profiled } from './plans.test-support.js'import { SESSION_ENDED_KIND } from '../src/superTabs/buddySessions.js'import { threadFacet } from '../src/superTabs/pageReads.js'import { CHAT_SETTING_DEFAULT_KEEP_MS, sweepChatSettings } from '../src/superTabs/chatSettings.js'@@ -619,3 +621,26 @@ describe('Chat notification settings', () => {expect((await db.superTabChatSetting.findMany({ where: { tabId: s.tabId }, select: { personId: true, notify: true } }))).toEqual([{ personId: peer.sub, notify: 'muted' }])})})++// Ken's 13:20 VN prod scan, 2026-11-05 (task #3146): SuperTabActivity {tabId, personId, createdAt} was in Atlas's slow log by time+// (79 ops, 23 s). Its explain(): the digest every person's read asks for examined each of the person's rows of 30 days, 865+// for the busiest, to return the 26 unread. Through the unread index it examines the unread rows alone.+describe('the unread-activity digest\'s plan (#3146)', () => {+ it('examines only the unread rows, a row with no readAt among them, and digests them as before', async () => {+ await listNamedIndexes(db)+ const tabId = `tab-${randomUUID()}`, personId = randomUUID().replaceAll('-', '').slice(0, 24), now = Date.now()+ const row = (i: number, readAt: Date | null) => ({ id: `act-${tabId}-${i}`, tabId, personId, kind: 'reply', conversationId: 'c', messageId: `m${i}`,+ threadId: null, actorKind: 'person', actorId: 'someone', emoji: null, createdAt: new Date(now - i * 60_000), readAt })+ await db.superTabActivity.createMany({ data: [...Array.from({ length: 300 }, (_, i) => row(i + 10, new Date(now))), row(1, null), row(2, null)] })+ // A row written before readAt was kept has no such field at all.+ await db.$runCommandRaw({ insert: 'SuperTabActivity', documents: [{ _id: `act-${tabId}-3`, tabId, personId, kind: 'reply', conversationId: 'c',+ messageId: 'm3', actorKind: 'person', actorId: 'someone', createdAt: { $date: new Date(now - 3 * 60_000).toISOString() } }] })+ let digest = ''+ const reads = (await profiled(db, async () => { digest = await activityDigest(db, tabId, personId, now) })).filter(e => e.ns.endsWith('.SuperTabActivity'))+ const unread = [`act-${tabId}-1`, `act-${tabId}-2`, `act-${tabId}-3`]+ expect(digest).toBe(createHash('sha256').update(unread.sort().join('\n')).digest('base64url').slice(0, 16))+ expect(reads.map(e => ({ plan: e.planSummary, keys: e.keysExamined, docs: e.docsExamined, returned: e.nreturned })))+ // `readAt: null` is two ranges of the index, null and missing: one scan of each.+ .toEqual([{ plan: expect.stringMatching(/^(IXSCAN \{ tabId: 1, personId: 1, readAt: 1, createdAt: 1 \}(, |$))+$/), keys: 3, docs: 3, returned: 3 }])+ })+})diff --git a/backend/integration/superTabCommandLoads.test.ts b/backend/integration/superTabCommandLoads.test.tsindex def511b08..0c0b71859 100644--- a/backend/integration/superTabCommandLoads.test.ts+++ b/backend/integration/superTabCommandLoads.test.ts@@ -11,6 +11,8 @@ import { SuperTabStore } from '../src/superTabs/store.js'import type { Command } from '../src/superTabs/commands.js'import { logger } from '../src/utils/logger.js'import type { Conversation } from '../src/superTabs/model.js'+import { listNamedIndexes } from '../src/superTabs/parts/indexed.js'+import { profiled } from './plans.test-support.js'// Ken's prod DB profile, 2026-11-02: a command naming the PM read every group the PM was ever in, 1,628 of them and growing with// every task, up to 1.66 s a read. Those that write its groups need them; those that only ask about it do not.@@ -30,14 +32,17 @@ beforeAll(async () => {db = new PrismaClient({ datasources: { db: { url: uri } } })await db.user.create({ data: { id: owner.sub, externalId: `fixture-${owner.sub}`, email: owner.email, autonomousEnv: owner.autonomousEnv } })await db.machine.create({ data: { machineId: 'mac', userId: owner.sub, apiKey: 'fixture-mac', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })+ // As each server does at startup (server.ts): reads name only the indexes the database has.+ await listNamedIndexes(db)}, 120_000)afterAll(async () => {await db?.$disconnect()await replica?.stop()})-/** A Swarm with a coordinator, a worker and [dormant] archived-free dormant groups holding the coordinator. */-async function swarm(dormant = 0) {+/** A Swarm with a coordinator, a worker, [dormant] archived-free dormant groups holding the coordinator, and [awake] more awake+ * ones, each with a brief the size of a real task's. */+async function swarm(dormant = 0, awake = 0) {const store = new SuperTabStore(db)const tab = (await store.create(owner, randomUUID(), 'Loads', 'public')).idconst execute = (command: Command, asAgentId?: string) => store.execute(owner, tab, { operationId: randomUUID(), command }, asAgentId)@@ -53,6 +58,12 @@ async function swarm(dormant = 0) {return { ...shape, id: `${tab}:group:${key}`, key, dormant: true, agentIds: [lead], executionMachineId: null,data: { ...part.data as object, id: key, title: `Old ${i}`, agentIds: [lead], leadId: lead, assignment: null, dormant: true } } as Prisma.SwarmGroupCreateManyInput})+ const brief = 'Implement it. '.repeat(300)+ rows.push(...Array.from({ length: awake }, (_, i) => {+ const key = `awake-${i}`+ return { ...shape, id: `${tab}:group:${key}`, key, dormant: false, agentIds: [lead], executionMachineId: null,+ data: { ...part.data as object, id: key, title: `Awake ${i}`, brief, agentIds: [lead], leadId: lead, assignment: null } } as Prisma.SwarmGroupCreateManyInput+ }))for (let i = 0; i < rows.length; i += 200) await db.swarmGroup.createMany({ data: rows.slice(i, i + 200) })return { store, tab, execute, lead, worker, group, harness }}@@ -128,6 +139,43 @@ describe('a command naming the coordinator', () => {})})+// Ken's 13:20 VN prod scan, 2026-11-05 (task #3146): SwarmGroup.aggregateRaw became a top-5 slow shape under POST commands.+// Every command read each awake conversation of its Swarm whole, 180 of about 5.6 KB in the busiest, to find the one or two+// that changed. A command now lists them from an index that holds every column it compares (no document read), and reads+// whole only those that changed since this server last read them.+describe('a command in a Swarm with many awake conversations (#3146)', () => {+ const AWAKE = 150+ const posted = (f: Awaited<ReturnType<typeof swarm>>) => f.execute({ type: 'message.send', conversationId: f.group, text: 'hello', replyTo: null, recipientIds: [] } as Command, f.worker)++ it('reads none of them whole once this server has read them, only the conversation it writes', async () => {+ const f = await swarm(0, AWAKE)+ await posted(f)+ const groups = (await profiled(db, () => posted(f))).filter(e => e.ns.endsWith('.SwarmGroup'))+ const examined = groups.reduce((n, e) => n + (e.docsExamined ?? 0), 0)+ expect(examined).toBeLessThanOrEqual(4)+ })++ it('lists them from the held index alone, examining no document', async () => {+ const f = await swarm(0, AWAKE)+ await posted(f)+ const groups = (await profiled(db, () => posted(f))).filter(e => e.ns.endsWith('.SwarmGroup'))+ const listing = groups.filter(e => JSON.stringify((e.command?.pipeline as unknown[] | undefined)?.[0] ?? null).includes('"dormant":false'))+ expect(listing.map(e => ({ plan: e.planSummary, docs: e.docsExamined, returned: e.nreturned })))+ .toEqual([{ plan: 'IXSCAN { tabId: 1, dormant: 1, deleted: 1, key: 1, version: 1, changedAt: 1 }', docs: 0, returned: AWAKE + 2 }])+ })++ it('still sees a conversation another server changed', async () => {+ const f = await swarm(0, AWAKE)+ await posted(f)+ const row = await db.swarmGroup.findFirstOrThrow({ where: { tabId: f.tab, key: 'awake-7' } })+ await db.swarmGroup.update({ where: { id: row.id }, data: { version: row.version + 1, changedAt: new Date(),+ data: { ...row.data as object, title: 'Renamed elsewhere' } } })+ await posted(f)+ const loaded = await f.store.load(db as never, owner, f.tab)+ expect(loaded.state.conversations.find(c => c.id === 'awake-7')?.title).toBe('Renamed elsewhere')+ })+})+describe('what still reads an agent\'s dormant groups', () => {it('a worker\'s alert finds its only group, asleep', async () => {const f = await swarm()diff --git a/backend/integration/superTabHeldParts.test.ts b/backend/integration/superTabHeldParts.test.tsindex 4b15dcf61..b8c7b14a5 100644--- a/backend/integration/superTabHeldParts.test.ts+++ b/backend/integration/superTabHeldParts.test.ts@@ -61,6 +61,8 @@ async function write(tabId: string, pick: (all: parts.SwarmParts) => parts.PartWconst liveOf = <T>(list: parts.Part<T>[]) => list.filter(p => !p.deleted)const observed = (tabId: string, held: HeldParts) => db.$transaction(tx => parts.loadObserved(tx, tabId, 'host', held))+/** A command's load (store.ts loadRow), naming one quiet conversation as a command naming it does. */+const commanded = (tabId: string, held: HeldParts | null, groupIds: string[] = []) => db.$transaction(tx => parts.loadActive(tx, tabId, { groupIds }, held))describe('a full report\'s held parts (B3, #2597)', () => {it('load what a server holding nothing loads, after every kind of write', { timeout: 120_000 }, async () => {@@ -139,3 +141,46 @@ describe('a full report\'s held parts (B3, #2597)', () => {.toEqual([[true, machines.length, machines.length], [true, 2, 2]])})})++describe('a command\'s held conversations (#3146)', () => {+ it('load what a server holding nothing loads, after every kind of conversation write', { timeout: 120_000 }, async () => {+ const tabId = await scene()+ const warm = new HeldParts(), tiny = new HeldParts(1)+ let n = 0+ const at = () => Date.now() + ++n+ const someGroup = (all: parts.SwarmParts) => liveOf(all.groups).find(x => !x.dormant && x.key !== 'lead')!+ const cases: Array<[string, () => Promise<boolean>]> = [+ ['a post in a conversation', () => write(tabId, all => { const g = someGroup(all)+ return { op: 'update', kind: 'group', key: g.key, seen: g.version, data: { ...g.data!, title: `T${n}` } } }, at())],+ ['a conversation touched (its version alone moves)', () => write(tabId, all => { const g = someGroup(all)+ return { op: 'touch', kind: 'group', key: g.key, seen: g.version } }, at())],+ ['a conversation put to sleep', () => write(tabId, all => { const g = liveOf(all.groups).find(x => x.key === 'lead')!+ return { op: 'update', kind: 'group', key: g.key, seen: g.version, data: { ...g.data!, dormant: true } } }, at())],+ ['and woken', () => write(tabId, all => { const g = liveOf(all.groups).find(x => x.key === 'lead')!+ return { op: 'update', kind: 'group', key: g.key, seen: g.version, data: { ...g.data!, dormant: false } } }, at())],+ ['a conversation removed', () => write(tabId, all => { const g = someGroup(all)+ return { op: 'remove', kind: 'group', key: g.key, seen: g.version, data: g.data } }, at())],+ ['a new conversation', () => write(tabId, all => { const g = someGroup(all)+ return { op: 'insert', kind: 'group', key: `g-${n}`, ordinal: 900 + n, data: { ...g.data!, id: `g-${n}`, title: `New ${n}` } } }, at())],+ ['a conversation rewritten at the version it had, under a new date', async () => {+ const row = await db.swarmGroup.findFirstOrThrow({ where: { tabId, deleted: false, dormant: false, key: { not: 'lead' } } })+ await db.swarmGroup.update({ where: { id: row.id }, data: { changedAt: new Date(at()), data: { ...row.data as object, title: 'rewritten' } } })+ return true+ }],+ ]+ // Against the load without held parts, as every command read before.+ expect(await commanded(tabId, warm)).toEqual(await commanded(tabId, null))+ for (const [name, change] of cases) {+ expect(await change(), name).toBe(true)+ const fresh = await commanded(tabId, null)+ expect(await commanded(tabId, warm), name).toEqual(fresh)+ expect(await commanded(tabId, tiny), name).toEqual(fresh)+ // A command naming a quiet conversation loads it too, and the next one without it does not.+ const quiet = (await db.swarmGroup.findFirst({ where: { tabId, deleted: false, dormant: true } }))?.key+ if (quiet) {+ expect(await commanded(tabId, warm, [quiet]), name).toEqual(await commanded(tabId, null, [quiet]))+ expect(await commanded(tabId, warm), name).toEqual(fresh)+ }+ }+ })+})diff --git a/backend/integration/superTabNoticeDigest.test.ts b/backend/integration/superTabNoticeDigest.test.tsindex 43551601e..32129954b 100644--- a/backend/integration/superTabNoticeDigest.test.ts+++ b/backend/integration/superTabNoticeDigest.test.ts@@ -429,9 +429,10 @@ describe('the same rules as a delivery, at digest time', () => {const f = await swarm()const id = await f.write({ kind: 'idle' }, f.lead, f.group, 'A secret notice.')await db.superTabMessage.update({ where: { id }, data: { text: 'A secret notice.' } })- // Archived with no command, as an older server could leave it: nothing cancelled it first.+ // Archived with no command, as an older server could leave it: nothing cancelled it first. Every server's write moves the+ // part's version (parts/write.ts), which is how a server holding the part sees the change (parts/held.ts).const part = await db.swarmGroup.findFirstOrThrow({ where: { tabId: f.tab, key: f.group } })- await db.swarmGroup.update({ where: { id: part.id }, data: { data: { ...part.data as object, archived: true } } })+ await db.swarmGroup.update({ where: { id: part.id }, data: { version: part.version + 1, changedAt: new Date(), data: { ...part.data as object, archived: true } } })f.clock.now += DUEawait f.poll()expect((await deliveryOf(id, f.lead)).status).toBe('cancelled')diff --git a/backend/integration/superTabRouteDecisions.test.ts b/backend/integration/superTabRouteDecisions.test.tsindex c900845d3..65f50fb9d 100644--- a/backend/integration/superTabRouteDecisions.test.ts+++ b/backend/integration/superTabRouteDecisions.test.ts@@ -179,9 +179,9 @@ describe('what the route-decision list gives each reader', () => {const w = await world(60)await decide(w, [[w.web, 60], [w.secretGroup, 600]])const got = await sameAnswers(w)- // Both windows, then every group.+ // Both windows, then every group: the quiet ones from the database, the awake ones as this server holds them (#3146).const [, profile] = await profiled(() => store.routeDecisions(peer, w.tabId))- expect(profile.SwarmGroup!.docsExamined).toBeGreaterThanOrEqual(await db.swarmGroup.count({ where: { tabId: w.tabId, deleted: false } }))+ expect(profile.SwarmGroup!.docsExamined).toBeGreaterThanOrEqual(await db.swarmGroup.count({ where: { tabId: w.tabId, deleted: false, dormant: true } }))expect(ids(got['a collaborator'])).toHaveLength(60)expect(ids(got['the owner'])).toHaveLength(100)})diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prismaindex 1906bed5a..5c36a4513 100644--- a/backend/prisma/schema.prisma+++ b/backend/prisma/schema.prisma@@ -1213,6 +1213,9 @@ model SuperTabActivity {@@index([tabId, personId, createdAt])/// One thread's or chat's rows of a person, for the read marks that clear them (activity.ts markActivityThreadRead).@@index([tabId, personId, conversationId, threadId])+ /// A person's unread rows, newest first: the digest each of their reads asks for (activity.ts activityDigest). Through the+ /// index above it examined every row of 30 days to find the unread (task #3146).+ @@index([tabId, personId, readAt, createdAt])/// The sweep's rows past their expiry (activity.ts sweepActivity); no TTL option, which the app user cannot set (#2238).@@index([createdAt])}@@ -1933,6 +1936,9 @@ model SwarmGroup {@@index([tabId, executionMachineId])/// The host signature (read/host.ts inputs) reads every conversation's digest and dormancy from this index alone.@@index([tabId, deleted, key, hostDigest, dormant])+ /// A command's awake conversations, listed by the columns it compares with what it holds (parts/held.ts) from this index alone:+ /// each command read every awake one whole before (task #3146).+ @@index([tabId, dormant, deleted, key, version, changedAt])/// Removed parts go at purgeAt (superTabs/sweeps.ts ensurePartExpiry). Declared so db push keeps the index; the worker sets its expiry.@@index([purgeAt], map: "part_purge")}diff --git a/backend/src/lib/productUsageReceiptSweep.test.ts b/backend/src/lib/productUsageReceiptSweep.test.tsnew file mode 100644index 000000000..20cb2be31--- /dev/null+++ b/backend/src/lib/productUsageReceiptSweep.test.ts@@ -0,0 +1,45 @@+import { describe, expect, it, vi } from 'vitest'+import { RECEIPT_SWEEP, sweepOldReceipts } from './productUsageReceiptSweep.js'++/** A database holding [left] old receipts: a find answers up to its limit of them, a delete removes the ids it names. Every+ * delete is recorded with when it was sent. */+function receipts(left: number) {+ const deletes: Array<{ ids: number; at: number }> = []+ let next = 0+ const db = { $runCommandRaw: vi.fn(async (command: Record<string, unknown>) => {+ if ('find' in command) {+ const take = Math.min(Number(command.limit), left)+ return { cursor: { firstBatch: Array.from({ length: take }, () => ({ _id: `r${next++}`, expiresAt: { $date: '2026-12-01T00:00:00.000Z' } })) } }+ }+ const ids = ((command.deletes as Array<{ q: { _id: { $in: string[] } } }>)[0]).q._id.$in+ deletes.push({ ids: ids.length, at: performance.now() })+ left -= ids.length+ return { n: ids.length }+ }) }+ return { db: db as never, deletes }+}++describe('the receipt sweep\'s pass (#3146: 112 deletes of 500 took 94 s of the primary in an hour, the longest 5.8 s)', () => {+ it('deletes at most 50 receipts at a time', async () => {+ const { db, deletes } = receipts(10_000)+ await sweepOldReceipts(db, new Date('2026-11-05T00:00:00Z'), { ...RECEIPT_SWEEP, pauseMs: 0, withoutRule: true })+ expect(deletes.length).toBeGreaterThan(1)+ expect(Math.max(...deletes.map(d => d.ids))).toBeLessThanOrEqual(50)+ })++ it('still removes 500 a pass, about 1,000 a minute, so the old receipts are gone as soon as before', async () => {+ const { db } = receipts(10_000)+ const pass = await sweepOldReceipts(db, new Date('2026-11-05T00:00:00Z'), { ...RECEIPT_SWEEP, pauseMs: 0, withoutRule: true })+ expect(pass.removed).toBe(500)+ expect(RECEIPT_SWEEP.chunk * RECEIPT_SWEEP.chunks * 60_000 / RECEIPT_SWEEP.everyMs).toBe(1_000)+ })++ it('waits between its deletes, and its waits end well inside the 30 s until the next pass', async () => {+ expect(RECEIPT_SWEEP.pauseMs).toBeGreaterThan(0)+ expect((RECEIPT_SWEEP.chunks - 1) * RECEIPT_SWEEP.pauseMs).toBeLessThanOrEqual(RECEIPT_SWEEP.everyMs * 0.6)+ const { db, deletes } = receipts(10_000)+ await sweepOldReceipts(db, new Date('2026-11-05T00:00:00Z'), { ...RECEIPT_SWEEP, pauseMs: 20, withoutRule: true })+ const gaps = deletes.slice(1).map((d, i) => d.at - deletes[i]!.at)+ expect(Math.min(...gaps)).toBeGreaterThanOrEqual(19)+ })+})diff --git a/backend/src/lib/productUsageReceiptSweep.ts b/backend/src/lib/productUsageReceiptSweep.tsindex 22c13b3c1..76296a6c4 100644--- a/backend/src/lib/productUsageReceiptSweep.ts+++ b/backend/src/lib/productUsageReceiptSweep.ts@@ -26,10 +26,11 @@ import { removeChunk, startChunkSweep } from './chunkSweep.js'const COLLECTION = 'product_usage_receipts'const DAY = 86_400_000const OLD_EXPIRY_DAYS = 36-/** One delete of 500 every 30 s, about 1,000 a minute. On production's M20 a delete of 500 old receipts took 12–17 s (cold- * index pages, read from disk; 2026-11-04), so the database spends about half its time on them at most, and a pass still- * running when the next is due skips it. The ~4M old receipts take about 3 days, against 2026-12-04 by expiry alone. */-export const RECEIPT_SWEEP = { chunk: 500, chunks: 1, pauseMs: 0, everyMs: 30_000 } as const+/** Ten deletes of 50 every 30 s, 2 s apart: about 1,000 a minute. One delete of 500 held production's primary up to 5.8 s at+ * peak (Atlas slow log, 2026-11-05 13:20 VN: 112 deletes, 94 s in an hour, task #3146), with commands queueing behind it;+ * a tenth of that is a short write, and the pauses leave the database to the app between them. A pass still running when+ * the next is due skips it. The ~4M old receipts take about 3 days, against 2026-12-04 by expiry alone. */+export const RECEIPT_SWEEP = { chunk: 50, chunks: 10, pauseMs: 2_000, everyMs: 30_000 } as const/** Every receipt write must carry its life; old receipts, never written again, are not checked. */export const RECEIPT_RULE = { validator: { $jsonSchema: { required: ['life'] } }, validationLevel: 'strict', validationAction: 'error' } as constdiff --git a/backend/src/superTabs/activity.ts b/backend/src/superTabs/activity.tsindex 437ee1e39..e86eaaf4d 100644--- a/backend/src/superTabs/activity.ts+++ b/backend/src/superTabs/activity.ts@@ -153,9 +153,12 @@ export async function writeReactionActivity(tx: Tx, write: { tabId: string; reac/** A digest of which of the person's rows are unread, moving when one is written, read or removed: a client reads its list* again when it moves, never on a timer. It carries no count and no chat: only that something changed. */export async function activityDigest(tx: Tx, tabId: string, personId: string, now: number): Promise<string> {- const rows = await tx.superTabActivity.findMany({ where: { tabId, personId, createdAt: { gt: new Date(now - ACTIVITY_EXPIRY_MS) }, ...unread },- orderBy: { createdAt: 'desc' }, take: ACTIVITY_DIGEST_ROWS, select: { id: true } })- return createHash('sha256').update(rows.map(r => r.id).sort().join('\n')).digest('base64url').slice(0, 16)+ // A plain `$match` through the unread index: `readAt: null` matches a row with no readAt too. Prisma's read sent an `$expr`+ // the index could bound only by person, and examined every row of theirs of 30 days (task #3146).+ const rows = await readBy(tx, 'SuperTabActivity', 'unread', [+ { $match: { tabId, personId, readAt: null, createdAt: { $gt: { $date: new Date(now - ACTIVITY_EXPIRY_MS).toISOString() } } } },+ { $sort: { createdAt: -1 } }, { $limit: ACTIVITY_DIGEST_ROWS }, { $project: { _id: 1 } }])+ return createHash('sha256').update(rows.map(r => r._id as string).sort().join('\n')).digest('base64url').slice(0, 16)}export interface ActivityItem {diff --git a/backend/src/superTabs/parts/held.ts b/backend/src/superTabs/parts/held.tsindex 7ce62b8d4..386b80846 100644--- a/backend/src/superTabs/parts/held.ts+++ b/backend/src/superTabs/parts/held.ts@@ -30,6 +30,9 @@ function columnsOf(row: Record<string, unknown>): string {}const byKey = (rows: PartRow[]) => rows.sort((a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0)const COLUMNS = { _id: 0, key: 1, version: 1, changedAt: 1, readAt: 1, dormant: 1 }+/** A group's columns: it has no readAt, and every other one is in the held index (parts/indexed.ts `held`), so a listing of the+ * awake ones through it reads no document. */+const GROUP_COLUMNS = { _id: 0, key: 1, version: 1, changedAt: 1, dormant: 1 }export class HeldParts {private readonly tabs = new Map<string, Tab>()@@ -88,8 +91,11 @@ export class HeldParts {}}-/** The process's held parts. */+/** The process's held parts: machine reports'. */export const HELD = new HeldParts()+/** Commands' (parts/load.ts loadActive), apart from the reports': each read lets go of what it did not select, and a report+ * selects only running work and Lead conversations, where a command selects every awake conversation. */+export const COMMAND_HELD = new HeldParts()export class HeldRead {constructor(private readonly owner: HeldParts, private readonly tab: Tab, private readonly at: number) {}@@ -124,7 +130,7 @@ export class HeldRead {async groups(tx: Transaction, reads: ReadonlyArray<readonly [GroupIndex, Where]>): Promise<Array<Record<string, unknown>>> {const { tabId } = this.tabconst read = (columns: boolean) => Promise.all(reads.map(([index, match]) => groupsBy(tx, index, [- { $match: { tabId, deleted: false, ...match } as Prisma.InputJsonValue }, ...columns ? [{ $project: COLUMNS }] : []])))+ { $match: { tabId, deleted: false, ...match } as Prisma.InputJsonValue }, ...columns ? [{ $project: GROUP_COLUMNS }] : []])))const once = (rows: Array<Record<string, unknown>>) => [...new Map(rows.map(row => [row.key as string, row])).values()]// Holding none of them, the whole reads alone, as before.if (!this.owner.holds(this.tab, 'group')) return this.kept('group', once((await read(false)).flat()), 'cold')diff --git a/backend/src/superTabs/parts/indexed.ts b/backend/src/superTabs/parts/indexed.tsindex da75dd596..8b75b1b14 100644--- a/backend/src/superTabs/parts/indexed.ts+++ b/backend/src/superTabs/parts/indexed.ts@@ -18,6 +18,8 @@ export const GROUP_INDEX = {agents: { tabId: 1, agentIds: 1 },project: { tabId: 1, projectId: 1, dormant: 1, lastActivityAt: 1, key: 1 },dormancy: { tabId: 1, dormant: 1, changedAt: 1 },+ /** A command's awake conversations by the columns a held part is compared by (parts/held.ts): the listing reads no document. */+ held: { tabId: 1, dormant: 1, deleted: 1, key: 1, version: 1, changedAt: 1 },changes: { tabId: 1, changedAt: 1 },/** A host's changed conversations: led by the host, so the wake check reads no other host's. */wake: { hostMachineIds: 1, tabId: 1, changedAt: 1 },@@ -98,7 +100,8 @@ export const NAMED_INDEX = {/** A page's thread summaries, by root id (pageReads.ts pageThreads). */SuperTabThread: { id: { _id: 1 } },/** `age`: Activity rows past their expiry (activity.ts sweepActivity). */- SuperTabActivity: { age: { createdAt: 1 } },+ /** `unread`: a person's unread rows, newest first: the digest each of their reads asks for (activity.ts activityDigest). */+ SuperTabActivity: { age: { createdAt: 1 }, unread: { tabId: 1, personId: 1, readAt: 1, createdAt: 1 } },/** `age`: chat settings back at the default, untouched for a week (chatSettings.ts sweepChatSettings). */SuperTabChatSetting: { age: { updatedAt: 1 } },/** A person's reminders (reminders.ts): `person`, the due list, the next one, the digest and the cap's count; `due`, thediff --git a/backend/src/superTabs/parts/load.ts b/backend/src/superTabs/parts/load.tsindex 41fe2b194..e84d25682 100644--- a/backend/src/superTabs/parts/load.ts+++ b/backend/src/superTabs/parts/load.ts@@ -7,7 +7,7 @@ import { recordDeltaRows } from '../cacheStats.js'import type { AgentStart } from '../model.js'import { ms, rawGroupRow, rawPartRow } from './light.js'import { groupsBy, readBy, type GroupIndex, type NamedCollection } from './indexed.js'-import { HELD, type HeldParts } from './held.js'+import { COMMAND_HELD, HELD, type HeldParts, type HeldRead } from './held.js'type Transaction = Prisma.TransactionClienttype Where = Record<string, unknown>@@ -17,19 +17,21 @@ type GroupReads = Array<readonly [GroupIndex, Where]>const toPart = <T>(kind: PartKind, row: PartRow): Part<T> => ({ kind, key: row.key, version: row.version, changedAt: row.changedAt.getTime(),ordinal: row.ordinal, data: row.data as T, ...(row.deleted ? { deleted: true as const } : {}), ...(row.dormant ? { dormant: true as const } : {}) })-/** The live groups these reads find, each once. */-async function liveGroups(tx: Transaction, tabId: string, reads: GroupReads): Promise<PartRow[]> {+/** The live groups these reads find, each once; through `held`, whole only those changed since it last read them. */+async function liveGroups(tx: Transaction, tabId: string, reads: GroupReads, held?: HeldRead): Promise<PartRow[]> {+ if (held) return (await held.groups(tx, reads)).map(rawGroupRow)const found = await Promise.all(reads.map(([index, match]) => groupsBy(tx, index, [{ $match: { tabId, deleted: false, ...match } as Prisma.InputJsonValue }])))return [...new Map(found.flat().map(row => [row.key as string, rawGroupRow(row)])).values()]}/** `null` for a kind leaves it out: none are loaded. Groups go by `where` (every part, tombstones too) or as named reads. */-async function assemble(tx: Transaction, tabId: string, where: Record<Exclude<PartKind, 'head' | 'group'>, Where | null> & { group: Where | GroupReads }): Promise<SwarmParts> {+async function assemble(tx: Transaction, tabId: string, where: Record<Exclude<PartKind, 'head' | 'group'>, Where | null> & { group: Where | GroupReads },+ held?: HeldRead): Promise<SwarmParts> {const head = await tx.swarmHead.findUnique({ where: { id: partId(tabId, 'head') } })if (!head) throw Object.assign(new Error(`Swarm ${tabId} has no parts`), { code: 'PARTS_MISSING' })const group = where.groupconst [members, agents, groups, machines, starts, progress] = await Promise.all((['member', 'agent', 'group', 'machine', 'start', 'progress'] as const)- .map(kind => kind === 'group' && Array.isArray(group) ? liveGroups(tx, tabId, group).then(rows => rows.map(row => toPart(kind, row)))+ .map(kind => kind === 'group' && Array.isArray(group) ? liveGroups(tx, tabId, group, held).then(rows => rows.map(row => toPart(kind, row))): where[kind] === null ? Promise.resolve([]): delegate(tx, kind).findMany({ where: { tabId, ...where[kind] as Where } }).then(rows => rows.map(row => toPart(kind, row)))))return { tabId, head: toPart<HeadData>('head', head as unknown as PartRow), members, agents, groups, machines, starts, progress,@@ -41,25 +43,30 @@ async function assemble(tx: Transaction, tabId: string, where: Record<Exclude<Pa* plus the groups named by id, the groups the named agents are in and the named projects' groups (dormant ones too),* and the finished starts named by id or by the agent they started.*/-export function loadActive(tx: Transaction, tabId: string,+export async function loadActive(tx: Transaction, tabId: string,opts: { groupIds?: string[]; agentIds?: string[]; projectIds?: string[]; startIds?: string[]; startAgentIds?: string[]/** Also the finished starts not yet expired (finished within 7 days), as a first load shows them (design section 4.3). */includeRecentStarts?: boolean/** Every live group, dormant ones too: for the rare reads that list across all of them (mentions, route decisions). */- allGroups?: boolean } = {}): Promise<SwarmParts> {+ allGroups?: boolean } = {}, held: HeldParts | null = COMMAND_HELD): Promise<SwarmParts> {const live = { deleted: false }const or = (always: Where, ...named: Array<Where | false>) => ({ ...live, OR: [always, ...named.filter((w): w is Where => !!w)] })// A group stored with no project belongs to the default one (projects.ts groupProject), as in a Swarm from before projects;// a null match is a missing field too.const projectIds = [...opts.projectIds ?? [], ...opts.projectIds?.includes('lead') ? [null] : []]- return assemble(tx, tabId, {+ // The awake groups through the held index, held (task #3146): each command read every one of them whole. Every live group+ // (allGroups) is a rare read, made whole as before.+ const read = held && !opts.allGroups ? held.read(tabId) : undefined+ const parts = await assemble(tx, tabId, {member: live, agent: live, machine: live, progress: live,- group: opts.allGroups ? [['live', {}]] : [['dormancy', { dormant: false }], ...opts.groupIds?.length ? [['key', { key: { $in: opts.groupIds } }] as const] : [],+ group: opts.allGroups ? [['live', {}]] : [['held', { dormant: false }], ...opts.groupIds?.length ? [['key', { key: { $in: opts.groupIds } }] as const] : [],...opts.agentIds?.length ? [['agents', { agentIds: { $in: opts.agentIds } }] as const] : [],...projectIds.length ? [['project', { projectId: { $in: projectIds } }] as const] : []],start: opts.includeRecentStarts ? live : or({ finishedAt: null }, !!opts.startIds?.length && { key: { in: opts.startIds } },!!opts.startAgentIds?.length && { agentId: { in: opts.startAgentIds } }),- })+ }, read)+ read?.done()+ return parts}/**diff --git a/docs/plans/2026-11-05-command-group-reads-plan.md b/docs/plans/2026-11-05-command-group-reads-plan.mdnew file mode 100644index 000000000..8b4477903--- /dev/null+++ b/docs/plans/2026-11-05-command-group-reads-plan.md@@ -0,0 +1,83 @@+# Commands stop reading every awake conversation: plan++> Written by the task's Engineer (Linden) and implemented by it, task by task, test first. Task #3146.++**Goal:** Bring `POST commands` p95 back under its 475 ms guard at peak by taking the whole-conversation read off every+command, making the unread-activity digest read only what it returns, and keeping the receipt sweep's deletes short.++**Architecture:** Every command loads its Swarm in a transaction (`store.ts loadRow` → `parts/load.ts loadActive`). That+load reads every awake conversation whole through the `dormancy` index: 180 documents of about 5.6 KB in production's+busiest Swarm, about 1 MB a command, to find the one or two that changed. Machine reports already avoid this with held+parts (`parts/held.ts`, #2597): list each part's version columns, and read whole only the parts whose columns moved. The+command load uses the same mechanism through its own `HeldParts` (so the reports' and commands' selections do not evict+each other), and a new index holds every column the listing reads, so the listing reads no document at all.++**Tech Stack:** TypeScript, Prisma 5 on MongoDB 7/8 (`aggregateRaw` with a named index hint), Vitest, mongodb-memory-server.++**Spec:** Task #3146 (Ken's 13:20 VN production scan in Watch Production #2773).++## Global Constraints++- No new settings or switches; optimizations ship on.+- Production: read-only login, explain() and `$planCacheStats` only. Each index build is stated with its size and build+ time and needs Ken's go; the backend release's schema step builds it.+- A hinted index the database lacks is never named (`parts/indexed.ts readBy` lists indexes first), so a pod running before+ the schema step, or after a rollback, reads as before.+- Swarm code stays out of `cli/src/core`. Temp files only in `mktemp -d` folders.++## Review Focus++- A command must load exactly what it loaded before: held records are frozen with the parts (`store.ts loadedFrom`) and+ are only reused at an unchanged version, changedAt and dormancy.+- The listing must be covered: no `FETCH` for the awake-conversation listing.+- The activity digest returns the same ids in the same order as the Prisma read it replaces.+- The receipt sweep removes at the same rate, in shorter deletes.++## Measured before (production, read-only, 2026-11-05 ~13:40 VN)++| Shape | Plan | Examined | Returned | Time |+|---|---|---|---|---|+| A command's awake conversations (busiest Swarm) | IXSCAN `tabId_dormant_changedAt`, FETCH | 180 docs (~1 MB) | 180 | every command |+| Unread activity digest, Prisma `$expr` (busiest person) | IXSCAN `tabId_personId_createdAt`, FETCH | 865 keys, 865 docs | 26 | 23 ms |+| Same, plain `$match` | same index | 865 / 865 | 26 | 5 ms |+| Dormant page of a project | IXSCAN `tabId_projectId_dormant_lastActivityAt_key` | 50 / 50 | 50 | 41 ms |+| Quiet conversations since Monday (board, on a cache miss) | same index | 1,779 / 1,779 | 1,779 | 12 ms |+| Receipt sweep delete | `_id` | 500 docs a delete | | max 5.8 s in Atlas |++## Task 1: Commands list awake conversations from a covering index++**Files:** `backend/src/superTabs/parts/held.ts`, `backend/src/superTabs/parts/load.ts`,+`backend/src/superTabs/parts/indexed.ts`, `backend/prisma/schema.prisma`, `backend/integration/superTabCommandLoads.test.ts`++**Interfaces:** `GROUP_INDEX.held = { tabId: 1, dormant: 1, deleted: 1, key: 1, version: 1, changedAt: 1 }`;+`loadActive(tx, tabId, opts, held = COMMAND_HELD)`; `HeldRead.groups` lists groups with `{ key, version, changedAt, dormant }`.++- [ ] Failing test: a warm command in a Swarm with 150 awake conversations examines at most a few SwarmGroup documents+ (today: every awake one), and its listing is `PROJECTION_COVERED`.+- [ ] Failing test: after every kind of conversation write, a command's load through held parts equals a load holding nothing.+- [ ] Code: the index, the group columns without `readAt`, `loadActive` through `COMMAND_HELD`.++## Task 2: The unread-activity digest reads only unread rows++**Files:** `backend/src/superTabs/activity.ts`, `backend/src/superTabs/parts/indexed.ts`, `backend/prisma/schema.prisma`,+`backend/integration/superTabActivity.test.ts`++**Interfaces:** `NAMED_INDEX.SuperTabActivity.unread = { tabId: 1, personId: 1, readAt: 1, createdAt: 1 }`;+`activityDigest(tx, tabId, personId, now)` unchanged.++- [ ] Failing test: with 300 read rows and 3 unread, the digest examines at most the unread rows, through `unread`.+- [ ] Code: `activityDigest` as a plain `aggregateRaw` `$match` (`readAt: null` matches a missing field too), named index.++## Task 3: The receipt sweep deletes in small spaced chunks++**Files:** `backend/src/lib/productUsageReceiptSweep.ts`, `backend/src/lib/productUsageReceiptSweep.test.ts`++**Interfaces:** `RECEIPT_SWEEP = { chunk: 50, chunks: 10, pauseMs: 2_000, everyMs: 30_000 }`.++- [ ] Failing test: one pass sends no delete of more than 50 receipts, waits between deletes, finishes inside its 30 s+ period, and still removes 500 a pass.+- [ ] Code: the new constants.++## Task 4: Evidence++- [ ] Red/green and the before/after explain() of each shape in a PR comment; the index sizes and build times for Ken.