Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

A Swarm's newest change reads one index key per collection, and the list caches it (DB 2.4) (#2383)

0xken158e5f8ef9f58e
diff --git a/backend/integration/superTabHotReads.test.ts b/backend/integration/superTabHotReads.test.ts
index 635c7bd17..b54de1329 100644
--- a/backend/integration/superTabHotReads.test.ts
+++ b/backend/integration/superTabHotReads.test.ts
@@ -177,6 +177,49 @@ it('reuses the host timestamp at the same stamp, including a repeated wake with
expect((await direct.changes(owner, s.machineId, { full: true })).tabs).toEqual(changed.tabs)
})
+/** What lastChangedAt answered before it read one key per Swarm: the newest changedAt over every part, by full scan. */
+async function newestByScan(tabIds: string[]): Promise<Map<string, number>> {
+ const newest = new Map<string, number>()
+ for (const name of ['SwarmHead', 'SwarmMember', 'SwarmAgent', 'SwarmGroup', 'SwarmMachine', 'SwarmStart', 'SwarmProgress']) {
+ const reply = await db.$runCommandRaw({ aggregate: name, cursor: {}, pipeline: [{ $match: { tabId: { $in: tabIds } } },
+ { $group: { _id: '$tabId', at: { $max: '$changedAt' } } }] }) as unknown as { cursor: { firstBatch: Array<{ _id: string; at: { $date: string } | null }> } }
+ for (const row of reply.cursor.firstBatch) {
+ const at = row.at ? Date.parse(row.at.$date) : 0
+ if (at > (newest.get(row._id) ?? 0)) newest.set(row._id, at)
+ }
+ }
+ return newest
+}
+
+it('reads each Swarm\'s newest change from one index key per part collection, with the answer a full scan gives', async () => {
+ const a = await scene(), b = await scene()
+ clock = Date.now()
+ await store.execute(owner, b.tabId, { operationId: randomUUID(), command: {
+ type: 'message.send', conversationId: b.web, text: 'A newer change', replyTo: null, recipientIds: [] } })
+ const expected = await newestByScan([a.tabId, b.tabId])
+ const entries = await profiled(async () => expect(await lastChangedAt(db, [a.tabId, b.tabId])).toEqual(expected))
+ const scans = entries.filter(e => timestampScans([e]).length)
+ expect(scans).toHaveLength(7)
+ // A busy Swarm's scan read every key of its parts (prod's largest: 1,300 groups, 873 starts, 721 agents, 2026-10-30).
+ for (const scan of scans) expect(scan.keysExamined ?? 0, `${scan.command?.aggregate} ${scan.planSummary}`).toBeLessThanOrEqual(4)
+ // A head is one document per Swarm, so its plan may stay an IXSCAN; collections with many parts per Swarm skip them.
+ expect(scans.find(s => s.command?.aggregate === 'SwarmGroup')?.planSummary).toMatch(/DISTINCT_SCAN/)
+})
+
+it('lists Swarms with their newest change read once per stamp, not on every list', async () => {
+ const s = await scene()
+ const reads = new SwarmReads(store, { now: () => Date.now() })
+ swarmReadCache()!.drop(s.tabId)
+ const first = await reads.list(owner)
+ expect(first.tabs.find(t => t.id === s.tabId)?.lastChangedAt).toBe((await newestByScan([s.tabId])).get(s.tabId))
+ expect(timestampScans(await profiled(() => reads.list(owner)))).toEqual([])
+ clock = Date.now()
+ await store.execute(owner, s.tabId, { operationId: randomUUID(), command: {
+ type: 'message.send', conversationId: s.web, text: 'Listed again', replyTo: null, recipientIds: [] } })
+ const again = await reads.list(owner)
+ expect(again.tabs.find(t => t.id === s.tabId)?.lastChangedAt).toBe((await newestByScan([s.tabId])).get(s.tabId))
+})
+
describe('delivery authority reads', () => {
it.each(['person', 'agent'] as const)('claim, hold and ack from a %s leave unrelated conversations and machine parts unread', async sender => {
const s = await scene()
diff --git a/backend/integration/superTabRead.test.ts b/backend/integration/superTabRead.test.ts
index c4b2bd6eb..e8d7464f3 100644
--- a/backend/integration/superTabRead.test.ts
+++ b/backend/integration/superTabRead.test.ts
@@ -1221,13 +1221,13 @@ describe('what a read costs the database (group 3b)', () => {
const s = await scene(f => { f.contribute(machineId, 'swarm') }, machineId)
const ops: string[] = []
const client = db.$extends({ query: { $allModels: { async $allOperations({ model, operation, args, query }) {
- ops.push(`${model}.${operation}${operation === 'aggregateRaw' && JSON.stringify(args).includes('"$max"') ? ':max' : ''}`)
+ ops.push(`${model}.${operation}${operation === 'aggregateRaw' && JSON.stringify(args).includes('"$first":"$changedAt"') ? ':newest' : ''}`)
return query(args)
} } } }) as unknown as PrismaClient
const hosts = new HostReads(new SuperTabStore(client), { now: () => clock })
clock = Date.now()
// The newest change is one Mongo `$group` per part kind, and never a Prisma groupBy.
- const newestReads = () => ops.filter(op => /^Swarm(Head|Member|Agent|Group|Machine|Start|Progress)\.aggregateRaw:max$/.test(op))
+ const newestReads = () => ops.filter(op => /^Swarm(Head|Member|Agent|Group|Machine|Start|Progress)\.aggregateRaw:newest$/.test(op))
const first = await hosts.changes(owner, machineId, { full: true })
expect(newestReads()).toHaveLength(7)
expect(ops.filter(op => op.endsWith('.groupBy'))).toEqual([])
diff --git a/backend/src/superTabs/parts/load.ts b/backend/src/superTabs/parts/load.ts
index 5a5f17e84..df4806954 100644
--- a/backend/src/superTabs/parts/load.ts
+++ b/backend/src/superTabs/parts/load.ts
@@ -199,8 +199,10 @@ export async function lastChangedAt(tx: Transaction, tabIds: string[]): Promise<
const newest = new Map<string, number>()
if (!tabIds.length) return newest
for (const kind of PART_KINDS) {
+ // Sorted as the (tabId, changedAt) index is, read backwards, the newest per Swarm is one key (DISTINCT_SCAN): a $max read every key of
+ // the Swarm's parts (prod's largest: 1,300 groups, 873 starts, 721 agents, 2026-10-30), on every list.
const rows = await delegate(tx, kind).aggregateRaw({ pipeline: [{ $match: { tabId: { $in: tabIds } } },
- { $group: { _id: '$tabId', at: { $max: '$changedAt' } } }] }) as Array<{ _id: string; at: unknown }>
+ { $sort: { tabId: -1, changedAt: -1 } }, { $group: { _id: '$tabId', at: { $first: '$changedAt' } } }] }) as Array<{ _id: string; at: unknown }>
for (const row of rows) {
const at = row.at ? ms(row.at) : 0
if (at > (newest.get(row._id) ?? 0)) newest.set(row._id, at)
diff --git a/backend/src/superTabs/read/service.ts b/backend/src/superTabs/read/service.ts
index 89b65f2ca..51fbebedb 100644
--- a/backend/src/superTabs/read/service.ts
+++ b/backend/src/superTabs/read/service.ts
@@ -14,7 +14,7 @@ import { readBy } from '../parts/indexed.js'
import type { SuperTabStore } from '../store.js'
import { transaction } from '../store.js'
import type { CacheView } from '../cache.js'
-import { readChanged, readLastChangedAt, readLight, readLightGroups, readParts } from '../cachedReads.js'
+import { readChanged, readLastChangedAt, readLastChangedAts, readLight, readLightGroups, readParts } from '../cachedReads.js'
import { migrateSwarm } from '../migration/migrate.js'
import * as parts from '../parts/index.js'
import type { GroupData, Part, SwarmParts } from '../parts/index.js'
@@ -303,7 +303,9 @@ export class SwarmReads {
const legacy = unsplit.length ? await this.db.superTab.findMany({ where: { id: { in: unsplit } }, select: { id: true, state: true } }) : []
const ids = page.filter(r => r.format === 2).map(r => r.id)
// Every head changed after the epoch: one read of each listed Swarm's head part.
- const [heads, newest] = await transaction(this.db, tx => Promise.all([parts.loadChanged(tx, ids, new Date(0), ['head']), parts.lastChangedAt(tx, ids)]))
+ // Each Swarm's newest change through the read cache, at its stamp: a list every few seconds no longer recomputes it.
+ const [heads, newest] = await Promise.all([transaction(this.db, tx => parts.loadChanged(tx, ids, new Date(0), ['head'])),
+ this.store.view(ids).then(view => readLastChangedAts(this.db, view, ids))])
const mentions = withMentions ? await this.mentionCounts(user, page.map(r => r.id)) : undefined
return { tabs: page.map(row => {
const head = (heads.get(row.id)?.[0]?.data ?? legacy.find(l => l.id === row.id)?.state) as { name: string; paused: boolean }