The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
perf(swarm): the host poll's Swarm lookup reads two machine indexes (DB roadmap Phase 3, part 3) (#2201)
0xken15b5264329a301
diff --git a/backend/integration/superTabHotReads.test.ts b/backend/integration/superTabHotReads.test.tsindex 3353ef9d9..635c7bd17 100644--- a/backend/integration/superTabHotReads.test.ts+++ b/backend/integration/superTabHotReads.test.ts@@ -12,6 +12,7 @@ import { listGroupIndexes } from '../src/superTabs/parts/indexed.js'import { migrateSwarm } from '../src/superTabs/migration/migrate.js'import { waitingIn } from '../src/superTabs/decisions.js'import { HostReads } from '../src/superTabs/read/host.js'+import { machineSwarms, swarmsIn } from '../src/superTabs/swarmRows.js'import { SwarmReads } from '../src/superTabs/read/service.js'import { cursorFor } from '../src/superTabs/read/cursor.js'import { lastChangedAt } from '../src/superTabs/parts/index.js'@@ -324,3 +325,64 @@ describe('the owner\'s grid poll', () => {expect((await store.hostGrids(owner)).find(g => g.tabId === s.tabId)!.members).toEqual([peer.email])})})++// Roadmap Phase 3.1: the host poll's Swarm lookup, once a second per host. Prisma sent `machineIds has` OR `cleanupMachineIds has`+// as $expr, sorted by updatedAt, which no index held: the slow log showed 17 Swarms examined per one returned.+describe('the host poll\'s Swarm lookup', () => {+ it('reads only the Swarms that list this machine, newest first, by the machine indexes', async () => {+ // Swarms of other machines: a lookup the machine does not bound examines these too.+ for (let i = 0; i < 6; i++) await scene()+ const s = await scene(), also = await scene({ sharedMachine: s.machineId })+ // A Swarm that removed this machine still lists it for cleanup until its host has cleaned up.+ const cleanup = await scene()+ const at = Date.now()+ await db.superTab.update({ where: { id: cleanup.tabId }, data: { machineIds: { set: [cleanup.machineId] }, cleanupMachineIds: { set: [s.machineId] },+ updatedAt: new Date(at + 30_000) } })+ await db.superTab.update({ where: { id: also.tabId }, data: { updatedAt: new Date(at + 60_000) } })+ await db.superTab.update({ where: { id: s.tabId }, data: { updatedAt: new Date(at + 10_000) } })+ const hosts = new HostReads(store, { now: () => Date.now() })+ let tabs: string[] = []+ const entries = await profiled(async () => { tabs = (await hosts.changes(owner, s.machineId, { full: true })).tabs.map(t => t.tabId) })+ // Newest write first, across both indexes: the cleanup Swarm sits between the two that list it as a machine.+ expect(tabs).toEqual([also.tabId, cleanup.tabId, s.tabId])+ const lookups = entries.filter(e => e.ns.endsWith('.SuperTab')).filter(e => /machineIds|cleanupMachineIds/.test(JSON.stringify(e.command?.pipeline ?? e.command?.filter ?? {})))+ expect(lookups.length, JSON.stringify(plan(lookups))).toBe(2)+ for (const lookup of lookups) {+ expect(lookup.docsExamined, JSON.stringify(plan(lookups))).toBeLessThanOrEqual(lookup.nreturned!)+ expect(lookup.command?.hint, `the lookup names its index: ${JSON.stringify(plan(lookups))}`).toBeTruthy()+ }+ })++ it('lists a Swarm found by both indexes once, and caps the merged list at the newest', async () => {+ const s = await scene(), both = await scene({ sharedMachine: s.machineId }), cleanupOnly = await scene()+ const at = Date.now()+ await db.superTab.update({ where: { id: s.tabId }, data: { updatedAt: new Date(at + 10_000) } })+ // Listed as a machine and for cleanup at once: found by both reads, returned once.+ await db.superTab.update({ where: { id: both.tabId }, data: { cleanupMachineIds: { set: [s.machineId] }, updatedAt: new Date(at + 20_000) } })+ await db.superTab.update({ where: { id: cleanupOnly.tabId }, data: { machineIds: { set: [cleanupOnly.machineId] },+ cleanupMachineIds: { set: [s.machineId] }, updatedAt: new Date(at + 30_000) } })+ expect(await machineSwarms(db, 'prod', s.machineId, 500)).toEqual([cleanupOnly.tabId, both.tabId, s.tabId])+ // Capped after merging: the newest two of the three, though each read alone returns up to two.+ expect(await machineSwarms(db, 'prod', s.machineId, 2)).toEqual([cleanupOnly.tabId, both.tabId])+ expect(await machineSwarms(db, 'prod', s.machineId, 1)).toEqual([cleanupOnly.tabId])+ })++ it('never returns a Swarm of another environment, one that dropped the machine, or one not yet in parts', async () => {+ const s = await scene()+ // The same machine id listed by a Swarm of another environment, as a machine and for cleanup.+ const dev = await scene({ sharedMachine: s.machineId }), devCleanup = await scene()+ await db.superTab.update({ where: { id: dev.tabId }, data: { autonomousEnv: 'dev' } })+ await db.superTab.update({ where: { id: devCleanup.tabId }, data: { autonomousEnv: 'dev', cleanupMachineIds: { set: [s.machineId] } } })+ // A Swarm whose host cleaned up after the machine left: it lists the machine nowhere.+ const dropped = await scene({ sharedMachine: s.machineId })+ await db.superTab.update({ where: { id: dropped.tabId }, data: { machineIds: { set: [] }, cleanupMachineIds: { set: [] } } })+ // A Swarm still in one row (format 1) is left for its migration.+ const legacy = await scene({ sharedMachine: s.machineId })+ await db.superTab.update({ where: { id: legacy.tabId }, data: { format: 1 } })+ expect(await machineSwarms(db, 'prod', s.machineId, 500)).toEqual([s.tabId])+ expect((await new HostReads(store, { now: () => Date.now() }).changes(owner, s.machineId, { full: true })).tabs.map(t => t.tabId)).toEqual([s.tabId])+ // The removed Swarms' check keeps only this environment's ids, whatever ids it is handed.+ expect(await swarmsIn(db, 'prod', [s.tabId, dev.tabId, devCleanup.tabId, 'no-such-swarm'])).toEqual([s.tabId])+ expect(await swarmsIn(db, 'prod', [])).toEqual([])+ })+})diff --git a/backend/integration/superTabRead.test.ts b/backend/integration/superTabRead.test.tsindex 90bdbe0cf..17110a6be 100644--- a/backend/integration/superTabRead.test.ts+++ b/backend/integration/superTabRead.test.ts@@ -1176,9 +1176,24 @@ describe('what a read costs the database (group 3b)', () => {const padding = 'old-state-'.repeat(2_000)await db.superTab.update({ where: { id: s.tabId }, data: { legacyState: { padding } } })const reads: Array<{ operation: string; select: Record<string, unknown> | undefined }> = []+ const rawRows: Array<Record<string, unknown>> = []+ // A named-index read (parts/indexed.ts readBy) names its fields in the $project that ends its pipeline: only an inclusion+ // projection counts, since one that only leaves fields out (`{ _id: 0 }`, `{ padding: 0 }`) still returns the old state.+ const included = (stage: { $project?: Record<string, unknown> } | undefined) => {+ const fields = Object.entries(stage?.$project ?? {})+ return fields.some(([, on]) => on === 1 || on === true)+ && fields.every(([field, on]) => on === 1 || on === true || (field === '_id' && (on === 0 || on === false))) ? stage!.$project : undefined+ }const client = db.$extends({ query: { $allModels: { async $allOperations({ model, operation, args, query }) {- if (model === 'SuperTab') reads.push({ operation, select: (args as { select?: Record<string, unknown> }).select })- return query(args)+ const raw = operation === 'aggregateRaw'+ if (model === 'SuperTab') {+ reads.push({ operation, select: raw ? included((args as { pipeline?: Array<{ $project?: Record<string, unknown> }> }).pipeline?.at(-1))+ : (args as { select?: Record<string, unknown> }).select })+ }+ const result = await query(args)+ // And what the database answered: no raw row holds the old state, whatever the pipeline says.+ if (model === 'SuperTab' && raw) rawRows.push(...result as unknown as Array<Record<string, unknown>>)+ return result} } } }) as unknown as PrismaClientconst hosts = new HostReads(new SuperTabStore(client), { now: () => clock })clock = Date.now()@@ -1189,10 +1204,12 @@ describe('what a read costs the database (group 3b)', () => {expect(first.tabs).toHaveLength(1)expect(JSON.stringify([first, idle])).not.toContain('old-state-')- // The poll reads the Swarm's row for its list and its owner, each time by naming the fields: none is the old state.+ // The poll reads the Swarm's row for its list and its owner, each time by naming the fields (Prisma's select, or a raw+ // read's closing inclusion $project): none is the old state, and no raw row the database returned holds it.expect(reads.length).toBeGreaterThan(0)for (const read of reads) expect(read.select, read.operation).toBeDefined()for (const read of reads) expect(read.select).not.toHaveProperty('legacyState')+ for (const row of rawRows) expect(Object.keys(row)).not.toContain('legacyState')})it('the host form asks for the newest change only of the Swarms it answers for', async () => {diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prismaindex f78c22d89..f27529564 100644--- a/backend/prisma/schema.prisma+++ b/backend/prisma/schema.prisma@@ -1055,8 +1055,9 @@ model SuperTab {@@index([autonomousEnv, memberIds, id])/// By id within an address: the invitation list reads the first 100 in id order (superTabs/invitedSwarms.ts).@@index([autonomousEnv, invitedEmails, id])- @@index([autonomousEnv, machineIds])- @@index([autonomousEnv, cleanupMachineIds])+ /// A machine's Swarms, newest first: the host poll's lookup, once a second per host (superTabs/swarmRows.ts machineSwarms).+ @@index([autonomousEnv, machineIds, updatedAt])+ @@index([autonomousEnv, cleanupMachineIds, updatedAt])}// A terminal retains its history when removed from a roster. These bindingsdiff --git a/backend/src/superTabs/hotFilters.allowed.json b/backend/src/superTabs/hotFilters.allowed.jsonindex ede6178eb..59225e045 100644--- a/backend/src/superTabs/hotFilters.allowed.json+++ b/backend/src/superTabs/hotFilters.allowed.json@@ -8,7 +8,6 @@"src/superTabs/migration/workspaces.ts swarmHead.findMany": 1,"src/superTabs/migration/workspaces.ts swarmMember.findMany": 1,"src/superTabs/native.ts superTabDelivery.updateMany": 2,- "src/superTabs/read/host.ts superTab.findMany": 3,"src/superTabs/read/host.ts swarmGroup.findMany": 1,"src/superTabs/read/host.ts swarmMachine.findMany": 2,"src/superTabs/read/service.ts superTab.findMany": 1,diff --git a/backend/src/superTabs/parts/indexed.ts b/backend/src/superTabs/parts/indexed.tsindex e3c828f2d..296084fcf 100644--- a/backend/src/superTabs/parts/indexed.ts+++ b/backend/src/superTabs/parts/indexed.ts@@ -36,6 +36,11 @@ export const NAMED_INDEX = {invited: { autonomousEnv: 1, invitedEmails: 1, _id: 1 },/** A member's Swarms in id order: Swarm lists and Workspace removals. */members: { autonomousEnv: 1, memberIds: 1, _id: 1 },+ /** A machine's Swarms, newest first, and those that still list it for cleanup: the host poll. */+ machines: { autonomousEnv: 1, machineIds: 1, updatedAt: 1 },+ cleanup: { autonomousEnv: 1, cleanupMachineIds: 1, updatedAt: 1 },+ /** Swarms by id. */+ id: { _id: 1 },},/** Each part kind's changes since a cursor; machines also by when they were last read. */SwarmHead: { changes: { tabId: 1, changedAt: 1 } },diff --git a/backend/src/superTabs/read/host.ts b/backend/src/superTabs/read/host.tsindex d187cc36e..4537442a0 100644--- a/backend/src/superTabs/read/host.ts+++ b/backend/src/superTabs/read/host.ts@@ -22,6 +22,7 @@ import { READ_FORMAT, type HostChanges, type HostChangesRequest } from './contraimport { cursorFor, readCursor } from './cursor.js'import { hostTab, type HostLookups } from './hostRows.js'import { confidentialSessions } from '../confidential.js'+import { machineSwarms, swarmsIn } from '../swarmRows.js'type Transaction = Prisma.TransactionClient/** The row hash's length at the front of an etag (read/hostEtag.ts); the input signature follows it. */@@ -56,9 +57,7 @@ export class HostReads {// Every agent's grant check of this read shares one read of each machine, share, contributor and Workspace membership.const grants = new GrantReads(owned)// Swarms still in one row are migrated when first opened; the job migrates every one before the new release serves.- const ids = (await tx.superTab.findMany({ where: { autonomousEnv: user.autonomousEnv, format: 2,- OR: [{ machineIds: { has: machineId } }, { cleanupMachineIds: { has: machineId } }] },- orderBy: { updatedAt: 'desc' }, take: FULL_TABS, select: { id: true } })).map(r => r.id)+ const ids = await machineSwarms(tx, user.autonomousEnv, machineId, FULL_TABS)const teams = isTeamSession(user) || this.options.teamAccountsIdle ? [] : await tx.machine.findMany({ where: { userId: user.sub,hostMachineId: machineId, sessionKind: 'team', autonomousEnv: user.autonomousEnv, ...machineAlive }, select: { machineId: true } })const full = 'reload' in from@@ -178,6 +177,6 @@ export class HostReads {const rows = await tx.swarmMachine.findMany({ where: { machineId, hostChangedAt: { gt: new Date(since) } }, select: { tabId: true } })const gone = [...new Set(rows.map(r => r.tabId))].filter(id => !listed.has(id))if (!gone.length) return []- return (await tx.superTab.findMany({ where: { id: { in: gone }, autonomousEnv }, select: { id: true } })).map(r => r.id)+ return swarmsIn(tx, autonomousEnv, gone)}}diff --git a/backend/src/superTabs/swarmRows.ts b/backend/src/superTabs/swarmRows.tsindex de3065463..94178ea88 100644--- a/backend/src/superTabs/swarmRows.ts+++ b/backend/src/superTabs/swarmRows.ts@@ -1,5 +1,6 @@import type { Prisma } from '@prisma/client'import { readBy } from './parts/indexed.js'+import { ms } from './parts/light.js'type Transaction = Prisma.TransactionClient@@ -21,3 +22,24 @@ export async function memberSwarms(db: Transaction, autonomousEnv: string, userIreturn rows.map(row => ({ id: row._id as string, ownerId: row.ownerId as string, revision: Number(row.revision ?? 0),format: Number(row.format ?? 1), workspaceId: (row.workspaceId ?? null) as string | null }))}++/**+ * The ids of up to `limit` Swarms in parts that list `machineId`, as a machine or for its cleanup, most recently written+ * first. Two reads on the machine indexes: through Prisma, the `has` OR `has` was one $expr the planner bounded by the+ * environment alone, 17 Swarms examined per one returned on the host poll (roadmap Phase 3.1).+ */+export async function machineSwarms(db: Transaction, autonomousEnv: string, machineId: string, limit: number): Promise<string[]> {+ const read = (index: 'machines' | 'cleanup', field: 'machineIds' | 'cleanupMachineIds') => readBy(db, 'SuperTab', index, [+ { $match: { autonomousEnv, [field]: machineId, format: 2 } }, { $sort: { updatedAt: -1 } }, { $limit: limit }, { $project: { updatedAt: 1 } }])+ const newest = new Map<string, number>()+ for (const row of [...await read('machines', 'machineIds'), ...await read('cleanup', 'cleanupMachineIds')]) {+ newest.set(row._id as string, Math.max(newest.get(row._id as string) ?? 0, ms(row.updatedAt)))+ }+ return [...newest].sort(([a, at], [b, bt]) => bt - at || (a < b ? -1 : 1)).slice(0, limit).map(([id]) => id)+}++/** Of these Swarm ids, those in `autonomousEnv`: a plain $in on the id index, never Prisma's $expr $or of each id. */+export async function swarmsIn(db: Transaction, autonomousEnv: string, ids: string[]): Promise<string[]> {+ if (!ids.length) return []+ return (await readBy(db, 'SuperTab', 'id', [{ $match: { _id: { $in: ids }, autonomousEnv } }, { $project: { _id: 1 } }])).map(row => row._id as string)+}