Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(backend): a person's invitation list reads through the invitedEmails index by name (313 ms planning → ~0.1 ms) (#2020)

0xken150a023916191f
diff --git a/backend/integration/superTabInvitationPlans.test.ts b/backend/integration/superTabInvitationPlans.test.ts
new file mode 100644
index 000000000..dab3b8222
--- /dev/null
+++ b/backend/integration/superTabInvitationPlans.test.ts
@@ -0,0 +1,184 @@
+import { randomUUID } from 'node:crypto'
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import { MongoMemoryReplSet } from 'mongodb-memory-server-core'
+import { PrismaClient } from '@prisma/client'
+import type { AuthUser } from '../src/lib/ssoAuth.js'
+import { SuperTabStore } from '../src/superTabs/store.js'
+import { invitedSwarms } from '../src/superTabs/invitedSwarms.js'
+import type { Command } from '../src/superTabs/commands.js'
+import { logger } from '../src/utils/logger.js'
+import { PROFILE_STAGES, queryShape } from './bench/profile.js'
+
+// Ken's prod profile (2026-10-28 20:12): a person's invitation list spent 313 ms planning its SuperTab read. Prisma sends
+// `invitedEmails: { has }` as $expr, which only the autonomousEnv prefix of an index can bound, so the planner races every
+// autonomousEnv index over every Swarm of the environment. The read runs here under Mongo's profiler with the plan cache
+// emptied first: it must name its index (never planned, no $expr), read only the Swarms it returns, and list the same
+// invitations as before. A fresh loopback replica set; never DATABASE_URL.
+let replica: MongoMemoryReplSet
+let db: PrismaClient
+let store: SuperTabStore
+const DAY = 86_400_000
+const NOW = Date.parse('2026-10-28T13:00:00.000Z')
+let clock = NOW
+const owner: AuthUser = { sub: '100000000000000000000001', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
+const devOwner: AuthUser = { ...owner, autonomousEnv: 'dev' }
+/** Signs in with their address in other case and spacing than the invitations hold it. */
+const invited: AuthUser = { sub: '100000000000000000000009', email: ' [email protected] ', autonomousEnv: 'prod', role: 'user' }
+/** Other Swarms of the environment, each with a member, a machine and someone else invited: what prod's planner raced over. */
+const OTHERS = 400
+/** The index the read names: by Swarm id within an email, the order Prisma's read capped them in. */
+const INVITED = { autonomousEnv: 1, invitedEmails: 1, _id: 1 }
+
+beforeAll(async () => {
+ replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },
+ replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })
+ const uri = replica.getUri(`swarm_invite_plans_${randomUUID().replaceAll('-', '_')}`)
+ await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {
+ env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,
+ })
+ db = new PrismaClient({ datasources: { db: { url: uri } } })
+ store = new SuperTabStore(db, () => clock)
+}, 120000)
+afterAll(async () => {
+ await db?.$disconnect()
+ await replica?.stop()
+})
+
+const run = (user: AuthUser, tab: string, command: Command) => store.execute(user, tab, { operationId: randomUUID(), command })
+/** A Swarm of `user`'s that invites [email protected] as `role`: its invitation's id and expiry. */
+async function inviting(user: AuthUser, name: string, role: 'collaborator' | 'viewer') {
+ const tabId = (await store.create(user, randomUUID(), name)).id
+ const inviteId = (await run(user, tabId, { type: 'invite.create', email: '[email protected]', role, conversationId: 'lead' })).result.inviteId as string
+ return { tabId, name, inviteId, role, expiresAt: clock + 7 * DAY }
+}
+
+/** One read of `collection` as the profiler saw it, with the plan cache emptied first: a read a test's planner serves from
+ * its cache is planned again in prod whenever the cache misses. */
+interface Read { shape: string; plans: string[]; multiPlanned: boolean; planningMicros: number; docsExamined: number; returned: number }
+async function profiled(collection: string, read: () => Promise<unknown>): Promise<Read[]> {
+ await db.$runCommandRaw({ planCacheClear: collection })
+ await db.$runCommandRaw({ profile: 0 })
+ await db.$runCommandRaw({ drop: 'system.profile' }).catch(() => {})
+ await db.$runCommandRaw({ create: 'system.profile', capped: true, size: 64 * 1024 * 1024 })
+ await db.$runCommandRaw({ profile: 2, slowms: 0 })
+ try {
+ await read()
+ } finally {
+ await db.$runCommandRaw({ profile: 0 })
+ }
+ const stages = [{ $match: { ns: { $regex: `\\.${collection}$` }, op: { $in: ['command', 'getmore', 'query'] } } },
+ { $project: { ...PROFILE_STAGES[0]!.$project, queryHash: 1, fromMultiPlanner: 1, planningTimeMicros: 1 } }]
+ const rows = (await db.$runCommandRaw({ aggregate: 'system.profile', pipeline: stages, cursor: { batchSize: 100000 } }) as unknown as
+ { cursor: { firstBatch: Array<{ match?: unknown; command?: { filter?: unknown }; planSummary?: string; docsExamined?: number; nreturned?: number
+ queryHash?: string; fromMultiPlanner?: boolean; planningTimeMicros?: number }> } }).cursor.firstBatch
+ const reads = new Map<string, Read>()
+ for (const row of rows) {
+ const id = row.queryHash ?? `${reads.size}`
+ const filter = row.match ?? row.command?.filter
+ const read = reads.get(id) ?? { shape: '', plans: [], multiPlanned: false, planningMicros: 0, docsExamined: 0, returned: 0 }
+ if (filter !== undefined && !read.shape) read.shape = queryShape(filter)
+ if (row.planSummary && !read.plans.includes(row.planSummary)) read.plans.push(row.planSummary)
+ read.multiPlanned ||= !!row.fromMultiPlanner
+ read.planningMicros += Number(row.planningTimeMicros ?? 0)
+ read.docsExamined += Number(row.docsExamined ?? 0)
+ read.returned += Number(row.nreturned ?? 0)
+ reads.set(id, read)
+ }
+ return [...reads.values()]
+}
+
+describe('a person\'s invitation list (Ken\'s prod profile, 2026-10-28)', () => {
+ let open: Array<Awaited<ReturnType<typeof inviting>>>
+ beforeAll(async () => {
+ // An invitation that expired, one accepted (no longer pending) and one in another environment: none is listed.
+ clock = NOW - 10 * DAY
+ await inviting(owner, 'Expired', 'viewer')
+ clock = NOW - DAY
+ const accepted = await inviting(owner, 'Accepted', 'collaborator')
+ await run({ ...invited, email: '[email protected]' }, accepted.tabId, { type: 'invite.accept', inviteId: accepted.inviteId })
+ await inviting(devOwner, 'Elsewhere', 'collaborator')
+ open = [await inviting(owner, 'Alpha', 'collaborator'), await inviting(owner, 'Beta', 'viewer')]
+ clock = NOW
+ await db.superTab.createMany({ data: Array.from({ length: OTHERS }, (_, i) => ({ id: `other-${String(i).padStart(3, '0')}`, ownerId: owner.sub,
+ autonomousEnv: 'prod', memberIds: [owner.sub, `member-${i}`], invitedEmails: [`someone-${i}@example.com`], machineIds: [`machine-${i}`],
+ workspaceId: `workspace-${i}`, format: 2, state: { version: 2 } })) })
+ })
+
+ it('lists the pending invitations of its environment, as before', async () => {
+ const listed = await store.invitations(invited)
+ expect(listed.sort((a, b) => a.name.localeCompare(b.name))).toEqual(open.map(i => ({ tabId: i.tabId, name: i.name, inviteId: i.inviteId,
+ role: i.role, expiresAt: i.expiresAt })))
+ })
+
+ it('reads the Swarms that invite them through the invitedEmails index by name, examining only those', async () => {
+ const reads = await profiled('SuperTab', () => store.invitations(invited))
+ expect(reads).toHaveLength(1)
+ const [read] = reads
+ expect(read!.shape, JSON.stringify(read)).not.toContain('$expr')
+ expect(read!.multiPlanned, JSON.stringify(read)).toBe(false)
+ expect(read!.plans).toEqual(['IXSCAN { autonomousEnv: 1, invitedEmails: 1, _id: 1 }'])
+ // Alpha, Beta, and the expired one: its expiry is in the Swarm's head, read after.
+ expect(read!.docsExamined, JSON.stringify(read)).toBeLessThanOrEqual(3)
+ })
+
+ // A rollback or a database whose index was never built: naming a missing index fails the read, so it reads without it.
+ it('answers without the index when the database lacks it, and says so', async () => {
+ const said = vi.spyOn(logger, 'error')
+ await withoutIndex(async () => {
+ const listed = await store.invitations(invited)
+ expect(listed.map(i => i.tabId).sort()).toEqual(open.map(i => i.tabId).sort())
+ expect(said).toHaveBeenCalledWith(expect.stringContaining('lacks the invitedEmails index'), { index: INVITED })
+ }).finally(() => said.mockRestore())
+ })
+})
+
+/** Runs `body` while the database lacks the index the read names, as after a rollback. */
+async function withoutIndex(body: () => Promise<void>) {
+ const listed = await db.$runCommandRaw({ listIndexes: 'SuperTab' }) as unknown as { cursor: { firstBatch: Array<{ key: Record<string, number>; name: string }> } }
+ const index = listed.cursor.firstBatch.find(i => JSON.stringify(i.key) === JSON.stringify(INVITED))!
+ await db.$runCommandRaw({ dropIndexes: 'SuperTab', index: index.name })
+ try {
+ await body()
+ } finally {
+ await db.$runCommandRaw({ createIndexes: 'SuperTab', indexes: [{ key: INVITED, name: index.name }] })
+ }
+}
+
+// Review of #2020: Prisma's read sorted by Swarm id before taking 100. A read capped in another order lists another 100,
+// and so other invitations, once more than 100 Swarms invite one address.
+describe('more than 100 Swarms invite one address', () => {
+ const CROWD = 105
+ beforeAll(async () => {
+ // Stored in reverse id order: a read that takes the index's or the collection's own order takes the wrong 100.
+ await db.superTab.createMany({ data: Array.from({ length: CROWD }, (_, i) => ({ id: `crowd-${String(CROWD - 1 - i).padStart(3, '0')}`, ownerId: owner.sub,
+ autonomousEnv: 'prod', memberIds: [owner.sub], invitedEmails: ['[email protected]'], machineIds: [], workspaceId: 'workspace-crowd',
+ format: 2, state: { version: 2 } })) })
+ })
+ const first100 = Array.from({ length: 100 }, (_, i) => `crowd-${String(i).padStart(3, '0')}`)
+ /** The Swarms the read before #2020 listed: Prisma's own `has` filter and cap. */
+ const prismaRead = async () => (await db.superTab.findMany({ where: { autonomousEnv: 'prod', invitedEmails: { has: '[email protected]' } }, take: 100,
+ select: { id: true } })).map(r => r.id)
+
+ it('reads the first 100 by Swarm id, as Prisma did, from the index without sorting them all', async () => {
+ let ids: string[] = []
+ const reads = await profiled('SuperTab', async () => { ids = (await invitedSwarms(db, 'prod', '[email protected]', 100)).map(r => r.id) })
+ expect(ids).toEqual(first100)
+ expect(await prismaRead()).toEqual(first100)
+ expect(reads).toHaveLength(1)
+ expect(reads[0]!.multiPlanned, JSON.stringify(reads[0])).toBe(false)
+ expect(reads[0]!.plans).toEqual(['IXSCAN { autonomousEnv: 1, invitedEmails: 1, _id: 1 }'])
+ expect(reads[0]!.docsExamined, JSON.stringify(reads[0])).toBeLessThanOrEqual(100)
+ })
+
+ it('reads the same 100 in the same order without the index', async () => {
+ const said = vi.spyOn(logger, 'error').mockImplementation(() => {})
+ await withoutIndex(async () => {
+ expect((await invitedSwarms(db, 'prod', '[email protected]', 100)).map(r => r.id)).toEqual(first100)
+ expect(await prismaRead()).toEqual(first100)
+ }).finally(() => said.mockRestore())
+ })
+})
diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma
index 25b04484f..91f476e28 100644
--- a/backend/prisma/schema.prisma
+++ b/backend/prisma/schema.prisma
@@ -1021,7 +1021,8 @@ model SuperTab {
@@index([autonomousEnv, workspaceId])
@@index([autonomousEnv, memberIds])
- @@index([autonomousEnv, invitedEmails])
+ /// By id within an address: the invitation list reads the first 100 in id order (superTabs/invitedSwarms.ts).
+ @@index([autonomousEnv, invitedEmails, id])
@@index([autonomousEnv, machineIds])
@@index([autonomousEnv, cleanupMachineIds])
}
diff --git a/backend/src/superTabs/invitedSwarms.ts b/backend/src/superTabs/invitedSwarms.ts
new file mode 100644
index 000000000..ebcaef826
--- /dev/null
+++ b/backend/src/superTabs/invitedSwarms.ts
@@ -0,0 +1,31 @@
+import type { PrismaClient } from '@prisma/client'
+import { logger } from '../utils/logger.js'
+
+// Ken's prod profile (2026-10-28): a person's invitation list spent 313 ms planning its SuperTab read. Prisma sends
+// `invitedEmails: { has }` as $expr, which bounds only the autonomousEnv prefix every SuperTab index shares: the planner raced
+// them all and picked the _id index, reading every Swarm of the environment. This read is a plain match that names the
+// invitedEmails index, so it is never planned and examines only the Swarms that invite the person. Prisma sorted by id before
+// its cap; the index holds each address's Swarms in id order, so the same first 100 come off it without a sort
+// (integration/superTabInvitationPlans.test.ts).
+
+/** The SuperTab index the read names, as prisma/schema.prisma declares it. */
+const INVITED = { autonomousEnv: 1, invitedEmails: 1, _id: 1 }
+
+/** The first `limit` Swarms by id of `autonomousEnv` with a pending invitation for `email` (already lowercased), as Prisma read them. */
+export async function invitedSwarms(db: Pick<PrismaClient, 'superTab'>, autonomousEnv: string, email: string, limit: number):
+ Promise<Array<{ id: string; format: number }>> {
+ const pipeline = [{ $match: { autonomousEnv, invitedEmails: email } }, { $sort: { _id: 1 } }, { $limit: limit }, { $project: { format: 1 } }]
+ const read = async (hint?: Record<string, number>) => await db.superTab.aggregateRaw({ pipeline, ...hint ? { options: { hint } } : {} }) as unknown as
+ Array<Record<string, unknown>>
+ let rows: Array<Record<string, unknown>>
+ try {
+ rows = await read(INVITED)
+ } catch (error) {
+ // A database without the index (a rollback, never pushed): naming it fails the read, which then plans as before.
+ if (!/hint provided does not correspond to an existing index/.test(error instanceof Error ? error.message : String(error))) throw error
+ logger.error('[swarm-reads] SuperTab lacks the invitedEmails index; invitation lists read without it', { index: INVITED })
+ rows = await read()
+ }
+ // A Swarm stored before `format` existed is whole in `state` (schema default 1).
+ return rows.map(row => ({ id: row._id as string, format: Number(row.format ?? 1) }))
+}
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index 0e888e57f..773b7a00a 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -48,6 +48,7 @@ import { closeAssignment, closeUnassignedStart, readDecision, recentDecisions, t
import { listMentions, markDone, mentionsWaiting, openTags, readMention, resolveByReply, writeMentions } from './mentions.js'
import { answerable, fullFor, listDecisions, type DecisionList, readAskedDecision, recordAnswer, repeatOf, waitingIn, waitingInProject, writeDecisions } from './decisions.js'
import { DEFAULT_ROUTING_POLICY, effectivePolicy, routingPolicySchema, type RoutingPolicy } from './routingPolicy.js'
+import { invitedSwarms } from './invitedSwarms.js'
import { authenticatedEmail, authorizeSwarmJoin, configureSharing, joinSwarmSchema, requireSharingOwner,
saveSharingSchema, sharingDetails, type JoinSwarm, type SaveSharing } from './sharing.js'
@@ -687,7 +688,7 @@ export class SuperTabStore {
async invitations(user: AuthUser) {
this.personOnly(user)
const email = user.email.trim().toLowerCase()
- const rows = await this.db.superTab.findMany({ where: { autonomousEnv: user.autonomousEnv, invitedEmails: { has: email } }, take: 100, select: SWARM_ROW })
+ const rows = await invitedSwarms(this.db, user.autonomousEnv, email, 100)
const heads = await this.heads(rows)
return rows.flatMap(row => {
const state = heads.get(row.id)
@@ -698,7 +699,7 @@ export class SuperTabStore {
}
/** The heads of Swarms already split into parts, by id. */
- private async heads(rows: SwarmRow[]): Promise<Map<string, HeadData>> {
+ private async heads(rows: Array<Pick<SwarmRow, 'id' | 'format'>>): Promise<Map<string, HeadData>> {
const heads = await this.db.swarmHead.findMany({ where: { id: { in: rows.filter(r => r.format === 2).map(r => `${r.id}:head`) } } })
return new Map(heads.map(head => [head.tabId, head.data as unknown as HeadData]))
}
diff --git a/backend/vitest.super-tabs.config.ts b/backend/vitest.super-tabs.config.ts
index e058983ee..45f795a6c 100644
--- a/backend/vitest.super-tabs.config.ts
+++ b/backend/vitest.super-tabs.config.ts
@@ -3,5 +3,5 @@ import { defineConfig } from 'vitest/config'
export default defineConfig({ test: { include: ['integration/superTabs.test.ts', 'integration/superTabSharing.test.ts', 'integration/superTabGithub.test.ts', 'integration/superTabTeam.test.ts', 'integration/superTabWorkspaces.test.ts',
'integration/superTabRead.test.ts', 'integration/superTabLeaderboard.test.ts', 'integration/superTabMigration.test.ts', 'integration/superTabLiveness.test.ts', 'integration/superTabTidy.test.ts',
'integration/superTabProfile.test.ts', 'integration/superTabLegacyDrop.test.ts', 'integration/shareRaces.test.ts',
- 'integration/superTabIndexes.test.ts', 'integration/superTabHotReads.test.ts', 'integration/superTabGroupPlans.test.ts', 'integration/superTabCache.test.ts', 'integration/superTabObserver.test.ts', 'integration/productUsageConflicts.test.ts'], hookTimeout: 120000, testTimeout: 30000, setupFiles: ['integration/cacheOn.test-support.ts'],
+ 'integration/superTabIndexes.test.ts', 'integration/superTabHotReads.test.ts', 'integration/superTabGroupPlans.test.ts', 'integration/superTabInvitationPlans.test.ts', 'integration/superTabCache.test.ts', 'integration/superTabObserver.test.ts', 'integration/productUsageConflicts.test.ts'], hookTimeout: 120000, testTimeout: 30000, setupFiles: ['integration/cacheOn.test-support.ts'],
fileParallelism: false, env: { NODE_ENV: 'test', HARNESS_CREDENTIAL_ENCRYPTION_KEY: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' } } })