The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(backend): a message page reads its deliveries and reactions through an index (#1914)
0xken15fd45904d66b2
diff --git a/backend/integration/superTabIndexes.test.ts b/backend/integration/superTabIndexes.test.tsnew file mode 100644index 000000000..6e455eac4--- /dev/null+++ b/backend/integration/superTabIndexes.test.ts@@ -0,0 +1,108 @@+import { randomUUID } from 'node:crypto'+import { execFile } from 'node:child_process'+import { promisify } from 'node:util'+import { join } from 'node:path'+import { tmpdir } from 'node:os'+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { MongoMemoryReplSet } from 'mongodb-memory-server-core'+import { PrismaClient } from '@prisma/client'+import { pageDeliveries, pageReactions } from '../src/superTabs/pageReads.js'++// A fresh loopback replica set, created here. This suite never reads DATABASE_URL or an existing database.+// Prod Mongo spent hours an hour on the message page's delivery read (2026-10-28): Prisma sends `messageId in [...]` as an+// $expr $or, which no index can bound, so each page read every delivery of the Swarm. These tests read the Mongo profiler+// for the page reads against a Swarm of thousands of deliveries: each must use its index and examine only the rows it returns.+let replica: MongoMemoryReplSet+let db: PrismaClient+const pad = (n: number) => String(n).padStart(3, '0')+const MESSAGES = 300, AGENTS = 10+const PAGE = Array.from({ length: 100 }, (_, i) => `A-m${pad(100 + i)}`)+const T0 = Date.parse('2026-10-28T05:00:00.000Z')++beforeAll(async () => {+ replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },+ replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })+ const uri = replica.getUri(`swarm_indexes_test_${randomUUID().replaceAll('-', '_')}`)+ await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {+ env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,+ })+ db = new PrismaClient({ datasources: { db: { url: uri } } })+ // Two Swarms of 300 messages, each sent to 10 agents, and two reactions on each message.+ const deliveries = [], reactions = []+ for (const tab of ['A', 'B']) {+ for (let m = 0; m < MESSAGES; m++) {+ for (let a = 0; a < AGENTS; a++) {+ deliveries.push({ id: `${tab}-${pad(m)}-${a}`, tabId: tab, conversationId: `g${m % 20}`, messageId: `${tab}-m${pad(m)}`,+ agentId: `agent-${a}`, machineId: 'host', membershipGeneration: 'gen', authorKind: 'person', authorId: 'owner', authorGeneration: 'gen',+ status: a % 3 ? 'answered' : 'delivered', createdAt: new Date(T0 + m * 1000), updatedAt: new Date(T0 + m * 1000 + 500) })+ }+ reactions.push({ id: `${tab}-${pad(m)}-r1`, tabId: tab, conversationId: `g${m % 20}`, messageId: `${tab}-m${pad(m)}`, actorKind: 'person', actorId: 'owner', emoji: 'tada' },+ { id: `${tab}-${pad(m)}-r2`, tabId: tab, conversationId: `g${m % 20}`, messageId: `${tab}-m${pad(m)}`, actorKind: 'agent', actorId: 'agent-1', emoji: 'eyes' })+ }+ }+ await db.superTabDelivery.createMany({ data: deliveries })+ await db.superTabReaction.createMany({ data: reactions })+ // One delivery with every optional field set, and one older record that never had the hold fields at all.+ await db.superTabDelivery.update({ where: { id: 'A-150-3' }, data: { status: 'delivered', receipt: 'typed', holdReason: 'busy',+ holdSince: new Date(T0 + 7000), deliveredAt: new Date(T0 + 8000), answeredAt: new Date(T0 + 9000) } })+ await db.$runCommandRaw({ insert: 'SuperTabDelivery', documents: [{ _id: 'A-150-old', tabId: 'A', conversationId: 'g10', messageId: 'A-m150',+ agentId: 'agent-old', machineId: 'host', membershipGeneration: 'gen', authorKind: 'person', authorId: 'owner', authorGeneration: 'gen',+ status: 'answered', createdAt: { $date: new Date(T0).toISOString() }, updatedAt: { $date: new Date(T0 + 1).toISOString() } }] })+}, 120000)+afterAll(async () => {+ await db?.$disconnect()+ await replica?.stop()+})++interface Profiled { plans: string[]; docsExamined: number; returned: number }+/** What Mongo did for one collection while `read` ran: its plans, the documents it examined and the rows it returned,+ * over the first batch and every getMore. */+async function profiled<T>(collection: string, read: () => Promise<T>): Promise<[T, Profiled]> {+ await db.$runCommandRaw({ profile: 2 })+ const result = await read()+ await db.$runCommandRaw({ profile: 0 })+ const found = await db.$runCommandRaw({ find: 'system.profile', filter: { 'command.profile': { $exists: false } }, batchSize: 10000 }) as+ unknown as { cursor: { firstBatch: Array<{ ns: string; planSummary?: string; docsExamined?: number; nreturned?: number }> } }+ await db.$runCommandRaw({ drop: 'system.profile' })+ const rows = found.cursor.firstBatch.filter(row => row.ns.endsWith(`.${collection}`))+ return [result, { plans: [...new Set(rows.flatMap(row => row.planSummary ? [row.planSummary] : []))],+ docsExamined: rows.reduce((sum, row) => sum + (row.docsExamined ?? 0), 0), returned: rows.reduce((sum, row) => sum + (row.nreturned ?? 0), 0) }]+}++describe('message page reads', () => {+ it('reads a page of deliveries through the tabId+messageId index, examining only the deliveries it returns', async () => {+ const [rows, profile] = await profiled('SuperTabDelivery', () => pageDeliveries(db, 'A', PAGE))+ expect(rows).toHaveLength(1001)+ expect(profile).toEqual({ plans: ['IXSCAN { tabId: 1, messageId: 1 }'], docsExamined: 1001, returned: 1001 })+ })++ it('reads a page of reactions through the tabId+messageId index, examining only the reactions it returns', async () => {+ const [rows, profile] = await profiled('SuperTabReaction', () => pageReactions(db, 'A', PAGE))+ expect(rows).toHaveLength(200)+ expect(profile).toEqual({ plans: ['IXSCAN { tabId: 1, messageId: 1 }'], docsExamined: 200, returned: 200 })+ })++ it('returns each delivery as the Prisma read did: ids, dates as Dates, absent fields as null', async () => {+ const rows = await pageDeliveries(db, 'A', ['A-m150'])+ expect(rows.find(row => row.id === 'A-150-3')).toEqual({ id: 'A-150-3', messageId: 'A-m150', agentId: 'agent-3', status: 'delivered',+ receipt: 'typed', holdReason: 'busy', holdSince: new Date(T0 + 7000), deliveredAt: new Date(T0 + 8000), answeredAt: new Date(T0 + 9000),+ createdAt: new Date(T0 + 150_000), updatedAt: expect.any(Date) })+ expect(rows.find(row => row.id === 'A-150-old')).toEqual({ id: 'A-150-old', messageId: 'A-m150', agentId: 'agent-old', status: 'answered',+ receipt: null, holdReason: null, holdSince: null, deliveredAt: null, answeredAt: null, createdAt: new Date(T0), updatedAt: new Date(T0 + 1) })+ expect(rows.find(row => row.id === 'A-150-4')).toEqual({ id: 'A-150-4', messageId: 'A-m150', agentId: 'agent-4', status: 'answered',+ receipt: null, holdReason: null, holdSince: null, deliveredAt: null, answeredAt: null,+ createdAt: new Date(T0 + 150_000), updatedAt: new Date(T0 + 150_500) })+ })++ // The delivery read never had an order (no orderBy: it followed whichever index the planner took), so it is compared by id.+ // Reactions keep the tabId+messageId index order, which their counts' `by` lists show.+ it('returns the same rows as the Prisma reads it replaces', async () => {+ const select = { id: true, messageId: true, agentId: true, status: true, receipt: true,+ holdReason: true, holdSince: true, deliveredAt: true, answeredAt: true, createdAt: true, updatedAt: true }+ const byId = (rows: Array<{ id: string }>) => [...rows].sort((a, b) => a.id.localeCompare(b.id))+ expect(byId(await pageDeliveries(db, 'A', PAGE))).toEqual(byId(await db.superTabDelivery.findMany({ where: { tabId: 'A', messageId: { in: PAGE } }, select })))+ expect(await pageReactions(db, 'A', PAGE)).toEqual(await db.superTabReaction.findMany({ where: { tabId: 'A', messageId: { in: PAGE } },+ select: { messageId: true, emoji: true, actorKind: true, actorId: true } }))+ expect(await pageDeliveries(db, 'B', PAGE)).toEqual([])+ })+})diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prismaindex 0fed7e3a8..a697123b5 100644--- a/backend/prisma/schema.prisma+++ b/backend/prisma/schema.prisma@@ -1200,6 +1200,8 @@ model SuperTabDelivery {@@index([tabId, agentId, status, createdAt])@@index([machineId, status])+ // A page of messages reads its deliveries by message (pageReads.ts).+ @@index([tabId, messageId])}// One independent harness owns at most one active assignment across all Super Tabs.diff --git a/backend/src/superTabs/pageReads.ts b/backend/src/superTabs/pageReads.tsnew file mode 100644index 000000000..463ce6c58--- /dev/null+++ b/backend/src/superTabs/pageReads.ts@@ -0,0 +1,30 @@+import type { Prisma, SuperTabDelivery } from '@prisma/client'+import { ms } from './parts/light.js'+import type { ReactionRow } from './reactions.js'++type Transaction = Prisma.TransactionClient++// A page's rows are read by a plain {tabId, messageId: {$in}} match. Prisma sends `messageId: { in }` as an $expr $or: Mongo+// cannot bound an index by it, and on 2026-10-28 each page read scanned every delivery of its Swarm (prod at 100% CPU).+// Each read here uses its tabId+messageId index and examines only the rows it returns (integration/superTabIndexes.test.ts).++/** The reactions on one page of messages. */+export async function pageReactions(tx: Transaction, tabId: string, messageIds: string[]): Promise<ReactionRow[]> {+ return await tx.superTabReaction.aggregateRaw({ pipeline: [{ $match: { tabId, messageId: { $in: messageIds } } },+ { $project: { _id: 0, messageId: 1, emoji: 1, actorKind: 1, actorId: 1 } }] }) as unknown as ReactionRow[]+}++export type PageDelivery = Pick<SuperTabDelivery, 'id' | 'messageId' | 'agentId' | 'status' | 'receipt' | 'holdReason' | 'holdSince'+ | 'deliveredAt' | 'answeredAt' | 'createdAt' | 'updatedAt'>++/** The deliveries of one page of messages: who each message went to and how far it got. As Prisma read them: dates as+ * Dates, and an optional field an older record lacks as null. */+export async function pageDeliveries(tx: Transaction, tabId: string, messageIds: string[]): Promise<PageDelivery[]> {+ const rows = await tx.superTabDelivery.aggregateRaw({ pipeline: [{ $match: { tabId, messageId: { $in: messageIds } } },+ { $project: { messageId: 1, agentId: 1, status: 1, receipt: 1, holdReason: 1, holdSince: 1, deliveredAt: 1, answeredAt: 1, createdAt: 1, updatedAt: 1 } }] }) as+ unknown as Array<Record<string, unknown>>+ const date = (value: unknown) => value == null ? null : new Date(ms(value))+ return rows.map(row => ({ id: row._id as string, messageId: row.messageId as string, agentId: row.agentId as string, status: row.status as string,+ receipt: (row.receipt ?? null) as string | null, holdReason: (row.holdReason ?? null) as string | null, holdSince: date(row.holdSince),+ deliveredAt: date(row.deliveredAt), answeredAt: date(row.answeredAt), createdAt: date(row.createdAt)!, updatedAt: date(row.updatedAt)! }))+}diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.tsindex 5dcf47788..8d47c90cc 100644--- a/backend/src/superTabs/store.ts+++ b/backend/src/superTabs/store.ts@@ -10,6 +10,7 @@ import { applyCommand, authorizeCommandAccess, operationSchema, PAIR_CHAIN_LIMITimport { redactCredentials } from './credentials.js'import type { AvatarUpload } from './profile.js'import { reactionCounts, type ReactionTarget } from './reactions.js'+import { pageDeliveries, pageReactions } from './pageReads.js'import { attachmentMime, attachmentName, AttachmentTooLarge, AVATAR_PURPOSE, AVATAR_TYPES, checkedType, MAX_ATTACHMENT_BYTES, MAX_AVATAR_BYTES, objectKeyFor, peekType, publicUrl,type Attachment, type Attachments, type AttachmentStore, type AvatarStore } from './attachments.js'import { randomBytes } from 'node:crypto'@@ -1562,11 +1563,8 @@ export class SuperTabStore {const page = rows.slice(0, 100)const next = rows.length > 100 ? { sequence: page.at(-1)!.sequence, id: page.at(-1)!.id } : nullif (!forward) page.reverse()- const reactions = page.length ? reactionCounts(state, await tx.superTabReaction.findMany({ where: { tabId, messageId: { in: page.map(m => m.id) } },- select: { messageId: true, emoji: true, actorKind: true, actorId: true } })) : new Map()- const deliveries = page.length ? await tx.superTabDelivery.findMany({ where: { tabId, messageId: { in: page.map(m => m.id) } },- select: { id: true, messageId: true, agentId: true, status: true, receipt: true,- holdReason: true, holdSince: true, deliveredAt: true, answeredAt: true, createdAt: true, updatedAt: true } }) : []+ const reactions = page.length ? reactionCounts(state, await pageReactions(tx, tabId, page.map(m => m.id))) : new Map()+ const deliveries = page.length ? await pageDeliveries(tx, tabId, page.map(m => m.id)) : []return { messages: page.map(m => ({ ...m, mentions: m.mentions ?? [], reactions: reactions.get(m.id) ?? [],deliveries: deliveries.filter(d => d.messageId === m.id) })), next,// A poll for new messages learns here that a reaction changed on a message it already shows (reactions.ts).diff --git a/backend/vitest.super-tabs.config.ts b/backend/vitest.super-tabs.config.tsindex 3265fcaeb..09463952c 100644--- a/backend/vitest.super-tabs.config.ts+++ b/backend/vitest.super-tabs.config.ts@@ -2,5 +2,6 @@ import { defineConfig } from 'vitest/config'export default defineConfig({ test: { include: ['integration/superTabs.test.ts', 'integration/superTabSharing.test.ts', 'integration/superTabGithub.test.ts', 'integration/superTabTeam.test.ts', 'integration/superTabWorkspaces.test.ts','integration/superTabRead.test.ts', 'integration/superTabLeaderboard.test.ts', 'integration/superTabMigration.test.ts', 'integration/superTabLiveness.test.ts', 'integration/superTabTidy.test.ts',- 'integration/superTabProfile.test.ts', 'integration/superTabLegacyDrop.test.ts', 'integration/shareRaces.test.ts'], hookTimeout: 120000, testTimeout: 30000,+ 'integration/superTabProfile.test.ts', 'integration/superTabLegacyDrop.test.ts', 'integration/shareRaces.test.ts',+ 'integration/superTabIndexes.test.ts'], hookTimeout: 120000, testTimeout: 30000,fileParallelism: false, env: { NODE_ENV: 'test', HARNESS_CREDENTIAL_ENCRYPTION_KEY: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' } } })