Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Swarm: one read for a person's unread threads and each chat's latest message (#2256)

0xken15780467198352
diff --git a/backend/integration/superTabInbox.test.ts b/backend/integration/superTabInbox.test.ts
new file mode 100644
index 000000000..903d7eb59
--- /dev/null
+++ b/backend/integration/superTabInbox.test.ts
@@ -0,0 +1,537 @@
+import { randomUUID } from 'node:crypto'
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { MongoMemoryReplSet } from 'mongodb-memory-server-core'
+import { PrismaClient } from '@prisma/client'
+import Fastify, { type FastifyInstance } from 'fastify'
+import type { AuthUser } from '../src/lib/ssoAuth.js'
+import { SuperTabStore } from '../src/superTabs/store.js'
+import { superTabRoutes } from '../src/routes/superTabs.js'
+import { errorHandler } from '../src/middlewares/errorHandler.js'
+import { migrateSwarm } from '../src/superTabs/migration/migrate.js'
+import * as parts from '../src/superTabs/parts/index.js'
+import { swarmTransaction } from '../src/superTabs/cache.js'
+import { guardedWrite } from '../src/superTabs/parts/write.js'
+import { SwarmReads } from '../src/superTabs/read/service.js'
+import { HostReads } from '../src/superTabs/read/host.js'
+import { buddyChatId } from '../src/superTabs/buddies.js'
+import { INBOX_READS, INBOX_WINDOW_MS, LATEST_CHATS, MAX_THREADS, PREVIEW_CHARS, resetInboxReads, THREAD_CHATS } from '../src/superTabs/read/inbox.js'
+import { stableId, type Author, type Conversation, type SuperTabState } from '../src/superTabs/model.js'
+import { writeLegacySwarm } from '../scripts/swarm-parts/legacy.js'
+import { NOW, outsider as outsiderActor, owner as ownerActor, peer as peerActor, swarm } from '../src/superTabs/read/fixture.test-support.js'
+import { PROFILE_STAGES, profileEntry, type ProfileEntry } from './bench/profile.js'
+
+// A fresh loopback replica set, created here. This suite never reads DATABASE_URL or an existing database.
+let replica: MongoMemoryReplSet
+let db: PrismaClient
+let app: FastifyInstance
+let store: SuperTabStore
+const user = (actor: { userId: string; email: string }): AuthUser => ({ sub: actor.userId, email: actor.email, autonomousEnv: 'prod', role: 'user' })
+const [owner, peer, outsider] = [ownerActor, peerActor, outsiderActor].map(user) as [AuthUser, AuthUser, AuthUser]
+const ownerTeam: AuthUser = { ...owner, harnessSessionId: 'owner-team-session', harnessSessionKind: 'team' }
+const signedIn = (authorization: string | undefined) =>
+ authorization === 'Bearer team' ? ownerTeam : [owner, peer, outsider].find(u => `Bearer ${u.sub}` === authorization)
+const MIGRATED = NOW - 60_000
+
+beforeAll(async () => {
+ replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },
+ replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })
+ const uri = replica.getUri(`swarm_inbox_test_${randomUUID().replaceAll('-', '_')}`)
+ await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {
+ env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,
+ })
+ db = new PrismaClient({ datasources: { db: { url: uri } } })
+ for (const each of [owner, peer, outsider]) await db.user.create({ data: { id: each.sub, externalId: `fixture-${each.sub}`, email: each.email, autonomousEnv: 'prod' } })
+ // The owner's own machine runs the Swarm's worker, so the owner may write as it.
+ await db.machine.create({ data: { machineId: 'host', userId: owner.sub, apiKey: 'fixture-only', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
+ store = new SuperTabStore(db)
+ app = Fastify()
+ app.setErrorHandler(errorHandler)
+ // External SSO is replaced with fixture accounts; every permission and write below is real.
+ app.addHook('onRequest', async (request, reply) => {
+ const found = signedIn(request.headers.authorization)
+ if (!found) return reply.code(401).send({ success: false })
+ request.user = found
+ })
+ await app.register(superTabRoutes, { store, github: null, reads: new SwarmReads(store), hosts: new HostReads(store) })
+ await app.ready()
+})
+afterAll(async () => {
+ await app?.close()
+ await db?.$disconnect()
+ await replica?.stop()
+})
+
+/** A Swarm in parts: a public project ("Web", with a worker on the owner's machine), a project private to its owner
+ * ("Secret work"), peer's Buddy chat, and peer's closed Chat with PM from before Buddies (nobody reads it). */
+async function scene(extra: (s: ReturnType<typeof swarm>) => void = () => {}) {
+ const s = swarm(MIGRATED)
+ s.join(peerActor)
+ s.join(outsiderActor)
+ const secret = s.run({ type: 'project.create', name: 'Secret', visibility: 'private', brief: '' }).projectId as string
+ const web = s.group('Web')
+ const worker = s.add('worker-harness', web, false, ownerActor, 'host', owner.sub)
+ s.run({ type: 'work.assign', conversationId: web, agentId: worker })
+ const secretGroup = s.group('Secret work', secret)
+ const closedId = s.group('Peer and PM')
+ let buddyId = s.group('Peer\'s Buddy')
+ s.edit(state => {
+ const generation = state.people.find(p => p.id === peer.sub)!.generation
+ state.conversations.find(c => c.id === closedId)!.audience = { people: [{ id: peer.sub, generation }] }
+ const chat = state.conversations.find(c => c.id === buddyId)!
+ buddyId = buddyChatId(chat.projectId ?? 'lead', { id: peer.sub, generation })
+ Object.assign(chat, { id: buddyId, kind: 'buddy', leadId: null, agentIds: [], audience: { people: [{ id: peer.sub, generation }] } })
+ })
+ extra(s)
+ const tabId = `tab-${randomUUID()}`
+ const state = { ...structuredClone(s.state), id: tabId } as SuperTabState
+ await writeLegacySwarm(db, { state, revision: 3 })
+ const report = await migrateSwarm(db, tabId, { now: () => MIGRATED })
+ if (!report.ok) throw new Error(report.error)
+ return { tabId, web, worker, secretGroup, closedId, buddyId, leads: state.leadsConversationId! }
+}
+
+const get = async (url: string, as: AuthUser | 'team' = owner) => {
+ const response = await app.inject({ method: 'GET', url, headers: { authorization: `Bearer ${as === 'team' ? 'team' : as.sub}` } })
+ return { status: response.statusCode, body: response.json() as { data: any; error?: { code: string; message: string } } }
+}
+const inbox = (tabId: string, as: AuthUser | 'team' = owner) => get(`/api/super-tabs/${tabId}/inbox`, as)
+
+/** Sends a message as a person (or as an agent the owner runs) through the real command, and answers its stored row. */
+async function send(tabId: string, conversationId: string, text: string, opts: { as?: AuthUser; agentId?: string; replyTo?: string } = {}) {
+ await store.execute(opts.as ?? owner, tabId, { operationId: randomUUID(), command: { type: 'message.send', conversationId, text,
+ replyTo: opts.replyTo ?? null, recipientIds: [] } }, opts.agentId)
+ return (await db.superTabMessage.findFirst({ where: { tabId, conversationId, text }, orderBy: { sequence: 'desc' } }))!
+}
+const agentAuthor = (id: string): Author => ({ kind: 'agent', id, name: id, membershipGeneration: 'fixture' })
+const personAuthor = (u: AuthUser): Author => ({ kind: 'person', id: u.sub, name: u.email, membershipGeneration: 'fixture' })
+
+/** Rows no command writes (a Buddy's reply, a Lead's tag, legacy notices), stored as the store stores them: numbered after the
+ * chat's newest, the chat's counters moved as numberMessages moves them for rows people read. */
+async function insert(tabId: string, conversationId: string, rows: Array<{ id?: string; text: string; author: Author; thread?: string | null;
+ source?: Record<string, unknown>; recipientIds?: string[]; mentions?: string[] }>) {
+ const ids: string[] = []
+ await swarmTransaction(db, async tx => {
+ const current = (await parts.loadActive(tx, tabId, { groupIds: [conversationId] })).groups.find(g => g.key === conversationId)!
+ const data = structuredClone(current.data!) as Conversation
+ for (const row of rows) {
+ const sequence = (data.messageSeq ?? 0) + 1
+ const id = row.id ?? `m-${randomUUID()}`
+ ids.push(id)
+ data.messageSeq = sequence
+ const hidden = !!row.source?.recipientsOnly
+ if (!hidden) {
+ data.lastMessageSequence = sequence
+ data.lastActivityAt = new Date().toISOString()
+ if (!row.thread || row.thread === id) data.lastMainSequence = sequence
+ }
+ await tx.superTabMessage.create({ data: { id, tabId, conversationId, sequence, author: row.author as never, text: row.text,
+ thread: row.thread === undefined ? id : row.thread, recipientIds: row.recipientIds ?? [], mentions: row.mentions ?? [],
+ ...(row.source ? { source: row.source as never } : {}), createdAt: new Date() } })
+ }
+ await guardedWrite(tx, tabId, { op: 'update', kind: 'group', key: conversationId, seen: current.version, data }, Date.now())
+ })
+ return ids
+}
+/** Changes one chat's stored data as a command would. */
+async function editChat(tabId: string, conversationId: string, edit: (data: Conversation) => void) {
+ await swarmTransaction(db, async tx => {
+ const current = (await parts.loadActive(tx, tabId, { groupIds: [conversationId] })).groups.find(g => g.key === conversationId)!
+ const data = structuredClone(current.data!) as Conversation
+ edit(data)
+ await guardedWrite(tx, tabId, { op: 'update', kind: 'group', key: conversationId, seen: current.version, data }, Date.now())
+ })
+}
+const sequenceOf = async (id: string) => (await db.superTabMessage.findUniqueOrThrow({ where: { id } })).sequence
+const markRead = async (tabId: string, conversationId: string, sequence: number, as: AuthUser) => {
+ const response = await app.inject({ method: 'POST', url: `/api/super-tabs/${tabId}/conversations/${conversationId}/read`,
+ headers: { authorization: `Bearer ${as.sub}` }, payload: { sequence } })
+ expect(response.statusCode).toBe(200)
+}
+const threadsOf = (data: any) => (data.threads as any[]).map(t => ({ chat: t.conversationId, root: t.root.text, unread: t.unreadReplies,
+ last: t.lastReplier?.id ?? null }))
+const latestOf = (data: any) => Object.fromEntries((data.latest as any[]).map(l => [l.conversationId, l.message.text]))
+
+/** Every chat gets a thread and a newer top-level message. Each reader marks each chat they read as read at its thread's root,
+ * before the replies: a chat never marked read counts as read.
+ * Web (public): owner's root tagging peer and outsider, two owner replies and one worker reply, then "Web latest".
+ * Secret work (owner's private project): owner's root, a worker reply, then "Secret latest".
+ * Peer's Buddy chat: peer's root, the Buddy's reply, then the Buddy's "Buddy latest".
+ * Peer's closed Chat with PM: a root, a reply and "Closed latest", read by nobody.
+ * Leads: the owner's "Leads note". */
+async function busyScene() {
+ const s = await scene()
+ const [webRootId] = await insert(s.tabId, s.web, [{ text: 'Web root', author: personAuthor(owner), mentions: [peer.sub, outsider.sub] }])
+ const [secretRoot] = await insert(s.tabId, s.secretGroup, [{ text: 'Secret root', author: personAuthor(owner) }])
+ const [buddyRoot] = await insert(s.tabId, s.buddyId, [{ text: 'Buddy root', author: personAuthor(peer) }])
+ const [closedRoot] = await insert(s.tabId, s.closedId, [{ text: 'Closed root', author: personAuthor(peer) }])
+ const webRoot = await db.superTabMessage.findUniqueOrThrow({ where: { id: webRootId! } })
+ for (const each of [owner, peer, outsider]) await markRead(s.tabId, s.web, webRoot.sequence, each)
+ await markRead(s.tabId, s.secretGroup, await sequenceOf(secretRoot!), owner)
+ await markRead(s.tabId, s.buddyId, await sequenceOf(buddyRoot!), peer)
+ await send(s.tabId, s.web, 'Web reply 1', { replyTo: webRoot.id })
+ await send(s.tabId, s.web, 'Web reply 2', { replyTo: webRoot.id })
+ await send(s.tabId, s.web, 'Web worker reply', { replyTo: webRoot.id, agentId: s.worker })
+ const webLatest = await send(s.tabId, s.web, 'Web latest')
+ await insert(s.tabId, s.secretGroup, [{ text: 'Secret reply', author: agentAuthor(s.worker), thread: secretRoot }, { text: 'Secret latest', author: personAuthor(owner) }])
+ await insert(s.tabId, s.buddyId, [{ text: 'Buddy reply', author: agentAuthor('buddy-agent'), thread: buddyRoot }, { text: 'Buddy latest', author: agentAuthor('buddy-agent') }])
+ await insert(s.tabId, s.closedId, [{ text: 'Closed reply', author: agentAuthor(s.worker), thread: closedRoot }, { text: 'Closed latest', author: personAuthor(peer) }])
+ await send(s.tabId, s.leads, 'Leads note')
+ return { ...s, webRoot, webLatest }
+}
+
+beforeEach(() => resetInboxReads())
+
+describe('a person\'s inbox: unread threads and each chat\'s latest message', () => {
+ it('gives a collaborator exactly the chats the messages read gives them: their own Buddy chat, never the private project or the closed chat', async () => {
+ const s = await busyScene()
+ const { status, body } = await inbox(s.tabId, peer)
+ expect(status).toBe(200)
+ expect(threadsOf(body.data)).toEqual([
+ { chat: s.buddyId, root: 'Buddy root', unread: 1, last: 'buddy-agent' },
+ { chat: s.web, root: 'Web root', unread: 3, last: s.worker },
+ ])
+ expect(latestOf(body.data)).toEqual({ [s.web]: 'Web latest', [s.buddyId]: 'Buddy latest', [s.leads]: 'Leads note' })
+ expect(body.data.more).toBe(false)
+ expect(JSON.stringify(body.data)).not.toMatch(/Secret|Closed|secret|closed/)
+ expect(JSON.stringify(body.data)).not.toContain(s.secretGroup)
+ expect(JSON.stringify(body.data)).not.toContain(s.closedId)
+ })
+
+ it('gives the owner the private project, never a collaborator\'s Buddy chat or closed chat, and never counts their own replies', async () => {
+ const s = await busyScene()
+ const { status, body } = await inbox(s.tabId, owner)
+ expect(status).toBe(200)
+ // In Web the owner wrote the root and two replies: only the worker's reply, after their own newest, is unread for them.
+ expect(threadsOf(body.data)).toEqual([
+ { chat: s.secretGroup, root: 'Secret root', unread: 1, last: s.worker },
+ { chat: s.web, root: 'Web root', unread: 1, last: s.worker },
+ ])
+ expect(latestOf(body.data)).toEqual({ [s.web]: 'Web latest', [s.secretGroup]: 'Secret latest', [s.leads]: 'Leads note' })
+ expect(JSON.stringify(body.data)).not.toMatch(/Buddy|Closed/)
+ expect(JSON.stringify(body.data)).not.toContain(s.buddyId)
+ })
+
+ it('gives someone outside the private project and the Buddy chat only the public chats', async () => {
+ const s = await busyScene()
+ const { status, body } = await inbox(s.tabId, outsider)
+ expect(status).toBe(200)
+ expect(threadsOf(body.data)).toEqual([{ chat: s.web, root: 'Web root', unread: 3, last: s.worker }])
+ expect(latestOf(body.data)).toEqual({ [s.web]: 'Web latest', [s.leads]: 'Leads note' })
+ expect(JSON.stringify(body.data)).not.toMatch(/Secret|Buddy|Closed/)
+ })
+
+ it('answers the root, the last reply time and the latest message as the messages read shows them', async () => {
+ const s = await busyScene()
+ const { body } = await inbox(s.tabId, outsider)
+ const page = (await get(`/api/super-tabs/${s.tabId}/conversations/${s.web}/messages`, outsider)).body.data.messages as any[]
+ const shown = (text: string) => page.find(m => m.text === text)
+ const thread = body.data.threads[0]
+ expect(thread).toEqual({ conversationId: s.web, rootId: s.webRoot.id,
+ root: { id: shown('Web root').id, sequence: shown('Web root').sequence, author: shown('Web root').author, text: 'Web root',
+ createdAt: shown('Web root').createdAt, attachments: 0 },
+ lastReplyAt: shown('Web worker reply').createdAt, lastReplySequence: shown('Web worker reply').sequence,
+ lastReplier: shown('Web worker reply').author, unreadReplies: 3 })
+ expect(body.data.latest.find((l: any) => l.conversationId === s.web)).toEqual({ conversationId: s.web,
+ message: { id: shown('Web latest').id, sequence: shown('Web latest').sequence, author: shown('Web latest').author, text: 'Web latest',
+ createdAt: shown('Web latest').createdAt, attachments: 0 } })
+ })
+
+ it('counts only replies past the person\'s read mark, and drops a thread they have read', async () => {
+ const s = await busyScene()
+ const reply1 = await db.superTabMessage.findFirstOrThrow({ where: { tabId: s.tabId, text: 'Web reply 1' } })
+ await markRead(s.tabId, s.web, reply1.sequence, peer)
+ expect(threadsOf((await inbox(s.tabId, peer)).body.data).filter(t => t.chat === s.web)).toEqual([{ chat: s.web, root: 'Web root', unread: 2, last: s.worker }])
+ await markRead(s.tabId, s.web, s.webLatest.sequence, peer)
+ const after = (await inbox(s.tabId, peer)).body.data
+ expect(threadsOf(after).filter(t => t.chat === s.web)).toEqual([])
+ // The latest message stays: the Buddies list shows it whether read or not.
+ expect(latestOf(after)[s.web]).toBe('Web latest')
+ })
+
+ it('never counts or names a row people cannot read, and never lists a thread whose root they cannot read', async () => {
+ const s = await busyScene()
+ // An agent-only notice in the Web thread (a reply only its recipient reads), and an agent-only root with a reply under it
+ // that tags the outsider, so only the hidden root keeps that thread out.
+ await insert(s.tabId, s.web, [{ text: 'Hidden reply', author: agentAuthor('hidden-agent'), thread: s.webRoot.id, source: { kind: 'note', recipientsOnly: true },
+ recipientIds: [s.worker] }])
+ const [hiddenRoot] = await insert(s.tabId, s.web, [{ text: 'Hidden root', author: agentAuthor('hidden-agent'), source: { kind: 'note', recipientsOnly: true },
+ recipientIds: [s.worker] }])
+ await insert(s.tabId, s.web, [{ text: 'Reply under a hidden root', author: agentAuthor(s.worker), thread: hiddenRoot, mentions: [outsider.sub] }])
+ const { body } = await inbox(s.tabId, outsider)
+ expect(threadsOf(body.data)).toEqual([{ chat: s.web, root: 'Web root', unread: 3, last: s.worker }])
+ expect(JSON.stringify(body.data)).not.toMatch(/Hidden|hidden-agent/)
+ })
+
+ it('shows as latest the message a page ends with when the newest row is a task chat copy of a Lead notice nobody reads', async () => {
+ const s = await busyScene()
+ // A lost-task notice stored before such copies were marked (pageReads.ts deliveredCopies), whose Lead copy reached the Lead.
+ const [copy] = await insert(s.tabId, s.web, [{ text: 'Lost task notice', author: agentAuthor('swarm'), source: { kind: 'work-lost' } }])
+ await insert(s.tabId, s.leads, [{ id: stableId(copy!, 'lead'), text: 'Lead copy', author: agentAuthor('swarm'), source: { kind: 'work-lost' },
+ recipientIds: ['lead-agent'] }])
+ const page = (await get(`/api/super-tabs/${s.tabId}/conversations/${s.web}/messages`, outsider)).body.data.messages as any[]
+ expect(page.map(m => m.text)).not.toContain('Lost task notice')
+ expect(latestOf((await inbox(s.tabId, outsider)).body.data)[s.web]).toBe('Web latest')
+ })
+
+ it('never names a last replier who can no longer read the chat', async () => {
+ const s = await busyScene()
+ // Outsider replies in Web, then leaves the Swarm: peer still sees the reply counted, but not who wrote it.
+ await send(s.tabId, s.web, 'Outsider reply', { as: outsider, replyTo: s.webRoot.id })
+ await store.execute(owner, s.tabId, { operationId: randomUUID(), command: { type: 'member.remove', userId: outsider.sub } })
+ const thread = (await inbox(s.tabId, peer)).body.data.threads.find((t: any) => t.conversationId === s.web)
+ expect(thread.unreadReplies).toBe(4)
+ // As thread summaries name them: the newest author who can still read the chat.
+ expect(thread.lastReplier.id).toBe(s.worker)
+ expect(JSON.stringify(thread)).not.toContain(outsider.sub)
+ })
+
+ it('refuses a removed member, a Team session and an agent alike, with nothing about the Swarm\'s chats', async () => {
+ const s = await busyScene()
+ await store.execute(owner, s.tabId, { operationId: randomUUID(), command: { type: 'member.remove', userId: peer.sub } })
+ const removed = await inbox(s.tabId, peer)
+ // Refused as the messages read refuses them, not answered as a member with nothing to read.
+ const page = await get(`/api/super-tabs/${s.tabId}/conversations/${s.web}/messages`, peer)
+ expect(page.status).toBe(403)
+ expect({ status: removed.status, code: removed.body.error?.code }).toEqual({ status: page.status, code: page.body.error?.code })
+ expect(removed.body.data).toBeUndefined()
+ expect(JSON.stringify(removed.body)).not.toMatch(/Web|Buddy|Leads/)
+ const team = await inbox(s.tabId, 'team')
+ expect(team.status).toBe(403)
+ expect(JSON.stringify(team.body)).not.toMatch(/Web|Secret|Leads/)
+ // An agent has no read marks: the read takes no agent at all.
+ expect((await get(`/api/super-tabs/${s.tabId}/inbox?asAgentId=${s.worker}`, owner)).status).toBe(400)
+ // Another Swarm's id reads as one that does not exist.
+ expect((await inbox(`tab-${randomUUID()}`, owner)).status).toBe(404)
+ })
+
+ it('does not list a thread whose only replies are the person\'s own', async () => {
+ const s = await busyScene()
+ const ownRoot = await send(s.tabId, s.web, 'Own root')
+ await send(s.tabId, s.web, 'Own reply', { replyTo: ownRoot.id })
+ const { body } = await inbox(s.tabId, owner)
+ expect(threadsOf(body.data)).toEqual([
+ { chat: s.secretGroup, root: 'Secret root', unread: 1, last: s.worker },
+ { chat: s.web, root: 'Web root', unread: 1, last: s.worker },
+ ])
+ expect(JSON.stringify(body.data.threads)).not.toMatch(/Own root|Own reply/)
+ })
+
+ it('shows a top-level message as latest, never a thread reply, when the newest row is a copy nobody reads', async () => {
+ const s = await scene()
+ const root = await send(s.tabId, s.web, 'Thread root')
+ await send(s.tabId, s.web, 'Thread reply', { replyTo: root.id })
+ const [copy] = await insert(s.tabId, s.web, [{ text: 'Lost task notice', author: agentAuthor('swarm'), source: { kind: 'work-lost' } }])
+ await insert(s.tabId, s.leads, [{ id: stableId(copy!, 'lead'), text: 'Lead copy', author: agentAuthor('swarm'), source: { kind: 'work-lost' },
+ recipientIds: ['lead-agent'] }])
+ expect(latestOf((await inbox(s.tabId, outsider)).body.data)[s.web]).toBe('Thread root')
+ })
+})
+
+describe('thread unread as Desktop marks it (controller.dart threadUnread)', () => {
+ it('counts a chat the person never marked read as read', async () => {
+ const s = await scene()
+ const [root] = await insert(s.tabId, s.web, [{ text: 'Tagged root', author: personAuthor(owner), mentions: [peer.sub] }])
+ await insert(s.tabId, s.web, [{ text: 'Worker reply', author: agentAuthor(s.worker), thread: root }])
+ const { body } = await inbox(s.tabId, peer)
+ expect(body.data.threads).toEqual([])
+ // Once they read it, the next reply counts.
+ await markRead(s.tabId, s.web, await sequenceOf(root!), peer)
+ expect(threadsOf((await inbox(s.tabId, peer)).body.data)).toEqual([{ chat: s.web, root: 'Tagged root', unread: 1, last: s.worker }])
+ })
+
+ it('lists only threads the person started, replied in or was tagged in, before or after their mark', async () => {
+ const s = await scene()
+ const [elsewhere, started, repliedBefore, taggedBefore, repliedAfter, taggedAfter] = await insert(s.tabId, s.web, [
+ { text: 'Not theirs', author: personAuthor(owner) },
+ { text: 'Started', author: personAuthor(peer) },
+ { text: 'Replied before', author: personAuthor(owner) },
+ { text: 'Tagged before', author: personAuthor(owner) },
+ { text: 'Replied after', author: personAuthor(owner) },
+ { text: 'Tagged after', author: personAuthor(owner) },
+ ])
+ await insert(s.tabId, s.web, [{ text: 'Peer reply', author: personAuthor(peer), thread: repliedBefore },
+ { text: 'Lead tags peer', author: agentAuthor(s.worker), thread: taggedBefore, mentions: [peer.sub] }])
+ await markRead(s.tabId, s.web, (await db.superTabMessage.findFirstOrThrow({ where: { tabId: s.tabId, conversationId: s.web }, orderBy: { sequence: 'desc' } })).sequence, peer)
+ await insert(s.tabId, s.web, [{ text: 'Peer reply after', author: personAuthor(peer), thread: repliedAfter },
+ { text: 'Lead tags peer after', author: agentAuthor(s.worker), thread: taggedAfter, mentions: [peer.sub] }])
+ await insert(s.tabId, s.web, [elsewhere, started, repliedBefore, taggedBefore, repliedAfter, taggedAfter]
+ .map(thread => ({ text: 'Worker reply', author: agentAuthor(s.worker), thread })))
+ const roots = threadsOf((await inbox(s.tabId, peer)).body.data).map(t => `${t.root}: ${t.unread}`).sort()
+ // After their own reply only the worker's counts; after a tag both the tag and the worker's reply do.
+ expect(roots).toEqual(['Replied after: 1', 'Replied before: 1', 'Started: 1', 'Tagged after: 2', 'Tagged before: 1'])
+ })
+
+ it('counts only others\' replies after the person\'s own newest reply, and skips a thread whose newest reply is theirs', async () => {
+ const s = await scene()
+ const [root] = await insert(s.tabId, s.web, [{ text: 'Peer root', author: personAuthor(peer) }])
+ await markRead(s.tabId, s.web, await sequenceOf(root!), peer)
+ await insert(s.tabId, s.web, [{ text: 'Worker 1', author: agentAuthor(s.worker), thread: root }, { text: 'Worker 2', author: agentAuthor(s.worker), thread: root },
+ { text: 'Peer answers', author: personAuthor(peer), thread: root }])
+ // Their reply is the newest: nothing in it waits for them, though their mark is older than the worker's replies.
+ expect((await inbox(s.tabId, peer)).body.data.threads).toEqual([])
+ await insert(s.tabId, s.web, [{ text: 'Worker 3', author: agentAuthor(s.worker), thread: root }])
+ expect(threadsOf((await inbox(s.tabId, peer)).body.data)).toEqual([{ chat: s.web, root: 'Peer root', unread: 1, last: s.worker }])
+ })
+
+ it('leaves archived chats out of the latest messages and the threads', async () => {
+ const s = await busyScene()
+ await editChat(s.tabId, s.web, data => { data.archived = true })
+ const { body } = await inbox(s.tabId, peer)
+ expect(threadsOf(body.data)).toEqual([{ chat: s.buddyId, root: 'Buddy root', unread: 1, last: 'buddy-agent' }])
+ expect(latestOf(body.data)).toEqual({ [s.buddyId]: 'Buddy latest', [s.leads]: 'Leads note' })
+ })
+
+ it('gives an older chat whose newest rows are all thread replies its latest top-level message', async () => {
+ const s = await scene()
+ const [root] = await insert(s.tabId, s.web, [{ text: 'Older root', author: personAuthor(owner) }])
+ await insert(s.tabId, s.web, Array.from({ length: 20 }, (_, i) => ({ text: `Reply ${i}`, author: agentAuthor(s.worker), thread: root })))
+ // Written before the server kept the main stream's newest number.
+ await editChat(s.tabId, s.web, data => { delete data.lastMainSequence })
+ const { body } = await inbox(s.tabId, peer)
+ expect(latestOf(body.data)[s.web]).toBe('Older root')
+ expect(body.data.more).toBe(false)
+ })
+})
+
+describe('the inbox\'s caps', () => {
+ it('cuts preview text to PREVIEW_CHARS characters in the database, counting a surrogate pair as one', async () => {
+ const s = await scene()
+ const long = '😀'.repeat(PREVIEW_CHARS + 50)
+ const [root] = await insert(s.tabId, s.web, [{ text: long, author: personAuthor(peer) }])
+ await markRead(s.tabId, s.web, await sequenceOf(root!), peer)
+ await insert(s.tabId, s.web, [{ text: 'Reply', author: agentAuthor(s.worker), thread: root }])
+ const { body } = await inbox(s.tabId, peer)
+ expect(body.data.threads[0].root.text).toBe('😀'.repeat(PREVIEW_CHARS))
+ expect(body.data.latest.find((l: any) => l.conversationId === s.web).message.text).toBe('😀'.repeat(PREVIEW_CHARS))
+ })
+
+ it(`answers the newest ${MAX_THREADS} threads and says there are more`, async () => {
+ const s = await scene(f => { for (let i = 0; i < 3; i++) f.group(`Busy ${i}`) })
+ const chats = (await store.readLoad(peer, s.tabId)).state.conversations.filter(c => c.title.startsWith('Busy ')).map(c => c.id)
+ for (const chat of chats) {
+ const roots = await insert(s.tabId, chat, Array.from({ length: 40 }, (_, i) => ({ text: `Root ${i}`, author: personAuthor(peer) })))
+ await markRead(s.tabId, chat, await sequenceOf(roots.at(-1)!), peer)
+ await insert(s.tabId, chat, roots.map(thread => ({ text: 'Reply', author: agentAuthor(s.worker), thread })))
+ }
+ const { body } = await inbox(s.tabId, peer)
+ expect(body.data.threads).toHaveLength(MAX_THREADS)
+ expect(body.data.more).toBe(true)
+ // Newest reply first: the last chat written holds the newest.
+ expect(new Set(body.data.threads.slice(0, 40).map((t: any) => t.conversationId))).toEqual(new Set([chats[2]]))
+ })
+
+ it(`answers the latest message of the ${LATEST_CHATS} most recently active chats and says there are more`, async () => {
+ const s = await scene(f => { for (let i = 0; i < LATEST_CHATS; i++) f.group(`Chat ${i}`) })
+ const chats = (await store.readLoad(peer, s.tabId)).state.conversations.filter(c => c.title.startsWith('Chat ')).map(c => c.id)
+ // Web and Leads get a message first, so they are the oldest active of LATEST_CHATS + 2 chats with a latest message.
+ await insert(s.tabId, s.web, [{ text: 'Older in Web', author: personAuthor(owner) }])
+ await insert(s.tabId, s.leads, [{ text: 'Older in Leads', author: personAuthor(owner) }])
+ for (const chat of chats) await insert(s.tabId, chat, [{ text: `Latest in ${chat}`, author: personAuthor(owner) }])
+ const { body } = await inbox(s.tabId, peer)
+ expect(body.data.latest).toHaveLength(LATEST_CHATS)
+ expect(new Set(body.data.latest.map((l: any) => l.conversationId))).toEqual(new Set(chats))
+ expect(body.data.more).toBe(true)
+ }, 60_000)
+
+ it(`allows a person ${INBOX_READS} reads of one Swarm's inbox per window, then answers 429 with Retry-After`, async () => {
+ const s = await busyScene()
+ vi.useFakeTimers({ toFake: ['Date'] })
+ try {
+ vi.setSystemTime(Math.ceil(Date.now() / INBOX_WINDOW_MS) * INBOX_WINDOW_MS + 1_000)
+ for (let i = 0; i < INBOX_READS; i++) expect((await inbox(s.tabId, peer)).status).toBe(200)
+ const refused = await app.inject({ method: 'GET', url: `/api/super-tabs/${s.tabId}/inbox`, headers: { authorization: `Bearer ${peer.sub}` } })
+ expect(refused.statusCode).toBe(429)
+ expect(refused.headers['retry-after']).toBe('9')
+ expect(refused.json().error.code).toBe('RATE_LIMITED')
+ expect(refused.body).not.toMatch(/Web|Buddy|Leads/)
+ // Counted per person and per Swarm.
+ expect((await inbox(s.tabId, owner)).status).toBe(200)
+ expect((await inbox((await busyScene()).tabId, peer)).status).toBe(200)
+ vi.setSystemTime(Date.now() + INBOX_WINDOW_MS)
+ expect((await inbox(s.tabId, peer)).status).toBe(200)
+ } finally { vi.useRealTimers() }
+ })
+})
+
+describe('what the inbox costs the database', () => {
+ const CHATS = 35
+ /** A person in CHATS active public chats, then one long chat of 4000 rows, the most recently active. Each of the CHATS has a
+ * thread they were tagged in, with 5 replies past their mark, then a top-level message. The long chat has 10 threads they
+ * are not in, each with 300 replies before their mark and 100 after. */
+ async function bigScene() {
+ const s = await scene(f => { for (let i = 0; i < CHATS; i++) f.group(`Chat ${i}`) })
+ const state = (await store.readLoad(peer, s.tabId)).state
+ const chats = state.conversations.filter(c => c.title.startsWith('Chat ')).map(c => c.id)
+ for (const chat of chats) {
+ const [root] = await insert(s.tabId, chat, [{ text: `Root in ${chat}`, author: personAuthor(owner), mentions: [peer.sub] }])
+ await markRead(s.tabId, chat, await sequenceOf(root!), peer)
+ await insert(s.tabId, chat, [...Array.from({ length: 5 }, (_, i) => ({ text: `Reply ${i}`, author: agentAuthor(s.worker), thread: root })),
+ { text: `Latest in ${chat}`, author: personAuthor(owner) }])
+ }
+ const roots = await insert(s.tabId, s.web, Array.from({ length: 10 }, (_, i) => ({ text: `Long root ${i}`, author: personAuthor(owner) })))
+ for (let batch = 0; batch < 3; batch++) {
+ await insert(s.tabId, s.web, Array.from({ length: 1000 }, (_, i) => ({ text: `Long reply ${batch}.${i}`, author: agentAuthor(s.worker), thread: roots[i % roots.length]! })))
+ }
+ await markRead(s.tabId, s.web, (await db.superTabMessage.findFirstOrThrow({ where: { tabId: s.tabId, conversationId: s.web }, orderBy: { sequence: 'desc' } })).sequence, peer)
+ await insert(s.tabId, s.web, Array.from({ length: 1000 }, (_, i) => ({ text: `Later reply ${i}`, author: agentAuthor(s.worker), thread: roots[i % roots.length]! })))
+ return { ...s, chats }
+ }
+ const sent = async () => {
+ const out = ((await db.$runCommandRaw({ serverStatus: 1 })) as { network: { bytesOut: unknown } }).network.bytesOut
+ return typeof out === 'number' ? out : Number((out as { $numberLong?: string; $numberInt?: string }).$numberLong ?? (out as { $numberInt?: string }).$numberInt)
+ }
+
+ it(`reads by index only and examines a bounded number of messages for a person with ${CHATS}+ active chats`, async () => {
+ const s = await bigScene()
+ expect(await db.superTabMessage.count({ where: { tabId: s.tabId } })).toBeGreaterThan(4200)
+ await inbox(s.tabId, peer)
+ await db.$runCommandRaw({ profile: 0 })
+ await db.$runCommandRaw({ drop: 'system.profile' }).catch(() => {})
+ await db.$runCommandRaw({ create: 'system.profile', capped: true, size: 16 * 1024 * 1024 })
+ await db.$runCommandRaw({ profile: 2, slowms: 0 })
+ let reply: Awaited<ReturnType<typeof inbox>>
+ try {
+ reply = await inbox(s.tabId, peer)
+ } finally { await db.$runCommandRaw({ profile: 0 }) }
+ expect(reply.status).toBe(200)
+ // What MongoDB sent for one more call, unprofiled, less what reading the counter itself costs (as superTabRead.test.ts).
+ const first = await sent(), own = await sent() - first
+ const start = await sent()
+ await inbox(s.tabId, peer)
+ const bytesFromDatabase = Math.max(0, await sent() - start - own)
+ const rows = (await db.$runCommandRaw({ aggregate: 'system.profile', pipeline: [{ $match: { ns: { $not: { $regex: 'system\\.' } } } }, ...PROFILE_STAGES],
+ cursor: {} }) as unknown as { cursor: { firstBatch: ProfileEntry[] } }).cursor.firstBatch.map(profileEntry)
+ const messages = rows.filter(r => r.ns.endsWith('.SuperTabMessage'))
+ expect(rows.filter(r => r.planSummary === 'COLLSCAN')).toEqual([])
+ for (const r of messages) expect(r.planSummary, JSON.stringify(r)).toMatch(/^IXSCAN/)
+ // A read's later batches (getmore) belong to that read.
+ const reads = rows.filter(r => r.op !== 'getmore')
+ const messageReads = messages.filter(r => r.op !== 'getmore')
+ const docsExamined = rows.reduce((n, r) => n + (r.docsExamined ?? 0), 0)
+ const messageDocs = messages.reduce((n, r) => n + (r.docsExamined ?? 0), 0)
+ const replyBytes = JSON.stringify(reply.body.data).length
+ console.info(`inbox cost: ${CHATS + 4} active chats, ${reads.length} database reads (${messageReads.length} of messages), ` +
+ `${docsExamined} docs examined (${messageDocs} messages), ${bytesFromDatabase} bytes from the database, ${replyBytes} reply bytes\n` +
+ reads.map(r => ` ${r.ns.split('.').at(-1)} ${r.op} ${r.planSummary ?? ''} docs=${r.docsExamined ?? 0}`).join('\n'))
+ // A fixed few reads whatever the chat count: the latest rows, the unread rows, their roots, the look-back for threads
+ // whose newer rows do not show the person (the read marks are another collection). Nothing new to check for delivered
+ // copies here, so no read for them.
+ expect(messageReads.length).toBeLessThanOrEqual(4)
+ expect(reads.length).toBeLessThanOrEqual(6)
+ // Bounded by the caps, never by history: the long chat's rows are read only to its newest UNREAD_SCAN + 1, and each of its
+ // threads the person is not in only to MEMBER_SCAN rows before the mark (not its 301).
+ expect(messageDocs).toBeLessThan(2_000)
+ expect(reply.body.data.latest).toHaveLength(CHATS + 1)
+ // The THREAD_CHATS most recently active unread chats are read: the long chat (none of its threads are the person's) and
+ // THREAD_CHATS - 1 others, each with one thread.
+ expect(reply.body.data.threads).toHaveLength(THREAD_CHATS - 1)
+ // More unread chats than one read takes, and the long chat had more unread rows than its scan: the reply says so.
+ expect(reply.body.data.more).toBe(true)
+ expect(replyBytes).toBeLessThan(60_000)
+ expect(bytesFromDatabase).toBeLessThan(250_000)
+ }, 120_000)
+})
diff --git a/backend/src/routes/superTabs.ts b/backend/src/routes/superTabs.ts
index fff01d1a2..3953aba40 100644
--- a/backend/src/routes/superTabs.ts
+++ b/backend/src/routes/superTabs.ts
@@ -24,6 +24,7 @@ import { attribution } from '../github/attribution.js'
import { githubTokensFromEnv, type GitHubTokens } from '../github/tokens.js'
import { logger } from '../utils/logger.js'
import { SwarmReads } from '../superTabs/read/service.js'
+import { inboxWait, readInbox } from '../superTabs/read/inbox.js'
import { HostReads } from '../superTabs/read/host.js'
import { UPDATE_REQUIRED } from '../superTabs/read/contract.js'
import { isTeamSession, TEAM_SESSION_LIMITED, TEAM_SESSION_MESSAGE } from '../lib/teamSession.js'
@@ -361,6 +362,13 @@ export async function superTabRoutes(app: FastifyInstance, options: { store?: Su
app.get<{ Params: z.infer<typeof tabParams>; Querystring: z.infer<typeof changesQuery> }>('/api/super-tabs/:tabId/changes',
{ preHandler: [validateParams(tabParams), validateQuery(changesQuery)] },
(req, reply) => respond(reply, () => reads.changes(req.user!, req.params.tabId, req.query.cursor, req.query.asAgentId)))
+ // A person's unread threads and each chat's latest message (read/inbox.ts): read when the change feed moves, never on a timer.
+ app.get<{ Params: z.infer<typeof tabParams>; Querystring: z.infer<typeof noQuery> }>('/api/super-tabs/:tabId/inbox',
+ { preHandler: [validateParams(tabParams), validateQuery(noQuery)] }, async (req, reply) => {
+ const wait = await inboxWait(req.user!, req.params.tabId)
+ if (wait) return sendError(reply.header('Retry-After', String(wait)), 'Too many inbox reads. Try again in a moment.', 'RATE_LIMITED', 429)
+ return respond(reply, () => readInbox(store, req.user!, req.params.tabId))
+ })
app.get<{ Params: z.infer<typeof tabParams>; Querystring: z.infer<typeof leaderboardQuery> }>('/api/super-tabs/:tabId/leaderboard',
{ preHandler: [validateParams(tabParams), validateQuery(leaderboardQuery)] },
(req, reply) => respond(reply, () => reads.leaderboard(req.user!, req.params.tabId, req.query.projectId, req.query.week)))
diff --git a/backend/src/superTabs/read/inbox.ts b/backend/src/superTabs/read/inbox.ts
new file mode 100644
index 000000000..cd7635f31
--- /dev/null
+++ b/backend/src/superTabs/read/inbox.ts
@@ -0,0 +1,252 @@
+/**
+ * A person's inbox in one Swarm (phone Home and Buddies): the threads with replies they have not read, and each chat's latest
+ * message, in one read instead of a page of 100 messages per chat. The chats are exactly those the messages read gives this
+ * person (projects.ts readConversation), among the Swarm's active, unarchived ones, and the rows are those a page shows a
+ * person (store.ts messages): never an agent-only notice or a delivered copy of a Lead's notice.
+ *
+ * A thread is unread as Desktop marks it (desktop super_tabs/controller.dart threadUnread): the person started it, replied in
+ * it or was tagged in it; its newest reply is someone else's; and it has replies by others after both the person's read mark
+ * on the chat and their own newest reply there. A chat they never marked read counts as read, as Desktop opens it at its end.
+ * Desktop also keeps a seen mark per thread on the device; the server has none yet, so the chat's mark stands in for it.
+ *
+ * Cost: every message read is bounded by an index range and a limit: the newest UNREAD_SCAN rows past the mark in at most
+ * THREAD_CHATS chats; the roots of the newest CANDIDATE_THREADS threads; MEMBER_SCAN rows before the mark of a thread whose
+ * newer rows do not show the person in it; one row (LATEST_SCAN for an older chat) at each of LATEST_CHATS chats' newest main
+ * message. Text is cut to PREVIEW_CHARS by the database. `more` says a cap was reached. The route allows a person
+ * INBOX_READS per INBOX_WINDOW_MS in each Swarm, counted by every pod together.
+ */
+import type { Prisma, PrismaClient } from '@prisma/client'
+import type { AuthUser } from '../../lib/ssoAuth.js'
+import { createRateLimiter } from '../../lib/rateLimit.js'
+import { isTeamSession, TEAM_SESSION_LIMITED, TEAM_SESSION_MESSAGE } from '../../lib/teamSession.js'
+import { agentOnly, author, fail, SuperTabError, type Actor, type Author, type Conversation, type SuperTabState } from '../model.js'
+import { readConversation } from '../projects.js'
+import { deliveredCopies, type ThreadReply } from '../pageReads.js'
+import { threadSummaries } from '../threads.js'
+import { readMarks } from '../readMarks.js'
+import { readBy } from '../parts/indexed.js'
+import { ms } from '../parts/light.js'
+import type { SuperTabStore } from '../store.js'
+
+/** Rows past the read mark read per chat: the newest ones, so the latest replies are always counted. */
+export const UNREAD_SCAN = 100
+/** Chats whose unread rows are read, those active most recently first. */
+export const THREAD_CHATS = 30
+/** Threads answered, newest reply first. */
+export const MAX_THREADS = 50
+/** Threads with unread replies whose roots are read, newest reply first: room for those the person turns out not to be in. */
+export const CANDIDATE_THREADS = 2 * MAX_THREADS
+/** Rows before the read mark read per thread, newest first, to learn whether the person wrote or was tagged there. */
+export const MEMBER_SCAN = 100
+/** Chats whose latest message is answered, those active most recently first. */
+export const LATEST_CHATS = 100
+/** Rows read back from an older chat's newest message for its newest main one: a chat written before the server kept its
+ * main stream's newest number (lastMainSequence), or one where an older backend stored a hidden copy there. */
+export const LATEST_SCAN = 50
+/** A preview's text, in characters. */
+export const PREVIEW_CHARS = 280
+/** Inbox reads a person may make in one Swarm per window: about one a second. */
+export const INBOX_READS = 10
+export const INBOX_WINDOW_MS = 10_000
+
+export interface InboxMessage { id: string; sequence: number; author: Author; text: string; createdAt: string; attachments: number }
+export interface InboxThread {
+ conversationId: string
+ rootId: string
+ root: InboxMessage
+ lastReplyAt: string
+ lastReplySequence: number
+ /** The newest reply's author who can still read the chat, as thread summaries name them; null when none can. */
+ lastReplier: Author | null
+ /** Replies by anyone else after the person's read mark and their own newest reply in the thread. */
+ unreadReplies: number
+}
+export interface InboxLatest { conversationId: string; message: InboxMessage }
+export interface Inbox { threads: InboxThread[]; latest: InboxLatest[]; more: boolean }
+
+interface Row { id: string; conversationId: string; sequence: number; author: Author; text: string; createdAt: Date; attachments: number; thread: string | null
+ alsoInChat: boolean; mentions: string[]; source: unknown; recipientIds: string[] }
+
+/** A preview's fields, its text cut by the database so a long message never leaves it whole. */
+const FIELDS = { sequence: 1, author: 1, text: { $substrCP: [{ $ifNull: ['$text', ''] }, 0, PREVIEW_CHARS] }, createdAt: 1, attachments: { $size: { $ifNull: ['$attachments', []] } },
+ thread: 1, alsoInChat: 1, mentions: 1, source: 1, recipientIds: 1 }
+/** A scan for unread replies: what a reply is counted, named and placed by, never its text. */
+const SCAN_FIELDS = { sequence: 1, author: 1, createdAt: 1, thread: 1, mentions: 1, source: 1, recipientIds: 1 }
+const row = (raw: Record<string, unknown>): Row => ({ id: raw._id as string, conversationId: raw.conversationId as string, sequence: raw.sequence as number, author: raw.author as Author,
+ text: (raw.text ?? '') as string, createdAt: new Date(ms(raw.createdAt)), attachments: typeof raw.attachments === 'number' ? raw.attachments : 0,
+ thread: (raw.thread ?? null) as string | null, alsoInChat: raw.alsoInChat === true, mentions: (raw.mentions ?? []) as string[], source: raw.source ?? null,
+ recipientIds: (raw.recipientIds ?? []) as string[] })
+const preview = (m: Row): InboxMessage => ({ id: m.id, sequence: m.sequence, author: m.author, text: m.text,
+ createdAt: m.createdAt.toISOString(), attachments: m.attachments })
+/** A thread reply: in a thread whose root is another row. */
+const isReply = (m: Row) => !!m.thread && m.thread !== m.id
+/** A row of the chat's main stream (parts/commit.ts numberMessages): a top-level message, or a reply also sent to the chat. */
+const isMain = (m: Row) => !isReply(m) || m.alsoInChat
+const threadKey = (conversationId: string, root: string) => `${conversationId}\u0000${root}`
+const newestFirst = (a: { at: number; id: string }, b: { at: number; id: string }) => b.at - a.at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)
+const activity = (g: Conversation) => Date.parse(g.lastActivityAt ?? '') || 0
+/** `rows` by `key`, each list in the rows' order. */
+function groupBy(rows: Row[], key: (m: Row) => string): Map<string, Row[]> {
+ const groups = new Map<string, Row[]>()
+ for (const m of rows) {
+ const group = groups.get(key(m))
+ if (group) group.push(m)
+ else groups.set(key(m), [m])
+ }
+ return groups
+}
+
+/** The rows of `rows` a person reads, as a page filters them for a person (store.ts messages, readsRow): no agent-only notice,
+ * and no task chat copy of a Lead's notice that reached the Lead (pageReads.ts deliveredCopies). */
+async function readable(db: PrismaClient, tabId: string, rows: Row[]): Promise<Row[]> {
+ const shown = rows.filter(m => !agentOnly(m.source, m.recipientIds))
+ const copies = shown.length ? await deliveredCopies(db, tabId, shown) : new Set<string>()
+ return shown.filter(m => !copies.has(m.id))
+}
+
+/** Whether the messages read lets `actor` read `group`: the same check. Only its "unavailable" answer means this chat is not
+ * theirs; any other refusal (no longer a member) refuses the whole read, as it refuses every chat's page. */
+function readsChat(state: SuperTabState, actor: Actor, group: Conversation): boolean {
+ try {
+ readConversation(state, actor, group.id)
+ return true
+ } catch (error) {
+ if (error instanceof SuperTabError && error.code === 'NOT_FOUND') return false
+ throw error
+ }
+}
+
+/** Ranges one read covers: each is a few stages of its pipeline, well under Mongo's limit on stages. */
+export const CHATS_PER_READ = 100
+
+/** Runs each range's own pipeline inside one aggregate per CHATS_PER_READ ranges ($unionWith), so the reads stay few whatever
+ * the chat count, and each range's rows are still bounded by its own index range and limit: one $in over every chat could not
+ * bound each by its own read mark, and would walk a busy chat's history for a chat never opened. */
+async function perRange(db: PrismaClient, pipelines: Prisma.InputJsonValue[][]): Promise<Row[]> {
+ const rows: Row[] = []
+ for (let i = 0; i < pipelines.length; i += CHATS_PER_READ) {
+ const [first, ...rest] = pipelines.slice(i, i + CHATS_PER_READ)
+ const raw = await db.superTabMessage.aggregateRaw({ pipeline: [...first!,
+ ...rest.map(pipeline => ({ $unionWith: { coll: 'SuperTabMessage', pipeline } }))] }) as unknown as Array<Record<string, unknown>>
+ rows.push(...raw.map(row))
+ }
+ return rows
+}
+
+/** Up to `limit` rows of one chat, newest first, in its (sequence, id) index range. */
+const newestIn = (tabId: string, conversationId: string, sequence: Record<string, number>, limit: number, fields: Record<string, unknown>): Prisma.InputJsonValue[] =>
+ [{ $match: { tabId, conversationId, sequence } }, { $sort: { sequence: -1, _id: -1 } }, { $limit: limit }, { $project: { ...fields, conversationId: 1 } }] as Prisma.InputJsonValue[]
+
+/** Each chat's newest main message the person reads: the row at its stored main sequence, read alone; the LATEST_SCAN rows
+ * before it only for a chat where that row is not one (an older chat's newest row may be a thread reply or a hidden copy).
+ * `more` when such a chat has none in that range. */
+async function latestIn(db: PrismaClient, tabId: string, chats: Conversation[]): Promise<{ latest: Map<string, Row>; more: boolean }> {
+ const tops = chats.flatMap(g => { const top = g.lastMainSequence ?? g.lastMessageSequence; return top ? [{ id: g.id, top }] : [] })
+ const pick = async (rows: Row[]) => {
+ const shown = (await readable(db, tabId, rows)).filter(isMain)
+ const newest = new Map<string, Row>()
+ for (const m of shown) if (!newest.has(m.conversationId)) newest.set(m.conversationId, m)
+ return newest
+ }
+ const read = (of: typeof tops, limit: number) => perRange(db, of.map(c => newestIn(tabId, c.id, { $lte: c.top }, limit, FIELDS)))
+ const latest = await pick(await read(tops, 1))
+ const missing = tops.filter(c => !latest.has(c.id))
+ if (!missing.length) return { latest, more: false }
+ const deeper = await read(missing, LATEST_SCAN)
+ for (const [id, m] of await pick(deeper)) latest.set(id, m)
+ // A chat with LATEST_SCAN rows and none of them a main row the person reads may have one further back.
+ const full = new Set([...groupBy(deeper, m => m.conversationId)].filter(([, rows]) => rows.length >= LATEST_SCAN).map(([id]) => id))
+ return { latest, more: missing.some(c => !latest.has(c.id) && full.has(c.id)) }
+}
+
+interface Candidate { conversationId: string; root: string; mark: number; replies: Row[]; unread: number; member: boolean; at: number; id: string }
+
+/** The threads of `chats` the person has unread replies in (see the file's comment), from each chat's newest UNREAD_SCAN rows
+ * past their mark. */
+async function threadsIn(db: PrismaClient, tabId: string, state: SuperTabState, chats: Conversation[], personId: string, marks: Map<string, number>):
+ Promise<{ threads: InboxThread[]; more: boolean }> {
+ if (!chats.length) return { threads: [], more: false }
+ const own = (a: Pick<Author, 'kind' | 'id'>) => a.kind === 'person' && a.id === personId
+ const inIt = (m: Row) => own(m.author) || m.mentions.includes(personId)
+ const scanned = await perRange(db, chats.map(g => newestIn(tabId, g.id, { $gt: marks.get(g.id)! }, UNREAD_SCAN + 1, SCAN_FIELDS)))
+ const byChat = groupBy(scanned, m => m.conversationId)
+ // A chat whose newest UNREAD_SCAN rows do not reach its mark: older unread replies are not counted.
+ let more = [...byChat.values()].some(rows => rows.length > UNREAD_SCAN)
+ const replies = await readable(db, tabId, [...byChat.values()].flatMap(rows => rows.slice(0, UNREAD_SCAN)).filter(isReply))
+ // Newest first within each thread, as the scan reads them.
+ const byThread = groupBy(replies, m => threadKey(m.conversationId, m.thread!))
+ const candidates: Candidate[] = []
+ for (const rows of byThread.values()) {
+ const newest = rows[0]!
+ // The person answered last: nothing in it waits for them.
+ if (own(newest.author)) continue
+ const answered = rows.find(m => own(m.author))?.sequence ?? 0
+ const unread = rows.filter(m => !own(m.author) && m.sequence > answered).length
+ candidates.push({ conversationId: newest.conversationId, root: newest.thread!, mark: marks.get(newest.conversationId)!, replies: rows, unread,
+ member: rows.some(inIt), at: newest.createdAt.getTime(), id: newest.id })
+ }
+ candidates.sort(newestFirst)
+ if (candidates.length > CANDIDATE_THREADS) more = true
+ const picked = candidates.slice(0, CANDIDATE_THREADS)
+ if (!picked.length) return { threads: [], more }
+ // Every root by its id, in one read: one the person cannot read hides its thread, as on a page (review 2 of #2168), and a
+ // root counts only in the chat its replies are in.
+ const found = (await readBy(db, 'SuperTabMessage', 'id', [{ $match: { _id: { $in: picked.map(c => c.root) }, tabId } }, { $project: { ...FIELDS, conversationId: 1 } }])).map(row)
+ const roots = new Map((await readable(db, tabId, found.filter(m => !isReply(m)))).map(m => [threadKey(m.conversationId, m.id), m]))
+ const withRoot = picked.filter(c => roots.has(threadKey(c.conversationId, c.root)))
+ for (const c of withRoot) if (inIt(roots.get(threadKey(c.conversationId, c.root))!)) c.member = true
+ // The rest: did the person write or get tagged before their mark? Each thread's newest MEMBER_SCAN rows there, by its thread
+ // index range; only the rows that name them come back.
+ const unknown = withRoot.filter(c => !c.member)
+ if (unknown.length) {
+ const named = await perRange(db, unknown.map(c => [{ $match: { tabId, conversationId: c.conversationId, thread: c.root, sequence: { $lte: c.mark } } },
+ { $sort: { sequence: -1 } }, { $limit: MEMBER_SCAN },
+ { $match: { $or: [{ 'author.kind': 'person', 'author.id': personId }, { mentions: personId }] } }, { $project: { ...SCAN_FIELDS, conversationId: 1 } }]))
+ const members = new Set((await readable(db, tabId, named)).map(m => threadKey(m.conversationId, m.thread!)))
+ for (const c of unknown) if (members.has(threadKey(c.conversationId, c.root))) c.member = true
+ }
+ const groups = new Map(chats.map(g => [g.id, g]))
+ const threads = withRoot.filter(c => c.member).flatMap(c => {
+ const root = roots.get(threadKey(c.conversationId, c.root))!, group = groups.get(c.conversationId)!
+ // Named as a page's thread summary names them: only authors who can still read the chat (threads.ts).
+ const rows: ThreadReply[] = [root, ...c.replies.toReversed()].map(m => ({ thread: c.root, id: m.id, author: m.author, mentions: m.mentions,
+ sequence: m.sequence, createdAt: m.createdAt, agentOnly: false, routing: false }))
+ const [summary] = threadSummaries(state, group, rows)
+ if (!summary) return []
+ return [{ conversationId: group.id, rootId: root.id, root: preview(root), lastReplyAt: summary.lastAt.toISOString(),
+ lastReplySequence: summary.lastSequence, lastReplier: summary.authors[0] ?? null, unreadReplies: c.unread }]
+ })
+ return { threads, more }
+}
+
+/** The inbox of the signed-in person: never an agent's (they have no read marks) or a Team session's (marks are the person's). */
+export async function readInbox(store: SuperTabStore, user: AuthUser, tabId: string): Promise<Inbox> {
+ if (isTeamSession(user)) fail(TEAM_SESSION_LIMITED, TEAM_SESSION_MESSAGE, 403)
+ const db = store.db
+ const { state } = await store.readLoad(user, tabId)
+ const actor = await store.actor(db, user, state)
+ // A person no longer in the Swarm is refused here, as the messages read refuses them, before any chat is looked at.
+ author(state, actor)
+ const chats = state.conversations.filter(g => !g.dormant && readsChat(state, actor, g)).filter(g => !g.archived)
+ .sort((a, b) => newestFirst({ at: activity(a), id: a.id }, { at: activity(b), id: b.id }))
+ const marks = await readMarks(db, tabId, user.sub)
+ // A chat never marked read counts as read: its threads wait for the person's first visit.
+ const unread = chats.filter(g => { const mark = marks.get(g.id) ?? 0; return mark > 0 && g.lastMessageSequence > mark })
+ const shown = chats.slice(0, LATEST_CHATS)
+ const [latest, threaded] = await Promise.all([latestIn(db, tabId, shown), threadsIn(db, tabId, state, unread.slice(0, THREAD_CHATS), user.sub, marks)])
+ const threads = threaded.threads.sort((a, b) => newestFirst({ at: Date.parse(a.lastReplyAt), id: a.rootId }, { at: Date.parse(b.lastReplyAt), id: b.rootId }))
+ const more = chats.length > LATEST_CHATS || latest.more || unread.length > THREAD_CHATS || threaded.more || threads.length > MAX_THREADS
+ return { threads: threads.slice(0, MAX_THREADS), more,
+ latest: shown.flatMap(g => { const m = latest.latest.get(g.id); return m ? [{ conversationId: g.id, message: preview(m) }] : [] }) }
+}
+
+const inboxReads = createRateLimiter('swarm-inbox', INBOX_WINDOW_MS, INBOX_READS)
+
+/** Seconds a person must wait before reading this Swarm's inbox again, or 0 when they may now. Apps read it when the change
+ * feed moves, which in a busy Swarm is several times a second; this keeps one person's reads to about one a second. */
+export async function inboxWait(user: AuthUser, tabId: string, now = Date.now()): Promise<number> {
+ if (await inboxReads.allows(`${user.sub}:${tabId}`, now)) return 0
+ return Math.max(1, Math.ceil((INBOX_WINDOW_MS - now % INBOX_WINDOW_MS) / 1000))
+}
+/** Test seam: forget this process's counts. */
+export const resetInboxReads = () => inboxReads.reset()
diff --git a/backend/vitest.super-tabs.config.ts b/backend/vitest.super-tabs.config.ts
index 85a57bf9a..942dfc3a5 100644
--- a/backend/vitest.super-tabs.config.ts
+++ b/backend/vitest.super-tabs.config.ts
@@ -1,7 +1,7 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({ test: { include: ['integration/superTabs.test.ts', 'integration/superTabSharing.test.ts', 'integration/superTabGithub.test.ts', 'integration/superTabTeam.test.ts', 'integration/superTabWorkspaces.test.ts',
- 'integration/superTabRead.test.ts', 'integration/superTabReadMarks.test.ts', 'integration/superTabLeaderboard.test.ts', 'integration/superTabMigration.test.ts', 'integration/superTabLiveness.test.ts',
+ 'integration/superTabRead.test.ts', 'integration/superTabReadMarks.test.ts', 'integration/superTabInbox.test.ts', 'integration/superTabLeaderboard.test.ts', 'integration/superTabMigration.test.ts', 'integration/superTabLiveness.test.ts',
'integration/superTabTidy.test.ts', 'integration/superTabProfile.test.ts', 'integration/superTabLegacyDrop.test.ts', 'integration/shareRaces.test.ts', 'integration/superTabIndexes.test.ts', 'integration/superTabRouteDecisions.test.ts',
'integration/superTabHotReads.test.ts', 'integration/swarmSweepStarts.test.ts', 'integration/superTabGroupPlans.test.ts', 'integration/superTabInvitationPlans.test.ts', 'integration/superTabCache.test.ts',
'integration/superTabObserver.test.ts', 'integration/productUsageConflicts.test.ts', 'integration/productUsageReceipts.test.ts', 'integration/schemaRelease.test.ts', 'integration/superTabBuddies.test.ts',