The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import type { Prisma, SuperTabDelivery, SuperTabMessage } from '@prisma/client'
import { routingRow } from './buddies.js'
import { ms } from './parts/light.js'
import type { ReactionRow } from './reactions.js'
import { agentOnly, stableId, type Author } from './model.js'
import { readBy } from './parts/indexed.js'
import { prismaRow } from './rawRows.js'
type Transaction = Prisma.TransactionClient
// A page's rows are read by a plain {tabId, messageId: {$in}} match. Prisma sends `messageId: { in }` as an $expr $or: Mongo
// cannot bound an index by it, and on 2026-10-28 each page read scanned every delivery of its Swarm (prod at 100% CPU).
// Each read here uses its tabId+messageId index and examines only the rows it returns (integration/superTabIndexes.test.ts).
/** The reactions on one page of messages. */
export async function pageReactions(tx: Transaction, tabId: string, messageIds: string[]): Promise<ReactionRow[]> {
return await tx.superTabReaction.aggregateRaw({ pipeline: [{ $match: { tabId, messageId: { $in: messageIds } } },
{ $project: { _id: 0, messageId: 1, emoji: 1, actorKind: 1, actorId: 1 } }] }) as unknown as ReactionRow[]
}
export type PageDelivery = Pick<SuperTabDelivery, 'id' | 'messageId' | 'agentId' | 'status' | 'receipt' | 'holdReason' | 'holdSince'
| 'claimFailingSince' | 'claimFailures' | 'deliveredAt' | 'answeredAt' | 'createdAt' | 'updatedAt'>
const date = (value: unknown) => value == null ? null : value instanceof Date ? value : new Date(ms(value))
/** The deliveries of one page of messages: who each message went to and how far it got, from the live rows and the
* compacted receipts (deliveryCompaction.ts), a live row winning for the same agent. As Prisma read them: dates as Dates,
* and an optional field an older record lacks as null. A compacted receipt gives only these fields, never its host's. */
export async function pageDeliveries(tx: Transaction, tabId: string, messageIds: string[]): Promise<PageDelivery[]> {
const rows = await tx.superTabDelivery.aggregateRaw({ pipeline: [{ $match: { tabId, messageId: { $in: messageIds } } },
{ $project: { messageId: 1, agentId: 1, status: 1, receipt: 1, holdReason: 1, holdSince: 1, claimFailingSince: 1, claimFailures: 1, deliveredAt: 1, answeredAt: 1, createdAt: 1, updatedAt: 1 } }] }) as
unknown as Array<Record<string, unknown>>
// After the rows, never beside them: a delivery compacted between the two reads is then in both, never in neither.
const compacted = await compactedReceipts(tx, tabId, messageIds)
const live = rows.map(row => pageDelivery({ ...row, id: row._id }))
const held = new Set(live.map(d => `${d.messageId}\0${d.agentId}`))
return [...live, ...compacted.filter(d => !held.has(`${d.messageId}\0${d.agentId}`)).map(pageDelivery)]
}
const pageDelivery = (row: Record<string, unknown>): PageDelivery => ({ id: row.id as string, messageId: row.messageId as string, agentId: row.agentId as string,
status: row.status as string, receipt: (row.receipt ?? null) as string | null, holdReason: (row.holdReason ?? null) as string | null, holdSince: date(row.holdSince),
claimFailingSince: date(row.claimFailingSince), claimFailures: (row.claimFailures ?? null) as number | null,
deliveredAt: date(row.deliveredAt), answeredAt: date(row.answeredAt), createdAt: date(row.createdAt)!, updatedAt: date(row.updatedAt)! })
/** The fields a receipt document keeps of each of its deliveries, rather than per receipt. */
const RECEIPT_DOCUMENT = new Set(['id', 'tabId', 'conversationId', 'messageId', 'agentId'])
/** A delivery row as its compacted receipt stores it in its message's SuperTabReceipt (deliveryCompaction.ts), for a raw `$set`
* (dates as extended JSON): the row as it was, under its agent's id and with its own as `deliveryId`, without what the
* document holds once (its message, Swarm and conversation). The whole row is kept: a host that still holds the delivery is
* answered from it (store.ts deliveryReceipt, deliver), as from the row. */
export function receiptFields(row: SuperTabDelivery): Prisma.InputJsonObject {
const stored: Record<string, Prisma.InputJsonValue | null> = { deliveryId: row.id }
for (const [key, value] of Object.entries(row)) {
if (!RECEIPT_DOCUMENT.has(key)) stored[key] = value instanceof Date ? { $date: value.toISOString() } : value as Prisma.InputJsonValue | null
}
return stored as Prisma.InputJsonObject
}
/** A receipt document as raw reads return it. */
interface ReceiptDocument { _id: string; tabId: string; conversationId: string; receipts?: Record<string, Record<string, unknown>> }
/** One compacted receipt as the delivery row it was: dates as Dates, a field the row did not have as null. */
function compactedRow(document: ReceiptDocument, agentId: string, stored: Record<string, unknown>): SuperTabDelivery {
const row: Record<string, unknown> = {}
for (const [key, value] of Object.entries(stored)) {
row[key] = value && typeof value === 'object' && '$date' in value ? date(value) : value
}
const { deliveryId, ...fields } = row
return { ...fields, id: deliveryId as string, tabId: document.tabId, conversationId: document.conversationId, messageId: document._id, agentId } as SuperTabDelivery
}
/** The compacted receipts of these messages, one per agent. By the primary key; the Swarm checked in the match. */
async function compactedReceipts(tx: Transaction, tabId: string, messageIds: string[]): Promise<SuperTabDelivery[]> {
const documents = await readBy(tx, 'SuperTabReceipt', 'id', [{ $match: { _id: { $in: messageIds }, tabId } }]) as unknown as ReceiptDocument[]
return documents.flatMap(document => Object.entries(document.receipts ?? {}).map(([agentId, stored]) => compactedRow(document, agentId, stored)))
}
/** A delivery of this agent that was compacted, as the row it was, found by its id (index [tabId, deliveryIds]); null when
* there is none. A host back after weeks offline still settles what its journal holds (store.ts deliveryReceipt, deliver). */
export async function compactedDelivery(tx: Transaction, tabId: string, agentId: string, deliveryId: string): Promise<SuperTabDelivery | null> {
const [document] = await readBy(tx, 'SuperTabReceipt', 'delivery', [{ $match: { tabId, deliveryIds: deliveryId } }, { $limit: 1 }]) as unknown as ReceiptDocument[]
const stored = document?.receipts?.[agentId]
return document && stored?.deliveryId === deliveryId ? compactedRow(document, agentId, stored) : null
}
/** The deliveries a host is still told about (the rest are answered, cancelled or confirmed). */
export const OPEN_DELIVERY = ['queued', 'claimed', 'uncertain'] as const
/** The deliveries a cancel ends: the open ones, and those gathered for a digest (noticeDigest.ts), which hosts are not told about
* yet: a gathered delivery outlives its reason no more than an open one. */
export const CANCELLABLE_DELIVERY = [...OPEN_DELIVERY, 'gathered'] as const
/** An agent's deliveries gathered for its digest, oldest first, at most [limit]: one indexed read, the agent's as its open ones are. */
export async function agentGatheredDeliveries(tx: Transaction, tabId: string, agentId: string, limit: number): Promise<SuperTabDelivery[]> {
const rows = await readBy(tx, 'SuperTabDelivery', 'agentOpen', [{ $match: { tabId, agentId, status: 'gathered' } }, { $sort: { createdAt: 1 } }, { $limit: limit }])
return inConversationOrder(tx, tabId, rows.map(row => prismaRow<SuperTabDelivery>('SuperTabDelivery', row)))
}
/** How many deliveries an agent has gathered: read only when a digest is full, to say how many more wait. */
export async function agentGatheredCount(tx: Transaction, tabId: string, agentId: string): Promise<number> {
return Number((await readBy(tx, 'SuperTabDelivery', 'agentOpen', [{ $match: { tabId, agentId, status: 'gathered' } }, { $count: 'n' }]))[0]?.n ?? 0)
}
/** An agent's open deliveries, oldest first and each conversation's tied rows in its order: one indexed read, and the tied
* messages' sequences only when there are ties. */
export async function agentOpenDeliveries(tx: Transaction, tabId: string, agentId: string, prioritized = false): Promise<SuperTabDelivery[]> {
// By the agent index (parts/indexed.ts): Prisma sent `status: { in }` as an $expr $or, planned as one scan per status.
const rows = await readBy(tx, 'SuperTabDelivery', 'agentOpen', [{ $match: { tabId, agentId, status: { $in: [...OPEN_DELIVERY] } } },
{ $sort: { createdAt: 1 } }, { $limit: 100 }])
const priority = prioritized ? await readBy(tx, 'SuperTabDelivery', 'agentProposal', [
{ $match: { tabId, agentId, status: { $in: [...OPEN_DELIVERY] }, inputPriority: { $gte: 0 } } },
{ $sort: { inputPriority: 1, createdAt: 1 } }, { $limit: 100 },
]) : []
const joined = [...new Map([...priority, ...rows].map(r => [String(r._id), r])).values()]
const ordered = await inConversationOrder(tx, tabId, joined.map(row => prismaRow<SuperTabDelivery>('SuperTabDelivery', row)))
// Old notices remain readable; missing priority means a normal server notice, below P0.
if (prioritized) ordered.sort((a, b) => (a.inputPriority ?? 1) - (b.inputPriority ?? 1) || a.createdAt.getTime() - b.createdAt.getTime())
return ordered
}
/** An agent's open deliveries, read by creation time, with each conversation's tied rows put in its message order. One
* write's deliveries share a creation time, and Mongo breaks that tie by insertion order alone. Each conversation keeps its
* places among the others; only tied rows have their messages' sequences read. */
export async function inConversationOrder<T extends Pick<SuperTabDelivery, 'conversationId' | 'messageId' | 'createdAt'>>(
tx: Transaction, tabId: string, rows: T[]): Promise<T[]> {
const ties = new Map<string, number[]>()
rows.forEach((row, index) => {
const key = `${row.conversationId}\0${row.createdAt.getTime()}`
ties.set(key, [...ties.get(key) ?? [], index])
})
const tied = [...ties.values()].filter(places => places.length > 1)
if (!tied.length) return rows
// Matched by id alone: with tabId in the match, the planner can cover it from the tabId+conversationId+sequence index
// and walk every message key of the Swarm. The Swarm is checked here instead.
const found = await tx.superTabMessage.aggregateRaw({ pipeline: [
{ $match: { _id: { $in: tied.flat().map(index => rows[index]!.messageId) } } }, { $project: { tabId: 1, sequence: 1 } }] }) as
unknown as Array<{ _id: string; tabId: string; sequence: number }>
const sequence = new Map(found.filter(message => message.tabId === tabId).map(message => [message._id, message.sequence]))
const ordered = [...rows]
for (const places of tied) {
const group = places.map(index => rows[index]!)
.sort((a, b) => (sequence.get(a.messageId) ?? Infinity) - (sequence.get(b.messageId) ?? Infinity))
places.forEach((index, at) => { ordered[index] = group[at]! })
}
return ordered
}
/** Where a page of messages starts: past this sequence, and within it past this id when one is given. */
export interface MessageEdge { sequence: number; id?: string }
/**
* Up to `limit` messages of one conversation past `edge` (from the newest or oldest without one), in (sequence, id) order
* forward or backward. A cursor inside a sequence is two reads its index bounds: the rest of that sequence by id, then the
* sequences past it. As one `$or`, the planner could not bound it: reading on from inside a sequence walked the whole
* conversation before it (roadmap Phase 3.5; integration/superTabNamedReads.test.ts).
*/
export async function messagesPast(tx: Transaction, at: { tabId: string; conversationId: string; messageId?: string; thread?: string },
edge: MessageEdge | undefined, forward: boolean, limit: number): Promise<SuperTabMessage[]> {
const [op, dir] = forward ? ['$gt', 1] as const : ['$lt', -1] as const
const index = at.messageId ? 'id' : at.thread ? 'thread' : 'sequence'
const read = async (sequence: unknown, after: string | undefined, take: number) => take <= 0 ? [] : (await readBy(tx, 'SuperTabMessage', index, [
{ $match: { tabId: at.tabId, conversationId: at.conversationId, ...at.thread ? { thread: at.thread } : {},
...at.messageId || after !== undefined ? { _id: { ...at.messageId ? { $eq: at.messageId } : {}, ...after !== undefined ? { [op]: after } : {} } } : {},
...sequence !== undefined ? { sequence } : {} } },
{ $sort: { sequence: dir, _id: dir } }, { $limit: take }])).map(row => prismaRow<SuperTabMessage>('SuperTabMessage', row))
const same = edge?.id !== undefined ? await read(edge.sequence, edge.id, limit) : []
return [...same, ...await read(edge ? { [op]: edge.sequence } : undefined, undefined, limit - same.length)]
}
/**
* Up to `limit` messages before `edge`, newest first, as the client `tx` holds them, or null when it does not hold the row
* at `edge` yet: an older page read on a secondary (task #2970). One read of `sequence <= edge`, so the row the reader
* pages from and the rows before it come from one member at once. A member that holds the edge holds every row before it:
* a Swarm's messages are numbered in the write that stores them, one write at a time, and a secondary reads whole applied
* writes. More than one row at the edge's sequence (one command's rows from before the store by parts) is left to the
* primary, as is an edge with no row, so a page never skips a row the primary has.
*/
export async function heldPast(tx: Transaction, at: { tabId: string; conversationId: string; thread?: string }, edge: MessageEdge,
limit: number): Promise<SuperTabMessage[] | null> {
const rows = (await readBy(tx, 'SuperTabMessage', at.thread ? 'thread' : 'sequence', [
{ $match: { tabId: at.tabId, conversationId: at.conversationId, ...at.thread ? { thread: at.thread } : {}, sequence: { $lte: edge.sequence } } },
{ $sort: { sequence: -1, _id: -1 } }, { $limit: limit + 1 }])).map(row => prismaRow<SuperTabMessage>('SuperTabMessage', row))
const atEdge = rows.filter(m => m.sequence === edge.sequence)
if (atEdge.length !== 1 || (edge.id !== undefined && atEdge[0]!.id !== edge.id)) return null
return rows.filter(m => m.sequence < edge.sequence)
}
/**
* The messages of one conversation with these ids, by the id index: what a page's rows name as their thread or reply-to but
* the page does not hold (store.ts messages), so what this reader may see of them can be checked. Named, not Prisma's `id in`,
* which is sent as $expr and left to the planner (roadmap Phase 3; plan test in integration/superTabNamedReads.test.ts).
*/
export async function messagesById(tx: Transaction, tabId: string, conversationId: string, ids: string[]):
Promise<Array<Pick<SuperTabMessage, 'id' | 'source' | 'recipientIds' | 'thread' | 'buddySession' | 'author' | 'buddyProject'>>> {
if (!ids.length) return []
// Its author too: a project session of a person's one Buddy chat reads only its person's words and its own (buddySessions.ts).
const rows = await readBy(tx, 'SuperTabMessage', 'id', [{ $match: { _id: { $in: ids }, tabId, conversationId } },
{ $project: { source: 1, recipientIds: 1, thread: 1, buddySession: 1, author: 1, buddyProject: 1 } }])
return rows.map(row => {
const { id, source, recipientIds, thread, buddySession, author, buddyProject } = prismaRow<SuperTabMessage>('SuperTabMessage', row)
return { id, source, recipientIds, thread: thread ?? null, buddySession: buddySession ?? null, author, buddyProject: buddyProject ?? null }
})
}
/**
* The stored rows with these ids, whichever Swarm or conversation holds them: what a machine report's or a Lead poll's time
* check compares its notices with before it writes them (store.ts writeMessages keepStored). A plain `_id` match through the id
* index; Prisma's `id: { in }` sent it as `$expr` (8% of the slow message reads in Ken's log of 11-04, ~1 s for 101 rows).
*/
export async function storedMessages(tx: Transaction, ids: string[]): Promise<Array<{ id: string; tabId: string; conversationId: string; source: unknown }>> {
if (!ids.length) return []
const rows = await readBy(tx, 'SuperTabMessage', 'id', [{ $match: { _id: { $in: ids } } }, { $project: { tabId: 1, conversationId: 1, source: 1 } }])
return rows.map(row => ({ id: row._id as string, tabId: row.tabId as string, conversationId: row.conversationId as string, source: row.source ?? null }))
}
/** The Lead's notices that leave a copy in a task's chat when the Lead hears them in its own chat (terminal.ts toLead). */
const LEAD_COPY_KINDS = new Set<unknown>(['work-lost', 'work-resumed'])
/** Of `rows`, the task chat copies stored before #2177 marked them (liveness.ts leadNoticeRows) whose project chat copy
* reached the Lead: that same notice, so read by no one either (re-check of #2177). A copy whose Lead copy reached no one,
* or that has none, stays shown: a person is then the only one who can act. Each Lead copy is read by its id alone, its
* Swarm checked here (inConversationOrder), and only when a row could be such a copy. */
export async function deliveredCopies(tx: Transaction, tabId: string, rows: ReadonlyArray<{ id: string; source: unknown; recipientIds: readonly string[] }>):
Promise<Set<string>> {
const copies = rows.filter(row => {
const source = row.source as { kind?: unknown; recipientsOnly?: unknown } | null | undefined
return !row.recipientIds.length && LEAD_COPY_KINDS.has(source?.kind) && source?.recipientsOnly !== true
})
if (!copies.length) return new Set()
const leads = await tx.superTabMessage.aggregateRaw({ pipeline: [
{ $match: { _id: { $in: copies.map(row => stableId(row.id, 'lead')) } } }, { $project: { tabId: 1, recipientIds: 1 } }] }) as
unknown as Array<{ _id: string; tabId: string; recipientIds?: string[] }>
const reached = new Set(leads.filter(lead => lead.tabId === tabId && lead.recipientIds?.length).map(lead => lead._id))
return new Set(copies.filter(row => reached.has(stableId(row.id, 'lead'))).map(row => row.id))
}
/** `routing`: a route, override or rule row (buddies.ts routingRow), which a Buddy never reads, as a reply or a thread's root.
* `count`: how many of the thread's rows this one stands for (pageThreads); a row without it is one. */
export interface ThreadReply { thread: string; id: string; author: Author; mentions: string[]; sequence: number; createdAt: Date; agentOnly: boolean; routing: boolean
count?: number
/** A reply in a Buddy thread session's thread (buddySessions.ts), which the main Buddy never counts. */
session?: boolean
/** Its source's kind, and what it says for its task's row (threads.ts). */
kind?: unknown
ask?: import('./taskLine.js').TaskAsk }
/** The rows of one thread that every reader sees or skips alike: one author, and one kind of row (agentOnly, routingRow). */
interface ThreadPart { thread: string; root: boolean; kind?: unknown; only?: unknown; to: boolean; session?: boolean | null
ask?: import('./taskLine.js').TaskAsk }
/** One row of a part, by its place in the thread. */
interface ThreadPlace { s: number; i: string; at: unknown; author: Author; slot?: number }
/** What the database sums up of some threads' rows (threadFacet), as it returns it: the same whoever reads. */
export interface ThreadFacet {
parts: Array<{ _id: ThreadPart; count: number; oldest: ThreadPlace; newest: ThreadPlace; root?: { source?: unknown; recipientIds?: string[] } | null }>
mentions: Array<{ _id: ThreadPart & { person: string }; first: ThreadPlace }>
}
/**
* The threads rooted at `roots` in one conversation, roots included, as the database sums them up: it groups the rows of each
* part (ThreadPart) and returns its oldest and newest row, how many it has, and where each person it tags is first tagged.
* That is every place a summary reads: first appearances, the newest reply of each author, the count and the last reply. So
* a poll that brings one reply of a 500-reply thread gets a few rows back, not 500, while the summaries stay what every
* row gave (integration/superTabIndexes.test.ts). Matched as the page reads are, so the thread index bounds it; the database
* still examines each of the threads' rows.
*/
export async function threadFacet(tx: Transaction, tabId: string, conversationId: string, roots: string[]): Promise<ThreadFacet> {
// No $sort: it makes the planner walk the conversation's sequence index instead.
return summed(tx, 'thread', { tabId, conversationId, thread: { $in: roots } })
}
/** threadFacet of the rows `ids` alone, read by `_id`: what a reply's rows add to their threads' summaries (foldReplies). */
async function rowsFacet(tx: Transaction, tabId: string, conversationId: string, ids: string[]): Promise<ThreadFacet> {
return summed(tx, 'id', { _id: { $in: ids }, tabId, conversationId })
}
async function summed(tx: Transaction, index: 'thread' | 'id', match: Prisma.InputJsonObject): Promise<ThreadFacet> {
const root = { $eq: ['$_id', '$thread'] }
const part = { thread: '$thread', root, kind: '$source.kind', only: '$source.recipientsOnly', ask: '$source.ask',
to: { $gt: [{ $size: { $ifNull: ['$recipientIds', []] } }, 0] }, author: { kind: '$author.kind', id: '$author.id' },
session: { $eq: ['$buddySession', true] } }
// (sequence, id) lead each place, so the database's min and max order rows as threadReplies sorts them (ids are ASCII).
const place = { s: '$sequence', i: '$_id', at: '$createdAt', author: '$author' }
const [found] = await readBy(tx, 'SuperTabMessage', index, [{ $match: match },
{ $facet: {
parts: [{ $group: { _id: part, count: { $sum: 1 }, oldest: { $min: place }, newest: { $max: place },
// A root is its part's only row: what deliveredCopies reads of it.
root: { $first: { $cond: [root, { source: '$source', recipientIds: '$recipientIds' }, '$$REMOVE'] } } } }],
mentions: [{ $unwind: { path: '$mentions', includeArrayIndex: 'slot' } },
{ $group: { _id: { ...part, person: '$mentions' }, first: { $min: { s: '$sequence', i: '$_id', slot: { $toInt: '$slot' }, at: '$createdAt', author: '$author' } } } }],
} }]) as unknown as Array<Partial<ThreadFacet>>
return { parts: found?.parts ?? [], mentions: found?.mentions ?? [] }
}
/** The rows a page's thread summaries are counted from (threads.ts), oldest first, from what threadFacet gave: a few rows that
* stand for every row of each thread (`count` on the newest of each part), with what each reader skips marked. */
export async function threadReplies(tx: Transaction, tabId: string, { parts, mentions }: ThreadFacet): Promise<ThreadReply[]> {
const copies = await deliveredCopies(tx, tabId, parts.flatMap(p => p._id.root && p.root
? [{ id: p._id.thread, source: p.root.source, recipientIds: p.root.recipientIds ?? [] }] : []))
// agentOnly reads only whether a row has recipients, not who they are.
const hidden = (key: ThreadPart) => (key.root && copies.has(key.thread))
|| agentOnly({ kind: key.kind, recipientsOnly: key.only }, key.to ? ['recipient'] : [])
const rows = new Map<string, ThreadReply & { slots: Array<[number, string]> }>()
const row = (key: ThreadPart, at: ThreadPlace) => {
let kept = rows.get(at.i)
if (!kept) rows.set(at.i, kept = { thread: key.thread, id: at.i, author: at.author, mentions: [], sequence: at.s, createdAt: new Date(ms(at.at)),
agentOnly: hidden(key), routing: routingRow({ kind: key.kind }), ...(key.session ? { session: true } : {}), kind: key.kind,
...(key.ask ? { ask: key.ask } : {}), count: 0, slots: [] })
return kept
}
for (const p of parts) {
row(p._id, p.oldest)
row(p._id, p.newest).count = p.count
}
for (const m of mentions) row(m._id, m.first).slots.push([m.first.slot!, m._id.person])
return [...rows.values()].map(({ slots, ...reply }) => ({ ...reply, mentions: slots.sort((a, b) => a[0] - b[0]).map(([, person]) => person) }))
.sort((a, b) => a.sequence - b.sequence || (a.id < b.id ? -1 : 1))
}
/**
* The rows the threads rooted at `roots` are summed up from (threadReplies). A Swarm whose summaries are backfilled
* (threadBackfill.ts) reads them by root id from SuperTabThread, one small document per thread with a reply; any other runs
* threadFacet over the threads' rows. A root with no summary has no reply, and threads.ts leaves such a thread out, so it is
* left out here too.
*/
export async function pageThreads(tx: Transaction, tabId: string, conversationId: string, roots: string[]): Promise<ThreadReply[]> {
return threadsAcross(tx, tabId, roots.map(root => ({ conversationId, root })))
}
/** pageThreads for threads of any conversations of one Swarm (read/followedThreads.ts): the summaries of all of them by root id
* in one read, each counted only in the Swarm and conversation named for it; threadFacet per conversation before the backfill.
* Matched by `_id` alone, so no other index of the collection (tabId, conversationId) can serve it and walk a Swarm's every
* summary. */
export async function threadsAcross(tx: Transaction, tabId: string, threads: ReadonlyArray<{ conversationId: string; root: string }>): Promise<ThreadReply[]> {
if (!threads.length) return []
if (!await threadsBackfilled(tx, tabId)) {
const byChat = new Map<string, string[]>()
for (const t of threads) byChat.set(t.conversationId, [...byChat.get(t.conversationId) ?? [], t.root])
const facets = []
for (const [conversationId, roots] of byChat) facets.push(await threadFacet(tx, tabId, conversationId, roots))
return threadReplies(tx, tabId, { parts: facets.flatMap(f => f.parts), mentions: facets.flatMap(f => f.mentions) })
}
const named = new Set(threads.map(t => `${t.conversationId}\u0000${t.root}`))
const stored = (await readBy(tx, 'SuperTabThread', 'id', [{ $match: { _id: { $in: [...new Set(threads.map(t => t.root))] } } },
{ $project: { tabId: 1, conversationId: 1, parts: 1, mentions: 1 } }]) as unknown as Array<ThreadFacet & { _id: string; tabId: string; conversationId: string }>)
.filter(t => t.tabId === tabId && named.has(`${t.conversationId}\u0000${t._id}`))
return threadReplies(tx, tabId, { parts: stored.flatMap(t => t.parts), mentions: stored.flatMap(t => t.mentions) })
}
/**
* Of the threads rooted at `threads`, those whose summary shows `personId` writing or tagged in a row a person reads (what
* threadReplies marks agentOnly is left out), as `${conversationId}\u0000${root}`. A backfilled Swarm's only (threadsBackfilled):
* its summaries hold every reply. One read by `_id`, as threadsAcross, that returns only the summaries naming the person, and
* of each only the parts and tags that do, without their rows' authors or the root's source: the member check needs no more
* (inbox.ts namedInThreads). Reading them whole made the threads route return 2.8x the database bytes per call (task #2838).
*/
export async function threadsNaming(tx: Transaction, tabId: string, threads: ReadonlyArray<{ conversationId: string; root: string }>,
personId: string): Promise<Set<string>> {
if (!threads.length) return new Set()
const named = new Set(threads.map(t => `${t.conversationId}\u0000${t.root}`))
// A part's key fields that decide whether a reader skips its rows (threadReplies hidden).
const key = (of: string) => ({ root: `${of}._id.root`, kind: `${of}._id.kind`, only: `${of}._id.only`, to: `${of}._id.to` })
const stored = (await readBy(tx, 'SuperTabThread', 'id', [
{ $match: { _id: { $in: [...new Set(threads.map(t => t.root))] },
$or: [{ parts: { $elemMatch: { '_id.author.kind': 'person', '_id.author.id': personId } } }, { 'mentions._id.person': personId }] } },
{ $project: { tabId: 1, conversationId: 1,
parts: { $map: { input: { $filter: { input: '$parts', as: 'p',
cond: { $and: [{ $eq: ['$$p._id.author.kind', 'person'] }, { $eq: ['$$p._id.author.id', personId] }] } } }, as: 'p', in: key('$$p') } },
mentions: { $map: { input: { $filter: { input: '$mentions', as: 'm', cond: { $eq: ['$$m._id.person', personId] } } }, as: 'm', in: key('$$m') } },
// The root's row, which deliveredCopies reads: only what it decides by.
root: { $map: { input: { $filter: { input: '$parts', as: 'p', cond: { $eq: ['$$p._id.root', true] } } }, as: 'p',
in: { kind: '$$p.root.source.kind', only: '$$p.root.source.recipientsOnly', to: { $gt: [{ $size: { $ifNull: ['$$p.root.recipientIds', []] } }, 0] } } } },
} }]) as unknown as Array<{ _id: string; tabId: string; conversationId: string; parts: NamingPart[]; mentions: NamingPart[]
root: Array<{ kind?: unknown; only?: unknown; to: boolean }> }>)
.filter(t => t.tabId === tabId && named.has(`${t.conversationId}\u0000${t._id}`))
const copies = await deliveredCopies(tx, tabId, stored.flatMap(t => t.root.slice(0, 1).map(r =>
({ id: t._id, source: { kind: r.kind, recipientsOnly: r.only }, recipientIds: r.to ? ['recipient'] : [] }))))
const hidden = (thread: string, p: NamingPart) => (p.root === true && copies.has(thread))
|| agentOnly({ kind: p.kind, recipientsOnly: p.only }, p.to ? ['recipient'] : [])
return new Set(stored.filter(t => [...t.parts, ...t.mentions].some(p => !hidden(t._id, p))).map(t => `${t.conversationId}\u0000${t._id}`))
}
/** What threadsNaming reads of a part or tag: what threadReplies decides agentOnly by. */
interface NamingPart { root?: boolean; kind?: unknown; only?: unknown; to?: boolean }
/**
* Rewrites the summaries of the threads rooted at `roots` in one conversation from their rows now: one document per thread
* with a reply, as threadFacet gives it, and none for a thread left with no reply. Run in the transaction that inserts a reply
* (refreshReplies) or deletes one, so a summary holds what its rows do: two replies at once write the same document, and one
* of their transactions runs again.
*/
export async function refreshThreads(tx: Transaction, tabId: string, conversationId: string, roots: string[]): Promise<ThreadFacet> {
if (!roots.length) return { parts: [], mentions: [] }
const facet = await threadFacet(tx, tabId, conversationId, roots), { parts, mentions } = facet
const updatedAt = { $date: new Date().toISOString() }
const updates = roots.flatMap(root => {
const own = parts.filter(p => p._id.thread === root)
if (!own.some(p => !p._id.root)) return []
return [{ q: { _id: root }, u: { $set: { tabId, conversationId, parts: own, mentions: mentions.filter(m => m._id.thread === root), updatedAt } },
upsert: true }]
})
// Raw, so the stored parts read back as the $facet returns them, dates as dates.
if (updates.length) await tx.$runCommandRaw({ update: 'SuperTabThread', updates: updates as unknown as Prisma.InputJsonArray })
// A thread whose last reply was deleted (messageEdits.ts) has no summary left.
const empty = roots.filter(root => !parts.some(p => p._id.thread === root && !p._id.root))
if (empty.length) await tx.$runCommandRaw({ delete: 'SuperTabThread', deletes: [{ q: { _id: { $in: empty }, tabId, conversationId }, limit: 0 }] as unknown as Prisma.InputJsonArray })
return facet
}
/** A part's or tag's key by value, whatever its fields' order: the database groups by the whole key. */
const keyOf = (key: unknown): string => JSON.stringify(key, (_, value: unknown) => value && typeof value === 'object' && !Array.isArray(value)
? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : value)
/** Places in the order the database's $min and $max give them: by sequence, then id, then a tag's slot. */
const before = (a: ThreadPlace, b: ThreadPlace) => Number(a.s) - Number(b.s) || (a.i < b.i ? -1 : a.i > b.i ? 1 : 0) || (a.slot ?? 0) - (b.slot ?? 0)
/** A thread's summary with the rows `added` sums up folded in, as threadFacet gives every row of both: each part's count adds,
* its oldest row is the older and its newest the newer; each person's first tag in a part is the earlier. */
export function foldFacet(stored: ThreadFacet, added: ThreadFacet): ThreadFacet {
const parts = new Map(stored.parts.map(p => [keyOf(p._id), { ...p }]))
for (const p of added.parts) {
const kept = parts.get(keyOf(p._id))
if (!kept) parts.set(keyOf(p._id), { ...p })
else Object.assign(kept, { count: kept.count + p.count, oldest: before(p.oldest, kept.oldest) < 0 ? p.oldest : kept.oldest,
newest: before(p.newest, kept.newest) > 0 ? p.newest : kept.newest })
}
const mentions = new Map(stored.mentions.map(m => [keyOf(m._id), { ...m }]))
for (const m of added.mentions) {
const kept = mentions.get(keyOf(m._id))
if (!kept || before(m.first, kept.first) < 0) mentions.set(keyOf(m._id), { ...m })
}
return { parts: [...parts.values()], mentions: [...mentions.values()] }
}
/**
* Folds the rows `ids`, just inserted in this transaction and replying in threads of one conversation, into those threads'
* stored summaries (foldFacet): it reads each summary by root id and only the new rows by `_id`, never the threads' other
* rows, so a reply in a 500-reply thread examines a few documents, not 500 (task #2974). A thread with no summary yet (its
* first reply, or a Swarm not backfilled) is counted whole once (refreshThreads). Two replies at once both write the same
* summary, so one transaction meets a write conflict and runs again, reading the summary the other committed. Answers the
* threads' whole facets, as refreshThreads does.
*/
export async function foldReplies(tx: Transaction, tabId: string, conversationId: string,
rows: ReadonlyArray<{ id: string; thread: string }>): Promise<ThreadFacet> {
const roots = [...new Set(rows.map(row => row.thread))]
const stored = new Map((await readBy(tx, 'SuperTabThread', 'id', [{ $match: { _id: { $in: roots } } },
{ $project: { tabId: 1, conversationId: 1, parts: 1, mentions: 1 } }]) as unknown as Array<ThreadFacet & { _id: string; tabId: string; conversationId: string }>)
.filter(t => t.tabId === tabId && t.conversationId === conversationId).map(t => [t._id, t]))
const whole = await refreshThreads(tx, tabId, conversationId, roots.filter(root => !stored.has(root)))
const folding = rows.filter(row => stored.has(row.thread))
if (!folding.length) return whole
const added = await rowsFacet(tx, tabId, conversationId, folding.map(row => row.id))
const updatedAt = { $date: new Date().toISOString() }
const folded = [...new Set(folding.map(row => row.thread))].map(root => {
const summary = stored.get(root)!
return { root, ...foldFacet(summary, { parts: added.parts.filter(p => p._id.thread === root), mentions: added.mentions.filter(m => m._id.thread === root) }) }
})
// Raw, as refreshThreads writes it: the stored parts read back as the $facet returns them, dates as dates.
await tx.$runCommandRaw({ update: 'SuperTabThread', updates: folded.map(({ root, parts, mentions }) =>
({ q: { _id: root, tabId, conversationId }, u: { $set: { parts, mentions, updatedAt } } })) as unknown as Prisma.InputJsonArray })
return { parts: [...whole.parts, ...folded.flatMap(f => f.parts)], mentions: [...whole.mentions, ...folded.flatMap(f => f.mentions)] }
}
/** Folds the rows `rows`, just inserted, into the summaries of the threads they reply in (foldReplies), and answers each
* thread's facet by its root, which the same transaction's Activity reads instead of the thread's rows (activity.ts
* writeReplyActivity). A row that is its own thread's root, or has none (written before threads), changes no summary. */
export async function refreshReplies(tx: Transaction, tabId: string,
rows: ReadonlyArray<{ id: string; conversationId: string; thread?: string | null }>): Promise<Map<string, ThreadFacet>> {
const replies = new Map<string, Array<{ id: string; thread: string }>>()
for (const row of rows) {
if (row.thread && row.thread !== row.id) replies.set(row.conversationId, [...replies.get(row.conversationId) ?? [], { id: row.id, thread: row.thread }])
}
const facets = new Map<string, ThreadFacet>()
for (const [conversationId, list] of replies) {
const facet = await foldReplies(tx, tabId, conversationId, list)
for (const root of new Set(list.map(row => row.thread))) {
facets.set(root, { parts: facet.parts.filter(p => p._id.thread === root), mentions: facet.mentions.filter(m => m._id.thread === root) })
}
}
return facets
}
/** How long this process trusts what it last read of a Swarm's backfill marker. */
const BACKFILL_CHECK_MS = 60_000
const backfilled = new Map<string, { done: boolean; at: number }>()
/** Whether the Swarm's thread summaries are backfilled (threadBackfill.ts), as this process read it in the last minute. */
export async function threadsBackfilled(tx: Transaction, tabId: string): Promise<boolean> {
const known = backfilled.get(tabId), now = Date.now()
if (known && now - known.at < BACKFILL_CHECK_MS) return known.done
const done = !!await tx.superTabThreadBackfill.findUnique({ where: { id: tabId }, select: { id: true } })
backfilled.set(tabId, { done, at: now })
return done
}
/** Notes that this process backfilled the Swarm's summaries, so its own reads use them at once. */
export function noteBackfilled(tabId: string): void {
backfilled.set(tabId, { done: true, at: Date.now() })
}