The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
/** How far each person has read each conversation, and each thread in one: the newest message sequence they saw there, so
* unread state follows them across devices. Marks live in SuperTabReadMark and SuperTabThreadMark, never in the tab state:
* writing one wakes no other reader, and an older backend reads and writes a tab that has them (it never reads the thread
* marks' collection). A person's marks go only to their own reads (read/service.ts); agents, Buddies included, have none. */
import { Prisma, type PrismaClient } from '@prisma/client'
import { ensureIndex } from '../lib/db/indexes.js'
import { removeChunk, startChunkSweep } from '../lib/chunkSweep.js'
import { logger } from '../utils/logger.js'
import { fail, stableId } from './model.js'
import { readBy } from './parts/indexed.js'
import { ms } from './parts/light.js'
/** A mark untouched this long is removed by a TTL index; the conversation then reads as the device last saw it. */
export const READ_MARK_EXPIRY_MS = 180 * 24 * 60 * 60_000
/** The most thread marks one person keeps in one Swarm: a new one past it removes the least recently moved. A thread whose
* mark went reads as far as its conversation's mark, as on a device that never kept one. */
export const THREAD_MARK_CAP = 500
/** Run by the singleton worker at startup, like the mention expiry; answers whether the expiry is set. Where the database user
* may not set it (production's, no collMod), startReadMarkSweep removes the old marks instead. Thread marks need no index
* option: the hourly sweep deletes them (sweepThreadMarks), as the app user may not set one on an index `db push` built (#2238). */
export async function ensureReadMarkExpiry(db: Pick<PrismaClient, '$runCommandRaw'>): Promise<boolean> {
return await ensureIndex(db, 'SuperTabReadMark', { key: { updatedAt: 1 }, name: 'read_mark_expiry',
expireAfterSeconds: READ_MARK_EXPIRY_MS / 1000 }) === 'ready'
}
/** Where the expiry is refused: one chunk of 500 marks every 30 s, as the usage-receipt sweep (lib/productUsageReceiptSweep.ts). */
export const READ_MARK_SWEEP = { chunk: 500, everyMs: 30_000 } as const
/** Removes up to [chunk] marks untouched for READ_MARK_EXPIRY_MS, oldest first, by the expiry index, and answers how many went.
* A mark moved between the read of the chunk and its delete stays (lib/chunkSweep.ts). */
export async function sweepReadMarks(db: Pick<PrismaClient, '$runCommandRaw'>, now: Date, chunk: number = READ_MARK_SWEEP.chunk): Promise<number> {
return removeChunk(db, { collection: 'SuperTabReadMark', sort: { updatedAt: 1 }, hint: 'read_mark_expiry', chunk,
filter: { updatedAt: { $lt: { $date: new Date(now.getTime() - READ_MARK_EXPIRY_MS).toISOString() } } } })
}
/** In the singleton worker, where the database refused the expiry (ensureReadMarkExpiry false: production's app user may not
* run collMod, Ken's prod scan 11-04), removes old marks itself: one chunk now and every READ_MARK_SWEEP.everyMs, for as long
* as the worker runs, since marks keep ageing. Where the expiry is set, it runs nothing. Answers its stop. */
export function startReadMarkSweep(db: Pick<PrismaClient, '$runCommandRaw'>, expirySet: boolean,
settings: Partial<{ chunk: number; everyMs: number; now: () => Date; log: Pick<typeof logger, 'info' | 'warn'> }> = {}): () => void {
if (expirySet) return () => {}
const { chunk, everyMs, now, log } = { ...READ_MARK_SWEEP, now: () => new Date(), log: logger, ...settings }
log.warn(`[read-marks] the database refused the read-mark expiry: removing marks untouched for ${READ_MARK_EXPIRY_MS / 86_400_000} days `
+ `here instead, ${chunk} every ${everyMs / 1000} s`)
return startChunkSweep({ label: '[read-marks]', everyMs, log, pass: async () => ({ removed: await sweepReadMarks(db, now(), chunk), done: false }) })
}
export interface ReadMarkWrite { tabId: string; personId: string; conversationId: string; sequence: number; now: number }
export interface ThreadMarkWrite extends ReadMarkWrite { threadId: string }
/** The calls `raise` makes, which both mark collections answer alike. */
interface MarkRows {
updateMany(args: { where: { id: string; sequence: { lt: number } }; data: { sequence: number; updatedAt: Date } & Record<string, unknown> }): Promise<{ count: number }>
findUnique(args: { where: { id: string }; select: { sequence: true } }): Promise<{ sequence: number } | null>
create(args: { data: Record<string, unknown> & { id: string } }): Promise<unknown>
}
/** Moves row `id` forward to `sequence` and answers the mark as stored, with whether this call created the row. An older
* mark never replaces a newer one: the update matches only a lower mark. When another device's write lands between the
* update and the look after it (two first writes racing to create the row, or one raising it), the update runs again;
* every retry follows a write that raised the mark, so a few settle it. `moved` is set on the row with the mark. */
async function raise(rows: MarkRows, id: string, sequence: number, updatedAt: Date, data: Record<string, unknown>, moved: Record<string, unknown> = {}):
Promise<{ sequence: number; created: boolean }> {
for (let attempt = 0; attempt < 4; attempt++) {
const raised = await rows.updateMany({ where: { id, sequence: { lt: sequence } }, data: { ...moved, sequence, updatedAt } })
if (raised.count) return { sequence, created: false }
const held = await rows.findUnique({ where: { id }, select: { sequence: true } })
if (held && held.sequence >= sequence) return { sequence: held.sequence, created: false }
if (held) continue
try {
await rows.create({ data: { id, ...data, sequence, updatedAt } })
return { sequence, created: true }
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
}
}
return fail('READ_MARK_BUSY', 'This conversation is being marked read on another device. Try again.', 503)
}
/** Moves the person's mark of a conversation forward to `sequence` and answers the mark as stored (`raise`). A read that moves
* it ends a Mark unread: the mark no longer reads as lowered (lowerReadMark). */
export async function writeReadMark(db: PrismaClient, mark: ReadMarkWrite): Promise<number> {
const { tabId, personId, conversationId, sequence } = mark
return (await raise(db.superTabReadMark as unknown as MarkRows, stableId(tabId, conversationId, personId), sequence, new Date(mark.now),
{ tabId, personId, conversationId }, { loweredAt: null })).sequence
}
/** The person's Mark unread: moves their mark of a conversation back to `sequence`, never forward (a chat already unread from
* earlier stays as it is), and answers the mark as stored. The one write that lowers a mark. It sets `loweredAt` with
* `updatedAt`, so their other devices, which keep the higher of a mark and the one they hold, take this one instead
* (readMarkRows `lowered`). A chat with no mark counts as read (read/inbox.ts), so it gets one. One conditional update; when
* another device creates or moves the row between it and the look after it, it runs again, as `raise` does. */
export async function lowerReadMark(db: PrismaClient, mark: ReadMarkWrite): Promise<number> {
const { tabId, personId, conversationId, sequence } = mark
const id = stableId(tabId, conversationId, personId), at = new Date(mark.now)
for (let attempt = 0; attempt < 4; attempt++) {
if ((await db.superTabReadMark.updateMany({ where: { id, sequence: { gt: sequence } }, data: { sequence, updatedAt: at, loweredAt: at } })).count) return sequence
const held = await db.superTabReadMark.findUnique({ where: { id }, select: { sequence: true } })
if (held && held.sequence <= sequence) return held.sequence
if (held) continue
try {
await db.superTabReadMark.create({ data: { id, tabId, personId, conversationId, sequence, updatedAt: at, loweredAt: at } })
return sequence
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
}
}
return fail('READ_MARK_BUSY', 'This conversation is being marked read on another device. Try again.', 503)
}
/** Moves the person's mark of one thread forward to `sequence` and answers the mark as stored (`raise`). A new mark past
* THREAD_MARK_CAP removes the person's least recently moved ones in this Swarm. Two first marks racing may each remove
* one; a removed mark only makes its thread read as far as its conversation's mark. */
export async function writeThreadMark(db: PrismaClient, mark: ThreadMarkWrite): Promise<number> {
const { tabId, personId, conversationId, threadId, sequence } = mark
const stored = await raise(db.superTabThreadMark as unknown as MarkRows, stableId(tabId, conversationId, threadId, personId), sequence,
new Date(mark.now), { tabId, personId, conversationId, threadId })
if (stored.created) await capThreadMarks(db, tabId, personId)
return stored.sequence
}
/** Gives one thread the person has no mark of a mark at `sequence` (read/threadMarkSeeds.ts: the chat's mark before a read
* moved it past a reply only in this thread), and answers whether it wrote one. A row a Follow alone wrote (sequence 0) takes
* it and keeps its Follow; a row with any mark keeps its own, lower or higher, so a thread opened on another device is never
* moved by a chat read. Each write matches only a row at 0 or creates one; when another device creates the row between the
* two, the match runs again on it. A new row counts toward THREAD_MARK_CAP as any mark. */
export async function seedThreadMark(db: PrismaClient, mark: ThreadMarkWrite): Promise<boolean> {
const { tabId, personId, conversationId, threadId, sequence } = mark
const id = stableId(tabId, conversationId, threadId, personId), updatedAt = new Date(mark.now)
for (let attempt = 0; attempt < 4; attempt++) {
if ((await db.superTabThreadMark.updateMany({ where: { id, sequence: 0 }, data: { sequence, updatedAt } })).count) return true
if (await db.superTabThreadMark.findUnique({ where: { id }, select: { sequence: true } })) return false
try {
await db.superTabThreadMark.create({ data: { id, tabId, personId, conversationId, threadId, sequence, updatedAt } })
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
continue
}
await capThreadMarks(db, tabId, personId)
return true
}
return fail('READ_MARK_BUSY', 'This conversation is being marked read on another device. Try again.', 503)
}
export interface ThreadFollowWrite { tabId: string; personId: string; conversationId: string; threadId: string; follow: boolean | null; now: number }
/** Stores the person's Follow (true), Unfollow (false) or neither (null) of one thread on its mark row, and answers it. The
* row keeps its sequence: one written here first starts at 0, which reads as no mark (every client keeps the higher of a
* mark and the one it holds). A Follow moves the row as a read does, so it counts toward THREAD_MARK_CAP and the 180-day
* expiry from now, and goes with the person when they leave the Swarm (sweepThreadMarks). Two first writes racing to create
* the row: the loser's create fails on the id and its update runs again on the winner's row. */
export async function writeThreadFollow(db: PrismaClient, write: ThreadFollowWrite): Promise<boolean | null> {
const { tabId, personId, conversationId, threadId, follow } = write
const id = stableId(tabId, conversationId, threadId, personId), updatedAt = new Date(write.now)
for (let attempt = 0; attempt < 4; attempt++) {
if ((await db.superTabThreadMark.updateMany({ where: { id }, data: { follow, updatedAt } })).count) return follow
try {
await db.superTabThreadMark.create({ data: { id, tabId, personId, conversationId, threadId, sequence: 0, follow, updatedAt } })
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
continue
}
await capThreadMarks(db, tabId, personId)
return follow
}
return fail('READ_MARK_BUSY', 'This thread is being changed on another device. Try again.', 503)
}
/** One of the person's thread mark rows: its mark, its Follow choice (null when none) and when it last moved. */
export interface ThreadMarkRow { conversationId: string; threadId: string; sequence: number; follow: boolean | null; updatedAt: number }
/** Every one of the person's thread mark rows in this tab (at most THREAD_MARK_CAP), most recently moved first, by one raw read
* of the (tabId, personId, updatedAt) index, as readThreadMarks. The caller keeps those of conversations the reader may read. */
export async function readThreadMarkRows(db: Prisma.TransactionClient, tabId: string, personId: string): Promise<ThreadMarkRow[]> {
const rows = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId } }, { $sort: { updatedAt: -1 } },
{ $project: { _id: 0, conversationId: 1, threadId: 1, sequence: 1, follow: 1, updatedAt: 1 } }])
return rows.map(row => ({ conversationId: row.conversationId as string, threadId: row.threadId as string, sequence: Number(row.sequence),
follow: typeof row.follow === 'boolean' ? row.follow : null, updatedAt: ms(row.updatedAt) }))
}
/** Removes the person's least recently moved thread marks in the Swarm past THREAD_MARK_CAP, by the (tabId, personId,
* updatedAt) index: it counts the person's marks and reads only the ids of those past the cap. */
async function capThreadMarks(db: PrismaClient, tabId: string, personId: string): Promise<void> {
const [counted] = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId } }, { $count: 'n' }])
const over = Number(counted?.n ?? 0) - THREAD_MARK_CAP
if (over <= 0) return
const oldest = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId } }, { $sort: { updatedAt: 1 } }, { $limit: over },
{ $project: { _id: 1 } }])
// A plain $in on _id: Prisma's `in` plans slowly (parts/indexed.ts).
await db.$runCommandRaw({ delete: 'SuperTabThreadMark', deletes: [{ q: { _id: { $in: oldest.map(row => row._id as string) } }, limit: 0 }] })
}
/** One of the person's read marks; `lowered` when its last move was their Mark unread (lowerReadMark). */
export interface ReadMarkRow { conversationId: string; sequence: number; lowered: boolean }
/** The person's marks in this tab; with `since`, only those moved at or after it. A Mark unread is its row's last move while
* `loweredAt` equals `updatedAt`: a read since clears it, and one from a backend that did not clear it moves `updatedAt`. */
export async function readMarkRows(db: PrismaClient, tabId: string, personId: string, since?: number): Promise<ReadMarkRow[]> {
const rows = await db.superTabReadMark.findMany({ where: { tabId, personId, ...(since === undefined ? {} : { updatedAt: { gte: new Date(since) } }) },
select: { conversationId: true, sequence: true, updatedAt: true, loweredAt: true } })
return rows.map(row => ({ conversationId: row.conversationId, sequence: row.sequence, lowered: row.loweredAt?.getTime() === row.updatedAt.getTime() }))
}
/** The person's marks in this tab, by conversation; with `since`, only those moved at or after it. */
export async function readMarks(db: PrismaClient, tabId: string, personId: string, since?: number): Promise<Map<string, number>> {
return new Map((await readMarkRows(db, tabId, personId, since)).map(row => [row.conversationId, row.sequence]))
}
/** The person's thread marks in this tab, by conversation then thread root: every one (at most THREAD_MARK_CAP), or with
* `since` those moved at or after it. One raw match by the (tabId, personId, updatedAt) index: Prisma's filters plan slowly
* at this size (parts/indexed.ts). The caller keeps those of conversations the reader may read. */
export async function readThreadMarks(db: Prisma.TransactionClient, tabId: string, personId: string, since?: number):
Promise<Map<string, Map<string, number>>> {
const marks = new Map<string, Map<string, number>>()
const rows = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId,
...(since === undefined ? {} : { updatedAt: { $gte: { $date: new Date(since).toISOString() } } }) } },
{ $project: { _id: 0, conversationId: 1, threadId: 1, sequence: 1 } }])
for (const row of rows) {
const conversationId = row.conversationId as string, threadId = row.threadId as string, sequence = Number(row.sequence)
let held = marks.get(conversationId)
if (!held) marks.set(conversationId, held = new Map())
held.set(threadId, sequence)
}
return marks
}
/** How many marks one sweep deletes at most, in batches, each kind: the rest wait for the next hour. */
export const THREAD_MARK_SWEEP = 20_000
const SWEEP_BATCH = 1_000
/** Deletes rows by id with a plain $in on _id (Prisma's `in` plans slowly). */
async function deleteIds(db: PrismaClient, ids: string[]): Promise<number> {
if (!ids.length) return 0
const reply = await db.$runCommandRaw({ delete: 'SuperTabThreadMark', deletes: [{ q: { _id: { $in: ids } }, limit: 0 }] }) as { n?: number }
return Number(reply.n ?? 0)
}
/** How many Swarms one sweep checks for marks of people no longer there; the next pass goes on after the last one. */
export const THREAD_MARK_SWARMS = 200
/** Where the people check reads on from: after the last Swarm whose check finished. */
export interface ThreadMarkWalk { after: string }
/** The worker's walk, kept in the database so a release goes on where the last process stopped. */
const CURSOR = 'thread-marks'
const storedWalk = async (db: PrismaClient): Promise<ThreadMarkWalk> =>
({ after: (await db.superTabSweepCursor.findUnique({ where: { id: CURSOR }, select: { after: true } }))?.after ?? '' })
const storeWalk = (db: PrismaClient, after: string, now: number) =>
db.superTabSweepCursor.upsert({ where: { id: CURSOR }, create: { id: CURSOR, after, updatedAt: new Date(now) }, update: { after, updatedAt: new Date(now) } })
/** The first mark's value of `field` past `after` within `match`, by one seek on the person index; null past the last. */
async function nextOf(db: PrismaClient, match: Record<string, string>, field: 'tabId' | 'personId', after: string): Promise<string | null> {
const [row] = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { ...match, [field]: { $gt: after } } }, { $sort: { tabId: 1, personId: 1 } },
{ $limit: 1 }, { $project: { _id: 0, [field]: 1 } }])
return (row?.[field] as string | undefined) ?? null
}
/**
* The hourly sweep of thread marks (run by the singleton worker with the Swarm sweeps, sweeps.ts): deletes marks untouched for
* READ_MARK_EXPIRY_MS, by the age index in batches of SWEEP_BATCH, at most THREAD_MARK_SWEEP a pass. Then it checks the next
* THREAD_MARK_SWARMS Swarms with marks, in id order from where the last pass stopped (the stored walk, or `walk` when given),
* one Swarm and one person at a time by index seeks; the place is stored after each Swarm's check finishes, so a release goes
* on from there and a Swarm whose check failed is checked again: a Swarm whose row is gone loses all its marks, a person no longer in its `memberIds` theirs. A Swarm
* row without `memberIds` is skipped, never read as having no members. A delete only makes a thread read as far as its
* conversation's mark, so a mark swept a moment before its person moves it again costs nothing.
*/
export async function sweepThreadMarks(db: PrismaClient, now: number, options: { walk?: ThreadMarkWalk; swarms?: number } = {}):
Promise<{ expired: number; lost: number; swarmsChecked: number }> {
const swarms = options.swarms ?? THREAD_MARK_SWARMS
const at = options.walk ?? await storedWalk(db)
const moveTo = async (after: string) => {
at.after = after
if (!options.walk) await storeWalk(db, after, now)
}
let expired = 0
const before = { $date: new Date(now - READ_MARK_EXPIRY_MS).toISOString() }
while (expired < THREAD_MARK_SWEEP) {
const rows = await readBy(db, 'SuperTabThreadMark', 'age', [{ $match: { updatedAt: { $lt: before } } }, { $limit: SWEEP_BATCH }, { $project: { _id: 1 } }])
expired += await deleteIds(db, rows.map(row => row._id as string))
if (rows.length < SWEEP_BATCH) break
}
let lost = 0, swarmsChecked = 0
const remove = async (q: Record<string, string>) => {
const reply = await db.$runCommandRaw({ delete: 'SuperTabThreadMark', deletes: [{ q, limit: 0 }] }) as { n?: number }
lost += Number(reply.n ?? 0)
}
while (swarmsChecked < swarms) {
const tabId = await nextOf(db, {}, 'tabId', at.after)
// Past the last Swarm: the next pass starts again from the first.
if (tabId === null) { await moveTo(''); break }
swarmsChecked++
const [row] = await readBy(db, 'SuperTab', 'id', [{ $match: { _id: tabId } }, { $project: { memberIds: 1, deletedAt: 1 } }])
if (!row) await remove({ tabId })
// A row without memberIds is skipped, never read as having no members. So is a deleted Swarm (deletion.ts): it lists
// nobody while it can be restored, and its marks come back with it; its purge removes them.
else if (Array.isArray(row.memberIds) && !row.deletedAt) {
const members = new Set(row.memberIds as string[])
for (let person = await nextOf(db, { tabId }, 'personId', ''); person !== null; person = await nextOf(db, { tabId }, 'personId', person)) {
if (!members.has(person)) await remove({ tabId, personId: person })
}
}
// Only now, its check done: a failure above leaves the place before it.
await moveTo(tabId)
}
return { expired, lost, swarmsChecked }
}