Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

21.3 KB
Raw Blame
/** How far each person has read each conversation, and each thread in one: the newest message sequence they saw there, so
 *  unread state follows them across devices. Marks live in SuperTabReadMark and SuperTabThreadMark, never in the tab state:
 *  writing one wakes no other reader, and an older backend reads and writes a tab that has them (it never reads the thread
 *  marks' collection). A person's marks go only to their own reads (read/service.ts); agents, Buddies included, have none. */
import { Prisma, type PrismaClient } from '@prisma/client'
import { ensureIndex } from '../lib/db/indexes.js'
import { removeChunk, startChunkSweep } from '../lib/chunkSweep.js'
import { logger } from '../utils/logger.js'
import { fail, stableId } from './model.js'
import { readBy } from './parts/indexed.js'
import { ms } from './parts/light.js'

/** A mark untouched this long is removed by a TTL index; the conversation then reads as the device last saw it. */
export const READ_MARK_EXPIRY_MS = 180 * 24 * 60 * 60_000

/** The most thread marks one person keeps in one Swarm: a new one past it removes the least recently moved. A thread whose
 *  mark went reads as far as its conversation's mark, as on a device that never kept one. */
export const THREAD_MARK_CAP = 500

/** Run by the singleton worker at startup, like the mention expiry; answers whether the expiry is set. Where the database user
 *  may not set it (production's, no collMod), startReadMarkSweep removes the old marks instead. Thread marks need no index
 *  option: the hourly sweep deletes them (sweepThreadMarks), as the app user may not set one on an index `db push` built (#2238). */
export async function ensureReadMarkExpiry(db: Pick<PrismaClient, '$runCommandRaw'>): Promise<boolean> {
  return await ensureIndex(db, 'SuperTabReadMark', { key: { updatedAt: 1 }, name: 'read_mark_expiry',
    expireAfterSeconds: READ_MARK_EXPIRY_MS / 1000 }) === 'ready'
}

/** Where the expiry is refused: one chunk of 500 marks every 30 s, as the usage-receipt sweep (lib/productUsageReceiptSweep.ts). */
export const READ_MARK_SWEEP = { chunk: 500, everyMs: 30_000 } as const

/** Removes up to [chunk] marks untouched for READ_MARK_EXPIRY_MS, oldest first, by the expiry index, and answers how many went.
 *  A mark moved between the read of the chunk and its delete stays (lib/chunkSweep.ts). */
export async function sweepReadMarks(db: Pick<PrismaClient, '$runCommandRaw'>, now: Date, chunk: number = READ_MARK_SWEEP.chunk): Promise<number> {
  return removeChunk(db, { collection: 'SuperTabReadMark', sort: { updatedAt: 1 }, hint: 'read_mark_expiry', chunk,
    filter: { updatedAt: { $lt: { $date: new Date(now.getTime() - READ_MARK_EXPIRY_MS).toISOString() } } } })
}

/** In the singleton worker, where the database refused the expiry (ensureReadMarkExpiry false: production's app user may not
 *  run collMod, Ken's prod scan 11-04), removes old marks itself: one chunk now and every READ_MARK_SWEEP.everyMs, for as long
 *  as the worker runs, since marks keep ageing. Where the expiry is set, it runs nothing. Answers its stop. */
export function startReadMarkSweep(db: Pick<PrismaClient, '$runCommandRaw'>, expirySet: boolean,
  settings: Partial<{ chunk: number; everyMs: number; now: () => Date; log: Pick<typeof logger, 'info' | 'warn'> }> = {}): () => void {
  if (expirySet) return () => {}
  const { chunk, everyMs, now, log } = { ...READ_MARK_SWEEP, now: () => new Date(), log: logger, ...settings }
  log.warn(`[read-marks] the database refused the read-mark expiry: removing marks untouched for ${READ_MARK_EXPIRY_MS / 86_400_000} days `
    + `here instead, ${chunk} every ${everyMs / 1000} s`)
  return startChunkSweep({ label: '[read-marks]', everyMs, log, pass: async () => ({ removed: await sweepReadMarks(db, now(), chunk), done: false }) })
}

export interface ReadMarkWrite { tabId: string; personId: string; conversationId: string; sequence: number; now: number }
export interface ThreadMarkWrite extends ReadMarkWrite { threadId: string }

/** The calls `raise` makes, which both mark collections answer alike. */
interface MarkRows {
  updateMany(args: { where: { id: string; sequence: { lt: number } }; data: { sequence: number; updatedAt: Date } & Record<string, unknown> }): Promise<{ count: number }>
  findUnique(args: { where: { id: string }; select: { sequence: true } }): Promise<{ sequence: number } | null>
  create(args: { data: Record<string, unknown> & { id: string } }): Promise<unknown>
}

/** Moves row `id` forward to `sequence` and answers the mark as stored, with whether this call created the row. An older
 *  mark never replaces a newer one: the update matches only a lower mark. When another device's write lands between the
 *  update and the look after it (two first writes racing to create the row, or one raising it), the update runs again;
 *  every retry follows a write that raised the mark, so a few settle it. `moved` is set on the row with the mark. */
async function raise(rows: MarkRows, id: string, sequence: number, updatedAt: Date, data: Record<string, unknown>, moved: Record<string, unknown> = {}):
  Promise<{ sequence: number; created: boolean }> {
  for (let attempt = 0; attempt < 4; attempt++) {
    const raised = await rows.updateMany({ where: { id, sequence: { lt: sequence } }, data: { ...moved, sequence, updatedAt } })
    if (raised.count) return { sequence, created: false }
    const held = await rows.findUnique({ where: { id }, select: { sequence: true } })
    if (held && held.sequence >= sequence) return { sequence: held.sequence, created: false }
    if (held) continue
    try {
      await rows.create({ data: { id, ...data, sequence, updatedAt } })
      return { sequence, created: true }
    } catch (error) {
      if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
    }
  }
  return fail('READ_MARK_BUSY', 'This conversation is being marked read on another device. Try again.', 503)
}

/** Moves the person's mark of a conversation forward to `sequence` and answers the mark as stored (`raise`). A read that moves
 *  it ends a Mark unread: the mark no longer reads as lowered (lowerReadMark). */
export async function writeReadMark(db: PrismaClient, mark: ReadMarkWrite): Promise<number> {
  const { tabId, personId, conversationId, sequence } = mark
  return (await raise(db.superTabReadMark as unknown as MarkRows, stableId(tabId, conversationId, personId), sequence, new Date(mark.now),
    { tabId, personId, conversationId }, { loweredAt: null })).sequence
}

/** The person's Mark unread: moves their mark of a conversation back to `sequence`, never forward (a chat already unread from
 *  earlier stays as it is), and answers the mark as stored. The one write that lowers a mark. It sets `loweredAt` with
 *  `updatedAt`, so their other devices, which keep the higher of a mark and the one they hold, take this one instead
 *  (readMarkRows `lowered`). A chat with no mark counts as read (read/inbox.ts), so it gets one. One conditional update; when
 *  another device creates or moves the row between it and the look after it, it runs again, as `raise` does. */
export async function lowerReadMark(db: PrismaClient, mark: ReadMarkWrite): Promise<number> {
  const { tabId, personId, conversationId, sequence } = mark
  const id = stableId(tabId, conversationId, personId), at = new Date(mark.now)
  for (let attempt = 0; attempt < 4; attempt++) {
    if ((await db.superTabReadMark.updateMany({ where: { id, sequence: { gt: sequence } }, data: { sequence, updatedAt: at, loweredAt: at } })).count) return sequence
    const held = await db.superTabReadMark.findUnique({ where: { id }, select: { sequence: true } })
    if (held && held.sequence <= sequence) return held.sequence
    if (held) continue
    try {
      await db.superTabReadMark.create({ data: { id, tabId, personId, conversationId, sequence, updatedAt: at, loweredAt: at } })
      return sequence
    } catch (error) {
      if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
    }
  }
  return fail('READ_MARK_BUSY', 'This conversation is being marked read on another device. Try again.', 503)
}

/** Moves the person's mark of one thread forward to `sequence` and answers the mark as stored (`raise`). A new mark past
 *  THREAD_MARK_CAP removes the person's least recently moved ones in this Swarm. Two first marks racing may each remove
 *  one; a removed mark only makes its thread read as far as its conversation's mark. */
export async function writeThreadMark(db: PrismaClient, mark: ThreadMarkWrite): Promise<number> {
  const { tabId, personId, conversationId, threadId, sequence } = mark
  const stored = await raise(db.superTabThreadMark as unknown as MarkRows, stableId(tabId, conversationId, threadId, personId), sequence,
    new Date(mark.now), { tabId, personId, conversationId, threadId })
  if (stored.created) await capThreadMarks(db, tabId, personId)
  return stored.sequence
}

/** Gives one thread the person has no mark of a mark at `sequence` (read/threadMarkSeeds.ts: the chat's mark before a read
 *  moved it past a reply only in this thread), and answers whether it wrote one. A row a Follow alone wrote (sequence 0) takes
 *  it and keeps its Follow; a row with any mark keeps its own, lower or higher, so a thread opened on another device is never
 *  moved by a chat read. Each write matches only a row at 0 or creates one; when another device creates the row between the
 *  two, the match runs again on it. A new row counts toward THREAD_MARK_CAP as any mark. */
export async function seedThreadMark(db: PrismaClient, mark: ThreadMarkWrite): Promise<boolean> {
  const { tabId, personId, conversationId, threadId, sequence } = mark
  const id = stableId(tabId, conversationId, threadId, personId), updatedAt = new Date(mark.now)
  for (let attempt = 0; attempt < 4; attempt++) {
    if ((await db.superTabThreadMark.updateMany({ where: { id, sequence: 0 }, data: { sequence, updatedAt } })).count) return true
    if (await db.superTabThreadMark.findUnique({ where: { id }, select: { sequence: true } })) return false
    try {
      await db.superTabThreadMark.create({ data: { id, tabId, personId, conversationId, threadId, sequence, updatedAt } })
    } catch (error) {
      if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
      continue
    }
    await capThreadMarks(db, tabId, personId)
    return true
  }
  return fail('READ_MARK_BUSY', 'This conversation is being marked read on another device. Try again.', 503)
}

export interface ThreadFollowWrite { tabId: string; personId: string; conversationId: string; threadId: string; follow: boolean | null; now: number }

/** Stores the person's Follow (true), Unfollow (false) or neither (null) of one thread on its mark row, and answers it. The
 *  row keeps its sequence: one written here first starts at 0, which reads as no mark (every client keeps the higher of a
 *  mark and the one it holds). A Follow moves the row as a read does, so it counts toward THREAD_MARK_CAP and the 180-day
 *  expiry from now, and goes with the person when they leave the Swarm (sweepThreadMarks). Two first writes racing to create
 *  the row: the loser's create fails on the id and its update runs again on the winner's row. */
export async function writeThreadFollow(db: PrismaClient, write: ThreadFollowWrite): Promise<boolean | null> {
  const { tabId, personId, conversationId, threadId, follow } = write
  const id = stableId(tabId, conversationId, threadId, personId), updatedAt = new Date(write.now)
  for (let attempt = 0; attempt < 4; attempt++) {
    if ((await db.superTabThreadMark.updateMany({ where: { id }, data: { follow, updatedAt } })).count) return follow
    try {
      await db.superTabThreadMark.create({ data: { id, tabId, personId, conversationId, threadId, sequence: 0, follow, updatedAt } })
    } catch (error) {
      if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error
      continue
    }
    await capThreadMarks(db, tabId, personId)
    return follow
  }
  return fail('READ_MARK_BUSY', 'This thread is being changed on another device. Try again.', 503)
}

/** One of the person's thread mark rows: its mark, its Follow choice (null when none) and when it last moved. */
export interface ThreadMarkRow { conversationId: string; threadId: string; sequence: number; follow: boolean | null; updatedAt: number }

/** Every one of the person's thread mark rows in this tab (at most THREAD_MARK_CAP), most recently moved first, by one raw read
 *  of the (tabId, personId, updatedAt) index, as readThreadMarks. The caller keeps those of conversations the reader may read. */
export async function readThreadMarkRows(db: Prisma.TransactionClient, tabId: string, personId: string): Promise<ThreadMarkRow[]> {
  const rows = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId } }, { $sort: { updatedAt: -1 } },
    { $project: { _id: 0, conversationId: 1, threadId: 1, sequence: 1, follow: 1, updatedAt: 1 } }])
  return rows.map(row => ({ conversationId: row.conversationId as string, threadId: row.threadId as string, sequence: Number(row.sequence),
    follow: typeof row.follow === 'boolean' ? row.follow : null, updatedAt: ms(row.updatedAt) }))
}

/** Removes the person's least recently moved thread marks in the Swarm past THREAD_MARK_CAP, by the (tabId, personId,
 *  updatedAt) index: it counts the person's marks and reads only the ids of those past the cap. */
async function capThreadMarks(db: PrismaClient, tabId: string, personId: string): Promise<void> {
  const [counted] = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId } }, { $count: 'n' }])
  const over = Number(counted?.n ?? 0) - THREAD_MARK_CAP
  if (over <= 0) return
  const oldest = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId } }, { $sort: { updatedAt: 1 } }, { $limit: over },
    { $project: { _id: 1 } }])
  // A plain $in on _id: Prisma's `in` plans slowly (parts/indexed.ts).
  await db.$runCommandRaw({ delete: 'SuperTabThreadMark', deletes: [{ q: { _id: { $in: oldest.map(row => row._id as string) } }, limit: 0 }] })
}

/** One of the person's read marks; `lowered` when its last move was their Mark unread (lowerReadMark). */
export interface ReadMarkRow { conversationId: string; sequence: number; lowered: boolean }

/** The person's marks in this tab; with `since`, only those moved at or after it. A Mark unread is its row's last move while
 *  `loweredAt` equals `updatedAt`: a read since clears it, and one from a backend that did not clear it moves `updatedAt`. */
export async function readMarkRows(db: PrismaClient, tabId: string, personId: string, since?: number): Promise<ReadMarkRow[]> {
  const rows = await db.superTabReadMark.findMany({ where: { tabId, personId, ...(since === undefined ? {} : { updatedAt: { gte: new Date(since) } }) },
    select: { conversationId: true, sequence: true, updatedAt: true, loweredAt: true } })
  return rows.map(row => ({ conversationId: row.conversationId, sequence: row.sequence, lowered: row.loweredAt?.getTime() === row.updatedAt.getTime() }))
}

/** The person's marks in this tab, by conversation; with `since`, only those moved at or after it. */
export async function readMarks(db: PrismaClient, tabId: string, personId: string, since?: number): Promise<Map<string, number>> {
  return new Map((await readMarkRows(db, tabId, personId, since)).map(row => [row.conversationId, row.sequence]))
}

/** The person's thread marks in this tab, by conversation then thread root: every one (at most THREAD_MARK_CAP), or with
 *  `since` those moved at or after it. One raw match by the (tabId, personId, updatedAt) index: Prisma's filters plan slowly
 *  at this size (parts/indexed.ts). The caller keeps those of conversations the reader may read. */
export async function readThreadMarks(db: Prisma.TransactionClient, tabId: string, personId: string, since?: number):
  Promise<Map<string, Map<string, number>>> {
  const marks = new Map<string, Map<string, number>>()
  const rows = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { tabId, personId,
    ...(since === undefined ? {} : { updatedAt: { $gte: { $date: new Date(since).toISOString() } } }) } },
  { $project: { _id: 0, conversationId: 1, threadId: 1, sequence: 1 } }])
  for (const row of rows) {
    const conversationId = row.conversationId as string, threadId = row.threadId as string, sequence = Number(row.sequence)
    let held = marks.get(conversationId)
    if (!held) marks.set(conversationId, held = new Map())
    held.set(threadId, sequence)
  }
  return marks
}

/** How many marks one sweep deletes at most, in batches, each kind: the rest wait for the next hour. */
export const THREAD_MARK_SWEEP = 20_000
const SWEEP_BATCH = 1_000

/** Deletes rows by id with a plain $in on _id (Prisma's `in` plans slowly). */
async function deleteIds(db: PrismaClient, ids: string[]): Promise<number> {
  if (!ids.length) return 0
  const reply = await db.$runCommandRaw({ delete: 'SuperTabThreadMark', deletes: [{ q: { _id: { $in: ids } }, limit: 0 }] }) as { n?: number }
  return Number(reply.n ?? 0)
}

/** How many Swarms one sweep checks for marks of people no longer there; the next pass goes on after the last one. */
export const THREAD_MARK_SWARMS = 200

/** Where the people check reads on from: after the last Swarm whose check finished. */
export interface ThreadMarkWalk { after: string }
/** The worker's walk, kept in the database so a release goes on where the last process stopped. */
const CURSOR = 'thread-marks'
const storedWalk = async (db: PrismaClient): Promise<ThreadMarkWalk> =>
  ({ after: (await db.superTabSweepCursor.findUnique({ where: { id: CURSOR }, select: { after: true } }))?.after ?? '' })
const storeWalk = (db: PrismaClient, after: string, now: number) =>
  db.superTabSweepCursor.upsert({ where: { id: CURSOR }, create: { id: CURSOR, after, updatedAt: new Date(now) }, update: { after, updatedAt: new Date(now) } })

/** The first mark's value of `field` past `after` within `match`, by one seek on the person index; null past the last. */
async function nextOf(db: PrismaClient, match: Record<string, string>, field: 'tabId' | 'personId', after: string): Promise<string | null> {
  const [row] = await readBy(db, 'SuperTabThreadMark', 'person', [{ $match: { ...match, [field]: { $gt: after } } }, { $sort: { tabId: 1, personId: 1 } },
    { $limit: 1 }, { $project: { _id: 0, [field]: 1 } }])
  return (row?.[field] as string | undefined) ?? null
}

/**
 * The hourly sweep of thread marks (run by the singleton worker with the Swarm sweeps, sweeps.ts): deletes marks untouched for
 * READ_MARK_EXPIRY_MS, by the age index in batches of SWEEP_BATCH, at most THREAD_MARK_SWEEP a pass. Then it checks the next
 * THREAD_MARK_SWARMS Swarms with marks, in id order from where the last pass stopped (the stored walk, or `walk` when given),
 * one Swarm and one person at a time by index seeks; the place is stored after each Swarm's check finishes, so a release goes
 * on from there and a Swarm whose check failed is checked again: a Swarm whose row is gone loses all its marks, a person no longer in its `memberIds` theirs. A Swarm
 * row without `memberIds` is skipped, never read as having no members. A delete only makes a thread read as far as its
 * conversation's mark, so a mark swept a moment before its person moves it again costs nothing.
 */
export async function sweepThreadMarks(db: PrismaClient, now: number, options: { walk?: ThreadMarkWalk; swarms?: number } = {}):
  Promise<{ expired: number; lost: number; swarmsChecked: number }> {
  const swarms = options.swarms ?? THREAD_MARK_SWARMS
  const at = options.walk ?? await storedWalk(db)
  const moveTo = async (after: string) => {
    at.after = after
    if (!options.walk) await storeWalk(db, after, now)
  }
  let expired = 0
  const before = { $date: new Date(now - READ_MARK_EXPIRY_MS).toISOString() }
  while (expired < THREAD_MARK_SWEEP) {
    const rows = await readBy(db, 'SuperTabThreadMark', 'age', [{ $match: { updatedAt: { $lt: before } } }, { $limit: SWEEP_BATCH }, { $project: { _id: 1 } }])
    expired += await deleteIds(db, rows.map(row => row._id as string))
    if (rows.length < SWEEP_BATCH) break
  }
  let lost = 0, swarmsChecked = 0
  const remove = async (q: Record<string, string>) => {
    const reply = await db.$runCommandRaw({ delete: 'SuperTabThreadMark', deletes: [{ q, limit: 0 }] }) as { n?: number }
    lost += Number(reply.n ?? 0)
  }
  while (swarmsChecked < swarms) {
    const tabId = await nextOf(db, {}, 'tabId', at.after)
    // Past the last Swarm: the next pass starts again from the first.
    if (tabId === null) { await moveTo(''); break }
    swarmsChecked++
    const [row] = await readBy(db, 'SuperTab', 'id', [{ $match: { _id: tabId } }, { $project: { memberIds: 1, deletedAt: 1 } }])
    if (!row) await remove({ tabId })
    // A row without memberIds is skipped, never read as having no members. So is a deleted Swarm (deletion.ts): it lists
    // nobody while it can be restored, and its marks come back with it; its purge removes them.
    else if (Array.isArray(row.memberIds) && !row.deletedAt) {
      const members = new Set(row.memberIds as string[])
      for (let person = await nextOf(db, { tabId }, 'personId', ''); person !== null; person = await nextOf(db, { tabId }, 'personId', person)) {
        if (!members.has(person)) await remove({ tabId, personId: person })
      }
    }
    // Only now, its check done: a failure above leaves the place before it.
    await moveTo(tabId)
  }
  return { expired, lost, swarmsChecked }
}