Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

20.9 KB
Raw Blame
import type { Prisma } from '@prisma/client'
import { agentOnly, conversation, fail, isLeads, type AgentStart, type Conversation, type MessageDraft, type SuperTabState } from '../model.js'
import { autoJoinOn } from '../autoJoinDomain.js'
import { wakeBuddies } from '../buddySleep.js'
import { split } from './shape.js'
import { guardedWrite, PartConflict, plainHost, type HeadColumns, type HostColumns, type PartWrite } from './write.js'
import type { GroupData, MachineData, Part, PartKind, SwarmParts } from './types.js'
import { RELEASED_CAP } from './types.js'

type Transaction = Prisma.TransactionClient

/** A part a command relies on without changing it: touched so a concurrent change to it conflicts (design section 1). */
export interface Touch { kind: PartKind; key: string }
/** A part a command wrote, at its new version: the command reply (design section 4.7). */
export interface Written { kind: PartKind; key: string; version: number }

/** MongoDB limits a document to 16 MiB. A part leaves room for its columns and indexes (design section 1.4). */
export const PART_LIMIT_BYTES = 4 * 1024 * 1024

const LISTS = { member: 'members', agent: 'agents', group: 'groups', machine: 'machines', start: 'starts', progress: 'progress' } as const
type ListKind = keyof typeof LISTS

/** JSON with sorted keys, so two equal records compare equal whatever order a command built them in. */
function canonical(value: unknown): string {
  return JSON.stringify(value, (_key, item: unknown) => item && typeof item === 'object' && !Array.isArray(item)
    ? Object.fromEntries(Object.entries(item as Record<string, unknown>).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : item)
}

/** A value as JSON writes it: a Date as its string, a number JSON cannot hold as null. */
const asJson = (value: unknown): unknown => {
  const plain = value && typeof value === 'object' && typeof (value as { toJSON?: unknown }).toJSON === 'function' ? (value as { toJSON(): unknown }).toJSON() : value
  return typeof plain === 'number' && !Number.isFinite(plain) ? null : plain
}
/** What JSON leaves out of an object, and writes as null in a list. */
const absent = (value: unknown) => value === undefined || typeof value === 'function' || typeof value === 'symbol'

/**
 * Whether two records hold the same data as stored, whatever order their keys were built in: what comparing their
 * `canonical` JSON says, without writing either out (Stability Roadmap 2.6: a write compared every part this way, twice).
 * It stops at the first difference.
 */
export function sameData(a: unknown, b: unknown): boolean {
  const x = asJson(a), y = asJson(b)
  if (x === y) return true
  if (absent(x) || absent(y)) return absent(x) && absent(y)
  if (!x || !y || typeof x !== 'object' || typeof y !== 'object' || Array.isArray(x) !== Array.isArray(y)) return false
  if (Array.isArray(x)) {
    const list = y as unknown[]
    return x.length === list.length && x.every((item, i) => sameData(absent(item) ? null : item, absent(list[i]) ? null : list[i]))
  }
  const left = x as Record<string, unknown>, right = y as Record<string, unknown>
  let count = 0
  for (const key in left) {
    if (!Object.hasOwn(left, key) || absent(left[key])) continue
    count++
    if (!Object.hasOwn(right, key) || absent(right[key]) || !sameData(left[key], right[key])) return false
  }
  for (const key in right) if (Object.hasOwn(right, key) && !absent(right[key])) count--
  return count === 0
}

/**
 * Who sees what changed (design section 4.2): a project's visibility, owner or members, a person removed or rejoined, the set of
 * Leads (the Leads conversation's members), a conversation's audience, or a project removed. Readers reload in full.
 */
export function visibilityChanged(before: SuperTabState, after: SuperTabState): boolean {
  const projects = (state: SuperTabState) => canonical((state.projects ?? []).map(p => [p.id, p.visibility, p.ownerId, p.members]))
  const people = (state: SuperTabState) => canonical(state.people.map(p => [p.id, p.generation, p.role]))
  const leads = (state: SuperTabState) => canonical(state.conversations.filter(isLeads).map(c => c.agentIds))
  const audiences = (state: SuperTabState) => new Map(state.conversations.map(c => [c.id, canonical(c.audience ?? null)]))
  if (projects(before) !== projects(after) || people(before) !== people(after) || leads(before) !== leads(after)) return true
  const was = audiences(before)
  return [...audiences(after)].some(([id, audience]) => was.has(id) && was.get(id) !== audience)
}

/** The Swarm's index row (SuperTab): what hosts, invitations and member lists are looked up by (design section 2.3). */
export function indexesOf(state: SuperTabState) {
  // A deleted Swarm (deletion.ts) lists nobody and is invited to by nobody, so it leaves every person's lists in this write; its
  // machines stay indexed until their hosts have closed its sessions and cleaned up.
  const deletion = state.deletion
  return {
    memberIds: deletion ? [] : state.people.map(p => p.id), invitedEmails: deletion ? [] : state.invitations.map(i => i.email),
    deletedAt: deletion ? new Date(deletion.at) : null, purgeAt: deletion ? new Date(deletion.purgeAt) : null,
    machineIds: [...new Set([...state.agents.map(a => a.machineId), ...(state.machines ?? []).map(m => m.machineId),
      ...(state.closingSessions ?? []).map(c => c.machineId), ...(state.machineCleanups ?? []).map(c => c.machineId)])],
    cleanupMachineIds: [...new Set([...(state.machines ?? []).flatMap(m => m.removal ? [m.machineId] : []),
      ...(state.machineCleanups ?? []).map(c => c.machineId)])],
    // Who joins by domain (autoJoin.ts): a deleted Swarm, or one whose owner turned it off, is found by nobody.
    autoJoinDomain: autoJoinOn(state), autoJoinDeclined: state.autoJoinDeclined ?? [],
  }
}

/** Whether [message], in [group], is a person's message to its project's coordinator: anywhere in the project chat, any of its
 *  threads, or in another chat of the project where it reaches the coordinator (task #3022, pmSchedule.ts). Who wrote and whom
 *  it reaches, never its words. Marked on [group] itself, which this write already writes (review of #3066, 1). */
function toCoordinator(state: SuperTabState, group: Conversation, message: MessageDraft): boolean {
  if (message.author.kind !== 'person' || message.source) return false
  const projects = state.projects ?? [{ id: 'lead', leadConversationId: 'lead' }]
  if (projects.some(p => p.leadConversationId === group.id)) return true
  const project = projects.find(p => p.id === (group.projectId ?? 'lead'))
  const coordinator = project && state.conversations.find(c => c.id === project.leadConversationId)?.leadId
  return !!coordinator && message.recipients.includes(coordinator)
}

/** How long a chat remembers that a person talked with an agent there: well past the 15 minutes an idle agent waits. */
export const TALKED_KEEP_MS = 60 * 60_000

/** Who a person talked with in [group] (task #3816): each agent a person's message reaches (`person`), and that agent's first
 *  post to people after it (`answer`): a post to no agent, once per person's message, so an agent never keeps itself or another
 *  agent from idling (review of #3389). Who wrote and whom it reached, never the words; a system row or notice marks nobody.
 *  Entries older than TALKED_KEEP_MS go first, before this message is read, so an old one never comes back. */
function markTalked(group: Conversation, message: MessageDraft, now: number): void {
  if (message.source || !group.talkedAt && message.author.kind !== 'person') return
  const talked = Object.fromEntries(Object.entries(group.talkedAt ?? {})
    .filter(([, at]) => now - Math.max(at.person, at.answer ?? 0) <= TALKED_KEEP_MS))
  if (message.author.kind === 'person') for (const id of message.recipients) talked[id] = { person: now }
  else if (message.author.kind === 'agent' && !message.recipients.length) {
    const entry = talked[message.author.id]
    if (entry && (entry.answer === undefined || entry.answer < entry.person)) talked[message.author.id] = { ...entry, answer: now }
  }
  if (group.talkedAt || Object.keys(talked).length) group.talkedAt = talked
}

/**
 * Numbers each message in its conversation (design section 1.1 D4): the conversation's counter moves by one per message, in
 * draft order. A shown message is the conversation's latest and wakes a dormant one; an agent-only notice is numbered but
 * shown to nobody. Returns each message's sequence.
 */
export function numberMessages(state: SuperTabState, messages: MessageDraft[], now: number): Map<string, number> {
  const sequences = new Map<string, number>()
  for (const message of messages) {
    const group = conversation(state, message.conversationId)
    const sequence = (group.messageSeq ?? state.sequenceBase ?? 0) + 1
    group.messageSeq = sequence
    sequences.set(message.id, sequence)
    wakeBuddies(state, message, now)
    if (agentOnly(message.source, message.recipients)) continue
    group.lastMessageSequence = sequence
    if (!message.thread || message.alsoInChat) {
      group.lastMainSequence = sequence
      if (message.author.kind === 'person' && !message.source) group.lastPersonSequence = sequence
    }
    if (toCoordinator(state, group, message)) group.heardAt = now
    markTalked(group, message, now)
    group.lastActivityAt = new Date(now).toISOString()
    delete group.dormant
  }
  return sequences
}

/** Keeps the newest RELEASED_CAP released harnesses of each machine (spec, Archive). */
export function capReleased(state: SuperTabState): void {
  const released = state.releasedHarnesses
  if (!released) return
  const count = new Map<string, number>()
  for (const entry of released) count.set(entry.machineId, (count.get(entry.machineId) ?? 0) + 1)
  if ([...count.values()].every(n => n <= RELEASED_CAP)) return
  const seen = new Map<string, number>()
  state.releasedHarnesses = released.filter(entry => {
    const index = (seen.get(entry.machineId) ?? 0) + 1
    seen.set(entry.machineId, index)
    return index > count.get(entry.machineId)! - RELEASED_CAP
  })
}

/**
 * Writes what a command changed (design section 3.1): each part whose data differs from what was loaded, each new part,
 * a tombstone for each removed one, a guarded touch for each part in `touches` it did not write, the head when its
 * counters or the visibility epoch move, and the index row when a person, invitation or machine reference was added or
 * removed. Every write is guarded by the version it was read at.
 */
/** A machine part without its machine's own reading, nor the Lent hours each reading credits (leaderboard/lent.ts): what
 *  that machine's host needs to hear about. */
const withoutReading = (data: unknown) => {
  const machine = data as MachineData | undefined
  return machine && { ...machine, contribution: machine.contribution && { ...machine.contribution, observation: undefined, lent: undefined, lentDays: undefined } }
}

/** What a reading moves all the time, measured on production (2026-11-04: of 128 parts moved in 40 s, `resources` in all of them): a
 *  client that asks gets a line for these, and reads them on demand (read/changes.ts, read/service.ts machineReadings). */
const VOLATILE_READING = ['receivedAt', 'resources', 'quotas', 'quotaReasons', 'profiles', 'privacy', 'harnessProgress', 'harnessTokens', 'harnessUncommitted'] as const
const steadyPart = (data: unknown) => {
  const machine = data as MachineData | undefined
  const observation = machine?.contribution?.observation
  if (!machine?.contribution) return machine
  const steady: Record<string, unknown> | undefined = observation && { ...observation }
  for (const field of VOLATILE_READING) if (steady) delete steady[field]
  return { ...machine, contribution: { ...machine.contribution, observation: steady, lent: undefined, lentDays: undefined } }
}
/** Whether a machine's write changes anything beyond those fields: running sessions, tools, readiness and all the rest. */
export const machineContentChanged = (before: unknown, after: unknown): boolean => !sameData(steadyPart(before), steadyPart(after))

/**
 * What a part write tells host reads (read/host.ts, review round 1), from the part's data before and after (undefined for
 * an insert or a removal): a machine's settings or ledger changed, not only its own reading; and which machines' hosts a
 * group's write concerned, by its agents and its work before and after, so an agent leaving its last group wakes its host.
 */
export function hostMarker(kind: PartKind, before: unknown, after: unknown, machineOf: ReadonlyMap<string, string>, stamp: number): { host?: HostColumns } {
  if (kind === 'machine') return sameData(withoutReading(before), withoutReading(after)) ? {} : { host: { hostChangedAt: new Date(stamp) } }
  if (kind !== 'group') return {}
  const machines = ([before, after] as Array<GroupData | undefined>).flatMap(g => g
    ? [...g.agentIds.flatMap(id => machineOf.get(id) ?? []), ...(g.assignment ? [g.assignment.execution.machineId] : [])] : [])
  return { host: { hostMachineIds: [...new Set(machines)] } }
}

/**
 * The machines whose hosts a commit's writes concern, by the same rules the host read wakes a Swarm for a machine
 * (read/host.ts woken): the head and members for every host of the Swarm before or after; an agent or a start for its
 * machine; a group for the machines its marker names; a machine's settings for that machine, the Mac it follows and the
 * team accounts that follow it. A machine's reading alone tells nobody. Each host is pushed these (lib/swarmHostPushes.ts).
 */
export function hostsToTell(writes: readonly PartWrite[], before: SuperTabState, after: SuperTabState): string[] {
  const hosts = new Set<string>()
  const pools = [...before.machines ?? [], ...after.machines ?? []]
  for (const write of writes) {
    if (write.op === 'touch') continue
    switch (write.kind) {
      case 'head': case 'member': case 'agent': case 'start': {
        const host = plainHost(write.kind, write.data)
        if (host === 'all') for (const state of [before, after]) for (const id of indexesOf(state).machineIds) hosts.add(id)
        else if (host) hosts.add(host)
        break
      }
      case 'group':
        for (const id of write.host?.hostMachineIds ?? []) hosts.add(id)
        break
      case 'machine': {
        if (!write.host?.hostChangedAt) break
        hosts.add(write.key)
        const follows = (write.data as MachineData | null)?.contribution?.hostMachineId
        if (follows) hosts.add(follows)
        for (const pool of pools) if (pool.hostMachineId === write.key) hosts.add(pool.machineId)
        break
      }
    }
  }
  return [...hosts]
}

/**
 * The people whose apps a commit's writes concern, and which of the Swarm's stamps it moved (parts/write.ts guardedWrite):
 * `machine` for a machine's own reading alone (every 15 s per machine), `main` for anything else. Everyone in the Swarm before
 * or after is told, so a person removed reads once more and finds it gone. Only the Swarm's id reaches them; what they may
 * read is decided by the read (read/views.ts readerView). Each person's apps are pushed these (lib/swarmClientPushes.ts).
 */
export function membersToTell(writes: readonly PartWrite[], before: SuperTabState, after: SuperTabState): { userIds: string[]; parts: Array<'main' | 'machine'> } {
  const written = writes.filter(write => write.op !== 'touch')
  if (!written.length) return { userIds: [], parts: [] }
  const reading = (write: PartWrite) => write.kind === 'machine' && write.op === 'update' && !write.host
  const parts = [...written.some(write => !reading(write)) ? ['main' as const] : [], ...written.some(reading) ? ['machine' as const] : []]
  return { userIds: [...new Set([...before.people, ...after.people].map(person => person.id))], parts }
}

/** The agents a group's write let go: in its members or its work before, in neither after. Each is written again, data
 *  unchanged, so its host hears of it whatever later writes to the group leave in that group's marker (review round 2). */
export function departedAgents(before: unknown, after: unknown): string[] {
  const of = (group: GroupData | undefined) => new Set(group ? [...group.agentIds, ...(group.assignment ? [group.assignment.agentId] : [])] : [])
  const kept = of(after as GroupData | undefined)
  return [...of(before as GroupData | undefined)].filter(id => !kept.has(id))
}

export async function commitParts(tx: Transaction, loaded: { parts: SwarmParts; state: SuperTabState; row: { id: string; revision: number } },
  next: SuperTabState, opts: { stamp: number; touches?: Touch[] }): Promise<{ written: Written[]; versions: Map<string, number>; hosts: string[]; members: ReturnType<typeof membersToTell> }> {
  const { parts } = loaded, { stamp } = opts
  capReleased(next)
  const after = split(next, { tabId: parts.tabId, version: 0, now: stamp, messageBase: parts.sequenceBase })
  const writes: PartWrite[] = []
  const first = parts.sequenceBase + 1
  // An agent removed by this write is still in the loaded state, so its machine is known for the host marker.
  const machineOf = new Map([...loaded.state.agents, ...next.agents].map(a => [a.id, a.machineId]))
  const host = (kind: PartKind, before: unknown, after: unknown) => hostMarker(kind, before, after, machineOf, stamp)
  let referencesChanged = false
  for (const kind of Object.keys(LISTS) as ListKind[]) {
    const before = new Map(parts[LISTS[kind]].filter(p => !p.deleted).map(p => [p.key, p as Part]))
    const now = new Map((after[LISTS[kind]] as Part[]).map(p => [p.key, p]))
    let added = 0
    for (const [key, part] of now) {
      const prior = before.get(key)
      if (!prior) {
        writes.push({ op: 'insert', kind, key, data: part.data, ordinal: stamp + added++ / 1000, first, ...host(kind, undefined, part.data) })
        if (kind !== 'group' && kind !== 'start' && kind !== 'progress') referencesChanged = true
      } else if (!sameData(prior.data, part.data)) {
        writes.push({ op: 'update', kind, key, seen: prior.version, data: part.data, ...host(kind, prior.data, part.data),
          ...kind === 'machine' && machineContentChanged(prior.data, part.data) ? { content: true as const } : {} })
      }
    }
    for (const [key, prior] of before) {
      if (now.has(key)) continue
      writes.push({ op: 'remove', kind, key, seen: prior.version, data: prior.data, ...host(kind, prior.data, undefined) })
      if (kind !== 'group' && kind !== 'start' && kind !== 'progress') referencesChanged = true
    }
  }
  const agentsWritten = new Set(writes.filter(w => w.kind === 'agent').map(w => w.key))
  for (const write of writes.filter((w): w is Extract<PartWrite, { op: 'update' }> => w.kind === 'group' && w.op === 'update')) {
    for (const id of departedAgents(parts.groups.find(g => g.key === write.key)!.data, write.data)) {
      const agent = parts.agents.find(a => a.key === id && !a.deleted), data = after.agents.find(a => a.key === id)?.data
      if (!agent || !data || agentsWritten.has(id)) continue
      agentsWritten.add(id)
      writes.push({ op: 'update', kind: 'agent', key: id, seen: agent.version, data })
    }
  }
  const epoch = parts.visibilityEpoch + (visibilityChanged(loaded.state, next) ? 1 : 0)
  const head: HeadColumns = { groupCounter: Math.max(parts.groupCounter, next.groupCounter ?? 0), visibilityEpoch: epoch, sequenceBase: parts.sequenceBase }
  if (!sameData(parts.head.data, after.head.data) || head.groupCounter !== parts.groupCounter || epoch !== parts.visibilityEpoch) {
    writes.push({ op: 'update', kind: 'head', key: 'head', seen: parts.head.version, data: after.head.data, head })
  }
  if (canonical(indexesOf(loaded.state)) !== canonical(indexesOf(next))) referencesChanged = true
  for (const write of writes) {
    if ('data' in write && Buffer.byteLength(JSON.stringify(write.data)) > PART_LIMIT_BYTES) {
      fail('PART_TOO_LARGE', 'This Swarm record has reached its size limit.', 413)
    }
  }
  const versions = new Map<string, number>()
  const written: Written[] = []
  for (const write of writes) {
    const version = await guardedWrite(tx, parts.tabId, write, stamp)
    versions.set(`${write.kind}:${write.key}`, version)
    written.push({ kind: write.kind, key: write.key, version })
  }
  const all = [parts.head as Part, ...parts.members, ...parts.agents, ...parts.groups, ...parts.machines, ...parts.starts, ...parts.progress]
  for (const touch of opts.touches ?? []) {
    const key = `${touch.kind}:${touch.key}`
    if (versions.has(key)) continue
    const part = all.find(p => p.kind === touch.kind && p.key === touch.key && !p.deleted)
    if (!part) continue
    versions.set(key, await guardedWrite(tx, parts.tabId, { op: 'touch', kind: touch.kind, key: touch.key, seen: part.version }, stamp))
  }
  if (referencesChanged) {
    // Always written when a reference is added or removed, so two such commands conflict instead of one losing an id.
    const updated = await tx.superTab.updateMany({ where: { id: loaded.row.id, revision: loaded.row.revision },
      data: { revision: loaded.row.revision + 1, ...indexesOf(next) } })
    if (!updated.count) throw new PartConflict('head', 'index')
  }
  return { written, versions, hosts: hostsToTell(writes, loaded.state, next), members: membersToTell(writes, loaded.state, next) }
}