Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

14.3 KB
Raw Blame
import 'dotenv/config'
import { closeBus, redisSwarmStamps } from './lib/bus.js'
import { configureSwarmStamps } from './superTabs/cache.js'
import { reconcileCommunityLikeCounts } from './lib/db/communityLikeCounts.js'
import { backfillMachineFreeEntitlements } from './lib/db/freeEntitlement.js'
import {
  backfillMachineAuthMode,
  backfillMachinePresenceTurnsStarted,
  migrateAutonomousEnvironment,
  migrateMachineRename,
  migrateUserIdentityByEmail,
  migrateUserPresencePerMachine,
  pruneLegacyDeviceBindings,
} from './lib/db/migrate.js'
import { ensurePlans } from './lib/db/seed.js'
import { prisma, prismaSecondary } from './lib/prisma.js'
import { startQueryStats } from './lib/queryStats.js'
import { startRetryStats } from './superTabs/retryStats.js'
import { releaseSchema } from './lib/db/schemaRelease.js'
import { startSwarmSweeps } from './superTabs/sweeps.js'
import { startDecisionSweep } from './superTabs/decisionSweep.js'
import { requeueCompactedAgents } from './superTabs/agentTombstones.js'
import { startLegacyDrop } from './superTabs/migration/dropLegacyJob.js'
import { startInstructionsRebase } from './superTabs/migration/rebase.js'
import { startInstructionsRestore } from './superTabs/migration/restoreRebase.js'
import { startDeliverySettle } from './superTabs/deliverySettle.js'
import { startDeliveryRepush } from './superTabs/deliveryRepush.js'
import { startDeliveryCompaction } from './superTabs/deliveryCompaction.js'
import { startThreadBackfill } from './superTabs/threadBackfill.js'
import { markFeedIds } from './superTabs/read/feedIds.js'
import { appPullRequests, markFinishedGroups, startSwarmArchive, unknownPullRequests } from './superTabs/archive.js'
import { appPullRequestWatch, startPullRequestWatch } from './superTabs/pullRequestEvents.js'
import { startTimeChecks } from './superTabs/timeChecks.js'
import { appReleaseReader, startReleaseWatch, unknownReleases } from './superTabs/releaseWatch.js'
import { recordTrainRules, startMergeTrain } from './superTabs/mergeTrain/run.js'
import { teamAccountRequired } from './superTabs/machines.js'
import { PrismaApprovals } from './github/approvals.js'
import { SuperTabStore } from './superTabs/store.js'
import { announceSwarmDeliveries, announceSwarmTerminal } from './lib/swarmDeliveryPushes.js'
import { announceSwarmHostChanges } from './lib/swarmHostPushes.js'
import { announceSwarmChanges } from './lib/swarmClientPushes.js'
import { GitHubApp, githubAppConfig } from './github/app.js'
import { env } from './config/env.js'
import { phoneSendersFromEnv, sendPhoneNotices } from './push/send.js'
import { machineBillingWorkerService } from './services/MachineBillingWorkerService.js'
import { productUsagePeopleRollupService } from './services/ProductUsagePeopleRollupService.js'
import { startReceiptSweep } from './lib/productUsageReceiptSweep.js'
import { startReadMarkSweep } from './superTabs/readMarks.js'
import { startPendingUploadSweep } from './lib/pendingUploadSweep.js'
import { startSwarmPurge } from './superTabs/purge.js'
import { startAttachmentSweepFrom } from './lib/attachmentSweep.js'
import { attachmentsFromEnv } from './superTabs/attachments.js'
import { startRoutingReflection } from './lib/routingReflection.js'
import { startScorecards } from './superTabs/scorecard/job.js'
import { logger } from './utils/logger.js'

// Singleton background process. HTTP/WS server.ts runs as a PM2 cluster and must never own schema
// writes, catalog syncs, migrations or periodic jobs; otherwise every live server worker races them.
async function start(): Promise<void> {
  // Every minute, this process's queries by model.action and filter shape (lib/queryStats.ts, DB roadmap Phase 1); its
  // jobs and startup migrations count as background.
  const stopQueryStats = startQueryStats()
  // Every minute, how often Swarm transactions ran again and commands loaded twice (superTabs/retryStats.ts, DB 2.1).
  const stopRetryStats = startRetryStats()
  // Existing production data was created against staging upstreams. Stamp it before Prisma swaps
  // the old global unique indexes for their per-environment replacements.
  await migrateAutonomousEnvironment()
  await migrateUserIdentityByEmail()
  // Same reason: the old (user, day) unique must be gone before the push installs its replacement.
  await migrateUserPresencePerMachine()
  const schema = await releaseSchema()
  // After the push, which adds the finished mark: parts written before it are marked once, so the archive pass finds them.
  await markFinishedGroups(prisma)
  // Likewise the Tasks feed's column (read/feedIds.ts): a person's older tasks show once it is set.
  await markFeedIds(prisma).then(count => logger.info('[swarm] Tasks feed ids set on older conversations', { count }), error =>
    logger.error('[swarm] Tasks feed ids not set on older conversations: their tasks stay out of people\'s Tasks feeds until the next worker start sets them',
      { error: error instanceof Error ? error.message : String(error) }))
  // AFTER the push: it creates the new collections + their unique indexes, and the copy uses $merge
  // so those indexes survive. Copying first would leave the unique index build to fail silently.
  await migrateMachineRename()
  // The copy can materialize rows from a collection the first pass never saw — stamp those too.
  await migrateAutonomousEnvironment()
  await ensurePlans()
  await backfillMachineAuthMode()
  await pruneLegacyDeviceBindings()
  await backfillMachineFreeEntitlements()
  await backfillMachinePresenceTurnsStarted()
  await reconcileCommunityLikeCounts()

  let stopSweeps = (): void => {}
  let stopDecisionSweep = (): void => {}
  let stopLegacyDrop = (): void => {}
  let stopInstructionsRebase = (): void => {}
  let stopInstructionsRestore = (): void => {}
  let stopArchive = (): void => {}
  let stopPullRequestWatch = (): void => {}
  let stopTimeChecks = (): void => {}
  let stopReleaseWatch = (): void => {}
  let stopMergeTrain = (): void => {}
  let stopDeliveries = (): void => {}
  let stopRepush = (): void => {}
  let stopCompaction = (): void => {}
  let stopThreadBackfill = (): void => {}
  let stopReceiptSweep = (): void => {}
  let stopReadMarkSweep = (): void => {}
  let stopPendingUploads = (): void => {}
  let stopSwarmPurge = (): void => {}
  let stopAttachmentSweep = (): void => {}
  let stopReflection = (): void => {}
  let stopScorecards = (): void => {}
  const shutdown = (signal: string): void => {
    logger.info(`Received ${signal}, stopping backend worker...`)
    const hardExit = setTimeout(() => process.exit(0), 5000)
    hardExit.unref()
    machineBillingWorkerService.stop()
    productUsagePeopleRollupService.stop()
    stopSweeps()
    stopDecisionSweep()
    stopLegacyDrop()
    stopInstructionsRebase()
    stopInstructionsRestore()
    stopArchive()
    stopPullRequestWatch()
    stopTimeChecks()
    stopReleaseWatch()
    stopMergeTrain()
    stopDeliveries()
    stopRepush()
    stopCompaction()
    stopReceiptSweep()
    stopReadMarkSweep()
    stopPendingUploads()
    stopSwarmPurge()
    stopAttachmentSweep()
    stopReflection()
    stopScorecards()
    stopThreadBackfill()
    // The last partial minute, before the database closes.
    stopQueryStats()
    stopRetryStats()
    void Promise.allSettled([prisma.$disconnect(), closeBus()]).finally(() => process.exit(0))
  }
  process.on('SIGINT', () => shutdown('SIGINT'))
  process.on('SIGTERM', () => shutdown('SIGTERM'))

  machineBillingWorkerService.start()
  productUsagePeopleRollupService.start()
  // Usage receipts from when they lived 36 days, a few at a time until none are left (lib/productUsageReceiptSweep.ts).
  stopReceiptSweep = startReceiptSweep(prisma)
  // Read marks untouched for 180 days, where the database refused their index expiry, a chunk every 30 s (superTabs/readMarks.ts).
  stopReadMarkSweep = startReadMarkSweep(prisma, schema.readMarkExpirySet)
  // Uploads begun on a link and never finished go an hour after the link expired, with what they stored (lib/pendingUploadSweep.ts).
  stopPendingUploads = startPendingUploadSweep(prisma, env)
  // Deleted Swarms past their 7 days go with their records and stored files (superTabs/purge.ts).
  stopSwarmPurge = startSwarmPurge(prisma, env)
  // Swarm uploads nobody sent within a day, row and stored file, once an hour (lib/attachmentSweep.ts). It logs whether it
  // is on; settings it cannot read turn off only the sweep.
  stopAttachmentSweep = startAttachmentSweepFrom(prisma, () => attachmentsFromEnv(env))
  // The worker's sweeps and archive write Swarms: they bump the stamps the API processes read by. It reads too little to keep a cache.
  configureSwarmStamps(redisSwarmStamps(), { reads: false })
  // Archive notes and Workspace removals write messages too: their recipients' hosts hear of them at once, and of the
  // groups they changed.
  // Their tags reach the tagged person's phones too (push/send.ts, task #2905).
  const phones = phoneSendersFromEnv(env)
  const swarms = new SuperTabStore(prisma, Date.now, null, { deliveriesWaiting: announceSwarmDeliveries, terminalWaiting: announceSwarmTerminal, hostsChanged: announceSwarmHostChanges,
    swarmChanged: announceSwarmChanges, phoneNotices: notices => { void sendPhoneNotices(prisma, phones, notices) } })
  // Decisions still waiting in work that was over before #2454 closed them where work ends: they counted toward the open
  // limit and came back on a reopen (store.ts withdrawEndedDecisions). Remove in the release after the one that runs it.
  logger.info('[swarm-decisions] closed waiting decisions of finished work', { closed: await swarms.withdrawEndedDecisions() })
  // Agent tombstones compacted for good before #2150 get a purge time, so the sweeps purge them (superTabs/agentTombstones.ts).
  // Remove in the release after the one that runs it.
  // A failure leaves them compacted, as before, and never stops the worker starting.
  await requeueCompactedAgents(prisma, Date.now()).then(queued => logger.info('[swarm-sweeps] queued compacted agent tombstones for purge', { queued }),
    error => logger.warn('[swarm-sweeps] queuing compacted agent tombstones failed', { error: error instanceof Error ? error.message : String(error) }))
  stopSweeps = startSwarmSweeps(prisma, undefined, () => swarms.workspaces.finishRemovals())
  stopDecisionSweep = startDecisionSweep(now => swarms.leaveOverdueDecisions(now))
  // The dead old state of migrated Swarms goes by itself, a few at a time (superTabs/migration/dropLegacyJob.ts).
  stopLegacyDrop = startLegacyDrop(prisma)
  // Every project's instructions move onto the new base once, by themselves (superTabs/migration/rebase.ts, task #3600).
  stopInstructionsRebase = startInstructionsRebase(prisma)
  // And the lines that first move lost come back, once, after it (superTabs/migration/restoreRebase.ts, task #3679).
  stopInstructionsRestore = startInstructionsRestore(prisma)
  // Finished groups archive once GitHub says their pull requests are merged or closed; without the app, only groups that
  // link none do (superTabs/archive.ts).
  const githubConfig = githubAppConfig(env), githubApp = githubConfig ? new GitHubApp(githubConfig) : null
  stopArchive = startSwarmArchive(prisma, swarms, () => githubApp ? appPullRequests(githubApp) : unknownPullRequests)
  // GitHub events on an open task's pull request reach its Engineer within a minute (superTabs/pullRequestEvents.ts): only
  // with the app, which reads them.
  if (githubApp) stopPullRequestWatch = startPullRequestWatch(prisma, swarms, () => appPullRequestWatch(githubApp))
  // Every 30 s, each active Swarm's time checks: lost and stalled work, failed starts and idle agents, told to its PM whether or
  // not the PM is up (superTabs/timeChecks.ts, task #3385). The PM's poll ran them before, so a hung PM stopped them.
  stopTimeChecks = startTimeChecks(prisma, swarms)
  // Every 5 minutes, merged tasks a verified release carries are marked released, and inside the release window a due release
  // step opens for the project's Integrator (superTabs/releaseWatch.ts, task #3405). Without the app nothing can be verified:
  // each pass says so for every project still waiting.
  stopReleaseWatch = startReleaseWatch(prisma, swarms, () => githubApp ? appReleaseReader(githubApp) : unknownReleases)
  // Every minute, each project whose merge train is switched on (its rules record; off until saved on) merges what passed at its head, in
  // batches through the app (superTabs/mergeTrain/, task #3402). Only where an agent's token could be had for the repository.
  if (githubApp) {
    const approvals = new PrismaApprovals(prisma)
    stopMergeTrain = startMergeTrain({ store: swarms, app: githubApp, rules: recordTrainRules(prisma),
      approved: async (project, repository) => !teamAccountRequired() || approvals.approved(project.autonomousEnv, project.tabId, project.projectId, repository) })
  }
  // A delivery that can no longer move ends as Not sent, instead of waiting for good (superTabs/deliverySettle.ts).
  stopDeliveries = startDeliverySettle(prisma, swarms)
  // Every 15 s, the machine of each agent with a delivery still queued after 10 s is told again, in case its push was lost:
  // hosts read an idle worker's list only when told (superTabs/deliveryRepush.ts, task #2800).
  stopRepush = startDeliveryRepush(prisma, announceSwarmDeliveries)
  // Every hour, finished deliveries 14 days old move into their messages' receipts, kept forever (superTabs/deliveryCompaction.ts).
  stopCompaction = startDeliveryCompaction(prisma, env.SWARM_DELIVERY_COMPACTION)
  // Every 15 minutes, thread summaries for replies no write summed up: all of a Swarm's at first, then only what changed
  // (superTabs/threadBackfill.ts). Pages read the summaries once a Swarm is backfilled.
  stopThreadBackfill = startThreadBackfill(prisma)
  // At 23:00 in each project's time zone, what its routed work cost and how it went, with routing changes asked of its owner
  // (lib/routingReflection.ts).
  stopReflection = startRoutingReflection(prisma, swarms)
  // Each project's scorecard for its local day, once the day has ended, from the Swarm's own records (superTabs/scorecard/job.ts).
  stopScorecards = startScorecards(prisma, prismaSecondary)
  // PM2 wait_ready gates the singleton worker's "online" state on schema/catalog/backfill startup.
  // Plain node/tsx runs do not provide process.send and simply continue.
  process.send?.('ready')
  logger.info('backend worker started')
}

start().catch((err) => {
  logger.error('Failed to start backend worker', err)
  process.exit(1)
})