Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { bindMaintenanceIo } from './maintenanceControl.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
import { noteReadinessChanged } from './readinessSignal.js';
/**
* Server boot orchestration.
*
* `server/index.js` owns HTTP/Socket.IO construction and route registration;
* everything about *starting the install up* lives here:
*
* 1. `bootstrapServices()` — pre-route boot: data migrations, collection
* schema verification, AI Toolkit construction + runner registration, and
* the background workers that must exist before any route handler runs.
* 2. `runBootSequence()` — post-route boot: fire-and-forget service inits +
* schedulers, then the ordered instance/sync/media-queue/DB chain that
* ends in `httpServer.listen()`.
* 3. `registerShutdownHandlers()` — the graceful-shutdown state machine.
*
* Ordering inside each phase is load-bearing. It lives in `bootstrapSequence.js`
* — a dependency-injected, service-free module — so it can be executed and
* asserted under test (#3451) without opening a listener or touching a real
* database. This file supplies the real implementations; that file decides when
* each one runs and which are awaited. Read both before reordering anything.
*
* NOTE (AGENTS.md "No cold-bootstrap LLM calls"): nothing in this file may
* queue an AI provider call. Boot only loads on-disk state and ARMS schedulers;
* every scheduler here is off by default or user-configured.
*/
import { isTestRunner, isSmokeBoot } from '../lib/runtimeEnv.js';
import { readPortosEnvValue } from '../lib/portosEnv.js';
import { join } from 'path';
import { resolveInstallRoot } from '../lib/dataRoot.js';
import { PORTS } from '../lib/ports.js';
import { getSelfHost } from '../lib/peerSelfHost.js';
import { getBuildIdentity, getCachedBuildIdentity, formatBuildIdentity } from '../lib/buildIdentity.js';
import { setupProcessErrorHandlers, asyncHandler, ServerError, errorEvents } from '../lib/errorHandler.js';
import { ERROR_CATEGORIES } from '../lib/aiToolkit/errorDetection.js';
import { createAIToolkit } from '../lib/aiToolkit/index.js';
import { attachLlmActivityHooks } from './systemActivityNotify.js';
import { verifyCollectionVersions } from '../lib/collectionStore.js';
import { startIdleReaper, stopIdleReaper } from '../lib/managedDaemon.js';
import { adoptNpmGlobalBinDir } from '../lib/npmGlobalBin.js';
import { conflictJournalStore } from '../lib/conflictJournal.js';
import { isHostShuttingDown, markHostShuttingDown, writeHostShutdownMarker } from '../lib/hostShutdown.js';
import { signalDetachedGroups } from '../lib/credentialBootstrap.js';
import { createHttpDrain } from '../lib/httpDrain.js';
import { createEventLoopLagProbe } from '../lib/eventLoopLagProbe.js';
import { quiesce as quiesceScheduledEvents, drain as drainScheduledEvents } from './eventScheduler.js';
import { setUserCatalogTypes } from '../lib/catalogTypes.js';
import { runMigrations } from '../../scripts/run-migrations.js';
import {
runPreRouteSequence,
runPostRouteSequence,
runPostListenSequence,
runDatabasePhase,
gateOnDatabase,
runDbAndCatalogMigrations,
warmMandatoryStores,
initCosAfterSpawner,
armCommissionScheduler,
gateStepsForSmokeBoot
} from './bootstrapSequence.js';
import { ensureBackendProvider, getBackend as getLocalLlmBackend } from './localLlm.js';
import { ensureRunning as ensureOllamaRunning } from './ollamaManager.js';
import { ensureProviderReadyForExecution } from './providerExecutionReadiness.js';
import { loadUsage, recordSession } from './usage.js';
import { recordCompletedRunUsage } from './usageReconciler.js';
import { setAIToolkit as setProvidersToolkit } from './providers.js';
import { setAIToolkit as setProviderStatusToolkit } from './providerStatus.js';
import { setAIToolkit as setRunnerToolkit, executeCliRun as executeCliRunFixed } from './runner.js';
import { reconcileDeadOwnerReservations } from './deadOwnerReservations.js';
import { setAIToolkit as setPromptsToolkit } from './promptService.js';
import { executeTuiRun as executeTuiRunFixed } from './tuiPromptRunner.js';
import { initAutoFixer } from './autoFixer.js';
import { initTaskLearning } from './taskLearning.js';
import { initSpawner } from './subAgentSpawner.js';
import { initCertRenewer } from './certRenewer.js';
import * as cos from './cos.js';
import { markCosAgentMigrationsSettled } from './cosState.js';
// Side-effect-free shared state module — importing it here does NOT pull in the
// agent-lifecycle orchestrator graph (that's why activeAgents lives in its own
// leaf module), so the shutdown handler can read the live agent set cheaply.
import { activeAgents, dispatchingAgentIds } from './agentState.js';
import * as automationScheduler from './automationScheduler.js';
import * as agentActionExecutor from './agentActionExecutor.js';
import * as telegram from './telegram.js';
import * as telegramBridge from './telegramBridge.js';
import { getSettings as getInitSettings } from './settings.js';
import { readUserTypes as readUserTypeSlice } from './catalogUserTypes/store.js';
import { getVoiceConfig } from './voice/config.js';
import { reconcile as reconcileVoice } from './voice/bootstrap.js';
import { initVoiceTimers } from './voice/timers.js';
import { startBackupScheduler } from './backupScheduler.js';
import { startAutoUpdateScheduler } from './autoUpdateScheduler.js';
import { startPrivacyRecheckScheduler } from './privacyRecheckScheduler.js';
import { startQuotaBurnScheduler } from './quotaBurnRunner.js';
import { startMaintenanceRunScheduler } from './maintenanceRun.js';
import { startNotificationResolveListener } from './githubNotificationResolveTask.js';
import { startSeriesAutopilotScheduler } from './seriesAutopilotScheduler.js';
import { startCommissionScheduler } from './creativeCommissions/scheduler.js';
import { startBeeperScheduler } from './beeperScheduler.js';
import { startEidoverseControllerSupervisor } from './eidoverseControllerRuntime.js';
import { reconcileOutboxOnBoot } from './beeperOutbox.js';
import { startBeeperSocket, stopBeeperSocket } from './beeperSocket.js';
import { startImessageScheduler } from './imessageScheduler.js';
import { startSignalScheduler } from './signalScheduler.js';
import { startSpotifyScheduler } from './spotifyScheduler.js';
import { startYoutubeScheduler } from './youtubeScheduler.js';
import { reconcileStackerNewsSchedulers } from './stackerNewsScheduler.js';
import { onProvidersSaved as onProvidersSavedForGraph } from './providerGraph.js';
import { reportRetiredModelPins } from './modelPinAuditNotifier.js';
import { startBrainScheduler } from './brainScheduler.js';
import { startActivityDigestScheduler } from './activityDigestScheduler.js';
import { startTwinEnrichmentScheduler } from './twinEnrichmentScheduler.js';
import { startUpdateScheduler, clearStaleUpdateInProgress, processUpdateMarker } from './updateChecker.js';
import { captureBootCommit } from './installState.js';
import { restoreLoops } from './loops.js';
import { startOrphanShellGc } from './importerOrphanGc.js';
import { startImageRefsGc } from './imageRefsGc.js';
import { startImageCleanTmpGc } from './imageCleanTmpGc.js';
import { startOrphanedPartialGc } from './orphanedPartialGc.js';
import { startBeeperAttachmentGc } from './beeperAttachmentGc.js';
import { startTribePurge } from './tribePurge.js';
import { startScreenshotsGc } from './screenshotsGc.js';
import { initBridge as initBrainMemoryBridge, replayPersistedResync as replayBrainMemoryResync } from './brainMemoryBridge.js';
import { flushAutoSync as flushIdeaLoomAutoSync } from './idealoomAutoSync.js';
import { initDrillCache } from './meatspacePostDrillCache.js';
import { registerPostReminderSchedule } from './meatspacePostReminder.js';
import { registerAutobiographyReminderSchedule } from './autobiographyReminder.js';
import { recoverInterruptedRepoClones, recoverStuckClassifications } from './brain.js';
import { recoverStuckAnalyses } from './writersRoom/evaluator.js';
import { recoverStuckGeneratingStages } from './pipeline/textStages.js';
import { recoverStuckAutoRuns } from './pipeline/autoRunner.js';
import { recoverStuckAutopilots } from './pipeline/seriesAutopilot.js';
import { recoverInFlightProjects } from './creativeDirector/recovery.js';
import { recoverStuckMusicVideoRenders } from './musicVideo/render.js';
import { recoverStuckMusicVideoExcerpts } from './musicVideo/excerptRender.js';
import { recoverInterruptedModels as recoverInterruptedThreejsModels } from './threejsModels/index.js';
import { recoverInterruptedModels as recoverInterruptedImageTo3dModels } from './imageTo3d/models.js';
import { startPolling } from './instances.js';
import { ensureSelf } from './instanceIdentity.js';
import { initSyncLog } from './brainSyncLog.js';
import { backfillOriginInstanceId, brainCollectionStores } from './brainStorage.js';
import { relayUnloggedRecords } from './brainReconcile.js';
import { initSyncOrchestrator } from './syncOrchestrator.js';
import { initMediaJobQueue, resumeMediaJobCompletions, flushMediaJobQueue, quiesceMediaJobQueue } from './mediaJobQueue/index.js';
import { initSpriteLocalAnimationHook } from './sprites/localAnimationJobHook.js';
import { initLoraTraining } from './loraTraining/index.js';
import { initSharing, quiesceSharingForExit, drainPeerSyncOperations } from './sharing/index.js';
import { initMortalLoomStore } from './mortalLoomStore.js';
import { initUniverseBuilderCollectionHook } from './universeBuilderCollectionHook.js';
import { initCatalogImageAttachHook } from './catalogImageAttachHook.js';
import { initWritersRoomSceneImageHook } from './writersRoomSceneImageHook.js';
import { startHostedSessionSweep, stopHostedSessionSweep } from './fableLoom/hostedSession.js';
import { initFableLoomSceneImageHook } from './fableLoomSceneImageHook.js';
import { initFableLoomSceneVideoHook } from './fableLoomSceneVideoHook.js';
import { initMusicVideoSceneImageHook } from './musicVideoSceneImageHook.js';
import { initMusicVideoSceneVideoHook } from './musicVideoSceneVideoHook.js';
import { initMusicVideoCollectionHook } from './musicVideoCollectionHook.js';
import { initMusicVideoCastSetsImageHook } from './musicVideoCastSetsImageHook.js';
import { initMusicVideoCoverArtImageHook } from './musicVideoCoverArtImageHook.js';
import { initCreativeDirectorMusicBedHook } from './creativeDirectorMusicBedHook.js';
import { initMusicStudioHook } from './musicStudioHook.js';
import { initImageGenQuotaHook } from './imageGenQuota.js';
import { initSpriteReferenceImageHook } from './spriteReferenceImageHook.js';
import { initMoodBoardItemRenderHook } from './moodBoardItemRenderHook.js';
import { initStoryboardPanelImageHook } from './storyboardPanelImageHook.js';
import { initDeckRenderHook } from './deckRenderHook.js';
import { initCreativeDirectorSceneImageHook } from './creativeDirectorSceneImageHook.js';
import { initCreativeDirectorRenderFailureHook } from './creativeDirector/renderFailureHook.js';
import { initComicPagesFilenameHook } from './pipeline/comicPagesFilenameHook.js';
import { initStoryboardsFilenameHook } from './pipeline/storyboardsFilenameHook.js';
import { initSeasonCoverFilenameHook } from './pipeline/seasonCoverFilenameHook.js';
import { universeStore } from './universeBuilder.js';
import { seriesStore } from './pipeline/series.js';
import { issueStore } from './pipeline/issues.js';
import { storyBuilderStore } from './storyBuilder.js';
import { writersRoomStore } from './writersRoom/store.js';
import { mediaCollectionStore } from './mediaCollections.js';
import { loraDatasetStore } from './loraDatasets.js';
import { rapidReaderLibraryStore } from './rapidReaderLibrary.js';
import { commissionStore, backfillAllCommissionFeedback } from './creativeCommissions/store.js';
import { backfillProjectCommissionIds } from './creativeCommissions/projectControl.js';
import { outcomesStore as liOutcomesStore } from './layeredIntelligenceOutcomes.js';
import * as gameStore from './games/store.js';
import * as fableLoomStore from './fableLoom/store.js';
import { prerequisitesMetForRouting } from './providerPrerequisites.js';
import { peekFallbackChain } from '../lib/providerFallbackChainCache.js';
import { initFallbackChain } from './providerFallbackChain.js';
import { localCachedModelIds } from './localCachedModels.js';
import { stopCodexAppServer } from './codexAppServer.js';
import { logFailureWithStack as logBootstrapFailure } from '../lib/failureLogging.js';
/**
* Pre-route boot. Everything a route handler may depend on being ready the
* moment the first request lands: applied data migrations, a constructed AI
* Toolkit (routes are built from it), and the spawner/autofixer/task-learning
* background workers.
*
* Returns `{ aiToolkit, spawnerReady }` — `aiToolkit` feeds the toolkit-backed
* route factories in index.js, `spawnerReady` gates CoS init in
* `runBootSequence` below.
*/
export const bootstrapServices = async ({ io, dataDir, dataReferenceDir, serverDir }) => {
bindMaintenanceIo(io);
// Lifecycle hooks shared between AI Toolkit and PortOS runner shim
const aiToolkitHooks = attachLlmActivityHooks({
ensureProviderReady: (provider) => ensureProviderReadyForExecution(provider),
onRunCreated: (metadata) => {
recordSession(metadata.providerId, metadata.providerName, metadata.model).catch(err => {
logBootstrapFailure('❌ Failed to record usage session', err);
});
},
onRunCompleted: (metadata, output) => {
// Reads the provider CLI's own transcript for real token counts (including
// the prompt-cache tiers that dominate cost), falling back to the
// prompt-length/stdout estimate when none is found. Owns its own error
// handling — a usage-accounting failure must not fail the run.
return recordCompletedRunUsage(metadata, output);
},
onRunFailed: (metadata, error) => {
const errorMessage = error?.message ?? String(error);
// A content/safety refusal is a known, self-explanatory outcome — not a
// provider fault. Emit a distinct code + warning severity so (a) the
// autofixer skips it (it only spawns investigation tasks for
// AI_PROVIDER_EXECUTION_FAILED) and (b) the client shows a calm "model
// declined, stopped" notice instead of a red error toast. Automatic retry
// and provider fallback are disabled.
const isBudget = metadata.errorAnalysis?.category === ERROR_CATEGORIES.RUNTIME_BUDGET_EXHAUSTED;
const isRefusal = metadata.errorAnalysis?.category === ERROR_CATEGORIES.CONTENT_REFUSAL;
errorEvents.emit('error', {
code: isBudget ? 'AI_RUNTIME_BUDGET_EXHAUSTED' : (isRefusal ? 'AI_PROVIDER_CONTENT_REFUSED' : 'AI_PROVIDER_EXECUTION_FAILED'),
message: isBudget ? errorMessage : isRefusal
? `${metadata.providerName} declined this prompt on content/safety grounds — stopped without automatic retry or provider fallback.`
: `AI provider ${metadata.providerName} execution failed: ${errorMessage}`,
severity: isBudget || isRefusal ? 'warning' : 'error',
canAutoFix: !isBudget && !isRefusal,
timestamp: Date.now(),
context: {
runId: metadata.id,
provider: metadata.providerName,
providerId: metadata.providerId,
model: metadata.model,
exitCode: metadata.exitCode,
duration: metadata.duration,
workspacePath: metadata.workspacePath,
workspaceName: metadata.workspaceName,
errorDetails: errorMessage,
errorAnalysis: metadata.errorAnalysis,
timeoutBound: metadata.timeoutBound,
timeoutOrigin: metadata.timeoutOrigin,
runtimeBudgetMs: metadata.runtimeBudgetMs,
// Note: promptPreview and outputTail intentionally omitted to avoid leaking sensitive data
}
});
}
});
// The ORDER these run in is `runPreRouteSequence`'s contract (see
// bootstrapSequence.js); this object is only the "what".
//
// Under the boot smoke (#8343) the steps that would start local-LLM backends,
// the auto-fixer, or the CoS spawner (runner connection + agent recovery) are
// no-ops — see SMOKE_BOOT_DISABLED_STEPS.
const smokeBoot = isSmokeBoot();
if (smokeBoot) console.log('🧪 Smoke boot: startup jobs, agent recovery, and external integrations are disabled');
return runPreRouteSequence(gateStepsForSmokeBoot({
// Apply pending data migrations BEFORE the AI toolkit reads stage-config.json
// and providers.json. Without this, a plain pull-and-restart (no update.sh)
// leaves new prompt stages and other shipped data changes unregistered —
// existing installs hit "Stage X not found" until the user manually runs
// `npm run migrations` or `npm run update`. Idempotent and cheap when the
// applied-list is already current.
// Prefer PORTOS_DATA_ROOT (set at real launch in ecosystem.config.cjs) over the
// import.meta.url-derived path so a server booted from inside a CoS agent
// worktree still resolves to the real install; runMigrations also skips a
// worktree-rooted path as a backstop (#1947).
applyDataMigrations: async () => {
await runMigrations({ rootDir: resolveInstallRoot(join(serverDir, '..')) });
// Local recovery only: no mail, browser, or LLM provider calls.
const { initializeMessageDrafts } = await import('./messageDrafts.js');
await initializeMessageDrafts();
},
// usage.js and migration 304 both update usage.json. Initialize the service
// only after migrations finish so their whole-file writes cannot race and
// replace a freshly normalized or rolled-up snapshot with stale bytes.
loadUsage,
// Verify every registered collection's on-disk type-level schemaVersion
// matches what the code expects. Mismatches mean a migration didn't run (or
// the user rolled the code back below a forward-only migration) — log loudly
// but DO NOT crash the server. PortOS is single-user (AGENTS.md "Security
// Model"); a hard exit on startup is worse than a noisy log the user can act
// on. Returns per-store statuses for downstream telemetry; we discard them.
verifyCollections: () => verifyCollectionVersions([universeStore(), seriesStore(), issueStore(), conflictJournalStore(), storyBuilderStore(), mediaCollectionStore(), loraDatasetStore, rapidReaderLibraryStore, liOutcomesStore(), commissionStore(), gameStore, fableLoomStore, ...brainCollectionStores()]),
createToolkit: () => createAIToolkit({
dataDir,
providersFile: 'providers.json',
runsDir: 'runs',
promptsDir: 'prompts',
screenshotsDir: join(dataDir, 'screenshots'),
sampleProvidersFile: join(dataReferenceDir, 'providers.json'),
io,
asyncHandler,
withAssetPublication: withBackupAssetPublication,
withRunAdmission: handler => (req, res) => maintenance.run('manual-run', 'Runs', () => handler(req, res)),
// Inject PortOS's ServerError so toolkit route errors normalize into the
// canonical `{ error, code, timestamp, context? }` envelope (issue #1084).
ServerError,
hooks: aiToolkitHooks,
// Keep the provider connection graph reconciled with any write to
// providers.json — an old client's PATCH, a model refresh, a delete.
// Injected rather than imported by the toolkit, which stays
// self-contained. A no-op until the database phase enables the graph.
//
// The same write is also the moment a catalog refresh retires a model a
// stored pin names, so it re-runs the derive-on-read pin audit and logs
// the pins that NEWLY went stale (#7328). allSettled, not all: neither
// consumer may skip or fail the other.
onProvidersSaved: () => Promise.allSettled([
// Covers direct toolkit route writes as well as service-shim callers.
noteReadinessChanged(),
onProvidersSavedForGraph(),
reportRetiredModelPins(),
]),
// Keep the fallback chain off providers whose CLI is not installed on
// this host (#4611), so a run falls through to the next candidate instead
// of dying at spawn time. Sync by contract: it reads the runtime probe's
// cache and never blocks.
prerequisitesMet: prerequisitesMetForRouting,
// The user's single ordered fallback list (Models > Fallbacks); a cache
// read, so it is sync as the toolkit requires.
getFallbackChain: peekFallbackChain,
// MTPLX and Slotstream each serve one checkpoint per process and report
// only that one, so a refresh could never surface a newly downloaded
// checkpoint — and for Slotstream it pruned the record's other shipped ids.
// This lets the refresh merge in what is actually on disk; it answers
// `null` for every other provider, reads a local cache rather than calling
// a model, and imports each runtime's cache reader lazily so neither subtree
// reaches the boot closure (see `services/localCachedModels.js`).
cachedModelIds: localCachedModelIds,
// A composite id (`harness.method@service[+bootstrap]`, #7564) names no
// stored record: the toolkit hands it here, and the host materializes it
// from the harness registry, the service instance and settings. Imported
// lazily so the graph store's subtree stays off the boot closure until a
// composite is actually asked for.
resolveCompositeProvider: async (id) => (await import('./compositeProviders.js')).resolveCompositeProvider(id)
}),
// Compatibility shims for services that import from the old service files.
registerToolkitShims: (aiToolkit) => {
setProvidersToolkit(aiToolkit);
setProviderStatusToolkit(aiToolkit);
setRunnerToolkit(aiToolkit, { dataDir, hooks: aiToolkitHooks });
setPromptsToolkit(aiToolkit);
// Note: the prompts service is initialized automatically by createAIToolkit().
},
// Warm the providers file at startup so the codex-sentinel migration runs
// before any inbound request can hit the providers cache. A pure READ of
// on-disk provider config — it dispatches nothing to a provider.
warmProviders: async (aiToolkit) => {
// Prime the sync fallback-list cache from settings.json (disk only).
await initFallbackChain().catch((err) => logBootstrapFailure('⚠️ Failed to load provider fallback list', err));
return aiToolkit.services.providers.getAllProviders();
},
// Ensure the provider paired with the active local-LLM backend (LLM_BACKEND in
// .env, chosen at setup time) is enabled, so a fresh install can use Ollama /
// LM Studio for runs without hand-toggling it in the Providers UI. Starting
// the Ollama server is not a provider CALL — nothing is inferred until the
// user asks for it.
ensureLocalLlmBackend: () => {
if (readPortosEnvValue('PORTOS_FLEET_LLM_ENABLED') === '1') return;
const activeLocalLlmBackend = getLocalLlmBackend();
ensureBackendProvider(activeLocalLlmBackend).catch((err) =>
logBootstrapFailure('⚠️ Failed to enable local LLM backend provider', err));
if (activeLocalLlmBackend === 'ollama') {
ensureOllamaRunning({ preferPersistent: true }).catch((err) =>
logBootstrapFailure('⚠️ Failed to start Ollama for active local LLM backend', err));
}
},
// Register PortOS's CLI + TUI runners through the toolkit's declared extension
// points (setCliRunner / setTuiRunner) instead of overwriting private props.
// The CLI variant adds per-provider argv building (Codex `exec -`, Antigravity
// `agy --print`, Claude Code `-p -`); the toolkit's in-tree implementation is
// also safe (no shell, prompt via stdin) — the variant exists for the per-CLI
// invocation conventions, not for security. The TUI runner has no toolkit
// built-in: registering it lets POST /api/runs with a TUI provider dispatch
// here instead of 400ing. Both runners track their child process / pty via the
// toolkit's external-run registry, so the toolkit's own stopRun/isRunActive/
// deleteRun account for live runs without any sibling-method monkey-patching.
registerRunners: (aiToolkit) => {
const executeApi = aiToolkit.services.runner.executeApiRun.bind(aiToolkit.services.runner);
aiToolkit.services.runner.executeApiRun = options => {
const permit = maintenance.admit('api-run', options.runId, { continuation: true });
return permit.run(() => executeApi({ ...options,
onPersistenceFailure: () => permit.markUnsettled(),
onSettled: () => permit.finish(),
})).catch(err => { permit.markUnsettled(); throw err; });
};
aiToolkit.services.runner.setCliRunner(executeCliRunFixed);
aiToolkit.services.runner.setTuiRunner(executeTuiRunFixed);
},
// Auto-fixer for error recovery.
initAutoFixer,
// Task learning system, tracking agent completions.
initTaskLearning,
// The CoS agent spawner (event wiring + orphan cleanup), initialized
// explicitly now that the runner registration + task learning are ready.
startSpawner: initSpawner
}, 'preRoute', smokeBoot));
};
/**
* Fire-and-forget service inits + scheduler arming. None of these block the
* server from listening; each logs its own failure and the boot continues.
*/
const startBackgroundServices = ({ spawnerReady, io, httpsEnabled, port }) => {
// A crashed process leaks settled reservations that finish() never reached.
// Stamp uncertain publications before later boot steps read the journal.
// Sync and non-throwing; CoS may be off. Smoke boot skips this whole function.
reconcileDeadOwnerReservations();
// Keep the opt-in agent API key file current (Settings > Security). A no-op
// beyond one settings read while the key is off. Disabled under smoke boot
// with the rest of this function, so a smoke run never writes to $HOME.
import('./agentKey.js')
.then(({ initAgentKey }) => initAgentKey({ httpsEnabled, port }))
.catch((err) => logBootstrapFailure('❌ Agent API key init failed', err));
// Put npm's global bin directory on PATH before anything spawns a provider
// CLI. npm's prefix need not be the directory the host's Node installer put
// on PATH, and a CLI installed there is invisible to the bare-name spawn a
// TUI provider uses until it is adopted. One `npm prefix -g` child, never an
// AI provider call — safe under AGENTS.md's no-cold-bootstrap rule, on the
// same footing as the `--version` probes providerPrerequisites.js already
// runs. The CoS runner is a separate process and adopts it separately.
adoptNpmGlobalBinDir().catch((err) => logBootstrapFailure('❌ npm global bin adoption failed', err));
// Explicit call (not a module-level side effect) so test imports of cos.js
// don't spin up its event listeners and timers. The spawner gate itself lives
// in bootstrapSequence.js.
initCosAfterSpawner({ spawnerReady, initCos: () => cos.init(), isStopping: isHostShuttingDown });
// Restore only an installed, opted-in world, then its bridge, resident
// presence, and pending design. No install, download, or AI provider call.
// startBackgroundServices is disabled during smoke boot.
import('./eidoverseRuntime.js')
.then(({ reconcileEidoverseRuntime }) => reconcileEidoverseRuntime())
.then((result) => {
if (result.running) console.log('🌐 Eidoverse runtime and resident presence restored');
})
.catch(err => logBootstrapFailure('⚠️ Eidoverse startup deferred', err));
// Docker containment (#12844): sweep containers/volumes a crash or restart
// left behind — only when the user turned the feature on, so a fresh install
// makes no Docker call at boot.
Promise.all([import('./instanceFeatures.js'), import('./dockerContainment.js')])
.then(async ([{ isInstanceFeatureEnabled }, { reconcileDockerContainment }]) => {
if (await isInstanceFeatureEnabled('docker-containment')) await reconcileDockerContainment({ enabled: true, reason: 'boot' });
})
.catch(err => logBootstrapFailure('⚠️ Docker containment boot reconcile deferred', err));
// Initialize agent automation scheduler and action executor
automationScheduler.init().catch(err => logBootstrapFailure('❌ Agent scheduler init failed', err));
// agentActionExecutor.init() is synchronous — guard with try/catch so a thrown
// error logs cleanly instead of crashing the server at module load.
try {
agentActionExecutor.init();
} catch (err) {
logBootstrapFailure('❌ agentActionExecutor init failed', err);
}
// Inbox recovery is deferred until after initSyncLog() (see the ensureSelf chain
// in runBootSequence) — it mutates inbox entries, which are now synced brain
// records, so its updateInboxLog() calls append to sync_log.jsonl and MUST run
// after the log's currentSeq is loaded, or they'd write low/duplicate sequence
// numbers and corrupt peer cursors.
recoverStuckAnalyses().catch(err => logBootstrapFailure('❌ Writers Room recovery failed', err));
recoverStuckGeneratingStages().catch(err => logBootstrapFailure('❌ Pipeline text-stage recovery failed', err));
recoverStuckAutoRuns().catch(err => logBootstrapFailure('❌ Pipeline auto-run recovery failed', err));
recoverStuckAutopilots().catch(err => logBootstrapFailure('❌ Pipeline autopilot recovery failed', err));
recoverStuckMusicVideoRenders().catch(err => logBootstrapFailure('❌ Music Video recovery failed', err));
recoverStuckMusicVideoExcerpts().catch(err => logBootstrapFailure('❌ Music Video excerpt recovery failed', err));
// A provider child cannot survive a server restart. Make interrupted
// Three.js generations retryable; this is state recovery only, never a
// cold-bootstrap provider call.
recoverInterruptedThreejsModels().catch(err => logBootstrapFailure('❌ Three.js model recovery failed', err));
recoverInterruptedImageTo3dModels().catch(err => logBootstrapFailure('❌ Image-to-3D model recovery failed', err));
// Arm the managed-model-server idle reaper. Timer only — it reads timestamps
// and may stop a PM2 process, and makes no AI provider call, so it is safe
// under AGENTS.md's "No cold-bootstrap LLM calls". Off in practice until the
// user sets an idle window (0 = never, the default).
startIdleReaper();
// Arm the FableLoom hosted-session sweeper. Timer only — it walks in-memory
// QR play sessions and tears down the ones past their TTL, making no AI
// provider call, so it is safe under AGENTS.md's "No cold-bootstrap LLM calls".
startHostedSessionSweep({ io });
// Initialize activity-digest scheduler — OFF by default; drafts daily-log
// auto-summaries from the Human Activity timeline only when the user enables it
// (Settings → Daily Log → Activity Digest). Silent + no LLM calls until then.
startActivityDigestScheduler();
// Initialize twin-enrichment scheduler — LLM-free daily rollup of observed
// taste + chronotype evidence from the Human Activity timeline (#2156). No
// provider calls; the AI interpretation is a separate explicit-button action.
startTwinEnrichmentScheduler();
// Initialize brain→memory bridge (mirrors brain data into CoS memory for semantic search)
initBrainMemoryBridge();
// Load any on-disk POST drill cache into memory. Does NOT trigger LLM calls —
// cache fill only happens on explicit user request (see meatspacePostRoutes.js).
initDrillCache().catch(err => logBootstrapFailure('❌ POST drill cache init failed', err));
// Register the optional daily POST reminder (opt-in, off by default) if the
// user has enabled it — deterministic cron nudge, no LLM calls.
// catchUpMissedSlot: true so a reminder whose slot elapsed while the server
// was down (or during a redeploy) still fires once we're back up, instead of
// silently waiting for tomorrow's tick (#2015).
registerPostReminderSchedule({ catchUpMissedSlot: true }).catch(err => logBootstrapFailure('❌ POST reminder init failed', err));
// Register the optional daily autobiography story prompt (opt-in, off by
// default, and only when the instance feature is on) — deterministic prompt
// bank, no LLM calls. Same catch-up rationale as the POST reminder above.
registerAutobiographyReminderSchedule({ catchUpMissedSlot: true }).catch(err => logBootstrapFailure('❌ Autobiography reminder init failed', err));
// Initialize backup scheduler for daily data backups
startBackupScheduler().catch(err => logBootstrapFailure('❌ Backup scheduler init failed', err));
// Automatic PortOS self-update — OFF by default. When the user turns it on
// (Update tab), it polls for a window in which the install is completely idle
// and then runs the SAME update the matching button runs. Registering the
// poll launches nothing by itself.
startAutoUpdateScheduler(io).catch(err => logBootstrapFailure('❌ Auto-update scheduler init failed', err));
// Initialize Privacy Center opt-out recheck scheduler — OFF by default; only
// re-runs the broker scan + opt-out pass when the user opts in via
// Settings → Privacy (sanctioned scheduled-automation exception) (#2145).
startPrivacyRecheckScheduler({ catchUpMissedSlot: true }).catch(err => logBootstrapFailure('❌ Privacy recheck scheduler init failed', err));
// Quota-burn loop — ONE install-level loop that spends subscription quota that
// would otherwise expire. OFF by default: the tick returns before touching a
// provider unless the user enabled it on the Quota Burn page, where the
// family, provider, model, and work are all named first (sanctioned
// scheduled-automation exception).
startQuotaBurnScheduler();
// Manual maintenance runs (Schedule tab → "Run maintenance now"): arms a
// listener and a slow sweep; dispatches only for a run the user started.
startMaintenanceRunScheduler();
// GitHub inbox Resolve with agent (#13191): marks a thread done when the
// agent run the operator started on it succeeds. Starts no agent itself.
startNotificationResolveListener();
// Initialize Series Autopilot scheduler — OFF by default; registers a cron per
// series only when the user configured + enabled one via Settings → Series
// Autopilot. Each scheduled run still passes through the cos autonomy gate +
// daily budget (sanctioned scheduled-automation exception) (#2174).
startSeriesAutopilotScheduler().catch(err => logBootstrapFailure('❌ Series Autopilot scheduler init failed', err));
// Autonomous Creation Engine (#2657) — arm a cron per enabled Creative
// Commission. Boot only ARMS timers; nothing fires until a cadence elapses, and
// each fire gates on creative autonomy `execute` + the daily cos budget (so an
// `off`/`dry-run` install generates nothing). Sanctioned scheduled-automation.
// The backfill is pure data movement (no LLM), idempotent, best-effort. The
// scripts/migrations runner executes before the DB pool is up, so the data move
// lives here (see the migration 194 registration stub); the table itself is
// created by ensureSchema. Backfill-before-arm ordering lives in
// bootstrapSequence.js.
armCommissionScheduler({
backfillCommissionFeedback: backfillAllCommissionFeedback,
backfillProjectCommissionIds,
startCommissionScheduler
});
// Executable Eidoverse world controllers (#7456) — arm the supervisor only
// when this install has at least one ARMED controller, so a world nobody has
// built in registers nothing and logs nothing. Boot arms a timer and nothing
// else: a controller step is synchronous by construction, so the tick path
// cannot reach an AI provider at all, and no install steps until its own
// cadence elapses.
startEidoverseControllerSupervisor().catch(err => logBootstrapFailure('❌ Eidoverse controller supervisor arming failed', err));
// OpenWorld's historical snapshot scheduler is retired with its UI. The
// legacy read/capture routes remain available for old clients, but Eidoverse
// is now the only automatic world projection path.
// Initialize iMessage sync scheduler — OFF by default; only polls chat.db when
// the user opts in from the iMessage Settings drawer on Comms → Messages → iMessage
// (needs macOS Full Disk Access) (#2151).
startImessageScheduler().catch(err => logBootstrapFailure('❌ iMessage sync scheduler init failed', err));
// Initialize Signal sync scheduler — OFF by default; only reads the SQLCipher
// chat DB (via the keychain-wrapped key) when the user opts in via
// Settings → Signal (#2154).
startSignalScheduler().catch(err => logBootstrapFailure('❌ Signal sync scheduler init failed', err));
// Initialize Spotify sync scheduler — OFF by default; only polls the
// recently-played API when the user connects Spotify + opts in via
// Settings → Spotify (#2152).
startSpotifyScheduler().catch(err => logBootstrapFailure('❌ Spotify sync scheduler init failed', err));
// Initialize YouTube watch-history sync scheduler — OFF by default; only scrapes
// the signed-in history page in the managed browser when the user opts in via
// Settings → YouTube (#2153).
startYoutubeScheduler().catch(err => logBootstrapFailure('❌ YouTube sync scheduler init failed', err));
// Initialize the Beeper ingestion sweep — OFF by default; registers only when
// the Beeper instance feature is on AND a token is configured, and then only
// runs when `settings.beeper.enabled` is set (fork issue #32). Deliberately
// NOT gated on Beeper's own `app.state`, which was measured reporting
// `initializing` for 105s while every account was connected.
startBeeperScheduler().catch(err => logBootstrapFailure('❌ Beeper sync scheduler init failed', err));
// Reconcile the outbox's two non-terminal states against the process that
// just died: a row left in `sending` becomes a failed row the user can act on
// (never an automatic resend — Beeper has no idempotency key), and a row left
// in `awaiting-confirmation` gets its lookup re-armed, which reads and never
// sends. Deliberately NOT on the ingestion gate above: a send stranded by a
// restart is stranded whether or not scheduled sync is switched on, and it is
// otherwise un-actionable forever, since nothing but this can move it.
reconcileOutboxOnBoot().catch(err => logBootstrapFailure('❌ Beeper outbox boot reconcile failed', err));
// Arm the Beeper realtime transport on the SAME gate as the sweep above
// (feature on + token present, never `app.state`) — one long-lived WebSocket
// to the local Beeper Desktop whose only job is to make ingestion prompt
// (fork issue #33). Correctness stays with the sweep: a reconnect, a `seq`
// gap or an `app.state` recovery each ask it to run early.
startBeeperSocket().catch(err => logBootstrapFailure('❌ Beeper realtime transport init failed', err));
// Periodically GC orphan zero-issue/zero-canon importer shells left by an
// abandoned analyze (issue #727).
startOrphanShellGc();
// Periodically GC orphan staged init/reference upload images that pile up in
// data/image-refs on every i2i/edit render and are never cleaned up (issue #1214).
startImageRefsGc();
// Periodically GC the Image Cleaner's GPU-clean temp working files (init/render/
// mask/original) that land in data/image-clean-tmp and are never long-lived
// (issue #2264). Age-gate only — nothing here is referenced after the fetch.
startImageCleanTmpGc();
// Periodically GC leftover `${dest}.partial` files from resumable weight
// downloads (issue #5855). Age-gated (7 days) so an overnight pause-and-resume
// is not punished; in-flight spec-decode dests are also pinned. Fire-and-forget
// like the other GCs — not part of the awaited warmStores pruneLegacyFiles
// chain, so a large sweep cannot stall listen().
startOrphanedPartialGc();
// Periodically bound the Beeper attachment mirror: evict least-recently-viewed
// bytes down to `settings.beeper.attachmentBudgetGb` (never evicting a file
// Beeper can no longer re-supply, and never one the user locked with `keep`),
// and heal the store both ways — orphan files no row points at, rows whose
// file is gone, abandoned partials (fork issue #37). Housekeeping over local
// bytes, so it is NOT gated on the ingestion toggle: turning scheduled sync
// off does not make an over-budget mirror stop being over budget.
startBeeperAttachmentGc();
// Erase Tribe people deleted more than 30 days ago — with their touchpoints,
// identities, memory links and audit snapshots — and expire older tribe audit
// snapshots (#8459). Deleting a contact must eventually erase a third party's data.
startTribePurge();
// Periodically expire images handed to agents via data/screenshots: shell
// image drops after 7 days, other screenshots after 30 unless a live CoS task
// still references them (issue #8461). Fails closed on an unreadable task store.
startScreenshotsGc();
// Warm the catalog user-type registry from the user-type store (Postgres as of
// #1001; the settings.json slice under the escape hatch) before any catalog
// request can land, so user-defined types validate + mint ids immediately on
// boot. The store's PG backend self-runs ensureSchema + the one-time settings→DB
// import, so this is safe even though it fires before the boot DB gate. No
// settings:updated listener anymore: the registry's only writers are the
// `/api/catalog/types` routes and the sync merge, both of which call
// setUserCatalogTypes(next) directly — a settings save no longer touches types,
// and a listener reading the now-absent settings key would wipe the registry.
readUserTypeSlice()
.then(list => setUserCatalogTypes(Array.isArray(list) ? list : []))
.catch(err => logBootstrapFailure('❌ Catalog user-type warm failed', err));
// Initialize Telegram (manual bot or MCP bridge based on settings)
getInitSettings().then(s => {
if (s.telegram?.method === 'mcp-bridge') {
telegramBridge.init().catch(err => logBootstrapFailure('❌ TG Bridge init failed', err));
} else {
telegram.init().catch(err => logBootstrapFailure('❌ Telegram init failed', err));
}
}).catch(err => logBootstrapFailure('❌ Telegram settings read failed', err));
// Reconcile voice stack (start portos-whisper if voice.enabled)
getVoiceConfig().then(cfg => reconcileVoice(cfg, { allowSetup: false })).catch(err => logBootstrapFailure('❌ Voice reconcile failed', err));
// Re-arm any voice timers that survived a restart (independent of voice.enabled —
// a pending reminder should still fire even if voice is currently off).
initVoiceTimers().catch(err => logBootstrapFailure('❌ Voice timer init failed', err));
// Check for update completion marker from a previous update cycle. The full
// read/validate/record/cleanup lifecycle lives in updateChecker.js.
processUpdateMarker().catch(err => logBootstrapFailure('❌ Update marker processing failed', err));
// Clear stale updateInProgress if the server was killed mid-update
clearStaleUpdateInProgress().catch(err => logBootstrapFailure('❌ Stale update recovery failed', err));
// Capture the commit this process booted at, so /api/update/status can detect
// a bare `git pull` that advanced on-disk HEAD without restarting (issue #1779).
// Best-effort — a tarball/non-git install just yields no boot commit.
captureBootCommit().catch(err => logBootstrapFailure('❌ Boot commit capture failed', err));
// Start periodic update checker (checks GitHub releases every 30 min)
startUpdateScheduler();
// Restore any active loops from previous session
restoreLoops().catch(err => logBootstrapFailure('❌ Loop restore failed', err));
};
/**
* Media-job-queue-dependent completion hooks. Every one of these files a
* finished render onto its owning record even if the requesting client
* unmounted mid-render, so they must be wired AFTER the queue has loaded its
* persisted jobs (otherwise they'd miss `completed` events for reloaded jobs).
*/
const initMediaJobDependentHooks = () => {
// Universe Builder needs the media job queue running before it can listen
// for `completed` events — so initialize the hook here.
initUniverseBuilderCollectionHook();
// Sprite local-render hook (#4876) — stages a finished MiniMax H3 clip into
// its animation run and files the outcome, for renders that outlive the
// request that started them. Its boot pass also reconciles jobs that settled
// while the server was down, which is the only thing that can rescue a
// multi-hour render interrupted by a restart.
initSpriteLocalAnimationHook();
// Catalog image-attach hook — durably files a queued render onto its target
// ingredient on completion, even if the editor page unmounted mid-render
// (#1359).
initCatalogImageAttachHook();
// Writers-Room scene-image hook — durably files a queued storyboard render
// onto its analysis snapshot + work collection on completion (#1363).
initWritersRoomSceneImageHook();
// FableLoom scene-image hook — durably files a queued scene render onto its
// loom episode's node on completion, even if the editor unmounted mid-render.
initFableLoomSceneImageHook();
// FableLoom scene-video hook — durably files a queued local clip onto its
// loom episode's node on completion, even if the editor unmounted.
initFableLoomSceneVideoHook();
// Music Video scene-image hook — durably files a queued reference-frame
// render onto its project scene's `referenceImageId` on completion, even if
// the director board unmounted mid-render (#1760 Phase 1b).
initMusicVideoSceneImageHook();
// Music Video scene-video hook — durably files a queued i2v scene clip onto
// its project scene's `videoHistoryId` on completion (#1760 Phase 1).
initMusicVideoSceneVideoHook();
// Music Video collection hook — files every render that names a music video
// (a shot's, Cast & Sets, or a tagged Image/Video Gen request) into its collection.
initMusicVideoCollectionHook();
// Music Video Cast & Sets hook — files a queued check-in reference image
// (character sheet, looks, set plates, in-set tests) onto its stage key.
// It only listens: nothing is generated at boot.
initMusicVideoCastSetsImageHook();
// Music Video cover art hook — composes the release cover from a cover
// source image the director asked for. It only listens.
initMusicVideoCoverArtImageHook();
// Creative Director music-bed hook — durably files a queued first-pass
// audio render onto its project's `musicBed` field on completion, even if
// the requesting client unmounted mid-render (#1928).
initCreativeDirectorMusicBedHook();
// Music Studio audio jobs attach to their track after completion so the
// library remains correct across a client remount or navigation.
initMusicStudioHook();
// Image-gen quota observation — the cloud image backends expose no quota API,
// so the Usage page's Image Gen card is built from renders we watched succeed
// or get refused. One bus subscriber, never per-provider edits.
initImageGenQuotaHook();
// Sprite reference-candidate hook — copies a completed reference/anchor
// render into the sprite record's reference/candidates/ with a generation
// sidecar (#2896).
initSpriteReferenceImageHook();
// Mood board note render hook — turns a rendered text note into its image
// item (#10531). It only listens.
initMoodBoardItemRenderHook();
// Storyboard panel frames — each finished frame becomes a panel take.
initStoryboardPanelImageHook();
// Deck card hook — files a completed card render onto its deck card row.
initDeckRenderHook();
// Creative Director scene-frame hook — durably files a queued first-pass
// reference-frame render onto its project scene's `sourceImageFile` on
// completion, even if no client is watching (#1867).
initCreativeDirectorSceneImageHook();
// Creative Director render failures are actionable code defects; queue one
// deduplicated internal repair task for every tagged CD media lane.
initCreativeDirectorRenderFailureHook();
// Pipeline filename hooks — stamp `filename` onto stage records on
// media-job completion so the UI can still render them after the
// 24h media-job archive TTL elapses.
initComicPagesFilenameHook();
initStoryboardsFilenameHook();
initSeasonCoverFilenameHook();
// Best-effort pre-materialize the MortalLoom iCloud store so the
// dashboard's proactive-alerts poll (and other readers) don't trigger
// on-demand downloads that surface as EAGAIN. `brctl download` only
// materializes the file — it does not pin against future eviction, so
// the retry-on-EAGAIN path inside the store is what guarantees the
// hardening. Fire-and-forget — failures are logged.
initMortalLoomStore().catch((err) => {
logBootstrapFailure('⚠️ MortalLoom store init failed', err, console.warn);
});
};
/**
* The database phase's dependencies. `runDatabasePhase` (bootstrapSequence.js)
* owns the gate → migrate → warm → arm ordering; this supplies the real db.js,
* the real migration scripts, and the real stores.
*
* Escape hatches for the health gate (dev/tests only, UNSUPPORTED for
* production): `MEMORY_BACKEND=file` (explicit file backend) and test runners
* (test suites boot without a database). Both downgrade "PostgreSQL is required"
* from a fail-fast to a warning.
*
* Every migration/warm step is imported lazily, at the moment it runs, exactly
* as before the extraction — nothing in this phase is pulled into the module
* graph on a boot that never reaches it.
*/
const runDatabaseBootPhase = () => runDatabasePhase({
gate: async () => {
const { checkHealth, ensureSchema } = await import('../lib/db.js');
const { dbReady } = await gateOnDatabase({
checkHealth,
ensureSchema,
escapeHatch: process.env.MEMORY_BACKEND === 'file' || isTestRunner()
});
// ensureSchema is threaded through to the migrations below so the phase
// resolves the db.js module exactly once.
return { dbReady, ensureSchema };
},
// Releases CoS reads that wait on the agent-record migrations (020/021) once the
// runner has had its turn — a failed migration exits the process instead.
migrate: ({ dbReady, ensureSchema }) => runDbAndCatalogMigrations({
dbReady,
ensureSchema,
// Versioned DB-migration runner (#1029): ordered schema-DELTA migrations
// (renames / type changes / data transforms / embedding-dim changes) that
// ensureSchema()'s additive IF NOT EXISTS gates can't express. Loading the
// runner is a separate step from running it because only the latter is
// fatal — a module that won't load falls to the group's log-and-continue.
loadDbMigrationRunner: async () => (await import('../scripts/run-db-migrations.js')).runDbMigrations,
migrateBibleToCatalog: async () => (await import('../scripts/migrateBibleToCatalog.js')).migrateBibleToCatalog(),
// One-time data repair: rewrite legacy machine universe tags
// (`from-universe`, `universe:<id>`) on backfilled rows into the friendly
// universe NAME tag. Marker-gated in data/catalog-universe-tags.applied.json.
repairUniverseTags: async () => (await import('../scripts/repairUniverseTags.js')).repairUniverseTags(),
// Per-record catalog payload-shape migration — walks rows whose stored
// payload.schemaVersion lags the registry-current and applies registered
// upgraders. No-ops via marker once an install is at the high-water version.
migrateCatalogPayload: async () => (await import('../scripts/migrateCatalogPayload.js')).migrateCatalogPayload(),
// One-time canon↔catalog reconciliation: collapse any pre-existing
// divergence between an embedded universe-canon entry and its catalog row
// (they were copy-on-write mirrors before the bidirectional projection
// landed). LWW on updatedAt; writes the winner to both sides. Marker-gated
// in data/catalog-canon-reconcile.applied.json.
reconcileCanonCatalog: async () => (await import('../scripts/reconcileCanonCatalog.js')).reconcileCanonCatalog(),
// Media asset index (#1000): subscribe the generation-completed hooks +
// reconcile the derived media_assets table against on-disk images/videos.
// Bytes + sidecars + video-history.json stay authoritative; this builds a
// queryable index over them. Idempotent, safe to run every boot.
initMediaAssetIndex: async () => (await import('./mediaAssetIndex/index.js')).initMediaAssetIndex()
}).finally(markCosAgentMigrationsSettled),
warmStores: () => warmMandatoryStores({
// Universe Builder PG warm (#1014): listIds() is the cheapest call that
// forces backend selection + the migrateUniversesToDB import.
warmUniverses: () => universeStore().listIds(),
// Pipeline series + issues PG warm (#1015): same contract.
warmSeries: () => seriesStore().listIds(),
warmIssues: () => issueStore().listIds(),
// Story Builder sessions PG warm (#1016): same contract.
warmStoryBuilder: () => storyBuilderStore().listIds(),
// Writers Room PG warm (#1017): listWorkIds() forces backend selection +
// migrateWritersRoomToDB. Draft .md bodies stay on disk (file-primary);
// only the metadata migrates.
warmWritersRoom: () => writersRoomStore().listWorkIds(),
// Authoritative catalog user-type warm (#1001): load the registry from the
// catalog_user_types store (runs the one-time settings→DB import on first
// access), so a normal install always serves with the registry warm even if
// the early fire-and-forget warm raced a cold DB.
warmCatalogUserTypes: async () => {
const warmTypes = await readUserTypeSlice();
setUserCatalogTypes(Array.isArray(warmTypes) ? warmTypes : []);
},
// Creative Director PG warm (#997): unlike the other stores, CD's file→DB
// import is triggered lazily on first backend access; at boot the only other
// trigger is a NOT-awaited fire-and-forget recoverInFlightProjects() in an
// earlier step, so it can still be in flight here. listProjects() forces
// selectBackend() → the (idempotent, marker-gated) import to completion,
// which the prune below depends on.
warmCreativeDirector: async () => (await import('./creativeDirector/local.js')).listProjects(),
// Legacy artifact prune: removes the `.imported` / `.bak-NNN` recovery
// copies the migrators parked aside, but ONLY when the live row count
// matches the marker's recorded import (a wiped/restored DB keeps the
// recovery files). Marker-gated in data/legacy-prune.applied.json.
pruneLegacyFiles: async () => (await import('../scripts/pruneImportedLegacyFiles.js')).pruneImportedLegacyFiles()
}),
// Provider connection graph (#6367): first run imports every provider record
// into ai_connections / ai_harness_bindings / ai_route_bindings; later runs
// reconcile against providers.json. Reads and writes local state only.
reconcileProviderGraph: async () => {
await (await import('./providerGraph.js')).initProviderGraph();
// Per-harness enablement (#7564): normalize the settings slice and prime
// the composite cache's revision. Settings-only — no provider is contacted.
await (await import('./harnessEnablement.js')).reconcileHarnessEnablement();
},
reconcileStackerNews: reconcileStackerNewsSchedulers
});
/** Log the canonical "where do I open this" banner and wire the HTTPS extras. */
const announceListening = ({ io, httpServer, localHttpServer, httpsEnabled, port }) => {
// One canonical "where do I open this" banner — :5555 is always user-facing
// (HTTP or HTTPS), :PORTOS_HTTP_PORT (default 5553) is the loopback HTTP
// mirror that only spawns when HTTPS is active. See docs/PORTS.md.
// A smoke boot requests port 0 (#12543), so log the port actually bound.
const boundPort = httpServer.address()?.port ?? port;
console.log(`🚀 PortOS listening on :${boundPort} (${httpsEnabled ? 'https' : 'http'})`);
// Which code is up. One PM2-managed portos-server serves :5555 for the whole
// machine while any number of worktrees hold different code, so `pm2 logs
// portos-server` should answer "which commit is this?" without a request
// (#4694). Logged here rather than fire-and-forget at module scope so it
// lands at a DETERMINISTIC position in the banner instead of racing it.
// Read synchronously from the cache primed at boot: this function's caller
// does not await it, so an await here would let the lines below print around
// the yield and scramble the banner. In the normal case boot (migrations, DB)
// far outlasts one git call, so the value is ready and lands in place.
//
// If it is NOT ready — a git wedged on a locked index or a slow network mount
// — defer rather than printing `unknown`, which would be a permanent lie: the
// probe resolves moments later and every other consumer reports it correctly.
// Losing banner adjacency beats logging a wrong answer that never updates.
const identity = getCachedBuildIdentity();
if (identity) {
console.log(` 🧬 build ${formatBuildIdentity(identity)}`);
} else {
getBuildIdentity()
.then((late) => {
// Only print once there is something true to print. A failed probe
// resolves to an all-null tuple, and formatting THAT would claim "no git
// metadata" — the wrong reason, permanently, about a checkout that is
// fine. Say the probe did not finish instead; the API and the UI report
// the real value as soon as a retry succeeds.
if (late?.shortCommit) console.log(` 🧬 build ${formatBuildIdentity(late)}`);
else console.log(` 🧬 build — git probe did not finish in time; /api/system/build will report it`);
})
.catch((err) => logBootstrapFailure('❌ Build identity probe failed', err));
}
if (!httpsEnabled) {
console.log(` 🌐 http://localhost:${boundPort}`);
console.log(`⚠️ HTTP only — getUserMedia (mic) won't work over Tailscale IP. Run "npm run setup:cert" to enable HTTPS.`);
return;
}
// Only the boot smoke may ask for a kernel-assigned mirror (#12543); elsewhere
// 0, empty or invalid keeps meaning the shipped default.
const localHttpPort = isSmokeBoot() && process.env.PORTOS_HTTP_PORT === '0'
? 0
: Number(process.env.PORTOS_HTTP_PORT) || PORTS.API_LOCAL;
// Lead with the URL that works from THIS machine with no cert warnings:
// the loopback HTTP mirror. http://localhost:${port} does NOT work in
// HTTPS mode (the :${port} socket speaks TLS only), so local users who
// type it land on a dead port — point them here instead.
console.log(` 👉 http://localhost:${localHttpPort} (open locally — no cert warnings)`);
// Only advertise the Tailscale hostname when it's actually usable. A cert
// provisioned while MagicDNS was down can carry a bogus host like
// "undefined.<tailnet>.ts.net"; printing it as "trusted" just misleads.
const selfHost = getSelfHost();
if (selfHost && !/^(undefined|null)\b/i.test(selfHost)) {
console.log(` ✅ https://${selfHost}:${port} (remote via Tailscale, trusted)`);
}
console.log(` 🔐 https://<tailscale-ip>:${port} (remote via Tailscale; cert warning unless using the hostname above)`);
initCertRenewer(httpServer);
if (localHttpServer) {
io.attach(localHttpServer);
localHttpServer.listen(localHttpPort, '127.0.0.1');
localHttpServer.on('error', (err) => {
console.warn(`⚠️ Loopback HTTP mirror on :${localHttpPort} failed: ${err.message} — http://localhost:${localHttpPort} won't work; use https://...:${port}`, err.stack || '');
});
}
};
/**
* Post-route boot. Kicks off the background services, then walks the ordered
* boot chain that ends in `httpServer.listen()` — the walk itself is
* `runPostRouteSequence` in bootstrapSequence.js, which owns which of these
* steps are awaited and which are fire-and-forget. Returns the chain's promise
* so a caller can await it; index.js intentionally does not (boot proceeds in
* the background and any fatal step exits the process itself).
*/
export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled, port, host, spawnerReady }) =>
runPostRouteSequence(gateStepsForSmokeBoot({
isStopping: isHostShuttingDown,
startBackgroundServices: () => startBackgroundServices({ spawnerReady, io, httpsEnabled, port }),
// Instance identity + sync log come up before requests are accepted, so a
// brain mutation can't arrive before the sync log is ready.
ensureSelf,
initSyncLog,
// Recover inbox classifications and repository clones interrupted by a
// previous crash. Must run AFTER initSyncLog() because both recovery paths
// append to the brain sync log — running them before currentSeq is loaded
// would mint colliding seqs and corrupt peer cursors.
recoverStuckClassifications,
recoverInterruptedRepoClones,
// Awaited by the sequence so data/ exists and the worker loop is running
// before /api/video-gen or /api/image-gen can enqueue (otherwise persist()
// can race with ensureDir). A failure here is fatal: the queue owns
// persistence + SSE + temp-file cleanup, and accepting requests with a
// half-init queue silently corrupts state.
initMediaJobQueue,
// LoRA run records reconcile against the live queue (interrupted runs →
// failed) and mirror queue-side cancels. Own step so the smoke boot can
// no-op it: the reconcile needs a database the hermetic root does not have.
initLoraTraining,
initMediaJobDependentHooks,
resumeMediaJobCompletions,
// Sharing: attach chokidar watchers to every registered share bucket so
// incoming manifests from peers are picked up live. Backlog processing
// (manifests that arrived while the server was offline) runs as part of
// initSharing.
initSharing: () => initSharing({ io }),
// Resume any Creative Director projects that were mid-flight when the server
// died. The queue reload above just reclassified their renders as 'failed
// (interrupted by restart)'; this nudges the orchestrator so projects don't
// sit frozen waiting for listeners that no longer exist.
// recoverInFlightProjects resolves cdRecoveryDone on success. On any failure
// path here, explicitly resolve it so cos.start's gate doesn't hit the 60s
// timeout fallback for nothing.
recoverCreativeDirectorProjects: () => recoverInFlightProjects().catch(async (e) => {
logBootstrapFailure('⚠️ CD boot recovery failed', e, console.log);
const { markRecoveryDone } = await import('./creativeDirector/recovery.js');
markRecoveryDone();
}),
runDatabasePhase: async () => {
await runDatabaseBootPhase();
// Brain→memory resyncs a previous process queued but never finished. Runs
// after initSyncLog and the DB phase (the memory index needs both), in the
// background so a large backlog can't delay accepting requests.
replayBrainMemoryResync().catch((err) => logBootstrapFailure('❌ Brain memory resync replay failed', err));
const { bindPeerExecutionIo, initializePeerExecution } = await import('./peerExecutionRuntime.js');
bindPeerExecutionIo(io);
await initializePeerExecution().catch(err => console.error(`❌ Peer execution recovery remains held: ${err.message}`));
},
// One-time series cover-thumbnail backfill: derive `series.coverImage` (the
// rendered volume/issue cover shown on the pipeline list) for series whose
// covers rendered before the feature shipped. Runs after the series + issues
// stores are warmed above so the derivation reads migrated records. Drives
// the services, so it works on both the PG backend and the file escape hatch.
// Marker-gated, so it runs at most once regardless. Loading it is a
// separate step from running it: the sequence awaits the load (a script
// that won't load is a real breakage) but never the backfill itself.
loadSeriesCoverBackfill: async () => (await import('../scripts/backfillSeriesCoverImages.js')).backfillSeriesCoverImages,
startListening: () => httpServer.listen(port, host, () => runPostListenSequence(gateStepsForSmokeBoot({
isStopping: isHostShuttingDown,
announceListening: () => announceListening({ io, httpServer, localHttpServer, httpsEnabled, port }),
// Process-level safety net, wired with the io instance so unhandled
// failures also surface in the UI.
setupProcessErrorHandlers: () => setupProcessErrorHandlers(io),
backfillOriginInstanceId,
// Relay any brain record whose current state never reached the sync log
// (a crash or failed append on a previous run) — #8351.
relayUnloggedBrainRecords: relayUnloggedRecords,
startPolling: () => {
startPolling();
// Best-effort: re-attach tailcat forwards persisted across restarts.
void import('./tailcatPeer.js')
.then(({ restoreForwards }) => restoreForwards())
.catch((err) => logBootstrapFailure('⚠️ tailcat forward restore failed', err, console.warn));
void import('./tailcatServe.js')
.then(({ restoreServe }) => restoreServe())
.catch((err) => logBootstrapFailure('⚠️ tailcat serve restore failed', err, console.warn));
},
initSyncOrchestrator,
// Brain digests/reviews call an AI provider, so the scheduler arms only
// once the server is listening and only acts when the user opted in
// (`scheduledReviewsEnabled`).
startBrainScheduler
}, 'postListen', isSmokeBoot())))
}, 'postRoute', isSmokeBoot()));
// Run an async close but resolve anyway after `ms` — so a close that never
// settles (e.g. a WebSocket-upgraded socket the server no longer tracks, or a
// leaked DB client) can't hang shutdown; process.exit() reclaims the resources at
// the OS level. Both callbacks passed to run resolve the same settle-once
// promise: finish logs success, finishWithError logs an error (and its stack)
// and continues.
// The backstop is .unref()'d so it never keeps the event loop alive on its own.
const withGrace = (label, ms, run) => new Promise((resolve) => {
let settled = false;
const complete = (msg, log, error) => {
if (settled) return;
settled = true;
if (msg) {
if (error) logBootstrapFailure(msg, error, log);
else log(msg);
}
resolve();
};
const finish = (msg) => complete(msg, console.log);
const finishWithError = (msg, error) => complete(msg, console.error, error);
run({ finish, finishWithError });
// Expected on most restarts (lingering upgraded sockets); the bounded wait is the handling.
setTimeout(() => finish(`⏱️ ${label} close exceeded ${ms}ms — proceeding`), ms).unref?.();
});
// graceMs is deliberately bounded: this runs after the request drain's deadline,
// and closeAllConnections() force-drops every connection still open, so there is
// no graceful drain left to wait for. The close callback
// still needs a short window for TLS/socket teardown — the only thing that can
// outlast it is a WebSocket-upgraded socket the server no longer tracks (and
// io.close()'s engine.close() already tore those down protocol-side; the OS reaps
// the TCP remnant on process.exit).
// ERR_SERVER_NOT_RUNNING means it was already closed (io.close() closes whichever
// server is its current this.httpServer) — success for us, not a failure.
const closeServer = (server, label, graceMs = 1000) => withGrace(label, graceMs, ({ finish, finishWithError }) => {
if (!server) return finish();
server.close((err) => {
if (err && err.code !== 'ERR_SERVER_NOT_RUNNING') finishWithError(`⚠️ Error closing ${label}`, err);
else finish(`✅ ${label} closed`);
});
// Order matters: close() above stops accepting NEW connections (the drain has
// normally done that already); NOW force-drop whatever outlived the drain
// window. (Node 18.2+.)
server.closeAllConnections?.();
});
// Hard ceiling on graceful shutdown: if Socket.IO/HTTP don't close within this
// window, force-exit so PM2 isn't left waiting on a hung process.
const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 10000;
// How long an already-accepted ordinary API request may keep running after the
// signal (#8323). Measured from the signal, so it overlaps the teardown steps
// before the HTTP close rather than adding to them: the drain (5s) + HTTP close
// grace (1s) + peer sync stop (1s) + DB pool grace (2.5s) stays under the ceiling above.
const HTTP_DRAIN_WINDOW_MS = 5000;
// Bound on stopping peer sync and settling its in-flight pushes before the DB
// pool closes; drain + HTTP close + this + DB pool grace stay under the ceiling.
const PEER_SYNC_STOP_GRACE_MS = 1000;
// A shutdown at or above this logs one numbers-only line naming where the time
// went (#12853). A step's loop lag is time the event loop could not run timers —
// a synchronous block — rather than asynchronous waiting.
const SLOW_SHUTDOWN_MS = 1000;
const describeSlowShutdown = (summary, settledMs) => {
if (summary.totalMs < SLOW_SHUTDOWN_MS) return null;
const phases = Object.entries(summary.phases);
const [slowest] = phases.reduce((max, entry) => (entry[1].ms > max[1].ms ? entry : max), phases[0]);
const steps = phases.map(([name, { ms, lagMs }]) => `${name}Ms=${ms} ${name}LagMs=${lagMs}`);
const settled = Object.entries(settledMs).map(([name, ms]) => `${name}SettledMs=${ms}`);
return `🐢 Graceful shutdown slow totalMs=${summary.totalMs} slowestStep=${slowest} loopLagMs=${summary.lagMs} loopLagMaxMs=${summary.lagMaxMs} ${[...steps, ...settled].join(' ')}`;
};
/**
* Wire SIGTERM/SIGINT to the graceful-shutdown state machine. Idempotent per
* signal: once shutdown starts, later signals are ignored.
*/
export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) => {
// Track in-flight requests from boot so shutdown knows what it is waiting on.
const httpDrain = createHttpDrain([httpServer, localHttpServer]);
let shuttingDown = false;
const shutdown = async (signal) => {
if (shuttingDown) return;
shuttingDown = true;
// Latch the host-shutdown flag BEFORE anything can await. pm2's TreeKill
// signals the whole descendant tree, so a server-owned agent PTY can exit
// microseconds from now — and its exit handler must already know the PTY
// died because PortOS is going down, not because the agent finished (#3202).
markHostShuttingDown();
// Stop request intake on BOTH listeners before the first await, so nothing
// new is accepted while the teardown below runs; requests already accepted
// keep running and are waited on (bounded) just before the HTTP close.
const requestsDrained = httpDrain.begin(HTTP_DRAIN_WINDOW_MS);
// Stop media-job dispatch before the first await too (#8691): a lane freed
// during teardown must not promote a waiting job the process is about to
// abandon — the next boot fails a running record but restores a queued one.
quiesceMediaJobQueue();
quiesceScheduledEvents();
// Start alongside the request drain so a stuck background handler spends
// no additional shutdown budget. The DB remains open until both settle.
const scheduledEventsDrained = withGrace('Scheduled events', HTTP_DRAIN_WINDOW_MS, ({ finish, finishWithError }) =>
drainScheduledEvents().then(
() => finish('✅ Scheduled events drained'),
(err) => finishWithError('⚠️ Scheduled events drain failed', err),
));
const timing = createEventLoopLagProbe().begin('tailcat');
// Concurrent steps settle out of phase order; recorded as ms since the signal.
const settledMs = {};
const recordSettled = (name) => () => { settledMs[name] = timing.snapshot().totalMs; };
const reportIfSlow = () => {
const line = describeSlowShutdown(timing.end(), settledMs);
if (line) console.log(line);
};
// Diagnostic context for the shutdown trigger. ppid tells us whether the
// signal came from PM2 (parent is the PM2 god process), a TTY (parent is
// the user's shell), or some external orchestrator. pm_* env vars are set
// by PM2 so their presence + a matching ppid is the smoking gun.
const pid = process.pid;
const ppid = process.ppid;
const tty = process.stdin.isTTY ? 'tty' : 'no-tty';
const pmId = process.env.pm_id ?? process.env.PM2_ID ?? '<not set>';
const pmExecPath = process.env.pm_exec_path ?? '<not set>';
console.log(`🛑 Received ${signal} - shutting down gracefully (pid=${pid} ppid=${ppid} ${tty} pm_id=${pmId})`);
if (pmExecPath !== '<not set>') console.log(` ↳ launched by PM2: pm_exec_path=${pmExecPath}`);
const forceExitTimer = setTimeout(() => {
console.error('⚠️ Graceful shutdown timed out, forcing exit');
reportIfSlow();
process.exit(1);
}, GRACEFUL_SHUTDOWN_TIMEOUT_MS);
// Don't let the safety timer itself keep the event loop alive — if every
// other handle has closed we should exit immediately, not wait out the timer.
forceExitTimer.unref?.();
// Record which agents this process owned, so the NEXT boot's orphan sweep can
// tell "PortOS was restarted out from under a healthy agent" apart from "the
// agent's own process died" and skip the retry-budget/cooldown penalty for a
// fault the agent didn't cause (#3202). Include dispatches whose durable
// registration exists but whose process handoff has not finished: shutdown
// destroys that setup too. Exclude `runnerAgents`; those handed-off runs
// survive this restart and are re-adopted on the next boot.
//
// Note `activeAgents` is NOT "direct-spawn only": runner-launched TUI agents
// register there too (`agentTuiSpawning.js`, both launch modes). The
// distinction that matters here is process ownership, not spawn mode — do
// not re-derive the snapshot from "was this spawned via the runner".
//
// Started here so the agent snapshot is taken before anything can await, but
// awaited just before exit so a stalled filesystem spends none of the graceful
// budget ahead of the teardown below. Best-effort throughout: the helper logs
// both outcomes and never rejects (hence `finish()` with no message), the
// `.catch` is the belt-and-suspenders for an out-of-lifecycle rejection, and a
// marker we fail to write only degrades recovery to the pre-existing orphan path.
const markerWritten = withGrace('Host-shutdown marker', 1500, ({ finish, finishWithError }) =>
writeHostShutdownMarker({ agentIds: [...activeAgents.keys(), ...dispatchingAgentIds], signal })
.then(() => finish(), (err) => finishWithError('⚠️ Host-shutdown marker failed', err)));
markerWritten.then(recordSettled('marker'));
// No teardown await may delay the hard deadline, recovery snapshot, or
// disarming timers that could launch work while shutdown is suspended.
stopIdleReaper();
stopHostedSessionSweep();
stopBeeperSocket();
await import('./tailcatPeer.js').then(({ stopAllForwards }) => stopAllForwards())
.catch((err) => logBootstrapFailure('❌ Tailcat forward shutdown failed', err));
await import('./tailcatServe.js').then(({ stopServeProcess }) => stopServeProcess())
.catch((err) => logBootstrapFailure('❌ Tailcat serve shutdown failed', err));
timing.mark('modelHost');
// Usage persistence is best effort; a stalled ledger write must leave
// time for the media queue and DB to settle inside the shutdown ceiling.
await withGrace('Dedicated model host', 1000, ({ finish, finishWithError }) =>
import('./fleetLlmHost.js').then(({ stopFleetLlmHost }) => stopFleetLlmHost()).then(
() => finish(),
(err) => finishWithError('❌ Dedicated model host shutdown failed', err),
));
// Reach every child spawned into its own process group — agent runs, CLI
// runs and vision calls alike. A credential-bootstrap-wrapped child is
// spawned `detached: true` so stop/timeout/cancel can signal its whole
// group (#7496); that also moves it OUT of this server's process group, and
// the one teardown route it thereby loses is a signal aimed at THAT group —
// Ctrl-C at an `npm start` terminal, or `kill -<pgid>`. (pm2's TreeKill
// walks the pid TREE, so the managed path is unaffected, and every child
// already survives a plain SIGTERM to the parent.) This restores exactly
// that lost reach rather than orphaning a harness holding a freshly minted
// credential. Before the teardown below, while the process still owns its
// state; best-effort, like everything else in this window.
const detachedGroups = signalDetachedGroups('SIGTERM', logBootstrapFailure);
if (detachedGroups) console.log(`🛑 Signalled ${detachedGroups} detached process group(s)`);
// Terminate the Codex app-server child before the socket teardown below.
// pm2's TreeKill would reap it anyway, but a direct SIGTERM keeps a manual
// `kill` of the server from orphaning a Codex process holding the user's
// sign-in, and it is a no-op when nothing was ever spawned.
timing.mark('codex');
await withGrace('Codex app-server', 2000, ({ finish, finishWithError }) =>
stopCodexAppServer().then(
() => finish(),
(err) => finishWithError('⚠️ Codex app-server stop failed', err),
));
// socket.io's io.close() closes engine.io AND its current this.httpServer — and
// every io.attach() reassigns this.httpServer (socket.io index.js:303), so with
// HTTPS on (io.attach(localHttpServer) at boot) io.close() closes the *mirror*,
// not the primary :5555 server. The historical bug was calling close() a second
// time on that already-closed server: Node registers the callback as a one-time
// 'close' listener for an event that already fired, so it never runs and shutdown
// hangs forever — the real cause of the reconcile "stopping apps" hang.
// The drain already stopped that server, so io.close() reporting
// ERR_SERVER_NOT_RUNNING is the expected outcome, not a failure. Its callback
// still waits for that server's connections to end, so while a slow request
// is draining this grace can lapse first — Socket.IO itself is torn down by
// then, and the drain wait below bounds the rest.
timing.mark('socketIo');
await withGrace('Socket.IO', 3000, ({ finish, finishWithError }) =>
io.close((err) => {
if (err && err.code !== 'ERR_SERVER_NOT_RUNNING') finishWithError('⚠️ Error closing Socket.IO', err);
else finish('✅ Socket.IO closed');
}));
timing.mark('ingress');
await import('./tailcatIngress.js').then(({ stopTailcatIngress }) => stopTailcatIngress())
.catch((err) => logBootstrapFailure('❌ Tailcat ingress shutdown failed', err));
// Let accepted requests finish (bounded by the drain window), then force the
// rest closed below — a handler mid-write loses its response only at the deadline.
timing.mark('requestDrain');
const { remaining } = await requestsDrained;
// The drain window is bounded on purpose, so closing what outlives it is the handling.
if (remaining > 0) console.log(`⏱️ ${remaining} request(s) still running after the ${HTTP_DRAIN_WINDOW_MS}ms drain — closing them`);
// Close BOTH servers explicitly. The drain already stopped both listeners
// (ERR_SERVER_NOT_RUNNING → treated as success by closeServer), and
// the bounded backstop in closeServer guarantees neither can hang shutdown even
// if a future socket.io version changes which server it owns.
// Upgraded (WebSocket) sockets are invisible to closeAllConnections() but
// keep server.close() pending; Socket.IO has already sent its close frames.
httpDrain.destroyUpgraded();
timing.mark('httpClose');
await Promise.all([
closeServer(httpServer, 'HTTP server'),
closeServer(localHttpServer, 'Local HTTP mirror')
]);
// An IdeaLoom vault export still inside its 5s debounce would otherwise be
// lost. A local file write, so it overlaps the DB close below inside that
// close's budget rather than adding to the shutdown ceiling; no request can
// schedule another export now that the servers are closed.
const ideaLoomFlushed = withGrace('IdeaLoom auto-sync flush', 2500, ({ finish, finishWithError }) =>
flushIdeaLoomAutoSync().then(() => finish('✅ IdeaLoom auto-sync flushed'), (err) => finishWithError('⚠️ IdeaLoom auto-sync flush failed', err)));
ideaLoomFlushed.then(recordSettled('ideaLoom'));
// Flush the media-job queue now that no request can admit another job
// (#8325): terminal transitions and the latest progress reach the snapshot a
// restart restores from. Not earlier: quiescing stops dispatch, not enqueue.
// Started ahead of the sharing and DB steps, so its write is already in
// flight when they run (#12853); awaited before exit and bounded by its own
// timeout so it spends no budget beyond the DB close's.
const mediaQueueFlushed = flushMediaJobQueue().then((outcome) => {
recordSettled('mediaFlush')();
if (outcome.ok) console.log('✅ Media job queue flushed');
else console.error(`❌ Media job queue flush failed: ${outcome.error}`);
});
// Stop peer sync before the pool ends: a peer:online convergence walk still
// running would otherwise log one pool-ended stack per remaining subscription.
// Abandoning it is safe — the next boot's peer:online pass repeats it (#11750).
// The exit form leaves share-bucket watchers open: closing them blocks the
// main thread for seconds on macOS, and process.exit releases them (#12853).
timing.mark('peerSync');
await withGrace('Peer sync', PEER_SYNC_STOP_GRACE_MS, ({ finish, finishWithError }) =>
Promise.resolve().then(quiesceSharingForExit).then(() => drainPeerSyncOperations()).then(
() => finish(),
(err) => finishWithError('⚠️ Peer sync shutdown failed', err),
));
await scheduledEventsDrained;
timing.mark('dbPool');
const { close } = await import('../lib/db.js');
if (typeof close === 'function') {
// Bound the DB pool close: pool.end() waits for every checked-out client to
// be released, so one hung/leaked connection (e.g. a LISTEN channel) would
// otherwise stall shutdown until the force-exit timer.
await withGrace('DB pool', 2500, ({ finish, finishWithError }) =>
close().then(() => finish('✅ DB pool closed'), (err) => finishWithError('⚠️ DB pool close failed', err)));
} else {
console.warn('ℹ️ DB pool close not available; skipping DB shutdown');
}
timing.mark('flushWait');
await markerWritten;
await mediaQueueFlushed;
await ideaLoomFlushed;
clearTimeout(forceExitTimer);
reportIfSlow();
process.exit(0);
};
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGINT', () => shutdown('SIGINT'));
};