Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: skip the LoRA boot reconcile during hermetic smoke (#12151)
[._.]/ Adam Eivy9e635f4137ad
diff --git a/scripts/smoke-boot.js b/scripts/smoke-boot.jsindex 35cbb737bf..709d67d913 100644--- a/scripts/smoke-boot.js+++ b/scripts/smoke-boot.js@@ -20,8 +20,9 @@// - a test-only database name, so even a reachable Postgres is never the// install's real one;// - PORTOS_SMOKE_BOOT=1, which makes boot skip schedulers, agent-// spawning/recovery, crash-recovery passes, peer polling/sync, and other-// external integrations (SMOKE_BOOT_DISABLED_STEPS in bootstrapSequence.js).+// spawning/recovery, crash-recovery passes (including the LoRA run+// reconcile), peer polling/sync, and other external integrations+// (SMOKE_BOOT_DISABLED_STEPS in bootstrapSequence.js).// The root is removed on every exit path, success or failure.import { spawn } from 'child_process';diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex 6fddc1ebe7..63d8170d55 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -675,10 +675,6 @@ const startBackgroundServices = ({ spawnerReady, io, httpsEnabled, port }) => {* persisted jobs (otherwise they'd miss `completed` events for reloaded jobs).*/const initMediaJobDependentHooks = () => {- // LoRA training run records reconcile against the live queue (interrupted- // runs → failed) and mirror queue-side cancels — must run after the queue- // has loaded its persisted jobs.- initLoraTraining().catch(err => logBootstrapFailure('❌ loraTraining init failed', err));// Universe Builder needs the media job queue running before it can listen// for `completed` events — so initialize the hook here.initUniverseBuilderCollectionHook();@@ -963,6 +959,10 @@ export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled,// persistence + SSE + temp-file cleanup, and accepting requests with a// half-init queue silently corrupts state.initMediaJobQueue,+ // LoRA run records reconcile against the live queue (interrupted runs →+ // failed) and mirror queue-side cancels. Own step so the smoke boot can+ // no-op it: the reconcile needs a database the hermetic root does not have.+ initLoraTraining,initMediaJobDependentHooks,// Sharing: attach chokidar watchers to every registered share bucket sodiff --git a/server/services/bootstrapSequence.js b/server/services/bootstrapSequence.jsindex 40d236ba0b..872d9f40d6 100644--- a/server/services/bootstrapSequence.js+++ b/server/services/bootstrapSequence.js@@ -47,6 +47,9 @@ export const SMOKE_BOOT_DISABLED_STEPS = Object.freeze({'startBackgroundServices','recoverStuckClassifications','recoverInterruptedRepoClones',+ // Interrupted-run reconcile needs the real database. A hermetic smoke names+ // a database that does not exist; skipping it keeps that miss off stderr.+ 'initLoraTraining','initSharing','recoverCreativeDirectorProjects']),@@ -353,9 +356,12 @@ export const runDatabasePhase = async ({ gate, migrate, warmStores, reconcilePro* Interrupted clone recovery is awaited so the first links request cannot* observe an orphaned `cloning` state; inbox recovery remains best-effort* in the background because nothing below reads its result.- * 4. `initMediaJobQueue` awaited before `initMediaJobDependentHooks` — the- * hooks must be listening before the queue replays `completed` events for- * reloaded jobs, and before a route can enqueue against a half-init queue.+ * 4. `initMediaJobQueue` awaited before `initLoraTraining` and+ * `initMediaJobDependentHooks` — the reconcile and the hooks read the+ * reloaded queue, and no route may enqueue against a half-init queue.+ * The lora reconcile is best-effort (a down database must not fail boot)+ * and smoke-disabled. The other hooks must be listening before the queue+ * replays `completed` events for reloaded jobs.* 5. The database phase before `startListening`, so no request can land on a* partially-migrated install.* 6. The series-cover backfill MODULE is awaited but the backfill RUN is not:@@ -372,6 +378,7 @@ export const runPostRouteSequence = ({recoverStuckClassifications,recoverInterruptedRepoClones,initMediaJobQueue,+ initLoraTraining,initMediaJobDependentHooks,initSharing,recoverCreativeDirectorProjects,@@ -390,6 +397,7 @@ export const runPostRouteSequence = ({return bestEffort(recoverInterruptedRepoClones(), logFailure('Brain clone recovery failed'));}).then(() => initMediaJobQueue())+ .then(() => bestEffort(initLoraTraining(), logFailure('loraTraining init failed'))).then(() => initMediaJobDependentHooks()).then(() => {// Sharing attaches chokidar watchers + drains the offline manifestdiff --git a/server/services/bootstrapSequence.test.js b/server/services/bootstrapSequence.test.jsindex 57e5efedd0..c1fbaf6ad4 100644--- a/server/services/bootstrapSequence.test.js+++ b/server/services/bootstrapSequence.test.js@@ -29,7 +29,8 @@ import {runDatabasePhase,runPostRouteSequence,runPostListenSequence,- gateStepsForSmokeBoot+ gateStepsForSmokeBoot,+ SMOKE_BOOT_DISABLED_STEPS} from './bootstrapSequence.js';const __dirname = dirname(fileURLToPath(import.meta.url));@@ -660,6 +661,7 @@ describe('runPostRouteSequence — post-route boot order', () => {recoverStuckClassifications: step('recoverStuckClassifications', () => Promise.resolve()),recoverInterruptedRepoClones: step('recoverInterruptedRepoClones', () => Promise.resolve()),initMediaJobQueue: step('initMediaJobQueue', () => Promise.resolve()),+ initLoraTraining: step('initLoraTraining', () => Promise.resolve()),initMediaJobDependentHooks: step('initMediaJobDependentHooks'),initSharing: step('initSharing', () => Promise.resolve()),recoverCreativeDirectorProjects: step('recoverCreativeDirectorProjects', () => Promise.resolve()),@@ -683,6 +685,7 @@ describe('runPostRouteSequence — post-route boot order', () => {'recoverStuckClassifications','recoverInterruptedRepoClones','initMediaJobQueue',+ 'initLoraTraining','initMediaJobDependentHooks','initSharing','recoverCreativeDirectorProjects',@@ -698,6 +701,7 @@ describe('runPostRouteSequence — post-route boot order', () => {const recorder = createRecorder();const deps = buildDeps(recorder);await runPostRouteSequence(gateStepsForSmokeBoot(deps, 'postRoute', true));+ expect(SMOKE_BOOT_DISABLED_STEPS.postRoute).toContain('initLoraTraining');expect(recorder.calls).toEqual(['ensureSelf','initSyncLog',@@ -708,6 +712,7 @@ describe('runPostRouteSequence — post-route boot order', () => {'backfillSeriesCoverImages','startListening']);+ expect(recorder.calls).not.toContain('initLoraTraining');expect(deps.onFatal).not.toHaveBeenCalled();});diff --git a/server/services/loraTraining/index.js b/server/services/loraTraining/index.jsindex 87db96ed6c..1255d0d759 100644--- a/server/services/loraTraining/index.js+++ b/server/services/loraTraining/index.js@@ -1306,8 +1306,21 @@ export function hasSurvivingTrainer(runId) {* queued/running, so the `!job || terminal` guard below skips it and the reap* path only touches runs that genuinely died.*/+/** Postgres never accepted the session: missing database or no listener. */+const loraDatabaseUnreachable = (err) => {+ const code = err?.code;+ if (code === '3D000' || code === 'ECONNREFUSED' || code === 'ENOTFOUND' || code === 'ETIMEDOUT') return true;+ return /database ".+" does not exist/i.test(String(err?.message ?? ''));+};+export async function initLoraTraining() {const active = await runsDb.listActiveRuns().catch((err) => {+ // Unreachable database is a skip (hermetic smoke, Postgres down). A query+ // that reached the database and then failed is a reconcile failure.+ if (loraDatabaseUnreachable(err)) {+ console.warn(`⚠️ loraTraining boot reconcile skipped: ${err?.message}`);+ return [];+ }console.error(`❌ loraTraining boot reconcile failed: ${err?.message}`);return [];});