Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: skip the LoRA boot reconcile during hermetic smoke (#12151)

[._.]/ Adam Eivy9e635f4137ad
diff --git a/scripts/smoke-boot.js b/scripts/smoke-boot.js
index 35cbb737bf..709d67d913 100644
--- a/scripts/smoke-boot.js
+++ b/scripts/smoke-boot.js
@@ -20,8 +20,9 @@
// - a test-only database name, so even a reachable Postgres is never the
// install's real one;
// - PORTOS_SMOKE_BOOT=1, which makes boot skip schedulers, agent
-// spawning/recovery, crash-recovery passes, peer polling/sync, and other
-// external integrations (SMOKE_BOOT_DISABLED_STEPS in bootstrapSequence.js).
+// spawning/recovery, crash-recovery passes (including the LoRA run
+// reconcile), peer polling/sync, and other external integrations
+// (SMOKE_BOOT_DISABLED_STEPS in bootstrapSequence.js).
// The root is removed on every exit path, success or failure.
import { spawn } from 'child_process';
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index 6fddc1ebe7..63d8170d55 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -675,10 +675,6 @@ const startBackgroundServices = ({ spawnerReady, io, httpsEnabled, port }) => {
* persisted jobs (otherwise they'd miss `completed` events for reloaded jobs).
*/
const initMediaJobDependentHooks = () => {
- // LoRA training run records reconcile against the live queue (interrupted
- // runs → failed) and mirror queue-side cancels — must run after the queue
- // has loaded its persisted jobs.
- initLoraTraining().catch(err => logBootstrapFailure('❌ loraTraining init failed', err));
// Universe Builder needs the media job queue running before it can listen
// for `completed` events — so initialize the hook here.
initUniverseBuilderCollectionHook();
@@ -963,6 +959,10 @@ export const runBootSequence = ({ io, httpServer, localHttpServer, httpsEnabled,
// persistence + SSE + temp-file cleanup, and accepting requests with a
// half-init queue silently corrupts state.
initMediaJobQueue,
+ // LoRA run records reconcile against the live queue (interrupted runs →
+ // failed) and mirror queue-side cancels. Own step so the smoke boot can
+ // no-op it: the reconcile needs a database the hermetic root does not have.
+ initLoraTraining,
initMediaJobDependentHooks,
// Sharing: attach chokidar watchers to every registered share bucket so
diff --git a/server/services/bootstrapSequence.js b/server/services/bootstrapSequence.js
index 40d236ba0b..872d9f40d6 100644
--- a/server/services/bootstrapSequence.js
+++ b/server/services/bootstrapSequence.js
@@ -47,6 +47,9 @@ export const SMOKE_BOOT_DISABLED_STEPS = Object.freeze({
'startBackgroundServices',
'recoverStuckClassifications',
'recoverInterruptedRepoClones',
+ // Interrupted-run reconcile needs the real database. A hermetic smoke names
+ // a database that does not exist; skipping it keeps that miss off stderr.
+ 'initLoraTraining',
'initSharing',
'recoverCreativeDirectorProjects'
]),
@@ -353,9 +356,12 @@ export const runDatabasePhase = async ({ gate, migrate, warmStores, reconcilePro
* Interrupted clone recovery is awaited so the first links request cannot
* observe an orphaned `cloning` state; inbox recovery remains best-effort
* in the background because nothing below reads its result.
- * 4. `initMediaJobQueue` awaited before `initMediaJobDependentHooks` — the
- * hooks must be listening before the queue replays `completed` events for
- * reloaded jobs, and before a route can enqueue against a half-init queue.
+ * 4. `initMediaJobQueue` awaited before `initLoraTraining` and
+ * `initMediaJobDependentHooks` — the reconcile and the hooks read the
+ * reloaded queue, and no route may enqueue against a half-init queue.
+ * The lora reconcile is best-effort (a down database must not fail boot)
+ * and smoke-disabled. The other hooks must be listening before the queue
+ * replays `completed` events for reloaded jobs.
* 5. The database phase before `startListening`, so no request can land on a
* partially-migrated install.
* 6. The series-cover backfill MODULE is awaited but the backfill RUN is not:
@@ -372,6 +378,7 @@ export const runPostRouteSequence = ({
recoverStuckClassifications,
recoverInterruptedRepoClones,
initMediaJobQueue,
+ initLoraTraining,
initMediaJobDependentHooks,
initSharing,
recoverCreativeDirectorProjects,
@@ -390,6 +397,7 @@ export const runPostRouteSequence = ({
return bestEffort(recoverInterruptedRepoClones(), logFailure('Brain clone recovery failed'));
})
.then(() => initMediaJobQueue())
+ .then(() => bestEffort(initLoraTraining(), logFailure('loraTraining init failed')))
.then(() => initMediaJobDependentHooks())
.then(() => {
// Sharing attaches chokidar watchers + drains the offline manifest
diff --git a/server/services/bootstrapSequence.test.js b/server/services/bootstrapSequence.test.js
index 57e5efedd0..c1fbaf6ad4 100644
--- a/server/services/bootstrapSequence.test.js
+++ b/server/services/bootstrapSequence.test.js
@@ -29,7 +29,8 @@ import {
runDatabasePhase,
runPostRouteSequence,
runPostListenSequence,
- gateStepsForSmokeBoot
+ gateStepsForSmokeBoot,
+ SMOKE_BOOT_DISABLED_STEPS
} from './bootstrapSequence.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -660,6 +661,7 @@ describe('runPostRouteSequence — post-route boot order', () => {
recoverStuckClassifications: step('recoverStuckClassifications', () => Promise.resolve()),
recoverInterruptedRepoClones: step('recoverInterruptedRepoClones', () => Promise.resolve()),
initMediaJobQueue: step('initMediaJobQueue', () => Promise.resolve()),
+ initLoraTraining: step('initLoraTraining', () => Promise.resolve()),
initMediaJobDependentHooks: step('initMediaJobDependentHooks'),
initSharing: step('initSharing', () => Promise.resolve()),
recoverCreativeDirectorProjects: step('recoverCreativeDirectorProjects', () => Promise.resolve()),
@@ -683,6 +685,7 @@ describe('runPostRouteSequence — post-route boot order', () => {
'recoverStuckClassifications',
'recoverInterruptedRepoClones',
'initMediaJobQueue',
+ 'initLoraTraining',
'initMediaJobDependentHooks',
'initSharing',
'recoverCreativeDirectorProjects',
@@ -698,6 +701,7 @@ describe('runPostRouteSequence — post-route boot order', () => {
const recorder = createRecorder();
const deps = buildDeps(recorder);
await runPostRouteSequence(gateStepsForSmokeBoot(deps, 'postRoute', true));
+ expect(SMOKE_BOOT_DISABLED_STEPS.postRoute).toContain('initLoraTraining');
expect(recorder.calls).toEqual([
'ensureSelf',
'initSyncLog',
@@ -708,6 +712,7 @@ describe('runPostRouteSequence — post-route boot order', () => {
'backfillSeriesCoverImages',
'startListening'
]);
+ expect(recorder.calls).not.toContain('initLoraTraining');
expect(deps.onFatal).not.toHaveBeenCalled();
});
diff --git a/server/services/loraTraining/index.js b/server/services/loraTraining/index.js
index 87db96ed6c..1255d0d759 100644
--- a/server/services/loraTraining/index.js
+++ b/server/services/loraTraining/index.js
@@ -1306,8 +1306,21 @@ export function hasSurvivingTrainer(runId) {
* queued/running, so the `!job || terminal` guard below skips it and the reap
* path only touches runs that genuinely died.
*/
+/** Postgres never accepted the session: missing database or no listener. */
+const loraDatabaseUnreachable = (err) => {
+ const code = err?.code;
+ if (code === '3D000' || code === 'ECONNREFUSED' || code === 'ENOTFOUND' || code === 'ETIMEDOUT') return true;
+ return /database ".+" does not exist/i.test(String(err?.message ?? ''));
+};
+
export async function initLoraTraining() {
const active = await runsDb.listActiveRuns().catch((err) => {
+ // Unreachable database is a skip (hermetic smoke, Postgres down). A query
+ // that reached the database and then failed is a reconcile failure.
+ if (loraDatabaseUnreachable(err)) {
+ console.warn(`⚠️ loraTraining boot reconcile skipped: ${err?.message}`);
+ return [];
+ }
console.error(`❌ loraTraining boot reconcile failed: ${err?.message}`);
return [];
});