Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: reclaim abandoned Ollama import staging once nothing holds it (#13124)
Adam Eivy35bb31b86f12
diff --git a/server/services/ollamaManager.js b/server/services/ollamaManager.jsindex dbeeff77a9..7542913b93 100644--- a/server/services/ollamaManager.js+++ b/server/services/ollamaManager.js@@ -43,7 +43,7 @@ import { isSafeHfRepoRelativePath } from '../lib/hfCache.js'import { assessDownloadPreflight, diskInsufficientError, DOWNLOAD_VERDICTS } from '../lib/downloadPreflight.js'import {awaitStagingRelease, blobStoreSignature, createImportStaging, findImportStaging, removeImportStaging,- runSupervisedOllamaCreate, stagedFilesComplete, updateImportStaging+ runSupervisedOllamaCreate, stagedFilesComplete, sweepImportStaging, updateImportStaging} from './ollamaSafetensorsImport.js'const execFileAsync = promisify(execFile)@@ -1637,6 +1637,8 @@ async function runSafetensorsImport({ modelId, repo, subdir, files, headers, onP// A staging directory from an earlier attempt (a stalled create, or a// PortOS restart mid-import) is reused when it holds the whole checkpoint;// an incomplete one is retired only once nothing holds it open.+ await sweepImportStaging({ modelId, activeModelIds: activeImports })+ .catch((err) => console.error(`❌ Safetensors staging sweep failed: ${err.message}`))const retained = await findImportStaging(modelId)if (retained) {const complete = await stagedFilesComplete(join(retained.tempRoot, 'model'), files)diff --git a/server/services/ollamaManager.test.js b/server/services/ollamaManager.test.jsindex 390e143ada..19ba0d9764 100644--- a/server/services/ollamaManager.test.js+++ b/server/services/ollamaManager.test.js@@ -971,6 +971,29 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' })})+ it('reclaims week-old retained staging for other models before importing, once nothing holds it', async () => {+ const day = 24 * 60 * 60_000+ const stage = async (modelId, retainedAgoMs) => {+ const root = await mkdtemp(join(tmpdir(), 'portos-ollama-hf-import-'))+ stagingRoots.add(root)+ const at = new Date(Date.now() - retainedAgoMs).toISOString()+ await writeFile(join(root, 'portos-import.json'), JSON.stringify({ modelId, phase: 'retained', createdAt: at, retainedAt: at }))+ return root+ }+ const abandoned = await stage('example/abandoned:4bit', 8 * day)+ const recent = await stage('example/recent:4bit', 1 * day)+ const { importModelFromHfSafetensors } = await loadManager()+ await scriptCreate((c, _cmd, args) => { stagingOf(args); queueMicrotask(() => c.exit(0)) })++ let result+ importModelFromHfSafetensors(specFor('fresh')).then((r) => { result = r })+ await advanceUntil(() => result, { stepMs: 0 })++ expect(result).toEqual({ success: true, modelId: 'example/fresh:4bit' })+ await expect(stat(abandoned)).rejects.toMatchObject({ code: 'ENOENT' })+ await expect(stat(recent)).resolves.toBeTruthy()+ })+it('retains staging another process still reads, and refuses to stack a second create on it', async () => {const modelSpec = specFor('daemon-held')const { importModelFromHfSafetensors } = await loadManager()diff --git a/server/services/ollamaSafetensorsImport.js b/server/services/ollamaSafetensorsImport.jsindex 915ac72a0a..3162e34146 100644--- a/server/services/ollamaSafetensorsImport.js+++ b/server/services/ollamaSafetensorsImport.js@@ -35,6 +35,7 @@ const CREATE_EXIT_BOUND_MS = 30_000const RELEASE_PROBE_ATTEMPTS = 4const RELEASE_PROBE_DELAY_MS = 15_000const OUTPUT_TAIL_CHARS = 8_192+const STAGING_MAX_AGE_MS = 7 * 24 * 60 * 60_000const SEEN_LINES_CAP = 5_000const ANSI_RE = /\x1b\[[0-9;?]*[A-Za-z]/g@@ -171,18 +172,62 @@ export async function updateImportStaging(tempRoot, fields) {await atomicWrite(markerPath(tempRoot), JSON.stringify({ ...marker, ...fields }))}-/** Newest staging directory left behind for `modelId`, or null. */-export async function findImportStaging(modelId) {+async function listImportStaging() {const names = await readdir(tmpdir()).catch(() => [])- let found = null+ const entries = []for (const name of names) {if (!name.startsWith(STAGING_PREFIX)) continueconst tempRoot = join(tmpdir(), name)- const marker = await readMarker(tempRoot)- if (marker?.modelId !== modelId) continue- if (!found || String(marker.createdAt) > String(found.marker.createdAt)) found = { tempRoot, marker }+ const mtimeMs = await stat(tempRoot).then((s) => s.mtimeMs, () => null)+ if (mtimeMs == null) continue+ entries.push({ tempRoot, mtimeMs, marker: await readMarker(tempRoot) })+ }+ return entries+}++function newestByModel(entries) {+ const newest = new Map()+ for (const entry of entries) {+ const id = entry.marker?.modelId+ if (!id) continue+ const current = newest.get(id)+ if (!current || String(entry.marker.createdAt) > String(current.marker.createdAt)) newest.set(id, entry)+ }+ return newest+}++/** Newest staging directory left behind for `modelId`, or null. */+export async function findImportStaging(modelId) {+ return newestByModel(await listImportStaging()).get(modelId) ?? null+}++/**+ * Reclaim staging nobody will reuse: older duplicates for a model, and any+ * directory untouched for `maxAgeMs` (including marker-less ones an older+ * PortOS left after a crash). `modelId`'s newest staging is left to the+ * caller, and models in `activeModelIds` are skipped. Each removal still+ * requires the open-file probe to prove nothing holds the directory.+ */+export async function sweepImportStaging({ modelId, activeModelIds = new Set(), maxAgeMs = STAGING_MAX_AGE_MS, now = Date.now() } = {}) {+ const entries = await listImportStaging()+ const newest = newestByModel(entries)+ for (const entry of entries) {+ const id = entry.marker?.modelId+ if (id && id !== modelId && activeModelIds.has(id)) continue+ const isNewest = id ? newest.get(id) === entry : false+ if (isNewest && id === modelId) continue+ const stampedAt = Date.parse(entry.marker?.retainedAt || entry.marker?.createdAt || '')+ const age = now - (Number.isFinite(stampedAt) ? stampedAt : entry.mtimeMs)+ // An older duplicate is never reused; anything else waits out the age limit.+ const ageGated = isNewest || !entry.marker+ if (ageGated && age <= maxAgeMs) continue+ // The probe's canary is the marker, so a marker-less directory gets one first.+ if (!entry.marker) await updateImportStaging(entry.tempRoot, { phase: 'orphaned' })+ const release = await awaitStagingRelease(entry.tempRoot, { attempts: 1 })+ if (!release.free) continue+ console.log(`🧹 Removing unused Safetensors import staging${id ? ` for ${id}` : ''}`)+ await removeImportStaging(entry.tempRoot)}- return found}/** True when every expected file is staged at its published size. */