Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: reclaim abandoned Ollama import staging once nothing holds it (#13124)

Adam Eivy35bb31b86f12
diff --git a/server/services/ollamaManager.js b/server/services/ollamaManager.js
index dbeeff77a9..7542913b93 100644
--- a/server/services/ollamaManager.js
+++ b/server/services/ollamaManager.js
@@ -43,7 +43,7 @@ import { isSafeHfRepoRelativePath } from '../lib/hfCache.js'
import { assessDownloadPreflight, diskInsufficientError, DOWNLOAD_VERDICTS } from '../lib/downloadPreflight.js'
import {
awaitStagingRelease, blobStoreSignature, createImportStaging, findImportStaging, removeImportStaging,
- runSupervisedOllamaCreate, stagedFilesComplete, updateImportStaging
+ runSupervisedOllamaCreate, stagedFilesComplete, sweepImportStaging, updateImportStaging
} from './ollamaSafetensorsImport.js'
const execFileAsync = promisify(execFile)
@@ -1637,6 +1637,8 @@ async function runSafetensorsImport({ modelId, repo, subdir, files, headers, onP
// A staging directory from an earlier attempt (a stalled create, or a
// PortOS restart mid-import) is reused when it holds the whole checkpoint;
// an incomplete one is retired only once nothing holds it open.
+ await sweepImportStaging({ modelId, activeModelIds: activeImports })
+ .catch((err) => console.error(`❌ Safetensors staging sweep failed: ${err.message}`))
const retained = await findImportStaging(modelId)
if (retained) {
const complete = await stagedFilesComplete(join(retained.tempRoot, 'model'), files)
diff --git a/server/services/ollamaManager.test.js b/server/services/ollamaManager.test.js
index 390e143ada..19ba0d9764 100644
--- a/server/services/ollamaManager.test.js
+++ b/server/services/ollamaManager.test.js
@@ -971,6 +971,29 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {
await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' })
})
+ it('reclaims week-old retained staging for other models before importing, once nothing holds it', async () => {
+ const day = 24 * 60 * 60_000
+ const stage = async (modelId, retainedAgoMs) => {
+ const root = await mkdtemp(join(tmpdir(), 'portos-ollama-hf-import-'))
+ stagingRoots.add(root)
+ const at = new Date(Date.now() - retainedAgoMs).toISOString()
+ await writeFile(join(root, 'portos-import.json'), JSON.stringify({ modelId, phase: 'retained', createdAt: at, retainedAt: at }))
+ return root
+ }
+ const abandoned = await stage('example/abandoned:4bit', 8 * day)
+ const recent = await stage('example/recent:4bit', 1 * day)
+ const { importModelFromHfSafetensors } = await loadManager()
+ await scriptCreate((c, _cmd, args) => { stagingOf(args); queueMicrotask(() => c.exit(0)) })
+
+ let result
+ importModelFromHfSafetensors(specFor('fresh')).then((r) => { result = r })
+ await advanceUntil(() => result, { stepMs: 0 })
+
+ expect(result).toEqual({ success: true, modelId: 'example/fresh:4bit' })
+ await expect(stat(abandoned)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(stat(recent)).resolves.toBeTruthy()
+ })
+
it('retains staging another process still reads, and refuses to stack a second create on it', async () => {
const modelSpec = specFor('daemon-held')
const { importModelFromHfSafetensors } = await loadManager()
diff --git a/server/services/ollamaSafetensorsImport.js b/server/services/ollamaSafetensorsImport.js
index 915ac72a0a..3162e34146 100644
--- a/server/services/ollamaSafetensorsImport.js
+++ b/server/services/ollamaSafetensorsImport.js
@@ -35,6 +35,7 @@ const CREATE_EXIT_BOUND_MS = 30_000
const RELEASE_PROBE_ATTEMPTS = 4
const RELEASE_PROBE_DELAY_MS = 15_000
const OUTPUT_TAIL_CHARS = 8_192
+const STAGING_MAX_AGE_MS = 7 * 24 * 60 * 60_000
const SEEN_LINES_CAP = 5_000
const ANSI_RE = /\x1b\[[0-9;?]*[A-Za-z]/g
@@ -171,18 +172,62 @@ export async function updateImportStaging(tempRoot, fields) {
await atomicWrite(markerPath(tempRoot), JSON.stringify({ ...marker, ...fields }))
}
-/** Newest staging directory left behind for `modelId`, or null. */
-export async function findImportStaging(modelId) {
+async function listImportStaging() {
const names = await readdir(tmpdir()).catch(() => [])
- let found = null
+ const entries = []
for (const name of names) {
if (!name.startsWith(STAGING_PREFIX)) continue
const tempRoot = join(tmpdir(), name)
- const marker = await readMarker(tempRoot)
- if (marker?.modelId !== modelId) continue
- if (!found || String(marker.createdAt) > String(found.marker.createdAt)) found = { tempRoot, marker }
+ const mtimeMs = await stat(tempRoot).then((s) => s.mtimeMs, () => null)
+ if (mtimeMs == null) continue
+ entries.push({ tempRoot, mtimeMs, marker: await readMarker(tempRoot) })
+ }
+ return entries
+}
+
+function newestByModel(entries) {
+ const newest = new Map()
+ for (const entry of entries) {
+ const id = entry.marker?.modelId
+ if (!id) continue
+ const current = newest.get(id)
+ if (!current || String(entry.marker.createdAt) > String(current.marker.createdAt)) newest.set(id, entry)
+ }
+ return newest
+}
+
+/** Newest staging directory left behind for `modelId`, or null. */
+export async function findImportStaging(modelId) {
+ return newestByModel(await listImportStaging()).get(modelId) ?? null
+}
+
+/**
+ * Reclaim staging nobody will reuse: older duplicates for a model, and any
+ * directory untouched for `maxAgeMs` (including marker-less ones an older
+ * PortOS left after a crash). `modelId`'s newest staging is left to the
+ * caller, and models in `activeModelIds` are skipped. Each removal still
+ * requires the open-file probe to prove nothing holds the directory.
+ */
+export async function sweepImportStaging({ modelId, activeModelIds = new Set(), maxAgeMs = STAGING_MAX_AGE_MS, now = Date.now() } = {}) {
+ const entries = await listImportStaging()
+ const newest = newestByModel(entries)
+ for (const entry of entries) {
+ const id = entry.marker?.modelId
+ if (id && id !== modelId && activeModelIds.has(id)) continue
+ const isNewest = id ? newest.get(id) === entry : false
+ if (isNewest && id === modelId) continue
+ const stampedAt = Date.parse(entry.marker?.retainedAt || entry.marker?.createdAt || '')
+ const age = now - (Number.isFinite(stampedAt) ? stampedAt : entry.mtimeMs)
+ // An older duplicate is never reused; anything else waits out the age limit.
+ const ageGated = isNewest || !entry.marker
+ if (ageGated && age <= maxAgeMs) continue
+ // The probe's canary is the marker, so a marker-less directory gets one first.
+ if (!entry.marker) await updateImportStaging(entry.tempRoot, { phase: 'orphaned' })
+ const release = await awaitStagingRelease(entry.tempRoot, { attempts: 1 })
+ if (!release.free) continue
+ console.log(`🧹 Removing unused Safetensors import staging${id ? ` for ${id}` : ''}`)
+ await removeImportStaging(entry.tempRoot)
}
- return found
}
/** True when every expected file is staged at its published size. */