Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: reuse retained Ollama staging only from the same source, reject duplicate imports up front (#13124)
Adam Eivy3d0f0ee6f7d9
diff --git a/server/services/ollamaManager.js b/server/services/ollamaManager.jsindex 7542913b93..53738e8d7a 100644--- a/server/services/ollamaManager.js+++ b/server/services/ollamaManager.js@@ -1558,6 +1558,21 @@ async function downloadHfImportFile({ repo, remotePath, destination, headers, pr* @param {(p: { status: string, percent: number|null, completed?: number, total?: number, importing?: boolean }) => void} [onProgress]*/async function importModelFromHfSafetensors(spec, onProgress) {+ const { modelId } = spec+ // One import per model at a time: a second would mistake the first's live+ // staging for one left behind. Checked before any network round-trip.+ if (activeImports.has(modelId)) {+ return { success: false, error: `${modelId} is already being imported.`, modelId, code: 'OLLAMA_IMPORT_IN_PROGRESS' }+ }+ activeImports.add(modelId)+ try {+ return await prepareSafetensorsImport(spec, onProgress)+ } finally {+ activeImports.delete(modelId)+ }+}++async function prepareSafetensorsImport(spec, onProgress) {const { modelId, repo, subdir, minVersion = '0.19.0' } = specif (!(await checkOllamaAvailable())) {return { success: false, error: 'Ollama not available', modelId }@@ -1607,20 +1622,10 @@ async function importModelFromHfSafetensors(spec, onProgress) {return { success: false, error: `The ${subdir} checkpoint is incomplete or unavailable on Hugging Face.`, modelId }}- // One import per model at a time: a second would mistake the first's live- // staging for one left behind.- if (activeImports.has(modelId)) {- return { success: false, error: `${modelId} is already being imported.`, modelId, code: 'OLLAMA_IMPORT_IN_PROGRESS' }- }- activeImports.add(modelId)- try {- // Resolve-not-throw, like every other failure here. A staging write that- // throws leaves its marker behind; the next attempt reuses or retires it.- return await runSafetensorsImport({ modelId, repo, subdir, files, headers, onProgress })- .catch((err) => ({ success: false, error: err.message, modelId }))- } finally {- activeImports.delete(modelId)- }+ // Resolve-not-throw, like every other failure here. A staging write that+ // throws leaves its marker behind; the next attempt reuses or retires it.+ return runSafetensorsImport({ modelId, repo, subdir, files, headers, onProgress })+ .catch((err) => ({ success: false, error: err.message, modelId }))}const heldImportResult = (modelId, error, reason) => ({@@ -1641,7 +1646,10 @@ async function runSafetensorsImport({ modelId, repo, subdir, files, headers, onP.catch((err) => console.error(`❌ Safetensors staging sweep failed: ${err.message}`))const retained = await findImportStaging(modelId)if (retained) {- const complete = await stagedFilesComplete(join(retained.tempRoot, 'model'), files)+ // Equal file sizes alone could pass a checkpoint from another repo or+ // subdir off as this one; a marker that cannot name its source is retired.+ const sameSource = retained.marker.repo === repo && retained.marker.subdir === subdir+ const complete = sameSource && await stagedFilesComplete(join(retained.tempRoot, 'model'), files)if (complete) {// Reusing is non-destructive, but stacking a second create on one the// daemon is still running is not; wait only for a holder we can see.@@ -1658,7 +1666,7 @@ async function runSafetensorsImport({ modelId, repo, subdir, files, headers, onP} else {const release = await awaitStagingRelease(retained.tempRoot, { attempts: 1 })if (!release.free) {- return heldImportResult(modelId, `A previous import of ${modelId} left an incomplete checkpoint.`, release.reason)+ return heldImportResult(modelId, `A previous import of ${modelId} left a checkpoint it cannot reuse.`, release.reason)}await removeImportStaging(retained.tempRoot)}diff --git a/server/services/ollamaManager.test.js b/server/services/ollamaManager.test.jsindex 19ba0d9764..9716cd64c2 100644--- a/server/services/ollamaManager.test.js+++ b/server/services/ollamaManager.test.js@@ -829,6 +829,7 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {let modelsDirlet lsoflet downloads+ let metadataFetcheslet stagingRoots// Each test imports its own model id so retained staging from a parallel@@ -842,11 +843,13 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'Date'] })hfTokenMock.token = 'hf_example_token'downloads = 0+ metadataFetches = 0stagingRoots = new Set()vi.stubGlobal('fetch', vi.fn(async (url) => {const href = String(url)if (href.endsWith('/api/version')) return versionResponse()if (href.includes('/api/models/example/Qwen-MLX')) {+ metadataFetches++return Response.json({ siblings: [{ rfilename: '4-bit/config.json', size: files.config.length },{ rfilename: '4-bit/model.safetensors', lfs: { size: files.weights.length } }@@ -971,6 +974,53 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' })})+ it('re-downloads instead of reusing retained staging staged from a different source', async () => {+ const modelSpec = specFor('moved-source')+ const stale = await mkdtemp(join(tmpdir(), 'portos-ollama-hf-import-'))+ stagingRoots.add(stale)+ await mkdir(join(stale, 'model'), { recursive: true })+ // Same file names and byte sizes as the current recipe, different repo.+ await writeFile(join(stale, 'model', 'config.json'), files.config)+ await writeFile(join(stale, 'model', 'model.safetensors'), files.weights)+ const at = new Date().toISOString()+ await writeFile(join(stale, 'portos-import.json'), JSON.stringify({+ modelId: modelSpec.modelId, repo: 'example/Other-MLX', subdir: '4-bit', phase: 'retained', createdAt: at, retainedAt: at+ }))+ const { importModelFromHfSafetensors } = await loadManager()+ let createdFrom+ await scriptCreate((c, _cmd, args) => { createdFrom = stagingOf(args); queueMicrotask(() => c.exit(0)) })++ let result+ importModelFromHfSafetensors(modelSpec).then((r) => { result = r })+ await advanceUntil(() => result, { stepMs: 0 })++ expect(result).toEqual({ success: true, modelId: modelSpec.modelId })+ expect(downloads).toBe(2)+ expect(createdFrom).not.toBe(stale)+ await expect(stat(stale)).rejects.toMatchObject({ code: 'ENOENT' })+ })++ it('rejects a second import of the same model before any Hugging Face request', async () => {+ const modelSpec = specFor('duplicate')+ const { importModelFromHfSafetensors } = await loadManager()+ let child+ await scriptCreate((c, _cmd, args) => { child = c; stagingOf(args) })++ let first+ importModelFromHfSafetensors(modelSpec).then((r) => { first = r })+ await advanceUntil(() => child, { stepMs: 0 })+ const fetchesBefore = globalThis.fetch.mock.calls.length++ const second = await importModelFromHfSafetensors(modelSpec)++ expect(second).toMatchObject({ success: false, code: 'OLLAMA_IMPORT_IN_PROGRESS' })+ expect(globalThis.fetch.mock.calls.length).toBe(fetchesBefore)+ expect(metadataFetches).toBe(1)+ child.exit(0)+ await advanceUntil(() => first, { stepMs: 0 })+ expect(first).toEqual({ success: true, modelId: modelSpec.modelId })+ })+it('reclaims week-old retained staging for other models before importing, once nothing holds it', async () => {const day = 24 * 60 * 60_000const stage = async (modelId, retainedAgoMs) => {