Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bound a stalled Ollama checkpoint import without losing staged weights (#13124)
Adam Eivy7538cf260f3d
diff --git a/server/lib/README.md b/server/lib/README.mdindex 51a5e5ede6..c5bdff8ca2 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -713,6 +713,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `portosUrls.js` | `PORTOS_UI_URL` / `PORTOS_API_URL` — the origins PortOS interpolates into prompts and links, resolved from `PORTOS_*` / `PORT` at module load. `PORTOS_API_URL_TOKEN` is the `{portosApiUrl}` placeholder agent prompts embed instead of `PORTOS_API_URL`; `buildAgentPrompt` expands it at spawn time to `localApiBaseUrl()` (the loopback mirror on HTTPS installs). Split out of `ports.js` so that module stays free of module-scope `process.env` reads and can be re-exported to the client. || `platform.js` | Platform/OS detection helpers — listening-port probes plus `isAppleSilicon()` (arm64 darwin; gates MLX model features, detect at the route boundary). || `processCwdProbe.js` | `hasProcessCwdInside(dir)` — true when any process (via `lsof -d cwd`) has its working directory inside `dir`. Fails CLOSED: no `lsof`, an unsupported platform or an empty/unreadable listing reads as occupied. Backs the ownerless-foreign-worktree release in `worktreeManager.js`. |+| `openFileHolders.js` | `findOpenFileHolders(files, { canary })` → `{ known: true, pids }` (other processes holding any of `files` open, via `lsof -F pn`) or `{ known: false, reason }`. Fails CLOSED: the probe holds `canary` open itself and rejects any listing that does not report its own PID, so a missing `lsof`, an unsupported platform or a truncated listing never reads as "nobody holds it". Gates the staged-checkpoint cleanup in `services/ollamaSafetensorsImport.js`. || `signalCrypto.js` | Pure, dependency-free crypto for Signal Desktop (#2154, #11149): SQLCipher-4 page decryption (`decryptSqlcipherDatabase`, `deriveSqlcipherKeys`, `sqlcipherPageHmac`) and Chromium/Electron `safeStorage` unwrap (`decryptSafeStorageValue`, `deriveSafeStorageKey`). Live sync does not decrypt a raw main-file copy — that misses committed WAL frames — and instead uses `services/signalSnapshot.js`. `isRawHexKey` still gates the key. All functions return `{ ok, ... }` reports (never throw). || `timezone.js` | Timezone utilities for scheduling. `getLocalParts(utcDate, timezone)` / `getUtcOffsetMs(utcDate, timezone)` are the primitives; `nextLocalTime(afterMs, hours, minutes, timezone)` finds the next UTC instant matching a local HH:MM. `anchorLocalMidnightUtc(dayStr, tz)` resolves the first UTC instant belonging to a local date, including zones whose DST transition skips 00:00; `localDayWindowUtc(timezone, atDate?)` exposes an inclusive ISO-string window, while `localDayRangeUtc(dateStr, timezone)` returns a validated half-open `{ start, end }` `Date` pair ending at the next local-date boundary so 23h/25h transition days remain exact. Also owns `HHMM_RE`/`HHMM_STRICT_RE`, `parseHHMM`, `dailyCronFromHHMM(time)` (the one strict "HH:MM" → `"M H * * *"` conversion every daily-reminder scheduler registers through, null on malformed input so the caller cancels rather than firing in some other slot), and `isWithinTimeWindow` (mirrored client-side in `client/src/utils/timeWindow.js`). || `tribeCadence.js` | Authoritative, pure Tribe care-cadence rules (single source of truth, mirrored to `client/src/lib/tribeCadence.js`): `cadenceStatus(entity)` → `{ state: external/missing/overdue/soon/steady, daysRemaining, daysOverdue }`, `daysSinceDate(dateStr)`, `DEFAULT_CADENCE_DAYS` (45), `SOON_WINDOW_DAYS` (7). Consumed by `personCadenceStatus` / `getCareSummary` in `services/tribe.js` (proactive alert + Care widget) and by the client Tribe page/map. |diff --git a/server/lib/index.js b/server/lib/index.jsindex 97ea05eace..aed97d9465 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -681,6 +681,7 @@ export * from './pgTimestamp.js';export * from './pgTools.js';export * from './platform.js';export * from './processCwdProbe.js';+export * from './openFileHolders.js';export * from './systemCapabilities.js';export * from './ports.js';export * from './portosUrls.js';diff --git a/server/lib/openFileHolders.js b/server/lib/openFileHolders.jsnew file mode 100644index 0000000000..e5e1ae596b--- /dev/null+++ b/server/lib/openFileHolders.js@@ -0,0 +1,47 @@+/**+ * Which OTHER processes still hold a set of files open?+ *+ * Used before deleting staged input another process (an `ollama create` CLI,+ * the Ollama daemon) may still be reading. The answer is tri-state and fails+ * CLOSED: `known: false` whenever the probe cannot prove its own listing is+ * complete — no `lsof`, an unsupported platform, or a listing that omits the+ * canary file this process holds open for the duration of the probe.+ */++import { open } from 'node:fs/promises';+import { promisify } from 'node:util';+import { execFile } from './childProcess.js';++const LSOF_TIMEOUT_MS = 15_000;+const LSOF_MAX_BUFFER = 4 * 1024 * 1024;++/** PIDs from `lsof -F pn` field output (`p<pid>` starts each process block). */+function parseLsofPids(stdout) {+ return String(stdout || '').split('\n')+ .filter((line) => line.startsWith('p'))+ .map((line) => Number(line.slice(1)))+ .filter((pid) => Number.isInteger(pid) && pid > 0);+}++/**+ * @param {string[]} files - paths to check+ * @param {{ canary: string, platform?: string, selfPid?: number,+ * run?: (file: string, args: string[], opts: object) => Promise<{ stdout: string }> }} options+ * `canary` must be an existing file; it is opened by this process so a+ * listing that does not report this PID on it is rejected as incomplete.+ * @returns {Promise<{ known: true, pids: number[] } | { known: false, reason: string }>}+ */+export async function findOpenFileHolders(files, { canary, platform = process.platform, selfPid = process.pid, run = promisify(execFile) } = {}) {+ if (platform !== 'darwin' && platform !== 'linux') return { known: false, reason: `open-file probe unsupported on ${platform}` };+ if (!canary) return { known: false, reason: 'no canary file' };+ const handle = await open(canary, 'r').catch(() => null);+ if (!handle) return { known: false, reason: 'canary file unreadable' };+ // lsof exits 1 when any named file is not open (the expected case for most+ // of them) while still printing what it found; judge the listing by content.+ const out = await run('lsof', ['-F', 'pn', '--', canary, ...files], { timeout: LSOF_TIMEOUT_MS, maxBuffer: LSOF_MAX_BUFFER })+ .then((r) => r.stdout, (err) => err?.stdout ?? null)+ .finally(() => handle.close().catch(() => {}));+ const pids = out == null ? [] : parseLsofPids(out);+ if (!pids.includes(selfPid)) return { known: false, reason: 'lsof unavailable or listing incomplete' };+ return { known: true, pids: [...new Set(pids.filter((pid) => pid !== selfPid))] };+}diff --git a/server/lib/openFileHolders.test.js b/server/lib/openFileHolders.test.jsnew file mode 100644index 0000000000..643a75ed32--- /dev/null+++ b/server/lib/openFileHolders.test.js@@ -0,0 +1,35 @@+import { describe, it, expect, beforeAll, afterAll } from 'vitest'+import { mkdtemp, rm, writeFile } from 'fs/promises'+import { tmpdir } from 'os'+import { join } from 'path'+import { findOpenFileHolders } from './openFileHolders.js'++// The result gates deleting staged files another process may still read, so a+// listing that cannot prove it is complete must never read as "no holders".+describe('findOpenFileHolders', () => {+ let dir+ let canary+ const staged = '/staging/model.safetensors'+ const listing = (stdout) => async () => ({ stdout })++ beforeAll(async () => {+ dir = await mkdtemp(join(tmpdir(), 'portos-open-file-holders-'))+ canary = join(dir, 'marker.json')+ await writeFile(canary, '{}')+ })+ afterAll(() => rm(dir, { recursive: true, force: true }))++ it('reports the other processes holding the files, never itself', async () => {+ const run = async () => { throw Object.assign(new Error('exit 1'), { stdout: `p100\nn${canary}\np200\nn${staged}\np200\nn${staged}\n` }) }+ await expect(findOpenFileHolders([staged], { canary, platform: 'linux', selfPid: 100, run }))+ .resolves.toEqual({ known: true, pids: [200] })+ })++ it.each([+ ['the listing omits the canary this process holds', { platform: 'darwin', run: listing('p200\nn/staging/x\n') }],+ ['lsof is missing', { platform: 'linux', run: async () => { throw Object.assign(new Error('spawn lsof ENOENT'), { code: 'ENOENT' }) } }],+ ['the platform has no lsof', { platform: 'win32', run: listing('p100\n') }]+ ])('fails closed when %s', async (_label, opts) => {+ await expect(findOpenFileHolders([staged], { canary, selfPid: 100, ...opts })).resolves.toMatchObject({ known: false })+ })+})diff --git a/server/services/ollamaManager.js b/server/services/ollamaManager.jsindex 7e5ffbf19d..dbeeff77a9 100644--- a/server/services/ollamaManager.js+++ b/server/services/ollamaManager.js@@ -18,7 +18,7 @@ import { invalidateModelObservation } from './modelObservation.js';import { homedir, tmpdir } from 'os'import { join, dirname } from 'path'import { createWriteStream } from 'fs'-import { readdir, stat, link, unlink, mkdtemp, rm } from 'fs/promises'+import { readdir, stat, link, unlink } from 'fs/promises'import { createHash } from 'crypto'import { Readable, Transform } from 'stream'import { pipeline } from 'stream/promises'@@ -41,6 +41,10 @@ import { compareSemver } from '../lib/versionUtils.js'import { createSingleFlight } from '../lib/singleFlight.js'import { isSafeHfRepoRelativePath } from '../lib/hfCache.js'import { assessDownloadPreflight, diskInsufficientError, DOWNLOAD_VERDICTS } from '../lib/downloadPreflight.js'+import {+ awaitStagingRelease, blobStoreSignature, createImportStaging, findImportStaging, removeImportStaging,+ runSupervisedOllamaCreate, stagedFilesComplete, updateImportStaging+} from './ollamaSafetensorsImport.js'const execFileAsync = promisify(execFile)const AVAILABILITY_CACHE_TTL_MS = 30_000@@ -74,6 +78,8 @@ const STOP_TIMEOUT_MS = 8_000const SERVICE_COMMAND_TIMEOUT_MS = 20_000const PROCESS_IDENTITY_TIMEOUT_MS = 2_000const HF_IMPORT_METADATA_TIMEOUT_MS = 180_000+// Model ids with a Safetensors import running in this process.+const activeImports = new Set()const DEFAULT_CONFIG = {// Ollama uses OLLAMA_HOST (host:port, no scheme) by convention; also accept@@ -1543,6 +1549,11 @@ async function downloadHfImportFile({ repo, remotePath, destination, headers, pr* Ollama's supported `ollama create` path. The caller supplies only catalog-owned* recipes; arbitrary free-text ids continue through pullModel instead.*+ * The create step is supervised (see ollamaSafetensorsImport.js): a silent+ * create resolves `OLLAMA_IMPORT_STALLED`, or `OLLAMA_IMPORT_HELD` with+ * `stagingRetained: true` when PortOS cannot prove nothing still reads the+ * staged checkpoint — a retry then reuses that staging.+ ** @param {{ modelId: string, repo: string, subdir: string, minVersion?: string }} spec* @param {(p: { status: string, percent: number|null, completed?: number, total?: number, importing?: boolean }) => void} [onProgress]*/@@ -1596,6 +1607,61 @@ async function importModelFromHfSafetensors(spec, onProgress) {return { success: false, error: `The ${subdir} checkpoint is incomplete or unavailable on Hugging Face.`, modelId }}+ // One import per model at a time: a second would mistake the first's live+ // staging for one left behind.+ if (activeImports.has(modelId)) {+ return { success: false, error: `${modelId} is already being imported.`, modelId, code: 'OLLAMA_IMPORT_IN_PROGRESS' }+ }+ activeImports.add(modelId)+ try {+ // Resolve-not-throw, like every other failure here. A staging write that+ // throws leaves its marker behind; the next attempt reuses or retires it.+ return await runSafetensorsImport({ modelId, repo, subdir, files, headers, onProgress })+ .catch((err) => ({ success: false, error: err.message, modelId }))+ } finally {+ activeImports.delete(modelId)+ }+}++const heldImportResult = (modelId, error, reason) => ({+ success: false,+ modelId,+ code: 'OLLAMA_IMPORT_HELD',+ stagingRetained: true,+ error: `${error} Ollama may still be reading the downloaded checkpoint (${reason}), so PortOS kept it instead of deleting it. Retry the installation to reuse it without downloading again.`+})++async function runSafetensorsImport({ modelId, repo, subdir, files, headers, onProgress }) {+ let tempRoot = null+ let reuse = false+ // A staging directory from an earlier attempt (a stalled create, or a+ // PortOS restart mid-import) is reused when it holds the whole checkpoint;+ // an incomplete one is retired only once nothing holds it open.+ const retained = await findImportStaging(modelId)+ if (retained) {+ const complete = await stagedFilesComplete(join(retained.tempRoot, 'model'), files)+ if (complete) {+ // Reusing is non-destructive, but stacking a second create on one the+ // daemon is still running is not; wait only for a holder we can see.+ const release = await awaitStagingRelease(retained.tempRoot, { attempts: 1 })+ if (release.pids?.length) {+ return heldImportResult(modelId, `A previous import of ${modelId} is still running.`, release.reason)+ }+ tempRoot = retained.tempRoot+ reuse = true+ console.log(`♻️ Reusing retained Safetensors staging for ${modelId}`)+ } else if (retained.marker.phase === 'downloading') {+ // `ollama create` never started, so no other process ever read it.+ await removeImportStaging(retained.tempRoot)+ } else {+ const release = await awaitStagingRelease(retained.tempRoot, { attempts: 1 })+ if (!release.free) {+ return heldImportResult(modelId, `A previous import of ${modelId} left an incomplete checkpoint.`, release.reason)+ }+ await removeImportStaging(retained.tempRoot)+ }+ }+// This import writes the checkpoint TWICE — once staged under `tmpdir()`,// once more when `ollama create` below lands it in the models dir — on// volumes that can differ (a temp partition, or OLLAMA_MODELS pointed at@@ -1603,51 +1669,77 @@ async function importModelFromHfSafetensors(spec, onProgress) {// copy against one of those volumes using a catalog-string size estimate;// this uses the real per-file bytes this import already fetched from HF.const total = files.reduce((sum, file) => sum + file.size, 0)- const stagingPreflight = await assessDownloadPreflight({ destPath: tmpdir(), expectedBytes: total })- if (stagingPreflight.verdict === DOWNLOAD_VERDICTS.INSUFFICIENT) {- return { success: false, error: diskInsufficientError(stagingPreflight).message, modelId, code: 'DISK_INSUFFICIENT' }+ if (!reuse) {+ const stagingPreflight = await assessDownloadPreflight({ destPath: tmpdir(), expectedBytes: total })+ if (stagingPreflight.verdict === DOWNLOAD_VERDICTS.INSUFFICIENT) {+ return { success: false, error: diskInsufficientError(stagingPreflight).message, modelId, code: 'DISK_INSUFFICIENT' }+ }}const modelsDirPreflight = await assessDownloadPreflight({ destPath: getModelsDir(), expectedBytes: total })if (modelsDirPreflight.verdict === DOWNLOAD_VERDICTS.INSUFFICIENT) {return { success: false, error: diskInsufficientError(modelsDirPreflight).message, modelId, code: 'DISK_INSUFFICIENT' }}- const tempRoot = await mkdtemp(join(tmpdir(), 'portos-ollama-hf-import-'))+ if (!tempRoot) tempRoot = await createImportStaging({ modelId, repo, subdir })const modelDir = join(tempRoot, 'model')- let completed = 0- let lastPercent = -1- const reportBytes = (bytes) => {- completed += bytes- const percent = total > 0 ? Math.min(100, Math.floor((completed / total) * 100)) : null- if (percent === lastPercent) return- lastPercent = percent- if (typeof onProgress === 'function') onProgress({ status: 'downloading MLX checkpoint', percent, completed, total })- }- const performImport = async () => {- for (const file of files) {- await downloadHfImportFile({- repo,- remotePath: file.remotePath,- destination: join(modelDir, ...file.relativePath.split('/')),- headers,- progress: reportBytes- })+ if (!reuse) {+ let completed = 0+ let lastPercent = -1+ const reportBytes = (bytes) => {+ completed += bytes+ const percent = total > 0 ? Math.min(100, Math.floor((completed / total) * 100)) : null+ if (percent === lastPercent) return+ lastPercent = percent+ if (typeof onProgress === 'function') onProgress({ status: 'downloading MLX checkpoint', percent, completed, total })}- const modelfile = join(tempRoot, 'Modelfile')- await atomicWrite(modelfile, `FROM ${modelDir}\n`)- if (typeof onProgress === 'function') {- onProgress({ status: 'importing checkpoint into Ollama', percent: null, importing: true })+ const downloadError = await (async () => {+ for (const file of files) {+ await downloadHfImportFile({+ repo,+ remotePath: file.remotePath,+ destination: join(modelDir, ...file.relativePath.split('/')),+ headers,+ progress: reportBytes+ })+ }+ })().then(() => null, (err) => err)+ if (downloadError) {+ await removeImportStaging(tempRoot)+ return { success: false, error: downloadError.message, modelId }}- await execFileAsync('ollama', ['create', modelId, '-f', modelfile], { timeout: 0, maxBuffer: 64 * 1024 * 1024 })+ }++ const modelfile = join(tempRoot, 'Modelfile')+ await atomicWrite(modelfile, `FROM ${modelDir}\n`)+ await updateImportStaging(tempRoot, { phase: 'creating' })+ const report = (status) => {+ if (typeof onProgress === 'function') onProgress({ status, percent: null, importing: true })+ }+ report('importing checkpoint into Ollama')+ const outcome = await runSupervisedOllamaCreate({+ modelId,+ modelfile,+ onStatus: (line) => report(`importing: ${line}`),+ probeProgress: () => blobStoreSignature(getModelsDir())+ })+ if (outcome.status === 'completed') {+ // A clean CLI exit means the daemon answered the whole create request.+ await removeImportStaging(tempRoot)installedModels = nullconsole.log(`✅ Ollama Safetensors import complete: ${modelId}`)return { success: true, modelId }}- return performImport()- .catch((err) => ({ success: false, error: err.stderr || err.message, modelId }))- .finally(() => rm(tempRoot, { recursive: true, force: true }).catch((err) => {- console.error(`⚠️ Ollama Safetensors import cleanup failed: ${err.message}`)- }))++ const release = outcome.exited === false+ ? { free: false, reason: 'the ollama create process did not exit after being stopped' }+ : await awaitStagingRelease(tempRoot)+ if (release.free) {+ await removeImportStaging(tempRoot)+ return { success: false, error: outcome.error, modelId, ...(outcome.status === 'stalled' ? { code: 'OLLAMA_IMPORT_STALLED' } : {}) }+ }+ await updateImportStaging(tempRoot, { phase: 'retained', retainedAt: new Date().toISOString(), reason: release.reason })+ console.log(`⏭️ Kept Safetensors staging for ${modelId}: ${release.reason}`)+ return heldImportResult(modelId, outcome.error, release.reason)}/**diff --git a/server/services/ollamaManager.test.js b/server/services/ollamaManager.test.jsindex 60cb329319..0e40a31a40 100644--- a/server/services/ollamaManager.test.js+++ b/server/services/ollamaManager.test.js@@ -2,8 +2,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'import { mkdtemp, mkdir, rm, readFile, writeFile, stat } from 'fs/promises'import { createHash } from 'crypto'import { tmpdir } from 'os'-import { join } from 'path'+import { join, dirname } from 'path'import { pinPlatform } from '../lib/testHelper.js'+import { EventEmitter } from 'events'// startPersistentService / getServiceStatus shell out via promisify(execFile).// Route every exec call through a per-test impl so the homebrew service flow can@@ -667,6 +668,64 @@ describe('ollamaManager.isBootstrapConflictError', () => {})})++// A controllable `ollama create` child: tests drive its output and exit, and+// record the signals PortOS sends it.+function makeFakeChild() {+ const child = new EventEmitter()+ child.stdout = new EventEmitter()+ child.stderr = new EventEmitter()+ child.pid = 424242+ child.exitCode = null+ child.signalCode = null+ child.signals = []+ child.onSignal = () => {}+ child.kill = (signal) => { child.signals.push(signal); child.onSignal(signal); return true }+ child.exit = (code, signal = null) => {+ child.exitCode = code+ child.signalCode = signal+ child.emit('close', code, signal)+ }+ return child+}++// Route `spawn('ollama', ['create', …])` to a fake child handed to `onCreate`.+async function scriptCreate(onCreate) {+ const { spawn } = await import('../lib/childProcess.js')+ const children = []+ spawn.mockReset().mockImplementation((cmd, args) => {+ const child = makeFakeChild()+ children.push(child)+ onCreate(child, cmd, args)+ return child+ })+ return children+}++// Fake timers drive the watchdog while real fs I/O (staging, blob-store+// probe, lsof bookkeeping) completes on the thread pool between steps, so each+// step yields a few real milliseconds before advancing fake time.+const realSetTimeout = globalThis.setTimeout+const ioPause = () => new Promise((resolve) => realSetTimeout(resolve, 2))++async function advanceUntil(predicate, { stepMs = 15_000, maxSteps = 500 } = {}) {+ for (let i = 0; i < maxSteps; i++) {+ await ioPause()+ if (predicate()) return+ await vi.advanceTimersByTimeAsync(stepMs)+ }+ throw new Error('condition never reached')+}++// Step the watchdog one check interval at a time so each blob-store probe+// settles before the next idle check.+async function advanceBy(ms, stepMs = 15_000) {+ for (let elapsed = 0; elapsed < ms; elapsed += stepMs) {+ await ioPause()+ await vi.advanceTimersByTimeAsync(stepMs)+ }+}+describe('ollamaManager curated Hugging Face Safetensors import', () => {const spec = {modelId: 'example/qwen-mlx:4bit',@@ -706,18 +765,18 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {let createCallexecMock.impl = (cmd, args, _opts, cb) => {if (cmd === 'ollama' && args.join(' ') === '--version') return cb(null, { stdout: 'ollama 0.31.0', stderr: '' })- if (cmd === 'ollama' && args[0] === 'create') {- createCall = { cmd, args }- readFile(args[3], 'utf8').then((modelfile) => {- createCall.modelfile = modelfile- cb(null, { stdout: '', stderr: '' })- }, cb)- return undefined- }return cb(new Error(`unexpected exec: ${cmd} ${args.join(' ')}`))}const onProgress = vi.fn()const { importModelFromHfSafetensors } = await loadManager()+ await scriptCreate((child, cmd, args) => {+ createCall = { cmd, args }+ readFile(args[3], 'utf8').then((modelfile) => {+ createCall.modelfile = modelfile+ child.stderr.emit('data', Buffer.from('gathering model components\n'))+ child.exit(0)+ })+ })const result = await importModelFromHfSafetensors(spec, onProgress)@@ -764,6 +823,187 @@ describe('ollamaManager curated Hugging Face Safetensors import', () => {})})+ describe('supervised ollama create', () => {+ const files = { config: Buffer.from('{"model_type":"qwen3_8"}'), weights: Buffer.from('example safetensors bytes') }+ let restorePlatform+ let modelsDir+ let lsof+ let downloads+ let stagingRoots++ // Each test imports its own model id so retained staging from a parallel+ // worker can never be picked up.+ const specFor = (name) => ({ ...spec, modelId: `example/${name}:4bit` })++ beforeEach(async () => {+ restorePlatform = pinPlatform('linux')+ modelsDir = await mkdtemp(join(tmpdir(), 'portos-ollama-models-'))+ vi.stubEnv('OLLAMA_MODELS', modelsDir)+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'Date'] })+ hfTokenMock.token = 'hf_example_token'+ downloads = 0+ stagingRoots = new Set()+ vi.stubGlobal('fetch', vi.fn(async (url) => {+ const href = String(url)+ if (href.endsWith('/api/version')) return versionResponse()+ if (href.includes('/api/models/example/Qwen-MLX')) {+ return Response.json({ siblings: [+ { rfilename: '4-bit/config.json', size: files.config.length },+ { rfilename: '4-bit/model.safetensors', lfs: { size: files.weights.length } }+ ] })+ }+ if (href.includes('/resolve/main/4-bit/')) {+ downloads+++ return new Response(href.endsWith('config.json') ? files.config : files.weights)+ }+ throw new Error(`unexpected fetch: ${href}`)+ }))+ // `lsof` answers for the staged files; `holders` are the other PIDs it reports.+ lsof = { available: true, holders: [], calls: 0 }+ execMock.impl = (cmd, args, _opts, cb) => {+ if (cmd === 'ollama' && args.join(' ') === '--version') return cb(null, { stdout: 'ollama 0.31.0', stderr: '' })+ if (cmd === 'lsof') {+ lsof.calls+++ if (!lsof.available) return cb(Object.assign(new Error('spawn lsof ENOENT'), { code: 'ENOENT' }))+ const [canary, ...staged] = args.slice(args.indexOf('--') + 1)+ const lines = [`p${process.pid}`, `n${canary}`, ...lsof.holders.flatMap((pid) => [`p${pid}`, `n${staged[0]}`])]+ return cb(Object.assign(new Error('lsof exit 1'), { stdout: `${lines.join('\n')}\n` }))+ }+ return cb(new Error(`unexpected exec: ${cmd} ${args.join(' ')}`))+ }+ })++ afterEach(async () => {+ vi.useRealTimers()+ vi.unstubAllEnvs()+ restorePlatform()+ await rm(modelsDir, { recursive: true, force: true })+ for (const root of stagingRoots) await rm(root, { recursive: true, force: true })+ })++ const stagingOf = (args) => {+ const root = dirname(args[3])+ stagingRoots.add(root)+ return root+ }++ it('lets a conversion that keeps reporting progress outlive the idle limit', async () => {+ const { importModelFromHfSafetensors } = await loadManager()+ let child+ await scriptCreate((c, _cmd, args) => { child = c; stagingOf(args) })++ const pending = importModelFromHfSafetensors(specFor('progressing'))+ await advanceUntil(() => child, { stepMs: 0 })+ // 40 minutes of wall time, a new progress line every 5 — four times the idle limit.+ for (let minute = 5; minute <= 40; minute += 5) {+ child.stderr.emit('data', Buffer.from(`\u280b copying file sha256:abc ${minute * 2}% 120 MB/s 3m10s\r`))+ child.stderr.emit('data', Buffer.from('\n'))+ await advanceBy(5 * 60_000)+ }+ child.exit(0)+ const result = await pending++ expect(result).toEqual({ success: true, modelId: 'example/progressing:4bit' })+ expect(child.signals).toEqual([])+ })++ it('stops a silent create, escalates past an ignored SIGTERM, and removes staging once nothing holds it', async () => {+ const { importModelFromHfSafetensors } = await loadManager()+ let child+ let root+ await scriptCreate((c, _cmd, args) => {+ child = c+ root = stagingOf(args)+ // Ignores SIGTERM; only SIGKILL ends it.+ c.onSignal = (signal) => { if (signal === 'SIGKILL') queueMicrotask(() => c.exit(null, 'SIGKILL')) }+ })+ // A spinner that redraws forever is not progress.+ const onProgress = vi.fn()++ let settled = 0+ const pending = importModelFromHfSafetensors(specFor('silent'), onProgress).finally(() => { settled++ })+ await advanceUntil(() => child, { stepMs: 0 })+ const spin = setInterval(() => child.stderr.emit('data', Buffer.from('\u280b converting model\r')), 1_000)+ await advanceUntil(() => child.signals.includes('SIGTERM'))+ clearInterval(spin)+ await advanceUntil(() => settled > 0, { stepMs: 5_000 })+ const result = await pending++ expect(child.signals).toEqual(['SIGTERM', 'SIGKILL'])+ expect(result).toMatchObject({ success: false, code: 'OLLAMA_IMPORT_STALLED', error: expect.stringMatching(/stopped reporting progress/) })+ expect(result.stagingRetained).toBeUndefined()+ await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' })+ expect(settled).toBe(1)+ })++ it('keeps the staged checkpoint when the stopped CLI never exits, and a retry reuses it without downloading', async () => {+ const modelSpec = specFor('unkillable')+ let { importModelFromHfSafetensors } = await loadManager()+ let child+ let root+ await scriptCreate((c, _cmd, args) => { child = c; root = stagingOf(args) })++ const pending = importModelFromHfSafetensors(modelSpec)+ await advanceUntil(() => child, { stepMs: 0 })+ let result+ pending.then((r) => { result = r })+ await advanceUntil(() => result)++ expect(child.signals).toEqual(['SIGTERM', 'SIGKILL'])+ expect(result).toMatchObject({ success: false, code: 'OLLAMA_IMPORT_HELD', stagingRetained: true })+ expect(lsof.calls).toBe(0) // an owned child that outlived SIGKILL is never probed as "free"+ const marker = JSON.parse(await readFile(join(root, 'portos-import.json'), 'utf8'))+ expect(marker).toMatchObject({ modelId: modelSpec.modelId, phase: 'retained' })+ expect(downloads).toBe(2)++ // After a restart, a retry on a host where ownership still cannot be+ // probed reuses the retained checkpoint: reading it is non-destructive.+ vi.useRealTimers()+ lsof.available = false+ ;({ importModelFromHfSafetensors } = await loadManager())+ let retryRoot+ await scriptCreate((c, _cmd, args) => { retryRoot = stagingOf(args); queueMicrotask(() => c.exit(0)) })+ const retried = await importModelFromHfSafetensors(modelSpec)++ expect(retried).toEqual({ success: true, modelId: modelSpec.modelId })+ expect(retryRoot).toBe(root)+ expect(downloads).toBe(2)+ await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' })+ })++ it('retains staging another process still reads, and refuses to stack a second create on it', async () => {+ const modelSpec = specFor('daemon-held')+ const { importModelFromHfSafetensors } = await loadManager()+ let root+ const children = await scriptCreate((c, _cmd, args) => {+ root = stagingOf(args)+ queueMicrotask(() => {+ c.stderr.emit('data', Buffer.from('Error: something went wrong in the daemon\n'))+ c.exit(1)+ })+ })+ lsof.holders = [31337]++ let result+ importModelFromHfSafetensors(modelSpec).then((r) => { result = r })+ await advanceUntil(() => result)++ expect(result).toMatchObject({+ success: false, code: 'OLLAMA_IMPORT_HELD', stagingRetained: true,+ error: expect.stringMatching(/something went wrong in the daemon.*still open in process 31337.*Retry/s)+ })+ expect(lsof.calls).toBe(4) // bounded settle window, then retain+ await expect(stat(join(root, 'model', 'model.safetensors'))).resolves.toBeTruthy()++ let retried+ importModelFromHfSafetensors(modelSpec).then((r) => { retried = r })+ await advanceUntil(() => retried)+ expect(retried).toMatchObject({ success: false, code: 'OLLAMA_IMPORT_HELD', error: expect.stringMatching(/still running/) })+ expect(children).toHaveLength(1)+ expect(downloads).toBe(2)+ })+ })+it('fails before downloading when the gated repository has no token', async () => {stubFetch([])const { importModelFromHfSafetensors } = await loadManager()diff --git a/server/services/ollamaSafetensorsImport.js b/server/services/ollamaSafetensorsImport.jsnew file mode 100644index 0000000000..915ac72a0a--- /dev/null+++ b/server/services/ollamaSafetensorsImport.js@@ -0,0 +1,228 @@+/**+ * Lifecycle pieces for the curated Safetensors import in ollamaManager.js:+ * the supervised `ollama create` child and the staged-checkpoint directory it+ * reads from.+ *+ * Neither the Ollama CLI nor its HTTP client puts a deadline on `create`, so a+ * wedged CLI/daemon pair would hold the install request forever. The supervisor+ * bounds SILENCE, not wall time: a conversion that keeps printing new progress,+ * or keeps growing the daemon's blob store, may run as long as it needs.+ *+ * Killing the CLI does not prove the daemon stopped reading the staged files,+ * so the staging directory is deleted only after an open-file probe shows no+ * other process holds any of it. When that cannot be shown the directory is+ * kept, its marker file (the durable recovery record — it lives and dies with+ * the staging it describes) is stamped `retained`, and the next install of the+ * same model reuses it instead of downloading again.+ */++import { tmpdir } from 'os'+import { join } from 'path'+import { mkdtemp, readdir, readFile, rm, stat } from 'fs/promises'+import { spawn } from '../lib/childProcess.js'+import { atomicWrite, safeJSONParse, sleep } from '../lib/fileUtils.js'+import { killWithEscalation } from '../lib/killWithEscalation.js'+import { findOpenFileHolders } from '../lib/openFileHolders.js'++export const STAGING_PREFIX = 'portos-ollama-hf-import-'+const MARKER_NAME = 'portos-import.json'+export const CREATE_IDLE_LIMIT_MS = 10 * 60_000+const CREATE_CHECK_INTERVAL_MS = 15_000+const CREATE_KILL_GRACE_MS = 10_000+// After SIGKILL a live child exits within milliseconds; still being there this+// long later means PortOS cannot claim the operation ended.+const CREATE_EXIT_BOUND_MS = 30_000+const RELEASE_PROBE_ATTEMPTS = 4+const RELEASE_PROBE_DELAY_MS = 15_000+const OUTPUT_TAIL_CHARS = 8_192+const SEEN_LINES_CAP = 5_000++const ANSI_RE = /\x1b\[[0-9;?]*[A-Za-z]/g+const SPINNER_RE = /[⠀-⣿]/g+// Ollama's progress bar redraws a transfer rate and ETA on every tick even when+// nothing moves; those must not count as progress.+const RATE_RE = /\b\d+(?:\.\d+)?\s?[KMGT]?i?B\/s\b/gi+const DURATION_RE = /\b(?:\d+h)?(?:\d+m)?\d+(?:\.\d+)?(?:ms|s)\b|\b\d+h\d+m\b|\b\d+m\b/g++/** Complete output segments, reduced to the text that changes only on real progress. */+export function normalizeCreateOutput(text) {+ return String(text).replace(ANSI_RE, '').split(/[\r\n]+/)+ .map((line) => line.replace(SPINNER_RE, '').replace(RATE_RE, '').replace(DURATION_RE, '').replace(/\s+/g, ' ').trim())+ .filter(Boolean)+}++function lastErrorLine(tail) {+ const lines = normalizeCreateOutput(tail)+ return lines.reverse().find((line) => /error/i.test(line)) || lines[0] || null+}++/** Size signature of the daemon's blob store — grows while the daemon writes a converted layer. */+export async function blobStoreSignature(modelsDir) {+ const dir = join(modelsDir, 'blobs')+ const names = await readdir(dir).catch(() => null)+ if (!names) return null+ let bytes = 0+ for (const name of names) bytes += await stat(join(dir, name)).then((s) => s.size, () => 0)+ return `${names.length}:${bytes}`+}++/**+ * Run `ollama create` until it exits, or until neither its output nor+ * `probeProgress()` has changed for `idleLimitMs`, in which case the child is+ * stopped (SIGTERM, then SIGKILL). Always resolves, exactly once:+ * { status: 'completed' } | { status: 'failed', error } |+ * { status: 'stalled', error, exited } — `exited: false` means the child+ * outlived SIGKILL and PortOS cannot claim the operation ended.+ */+export function runSupervisedOllamaCreate({+ modelId, modelfile, onStatus, probeProgress,+ idleLimitMs = CREATE_IDLE_LIMIT_MS, checkIntervalMs = CREATE_CHECK_INTERVAL_MS+}) {+ return new Promise((resolve) => {+ const child = spawn('ollama', ['create', modelId, '-f', modelfile], { stdio: ['ignore', 'pipe', 'pipe'] })+ let settled = false+ let exited = false+ let stalled = false+ let probing = false+ let lastProgressAt = Date.now()+ let lastSignature+ let partial = ''+ let tail = ''+ let exitTimer = null+ let interval = null+ const seen = new Set()+ const idleMinutes = Math.round(idleLimitMs / 60_000)+ const stallError = `Ollama stopped reporting progress while importing ${modelId} (nothing for ${idleMinutes} min), so PortOS stopped the import.`++ const finish = (outcome) => {+ if (settled) return+ settled = true+ clearInterval(interval)+ clearTimeout(exitTimer)+ resolve(outcome)+ }+ const onOutput = (chunk) => {+ const text = partial + chunk.toString()+ tail = (tail + chunk.toString()).slice(-OUTPUT_TAIL_CHARS)+ // Only judge complete segments, so a redraw split across two chunks+ // cannot masquerade as a new line.+ const cut = Math.max(text.lastIndexOf('\n'), text.lastIndexOf('\r'))+ partial = text.slice(cut + 1).slice(-OUTPUT_TAIL_CHARS)+ for (const line of normalizeCreateOutput(text.slice(0, cut + 1))) {+ if (seen.has(line)) continue+ if (seen.size >= SEEN_LINES_CAP) seen.clear()+ seen.add(line)+ lastProgressAt = Date.now()+ try { onStatus?.(line) } catch (err) { console.error(`❌ ollama create progress handler failed: ${err.message}`) }+ }+ }+ child.stdout?.on('data', onOutput)+ child.stderr?.on('data', onOutput)+ child.on('error', (err) => {+ exited = true+ finish({ status: 'failed', error: `Could not run ollama create: ${err.message}` })+ })+ child.on('close', (code, signal) => {+ exited = true+ if (stalled) return finish({ status: 'stalled', error: stallError, exited: true })+ if (code === 0) return finish({ status: 'completed' })+ finish({ status: 'failed', error: lastErrorLine(tail + partial) || `ollama create exited with ${code ?? signal}` })+ })++ const tick = async () => {+ if (stalled || exited || probing) return+ probing = true+ const signature = await Promise.resolve(probeProgress?.()).catch(() => null).finally(() => { probing = false })+ if (signature != null && signature !== lastSignature) {+ if (lastSignature !== undefined) lastProgressAt = Date.now()+ lastSignature = signature+ }+ if (stalled || exited || Date.now() - lastProgressAt < idleLimitMs) return+ stalled = true+ console.warn(`⚠️ ollama create ${modelId} made no progress for ${idleMinutes} min — stopping it`)+ killWithEscalation(child, { label: `ollama create ${modelId}`, stillRunning: () => !exited, delayMs: CREATE_KILL_GRACE_MS })+ exitTimer = setTimeout(() => finish({ status: 'stalled', error: stallError, exited: false }), CREATE_KILL_GRACE_MS + CREATE_EXIT_BOUND_MS)+ exitTimer.unref?.()+ }+ interval = setInterval(() => {+ tick().catch((err) => console.error(`❌ ollama create watchdog failed: ${err.message}`))+ }, checkIntervalMs)+ interval.unref?.()+ })+}++const markerPath = (tempRoot) => join(tempRoot, MARKER_NAME)++/** Create a staging directory whose marker names the import it belongs to. */+export async function createImportStaging({ modelId, repo, subdir }) {+ const tempRoot = await mkdtemp(join(tmpdir(), STAGING_PREFIX))+ await atomicWrite(markerPath(tempRoot), JSON.stringify({ modelId, repo, subdir, phase: 'downloading', createdAt: new Date().toISOString() }))+ return tempRoot+}++async function readMarker(tempRoot) {+ const raw = await readFile(markerPath(tempRoot), 'utf8').catch(() => null)+ const marker = raw == null ? null : safeJSONParse(raw, null)+ return marker && typeof marker === 'object' ? marker : null+}++export async function updateImportStaging(tempRoot, fields) {+ const marker = await readMarker(tempRoot)+ await atomicWrite(markerPath(tempRoot), JSON.stringify({ ...marker, ...fields }))+}++/** Newest staging directory left behind for `modelId`, or null. */+export async function findImportStaging(modelId) {+ const names = await readdir(tmpdir()).catch(() => [])+ let found = null+ for (const name of names) {+ if (!name.startsWith(STAGING_PREFIX)) continue+ const tempRoot = join(tmpdir(), name)+ const marker = await readMarker(tempRoot)+ if (marker?.modelId !== modelId) continue+ if (!found || String(marker.createdAt) > String(found.marker.createdAt)) found = { tempRoot, marker }+ }+ return found+}++/** True when every expected file is staged at its published size. */+export async function stagedFilesComplete(modelDir, files) {+ for (const file of files) {+ if (!(file.size > 0)) return false+ const size = await stat(join(modelDir, ...file.relativePath.split('/'))).then((s) => s.size, () => -1)+ if (size !== file.size) return false+ }+ return true+}++async function listFilesRecursive(dir) {+ const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])+ const nested = await Promise.all(entries.map((entry) => {+ const path = join(dir, entry.name)+ return entry.isDirectory() ? listFilesRecursive(path) : [path]+ }))+ return nested.flat()+}++/**+ * Wait (bounded) for every other process to let go of the staging directory.+ * @returns {Promise<{ free: true } | { free: false, reason: string, pids?: number[] }>}+ */+export async function awaitStagingRelease(tempRoot, { attempts = RELEASE_PROBE_ATTEMPTS, delayMs = RELEASE_PROBE_DELAY_MS, probe = findOpenFileHolders } = {}) {+ const canary = markerPath(tempRoot)+ const files = (await listFilesRecursive(tempRoot)).filter((path) => path !== canary)+ for (let attempt = 1; attempt <= attempts; attempt++) {+ const result = await probe(files, { canary })+ if (!result.known) return { free: false, reason: result.reason }+ if (result.pids.length === 0) return { free: true }+ if (attempt === attempts) return { free: false, reason: `still open in process ${result.pids.join(', ')}`, pids: result.pids }+ await sleep(delayMs)+ }+ return { free: false, reason: 'not probed' }+}++export async function removeImportStaging(tempRoot) {+ await rm(tempRoot, { recursive: true, force: true }).catch((err) => {+ console.error(`❌ Ollama Safetensors import cleanup failed: ${err.message}`)+ })+}