VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(backend): survive truncated weights and native probe crashes, name port-denied and stalls
velixio8b305ac666c4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex abbe0d4ad..73bfc1f7d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,6 +16,10 @@ metadata and the backend fallback mirror it.- Preview iOS and Android companion apps: drive your own VoiceStudio from your phone (speak, voices and cloning, dubbing, dictation) after pairing by QR code; build from source for now ([guide](docs/mobile.md)) (#2965)+### Fixed++- Backend no longer dies with an access violation on a truncated model file; it re-downloads it, and hangs, port-denied and slow-start failures now name the cause and what to do (#PRNUM)+### Docs- Contributor guides and the pull request template explain how to sign the CLA before opening a first pull request (#2676)diff --git a/backend/core/loop_stall_monitor.py b/backend/core/loop_stall_monitor.pynew file mode 100644index 000000000..dd421b798--- /dev/null+++ b/backend/core/loop_stall_monitor.py@@ -0,0 +1,80 @@+"""Name the code that blocks the event loop when /health stops answering.++The desktop shell polls ``/health`` with a 1.5 s deadline. When a synchronous+call (or a GIL-hungry thread) holds the loop longer than that, the shell+reports "backend busy / not responding" while the log shows only unrelated 200s+- nothing names the culprit (#2970, #2876, #2840, #2794, #2756). A daemon+thread pings the loop and, if a ping goes unanswered for the threshold, dumps+every thread's Python stack once per stall. Pure diagnostics: never raises into+startup, and ``OMNIVOICE_LOOP_STALL_S=0`` disables it.+"""+from __future__ import annotations++import asyncio+import logging+import os+import sys+import threading+import time++logger = logging.getLogger("omnivoice.loop_stall")+_DEFAULT_THRESHOLD_S = 4.0+_POLL_S = 1.0+++def stall_threshold() -> float:+ try:+ return float(os.environ.get("OMNIVOICE_LOOP_STALL_S", _DEFAULT_THRESHOLD_S))+ except ValueError:+ return _DEFAULT_THRESHOLD_S+++def start_loop_stall_monitor(+ loop: asyncio.AbstractEventLoop | None = None,+ *,+ threshold: float | None = None,+ poll: float = _POLL_S,+ dump=None,+) -> threading.Event | None:+ """Start the monitor; returns its stop event, or None when disabled."""+ limit = stall_threshold() if threshold is None else threshold+ if limit <= 0:+ return None+ loop = loop or asyncio.get_running_loop()+ stop = threading.Event()+ last_seen = [time.monotonic()]++ def _dump_threads() -> None:+ import faulthandler++ faulthandler.dump_traceback(file=sys.stderr, all_threads=True)++ emit = dump or _dump_threads++ def _beat() -> None:+ last_seen[0] = time.monotonic()++ def _run() -> None:+ reported = False+ while not stop.wait(poll):+ try:+ loop.call_soon_threadsafe(_beat)+ except RuntimeError: # loop closed: the process is exiting+ return+ lag = time.monotonic() - last_seen[0]+ if lag < limit:+ reported = False+ elif not reported:+ reported = True+ logger.warning(+ "Event loop has not run for %.1fs; /health cannot answer. "+ "Stacks of all threads follow - the main thread's frame is "+ "the blocking call.", lag,+ )+ try:+ emit()+ except Exception: # diagnostics must never take the app down+ logger.debug("stall thread dump failed", exc_info=True)++ threading.Thread(target=_run, name="loop-stall-monitor", daemon=True).start()+ return stopdiff --git a/backend/main.py b/backend/main.pyindex d17f0a667..2260dd494 100644--- a/backend/main.py+++ b/backend/main.py@@ -1218,7 +1218,15 @@ async def lifespan(app: FastAPI):# StartupGate. The watchdog stays armed until the deferred task# disarms it, so a hang inside Phase A/B still gets its thread dump.app.state.startup_task = asyncio.create_task(_deferred_startup(app))+ try:+ from core.loop_stall_monitor import start_loop_stall_monitor++ _loop_stall_stop = start_loop_stall_monitor()+ except Exception:+ _loop_stall_stop = None # diagnostics-onlyyield+ if _loop_stall_stop is not None:+ _loop_stall_stop.set()# ── Graceful shutdown (SIGTERM from Tauri, Ctrl+C, etc.) ────────────# Retire the run sentinel FIRST, before any bounded wait below (#1895):# once uvicorn has begun graceful shutdown the exit is deliberate bydiff --git a/backend/services/hf_cache_repair.py b/backend/services/hf_cache_repair.pyindex 3849fe7a0..19a3cefc3 100644--- a/backend/services/hf_cache_repair.py+++ b/backend/services/hf_cache_repair.py@@ -27,8 +27,10 @@ Conservative by design, repairing STATE rather than chasing one cause:"""from __future__ import annotations+import jsonimport loggingimport os+import structlogger = logging.getLogger("omnivoice.hf_cache_repair")@@ -121,6 +123,64 @@ def find_dangling_entries(repo_cache_dir: str) -> list[str]:return broken+_SAFETENSORS_MAX_HEADER = 100 * 1024 * 1024 # the format's own sanity cap+++def is_truncated_safetensors(path: str) -> bool:+ """True when a ``.safetensors`` file is shorter than its own header says.++ A cut-off download (killed app, full disk, antivirus quarantine) leaves a+ file whose header promises more tensor bytes than exist. transformers+ memory-maps it and the first read past EOF is not a Python exception: on+ Windows it is an access violation (0xC0000005) that kills the whole+ backend mid-load, below anything ``except`` can catch (#3015, #2960,+ #2957, #2382, #2250). Reading only the 8-byte length prefix and the JSON+ header is enough to know, so this costs a few KB per file. Anything this+ cannot parse as safetensors is reported truncated too: it would not load.+ Never raises.+ """+ try:+ size = os.path.getsize(path)+ with open(path, "rb") as fh:+ prefix = fh.read(8)+ if len(prefix) < 8:+ return True+ (header_len,) = struct.unpack("<Q", prefix)+ if header_len <= 0 or header_len > _SAFETENSORS_MAX_HEADER or 8 + header_len > size:+ return True+ header = json.loads(fh.read(header_len))+ end = 0+ for name, info in header.items():+ if name == "__metadata__":+ continue+ end = max(end, int(info["data_offsets"][1]))+ return size < 8 + header_len + end+ except OSError:+ return False # unreadable right now is not evidence of truncation+ except (ValueError, KeyError, TypeError, IndexError, AttributeError):+ return True+++def find_truncated_weights(root: str) -> list[str]:+ """Truncated ``.safetensors`` files under ``root`` (a ``snapshots/`` tree+ or a local checkpoint directory). Dangling links are skipped (they have+ their own heal). Never raises."""+ found: list[str] = []+ try:+ for dirpath, _dirs, files in os.walk(root, followlinks=False):+ for name in files:+ path = os.path.join(dirpath, name)+ if (+ name.lower().endswith(".safetensors")+ and os.path.exists(path)+ and is_truncated_safetensors(path)+ ):+ found.append(path)+ except OSError: # pragma: no cover - defensive+ pass+ return found++def _force_copy_mode(cache_root: str) -> bool:"""Best-effort: make huggingface_hub materialize snapshot entries as realfile COPIES instead of symlinks for the rest of this process.@@ -190,6 +250,8 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict:repo_dir = repo_cache_dir(repo_id, cache_root)summary["repo_dir"] = repo_dirbroken = find_dangling_entries(repo_dir)+ truncated = find_truncated_weights(os.path.join(repo_dir, "snapshots"))+ broken += [p for p in truncated if p not in broken]summary["found"] = len(broken)if not broken:summary["ok"] = True # nothing broken → nothing to do@@ -231,6 +293,16 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict:)return n+ # The cut-off bytes live in blobs/; hub trusts a blob that already+ # exists, so it must go too or the restore would be a no-op.+ blobs_dir = os.path.realpath(os.path.join(repo_dir, "blobs"))+ for path in truncated:+ target = os.path.realpath(path)+ if os.path.dirname(target) == blobs_dir:+ try:+ os.remove(target)+ except OSError as rm_err:+ logger.warning("could not remove truncated blob %s: %s", target, rm_err)summary["removed"] = _remove(broken)if summary["removed"] == 0:summary["error"] = "broken entries could not be removed"diff --git a/backend/services/model_manager.py b/backend/services/model_manager.pyindex 85a3a7be8..9998e0be0 100644--- a/backend/services/model_manager.py+++ b/backend/services/model_manager.py@@ -2175,6 +2175,39 @@ def _selfheal_broken_snapshot_links(checkpoint: str) -> bool:return False+def _preflight_truncated_weights(checkpoint: str) -> None:+ """Refuse to memory-map a cut-off ``.safetensors`` file (#3015, #2382).++ Reading past the end of a truncated mapped file is an access violation on+ Windows that kills the backend with no Python traceback. A header check+ first turns that into a re-download (hub cache) or a clear error (local+ directory). Never raises for healthy caches.+ """+ from services import hf_cache_repair as repair++ if os.path.isdir(checkpoint):+ bad = repair.find_truncated_weights(checkpoint)+ if bad:+ raise RuntimeError(+ "These model weight files are incomplete (a cut-off download): "+ + ", ".join(bad)+ + ". Delete them and download the model again."+ )+ return+ if not repair.find_truncated_weights(+ os.path.join(repair.repo_cache_dir(checkpoint), "snapshots")+ ):+ return+ logger.warning("Model cache for %s has a truncated weight file; re-downloading it.", checkpoint)+ summary = repair.repair_repo_cache(checkpoint)+ if not summary.get("ok"):+ raise RuntimeError(+ f"The model weights for {checkpoint} are incomplete and could not be "+ f"re-downloaded ({summary.get('error') or 'unknown'})."+ + _manual_cache_delete_hint(checkpoint)+ )++def _manual_cache_delete_hint(checkpoint: str) -> str:"""Names the exact on-disk folder to delete when every auto-repair rungfailed — "delete the model" is only actionable if the user can find it.@@ -2583,6 +2616,8 @@ def _load_model_sync():logger.info("Preloading PyTorch Whisper after TTS model load.")else:logger.info("Skipping PyTorch Whisper preload; ASR will load on demand.")+ _preflight_truncated_weights(checkpoint)+def _load():return VoiceStudio.from_pretrained(checkpoint, device_map=device,diff --git a/docs/install/troubleshooting.md b/docs/install/troubleshooting.mdindex a11e1cc23..82209dd94 100644--- a/docs/install/troubleshooting.md+++ b/docs/install/troubleshooting.md@@ -1383,4 +1383,27 @@ If an older installer exits with code `3221225477` at this probe, install theupdated desktop build and retry **Install local runtime**. CPU and ROCm setupnow skip the native CUDA compatibility probe; runtime imports are still verifiedbefore setup finishes. If verification also fails, include the new setup log in-your report rather than deleting saved voices or projects.+your report rather than deleting saved voices or projects. The probe is also+advisory now: if it still crashes natively, setup continues and the import+verification decides whether the runtime works.++### "Backend exited unexpectedly (exit code 3221225477)" while loading a model++A cut-off model download makes Windows kill the backend with an access violation+(`0xC0000005`) when the weights are read. VoiceStudio now checks each+`.safetensors` file's header against its size before loading and re-downloads a+truncated file automatically. If you point `OMNIVOICE_CHECKPOINT` at your own+folder, it names the incomplete file instead; delete it and download it again.++### "Backend is running but busy / not responding on port 3900"++If the log shows `Event loop has not run for N s`, the stack dump right below it+names the call that blocked the backend; include it in your report. Set+`OMNIVOICE_LOOP_STALL_S` to change the 4 s threshold (`0` turns it off).++### Backend exits with code 3 and "[Errno 13] error while attempting to bind"++Windows or another program refused access to the port, often a reserved range+(Hyper-V, WSL, Docker). Check with+`netsh interface ipv4 show excludedportrange protocol=tcp`, then set+`OMNIVOICE_PORT` to a free port such as 3901.diff --git a/electron/src/main/backend-launch.test.ts b/electron/src/main/backend-launch.test.tsindex f4ef7fd9c..37ba8fa03 100644--- a/electron/src/main/backend-launch.test.ts+++ b/electron/src/main/backend-launch.test.ts@@ -180,8 +180,8 @@ it('says what the launch did in a startup-budget failure', () => {const budget = 'Backend did not answer on port 3900 within 300 s (OMNIVOICE_STARTUP_BUDGET_S).';expect(startupTimeoutMessage(3900, 300_000, { owned: true, lastOutput: 'INFO: loading model' }),- ).toBe(`${budget} Last output: INFO: loading model`);- expect(startupTimeoutMessage(3900, 300_000, { owned: true })).toBe(+ ).toContain(`${budget} Last output: INFO: loading model`);+ expect(startupTimeoutMessage(3900, 300_000, { owned: true })).toContain(`${budget} It printed no output.`,);expect(startupTimeoutMessage(3900, 300_000, { owned: false })).toBe(diff --git a/electron/src/main/backend-port.test.ts b/electron/src/main/backend-port.test.tsindex 2321a7097..13ed5c783 100644--- a/electron/src/main/backend-port.test.ts+++ b/electron/src/main/backend-port.test.ts@@ -44,7 +44,7 @@ vi.mock('./runtime-project', () => ({stageRuntimeSources: async () => {},runtimePython: () => '/runtime/python',}));-import { BackendSupervisor } from './backend';+import { BackendSupervisor, bindFailureFromLog } from './backend';import { availableBackendPort } from './backend-port';afterEach(() => {@@ -359,3 +359,14 @@ it.each(['', '3900'])(}},);++it.each([+ ['[Errno 13] error while attempting to bind on address (\'127.0.0.1\', 3900): access denied', 'denied'],+ ['ERROR: [Errno 10013] error while attempting to bind on address (\'127.0.0.1\', 3900)', 'denied'],+ ['[Errno 10048] error while attempting to bind on address (\'127.0.0.1\', 3900)', 'in_use'],+ ['[Errno 98] error while attempting to bind on address (\'127.0.0.1\', 3900)', 'in_use'],+ ['[Errno 2] No such file or directory', null],+ ['INFO: Application shutdown complete.', null],+])('classifies uvicorn bind failures by errno (#3020): %s', (log, expected) => {+ expect(bindFailureFromLog(log)).toBe(expected);+});diff --git a/electron/src/main/backend.ts b/electron/src/main/backend.tsindex 00a336857..a3a0c9e39 100644--- a/electron/src/main/backend.ts+++ b/electron/src/main/backend.ts@@ -86,6 +86,22 @@ const LOG_RING_LINES = 200;const LOG_TAIL_LINES = 40;/** EX_CONFIG (sysexits.h): backend/main.py exits with it when the port is taken (#1223). */const EXIT_PORT_IN_USE = 78;++/**+ * The desktop shell starts `python -m uvicorn`, not backend/main.py's+ * `__main__`, so main.py's own bind diagnostics (exit 78 / permission-denied+ * text) never run. uvicorn instead exits 3 after logging asyncio's+ * "[Errno N] error while attempting to bind on address ..." line. The errno+ * is locale-independent, so classify on it (#3020).+ */+export function bindFailureFromLog(text: string): 'in_use' | 'denied' | null {+ const match = /\[Errno (-?\d+)\] error while attempting to bind on address/.exec(text);+ if (!match) return null;+ const errno = Number(match[1]);+ if ([48, 98, 10048].includes(errno)) return 'in_use';+ if ([13, 10013].includes(errno)) return 'denied';+ return null;+}const POSIX_SIGKILL_AFTER_MS = 2000;const TAURI_APP_ID = 'com.debpalash.omnivoice-studio';const RUNTIME_LOCATION_FILE = 'runtime-location.json';@@ -552,7 +568,13 @@ export function startupTimeoutMessage(`Backend did not answer on port ${port} within ${Math.round(budgetMs / 1000)} s ` +'(OMNIVOICE_STARTUP_BUDGET_S).';if (!owned) return `${base} Nothing was spawned for this attempt.`;- return lastOutput ? `${base} Last output: ${lastOutput}` : `${base} It printed no output.`;+ const advice =+ ' Slow disks and antivirus scanning of the runtime folder are the usual cause: click Retry, ' ++ 'exclude the VoiceStudio runtime folder from real-time scanning, or raise OMNIVOICE_STARTUP_BUDGET_S ' ++ '(for example 900).';+ return lastOutput+ ? `${base} Last output: ${lastOutput}${advice}`+ : `${base} It printed no output.${advice}`;}function delay(ms: number): Promise<void> {@@ -921,7 +943,13 @@ export class BackendSupervisor extends EventEmitter<{child.on('close', (code) => {if (this.child === child) this.child = null;if (code === 0) resolve(capturedStdout);- else reject(new Error(`Runtime setup exited with code ${code}`));+ else+ reject(+ Object.assign(new Error(`Runtime setup exited with code ${code}`), {+ code,+ stdout: capturedStdout,+ }),+ );});}),controller.signal,@@ -1318,12 +1346,25 @@ export class BackendSupervisor extends EventEmitter<{const recorded = this.crashes.record(code, signal, Date.now() - this.startedAt, this.log);this.exitCode = code;this.exitSignal = signal;- if (code === EXIT_PORT_IN_USE) {+ const bindFailure =+ code === EXIT_PORT_IN_USE+ ? 'in_use'+ : bindFailureFromLog(this.childLog.join('\n'));+ if (bindFailure === 'in_use') {this.setStage('port_in_use', {message: `Port ${this.port} is already in use by another process. Stop it or set OMNIVOICE_PORT.`,});return;}+ if (bindFailure === 'denied') {+ this.setStage('port_in_use', {+ message:+ `Windows or another program refused access to port ${this.port}. It is often in a reserved ` ++ 'port range (Hyper-V, WSL, Docker) - run "netsh interface ipv4 show excludedportrange protocol=tcp" ' ++ 'to check - or held exclusively. Set OMNIVOICE_PORT to a free port (for example 3901) and restart.',+ });+ return;+ }// A native fault's last line is whatever followed the dump: the stdlib// frame that started the process, an extension-module list or an access// log line. Name the fault and where it happened instead (#2382, #2187).diff --git a/electron/src/main/runtime-project.test.ts b/electron/src/main/runtime-project.test.tsindex 9e6a9b411..a0e98bc5b 100644--- a/electron/src/main/runtime-project.test.ts+++ b/electron/src/main/runtime-project.test.ts@@ -108,6 +108,30 @@ describe('packaged runtime setup', () => {expect(await runtimeReady(bundle, project)).toBe(true);},);+ it.each([+ ['after printing its result', true],+ ['before printing anything', false],+ ])('keeps setting up when the CUDA compatibility probe crashes natively %s', async (_name, printed) => {+ forcePlatform('win32');+ vi.spyOn(process, 'arch', 'get').mockReturnValue('x64');+ vi.stubEnv('OMNIVOICE_TORCH_VARIANT', 'cuda');+ const { bundle, project } = await fixture();+ const sitePackages = join(project, '.venv', 'Lib', 'site-packages');+ const run = vi.fn(async (_command: string, args: string[]) => {+ if (args[0] === 'sync') await interpreter(project);+ if (args[1]?.includes('VOICESTUDIO_CUDNN8_PROBE=')) {+ throw Object.assign(new Error('Runtime setup exited with code 3221225477'), {+ stdout: printed+ ? `VOICESTUDIO_CUDNN8_PROBE=${JSON.stringify({ device: 'none', sitePackages })}\n`+ : '',+ });+ }+ return undefined;+ });+ await installRuntime(bundle, project, 'uv', run, new AbortController().signal, undefined, 'global');+ expect(run.mock.calls.some(([, args]) => args[1] === RUNTIME_IMPORT_PROBE)).toBe(true);+ expect(await runtimeReady(bundle, project)).toBe(true);+ });it.each([0, 1])('verifies the runtime when .pth repair attempt %i fails',async (failedAttempt) => {diff --git a/electron/src/main/runtime-project.ts b/electron/src/main/runtime-project.tsindex f5fa1ab5e..428c51224 100644--- a/electron/src/main/runtime-project.ts+++ b/electron/src/main/runtime-project.ts@@ -291,9 +291,25 @@ async function ensureCudnn8Compat(' pass',]: [`device = '${variant === 'rocm' ? 'hip' : 'none'}'`]),- `print('${CUDNN8_PROBE_PREFIX}' + json.dumps({'device': device, 'sitePackages': sysconfig.get_paths()['purelib']}))`,+ `print('${CUDNN8_PROBE_PREFIX}' + json.dumps({'device': device, 'sitePackages': sysconfig.get_paths()['purelib']}), flush=True)`,+ // Skip interpreter teardown: unloading native Torch/CUDA DLLs at exit is+ // where Windows access violations (0xC0000005) surface after the result+ // has already been printed (#2942).+ 'import os',+ 'os._exit(0)',].join('\n');- const probe = parseCudnn8Probe(await run(python, ['-c', script], project, env));+ // The probe is advisory. A native crash inside it must never abort runtime+ // setup: fall back to "no CUDA compat libraries needed" and let the real+ // import verification below decide whether the runtime works.+ let probeOutput: string | void;+ try {+ probeOutput = await run(python, ['-c', script], project, env);+ } catch (error) {+ signal.throwIfAborted();+ probeOutput = (error as { stdout?: string }).stdout;+ if (!parseCudnn8Probe(probeOutput)) return;+ }+ const probe = parseCudnn8Probe(probeOutput);signal.throwIfAborted();if (!probe) return;await clearCtranslate2ExecutableStack(probe.sitePackages);diff --git a/tests/test_loop_stall_monitor.py b/tests/test_loop_stall_monitor.pynew file mode 100644index 000000000..f2cf17f57--- /dev/null+++ b/tests/test_loop_stall_monitor.py@@ -0,0 +1,36 @@+"""A blocked event loop must leave a thread dump naming the blocker."""+import asyncio+import time++from core.loop_stall_monitor import start_loop_stall_monitor+++def test_dumps_once_when_loop_is_blocked():+ dumps = []++ async def scenario():+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))+ await asyncio.sleep(0.2)+ time.sleep(0.9) # the synchronous call that blocks /health+ await asyncio.sleep(0.2)+ stop.set()++ asyncio.run(scenario())+ assert len(dumps) == 1+++def test_silent_when_loop_is_healthy():+ dumps = []++ async def scenario():+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))+ await asyncio.sleep(0.6)+ stop.set()++ asyncio.run(scenario())+ assert dumps == []+++def test_disabled_by_zero(monkeypatch):+ monkeypatch.setenv("OMNIVOICE_LOOP_STALL_S", "0")+ assert start_loop_stall_monitor(loop=object()) is Nonediff --git a/tests/test_truncated_safetensors_preflight.py b/tests/test_truncated_safetensors_preflight.pynew file mode 100644index 000000000..796b0e5d5--- /dev/null+++ b/tests/test_truncated_safetensors_preflight.py@@ -0,0 +1,71 @@+"""A cut-off .safetensors must be caught before mmap, not by an access violation.++transformers memory-maps weights; reading past EOF of a truncated file is a+Windows 0xC0000005 that kills the backend with no Python traceback+(#3015, #2960, #2957, #2382, #2250).+"""+from __future__ import annotations++import json+import struct++import pytest++from services import hf_cache_repair as repair+from services import model_manager+++def _st(path, payload_bytes=64, keep=None):+ header = json.dumps({"w": {"dtype": "U8", "shape": [payload_bytes],+ "data_offsets": [0, payload_bytes]}}).encode()+ data = struct.pack("<Q", len(header)) + header + b"\0" * payload_bytes+ path.write_bytes(data if keep is None else data[:keep])+ return path+++def test_complete_file_is_not_truncated(tmp_path):+ assert not repair.is_truncated_safetensors(str(_st(tmp_path / "a.safetensors")))++[email protected]("keep", [0, 5, 20, 100])+def test_cut_off_file_is_truncated(tmp_path, keep):+ assert repair.is_truncated_safetensors(str(_st(tmp_path / "a.safetensors", keep=keep)))+++def test_find_truncated_ignores_healthy(tmp_path):+ _st(tmp_path / "ok.safetensors")+ bad = _st(tmp_path / "bad.safetensors", keep=90)+ assert repair.find_truncated_weights(str(tmp_path)) == [str(bad)]+++def test_preflight_refuses_truncated_local_checkpoint(tmp_path):+ _st(tmp_path / "model.safetensors", keep=90)+ with pytest.raises(RuntimeError, match="incomplete"):+ model_manager._preflight_truncated_weights(str(tmp_path))+++def test_preflight_passes_healthy_local_checkpoint(tmp_path):+ _st(tmp_path / "model.safetensors")+ model_manager._preflight_truncated_weights(str(tmp_path))+++def test_repair_removes_truncated_blob_and_link(tmp_path, monkeypatch):+ from services import hf_revisions+ monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)+ monkeypatch.setitem(hf_revisions.CURATED_REVISIONS, "test/checkpoint", "a" * 40)+ repo = tmp_path / "models--test--checkpoint"+ (repo / "blobs").mkdir(parents=True)+ snap = repo / "snapshots" / "abc"+ snap.mkdir(parents=True)+ blob = _st(repo / "blobs" / "b1", keep=90)+ try:+ (snap / "model.safetensors").symlink_to(blob)+ except (OSError, NotImplementedError):+ pytest.skip("symlinks unavailable")+ calls = []+ import huggingface_hub+ monkeypatch.setattr(huggingface_hub, "snapshot_download", lambda **kw: calls.append(kw))+ out = repair.repair_repo_cache("test/checkpoint", cache_dir=str(tmp_path))+ assert out["found"] == 1 and out["restored"] and calls+ assert not blob.exists() and not (snap / "model.safetensors").is_symlink()