Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(backend): survive truncated weights and native probe crashes, name port-denied and stalls

velixio8b305ac666c4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index abbe0d4ad..73bfc1f7d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,10 @@ metadata and the backend fallback mirror it.
- Preview iOS and Android companion apps: drive your own VoiceStudio from your phone (speak, voices and cloning, dubbing, dictation) after pairing by QR code; build from source for now ([guide](docs/mobile.md)) (#2965)
+### Fixed
+
+- Backend no longer dies with an access violation on a truncated model file; it re-downloads it, and hangs, port-denied and slow-start failures now name the cause and what to do (#PRNUM)
+
### Docs
- Contributor guides and the pull request template explain how to sign the CLA before opening a first pull request (#2676)
diff --git a/backend/core/loop_stall_monitor.py b/backend/core/loop_stall_monitor.py
new file mode 100644
index 000000000..dd421b798
--- /dev/null
+++ b/backend/core/loop_stall_monitor.py
@@ -0,0 +1,80 @@
+"""Name the code that blocks the event loop when /health stops answering.
+
+The desktop shell polls ``/health`` with a 1.5 s deadline. When a synchronous
+call (or a GIL-hungry thread) holds the loop longer than that, the shell
+reports "backend busy / not responding" while the log shows only unrelated 200s
+- nothing names the culprit (#2970, #2876, #2840, #2794, #2756). A daemon
+thread pings the loop and, if a ping goes unanswered for the threshold, dumps
+every thread's Python stack once per stall. Pure diagnostics: never raises into
+startup, and ``OMNIVOICE_LOOP_STALL_S=0`` disables it.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import sys
+import threading
+import time
+
+logger = logging.getLogger("omnivoice.loop_stall")
+_DEFAULT_THRESHOLD_S = 4.0
+_POLL_S = 1.0
+
+
+def stall_threshold() -> float:
+ try:
+ return float(os.environ.get("OMNIVOICE_LOOP_STALL_S", _DEFAULT_THRESHOLD_S))
+ except ValueError:
+ return _DEFAULT_THRESHOLD_S
+
+
+def start_loop_stall_monitor(
+ loop: asyncio.AbstractEventLoop | None = None,
+ *,
+ threshold: float | None = None,
+ poll: float = _POLL_S,
+ dump=None,
+) -> threading.Event | None:
+ """Start the monitor; returns its stop event, or None when disabled."""
+ limit = stall_threshold() if threshold is None else threshold
+ if limit <= 0:
+ return None
+ loop = loop or asyncio.get_running_loop()
+ stop = threading.Event()
+ last_seen = [time.monotonic()]
+
+ def _dump_threads() -> None:
+ import faulthandler
+
+ faulthandler.dump_traceback(file=sys.stderr, all_threads=True)
+
+ emit = dump or _dump_threads
+
+ def _beat() -> None:
+ last_seen[0] = time.monotonic()
+
+ def _run() -> None:
+ reported = False
+ while not stop.wait(poll):
+ try:
+ loop.call_soon_threadsafe(_beat)
+ except RuntimeError: # loop closed: the process is exiting
+ return
+ lag = time.monotonic() - last_seen[0]
+ if lag < limit:
+ reported = False
+ elif not reported:
+ reported = True
+ logger.warning(
+ "Event loop has not run for %.1fs; /health cannot answer. "
+ "Stacks of all threads follow - the main thread's frame is "
+ "the blocking call.", lag,
+ )
+ try:
+ emit()
+ except Exception: # diagnostics must never take the app down
+ logger.debug("stall thread dump failed", exc_info=True)
+
+ threading.Thread(target=_run, name="loop-stall-monitor", daemon=True).start()
+ return stop
diff --git a/backend/main.py b/backend/main.py
index d17f0a667..2260dd494 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -1218,7 +1218,15 @@ async def lifespan(app: FastAPI):
# StartupGate. The watchdog stays armed until the deferred task
# disarms it, so a hang inside Phase A/B still gets its thread dump.
app.state.startup_task = asyncio.create_task(_deferred_startup(app))
+ try:
+ from core.loop_stall_monitor import start_loop_stall_monitor
+
+ _loop_stall_stop = start_loop_stall_monitor()
+ except Exception:
+ _loop_stall_stop = None # diagnostics-only
yield
+ if _loop_stall_stop is not None:
+ _loop_stall_stop.set()
# ── Graceful shutdown (SIGTERM from Tauri, Ctrl+C, etc.) ────────────
# Retire the run sentinel FIRST, before any bounded wait below (#1895):
# once uvicorn has begun graceful shutdown the exit is deliberate by
diff --git a/backend/services/hf_cache_repair.py b/backend/services/hf_cache_repair.py
index 3849fe7a0..19a3cefc3 100644
--- a/backend/services/hf_cache_repair.py
+++ b/backend/services/hf_cache_repair.py
@@ -27,8 +27,10 @@ Conservative by design, repairing STATE rather than chasing one cause:
"""
from __future__ import annotations
+import json
import logging
import os
+import struct
logger = logging.getLogger("omnivoice.hf_cache_repair")
@@ -121,6 +123,64 @@ def find_dangling_entries(repo_cache_dir: str) -> list[str]:
return broken
+_SAFETENSORS_MAX_HEADER = 100 * 1024 * 1024 # the format's own sanity cap
+
+
+def is_truncated_safetensors(path: str) -> bool:
+ """True when a ``.safetensors`` file is shorter than its own header says.
+
+ A cut-off download (killed app, full disk, antivirus quarantine) leaves a
+ file whose header promises more tensor bytes than exist. transformers
+ memory-maps it and the first read past EOF is not a Python exception: on
+ Windows it is an access violation (0xC0000005) that kills the whole
+ backend mid-load, below anything ``except`` can catch (#3015, #2960,
+ #2957, #2382, #2250). Reading only the 8-byte length prefix and the JSON
+ header is enough to know, so this costs a few KB per file. Anything this
+ cannot parse as safetensors is reported truncated too: it would not load.
+ Never raises.
+ """
+ try:
+ size = os.path.getsize(path)
+ with open(path, "rb") as fh:
+ prefix = fh.read(8)
+ if len(prefix) < 8:
+ return True
+ (header_len,) = struct.unpack("<Q", prefix)
+ if header_len <= 0 or header_len > _SAFETENSORS_MAX_HEADER or 8 + header_len > size:
+ return True
+ header = json.loads(fh.read(header_len))
+ end = 0
+ for name, info in header.items():
+ if name == "__metadata__":
+ continue
+ end = max(end, int(info["data_offsets"][1]))
+ return size < 8 + header_len + end
+ except OSError:
+ return False # unreadable right now is not evidence of truncation
+ except (ValueError, KeyError, TypeError, IndexError, AttributeError):
+ return True
+
+
+def find_truncated_weights(root: str) -> list[str]:
+ """Truncated ``.safetensors`` files under ``root`` (a ``snapshots/`` tree
+ or a local checkpoint directory). Dangling links are skipped (they have
+ their own heal). Never raises."""
+ found: list[str] = []
+ try:
+ for dirpath, _dirs, files in os.walk(root, followlinks=False):
+ for name in files:
+ path = os.path.join(dirpath, name)
+ if (
+ name.lower().endswith(".safetensors")
+ and os.path.exists(path)
+ and is_truncated_safetensors(path)
+ ):
+ found.append(path)
+ except OSError: # pragma: no cover - defensive
+ pass
+ return found
+
+
def _force_copy_mode(cache_root: str) -> bool:
"""Best-effort: make huggingface_hub materialize snapshot entries as real
file COPIES instead of symlinks for the rest of this process.
@@ -190,6 +250,8 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict:
repo_dir = repo_cache_dir(repo_id, cache_root)
summary["repo_dir"] = repo_dir
broken = find_dangling_entries(repo_dir)
+ truncated = find_truncated_weights(os.path.join(repo_dir, "snapshots"))
+ broken += [p for p in truncated if p not in broken]
summary["found"] = len(broken)
if not broken:
summary["ok"] = True # nothing broken → nothing to do
@@ -231,6 +293,16 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict:
)
return n
+ # The cut-off bytes live in blobs/; hub trusts a blob that already
+ # exists, so it must go too or the restore would be a no-op.
+ blobs_dir = os.path.realpath(os.path.join(repo_dir, "blobs"))
+ for path in truncated:
+ target = os.path.realpath(path)
+ if os.path.dirname(target) == blobs_dir:
+ try:
+ os.remove(target)
+ except OSError as rm_err:
+ logger.warning("could not remove truncated blob %s: %s", target, rm_err)
summary["removed"] = _remove(broken)
if summary["removed"] == 0:
summary["error"] = "broken entries could not be removed"
diff --git a/backend/services/model_manager.py b/backend/services/model_manager.py
index 85a3a7be8..9998e0be0 100644
--- a/backend/services/model_manager.py
+++ b/backend/services/model_manager.py
@@ -2175,6 +2175,39 @@ def _selfheal_broken_snapshot_links(checkpoint: str) -> bool:
return False
+def _preflight_truncated_weights(checkpoint: str) -> None:
+ """Refuse to memory-map a cut-off ``.safetensors`` file (#3015, #2382).
+
+ Reading past the end of a truncated mapped file is an access violation on
+ Windows that kills the backend with no Python traceback. A header check
+ first turns that into a re-download (hub cache) or a clear error (local
+ directory). Never raises for healthy caches.
+ """
+ from services import hf_cache_repair as repair
+
+ if os.path.isdir(checkpoint):
+ bad = repair.find_truncated_weights(checkpoint)
+ if bad:
+ raise RuntimeError(
+ "These model weight files are incomplete (a cut-off download): "
+ + ", ".join(bad)
+ + ". Delete them and download the model again."
+ )
+ return
+ if not repair.find_truncated_weights(
+ os.path.join(repair.repo_cache_dir(checkpoint), "snapshots")
+ ):
+ return
+ logger.warning("Model cache for %s has a truncated weight file; re-downloading it.", checkpoint)
+ summary = repair.repair_repo_cache(checkpoint)
+ if not summary.get("ok"):
+ raise RuntimeError(
+ f"The model weights for {checkpoint} are incomplete and could not be "
+ f"re-downloaded ({summary.get('error') or 'unknown'})."
+ + _manual_cache_delete_hint(checkpoint)
+ )
+
+
def _manual_cache_delete_hint(checkpoint: str) -> str:
"""Names the exact on-disk folder to delete when every auto-repair rung
failed — "delete the model" is only actionable if the user can find it.
@@ -2583,6 +2616,8 @@ def _load_model_sync():
logger.info("Preloading PyTorch Whisper after TTS model load.")
else:
logger.info("Skipping PyTorch Whisper preload; ASR will load on demand.")
+ _preflight_truncated_weights(checkpoint)
+
def _load():
return VoiceStudio.from_pretrained(
checkpoint, device_map=device,
diff --git a/docs/install/troubleshooting.md b/docs/install/troubleshooting.md
index a11e1cc23..82209dd94 100644
--- a/docs/install/troubleshooting.md
+++ b/docs/install/troubleshooting.md
@@ -1383,4 +1383,27 @@ If an older installer exits with code `3221225477` at this probe, install the
updated desktop build and retry **Install local runtime**. CPU and ROCm setup
now skip the native CUDA compatibility probe; runtime imports are still verified
before setup finishes. If verification also fails, include the new setup log in
-your report rather than deleting saved voices or projects.
+your report rather than deleting saved voices or projects. The probe is also
+advisory now: if it still crashes natively, setup continues and the import
+verification decides whether the runtime works.
+
+### "Backend exited unexpectedly (exit code 3221225477)" while loading a model
+
+A cut-off model download makes Windows kill the backend with an access violation
+(`0xC0000005`) when the weights are read. VoiceStudio now checks each
+`.safetensors` file's header against its size before loading and re-downloads a
+truncated file automatically. If you point `OMNIVOICE_CHECKPOINT` at your own
+folder, it names the incomplete file instead; delete it and download it again.
+
+### "Backend is running but busy / not responding on port 3900"
+
+If the log shows `Event loop has not run for N s`, the stack dump right below it
+names the call that blocked the backend; include it in your report. Set
+`OMNIVOICE_LOOP_STALL_S` to change the 4 s threshold (`0` turns it off).
+
+### Backend exits with code 3 and "[Errno 13] error while attempting to bind"
+
+Windows or another program refused access to the port, often a reserved range
+(Hyper-V, WSL, Docker). Check with
+`netsh interface ipv4 show excludedportrange protocol=tcp`, then set
+`OMNIVOICE_PORT` to a free port such as 3901.
diff --git a/electron/src/main/backend-launch.test.ts b/electron/src/main/backend-launch.test.ts
index f4ef7fd9c..37ba8fa03 100644
--- a/electron/src/main/backend-launch.test.ts
+++ b/electron/src/main/backend-launch.test.ts
@@ -180,8 +180,8 @@ it('says what the launch did in a startup-budget failure', () => {
const budget = 'Backend did not answer on port 3900 within 300 s (OMNIVOICE_STARTUP_BUDGET_S).';
expect(
startupTimeoutMessage(3900, 300_000, { owned: true, lastOutput: 'INFO: loading model' }),
- ).toBe(`${budget} Last output: INFO: loading model`);
- expect(startupTimeoutMessage(3900, 300_000, { owned: true })).toBe(
+ ).toContain(`${budget} Last output: INFO: loading model`);
+ expect(startupTimeoutMessage(3900, 300_000, { owned: true })).toContain(
`${budget} It printed no output.`,
);
expect(startupTimeoutMessage(3900, 300_000, { owned: false })).toBe(
diff --git a/electron/src/main/backend-port.test.ts b/electron/src/main/backend-port.test.ts
index 2321a7097..13ed5c783 100644
--- a/electron/src/main/backend-port.test.ts
+++ b/electron/src/main/backend-port.test.ts
@@ -44,7 +44,7 @@ vi.mock('./runtime-project', () => ({
stageRuntimeSources: async () => {},
runtimePython: () => '/runtime/python',
}));
-import { BackendSupervisor } from './backend';
+import { BackendSupervisor, bindFailureFromLog } from './backend';
import { availableBackendPort } from './backend-port';
afterEach(() => {
@@ -359,3 +359,14 @@ it.each(['', '3900'])(
}
},
);
+
+it.each([
+ ['[Errno 13] error while attempting to bind on address (\'127.0.0.1\', 3900): access denied', 'denied'],
+ ['ERROR: [Errno 10013] error while attempting to bind on address (\'127.0.0.1\', 3900)', 'denied'],
+ ['[Errno 10048] error while attempting to bind on address (\'127.0.0.1\', 3900)', 'in_use'],
+ ['[Errno 98] error while attempting to bind on address (\'127.0.0.1\', 3900)', 'in_use'],
+ ['[Errno 2] No such file or directory', null],
+ ['INFO: Application shutdown complete.', null],
+])('classifies uvicorn bind failures by errno (#3020): %s', (log, expected) => {
+ expect(bindFailureFromLog(log)).toBe(expected);
+});
diff --git a/electron/src/main/backend.ts b/electron/src/main/backend.ts
index 00a336857..a3a0c9e39 100644
--- a/electron/src/main/backend.ts
+++ b/electron/src/main/backend.ts
@@ -86,6 +86,22 @@ const LOG_RING_LINES = 200;
const LOG_TAIL_LINES = 40;
/** EX_CONFIG (sysexits.h): backend/main.py exits with it when the port is taken (#1223). */
const EXIT_PORT_IN_USE = 78;
+
+/**
+ * The desktop shell starts `python -m uvicorn`, not backend/main.py's
+ * `__main__`, so main.py's own bind diagnostics (exit 78 / permission-denied
+ * text) never run. uvicorn instead exits 3 after logging asyncio's
+ * "[Errno N] error while attempting to bind on address ..." line. The errno
+ * is locale-independent, so classify on it (#3020).
+ */
+export function bindFailureFromLog(text: string): 'in_use' | 'denied' | null {
+ const match = /\[Errno (-?\d+)\] error while attempting to bind on address/.exec(text);
+ if (!match) return null;
+ const errno = Number(match[1]);
+ if ([48, 98, 10048].includes(errno)) return 'in_use';
+ if ([13, 10013].includes(errno)) return 'denied';
+ return null;
+}
const POSIX_SIGKILL_AFTER_MS = 2000;
const TAURI_APP_ID = 'com.debpalash.omnivoice-studio';
const RUNTIME_LOCATION_FILE = 'runtime-location.json';
@@ -552,7 +568,13 @@ export function startupTimeoutMessage(
`Backend did not answer on port ${port} within ${Math.round(budgetMs / 1000)} s ` +
'(OMNIVOICE_STARTUP_BUDGET_S).';
if (!owned) return `${base} Nothing was spawned for this attempt.`;
- return lastOutput ? `${base} Last output: ${lastOutput}` : `${base} It printed no output.`;
+ const advice =
+ ' Slow disks and antivirus scanning of the runtime folder are the usual cause: click Retry, ' +
+ 'exclude the VoiceStudio runtime folder from real-time scanning, or raise OMNIVOICE_STARTUP_BUDGET_S ' +
+ '(for example 900).';
+ return lastOutput
+ ? `${base} Last output: ${lastOutput}${advice}`
+ : `${base} It printed no output.${advice}`;
}
function delay(ms: number): Promise<void> {
@@ -921,7 +943,13 @@ export class BackendSupervisor extends EventEmitter<{
child.on('close', (code) => {
if (this.child === child) this.child = null;
if (code === 0) resolve(capturedStdout);
- else reject(new Error(`Runtime setup exited with code ${code}`));
+ else
+ reject(
+ Object.assign(new Error(`Runtime setup exited with code ${code}`), {
+ code,
+ stdout: capturedStdout,
+ }),
+ );
});
}),
controller.signal,
@@ -1318,12 +1346,25 @@ export class BackendSupervisor extends EventEmitter<{
const recorded = this.crashes.record(code, signal, Date.now() - this.startedAt, this.log);
this.exitCode = code;
this.exitSignal = signal;
- if (code === EXIT_PORT_IN_USE) {
+ const bindFailure =
+ code === EXIT_PORT_IN_USE
+ ? 'in_use'
+ : bindFailureFromLog(this.childLog.join('\n'));
+ if (bindFailure === 'in_use') {
this.setStage('port_in_use', {
message: `Port ${this.port} is already in use by another process. Stop it or set OMNIVOICE_PORT.`,
});
return;
}
+ if (bindFailure === 'denied') {
+ this.setStage('port_in_use', {
+ message:
+ `Windows or another program refused access to port ${this.port}. It is often in a reserved ` +
+ 'port range (Hyper-V, WSL, Docker) - run "netsh interface ipv4 show excludedportrange protocol=tcp" ' +
+ 'to check - or held exclusively. Set OMNIVOICE_PORT to a free port (for example 3901) and restart.',
+ });
+ return;
+ }
// A native fault's last line is whatever followed the dump: the stdlib
// frame that started the process, an extension-module list or an access
// log line. Name the fault and where it happened instead (#2382, #2187).
diff --git a/electron/src/main/runtime-project.test.ts b/electron/src/main/runtime-project.test.ts
index 9e6a9b411..a0e98bc5b 100644
--- a/electron/src/main/runtime-project.test.ts
+++ b/electron/src/main/runtime-project.test.ts
@@ -108,6 +108,30 @@ describe('packaged runtime setup', () => {
expect(await runtimeReady(bundle, project)).toBe(true);
},
);
+ it.each([
+ ['after printing its result', true],
+ ['before printing anything', false],
+ ])('keeps setting up when the CUDA compatibility probe crashes natively %s', async (_name, printed) => {
+ forcePlatform('win32');
+ vi.spyOn(process, 'arch', 'get').mockReturnValue('x64');
+ vi.stubEnv('OMNIVOICE_TORCH_VARIANT', 'cuda');
+ const { bundle, project } = await fixture();
+ const sitePackages = join(project, '.venv', 'Lib', 'site-packages');
+ const run = vi.fn(async (_command: string, args: string[]) => {
+ if (args[0] === 'sync') await interpreter(project);
+ if (args[1]?.includes('VOICESTUDIO_CUDNN8_PROBE=')) {
+ throw Object.assign(new Error('Runtime setup exited with code 3221225477'), {
+ stdout: printed
+ ? `VOICESTUDIO_CUDNN8_PROBE=${JSON.stringify({ device: 'none', sitePackages })}\n`
+ : '',
+ });
+ }
+ return undefined;
+ });
+ await installRuntime(bundle, project, 'uv', run, new AbortController().signal, undefined, 'global');
+ expect(run.mock.calls.some(([, args]) => args[1] === RUNTIME_IMPORT_PROBE)).toBe(true);
+ expect(await runtimeReady(bundle, project)).toBe(true);
+ });
it.each([0, 1])(
'verifies the runtime when .pth repair attempt %i fails',
async (failedAttempt) => {
diff --git a/electron/src/main/runtime-project.ts b/electron/src/main/runtime-project.ts
index f5fa1ab5e..428c51224 100644
--- a/electron/src/main/runtime-project.ts
+++ b/electron/src/main/runtime-project.ts
@@ -291,9 +291,25 @@ async function ensureCudnn8Compat(
' pass',
]
: [`device = '${variant === 'rocm' ? 'hip' : 'none'}'`]),
- `print('${CUDNN8_PROBE_PREFIX}' + json.dumps({'device': device, 'sitePackages': sysconfig.get_paths()['purelib']}))`,
+ `print('${CUDNN8_PROBE_PREFIX}' + json.dumps({'device': device, 'sitePackages': sysconfig.get_paths()['purelib']}), flush=True)`,
+ // Skip interpreter teardown: unloading native Torch/CUDA DLLs at exit is
+ // where Windows access violations (0xC0000005) surface after the result
+ // has already been printed (#2942).
+ 'import os',
+ 'os._exit(0)',
].join('\n');
- const probe = parseCudnn8Probe(await run(python, ['-c', script], project, env));
+ // The probe is advisory. A native crash inside it must never abort runtime
+ // setup: fall back to "no CUDA compat libraries needed" and let the real
+ // import verification below decide whether the runtime works.
+ let probeOutput: string | void;
+ try {
+ probeOutput = await run(python, ['-c', script], project, env);
+ } catch (error) {
+ signal.throwIfAborted();
+ probeOutput = (error as { stdout?: string }).stdout;
+ if (!parseCudnn8Probe(probeOutput)) return;
+ }
+ const probe = parseCudnn8Probe(probeOutput);
signal.throwIfAborted();
if (!probe) return;
await clearCtranslate2ExecutableStack(probe.sitePackages);
diff --git a/tests/test_loop_stall_monitor.py b/tests/test_loop_stall_monitor.py
new file mode 100644
index 000000000..f2cf17f57
--- /dev/null
+++ b/tests/test_loop_stall_monitor.py
@@ -0,0 +1,36 @@
+"""A blocked event loop must leave a thread dump naming the blocker."""
+import asyncio
+import time
+
+from core.loop_stall_monitor import start_loop_stall_monitor
+
+
+def test_dumps_once_when_loop_is_blocked():
+ dumps = []
+
+ async def scenario():
+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))
+ await asyncio.sleep(0.2)
+ time.sleep(0.9) # the synchronous call that blocks /health
+ await asyncio.sleep(0.2)
+ stop.set()
+
+ asyncio.run(scenario())
+ assert len(dumps) == 1
+
+
+def test_silent_when_loop_is_healthy():
+ dumps = []
+
+ async def scenario():
+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))
+ await asyncio.sleep(0.6)
+ stop.set()
+
+ asyncio.run(scenario())
+ assert dumps == []
+
+
+def test_disabled_by_zero(monkeypatch):
+ monkeypatch.setenv("OMNIVOICE_LOOP_STALL_S", "0")
+ assert start_loop_stall_monitor(loop=object()) is None
diff --git a/tests/test_truncated_safetensors_preflight.py b/tests/test_truncated_safetensors_preflight.py
new file mode 100644
index 000000000..796b0e5d5
--- /dev/null
+++ b/tests/test_truncated_safetensors_preflight.py
@@ -0,0 +1,71 @@
+"""A cut-off .safetensors must be caught before mmap, not by an access violation.
+
+transformers memory-maps weights; reading past EOF of a truncated file is a
+Windows 0xC0000005 that kills the backend with no Python traceback
+(#3015, #2960, #2957, #2382, #2250).
+"""
+from __future__ import annotations
+
+import json
+import struct
+
+import pytest
+
+from services import hf_cache_repair as repair
+from services import model_manager
+
+
+def _st(path, payload_bytes=64, keep=None):
+ header = json.dumps({"w": {"dtype": "U8", "shape": [payload_bytes],
+ "data_offsets": [0, payload_bytes]}}).encode()
+ data = struct.pack("<Q", len(header)) + header + b"\0" * payload_bytes
+ path.write_bytes(data if keep is None else data[:keep])
+ return path
+
+
+def test_complete_file_is_not_truncated(tmp_path):
+ assert not repair.is_truncated_safetensors(str(_st(tmp_path / "a.safetensors")))
+
+
[email protected]("keep", [0, 5, 20, 100])
+def test_cut_off_file_is_truncated(tmp_path, keep):
+ assert repair.is_truncated_safetensors(str(_st(tmp_path / "a.safetensors", keep=keep)))
+
+
+def test_find_truncated_ignores_healthy(tmp_path):
+ _st(tmp_path / "ok.safetensors")
+ bad = _st(tmp_path / "bad.safetensors", keep=90)
+ assert repair.find_truncated_weights(str(tmp_path)) == [str(bad)]
+
+
+def test_preflight_refuses_truncated_local_checkpoint(tmp_path):
+ _st(tmp_path / "model.safetensors", keep=90)
+ with pytest.raises(RuntimeError, match="incomplete"):
+ model_manager._preflight_truncated_weights(str(tmp_path))
+
+
+def test_preflight_passes_healthy_local_checkpoint(tmp_path):
+ _st(tmp_path / "model.safetensors")
+ model_manager._preflight_truncated_weights(str(tmp_path))
+
+
+def test_repair_removes_truncated_blob_and_link(tmp_path, monkeypatch):
+ from services import hf_revisions
+ monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)
+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)
+ monkeypatch.setitem(hf_revisions.CURATED_REVISIONS, "test/checkpoint", "a" * 40)
+ repo = tmp_path / "models--test--checkpoint"
+ (repo / "blobs").mkdir(parents=True)
+ snap = repo / "snapshots" / "abc"
+ snap.mkdir(parents=True)
+ blob = _st(repo / "blobs" / "b1", keep=90)
+ try:
+ (snap / "model.safetensors").symlink_to(blob)
+ except (OSError, NotImplementedError):
+ pytest.skip("symlinks unavailable")
+ calls = []
+ import huggingface_hub
+ monkeypatch.setattr(huggingface_hub, "snapshot_download", lambda **kw: calls.append(kw))
+ out = repair.repair_repo_cache("test/checkpoint", cache_dir=str(tmp_path))
+ assert out["found"] == 1 and out["restored"] and calls
+ assert not blob.exists() and not (snap / "model.safetensors").is_symlink()