Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: send a download over Xet when HTTPS cannot fetch its largest file (#11061)
oobaboogab28746ff29e5
diff --git a/studio/backend/hub/services/datasets/downloads.py b/studio/backend/hub/services/datasets/downloads.pyindex e0c50578155..ca0c4a3c603 100644--- a/studio/backend/hub/services/datasets/downloads.py+++ b/studio/backend/hub/services/datasets/downloads.py@@ -235,14 +235,21 @@ async def download_dataset_response(repo_id = await asyncio.to_thread(resolve_cached_repo_id_case, repo_id, repo_type = "dataset")key = _download_job_key(repo_id)- # Off the event loop: resolving "auto" can run the Xet reachability probe, and a blackholed DNS- # makes that outlast its 3s budget while every other request waits behind it.+ # Size and Auto resolution may perform network probes, so keep both off the event loop.+ largest_file_bytes = await asyncio.to_thread(+ download_lifecycle.largest_download_file_bytes,+ "dataset",+ repo_id,+ hf_token = hf_token,+ allow_ambient_token = allow_ambient_token,+ )use_xet, transport_reason = await asyncio.to_thread(download_lifecycle.resolve_requested_use_xet,getattr(body, "transport_mode", None),body.use_xet,+ largest_file_bytes = largest_file_bytes,)- transport = download_lifecycle.resolve_transport(use_xet)+ transport = download_lifecycle.resolve_transport(use_xet, largest_file_bytes = largest_file_bytes)logger.info("Download transport for %s: %s (%s)", repo_id, transport, transport_reason)from utils.hf_cache_settings import get_hf_cache_pathsdiff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.pyindex 597fc64aab1..46ebd4195df 100644--- a/studio/backend/hub/services/download_lifecycle.py+++ b/studio/backend/hub/services/download_lifecycle.py@@ -23,6 +23,7 @@ from hub.utils import download_manifestfrom hub.utils import download_registryfrom hub.utils import inventory_scan as hf_cache_scanfrom hub.utils.hf_cache_state import EXIT_CANCELLED+from hub.utils.hf_tokens import HfTokenArg, hf_token_argfrom hub.utils.state_dir import RepoTypelogger = logging.getLogger(__name__)@@ -67,6 +68,172 @@ def backend_dir() -> Path:return Path(__file__).resolve().parent.parent.parent+_REPO_SIBLINGS_TTL_SECONDS = 60.0+_REPO_SIBLINGS_MAX = 64+_REPO_SIBLINGS_TIMEOUT_SECONDS = 5.0+_repo_siblings_lock = threading.Lock()+# (repo_type, repo_id, token fingerprint) -> (siblings, fetched_at)+_REPO_SIBLINGS: "dict[tuple[str, str, str], tuple[tuple, float]]" = {}+++def _repo_siblings(repo_type: str, repo_id: str, hf_token: HfTokenArg) -> tuple:+ """Return a briefly cached file listing with sizes, or an empty tuple when none was ever read.++ A failed refresh serves the last listing instead: sizes at a revision do not change, and+ discarding them would report an oversized download as unmeasured, which reads as+ HTTP-eligible. What the refresh is really for, whether the oversized file is now cached, is+ decided against the local cache by ``largest_download_file_bytes``.+ """+ key = (str(repo_type), repo_id.lower(), hf_cache_scan.token_fingerprint(hf_token))+ now = time.monotonic()+ with _repo_siblings_lock:+ cached = _REPO_SIBLINGS.pop(key, None)+ if cached is not None:+ # Re-insert so the dict stays in least-recently-used order for eviction.+ _REPO_SIBLINGS[key] = cached+ if (now - cached[1]) < _REPO_SIBLINGS_TTL_SECONDS:+ return cached[0]+ try:+ from huggingface_hub import HfApi+ info = HfApi(token = hf_token).repo_info(+ repo_id,+ repo_type = repo_type,+ files_metadata = True,+ timeout = _REPO_SIBLINGS_TIMEOUT_SECONDS,+ )+ siblings = tuple(info.siblings or ())+ except Exception as exc: # noqa: BLE001 - a size probe must never be what fails a download+ logger.debug(+ "Could not read file sizes for %s %s: %s",+ repo_type,+ repo_id,+ download_registry.scrub_secrets(str(exc), hf_token = hf_token),+ )+ return cached[0] if cached is not None else ()+ with _repo_siblings_lock:+ if key not in _REPO_SIBLINGS and len(_REPO_SIBLINGS) >= _REPO_SIBLINGS_MAX:+ # Evict least-recently-used, not everything: a clear drops the listings that in-flight+ # downloads still depend on.+ _REPO_SIBLINGS.pop(next(iter(_REPO_SIBLINGS)), None)+ _REPO_SIBLINGS[key] = (siblings, now)+ return siblings+++def largest_download_file_bytes(+ repo_type: str,+ repo_id: str,+ *,+ variant: Optional[str] = None,+ files: Optional[Sequence[str]] = None,+ hf_token: Optional[str] = None,+ allow_ambient_token: bool = True,+ hub_cache: Optional[str] = None,+) -> Optional[int]:+ """Return the largest selected, uncached file, or None when it cannot be measured.++ Selection matches the worker: explicit files, a GGUF variant plan, filtered model snapshots, or+ every dataset file. Finalized blobs are excluded because ``snapshot_download`` skips them, so+ ``0`` means the job has nothing left to fetch. It is not ``None``: only ``None`` says the size+ is unknown, and a caller carrying an earlier measurement forward needs to tell those apart.++ ``allow_ambient_token`` is the caller's boundary, applied exactly as ``spawn_worker`` applies+ it: a caller denied the backend's own login measures anonymously, so this probe can never read+ a repository the download it is deciding for could not.+ """+ siblings = _repo_siblings(+ repo_type, repo_id, hf_token_arg(hf_token, allow_ambient_token = allow_ambient_token)+ )+ if not siblings:+ return None+ from hub.utils.gguf_plan import (+ build_gguf_variant_plans,+ expected_file_from_sibling,+ plan_for_variant,+ )++ wanted = {f for f in (files or []) if f}+ if wanted:+ chosen = [s for s in siblings if getattr(s, "rfilename", None) in wanted]+ elif variant:+ plan = plan_for_variant(build_gguf_variant_plans(siblings), variant)+ if plan is None:+ # An unresolvable variant is the worker's error to report, not a transport verdict.+ return None+ chosen = None+ candidates = list(plan.expected_files)+ elif repo_type == "model":+ from hub.utils.snapshot_filters import snapshot_download_siblings+ chosen = snapshot_download_siblings(siblings)+ else:+ chosen = list(siblings)+ if chosen is not None:+ candidates = [f for s in chosen if (f := expected_file_from_sibling(s)) is not None]++ cached = download_registry.finalized_blob_hashes(+ repo_type,+ repo_id,+ frozenset(file.sha256 for file in candidates if file.sha256),+ root = Path(hub_cache) if hub_cache else None,+ )+ return max(+ (max(0, int(file.size or 0)) for file in candidates if file.sha256 not in cached),+ default = 0,+ )+++def _largest_file_bytes_for_job(+ repo_type: str,+ repo_id: str,+ metadata = None,+ *,+ hf_token: Optional[str] = None,+ allow_ambient_token: bool = True,+) -> Optional[int]:+ """Measure a registered job, including its scoped file list, without failing the watcher."""+ try:+ return largest_download_file_bytes(+ repo_type,+ repo_id,+ variant = getattr(metadata, "variant", None),+ files = getattr(metadata, "scoped_files", None) or None,+ hf_token = hf_token,+ allow_ambient_token = allow_ambient_token,+ hub_cache = getattr(metadata, "hub_cache", None),+ )+ except Exception as exc: # noqa: BLE001 - unknown size, same as it was before the probe existed+ logger.debug("Largest-file probe failed for %s %s: %s", repo_type, repo_id, exc)+ return None+++def http_rung_reason(+ repo_type: str,+ repo_id: str,+ metadata = None,+ *,+ hf_token: Optional[str] = None,+ allow_ambient_token: bool = True,+ known_largest_file_bytes: Optional[int] = None,+) -> Optional[str]:+ """Return why this job cannot use the recovery ladder's HTTP rung, if applicable.++ The measurement is taken again here because the worker may have finalized the oversized file.+ ``known_largest_file_bytes`` is what the job measured before it ran, and it stands when the+ fresh attempt cannot measure at all: the sibling cache that would otherwise carry it is+ bounded, so 64 other repositories are enough to evict an hours-long download's only listing.+ """+ measured = _largest_file_bytes_for_job(+ repo_type,+ repo_id,+ metadata,+ hf_token = hf_token,+ allow_ambient_token = allow_ambient_token,+ )+ return download_registry.download_transport_unavailable_reason(+ download_registry.TRANSPORT_HTTP,+ largest_file_bytes = known_largest_file_bytes if measured is None else measured,+ )++def resolve_effective_use_xet(use_xet: bool) -> bool:"""Downgrade an Xet request to HTTP when hf_xet is unavailable, so a defaulted or explicit Xet request never hard-fails on installs without the Xet extra."""if not use_xet:@@ -80,23 +247,32 @@ def resolve_effective_use_xet(use_xet: bool) -> bool:return False-def resolve_requested_use_xet(transport_mode: Optional[str], use_xet: bool) -> tuple[bool, str]:- """Turn a download request's transport preference into ``(use_xet, reason)``. ``transport_mode`` is the current field; ``use_xet`` is the older boolean, still honoured so an older frontend or scripted caller keeps working. An explicit "xet" is respected even on a machine the health check dislikes, but still gets the memory caps and the stall fallback."""+def resolve_requested_use_xet(+ transport_mode: Optional[str],+ use_xet: bool,+ *,+ largest_file_bytes: Optional[int] = None,+) -> tuple[bool, str]:+ """Resolve the current transport mode, falling back to the legacy ``use_xet`` flag."""mode = (transport_mode or "").strip().lower()if mode == download_registry.TRANSPORT_HTTP:return (False, "HTTP (requested)")if mode == download_registry.TRANSPORT_XET:return (resolve_effective_use_xet(True), "Xet (requested)")if mode == download_registry.TRANSPORT_AUTO:- return resolve_auto_use_xet()+ return resolve_auto_use_xet(largest_file_bytes = largest_file_bytes)resolved = resolve_effective_use_xet(use_xet)return (resolved, "Xet" if resolved else "HTTP")-def resolve_auto_use_xet() -> tuple[bool, str]:+def resolve_auto_use_xet(*, largest_file_bytes: Optional[int] = None) -> tuple[bool, str]:"""Pick a transport for a download the user left on "Auto". Returns ``(use_xet, reason)``. Server-side on purpose: only the backend can see this machine's RAM, its hf_xet build, and whether Xet has been failing here. Probing IS allowed here, unlike in the capabilities endpoint the UI polls on render, because this runs once per download request and the verdict is memoized."""if not resolve_effective_use_xet(True):return (False, "hf_xet is not installed")+ http_ceiling_reason = download_registry.http_size_ceiling_reason(largest_file_bytes)+ if http_ceiling_reason is not None:+ # Health and RAM preferences cannot select HTTP when the file exceeds its limit.+ return (True, "Xet (HTTPS cannot fetch a file this large)")try:from utils.hf_xet_fallback import xet_healthhealth = xet_health()@@ -148,10 +324,19 @@ def _allow_high_performance() -> bool:)-def resolve_transport(use_xet: bool) -> str:+def resolve_transport(use_xet: bool, *, largest_file_bytes: Optional[int] = None) -> str:transport = download_registry.TRANSPORT_XET if use_xet else download_registry.TRANSPORT_HTTP- unavailable_reason = download_registry.download_transport_unavailable_reason(transport)+ unavailable_reason = download_registry.download_transport_unavailable_reason(+ transport, largest_file_bytes = largest_file_bytes+ )if unavailable_reason is not None:+ if transport == download_registry.TRANSPORT_HTTP:+ # Include both failures when the required Xet transport is also unavailable.+ xet_reason = download_registry.download_transport_unavailable_reason(+ download_registry.TRANSPORT_XET+ )+ if xet_reason is not None:+ unavailable_reason = f"{unavailable_reason} {xet_reason}"raise HTTPException(status_code = 400, detail = unavailable_reason)return transport@@ -376,8 +561,13 @@ def finalize_worker_exit(transport: Optional[str] = None,cancel_marker_transport: Optional[str] = None,defer_error: bool = False,+ deferred_error_out: "Optional[list[str]]" = None,+ largest_file_bytes: Optional[int] = None,) -> str:- """Block until *proc* exits, then record the job's terminal state in *registry*. Drains and scrubs stderr first, then classifies the exit code. A no-op when the process was already dropped (e.g. superseded). No stall watchdog: huggingface_hub already times out chunk reads and raises a resumable error on a dead connection, so the worker's exit code is the single source of truth."""+ """Reap *proc* and record its terminal state.++ ``deferred_error_out`` captures a deferred error for callers that exhaust all retry rungs.+ """stderr_data = drain_stderr_excerpt(proc.stderr)rc = proc.wait()_cleanup_worker_files_manifest(proc)@@ -445,12 +635,15 @@ def finalize_worker_exit(logger = logger,)else:+ # Rewrite the misleading >50GB dependency error for the UI; logs retain raw stderr.+ error_text = download_registry.humanize_worker_error(+ stderr_text or f"worker exited with code {rc}",+ largest_file_bytes = largest_file_bytes,+ )+ if deferred_error_out is not None:+ deferred_error_out.append(error_text)if not defer_error:- registry.set_job(- key,- "error",- stderr_text or f"worker exited with code {rc}",- )+ registry.set_job(key, "error", error_text)logger.error(f"{log_prefix} failed for {label} (rc={rc}): {stderr_text}",)@@ -521,8 +714,9 @@ def _try_transport_retry(pending_xet_failure: Optional[str] = None,bytes_before: "Optional[int]" = _UNSAMPLED,allow_ambient_token: bool = True,+ largest_file_bytes: Optional[int] = None,) -> bool:- """Reclaim *key* under *retry_transport* and spawn a recovery worker. Returns ``True`` when the recovery worker was successfully registered. Caller is responsible for ensuring this is only called when the job is in ``"error"`` state, the original transport was XET, and the target transport is available. Two directions share this body: ``TRANSPORT_HTTP`` is terminal, so the transport changes and the worker's own ``prepare_cache_for_transport`` purges the XET partial an HTTP resume would corrupt, while ``TRANSPORT_XET`` is the stall retry, same transport, same marker, one more child, bounded by *xet_attempt* rather than by the transport check that stops the HTTP one. *allow_ambient_token* rides along unchanged, so a job that started anonymous cannot pick the backend's own HF_TOKEN up on a lower rung. *pending_xet_failure* is a stall verdict held back from the health tracker and carried into the next worker, so a download that recovers on its second Xet attempt reports nothing and one that does not reports exactly one failure. *bytes_before* is the ORIGINAL pre-Xet baseline: resampling would fold the killed worker's partial writes in and make a recovered attempt read as a cached no-op. Derives variant and blob-hash metadata from the registry entry written by the original XET claim so callers do not re-construct worker arguments, and re-queries peer protection hashes at spawn time to reflect concurrent sibling changes."""+ """Reclaim *key* under *retry_transport* and spawn a recovery worker. Returns ``True`` when the recovery worker was successfully registered. Caller is responsible for ensuring this is only called when the job is in ``"error"`` state, the original transport was XET, and the target transport is available. Two directions share this body: ``TRANSPORT_HTTP`` is terminal, so the transport changes and the worker's own ``prepare_cache_for_transport`` purges the XET partial an HTTP resume would corrupt, while ``TRANSPORT_XET`` is the stall retry, same transport, same marker, one more child, bounded by *xet_attempt* rather than by the transport check that stops the HTTP one. *allow_ambient_token* rides along unchanged, so a job that started anonymous cannot pick the backend's own HF_TOKEN up on a lower rung. *pending_xet_failure* is a stall verdict held back from the health tracker and carried into the next worker, so a download that recovers on its second Xet attempt reports nothing and one that does not reports exactly one failure. *bytes_before* is the ORIGINAL pre-Xet baseline: resampling would fold the killed worker's partial writes in and make a recovered attempt read as a cached no-op. *largest_file_bytes* is the size the job measured before it ran, carried down the ladder the same way, so a rung whose own probe cannot measure still knows HTTP cannot serve this download. Derives variant and blob-hash metadata from the registry entry written by the original XET claim so callers do not re-construct worker arguments, and re-queries peer protection hashes at spawn time to reflect concurrent sibling changes."""retry_over_xet = retry_transport == download_registry.TRANSPORT_XETretry_name = "XET" if retry_over_xet else "HTTP"@@ -697,7 +891,20 @@ def _try_transport_retry(scrubbed,)registry.update_job_transport(key, original_metadata.transport)- if retry_over_xet:+ # A failed Xet respawn may fall back only when HTTP can fetch the remaining files.+ rung_reason = (+ http_rung_reason(+ repo_type,+ repo_id,+ original_metadata,+ hf_token = hf_token,+ allow_ambient_token = allow_ambient_token,+ known_largest_file_bytes = largest_file_bytes,+ )+ if retry_over_xet+ else None+ )+ if retry_over_xet and rung_reason is None:# A failed EXTRA worker spawn is a local failure, not a verdict on the download, so drop to the rung this job would have taken without the retry instead of stranding it in "error".logger.warning("%s XET retry could not be spawned for %s; falling back to HTTP",@@ -719,12 +926,13 @@ def _try_transport_retry(pending_xet_failure = pending_xet_failure,bytes_before = bytes_before,allow_ambient_token = allow_ambient_token,+ largest_file_bytes = largest_file_bytes,)_give_up()_set_retry_failure_state(registry,key,- scrubbed,+ f"{scrubbed}\n{rung_reason}" if rung_reason else scrubbed,repo_type = repo_type,repo_id = repo_id,fallback_variant = variant,@@ -750,6 +958,7 @@ def _try_transport_retry(pending_xet_failure = pending_xet_failure,bytes_before = bytes_before,allow_ambient_token = allow_ambient_token,+ largest_file_bytes = largest_file_bytes,)@@ -920,8 +1129,9 @@ def register_worker(xet_attempt: int = 1,pending_xet_failure: Optional[str] = None,allow_ambient_token: bool = True,+ largest_file_bytes: Optional[int] = None,) -> bool:- """Watch *proc* to completion and drive the recovery ladder off its exit. *xet_attempt* (1-based) bounds the XET->XET stall retry, the way ``transport == TRANSPORT_XET`` bounds the terminal XET->HTTP one. *pending_xet_failure* is an earlier attempt's stall verdict, held back from the health tracker until the XET phase ends so one download can never spend the two consecutive failures that demote a machine. *allow_ambient_token* is the token policy this job was started under, carried onto every rung of the ladder."""+ """Watch *proc* to completion and drive the recovery ladder off its exit. *xet_attempt* (1-based) bounds the XET->XET stall retry, the way ``transport == TRANSPORT_XET`` bounds the terminal XET->HTTP one. *pending_xet_failure* is an earlier attempt's stall verdict, held back from the health tracker until the XET phase ends so one download can never spend the two consecutive failures that demote a machine. *allow_ambient_token* is the token policy this job was started under, carried onto every rung of the ladder. *largest_file_bytes* is an earlier rung's measurement, kept only for as long as this one cannot take its own."""if not registry.register_process(key, proc):kill_and_reap_process(proc, label = label, logger = logger)return False@@ -952,16 +1162,23 @@ def register_worker(def _watch() -> None:stalled: list[str] = []watchdog_stop = None+ deferred_error: list[str] = []try:- can_retry_http = (- transport == download_registry.TRANSPORT_XET- and download_registry.download_transport_unavailable_reason(- download_registry.TRANSPORT_HTTP- )- is None+ started_on_xet = transport == download_registry.TRANSPORT_XET+ # Keep the pre-run size for error reporting. Retry eligibility is measured after exit.+ measured_before_run = _largest_file_bytes_for_job(+ repo_type,+ repo_id,+ _metadata,+ hf_token = worker_token,+ allow_ambient_token = allow_ambient_token,+ )+ # An earlier rung's measurement stands until this one takes its own.+ known_largest_file_bytes = (+ largest_file_bytes if measured_before_run is None else measured_before_run)- # A Xet transfer that hangs with no progress and no error never produces an exit code, so watch the cache for byte-level progress and kill it; the SIGKILL surfaces as "error".- if can_retry_http:+ # Watch every Xet attempt so even a final hung worker becomes terminal.+ if started_on_xet:watchdog_stop = _start_stall_watchdog(registry,key,@@ -985,18 +1202,36 @@ def register_worker(repo_id = repo_id,transport = transport,cancel_marker_transport = cancel_marker_transport,- defer_error = can_retry_http,+ # Retry eligibility is known only after the worker leaves its final cache state.+ defer_error = started_on_xet,+ deferred_error_out = deferred_error,+ largest_file_bytes = known_largest_file_bytes,)if watchdog_stop is not None:# Stop measuring once the worker is reaped: post-download symlinking and verification make no byte-level progress and must not read as a stall.watchdog_stop.set()+ # Recheck after exit because this worker may have finalized the oversized shard.+ rung_reason = (+ http_rung_reason(+ repo_type,+ repo_id,+ _metadata,+ hf_token = worker_token,+ allow_ambient_token = allow_ambient_token,+ known_largest_file_bytes = known_largest_file_bytes,+ )+ if started_on_xet+ else None+ )+ can_retry_http = started_on_xet and rung_reason is None+ # HTTP availability and the Xet attempt budget are independent.+ can_retry_xet = started_on_xet and xet_attempt < _xet_attempt_budget()# Spend one more XET worker only on a DATA-phase verdict: retrying a pre-byte trip would buy a second full 600s connect window, and that trip is as likely slow metadata as a broken Xet.retry_xet = (- can_retry_http+ can_retry_xetand state == "error"and bool(stalled)and _is_data_phase_stall(stalled[0])- and xet_attempt < _xet_attempt_budget())# Evidence for the WHOLE Xet phase, reported once when the phase ends.xet_failure = pending_xet_failure@@ -1028,29 +1263,45 @@ def register_worker(and bytes_after > _bytes_before):_record_xet_success(logger)- # One guard per direction: transport == TRANSPORT_XET makes the HTTP rung terminal, xet_attempt bounds the XET one.- if can_retry_http and state == "error":- _try_transport_retry(- registry,- key,- hf_token = worker_token,- label = label,- log_prefix = log_prefix,- logger = logger,- repo_type = repo_type,- repo_id = repo_id,- watch_name = watch_name,- retry_transport = (- download_registry.TRANSPORT_XET- if retry_xet- else download_registry.TRANSPORT_HTTP- ),- xet_attempt = xet_attempt + 1 if retry_xet else xet_attempt,- pending_xet_failure = xet_failure,- # The ORIGINAL pre-Xet baseline: resampling would fold the killed worker's partial writes in, so a recovered attempt would read as a cached no-op.- bytes_before = _bytes_before,- allow_ambient_token = allow_ambient_token,- )+ if started_on_xet and state == "error":+ if retry_xet or can_retry_http:+ _try_transport_retry(+ registry,+ key,+ hf_token = worker_token,+ label = label,+ log_prefix = log_prefix,+ logger = logger,+ repo_type = repo_type,+ repo_id = repo_id,+ watch_name = watch_name,+ retry_transport = (+ download_registry.TRANSPORT_XET+ if retry_xet+ else download_registry.TRANSPORT_HTTP+ ),+ xet_attempt = xet_attempt + 1 if retry_xet else xet_attempt,+ pending_xet_failure = xet_failure,+ # Preserve the pre-Xet baseline across retries.+ bytes_before = _bytes_before,+ allow_ambient_token = allow_ambient_token,+ largest_file_bytes = known_largest_file_bytes,+ )+ else:+ # No retry remains, so publish the deferred worker error.+ failure = (deferred_error[0] if deferred_error else "") or "download failed"+ if rung_reason and rung_reason not in failure:+ failure = f"{failure}\n{rung_reason}"+ _set_retry_failure_state(+ registry,+ key,+ failure,+ repo_type = repo_type,+ repo_id = repo_id,+ fallback_variant = download_registry.variant_from_key(key),+ fallback_transport = transport,+ logger = logger,+ )except Exception:if watchdog_stop is not None:watchdog_stop.set()diff --git a/studio/backend/hub/services/models/downloads.py b/studio/backend/hub/services/models/downloads.pyindex da2ea7df478..4bc4d98978f 100644--- a/studio/backend/hub/services/models/downloads.py+++ b/studio/backend/hub/services/models/downloads.py@@ -214,13 +214,24 @@ async def download_model_response(raise HTTPException(status_code = 400, detail = f"Invalid scope_id: {body.scope_id!r}")variant = scope_variantkey = _download_job_key(repo_id, variant)- # Off the event loop: resolving "auto" can run the Xet reachability probe, and a blackholed DNS makes that outlast its 3s budget while every other request waits behind it.+ # Size and Auto resolution may perform network probes, so keep both off the event loop.+ largest_file_bytes = await asyncio.to_thread(+ download_lifecycle.largest_download_file_bytes,+ "model",+ repo_id,+ variant = variant,+ # Mirror the claim and the worker below: files are honoured only under a scope_id.+ files = scoped_files if scope_variant is not None else None,+ hf_token = hf_token,+ allow_ambient_token = allow_ambient_token,+ )use_xet, transport_reason = await asyncio.to_thread(download_lifecycle.resolve_requested_use_xet,getattr(body, "transport_mode", None),body.use_xet,+ largest_file_bytes = largest_file_bytes,)- transport = download_lifecycle.resolve_transport(use_xet)+ transport = download_lifecycle.resolve_transport(use_xet, largest_file_bytes = largest_file_bytes)logger.info("Download transport for %s: %s (%s)", repo_id, transport, transport_reason)from utils.hf_cache_settings import get_hf_cache_pathsdiff --git a/studio/backend/hub/tests/test_dataset_services.py b/studio/backend/hub/tests/test_dataset_services.pyindex ed4a77b7fbd..b56a93b0f45 100644--- a/studio/backend/hub/tests/test_dataset_services.py+++ b/studio/backend/hub/tests/test_dataset_services.py@@ -1183,7 +1183,7 @@ def test_dataset_claim_register_cancel_uses_registry_marker_owner(monkeypatch):monkeypatch.setattr(downloads.download_registry,"download_transport_unavailable_reason",- lambda _transport: None,+ lambda _transport, **_kwargs: None,)monkeypatch.setattr(downloads.download_lifecycle,diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.pyindex 44b6e22da16..19fe3ed327a 100644--- a/studio/backend/hub/tests/test_model_services.py+++ b/studio/backend/hub/tests/test_model_services.py@@ -237,7 +237,7 @@ def _shared_setup_18(monkeypatch):monkeypatch.setattr(downloads.download_registry,"download_transport_unavailable_reason",- lambda _transport: None,+ lambda _transport, **_kwargs: None,)diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.pyindex 681bb1f75a3..9fdccc153c0 100644--- a/studio/backend/hub/utils/download_registry.py+++ b/studio/backend/hub/utils/download_registry.py@@ -60,6 +60,56 @@ from hub.utils.hf_cache_state import ()+# HTTP cannot fetch files above huggingface_hub's limit. Read the installed value when available.+_HTTP_MAX_FILE_BYTES_FALLBACK = 50 * 1000 * 1000 * 1000++_HTTP_SIZE_CEILING_MARKER = "too large to be downloaded using the regular download method"+++def http_max_file_bytes() -> int:+ """The largest single file the HTTP transport can fetch."""+ try:+ from huggingface_hub import constants as hf_constants+ value = int(getattr(hf_constants, "MAX_HTTP_DOWNLOAD_SIZE", 0) or 0)+ except Exception: # noqa: BLE001 - an unreadable constant is not a reason to fail a download+ value = 0+ return value if value > 0 else _HTTP_MAX_FILE_BYTES_FALLBACK+++def http_size_ceiling_reason(largest_file_bytes: Optional[int]) -> Optional[str]:+ """Why HTTP cannot serve a download whose biggest file is *largest_file_bytes*, or None.++ An unknown size (``None``/0, e.g. metadata that could not be read) is not evidence of anything and+ leaves the transport choice exactly as it was.+ """+ try:+ largest = int(largest_file_bytes or 0)+ except (TypeError, ValueError):+ return None+ ceiling = http_max_file_bytes()+ if largest <= ceiling:+ return None+ return (+ f"HTTPS cannot fetch this download: its largest file is {largest / 1e9:.1f}GB and "+ f"huggingface_hub refuses a plain HTTPS transfer above {ceiling / 1e9:.0f}GB. "+ "Only the Xet transport can fetch a file this large."+ )+++def humanize_worker_error(text: str, *, largest_file_bytes: Optional[int] = None) -> str:+ """Replace the hub's misleading >50GB dependency error with the transport limit."""+ if not text or _HTTP_SIZE_CEILING_MARKER not in text:+ return text+ reason = http_size_ceiling_reason(largest_file_bytes)+ if reason is not None:+ return reason+ return (+ "HTTPS cannot fetch this download: one of its files is larger than the "+ f"{http_max_file_bytes() / 1e9:.0f}GB limit huggingface_hub allows over plain HTTPS. "+ "Only the Xet transport can fetch a file this large."+ )++@dataclass(frozen = True)class DownloadTransportCapability:available: bool@@ -78,10 +128,18 @@ class DownloadTransportCapabilities:def get_download_transport_capabilities(- *, probe: bool = False, ram_gate: bool = False+ *,+ probe: bool = False,+ ram_gate: bool = False,+ largest_file_bytes: Optional[int] = None,) -> DownloadTransportCapabilities:- """What this machine can do, and what Auto resolves to on it. ``probe`` runs the live Xet health check and is only for the frontend resolving Auto at download start; ``ram_gate`` applies the free-RAM half of that same verdict WITHOUT the network probe, for a surface that has to state what the next download will pick, since the settings row said "Auto is using Xet" while the download path, which probes, chose HTTP."""+ """Return transport availability and the current Auto choice.++ ``probe`` checks live Xet health, ``ram_gate`` applies memory pressure, and+ ``largest_file_bytes`` applies the HTTP size limit.+ """xet_available = importlib.util.find_spec("hf_xet") is not None+ http_reason = http_size_ceiling_reason(largest_file_bytes)auto_transport = TRANSPORT_XET if xet_available else TRANSPORT_HTTPauto_reason: Optional[str] = Noneauto_forced = False@@ -119,8 +177,12 @@ def get_download_transport_capabilities(if pressure is not None:auto_transport = TRANSPORT_HTTPauto_reason = pressure+ if http_reason is not None and xet_available:+ # The size limit overrides preferences for HTTP based on health or RAM.+ auto_transport = TRANSPORT_XET+ auto_reason = "Xet (HTTPS cannot fetch a file this large)"return DownloadTransportCapabilities(- http = DownloadTransportCapability(available = True),+ http = DownloadTransportCapability(available = http_reason is None, reason = http_reason),xet = DownloadTransportCapability(available = xet_available,reason = None@@ -133,9 +195,11 @@ def get_download_transport_capabilities()-def download_transport_unavailable_reason(transport: str) -> Optional[str]:+def download_transport_unavailable_reason(+ transport: str, *, largest_file_bytes: Optional[int] = None+) -> Optional[str]:if transport == TRANSPORT_HTTP:- return None+ return http_size_ceiling_reason(largest_file_bytes)if transport == TRANSPORT_XET:caps = get_download_transport_capabilities().xetreturn None if caps.available else caps.reason@@ -969,6 +1033,32 @@ def completed_blob_bytes(return total+def finalized_blob_hashes(+ repo_type: str,+ repo_id: str,+ blob_hashes: frozenset[str],+ *,+ root: Optional[Path] = None,+) -> frozenset[str]:+ """Return requested blob hashes that are finalized in the active cache."""+ if not blob_hashes:+ return frozenset()+ found: set[str] = set()+ for entry in iter_active_repo_cache_dirs(repo_type, repo_id, root = root):+ blobs_dir = entry / "blobs"+ if not blobs_dir.is_dir():+ continue+ for blob_hash in blob_hashes:+ if blob_hash in found:+ continue+ try:+ if (blobs_dir / blob_hash).is_file():+ found.add(blob_hash)+ except OSError:+ continue+ return frozenset(found)++def existing_blob_bytes(repo_type: str,repo_id: str,diff --git a/studio/backend/tests/test_account_download_retirement_race.py b/studio/backend/tests/test_account_download_retirement_race.pyindex e6fadd3978b..034dabecacf 100644--- a/studio/backend/tests/test_account_download_retirement_race.py+++ b/studio/backend/tests/test_account_download_retirement_race.py@@ -39,7 +39,9 @@ def isolated(monkeypatch, tmp_path):monkeypatch.setattr(jobs, "_services", [])monkeypatch.setattr(jobs, "_retired", set())monkeypatch.setattr(- download_lifecycle, "resolve_requested_use_xet", lambda mode, use_xet: (False, "HTTP")+ download_lifecycle,+ "resolve_requested_use_xet",+ lambda mode, use_xet, **_kw: (False, "HTTP"),)from core import research_runsfrom core.rag import folder_sync, ingestiondiff --git a/studio/backend/tests/test_download_adoption_transport.py b/studio/backend/tests/test_download_adoption_transport.pyindex c9d56145a59..18b126a6331 100644--- a/studio/backend/tests/test_download_adoption_transport.py+++ b/studio/backend/tests/test_download_adoption_transport.py@@ -49,7 +49,7 @@ def test_a_fresh_start_reports_the_transport_the_backend_resolved(monkeypatch):monkeypatch.setattr(download_lifecycle.download_registry,"download_transport_unavailable_reason",- lambda transport: "hf_xet is not installed" if transport == "xet" else None,+ lambda transport, **_kw: "hf_xet is not installed" if transport == "xet" else None,)use_xet, _reason = download_lifecycle.resolve_requested_use_xet("xet", True)assert use_xet is False, "the downgrade this reports is what the client must be told"@@ -62,7 +62,7 @@ def test_an_available_xet_request_is_left_alone(monkeypatch):monkeypatch.setattr(download_lifecycle.download_registry,"download_transport_unavailable_reason",- lambda transport: None,+ lambda transport, **_kw: None,)use_xet, _reason = download_lifecycle.resolve_requested_use_xet("xet", True)assert download_lifecycle.resolve_transport(use_xet) == "xet"diff --git a/studio/backend/tests/test_hub_download_http_size_ceiling.py b/studio/backend/tests/test_hub_download_http_size_ceiling.pynew file mode 100644index 00000000000..a18257c7c67--- /dev/null+++ b/studio/backend/tests/test_hub_download_http_size_ceiling.py@@ -0,0 +1,936 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""HTTP size-limit transport selection and recovery tests."""++from __future__ import annotations++import sys+import types as _types+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from fastapi import HTTPException++from hub.services import download_lifecycle as dl+from hub.utils import download_registry++# Sizes from the repository reported in #10840.+_OVERSIZED = 54_400_261_312+_UNDER_THE_CEILING = 49_859_583_136+++class _Sibling:+ def __init__(self, rfilename: str, size: int):+ self.rfilename = rfilename+ self.size = size+ self.lfs = {"sha256": f"{abs(hash(rfilename)):064x}"[:64]}+++def _flash_next_siblings() -> list[_Sibling]:+ """A two-variant GGUF repo: one variant past the ceiling, one under it."""+ return [+ _Sibling("UD-Q5_K_XL/M-UD-Q5_K_XL-00001-of-00002.gguf", _OVERSIZED),+ _Sibling("UD-Q5_K_XL/M-UD-Q5_K_XL-00002-of-00002.gguf", 43_500_000_000),+ _Sibling("UD-Q4_K_XL/M-UD-Q4_K_XL-00001-of-00002.gguf", _UNDER_THE_CEILING),+ _Sibling("UD-Q4_K_XL/M-UD-Q4_K_XL-00002-of-00002.gguf", 20_000_000_000),+ _Sibling("README.md", 1_024),+ ]+++# --------------------------------------------------------------------------------------------+# The ceiling itself+# --------------------------------------------------------------------------------------------+++def test_the_ceiling_is_read_from_the_installed_hub():+ from huggingface_hub import constants as hf_constants+ assert download_registry.http_max_file_bytes() == hf_constants.MAX_HTTP_DOWNLOAD_SIZE+++def test_only_a_file_past_the_ceiling_refuses_http():+ ceiling = download_registry.http_max_file_bytes()+ assert download_registry.http_size_ceiling_reason(ceiling) is None+ assert download_registry.http_size_ceiling_reason(ceiling + 1) is not None+ # Unknown sizes preserve the previous transport choice.+ assert download_registry.http_size_ceiling_reason(None) is None+ assert download_registry.http_size_ceiling_reason(0) is None+++def test_the_refusal_names_the_size_and_the_limit():+ reason = download_registry.http_size_ceiling_reason(_OVERSIZED)+ assert "54.4GB" in reason and "50GB" in reason and "Xet" in reason+++def test_transport_unavailable_reason_is_size_aware():+ assert (+ download_registry.download_transport_unavailable_reason(download_registry.TRANSPORT_HTTP)+ is None+ )+ assert (+ download_registry.download_transport_unavailable_reason(+ download_registry.TRANSPORT_HTTP, largest_file_bytes = _OVERSIZED+ )+ is not None+ )+++# --------------------------------------------------------------------------------------------+# Which file the ceiling is asked about+# --------------------------------------------------------------------------------------------+++def test_the_size_is_measured_per_variant_not_per_repo(monkeypatch):+ """The 46GB variant in a repo that also holds a 54GB one keeps HTTP."""+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(_flash_next_siblings()))+ assert (+ dl.largest_download_file_bytes("model", "unsloth/M-GGUF", variant = "ud-q5_k_xl")+ == _OVERSIZED+ )+ assert (+ dl.largest_download_file_bytes("model", "unsloth/M-GGUF", variant = "ud-q4_k_xl")+ == _UNDER_THE_CEILING+ )+++def test_a_scoped_file_list_is_measured_on_its_own_files(monkeypatch):+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(_flash_next_siblings()))+ assert (+ dl.largest_download_file_bytes(+ "model",+ "unsloth/M-GGUF",+ files = ["UD-Q4_K_XL/M-UD-Q4_K_XL-00002-of-00002.gguf"],+ )+ == 20_000_000_000+ )+++def test_a_model_snapshot_is_measured_on_the_files_it_actually_fetches(monkeypatch):+ """Ignored exports must not affect snapshot transport selection."""+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(_flash_next_siblings()))+ assert dl.largest_download_file_bytes("model", "unsloth/M-GGUF") == 1_024+++def test_an_ignored_oversized_export_does_not_decide_the_transport(monkeypatch):+ siblings = [+ _Sibling("model-00001-of-00002.safetensors", 30_000_000_000),+ _Sibling("model-00002-of-00002.safetensors", 20_000_000_000),+ _Sibling("onnx/model.onnx", _OVERSIZED),+ _Sibling("mlx/weights.safetensors", _OVERSIZED),+ ]+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(siblings))+ assert dl.largest_download_file_bytes("model", "org/Weights") == 30_000_000_000+++def test_a_dataset_is_measured_unfiltered(monkeypatch):+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: (_Sibling("shard.onnx", _OVERSIZED),))+ assert dl.largest_download_file_bytes("dataset", "org/Data") == _OVERSIZED+++def test_unreadable_metadata_measures_nothing(monkeypatch):+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: ())+ assert dl.largest_download_file_bytes("model", "unsloth/M-GGUF") is None+++def test_an_unresolvable_variant_measures_nothing(monkeypatch):+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(_flash_next_siblings()))+ assert dl.largest_download_file_bytes("model", "unsloth/M-GGUF", variant = "nope") is None+++def test_a_finalized_blob_is_not_a_file_this_download_has_to_fetch(monkeypatch, tmp_path):+ """A finalized oversized shard must not block HTTP for smaller remaining files."""+ siblings = _flash_next_siblings()+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(siblings))+ oversized = next(s for s in siblings if s.size == _OVERSIZED)++ blobs = tmp_path / "models--unsloth--M-GGUF" / "blobs"+ blobs.mkdir(parents = True)+ (blobs / oversized.lfs["sha256"]).write_bytes(b"")+ monkeypatch.setattr(+ download_registry,+ "iter_active_repo_cache_dirs",+ lambda *a, **k: iter([blobs.parent]),+ )++ assert (+ dl.largest_download_file_bytes("model", "unsloth/M-GGUF", variant = "ud-q5_k_xl")+ == 43_500_000_000+ )+++def test_a_scoped_job_is_measured_on_its_own_file_list(monkeypatch):+ """Scoped jobs must use metadata files instead of their synthetic variant key."""+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(_flash_next_siblings()))+ metadata = _types.SimpleNamespace(+ variant = "@ab-big",+ scoped_files = ("UD-Q5_K_XL/M-UD-Q5_K_XL-00001-of-00002.gguf",),+ hub_cache = None,+ )+ assert dl._largest_file_bytes_for_job("model", "unsloth/M-GGUF", metadata) == _OVERSIZED+ assert (+ dl._largest_file_bytes_for_job(+ "model",+ "unsloth/M-GGUF",+ _types.SimpleNamespace(variant = "@ab-big", scoped_files = (), hub_cache = None),+ )+ is None+ )+++def test_a_probe_failure_never_escapes_into_the_watcher(monkeypatch):+ monkeypatch.setattr(dl, "largest_download_file_bytes", _raise)+ assert dl._largest_file_bytes_for_job("model", "unsloth/M-GGUF") is None+++def _raise(*a, **k):+ raise RuntimeError("hub is down")+++# --------------------------------------------------------------------------------------------+# What the transport choice does with it+# --------------------------------------------------------------------------------------------+++def test_auto_takes_xet_over_a_demoted_verdict_when_http_cannot_serve(monkeypatch):+ monkeypatch.setattr(dl, "resolve_effective_use_xet", lambda requested: requested)+ fake = _types.ModuleType("utils.hf_xet_fallback")+ fake.xet_health = lambda **kw: _types.SimpleNamespace(+ use_xet = False, reason = "Xet stalled twice on this machine"+ )+ fake.free_ram_pressure_reason = lambda: "HTTP: only 1.0GB RAM free"+ monkeypatch.setitem(sys.modules, "utils.hf_xet_fallback", fake)++ assert dl.resolve_auto_use_xet()[0] is False+ use_xet, reason = dl.resolve_auto_use_xet(largest_file_bytes = _OVERSIZED)+ assert use_xet is True+ assert "HTTPS cannot fetch" in reason+++def test_auto_still_defers_to_the_verdict_under_the_ceiling(monkeypatch):+ monkeypatch.setattr(dl, "resolve_effective_use_xet", lambda requested: requested)+ monkeypatch.setattr(dl, "resolve_auto_use_xet", lambda **kw: (False, "demoted"))+ assert dl.resolve_requested_use_xet("auto", True, largest_file_bytes = _UNDER_THE_CEILING) == (+ False,+ "demoted",+ )+++def test_an_explicit_https_request_is_rejected_by_name(monkeypatch):+ """An impossible explicit HTTP request should fail instead of switching transport."""+ monkeypatch.setattr(dl, "resolve_effective_use_xet", lambda requested: requested)+ use_xet, _ = dl.resolve_requested_use_xet("http", False, largest_file_bytes = _OVERSIZED)+ assert use_xet is False+ with pytest.raises(HTTPException) as excinfo:+ dl.resolve_transport(use_xet, largest_file_bytes = _OVERSIZED)+ assert excinfo.value.status_code == 400+ assert "HTTPS cannot fetch" in excinfo.value.detail+ assert (+ dl.resolve_transport(False, largest_file_bytes = _UNDER_THE_CEILING)+ == download_registry.TRANSPORT_HTTP+ )+++def test_an_install_without_hf_xet_is_told_that_too(monkeypatch):+ """Report when the only suitable transport is also unavailable."""+ monkeypatch.setattr(+ download_registry.importlib.util,+ "find_spec",+ lambda name: None if name == "hf_xet" else object(),+ )+ use_xet, _ = dl.resolve_requested_use_xet("auto", True, largest_file_bytes = _OVERSIZED)+ assert use_xet is False+ with pytest.raises(HTTPException) as excinfo:+ dl.resolve_transport(use_xet, largest_file_bytes = _OVERSIZED)+ assert "HTTPS cannot fetch" in excinfo.value.detail+ assert "hf_xet is not installed" in excinfo.value.detail+++def test_capabilities_mark_http_unavailable_for_an_oversized_download(monkeypatch):+ monkeypatch.setattr(download_registry.importlib.util, "find_spec", lambda name: object())+ caps = download_registry.get_download_transport_capabilities(largest_file_bytes = _OVERSIZED)+ assert caps.http.available is False+ assert "HTTPS cannot fetch" in caps.http.reason+ assert caps.auto_resolves_to == download_registry.TRANSPORT_XET+ assert (+ download_registry.get_download_transport_capabilities(+ largest_file_bytes = _UNDER_THE_CEILING+ ).http.available+ is True+ )+++# --------------------------------------------------------------------------------------------+# What the user is told+# --------------------------------------------------------------------------------------------+++_HUB_REFUSAL = (+ "ValueError: The file is too large to be downloaded using the regular download method. "+ " Install `hf_xet` with `pip install hf_xet` for xet-powered downloads."+)+++def test_the_hub_refusal_is_rewritten_into_what_happened():+ rewritten = download_registry.humanize_worker_error(_HUB_REFUSAL)+ assert "hf_xet" not in rewritten+ assert "HTTPS cannot fetch" in rewritten+++def test_a_known_size_makes_the_rewrite_specific():+ rewritten = download_registry.humanize_worker_error(_HUB_REFUSAL, largest_file_bytes = _OVERSIZED)+ assert "54.4GB" in rewritten+++def test_every_other_failure_is_passed_through():+ assert download_registry.humanize_worker_error("404 Client Error") == "404 Client Error"+ assert download_registry.humanize_worker_error("") == ""+++# --------------------------------------------------------------------------------------------+# The recovery ladder+# --------------------------------------------------------------------------------------------+++class _Proc:+ pid = 4242++ def __init__(+ self,+ rc,+ stderr = b"",+ ):+ import io++ self.rc = rc+ self.stderr = io.BytesIO(stderr)+ self.waited = False++ def poll(self):+ return self.rc if self.waited else None++ def wait(self, timeout = None):+ self.waited = True+ return self.rc++ def kill(self):+ pass+++class _ImmediateThread:+ def __init__(+ self,+ *,+ target,+ args = (),+ kwargs = None,+ **_kwargs,+ ):+ self.target, self.args, self.kwargs = target, args, kwargs or {}++ def start(self):+ self.target(*self.args, **self.kwargs)+++def _claimed_registry():+ registry = download_registry.DownloadRegistry()+ key = download_registry.normalize_job_key("unsloth/M-GGUF::UD-Q5_K_XL")+ assert registry.claim(+ key,+ download_registry.TRANSPORT_XET,+ repo_type = "model",+ repo_id = "unsloth/M-GGUF",+ variant = "UD-Q5_K_XL",+ blob_hashes = frozenset({"blob"}),+ )[0]+ return key, registry+++_WATCHDOG_STARTS: list = []+++def _ladder_setup(+ monkeypatch,+ tmp_path,+ *,+ largest_file_bytes,+ stall_message = None,+ probe = True,+):+ """Set up the recovery ladder without a real worker."""+ from hub.utils import state_dir++ _WATCHDOG_STARTS.clear()+ monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")+ monkeypatch.setattr(dl.threading, "Thread", _ImmediateThread)+ if probe:+ monkeypatch.setattr(dl, "_largest_file_bytes_for_job", lambda *a, **k: largest_file_bytes)++ def _start(*_a, on_stall, **_k):+ _WATCHDOG_STARTS.append(True)+ if stall_message is not None:+ on_stall(stall_message)+ return None++ monkeypatch.setattr(dl, "_start_stall_watchdog", _start)+++def _run_worker(registry, key, proc):+ return dl.register_worker(+ registry,+ key,+ proc,+ hf_token = None,+ label = "unsloth/M-GGUF",+ log_prefix = "Download",+ logger = dl.logger,+ repo_type = "model",+ repo_id = "unsloth/M-GGUF",+ transport = download_registry.TRANSPORT_XET,+ watch_name = "model-watch",+ )+++def test_the_ladder_does_not_spend_the_job_on_a_doomed_http_retry(monkeypatch, tmp_path):+ """An oversized download must not fall back to HTTP."""+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = _OVERSIZED)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 2)+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"xet transport failed"))++ assert rungs == [], "an oversized download was retried over a transport that cannot serve it"+ state, error, _generation = dl.idle_status(+ registry, key, repo_type = "model", repo_id = "unsloth/M-GGUF", variant = "UD-Q5_K_XL"+ )+ assert state == "error"+ assert "xet transport failed" in error+ assert "HTTPS cannot fetch" in error+++def test_an_oversized_download_still_gets_its_xet_retry(monkeypatch, tmp_path):+ """HTTP unavailability must not remove the Xet retry."""+ _ladder_setup(+ monkeypatch,+ tmp_path,+ largest_file_bytes = _OVERSIZED,+ stall_message = "Download appears stalled (xet transport) -- no progress for 30s",+ )+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 2)+ rungs: list = []+ monkeypatch.setattr(+ dl,+ "_try_transport_retry",+ lambda *a, **kw: rungs.append((kw.get("retry_transport"), kw.get("xet_attempt"))),+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"stalled"))++ assert rungs == [(download_registry.TRANSPORT_XET, 2)]+++def test_a_download_under_the_ceiling_keeps_its_http_rung(monkeypatch, tmp_path):+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = _UNDER_THE_CEILING)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"xet transport failed"))++ assert rungs == [download_registry.TRANSPORT_HTTP]+++def test_a_failed_xet_respawn_does_not_drop_to_https_either(monkeypatch, tmp_path):+ """A failed Xet respawn must not bypass HTTP eligibility checks."""+ from hub.utils import state_dir++ monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")+ monkeypatch.setattr(dl, "_largest_file_bytes_for_job", lambda *a, **k: _OVERSIZED)+ monkeypatch.setattr(dl, "spawn_worker", _boom_spawn)+ key, registry = _claimed_registry()+ rungs: list = []+ real_retry = dl._try_transport_retry++ def _record(registry_, key_, **kwargs):+ rungs.append(kwargs.get("retry_transport"))+ return real_retry(registry_, key_, **kwargs)++ monkeypatch.setattr(dl, "_try_transport_retry", _record)++ assert (+ real_retry(+ registry,+ key,+ hf_token = None,+ label = "unsloth/M-GGUF",+ log_prefix = "Download",+ logger = dl.logger,+ repo_type = "model",+ repo_id = "unsloth/M-GGUF",+ watch_name = "model-watch",+ retry_transport = download_registry.TRANSPORT_XET,+ xet_attempt = 2,+ )+ is False+ )++ assert download_registry.TRANSPORT_HTTP not in rungs+ state, error, _generation = dl.idle_status(+ registry, key, repo_type = "model", repo_id = "unsloth/M-GGUF", variant = "UD-Q5_K_XL"+ )+ assert state == "error"+ assert "HTTPS cannot fetch" in error+++def _boom_spawn(*_args, **_kwargs):+ raise RuntimeError("could not fork")+++def test_the_rung_is_judged_on_the_cache_the_worker_leaves_behind(monkeypatch, tmp_path):+ """Judge HTTP eligibility from the cache left by the failed Xet worker."""+ siblings = _flash_next_siblings()+ oversized = next(s for s in siblings if s.size == _OVERSIZED)+ blobs = tmp_path / "models--unsloth--M-GGUF" / "blobs"+ blobs.mkdir(parents = True)++ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(siblings))+ monkeypatch.setattr(+ download_registry,+ "iter_active_repo_cache_dirs",+ lambda *a, **k: iter([blobs.parent]),+ )+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = None, probe = False)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ assert dl._largest_file_bytes_for_job(+ "model", "unsloth/M-GGUF", _metadata_of(registry, key)+ ) == (_OVERSIZED)++ class _LandsTheBigShard(_Proc):+ def wait(self, timeout = None):+ (blobs / oversized.lfs["sha256"]).write_bytes(b"")+ return super().wait(timeout)++ assert _run_worker(registry, key, _LandsTheBigShard(1, b"xet failed on the second shard"))++ assert rungs == [+ download_registry.TRANSPORT_HTTP+ ], "the HTTP rung stayed closed on a measurement the worker had already invalidated"+++def test_a_final_xet_attempt_with_no_http_rung_is_still_watched(monkeypatch, tmp_path):+ """Watch the final Xet attempt even when no retry remains."""+ _ladder_setup(+ monkeypatch,+ tmp_path,+ largest_file_bytes = _OVERSIZED,+ stall_message = "Download appears stalled (xet transport) -- no progress for 30s",+ )+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"killed"))++ assert _WATCHDOG_STARTS, "a final Xet worker was left unwatched"+ assert rungs == []+ state, error, _generation = dl.idle_status(+ registry, key, repo_type = "model", repo_id = "unsloth/M-GGUF", variant = "UD-Q5_K_XL"+ )+ assert state == "error"+ assert "HTTPS cannot fetch" in error+++def _metadata_of(registry, key):+ return registry.get_job_metadata(key)+++# --------------------------------------------------------------------------------------------+# A metadata refresh that fails+# --------------------------------------------------------------------------------------------+++class _OneShotHfApi:+ """Answer ``repo_info`` once, then fail the way a dropped connection does."""++ calls = 0++ def __init__(self, token = None):+ self.token = token++ def repo_info(self, *_args, **_kwargs):+ type(self).calls += 1+ if type(self).calls > 1:+ raise RuntimeError("connection reset by peer")+ return _types.SimpleNamespace(siblings = _flash_next_siblings())+++def _hub_answers_once(monkeypatch):+ """Install a hub whose second listing fails, with the sibling TTL already elapsed."""+ import time as _time+ import huggingface_hub++ _OneShotHfApi.calls = 0+ dl._REPO_SIBLINGS.clear()+ monkeypatch.setattr(huggingface_hub, "HfApi", _OneShotHfApi)+ clock = iter([0.0])+ monkeypatch.setattr(+ dl,+ "time",+ _types.SimpleNamespace(+ monotonic = lambda: next(clock, dl._REPO_SIBLINGS_TTL_SECONDS * 100),+ sleep = _time.sleep,+ ),+ )+++def test_a_failed_refresh_serves_the_listing_it_already_read(monkeypatch):+ """A dropped refresh must not turn a measured size into an unmeasured one."""+ _hub_answers_once(monkeypatch)++ first = dl.largest_download_file_bytes("model", "unsloth/M-GGUF", variant = "ud-q5_k_xl")+ second = dl.largest_download_file_bytes("model", "unsloth/M-GGUF", variant = "ud-q5_k_xl")++ assert _OneShotHfApi.calls == 2, "the second measurement must really have gone to the hub"+ assert first == _OVERSIZED+ assert second == _OVERSIZED+++def test_filling_the_cache_evicts_one_entry_not_all_of_them(monkeypatch):+ """An in-flight download's listing must survive unrelated traffic."""+ import huggingface_hub++ dl._REPO_SIBLINGS.clear()+ monkeypatch.setattr(+ huggingface_hub,+ "HfApi",+ lambda token = None: _types.SimpleNamespace(+ repo_info = lambda *a, **k: _types.SimpleNamespace(siblings = ())+ ),+ )+ for i in range(dl._REPO_SIBLINGS_MAX + 1):+ dl._repo_siblings("model", f"org/r{i}", None)++ assert len(dl._REPO_SIBLINGS) == dl._REPO_SIBLINGS_MAX+ assert ("model", "org/r0", dl.hf_cache_scan.token_fingerprint(None)) not in dl._REPO_SIBLINGS+++def test_the_http_rung_stays_closed_when_the_refresh_fails(monkeypatch, tmp_path):+ """The post-exit recheck must not reopen HTTP on metadata it could not refresh."""+ _hub_answers_once(monkeypatch)+ blobs = tmp_path / "models--unsloth--M-GGUF" / "blobs"+ blobs.mkdir(parents = True)+ monkeypatch.setattr(+ download_registry,+ "iter_active_repo_cache_dirs",+ lambda *a, **k: iter([blobs.parent]),+ )+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = None, probe = False)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"xet transport failed"))++ assert rungs == [], "a doomed HTTP retry was spent on a size the refresh could not re-read"+ state, error, _generation = dl.idle_status(+ registry, key, repo_type = "model", repo_id = "unsloth/M-GGUF", variant = "UD-Q5_K_XL"+ )+ assert state == "error"+ assert "HTTPS cannot fetch" in error+++def test_a_finalized_shard_still_reopens_http_on_a_failed_refresh(monkeypatch, tmp_path):+ """Whether the oversized shard landed is read from the local cache, not from the hub."""+ _hub_answers_once(monkeypatch)+ oversized = next(s for s in _flash_next_siblings() if s.size == _OVERSIZED)+ blobs = tmp_path / "models--unsloth--M-GGUF" / "blobs"+ blobs.mkdir(parents = True)+ monkeypatch.setattr(+ download_registry,+ "iter_active_repo_cache_dirs",+ lambda *a, **k: iter([blobs.parent]),+ )+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = None, probe = False)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ class _LandsTheBigShard(_Proc):+ def wait(self, timeout = None):+ (blobs / oversized.lfs["sha256"]).write_bytes(b"")+ return super().wait(timeout)++ assert _run_worker(registry, key, _LandsTheBigShard(1, b"xet failed on the second shard"))++ assert rungs == [download_registry.TRANSPORT_HTTP]+++# --------------------------------------------------------------------------------------------+# Whose credential the probe measures with+# --------------------------------------------------------------------------------------------+++def _record_probe_tokens(monkeypatch) -> list:+ """Capture the token every ``repo_info`` call is made under."""+ import huggingface_hub++ seen: list = []+ dl._REPO_SIBLINGS.clear()++ class _Recorder:+ def __init__(self, token = None):+ self.token = token++ def repo_info(self, *_args, **_kwargs):+ seen.append(self.token)+ return _types.SimpleNamespace(siblings = _flash_next_siblings())++ monkeypatch.setattr(huggingface_hub, "HfApi", _Recorder)+ return seen+++def test_a_caller_denied_the_ambient_login_measures_anonymously(monkeypatch):+ """``token=None`` would borrow the backend's saved login; ``False`` is the anonymous sentinel."""+ seen = _record_probe_tokens(monkeypatch)++ dl.largest_download_file_bytes(+ "model", "unsloth/M-GGUF", variant = "ud-q5_k_xl", allow_ambient_token = False+ )++ assert seen == [False]+++def test_a_ui_session_still_measures_under_the_ambient_login(monkeypatch):+ seen = _record_probe_tokens(monkeypatch)++ dl.largest_download_file_bytes(+ "model", "unsloth/M-GGUF", variant = "ud-q5_k_xl", allow_ambient_token = True+ )++ assert seen == [None]+++def test_an_anonymous_probe_does_not_read_an_ambient_cache_entry(monkeypatch):+ """The two boundaries take separate cache identities, so neither serves the other."""+ seen = _record_probe_tokens(monkeypatch)++ dl.largest_download_file_bytes("model", "unsloth/M-GGUF", allow_ambient_token = True)+ dl.largest_download_file_bytes("model", "unsloth/M-GGUF", allow_ambient_token = False)++ assert seen == [None, False]+ assert len(dl._REPO_SIBLINGS) == 2+++def test_the_watcher_probes_under_the_jobs_own_boundary(monkeypatch, tmp_path):+ """Every rung of the ladder measures with the token policy the job started under."""+ seen = _record_probe_tokens(monkeypatch)+ blobs = tmp_path / "models--unsloth--M-GGUF" / "blobs"+ blobs.mkdir(parents = True)+ monkeypatch.setattr(+ download_registry,+ "iter_active_repo_cache_dirs",+ lambda *a, **k: iter([blobs.parent]),+ )+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = None, probe = False)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ monkeypatch.setattr(dl, "_try_transport_retry", lambda *a, **kw: None)+ key, registry = _claimed_registry()++ assert dl.register_worker(+ registry,+ key,+ _Proc(1, b"xet transport failed"),+ hf_token = None,+ label = "unsloth/M-GGUF",+ log_prefix = "Download",+ logger = dl.logger,+ repo_type = "model",+ repo_id = "unsloth/M-GGUF",+ transport = download_registry.TRANSPORT_XET,+ watch_name = "model-watch",+ allow_ambient_token = False,+ )++ assert seen, "the watcher never measured"+ assert set(seen) == {False}, "a watcher probe borrowed the backend's own login"+++# --------------------------------------------------------------------------------------------+# The request path measures what the worker will actually fetch+# --------------------------------------------------------------------------------------------+++def _request_path(monkeypatch):+ """Take a model download request as far as the transport decision, then stop."""+ from hub.services.models import downloads as model_downloads++ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(_flash_next_siblings()))+ monkeypatch.setattr(model_downloads, "_reject_if_load_in_flight", lambda repo_id: None)+ monkeypatch.setattr(model_downloads, "resolve_cached_repo_id_case", lambda repo, **k: repo)+ monkeypatch.setattr(model_downloads, "scoped_file_blob_hashes", lambda *a, **k: frozenset())+ monkeypatch.setattr(dl, "launch_worker", lambda *a, **k: "running")+ return model_downloads+++def _download(model_downloads, **over):+ import asyncio++ from hub.schemas.downloads import DownloadModelRequest++ body = {"repo_id": "unsloth/M-GGUF", "transport_mode": "http"}+ body.update(over)+ return asyncio.run(model_downloads.download_model_response(DownloadModelRequest(**body)))+++_BIG_SHARD = "UD-Q5_K_XL/M-UD-Q5_K_XL-00001-of-00002.gguf"+++def test_files_without_a_scope_id_do_not_decide_the_transport(monkeypatch):+ """``files`` is ignored without ``scope_id``, so it must not shrink the measured size."""+ model_downloads = _request_path(monkeypatch)++ with pytest.raises(HTTPException) as excinfo:+ _download(+ model_downloads,+ gguf_variant = "UD-Q5_K_XL",+ files = ["README.md"],+ )++ assert excinfo.value.status_code == 400+ assert "HTTPS cannot fetch" in excinfo.value.detail+++def test_an_ignored_file_list_does_not_refuse_a_download_that_fits(monkeypatch):+ """The other direction: an ignored oversized name must not reject a valid snapshot."""+ model_downloads = _request_path(monkeypatch)++ result = _download(model_downloads, files = [_BIG_SHARD])++ assert result["transport"] == download_registry.TRANSPORT_HTTP+++def test_a_real_scope_is_still_measured_on_its_files(monkeypatch):+ model_downloads = _request_path(monkeypatch)++ with pytest.raises(HTTPException) as excinfo:+ _download(model_downloads, scope_id = "diffusion", files = [_BIG_SHARD])++ assert excinfo.value.status_code == 400+ assert "HTTPS cannot fetch" in excinfo.value.detail+++# --------------------------------------------------------------------------------------------+# A measurement the cache no longer holds+# --------------------------------------------------------------------------------------------+++def test_nothing_left_to_fetch_is_zero_not_unknown(monkeypatch, tmp_path):+ """A carried measurement only stands over an unknown, so the two must not share a value."""+ siblings = [_Sibling("only.gguf", _OVERSIZED)]+ monkeypatch.setattr(dl, "_repo_siblings", lambda *a, **k: tuple(siblings))+ blobs = tmp_path / "models--unsloth--M-GGUF" / "blobs"+ blobs.mkdir(parents = True)+ (blobs / siblings[0].lfs["sha256"]).write_bytes(b"")+ monkeypatch.setattr(+ download_registry,+ "iter_active_repo_cache_dirs",+ lambda *a, **k: iter([blobs.parent]),+ )++ assert dl.largest_download_file_bytes("model", "unsloth/M-GGUF") == 0+++def _evicting_probe(values):+ """A probe that measures once and is then unable to measure at all."""+ remaining = list(values)++ def _probe(*_a, **_k):+ return remaining.pop(0) if remaining else None++ return _probe+++def test_an_evicted_listing_does_not_reopen_the_http_rung(monkeypatch, tmp_path):+ """Losing the cached listing must not turn a measured size back into an unknown one."""+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = None, probe = False)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ monkeypatch.setattr(dl, "_largest_file_bytes_for_job", _evicting_probe([_OVERSIZED]))+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"xet transport failed"))++ assert rungs == [], "an evicted listing reopened a rung that cannot serve this download"+ state, error, _generation = dl.idle_status(+ registry, key, repo_type = "model", repo_id = "unsloth/M-GGUF", variant = "UD-Q5_K_XL"+ )+ assert state == "error"+ assert "HTTPS cannot fetch" in error+++def test_a_finalized_shard_still_overrides_the_carried_size(monkeypatch, tmp_path):+ """A fresh measurement always wins; only an unmeasurable one defers to the carried size."""+ _ladder_setup(monkeypatch, tmp_path, largest_file_bytes = None, probe = False)+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 1)+ monkeypatch.setattr(dl, "_largest_file_bytes_for_job", _evicting_probe([_OVERSIZED, 0]))+ rungs: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: rungs.append(kw.get("retry_transport"))+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"xet failed on the second shard"))++ assert rungs == [download_registry.TRANSPORT_HTTP]+++def test_the_carried_size_survives_a_xet_retry(monkeypatch, tmp_path):+ """The next rung inherits it, so an hours-long job never measures from scratch alone."""+ _ladder_setup(+ monkeypatch,+ tmp_path,+ largest_file_bytes = None,+ probe = False,+ stall_message = "Download appears stalled (xet transport) -- no progress for 30s",+ )+ monkeypatch.setattr(dl, "_xet_attempt_budget", lambda: 2)+ monkeypatch.setattr(dl, "_largest_file_bytes_for_job", _evicting_probe([_OVERSIZED]))+ carried: list = []+ monkeypatch.setattr(+ dl, "_try_transport_retry", lambda *a, **kw: carried.append(kw.get("largest_file_bytes"))+ )+ key, registry = _claimed_registry()++ assert _run_worker(registry, key, _Proc(1, b"stalled"))++ assert carried == [_OVERSIZED]diff --git a/studio/backend/tests/test_hub_download_token_off_event_loop.py b/studio/backend/tests/test_hub_download_token_off_event_loop.pyindex 9fa3912328a..fed855460e1 100644--- a/studio/backend/tests/test_hub_download_token_off_event_loop.py+++ b/studio/backend/tests/test_hub_download_token_off_event_loop.py@@ -129,13 +129,13 @@ def test_cancellation_while_executor_busy_leaves_no_claim(monkeypatch, download)task = asyncio.current_task()blocker = None- def transport(use_xet):+ def transport(use_xet, **kwargs):nonlocal blocker# Earlier preparation awaits have finished. Occupy the only thread so# the next offloaded operation is queued when the request is cancelled.blocker = loop.run_in_executor(None, release.wait, 15)loop.call_soon(task.cancel)- return original_transport(use_xet)+ return original_transport(use_xet, **kwargs)monkeypatch.setattr(download_lifecycle, "resolve_transport", transport)try:diff --git a/studio/backend/tests/test_hub_download_transport_auto.py b/studio/backend/tests/test_hub_download_transport_auto.pyindex 6f94949246c..90d89b532e8 100644--- a/studio/backend/tests/test_hub_download_transport_auto.py+++ b/studio/backend/tests/test_hub_download_transport_auto.py@@ -59,12 +59,12 @@ def test_explicit_xet_beats_an_unhealthy_verdict(monkeypatch):"""An explicit choice is not overruled by the health verdict; it still gets the memory caps andthe stall fallback."""monkeypatch.setattr(dl, "resolve_effective_use_xet", lambda requested: requested)- monkeypatch.setattr(dl, "resolve_auto_use_xet", lambda: (False, "demoted"))+ monkeypatch.setattr(dl, "resolve_auto_use_xet", lambda **kw: (False, "demoted"))assert dl.resolve_requested_use_xet("xet", True)[0] is Truedef test_auto_defers_to_the_health_verdict(monkeypatch):- monkeypatch.setattr(dl, "resolve_auto_use_xet", lambda: (False, "Xet stalled twice"))+ monkeypatch.setattr(dl, "resolve_auto_use_xet", lambda **kw: (False, "Xet stalled twice"))use_xet, reason = dl.resolve_requested_use_xet("auto", True)assert use_xet is Falseassert reason == "Xet stalled twice"diff --git a/studio/backend/tests/test_job_accounts.py b/studio/backend/tests/test_job_accounts.pyindex 135a224c460..ea1ede0b3d2 100644--- a/studio/backend/tests/test_job_accounts.py+++ b/studio/backend/tests/test_job_accounts.py@@ -1138,7 +1138,7 @@ def test_a_second_account_cannot_start_the_same_dataset_download(monkeypatch):monkeypatch.setattr(downloads, "_deleting", set(), raising = False)monkeypatch.setattr(downloads, "resolve_cached_repo_id_case", lambda repo_id, **_k: repo_id)monkeypatch.setattr(- downloads.download_registry, "download_transport_unavailable_reason", lambda _t: None+ downloads.download_registry, "download_transport_unavailable_reason", lambda _t, **_kw: None)monkeypatch.setattr(downloads.download_manifest, "clear_cancel_marker", lambda *a, **k: None)monkeypatch.setattr(downloads.account_access, "authorize_download", lambda *a, **k: None)@@ -1167,7 +1167,7 @@ def test_dataset_download_request_authorizes_before_reporting_a_foreign_job(monkmonkeypatch.setattr(downloads, "_deleting", set(), raising = False)monkeypatch.setattr(downloads, "resolve_cached_repo_id_case", lambda repo_id, **_k: repo_id)monkeypatch.setattr(- downloads.download_registry, "download_transport_unavailable_reason", lambda _t: None+ downloads.download_registry, "download_transport_unavailable_reason", lambda _t, **_kw: None)monkeypatch.setattr(downloads.download_manifest, "clear_cancel_marker", lambda *a, **k: None)monkeypatch.setattr(