Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(docker): bake checkpoints from ModelScope at build time
gqchen3be99e7707d6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 39843b20..6de66e70 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -138,6 +138,7 @@ jobs:python tests/test_context_manager.pypython tests/test_criteria_normalization.pypython tests/test_docker_entrypoint.py+ python tests/test_modelscope_prefetch.pypython tests/test_empty_questions.pypython tests/test_router_memory.pypython tests/test_shortlist_cosine.pydiff --git a/.github/workflows/docker.yml b/.github/workflows/docker.ymlindex e7b814d0..c1a3bfdb 100644--- a/.github/workflows/docker.yml+++ b/.github/workflows/docker.yml@@ -59,6 +59,12 @@ jobs:docker compose -f compose.yaml -f compose.cuda.yaml config --quietdocker compose -f compose.yaml -f compose.http.yaml config --quietdocker compose -f compose.yaml -f compose.http.yaml -f compose.cuda.yaml config --quiet+ # ModelScope prefetch, alone and stacked on top of every other combination a+ # deployment reaches for.+ docker compose -f compose.yaml -f compose.modelscope.yaml config --quiet+ docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml config --quiet+ docker compose -f compose.yaml -f compose.http.yaml -f compose.cuda.yaml -f compose.modelscope.yaml config --quiet+ docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml -f compose.modelscope.yaml config --quietdocker build --pull -t laya:smoke .- name: Check installed runtime and fresh cache permissionsrun: |diff --git a/Dockerfile b/Dockerfileindex 3806d4b1..a940c1f4 100644--- a/Dockerfile+++ b/Dockerfile@@ -29,6 +29,13 @@ RUN pip install ".[serve]" && pip checkFROM ${PYTHON_IMAGE} AS runtime+# Optional ModelScope prefetch: with `--build-arg MODELSCOPE_MODEL=multilingual` the checkpoint is+# baked into the hub cache during the build, so the image never depends on huggingface.co. The+# argument takes a checkpoint type (multilingual, english, typed-decisions, all) or a comma- or+# space-separated list of `repo[:subfolder]` specs; empty by default leaves the image as it was.+ARG MODELSCOPE_MODEL=""+ARG MODELSCOPE_REVISION="master"+LABEL org.opencontainers.image.title="Laya Docker quickstart" \org.opencontainers.image.source="https://github.com/NandhaKishorM/laya" \org.opencontainers.image.licenses="Apache-2.0"@@ -57,6 +64,21 @@ COPY --from=build /opt/venv /opt/venvCOPY LICENSE /usr/share/doc/laya/LICENSECOPY examples/docker/ /opt/laya/examples/COPY docker/entrypoint.py /opt/laya/entrypoint.py+COPY docker/prefetch_modelscope.py /opt/laya/prefetch_modelscope.py++# Bake the requested ModelScope checkpoints into the hub cache ($HF_HOME/hub), laid out the way+# `snapshot_download` reads them offline, so no entry point changes: the quickstart's Router, the+# HTTP server's Router and `laya.cli` all resolve their repo ids to the baked snapshot. The cache is+# handed to the runtime user afterwards, because the tokenizer-compatibility fix writes into the+# snapshot on first load.+RUN if [ -n "$MODELSCOPE_MODEL" ]; then \+ python /opt/laya/prefetch_modelscope.py \+ --model "$MODELSCOPE_MODEL" \+ --revision "$MODELSCOPE_REVISION" \+ --cache-dir "$HF_HOME/hub" \+ && chown -R laya:laya /home/laya/.cache; \+ fi+USER layaWORKDIR /home/layadiff --git a/compose.modelscope.yaml b/compose.modelscope.yamlnew file mode 100644index 00000000..aa79331b--- /dev/null+++ b/compose.modelscope.yaml@@ -0,0 +1,43 @@+# Bake the checkpoint from ModelScope (https://modelscope.cn) into the image, so neither the build+# nor the running container has to reach huggingface.co:+#+# docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve+# curl -s localhost:8000/health+# curl -s localhost:8000/v1/systemone -H 'content-type: application/json' \+# --data @examples/docker/request.json+#+# The weights land in the hub cache the image already mounts as the model-cache volume, laid out the+# way `snapshot_download` reads them offline, so the quickstart, the HTTP server's Router and the CLI+# keep their repo ids and resolve them to the baked snapshot: docker/prefetch_modelscope.py does it+# during the build and stops it on any size or digest mismatch.+#+# For NVIDIA, append -f compose.cuda.yaml; it repeats the build args and the device reservation for+# both services, because an override for `laya` never reaches `laya-serve`.+#+# MODELSCOPE_MODEL takes a checkpoint type -- multilingual (the default), english, typed-decisions,+# all -- or comma- or space-separated `repo[:subfolder]` specs for a standalone repository or a+# fine-tuned checkpoint. A type maps to that checkpoint's path in the bundled repository, which is+# what the Router and the quickstart load by default, so a single-argument build serves it with no+# further change. The default bakes only the multilingual checkpoint (~650 MB); see docs/docker.md+# for the other two and for what a baked-only image can still load.+services:+ laya:+ build:+ args:+ MODELSCOPE_MODEL: "${MODELSCOPE_MODEL:-multilingual}"+ MODELSCOPE_REVISION: "${MODELSCOPE_REVISION:-master}"+ environment:+ # The baked snapshot is keyed by a ModelScope commit, which the Hub cannot confirm; going+ # online would download the same weights again instead of using the baked copy.+ HF_HUB_OFFLINE: "${HF_HUB_OFFLINE:-1}"+ laya-serve:+ build:+ args:+ MODELSCOPE_MODEL: "${MODELSCOPE_MODEL:-multilingual}"+ MODELSCOPE_REVISION: "${MODELSCOPE_REVISION:-master}"+ environment:+ HF_HUB_OFFLINE: "${HF_HUB_OFFLINE:-1}"+ # Restrains preload to the checkpoint that was baked. Routing itself still selects the+ # checkpoint that fits the text, so an English-serving deployment bakes that one too and+ # names it here: `LAYA_PRELOAD=1` with a list an image cannot serve fails on the first one.+ LAYA_MODELS: "${LAYA_MODELS:-multilingual}"diff --git a/docker/prefetch_modelscope.py b/docker/prefetch_modelscope.pynew file mode 100644index 00000000..5f42a475--- /dev/null+++ b/docker/prefetch_modelscope.py@@ -0,0 +1,292 @@+"""Bake Laya checkpoints into a Hugging Face cache from ModelScope, during an image build.++The runtime resolves a repo id through `huggingface_hub.snapshot_download`, which reads its+local cache. With `HF_HUB_OFFLINE=1` -- or a network that cannot reach huggingface.co -- that+resolution is purely local: `refs/<revision>` names a commit and an existing+`snapshots/<sha>` directory is returned as it is, with no repo listing at all+(`huggingface_hub/_snapshot_download.py`). A mirror therefore only has to lay the files out the+way the Hub would have, and every code path that names a checkpoint keeps its ids and its+revision handling: `Agent(...)`, the `Router` laya-serve builds, `laya.cli`, the integrations.++Only the standard library is used, because an image build has no reason to install the+ModelScope SDK for a handful of HTTP calls. Every file is streamed with resume and its size and SHA-256+are checked against the values the repository API reports, so a truncated or substituted+download fails the build instead of shipping inside a layer that looks healthy.++Usage inside a Dockerfile RUN:++ python prefetch_modelscope.py --models convaiinnovations/laya:multilingual \+ --revision master --cache-dir "$HF_HOME/hub"++Each spec is `repo[:subfolder]`; the subfolder selects one checkpoint from the bundled+repository, the same way `Agent("convaiinnovations/laya", subfolder="multilingual")` does.+"""+import argparse+import hashlib+import http.client+import json+import os+import re+import shutil+import sys+import time+import urllib.error+import urllib.parse+import urllib.request+from pathlib import Path++FILES_API = "https://modelscope.cn/api/v1/models/{repo}/repo/files"+FILE_API = "https://modelscope.cn/api/v1/models/{repo}/repo"+# What a Laya checkpoint needs: the same set laya/agent.py hands snapshot_download, so a baked+# snapshot carries exactly the files a load would have fetched and none of its siblings.+NEEDED = ("rl_agent_config.json", "model.safetensors")+FOLDERS = ("tokenizer/", "encoder/")+CHUNK = 1 << 20+RETRIES = 4+BACKOFF = 2.0+TIMEOUT = 60+# The revision the runtime asks for by default. The mirror publishes `master`, the Hub publishes+# `main`, and a baked snapshot answers both: this is the name `refs/` is written under.+DEFAULT_BRANCH = "main"++# A checkpoint type and the path it means inside the bundled repository. These are the paths the+# `Router` and the one-shot quickstart load by default (laya/router.py: DEFAULT_MODELS), which is+# why a type bakes them rather than the standalone repositories of the same checkpoints: a baked+# image then serves the default configuration with no change at all.+TYPES = {+ "english": ["convaiinnovations/laya:"],+ "multilingual": ["convaiinnovations/laya:multilingual"],+ "typed-decisions": ["convaiinnovations/laya:typed-decisions"],+}+# The spellings laya/router.py accepts for the same three checkpoints, so an operator who copies a+# name out of that mapping is not told it is unknown here.+ALIASES = {"en": "english", "laya": "english", "default": "english",+ "multi": "multilingual", "ml": "multilingual",+ "typed": "typed-decisions", "decisions": "typed-decisions"}+++def expand_model(value: str):+ """A type name, or one or more `repo[:subfolder]` specs, into the specs to bake.++ `all` is the whole family. Anything holding a `/` is taken as a spec, which is how a standalone+ repository (`convaiinnovations/laya-multilingual`) or a fine-tuned checkpoint is baked; specs+ and types can be mixed in one comma- or space-separated value.+ """+ parts = [part for part in re.split(r"[,\s]+", (value or "").strip()) if part]+ if not parts:+ raise SystemExit("laya: --model is empty; expected a checkpoint type (%s) or a "+ "repo[:subfolder] spec" % ", ".join(sorted(TYPES) + ["all"]))+ specs = []+ for part in parts:+ if "/" in part:+ specs.append(part)+ continue+ name = ALIASES.get(part.lower(), part.lower())+ if name == "all":+ specs += [spec for key in sorted(TYPES) for spec in TYPES[key]]+ elif name in TYPES:+ specs += TYPES[name]+ else:+ raise SystemExit("laya: unknown checkpoint type %r; choose one of %s, all, or a "+ "repo[:subfolder] spec" % (part, sorted(TYPES)))+ return specs+++def wanted(rel: str) -> bool:+ """Whether a path inside the subfolder is one of the files a checkpoint load fetches."""+ return rel in NEEDED or rel.startswith(FOLDERS)+++def split_spec(spec: str):+ """`repo[:subfolder]` -> (repo, subfolder); `:` with nothing after it means the root."""+ repo, sep, sub = spec.partition(":")+ return repo.strip(), sub.strip() if sep else ""+++def cache_root(override):+ """The hub cache directory: an explicit path, `HF_HUB_CACHE`, or `$HF_HOME/hub`."""+ if override:+ return Path(override)+ explicit = os.environ.get("HF_HUB_CACHE")+ if explicit:+ return Path(explicit)+ home = os.environ.get("HF_HOME") or os.path.join(os.path.expanduser("~"), ".cache", "huggingface")+ return Path(home) / "hub"+++def repo_folder(repo: str) -> str:+ """The cache directory name the Hub derives from a repo id: `ns/name` -> `models--ns--name`."""+ return "models--" + repo.replace("/", "--")+++def list_repo(repo: str, revision: str, subfolder: str):+ """(tip commit, entries) for the revision's own files inside `subfolder`.++ Each entry reports the path as the API names it, the path relative to the checkpoint -- what+ the Hub caches -- its size, and its SHA-256 when the API publishes one. The listing API returns+ one directory at a time and names subdirectories as `tree` entries, so the walk follows them:+ the bundled repository keeps each checkpoint's tokenizer and encoder in their own directories.+ """+ entries, tip, queue = [], "", [subfolder]+ prefix = subfolder + "/" if subfolder else ""+ while queue:+ query = urllib.parse.urlencode({"Revision": revision, "Root": queue.pop(0)})+ try:+ with urllib.request.urlopen(FILES_API.format(repo=repo) + "?" + query,+ timeout=TIMEOUT) as r:+ payload = json.loads(r.read().decode("utf-8"))+ except (urllib.error.URLError, urllib.error.HTTPError, ValueError, OSError) as exc:+ raise SystemExit("laya: cannot list %s at revision %r: %s" % (repo, revision, exc))+ if not payload.get("Success"):+ raise SystemExit("laya: ModelScope returned %r for %s -- check the repo id and revision"+ % (payload.get("Message"), repo))+ data = payload.get("Data") or {}+ tip = tip or ((data.get("LatestCommitter") or {}).get("Revision") or "").strip()+ for item in data.get("Files") or []:+ path, kind = item.get("Path", ""), item.get("Type")+ if kind == "tree":+ if path.startswith(prefix):+ queue.append(path)+ continue+ if kind != "blob" or not path.startswith(prefix):+ continue+ rel = path[len(prefix):]+ if not wanted(rel):+ continue+ entries.append({+ "path": path,+ "rel": rel,+ "size": int(item.get("Size") or 0),+ "sha256": (item.get("Sha256") or "").strip(),+ "revision": (item.get("Revision") or "").strip(),+ })+ return tip, entries+++def snapshot_commit(tip: str, entries: list) -> str:+ """The commit to key the baked snapshot by: the weights' own, else the revision's tip.++ The weights' commit is the one a caller would pin to get *these* bytes back, so it is the+ more useful key. A mixed-commit fetch still lands as one snapshot directory, because the+ offline path never compares the file list against a commit -- but that is a warning, not a+ failure: a maintenance commit to a README must not break a build.+ """+ commits = {entry["revision"] for entry in entries if entry["revision"]}+ weights = [entry["revision"] for entry in entries+ if entry["rel"] == "model.safetensors" and entry["revision"]]+ if len(commits) > 1:+ print(" note: files span commits %s; keying the snapshot by the weights' commit"+ % sorted(commits), file=sys.stderr)+ return (weights[0] if weights else "") or tip+++def download(repo: str, revision: str, entry: dict, dest: Path) -> None:+ """Stream one file into `dest`, resuming a partial copy and verifying what landed."""+ query = urllib.parse.urlencode({"FilePath": entry["path"], "Revision": revision})+ url = FILE_API.format(repo=repo) + "?" + query+ for attempt in range(1, RETRIES + 1):+ have = dest.stat().st_size if dest.exists() else 0+ headers = {"Range": "bytes=%d-" % have} if have else {}+ try:+ with urllib.request.urlopen(urllib.request.Request(url, headers=headers),+ timeout=TIMEOUT) as response:+ # A 206 answers the Range request; anything else starts the file over, because+ # appending a full body to a partial one would corrupt it.+ mode = "ab" if response.status == 206 and have else "wb"+ with open(dest, mode) as fh:+ while True:+ chunk = response.read(CHUNK)+ if not chunk:+ break+ fh.write(chunk)+ except (urllib.error.URLError, urllib.error.HTTPError, OSError,+ http.client.HTTPException) as exc:+ print(" retry %d/%d after %s: %s" % (attempt, RETRIES, type(exc).__name__, exc),+ file=sys.stderr)+ # Written outside the `try`, so a response that closed early still re-asks for the rest+ # instead of being treated as a complete file.+ on_disk = dest.stat().st_size if dest.exists() else 0+ if entry["size"] and on_disk == entry["size"]:+ break+ if attempt == RETRIES:+ raise SystemExit("laya: %s: %d of %d bytes after %d attempts -- check the connection "+ "to modelscope.cn" % (entry["path"], on_disk, entry["size"], RETRIES))+ time.sleep(BACKOFF * attempt)++ digest = hashlib.sha256()+ with open(dest, "rb") as fh:+ while True:+ chunk = fh.read(CHUNK)+ if not chunk:+ break+ digest.update(chunk)+ if entry["sha256"] and digest.hexdigest() != entry["sha256"]:+ raise SystemExit("laya: %s: sha256 mismatch -- the repository reports %s, the download "+ "has %s" % (entry["path"], entry["sha256"], digest.hexdigest()))+++def seed(cache: Path, repo: str, commit: str, revision: str, staging: Path) -> None:+ """Publish a verified download as a hub snapshot, then name its commit in `refs/`."""+ repo_dir = cache / repo_folder(repo)+ snapshot = repo_dir / "snapshots" / commit+ refs = repo_dir / "refs"+ refs.mkdir(parents=True, exist_ok=True)+ shutil.rmtree(snapshot, ignore_errors=True)+ # Renamed into place only once it holds every file it claims to, and the ref is written only+ # after the rename: an offline snapshot_download returns whatever `snapshots/<sha>` already+ # contains, so a half-published snapshot would otherwise be served as if complete.+ os.replace(str(staging), str(snapshot))+ for name in (DEFAULT_BRANCH, revision):+ if name and len(name) < 64:+ (refs / name).write_text(commit, encoding="utf-8")+++def prefetch(spec: str, revision: str, cache: Path) -> None:+ repo, subfolder = split_spec(spec)+ tip, entries = list_repo(repo, revision, subfolder)+ if not entries:+ raise SystemExit("laya: %s%s holds no checkpoint files at revision %r"+ % (repo, "/" + subfolder if subfolder else "", revision))+ commit = snapshot_commit(tip, entries)+ if not commit:+ raise SystemExit("laya: %s%s names no commit at revision %r, so there is nothing to key "+ "the baked snapshot by" % (repo, "/" + subfolder if subfolder else "",+ revision))+ repo_dir = cache / repo_folder(repo)+ staging = repo_dir / "snapshots" / (".staging-" + str(os.getpid()))+ try:+ for entry in entries:+ # The repository-relative path, not the checkpoint-relative one: `Agent` appends the+ # subfolder itself (`model_dir/<subfolder>`), so the snapshot has to mirror the repo.+ dest = staging / entry["path"]+ dest.parent.mkdir(parents=True, exist_ok=True)+ print(" %s: %d bytes" % (entry["path"], entry["size"]))+ download(repo, revision, entry, dest)+ seed(cache, repo, commit, revision, staging)+ finally:+ shutil.rmtree(staging, ignore_errors=True)+ print(" baked as models--%s/snapshots/%s" % (repo.replace("/", "--"), commit))+++def main() -> int:+ parser = argparse.ArgumentParser(description="Bake ModelScope checkpoints into a hub cache")+ parser.add_argument("--model", required=True, metavar="TYPE_OR_SPEC",+ help="checkpoint type (multilingual, english, typed-decisions, all) or one "+ "or more comma- or space-separated repo[:subfolder] specs")+ parser.add_argument("--revision", default="master",+ help="ModelScope revision (branch, tag or commit); default master")+ parser.add_argument("--cache-dir", default=None,+ help="hub cache directory; defaults to HF_HUB_CACHE or $HF_HOME/hub")+ args = parser.parse_args()+ specs = expand_model(args.model)+ cache = cache_root(args.cache_dir)+ cache.mkdir(parents=True, exist_ok=True)+ for spec in specs:+ print("modelscope/%s at revision %s" % (spec, args.revision))+ prefetch(spec, args.revision, cache)+ print("baked %d checkpoint(s) into %s" % (len(specs), cache))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/docs/docker.md b/docs/docker.mdindex 41eec446..1c9dbb31 100644--- a/docs/docker.md+++ b/docs/docker.md@@ -161,6 +161,78 @@ are mutually exclusive. The local-path response comes from the Agent and has noRouter `routing` metadata. These settings also work with the CUDA override.Evaluate fine-tuned checkpoints on held-out examples before relying on them.+## Models from ModelScope++When a host cannot reach huggingface.co, the checkpoint can come from+[ModelScope](https://modelscope.cn) and be baked into the image at build time. One+argument selects which checkpoint, and it defaults to the multilingual one. The+Compose override adds the prefetch arguments to both services and keeps the+container off the Hub:++```bash+docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve+```++For NVIDIA, add `-f compose.cuda.yaml` before `up`; it repeats its own arguments for+both services, so the ordering between the two overrides does not matter. Plain+Docker takes the arguments directly:++```bash+docker build --build-arg MODELSCOPE_MODEL=multilingual \+ -t laya:local .+docker run --rm -e HF_HUB_OFFLINE=1 -p 127.0.0.1:8000:8000 laya:local laya-serve+```++[`docker/prefetch_modelscope.py`](https://github.com/NandhaKishorM/laya/blob/main/docker/prefetch_modelscope.py)+lists the repository on modelscope.cn, downloads the checkpoint's own files -- the+same set `laya/agent.py` asks the Hub for, so no sibling checkpoint is pulled -- and+writes them into the image's hub cache the way `snapshot_download` lays out a+snapshot. Nothing else changes: `Agent`, the `Router` that `laya-serve` builds,+`laya.cli` and the integrations keep their repo ids and resolve them to the baked+snapshot, so a container built this way needs no network at all. Each file's size+and SHA-256 are checked against what the repository reports before the snapshot is+published, so a truncated or substituted download fails the build instead of+shipping inside a layer that looks healthy.++| Variable | Default | Purpose |+| --- | --- | --- |+| `MODELSCOPE_MODEL` | `multilingual` (Compose); empty in the Dockerfile | Checkpoint to bake: `multilingual`, `english`, `typed-decisions` or `all`. Empty means no prefetch and the unchanged image |+| `MODELSCOPE_REVISION` | `master` | ModelScope branch, tag or commit to bake |+| `HF_HUB_OFFLINE` | `1` in the Compose override | `1` never contacts the Hub, so the baked copy is served |++A type expands to that checkpoint's path inside the bundled repository, which is what+the `Router` and the one-shot quickstart load by default, so a build that names one+type serves it with no further change:++```bash+MODELSCOPE_MODEL=english docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve+MODELSCOPE_MODEL=all docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve+```++`all` is the whole family, about 2.4 GB of weights. The override also sets+`LAYA_MODELS=multilingual`, because `LAYA_PRELOAD=1` with the default list would try+to build every checkpoint and fail on the first one that was not baked; set+`LAYA_MODELS` to the list you baked when you bake more, and `MODELSCOPE_MODEL=all`+when a deployment really does serve the family.++Beyond the four types the argument also takes `repo[:subfolder]` specs, comma- or+space-separated, which is how the mirror's standalone repositories+([laya](https://modelscope.cn/models/convaiinnovations/laya),+[laya-multilingual](https://modelscope.cn/models/convaiinnovations/laya-multilingual),+[laya-typed-decisions](https://modelscope.cn/models/convaiinnovations/laya-typed-decisions))+or a fine-tuned checkpoint is baked. A standalone repository is what+`Agent("convaiinnovations/laya-multilingual")` loads directly; the `Router`'s default+is the bundled path, so a type is normally what a serving image wants.++Two details about pins and provenance. The build prints the commit the snapshot is+keyed by -- pass that SHA as `revision=` or `LAYA_REVISION` to pin a load to exactly+what was baked. Hub-side pins do not describe a mirror snapshot: `reviewed` names+Hugging Face commits, and the SHA-256 digest map is keyed to Hub artifact hashes, so+neither applies here. The build already refuses a download that does not match the+mirror's own reported size and digest, which is the equivalent guarantee for this+path. And the weights come from whichever mirror account the argument names, which is+its own supply-chain decision for the deployment to make.+## Development and cleanupOpen a Python prompt with `docker compose run --rm laya python`. To run thediff --git a/tests/test_modelscope_prefetch.py b/tests/test_modelscope_prefetch.pynew file mode 100644index 00000000..143b5f49--- /dev/null+++ b/tests/test_modelscope_prefetch.py@@ -0,0 +1,384 @@+"""Bake-from-ModelScope has to produce a cache the offline loader accepts -- exactly that.++`docker/prefetch_modelscope.py` writes a Hugging Face snapshot cache during an image build so that+a host that cannot reach huggingface.co can still ship and serve a checkpoint. The whole design+rests on one property, and it is not visible in the script: with `HF_HUB_OFFLINE=1`,+`snapshot_download` returns `<cache>/snapshots/<sha>` whenever `refs/<revision>` names a directory+that already exists, without listing the repository (`huggingface_hub/_snapshot_download.py`).+These tests therefore bake from a stubbed ModelScope API and then ask the real `snapshot_download`+what it resolves, so a layout that merely looks right cannot pass.++The second property worth pinning is the failure direction. A snapshot directory is *served*, not+validated, so anything half-written would be loaded as weights: a download that fails its size or+digest check must leave nothing behind for the loader to find.++No weights, no network, no docker: the file listing and the file bodies are fixtures.+"""+import hashlib+import http.client+import importlib.util+import json+import os+import tempfile+import unittest+import urllib.parse+from pathlib import Path+from unittest.mock import patch++ROOT = Path(__file__).resolve().parents[1]+SCRIPT = ROOT / "docker" / "prefetch_modelscope.py"+spec = importlib.util.spec_from_file_location("prefetch_modelscope", SCRIPT)+prefetch = importlib.util.module_from_spec(spec)+spec.loader.exec_module(prefetch)++WEIGHTS_COMMIT = "a" * 40+TIP_COMMIT = "b" * 40+CONTENTS = {+ "multilingual/rl_agent_config.json": b'{"encoder": "jhu-clsp/mmBERT-base"}',+ "multilingual/model.safetensors": b"WEIGHTS" * 64,+ "multilingual/tokenizer/tokenizer.json": b'{"model": {}}',+ "multilingual/encoder/config.json": b'{"hidden_size": 768}',+ "multilingual/README.md": b"not a checkpoint file",+ "README.md": b"outside the subfolder",+ # A standalone repository keeps the same checkpoint at its root.+ "model.safetensors": b"STANDALONE" * 8,+ "rl_agent_config.json": b"standalone config",+ "tokenizer/tokenizer.json": b"standalone tokenizer",+ "encoder/config.json": b"standalone encoder",+}+++def entry(path, commit):+ body = CONTENTS[path]+ return {"Type": "blob", "Path": path, "Size": len(body),+ "Sha256": hashlib.sha256(body).hexdigest(), "Revision": commit}+++def tree(path):+ return {"Type": "tree", "Path": path, "Size": 0, "Revision": WEIGHTS_COMMIT}+++# The listing API answers one directory at a time and names subdirectories as `tree` entries, the+# way the mirror does: the checkpoint's tokenizer and encoder are pages of their own, and a sibling+# checkpoint directory is a page this bake must not walk into.+PAGES = {+ "multilingual": [+ entry("multilingual/model.safetensors", WEIGHTS_COMMIT),+ entry("multilingual/rl_agent_config.json", WEIGHTS_COMMIT),+ entry("multilingual/README.md", WEIGHTS_COMMIT),+ entry("README.md", TIP_COMMIT),+ tree("multilingual/tokenizer"),+ tree("multilingual/encoder"),+ tree("typed-decisions"),+ ],+ "multilingual/tokenizer": [entry("multilingual/tokenizer/tokenizer.json", WEIGHTS_COMMIT)],+ "multilingual/encoder": [entry("multilingual/encoder/config.json", WEIGHTS_COMMIT)],+}+++def page(root):+ listing = {"Success": True,+ "Data": {"Files": PAGES[root], "LatestCommitter": {"Revision": TIP_COMMIT}}}+ return json.dumps(listing).encode("utf-8")+++def params(url):+ # keep_blank_values: the root listing carries `Root=`, which parse_qsl otherwise drops.+ pairs = urllib.parse.parse_qsl(urllib.parse.urlparse(url).query, keep_blank_values=True)+ return dict(pairs)+++class FakeResponse:+ """Serves the listing API, the file API and HTTP ranges, the way the mirror's CDN does."""++ status = 200++ def __init__(self, request, **_):+ self.headers = {key.lower(): value for key, value in request.headers.items()}+ self.url = request.full_url+ self.served = 0+ if "/repo/files" in self.url:+ self.body = page(params(self.url)["Root"])+ return+ self.body = CONTENTS[params(self.url)["FilePath"]]+ start = 0+ header = self.headers.get("range", "")+ if header.startswith("bytes="):+ start = int(header[len("bytes="):].split("-")[0])+ if start:+ self.body = self.body[start:]+ self.status = 206++ def read(self, size=-1):+ size = size if size and size > 0 else len(self.body)+ chunk = self.body[self.served:self.served + size]+ self.served += len(chunk)+ return chunk++ def __enter__(self):+ return self++ def __exit__(self, *_):+ return False+++class Dropping(FakeResponse):+ """Hands over the first 32 bytes of the weights file, then drops the connection mid-body."""++ dropped = False++ def read(self, size=-1):+ if "FilePath" not in self.url:+ return FakeResponse.read(self, size)+ if not params(self.url)["FilePath"].endswith("model.safetensors"):+ return FakeResponse.read(self, size)+ if self.served < 32:+ self.served = 32+ return self.body[:32]+ if not Dropping.dropped and "range" not in self.headers:+ Dropping.dropped = True+ raise http.client.IncompleteRead(b"dropped mid-body")+ return FakeResponse.read(self, size)+++class ShortBody(FakeResponse):+ """Every body is one byte short of the size the listing promises."""++ def __init__(self, request, **_):+ FakeResponse.__init__(self, request, **_)+ if "/repo/files" not in self.url:+ self.body = self.body[:-1]+++class WrongDigest(FakeResponse):+ """Every body hashes to something the listing does not report."""++ def __init__(self, request, **_):+ FakeResponse.__init__(self, request, **_)+ if "/repo/files" not in self.url:+ self.body = b"X" + self.body[1:]+++class EmptyListing(FakeResponse):+ """A repository with no checkpoint files at all."""++ def __init__(self, request, **_):+ self.status = 200+ self.headers, self.url, self.served = {}, request.full_url, 0+ if "/repo/files" in self.url:+ self.body = json.dumps(+ {"Success": True,+ "Data": {"Files": [], "LatestCommitter": {"Revision": TIP_COMMIT}}}).encode("utf-8")+ return+ raise AssertionError("nothing should be downloaded")+++class TypeTests(unittest.TestCase):+ """One argument selects the checkpoint, and it defaults to the multilingual one."""++ SPEC = "convaiinnovations/laya"++ def test_each_type_is_its_own_path_in_the_bundled_repository(self):+ # The Router and the one-shot quickstart load these paths by default, so a type that meant+ # anything else would bake a snapshot no caller resolves.+ for name, subfolder in (("english", ""), ("multilingual", "multilingual"),+ ("typed-decisions", "typed-decisions")):+ specs = prefetch.expand_model(name)+ self.assertEqual(specs, ["%s:%s" % (self.SPEC, subfolder)], name)++ def test_default_and_aliases(self):+ expected = ["convaiinnovations/laya:multilingual"]+ for value in ("multilingual", "multi", "ml", "Multilingual", " multilingual "):+ self.assertEqual(prefetch.expand_model(value), expected, value)++ def test_all_is_the_family(self):+ self.assertEqual(prefetch.expand_model("all"),+ ["convaiinnovations/laya:", "convaiinnovations/laya:multilingual",+ "convaiinnovations/laya:typed-decisions"])+ self.assertEqual(prefetch.expand_model("ALL"), prefetch.expand_model("all"))++ def test_specs_pass_through_and_mix_with_types(self):+ self.assertEqual(prefetch.expand_model("convaiinnovations/laya-multilingual"),+ ["convaiinnovations/laya-multilingual"])+ self.assertEqual(prefetch.expand_model("english,convaiinnovations/laya-typed-decisions"),+ ["convaiinnovations/laya:", "convaiinnovations/laya-typed-decisions"])+ self.assertEqual(prefetch.expand_model("models/custom:typed-decisions models/other"),+ ["models/custom:typed-decisions", "models/other"])++ def test_a_bad_value_is_refused_with_the_valid_names(self):+ for value in ("", "french", "multilingual,french", "ml,spanish"):+ with self.assertRaises(SystemExit) as caught:+ prefetch.expand_model(value)+ self.assertIn("multilingual", str(caught.exception), value)+ self.assertIn("english", str(caught.exception), value)+++class SelectionTests(unittest.TestCase):+ def test_wanted_keeps_exactly_what_a_load_fetches(self):+ keep = ["rl_agent_config.json", "model.safetensors", "tokenizer/tokenizer.json",+ "encoder/config.json"]+ drop = ["README.md", ".gitattributes", "configuration.json", "email_utils.py",+ "rl_agent_api.py"]+ for rel in keep:+ self.assertTrue(prefetch.wanted(rel), rel)+ for rel in drop:+ self.assertFalse(prefetch.wanted(rel), rel)++ def test_specs(self):+ self.assertEqual(prefetch.split_spec("convaiinnovations/laya"), ("convaiinnovations/laya", ""))+ self.assertEqual(prefetch.split_spec("convaiinnovations/laya:multilingual"),+ ("convaiinnovations/laya", "multilingual"))+ self.assertEqual(prefetch.split_spec("convaiinnovations/laya:"),+ ("convaiinnovations/laya", ""))++ def test_cache_root_prefers_the_argument_then_the_hf_variables(self):+ self.assertEqual(prefetch.cache_root("/explicit"), Path("/explicit"))+ with patch.dict(os.environ, {"HF_HUB_CACHE": "/hub-cache", "HF_HOME": "/home"}):+ self.assertEqual(prefetch.cache_root(None), Path("/hub-cache"))+ with patch.dict(os.environ, {"HF_HUB_CACHE": "", "HF_HOME": "/home"}):+ self.assertEqual(prefetch.cache_root(None), Path("/home/hub"))+ self.assertEqual(prefetch.repo_folder("convaiinnovations/laya"),+ "models--convaiinnovations--laya")+++class BakeTests(unittest.TestCase):+ SPEC = "convaiinnovations/laya:multilingual"++ def setUp(self):+ self.cache = Path(tempfile.mkdtemp()) / "hub"+ self.cache.mkdir(parents=True)+ Dropping.dropped = False++ def folder(self):+ """The cache directory this bake writes, derived from the spec it is baking."""+ repo, _ = prefetch.split_spec(self.SPEC)+ return self.cache / ("models--" + repo.replace("/", "--"))++ def bake(self, response=FakeResponse):+ def urlopen(target, **_):+ # `list_repo` passes a URL, `download` passes a Request; the mirror answers both.+ request = target if isinstance(target, urllib.request.Request) \+ else urllib.request.Request(target)+ return response(request)++ with patch.object(prefetch.urllib.request, "urlopen", urlopen):+ prefetch.prefetch(self.SPEC, "master", self.cache)+ return self.folder()++ def published(self, repo_dir):+ """Snapshot directories a loader could resolve, ignoring an in-progress staging one."""+ snapshots = repo_dir / "snapshots"+ if not snapshots.exists():+ return []+ return sorted(p.name for p in snapshots.iterdir() if not p.name.startswith(".staging-"))++ def test_seeded_layout_matches_what_the_hub_writes(self):+ repo_dir = self.bake()+ snapshot = repo_dir / "snapshots" / WEIGHTS_COMMIT+ self.assertTrue(snapshot.is_dir(), "no snapshot for the weights' commit")+ for rel in ("multilingual/rl_agent_config.json", "multilingual/model.safetensors",+ "multilingual/tokenizer/tokenizer.json", "multilingual/encoder/config.json"):+ self.assertTrue(snapshot.joinpath(rel).is_file(), rel)+ # The repo-relative layout, because `Agent` appends the subfolder itself: a snapshot that+ # drops `multilingual/` resolves and then fails with "Subfolder not found".+ self.assertEqual(snapshot.joinpath("multilingual", "model.safetensors").read_bytes(),+ CONTENTS["multilingual/model.safetensors"])+ # The runtime asks for `main`, the mirror publishes `master`: both names resolve.+ self.assertEqual((repo_dir / "refs" / "main").read_text(), WEIGHTS_COMMIT)+ self.assertEqual((repo_dir / "refs" / "master").read_text(), WEIGHTS_COMMIT)+ # Sibling files never enter the snapshot.+ self.assertFalse(snapshot.joinpath("multilingual", "README.md").exists())+ self.assertFalse(snapshot.joinpath("README.md").exists())+ self.assertEqual(self.published(repo_dir), [WEIGHTS_COMMIT])++ def test_a_standalone_repo_lands_at_the_snapshot_root(self):+ """No subfolder: the checkpoint's own files sit at the snapshot root, as on the Hub."""+ self.SPEC = "convaiinnovations/laya-multilingual"+ pages = {+ "": [+ entry("model.safetensors", WEIGHTS_COMMIT),+ entry("rl_agent_config.json", WEIGHTS_COMMIT),+ entry("README.md", TIP_COMMIT),+ tree("tokenizer"),+ tree("encoder"),+ ],+ "tokenizer": [entry("tokenizer/tokenizer.json", WEIGHTS_COMMIT)],+ "encoder": [entry("encoder/config.json", WEIGHTS_COMMIT)],+ }+ with patch.dict(PAGES, pages, clear=True):+ repo_dir = self.bake()+ snapshot = repo_dir / "snapshots" / WEIGHTS_COMMIT+ self.assertTrue(snapshot.joinpath("model.safetensors").is_file())+ self.assertTrue(snapshot.joinpath("rl_agent_config.json").is_file())+ self.assertTrue(snapshot.joinpath("tokenizer", "tokenizer.json").is_file())+ self.assertFalse(snapshot.joinpath("README.md").exists())+ self.assertFalse(snapshot.joinpath("multilingual").exists())++ def test_offline_snapshot_download_resolves_the_baked_checkpoint(self):+ """The load-bearing property, checked against the real hub resolver.++ `local_files_only` is what `HF_HUB_OFFLINE=1` resolves to, and passing it beats patching the+ variable: the hub reads that one at import time, so a later patch would still dial out.+ """+ from huggingface_hub import snapshot_download++ self.bake()+ resolved = snapshot_download(+ "convaiinnovations/laya",+ allow_patterns=["multilingual/rl_agent_config.json", "multilingual/model.safetensors",+ "multilingual/tokenizer/*", "multilingual/encoder/*"],+ cache_dir=str(self.cache), local_files_only=True,+ )+ expected = self.cache / "models--convaiinnovations--laya" / "snapshots" / WEIGHTS_COMMIT+ self.assertEqual(Path(os.path.realpath(resolved)), Path(os.path.realpath(str(expected))))++ def test_the_commit_the_bake_keyed_resolves_as_a_pin(self):+ """`revision=` reaching the weights' own commit must hit the same snapshot."""+ from huggingface_hub import snapshot_download++ self.bake()+ resolved = snapshot_download("convaiinnovations/laya", revision=WEIGHTS_COMMIT,+ cache_dir=str(self.cache), local_files_only=True)+ self.assertEqual(Path(os.path.realpath(resolved)).name, WEIGHTS_COMMIT)++ def test_a_commit_the_bake_did_not_key_misses(self):+ """A commit that was not baked must miss, not silently serve the weights' snapshot."""+ from huggingface_hub import snapshot_download+ from huggingface_hub.errors import LocalEntryNotFoundError++ self.bake()+ with self.assertRaises(LocalEntryNotFoundError):+ snapshot_download("convaiinnovations/laya", revision=TIP_COMMIT,+ cache_dir=str(self.cache), local_files_only=True)++ def test_a_dropped_connection_is_retried_and_resumed(self):+ """A mid-body drop must resume byte-aligned, not restart or duplicate the file."""+ repo_dir = self.bake(Dropping)+ snapshot = repo_dir / "snapshots" / WEIGHTS_COMMIT+ self.assertEqual(snapshot.joinpath("multilingual", "model.safetensors").read_bytes(),+ CONTENTS["multilingual/model.safetensors"])++ def test_truncated_download_refuses_to_publish(self):+ with patch.object(prefetch, "RETRIES", 2), patch.object(prefetch, "BACKOFF", 0):+ with self.assertRaises(SystemExit):+ self.bake(ShortBody)+ self.assertEqual(self.published(self.folder()), [])++ def test_digest_mismatch_refuses_to_publish(self):+ with self.assertRaises(SystemExit):+ self.bake(WrongDigest)+ self.assertEqual(self.published(self.folder()), [])++ def test_empty_selection_is_refused(self):+ with self.assertRaises(SystemExit):+ self.bake(EmptyListing)+ self.assertEqual(self.published(self.folder()), [])++ def test_rebaking_the_same_commit_replaces_the_snapshot(self):+ self.bake()+ repo_dir = self.bake()+ self.assertEqual(self.published(repo_dir), [WEIGHTS_COMMIT])+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_packaging.py b/tests/test_packaging.pyindex 656a2cfa..2f356cc4 100644--- a/tests/test_packaging.py+++ b/tests/test_packaging.py@@ -303,9 +303,49 @@ check_true("compose.cuda/no stale reference to a missing file",# Every file the Docker workflow validates must exist.for name in ("compose.yaml", "compose.example.yml", "compose.cuda.yaml", "compose.http.yaml",- "compose.spark.yaml"):+ "compose.spark.yaml", "compose.modelscope.yaml"):check_true("compose/%s exists" % name, os.path.exists(name))+# The ModelScope bake is a build argument, so a deployment that wants it has to carry the+# arguments into *both* services -- `laya-serve` is its own service and an override for `laya`+# never reaches it, which is the same trap compose.cuda.yaml documents. And the image has to stay+# offline: a baked snapshot is keyed by a ModelScope commit the Hub cannot confirm, so going online+# downloads the same weights again instead of serving the baked copy.+ms = read("compose.modelscope.yaml")+check_true("compose.modelscope/covers laya-serve too",+ re.search(r"^\s{2}laya-serve:", ms, re.M) is not None,+ "compose.modelscope.yaml does not mention laya-serve, so served checkpoints come from "+ "the Hub and a host without Hugging Face access cannot serve at all")+# One argument selects the checkpoint, and it defaults to the multilingual one, the checkpoint+# every Laya caller routes to without being asked.+check("compose.modelscope/repeats the prefetch args for both services",+ len(re.findall(r'MODELSCOPE_MODEL: "\$\{MODELSCOPE_MODEL:-multilingual\}"', ms)), 2)+check("compose.modelscope/repeats the revision for both services",+ len(re.findall(r'MODELSCOPE_REVISION: "\$\{MODELSCOPE_REVISION:-master\}"', ms)), 2)+check("compose.modelscope/keeps every service on the baked cache",+ len(re.findall(r'HF_HUB_OFFLINE: "\$\{HF_HUB_OFFLINE:-1\}"', ms)), 2)+check_true("compose.modelscope/preloads only what can load offline",+ 'LAYA_MODELS: "${LAYA_MODELS:-multilingual}"' in ms,+ "LAYA_PRELOAD=1 with the family-wide default would fail on the first checkpoint that "+ "was not baked")++# The default build must stay exactly what it was: no prefetch, so a plain+# `docker build .` hits the Hub as before, and the RUN is a no-op for the empty argument.+dockerfile_runtime = dockerfile.partition("AS runtime")[2]+check_true("Dockerfile/prefetch args default to off",+ re.search(r'^ARG MODELSCOPE_MODEL=""', dockerfile_runtime, re.M) is not None,+ "a non-empty default would change every existing build")+check_true("Dockerfile/copies the prefetch script",+ "docker/prefetch_modelscope.py" in dockerfile, "the RUN below references a missing file")+check_true("Dockerfile/prefetch step is conditional",+ re.search(r'^\s*RUN if \[ -n "\$MODELSCOPE_MODEL" \]; then', dockerfile_runtime, re.M) is not None,+ "an unconditional RUN would make every build depend on modelscope.cn")+check_true("Dockerfile/prefetch hands the cache to the runtime user",+ "chown -R laya:laya /home/laya/.cache" in dockerfile_runtime,+ "the tokenizer-compatibility fix writes into the snapshot on first load, and the image "+ "runs as UID 10001")+check_true("docker/prefetch_modelscope.py exists", os.path.exists("docker/prefetch_modelscope.py"))+# --------------------------------------------------------------- nix: the deployment layer# Nix builds and evaluates nothing here in CI (`grep -rn nix .github/workflows/` is empty), so