Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(docker): bake checkpoints from ModelScope at build time

gqchen3be99e7707d6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 39843b20..6de66e70 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -138,6 +138,7 @@ jobs:
python tests/test_context_manager.py
python tests/test_criteria_normalization.py
python tests/test_docker_entrypoint.py
+ python tests/test_modelscope_prefetch.py
python tests/test_empty_questions.py
python tests/test_router_memory.py
python tests/test_shortlist_cosine.py
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index e7b814d0..c1a3bfdb 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -59,6 +59,12 @@ jobs:
docker compose -f compose.yaml -f compose.cuda.yaml config --quiet
docker compose -f compose.yaml -f compose.http.yaml config --quiet
docker compose -f compose.yaml -f compose.http.yaml -f compose.cuda.yaml config --quiet
+ # ModelScope prefetch, alone and stacked on top of every other combination a
+ # deployment reaches for.
+ docker compose -f compose.yaml -f compose.modelscope.yaml config --quiet
+ docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml config --quiet
+ docker compose -f compose.yaml -f compose.http.yaml -f compose.cuda.yaml -f compose.modelscope.yaml config --quiet
+ docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml -f compose.modelscope.yaml config --quiet
docker build --pull -t laya:smoke .
- name: Check installed runtime and fresh cache permissions
run: |
diff --git a/Dockerfile b/Dockerfile
index 3806d4b1..a940c1f4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,6 +29,13 @@ RUN pip install ".[serve]" && pip check
FROM ${PYTHON_IMAGE} AS runtime
+# Optional ModelScope prefetch: with `--build-arg MODELSCOPE_MODEL=multilingual` the checkpoint is
+# baked into the hub cache during the build, so the image never depends on huggingface.co. The
+# argument takes a checkpoint type (multilingual, english, typed-decisions, all) or a comma- or
+# space-separated list of `repo[:subfolder]` specs; empty by default leaves the image as it was.
+ARG MODELSCOPE_MODEL=""
+ARG MODELSCOPE_REVISION="master"
+
LABEL org.opencontainers.image.title="Laya Docker quickstart" \
org.opencontainers.image.source="https://github.com/NandhaKishorM/laya" \
org.opencontainers.image.licenses="Apache-2.0"
@@ -57,6 +64,21 @@ COPY --from=build /opt/venv /opt/venv
COPY LICENSE /usr/share/doc/laya/LICENSE
COPY examples/docker/ /opt/laya/examples/
COPY docker/entrypoint.py /opt/laya/entrypoint.py
+COPY docker/prefetch_modelscope.py /opt/laya/prefetch_modelscope.py
+
+# Bake the requested ModelScope checkpoints into the hub cache ($HF_HOME/hub), laid out the way
+# `snapshot_download` reads them offline, so no entry point changes: the quickstart's Router, the
+# HTTP server's Router and `laya.cli` all resolve their repo ids to the baked snapshot. The cache is
+# handed to the runtime user afterwards, because the tokenizer-compatibility fix writes into the
+# snapshot on first load.
+RUN if [ -n "$MODELSCOPE_MODEL" ]; then \
+ python /opt/laya/prefetch_modelscope.py \
+ --model "$MODELSCOPE_MODEL" \
+ --revision "$MODELSCOPE_REVISION" \
+ --cache-dir "$HF_HOME/hub" \
+ && chown -R laya:laya /home/laya/.cache; \
+ fi
+
USER laya
WORKDIR /home/laya
diff --git a/compose.modelscope.yaml b/compose.modelscope.yaml
new file mode 100644
index 00000000..aa79331b
--- /dev/null
+++ b/compose.modelscope.yaml
@@ -0,0 +1,43 @@
+# Bake the checkpoint from ModelScope (https://modelscope.cn) into the image, so neither the build
+# nor the running container has to reach huggingface.co:
+#
+# docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve
+# curl -s localhost:8000/health
+# curl -s localhost:8000/v1/systemone -H 'content-type: application/json' \
+# --data @examples/docker/request.json
+#
+# The weights land in the hub cache the image already mounts as the model-cache volume, laid out the
+# way `snapshot_download` reads them offline, so the quickstart, the HTTP server's Router and the CLI
+# keep their repo ids and resolve them to the baked snapshot: docker/prefetch_modelscope.py does it
+# during the build and stops it on any size or digest mismatch.
+#
+# For NVIDIA, append -f compose.cuda.yaml; it repeats the build args and the device reservation for
+# both services, because an override for `laya` never reaches `laya-serve`.
+#
+# MODELSCOPE_MODEL takes a checkpoint type -- multilingual (the default), english, typed-decisions,
+# all -- or comma- or space-separated `repo[:subfolder]` specs for a standalone repository or a
+# fine-tuned checkpoint. A type maps to that checkpoint's path in the bundled repository, which is
+# what the Router and the quickstart load by default, so a single-argument build serves it with no
+# further change. The default bakes only the multilingual checkpoint (~650 MB); see docs/docker.md
+# for the other two and for what a baked-only image can still load.
+services:
+ laya:
+ build:
+ args:
+ MODELSCOPE_MODEL: "${MODELSCOPE_MODEL:-multilingual}"
+ MODELSCOPE_REVISION: "${MODELSCOPE_REVISION:-master}"
+ environment:
+ # The baked snapshot is keyed by a ModelScope commit, which the Hub cannot confirm; going
+ # online would download the same weights again instead of using the baked copy.
+ HF_HUB_OFFLINE: "${HF_HUB_OFFLINE:-1}"
+ laya-serve:
+ build:
+ args:
+ MODELSCOPE_MODEL: "${MODELSCOPE_MODEL:-multilingual}"
+ MODELSCOPE_REVISION: "${MODELSCOPE_REVISION:-master}"
+ environment:
+ HF_HUB_OFFLINE: "${HF_HUB_OFFLINE:-1}"
+ # Restrains preload to the checkpoint that was baked. Routing itself still selects the
+ # checkpoint that fits the text, so an English-serving deployment bakes that one too and
+ # names it here: `LAYA_PRELOAD=1` with a list an image cannot serve fails on the first one.
+ LAYA_MODELS: "${LAYA_MODELS:-multilingual}"
diff --git a/docker/prefetch_modelscope.py b/docker/prefetch_modelscope.py
new file mode 100644
index 00000000..5f42a475
--- /dev/null
+++ b/docker/prefetch_modelscope.py
@@ -0,0 +1,292 @@
+"""Bake Laya checkpoints into a Hugging Face cache from ModelScope, during an image build.
+
+The runtime resolves a repo id through `huggingface_hub.snapshot_download`, which reads its
+local cache. With `HF_HUB_OFFLINE=1` -- or a network that cannot reach huggingface.co -- that
+resolution is purely local: `refs/<revision>` names a commit and an existing
+`snapshots/<sha>` directory is returned as it is, with no repo listing at all
+(`huggingface_hub/_snapshot_download.py`). A mirror therefore only has to lay the files out the
+way the Hub would have, and every code path that names a checkpoint keeps its ids and its
+revision handling: `Agent(...)`, the `Router` laya-serve builds, `laya.cli`, the integrations.
+
+Only the standard library is used, because an image build has no reason to install the
+ModelScope SDK for a handful of HTTP calls. Every file is streamed with resume and its size and SHA-256
+are checked against the values the repository API reports, so a truncated or substituted
+download fails the build instead of shipping inside a layer that looks healthy.
+
+Usage inside a Dockerfile RUN:
+
+ python prefetch_modelscope.py --models convaiinnovations/laya:multilingual \
+ --revision master --cache-dir "$HF_HOME/hub"
+
+Each spec is `repo[:subfolder]`; the subfolder selects one checkpoint from the bundled
+repository, the same way `Agent("convaiinnovations/laya", subfolder="multilingual")` does.
+"""
+import argparse
+import hashlib
+import http.client
+import json
+import os
+import re
+import shutil
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+FILES_API = "https://modelscope.cn/api/v1/models/{repo}/repo/files"
+FILE_API = "https://modelscope.cn/api/v1/models/{repo}/repo"
+# What a Laya checkpoint needs: the same set laya/agent.py hands snapshot_download, so a baked
+# snapshot carries exactly the files a load would have fetched and none of its siblings.
+NEEDED = ("rl_agent_config.json", "model.safetensors")
+FOLDERS = ("tokenizer/", "encoder/")
+CHUNK = 1 << 20
+RETRIES = 4
+BACKOFF = 2.0
+TIMEOUT = 60
+# The revision the runtime asks for by default. The mirror publishes `master`, the Hub publishes
+# `main`, and a baked snapshot answers both: this is the name `refs/` is written under.
+DEFAULT_BRANCH = "main"
+
+# A checkpoint type and the path it means inside the bundled repository. These are the paths the
+# `Router` and the one-shot quickstart load by default (laya/router.py: DEFAULT_MODELS), which is
+# why a type bakes them rather than the standalone repositories of the same checkpoints: a baked
+# image then serves the default configuration with no change at all.
+TYPES = {
+ "english": ["convaiinnovations/laya:"],
+ "multilingual": ["convaiinnovations/laya:multilingual"],
+ "typed-decisions": ["convaiinnovations/laya:typed-decisions"],
+}
+# The spellings laya/router.py accepts for the same three checkpoints, so an operator who copies a
+# name out of that mapping is not told it is unknown here.
+ALIASES = {"en": "english", "laya": "english", "default": "english",
+ "multi": "multilingual", "ml": "multilingual",
+ "typed": "typed-decisions", "decisions": "typed-decisions"}
+
+
+def expand_model(value: str):
+ """A type name, or one or more `repo[:subfolder]` specs, into the specs to bake.
+
+ `all` is the whole family. Anything holding a `/` is taken as a spec, which is how a standalone
+ repository (`convaiinnovations/laya-multilingual`) or a fine-tuned checkpoint is baked; specs
+ and types can be mixed in one comma- or space-separated value.
+ """
+ parts = [part for part in re.split(r"[,\s]+", (value or "").strip()) if part]
+ if not parts:
+ raise SystemExit("laya: --model is empty; expected a checkpoint type (%s) or a "
+ "repo[:subfolder] spec" % ", ".join(sorted(TYPES) + ["all"]))
+ specs = []
+ for part in parts:
+ if "/" in part:
+ specs.append(part)
+ continue
+ name = ALIASES.get(part.lower(), part.lower())
+ if name == "all":
+ specs += [spec for key in sorted(TYPES) for spec in TYPES[key]]
+ elif name in TYPES:
+ specs += TYPES[name]
+ else:
+ raise SystemExit("laya: unknown checkpoint type %r; choose one of %s, all, or a "
+ "repo[:subfolder] spec" % (part, sorted(TYPES)))
+ return specs
+
+
+def wanted(rel: str) -> bool:
+ """Whether a path inside the subfolder is one of the files a checkpoint load fetches."""
+ return rel in NEEDED or rel.startswith(FOLDERS)
+
+
+def split_spec(spec: str):
+ """`repo[:subfolder]` -> (repo, subfolder); `:` with nothing after it means the root."""
+ repo, sep, sub = spec.partition(":")
+ return repo.strip(), sub.strip() if sep else ""
+
+
+def cache_root(override):
+ """The hub cache directory: an explicit path, `HF_HUB_CACHE`, or `$HF_HOME/hub`."""
+ if override:
+ return Path(override)
+ explicit = os.environ.get("HF_HUB_CACHE")
+ if explicit:
+ return Path(explicit)
+ home = os.environ.get("HF_HOME") or os.path.join(os.path.expanduser("~"), ".cache", "huggingface")
+ return Path(home) / "hub"
+
+
+def repo_folder(repo: str) -> str:
+ """The cache directory name the Hub derives from a repo id: `ns/name` -> `models--ns--name`."""
+ return "models--" + repo.replace("/", "--")
+
+
+def list_repo(repo: str, revision: str, subfolder: str):
+ """(tip commit, entries) for the revision's own files inside `subfolder`.
+
+ Each entry reports the path as the API names it, the path relative to the checkpoint -- what
+ the Hub caches -- its size, and its SHA-256 when the API publishes one. The listing API returns
+ one directory at a time and names subdirectories as `tree` entries, so the walk follows them:
+ the bundled repository keeps each checkpoint's tokenizer and encoder in their own directories.
+ """
+ entries, tip, queue = [], "", [subfolder]
+ prefix = subfolder + "/" if subfolder else ""
+ while queue:
+ query = urllib.parse.urlencode({"Revision": revision, "Root": queue.pop(0)})
+ try:
+ with urllib.request.urlopen(FILES_API.format(repo=repo) + "?" + query,
+ timeout=TIMEOUT) as r:
+ payload = json.loads(r.read().decode("utf-8"))
+ except (urllib.error.URLError, urllib.error.HTTPError, ValueError, OSError) as exc:
+ raise SystemExit("laya: cannot list %s at revision %r: %s" % (repo, revision, exc))
+ if not payload.get("Success"):
+ raise SystemExit("laya: ModelScope returned %r for %s -- check the repo id and revision"
+ % (payload.get("Message"), repo))
+ data = payload.get("Data") or {}
+ tip = tip or ((data.get("LatestCommitter") or {}).get("Revision") or "").strip()
+ for item in data.get("Files") or []:
+ path, kind = item.get("Path", ""), item.get("Type")
+ if kind == "tree":
+ if path.startswith(prefix):
+ queue.append(path)
+ continue
+ if kind != "blob" or not path.startswith(prefix):
+ continue
+ rel = path[len(prefix):]
+ if not wanted(rel):
+ continue
+ entries.append({
+ "path": path,
+ "rel": rel,
+ "size": int(item.get("Size") or 0),
+ "sha256": (item.get("Sha256") or "").strip(),
+ "revision": (item.get("Revision") or "").strip(),
+ })
+ return tip, entries
+
+
+def snapshot_commit(tip: str, entries: list) -> str:
+ """The commit to key the baked snapshot by: the weights' own, else the revision's tip.
+
+ The weights' commit is the one a caller would pin to get *these* bytes back, so it is the
+ more useful key. A mixed-commit fetch still lands as one snapshot directory, because the
+ offline path never compares the file list against a commit -- but that is a warning, not a
+ failure: a maintenance commit to a README must not break a build.
+ """
+ commits = {entry["revision"] for entry in entries if entry["revision"]}
+ weights = [entry["revision"] for entry in entries
+ if entry["rel"] == "model.safetensors" and entry["revision"]]
+ if len(commits) > 1:
+ print(" note: files span commits %s; keying the snapshot by the weights' commit"
+ % sorted(commits), file=sys.stderr)
+ return (weights[0] if weights else "") or tip
+
+
+def download(repo: str, revision: str, entry: dict, dest: Path) -> None:
+ """Stream one file into `dest`, resuming a partial copy and verifying what landed."""
+ query = urllib.parse.urlencode({"FilePath": entry["path"], "Revision": revision})
+ url = FILE_API.format(repo=repo) + "?" + query
+ for attempt in range(1, RETRIES + 1):
+ have = dest.stat().st_size if dest.exists() else 0
+ headers = {"Range": "bytes=%d-" % have} if have else {}
+ try:
+ with urllib.request.urlopen(urllib.request.Request(url, headers=headers),
+ timeout=TIMEOUT) as response:
+ # A 206 answers the Range request; anything else starts the file over, because
+ # appending a full body to a partial one would corrupt it.
+ mode = "ab" if response.status == 206 and have else "wb"
+ with open(dest, mode) as fh:
+ while True:
+ chunk = response.read(CHUNK)
+ if not chunk:
+ break
+ fh.write(chunk)
+ except (urllib.error.URLError, urllib.error.HTTPError, OSError,
+ http.client.HTTPException) as exc:
+ print(" retry %d/%d after %s: %s" % (attempt, RETRIES, type(exc).__name__, exc),
+ file=sys.stderr)
+ # Written outside the `try`, so a response that closed early still re-asks for the rest
+ # instead of being treated as a complete file.
+ on_disk = dest.stat().st_size if dest.exists() else 0
+ if entry["size"] and on_disk == entry["size"]:
+ break
+ if attempt == RETRIES:
+ raise SystemExit("laya: %s: %d of %d bytes after %d attempts -- check the connection "
+ "to modelscope.cn" % (entry["path"], on_disk, entry["size"], RETRIES))
+ time.sleep(BACKOFF * attempt)
+
+ digest = hashlib.sha256()
+ with open(dest, "rb") as fh:
+ while True:
+ chunk = fh.read(CHUNK)
+ if not chunk:
+ break
+ digest.update(chunk)
+ if entry["sha256"] and digest.hexdigest() != entry["sha256"]:
+ raise SystemExit("laya: %s: sha256 mismatch -- the repository reports %s, the download "
+ "has %s" % (entry["path"], entry["sha256"], digest.hexdigest()))
+
+
+def seed(cache: Path, repo: str, commit: str, revision: str, staging: Path) -> None:
+ """Publish a verified download as a hub snapshot, then name its commit in `refs/`."""
+ repo_dir = cache / repo_folder(repo)
+ snapshot = repo_dir / "snapshots" / commit
+ refs = repo_dir / "refs"
+ refs.mkdir(parents=True, exist_ok=True)
+ shutil.rmtree(snapshot, ignore_errors=True)
+ # Renamed into place only once it holds every file it claims to, and the ref is written only
+ # after the rename: an offline snapshot_download returns whatever `snapshots/<sha>` already
+ # contains, so a half-published snapshot would otherwise be served as if complete.
+ os.replace(str(staging), str(snapshot))
+ for name in (DEFAULT_BRANCH, revision):
+ if name and len(name) < 64:
+ (refs / name).write_text(commit, encoding="utf-8")
+
+
+def prefetch(spec: str, revision: str, cache: Path) -> None:
+ repo, subfolder = split_spec(spec)
+ tip, entries = list_repo(repo, revision, subfolder)
+ if not entries:
+ raise SystemExit("laya: %s%s holds no checkpoint files at revision %r"
+ % (repo, "/" + subfolder if subfolder else "", revision))
+ commit = snapshot_commit(tip, entries)
+ if not commit:
+ raise SystemExit("laya: %s%s names no commit at revision %r, so there is nothing to key "
+ "the baked snapshot by" % (repo, "/" + subfolder if subfolder else "",
+ revision))
+ repo_dir = cache / repo_folder(repo)
+ staging = repo_dir / "snapshots" / (".staging-" + str(os.getpid()))
+ try:
+ for entry in entries:
+ # The repository-relative path, not the checkpoint-relative one: `Agent` appends the
+ # subfolder itself (`model_dir/<subfolder>`), so the snapshot has to mirror the repo.
+ dest = staging / entry["path"]
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ print(" %s: %d bytes" % (entry["path"], entry["size"]))
+ download(repo, revision, entry, dest)
+ seed(cache, repo, commit, revision, staging)
+ finally:
+ shutil.rmtree(staging, ignore_errors=True)
+ print(" baked as models--%s/snapshots/%s" % (repo.replace("/", "--"), commit))
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Bake ModelScope checkpoints into a hub cache")
+ parser.add_argument("--model", required=True, metavar="TYPE_OR_SPEC",
+ help="checkpoint type (multilingual, english, typed-decisions, all) or one "
+ "or more comma- or space-separated repo[:subfolder] specs")
+ parser.add_argument("--revision", default="master",
+ help="ModelScope revision (branch, tag or commit); default master")
+ parser.add_argument("--cache-dir", default=None,
+ help="hub cache directory; defaults to HF_HUB_CACHE or $HF_HOME/hub")
+ args = parser.parse_args()
+ specs = expand_model(args.model)
+ cache = cache_root(args.cache_dir)
+ cache.mkdir(parents=True, exist_ok=True)
+ for spec in specs:
+ print("modelscope/%s at revision %s" % (spec, args.revision))
+ prefetch(spec, args.revision, cache)
+ print("baked %d checkpoint(s) into %s" % (len(specs), cache))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/docs/docker.md b/docs/docker.md
index 41eec446..1c9dbb31 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -161,6 +161,78 @@ are mutually exclusive. The local-path response comes from the Agent and has no
Router `routing` metadata. These settings also work with the CUDA override.
Evaluate fine-tuned checkpoints on held-out examples before relying on them.
+## Models from ModelScope
+
+When a host cannot reach huggingface.co, the checkpoint can come from
+[ModelScope](https://modelscope.cn) and be baked into the image at build time. One
+argument selects which checkpoint, and it defaults to the multilingual one. The
+Compose override adds the prefetch arguments to both services and keeps the
+container off the Hub:
+
+```bash
+docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve
+```
+
+For NVIDIA, add `-f compose.cuda.yaml` before `up`; it repeats its own arguments for
+both services, so the ordering between the two overrides does not matter. Plain
+Docker takes the arguments directly:
+
+```bash
+docker build --build-arg MODELSCOPE_MODEL=multilingual \
+ -t laya:local .
+docker run --rm -e HF_HUB_OFFLINE=1 -p 127.0.0.1:8000:8000 laya:local laya-serve
+```
+
+[`docker/prefetch_modelscope.py`](https://github.com/NandhaKishorM/laya/blob/main/docker/prefetch_modelscope.py)
+lists the repository on modelscope.cn, downloads the checkpoint's own files -- the
+same set `laya/agent.py` asks the Hub for, so no sibling checkpoint is pulled -- and
+writes them into the image's hub cache the way `snapshot_download` lays out a
+snapshot. Nothing else changes: `Agent`, the `Router` that `laya-serve` builds,
+`laya.cli` and the integrations keep their repo ids and resolve them to the baked
+snapshot, so a container built this way needs no network at all. Each file's size
+and SHA-256 are checked against what the repository reports before the snapshot is
+published, so a truncated or substituted download fails the build instead of
+shipping inside a layer that looks healthy.
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `MODELSCOPE_MODEL` | `multilingual` (Compose); empty in the Dockerfile | Checkpoint to bake: `multilingual`, `english`, `typed-decisions` or `all`. Empty means no prefetch and the unchanged image |
+| `MODELSCOPE_REVISION` | `master` | ModelScope branch, tag or commit to bake |
+| `HF_HUB_OFFLINE` | `1` in the Compose override | `1` never contacts the Hub, so the baked copy is served |
+
+A type expands to that checkpoint's path inside the bundled repository, which is what
+the `Router` and the one-shot quickstart load by default, so a build that names one
+type serves it with no further change:
+
+```bash
+MODELSCOPE_MODEL=english docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve
+MODELSCOPE_MODEL=all docker compose -f compose.yaml -f compose.http.yaml -f compose.modelscope.yaml up --build laya-serve
+```
+
+`all` is the whole family, about 2.4 GB of weights. The override also sets
+`LAYA_MODELS=multilingual`, because `LAYA_PRELOAD=1` with the default list would try
+to build every checkpoint and fail on the first one that was not baked; set
+`LAYA_MODELS` to the list you baked when you bake more, and `MODELSCOPE_MODEL=all`
+when a deployment really does serve the family.
+
+Beyond the four types the argument also takes `repo[:subfolder]` specs, comma- or
+space-separated, which is how the mirror's standalone repositories
+([laya](https://modelscope.cn/models/convaiinnovations/laya),
+[laya-multilingual](https://modelscope.cn/models/convaiinnovations/laya-multilingual),
+[laya-typed-decisions](https://modelscope.cn/models/convaiinnovations/laya-typed-decisions))
+or a fine-tuned checkpoint is baked. A standalone repository is what
+`Agent("convaiinnovations/laya-multilingual")` loads directly; the `Router`'s default
+is the bundled path, so a type is normally what a serving image wants.
+
+Two details about pins and provenance. The build prints the commit the snapshot is
+keyed by -- pass that SHA as `revision=` or `LAYA_REVISION` to pin a load to exactly
+what was baked. Hub-side pins do not describe a mirror snapshot: `reviewed` names
+Hugging Face commits, and the SHA-256 digest map is keyed to Hub artifact hashes, so
+neither applies here. The build already refuses a download that does not match the
+mirror's own reported size and digest, which is the equivalent guarantee for this
+path. And the weights come from whichever mirror account the argument names, which is
+its own supply-chain decision for the deployment to make.
+
## Development and cleanup
Open a Python prompt with `docker compose run --rm laya python`. To run the
diff --git a/tests/test_modelscope_prefetch.py b/tests/test_modelscope_prefetch.py
new file mode 100644
index 00000000..143b5f49
--- /dev/null
+++ b/tests/test_modelscope_prefetch.py
@@ -0,0 +1,384 @@
+"""Bake-from-ModelScope has to produce a cache the offline loader accepts -- exactly that.
+
+`docker/prefetch_modelscope.py` writes a Hugging Face snapshot cache during an image build so that
+a host that cannot reach huggingface.co can still ship and serve a checkpoint. The whole design
+rests on one property, and it is not visible in the script: with `HF_HUB_OFFLINE=1`,
+`snapshot_download` returns `<cache>/snapshots/<sha>` whenever `refs/<revision>` names a directory
+that already exists, without listing the repository (`huggingface_hub/_snapshot_download.py`).
+These tests therefore bake from a stubbed ModelScope API and then ask the real `snapshot_download`
+what it resolves, so a layout that merely looks right cannot pass.
+
+The second property worth pinning is the failure direction. A snapshot directory is *served*, not
+validated, so anything half-written would be loaded as weights: a download that fails its size or
+digest check must leave nothing behind for the loader to find.
+
+No weights, no network, no docker: the file listing and the file bodies are fixtures.
+"""
+import hashlib
+import http.client
+import importlib.util
+import json
+import os
+import tempfile
+import unittest
+import urllib.parse
+from pathlib import Path
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "docker" / "prefetch_modelscope.py"
+spec = importlib.util.spec_from_file_location("prefetch_modelscope", SCRIPT)
+prefetch = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(prefetch)
+
+WEIGHTS_COMMIT = "a" * 40
+TIP_COMMIT = "b" * 40
+CONTENTS = {
+ "multilingual/rl_agent_config.json": b'{"encoder": "jhu-clsp/mmBERT-base"}',
+ "multilingual/model.safetensors": b"WEIGHTS" * 64,
+ "multilingual/tokenizer/tokenizer.json": b'{"model": {}}',
+ "multilingual/encoder/config.json": b'{"hidden_size": 768}',
+ "multilingual/README.md": b"not a checkpoint file",
+ "README.md": b"outside the subfolder",
+ # A standalone repository keeps the same checkpoint at its root.
+ "model.safetensors": b"STANDALONE" * 8,
+ "rl_agent_config.json": b"standalone config",
+ "tokenizer/tokenizer.json": b"standalone tokenizer",
+ "encoder/config.json": b"standalone encoder",
+}
+
+
+def entry(path, commit):
+ body = CONTENTS[path]
+ return {"Type": "blob", "Path": path, "Size": len(body),
+ "Sha256": hashlib.sha256(body).hexdigest(), "Revision": commit}
+
+
+def tree(path):
+ return {"Type": "tree", "Path": path, "Size": 0, "Revision": WEIGHTS_COMMIT}
+
+
+# The listing API answers one directory at a time and names subdirectories as `tree` entries, the
+# way the mirror does: the checkpoint's tokenizer and encoder are pages of their own, and a sibling
+# checkpoint directory is a page this bake must not walk into.
+PAGES = {
+ "multilingual": [
+ entry("multilingual/model.safetensors", WEIGHTS_COMMIT),
+ entry("multilingual/rl_agent_config.json", WEIGHTS_COMMIT),
+ entry("multilingual/README.md", WEIGHTS_COMMIT),
+ entry("README.md", TIP_COMMIT),
+ tree("multilingual/tokenizer"),
+ tree("multilingual/encoder"),
+ tree("typed-decisions"),
+ ],
+ "multilingual/tokenizer": [entry("multilingual/tokenizer/tokenizer.json", WEIGHTS_COMMIT)],
+ "multilingual/encoder": [entry("multilingual/encoder/config.json", WEIGHTS_COMMIT)],
+}
+
+
+def page(root):
+ listing = {"Success": True,
+ "Data": {"Files": PAGES[root], "LatestCommitter": {"Revision": TIP_COMMIT}}}
+ return json.dumps(listing).encode("utf-8")
+
+
+def params(url):
+ # keep_blank_values: the root listing carries `Root=`, which parse_qsl otherwise drops.
+ pairs = urllib.parse.parse_qsl(urllib.parse.urlparse(url).query, keep_blank_values=True)
+ return dict(pairs)
+
+
+class FakeResponse:
+ """Serves the listing API, the file API and HTTP ranges, the way the mirror's CDN does."""
+
+ status = 200
+
+ def __init__(self, request, **_):
+ self.headers = {key.lower(): value for key, value in request.headers.items()}
+ self.url = request.full_url
+ self.served = 0
+ if "/repo/files" in self.url:
+ self.body = page(params(self.url)["Root"])
+ return
+ self.body = CONTENTS[params(self.url)["FilePath"]]
+ start = 0
+ header = self.headers.get("range", "")
+ if header.startswith("bytes="):
+ start = int(header[len("bytes="):].split("-")[0])
+ if start:
+ self.body = self.body[start:]
+ self.status = 206
+
+ def read(self, size=-1):
+ size = size if size and size > 0 else len(self.body)
+ chunk = self.body[self.served:self.served + size]
+ self.served += len(chunk)
+ return chunk
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_):
+ return False
+
+
+class Dropping(FakeResponse):
+ """Hands over the first 32 bytes of the weights file, then drops the connection mid-body."""
+
+ dropped = False
+
+ def read(self, size=-1):
+ if "FilePath" not in self.url:
+ return FakeResponse.read(self, size)
+ if not params(self.url)["FilePath"].endswith("model.safetensors"):
+ return FakeResponse.read(self, size)
+ if self.served < 32:
+ self.served = 32
+ return self.body[:32]
+ if not Dropping.dropped and "range" not in self.headers:
+ Dropping.dropped = True
+ raise http.client.IncompleteRead(b"dropped mid-body")
+ return FakeResponse.read(self, size)
+
+
+class ShortBody(FakeResponse):
+ """Every body is one byte short of the size the listing promises."""
+
+ def __init__(self, request, **_):
+ FakeResponse.__init__(self, request, **_)
+ if "/repo/files" not in self.url:
+ self.body = self.body[:-1]
+
+
+class WrongDigest(FakeResponse):
+ """Every body hashes to something the listing does not report."""
+
+ def __init__(self, request, **_):
+ FakeResponse.__init__(self, request, **_)
+ if "/repo/files" not in self.url:
+ self.body = b"X" + self.body[1:]
+
+
+class EmptyListing(FakeResponse):
+ """A repository with no checkpoint files at all."""
+
+ def __init__(self, request, **_):
+ self.status = 200
+ self.headers, self.url, self.served = {}, request.full_url, 0
+ if "/repo/files" in self.url:
+ self.body = json.dumps(
+ {"Success": True,
+ "Data": {"Files": [], "LatestCommitter": {"Revision": TIP_COMMIT}}}).encode("utf-8")
+ return
+ raise AssertionError("nothing should be downloaded")
+
+
+class TypeTests(unittest.TestCase):
+ """One argument selects the checkpoint, and it defaults to the multilingual one."""
+
+ SPEC = "convaiinnovations/laya"
+
+ def test_each_type_is_its_own_path_in_the_bundled_repository(self):
+ # The Router and the one-shot quickstart load these paths by default, so a type that meant
+ # anything else would bake a snapshot no caller resolves.
+ for name, subfolder in (("english", ""), ("multilingual", "multilingual"),
+ ("typed-decisions", "typed-decisions")):
+ specs = prefetch.expand_model(name)
+ self.assertEqual(specs, ["%s:%s" % (self.SPEC, subfolder)], name)
+
+ def test_default_and_aliases(self):
+ expected = ["convaiinnovations/laya:multilingual"]
+ for value in ("multilingual", "multi", "ml", "Multilingual", " multilingual "):
+ self.assertEqual(prefetch.expand_model(value), expected, value)
+
+ def test_all_is_the_family(self):
+ self.assertEqual(prefetch.expand_model("all"),
+ ["convaiinnovations/laya:", "convaiinnovations/laya:multilingual",
+ "convaiinnovations/laya:typed-decisions"])
+ self.assertEqual(prefetch.expand_model("ALL"), prefetch.expand_model("all"))
+
+ def test_specs_pass_through_and_mix_with_types(self):
+ self.assertEqual(prefetch.expand_model("convaiinnovations/laya-multilingual"),
+ ["convaiinnovations/laya-multilingual"])
+ self.assertEqual(prefetch.expand_model("english,convaiinnovations/laya-typed-decisions"),
+ ["convaiinnovations/laya:", "convaiinnovations/laya-typed-decisions"])
+ self.assertEqual(prefetch.expand_model("models/custom:typed-decisions models/other"),
+ ["models/custom:typed-decisions", "models/other"])
+
+ def test_a_bad_value_is_refused_with_the_valid_names(self):
+ for value in ("", "french", "multilingual,french", "ml,spanish"):
+ with self.assertRaises(SystemExit) as caught:
+ prefetch.expand_model(value)
+ self.assertIn("multilingual", str(caught.exception), value)
+ self.assertIn("english", str(caught.exception), value)
+
+
+class SelectionTests(unittest.TestCase):
+ def test_wanted_keeps_exactly_what_a_load_fetches(self):
+ keep = ["rl_agent_config.json", "model.safetensors", "tokenizer/tokenizer.json",
+ "encoder/config.json"]
+ drop = ["README.md", ".gitattributes", "configuration.json", "email_utils.py",
+ "rl_agent_api.py"]
+ for rel in keep:
+ self.assertTrue(prefetch.wanted(rel), rel)
+ for rel in drop:
+ self.assertFalse(prefetch.wanted(rel), rel)
+
+ def test_specs(self):
+ self.assertEqual(prefetch.split_spec("convaiinnovations/laya"), ("convaiinnovations/laya", ""))
+ self.assertEqual(prefetch.split_spec("convaiinnovations/laya:multilingual"),
+ ("convaiinnovations/laya", "multilingual"))
+ self.assertEqual(prefetch.split_spec("convaiinnovations/laya:"),
+ ("convaiinnovations/laya", ""))
+
+ def test_cache_root_prefers_the_argument_then_the_hf_variables(self):
+ self.assertEqual(prefetch.cache_root("/explicit"), Path("/explicit"))
+ with patch.dict(os.environ, {"HF_HUB_CACHE": "/hub-cache", "HF_HOME": "/home"}):
+ self.assertEqual(prefetch.cache_root(None), Path("/hub-cache"))
+ with patch.dict(os.environ, {"HF_HUB_CACHE": "", "HF_HOME": "/home"}):
+ self.assertEqual(prefetch.cache_root(None), Path("/home/hub"))
+ self.assertEqual(prefetch.repo_folder("convaiinnovations/laya"),
+ "models--convaiinnovations--laya")
+
+
+class BakeTests(unittest.TestCase):
+ SPEC = "convaiinnovations/laya:multilingual"
+
+ def setUp(self):
+ self.cache = Path(tempfile.mkdtemp()) / "hub"
+ self.cache.mkdir(parents=True)
+ Dropping.dropped = False
+
+ def folder(self):
+ """The cache directory this bake writes, derived from the spec it is baking."""
+ repo, _ = prefetch.split_spec(self.SPEC)
+ return self.cache / ("models--" + repo.replace("/", "--"))
+
+ def bake(self, response=FakeResponse):
+ def urlopen(target, **_):
+ # `list_repo` passes a URL, `download` passes a Request; the mirror answers both.
+ request = target if isinstance(target, urllib.request.Request) \
+ else urllib.request.Request(target)
+ return response(request)
+
+ with patch.object(prefetch.urllib.request, "urlopen", urlopen):
+ prefetch.prefetch(self.SPEC, "master", self.cache)
+ return self.folder()
+
+ def published(self, repo_dir):
+ """Snapshot directories a loader could resolve, ignoring an in-progress staging one."""
+ snapshots = repo_dir / "snapshots"
+ if not snapshots.exists():
+ return []
+ return sorted(p.name for p in snapshots.iterdir() if not p.name.startswith(".staging-"))
+
+ def test_seeded_layout_matches_what_the_hub_writes(self):
+ repo_dir = self.bake()
+ snapshot = repo_dir / "snapshots" / WEIGHTS_COMMIT
+ self.assertTrue(snapshot.is_dir(), "no snapshot for the weights' commit")
+ for rel in ("multilingual/rl_agent_config.json", "multilingual/model.safetensors",
+ "multilingual/tokenizer/tokenizer.json", "multilingual/encoder/config.json"):
+ self.assertTrue(snapshot.joinpath(rel).is_file(), rel)
+ # The repo-relative layout, because `Agent` appends the subfolder itself: a snapshot that
+ # drops `multilingual/` resolves and then fails with "Subfolder not found".
+ self.assertEqual(snapshot.joinpath("multilingual", "model.safetensors").read_bytes(),
+ CONTENTS["multilingual/model.safetensors"])
+ # The runtime asks for `main`, the mirror publishes `master`: both names resolve.
+ self.assertEqual((repo_dir / "refs" / "main").read_text(), WEIGHTS_COMMIT)
+ self.assertEqual((repo_dir / "refs" / "master").read_text(), WEIGHTS_COMMIT)
+ # Sibling files never enter the snapshot.
+ self.assertFalse(snapshot.joinpath("multilingual", "README.md").exists())
+ self.assertFalse(snapshot.joinpath("README.md").exists())
+ self.assertEqual(self.published(repo_dir), [WEIGHTS_COMMIT])
+
+ def test_a_standalone_repo_lands_at_the_snapshot_root(self):
+ """No subfolder: the checkpoint's own files sit at the snapshot root, as on the Hub."""
+ self.SPEC = "convaiinnovations/laya-multilingual"
+ pages = {
+ "": [
+ entry("model.safetensors", WEIGHTS_COMMIT),
+ entry("rl_agent_config.json", WEIGHTS_COMMIT),
+ entry("README.md", TIP_COMMIT),
+ tree("tokenizer"),
+ tree("encoder"),
+ ],
+ "tokenizer": [entry("tokenizer/tokenizer.json", WEIGHTS_COMMIT)],
+ "encoder": [entry("encoder/config.json", WEIGHTS_COMMIT)],
+ }
+ with patch.dict(PAGES, pages, clear=True):
+ repo_dir = self.bake()
+ snapshot = repo_dir / "snapshots" / WEIGHTS_COMMIT
+ self.assertTrue(snapshot.joinpath("model.safetensors").is_file())
+ self.assertTrue(snapshot.joinpath("rl_agent_config.json").is_file())
+ self.assertTrue(snapshot.joinpath("tokenizer", "tokenizer.json").is_file())
+ self.assertFalse(snapshot.joinpath("README.md").exists())
+ self.assertFalse(snapshot.joinpath("multilingual").exists())
+
+ def test_offline_snapshot_download_resolves_the_baked_checkpoint(self):
+ """The load-bearing property, checked against the real hub resolver.
+
+ `local_files_only` is what `HF_HUB_OFFLINE=1` resolves to, and passing it beats patching the
+ variable: the hub reads that one at import time, so a later patch would still dial out.
+ """
+ from huggingface_hub import snapshot_download
+
+ self.bake()
+ resolved = snapshot_download(
+ "convaiinnovations/laya",
+ allow_patterns=["multilingual/rl_agent_config.json", "multilingual/model.safetensors",
+ "multilingual/tokenizer/*", "multilingual/encoder/*"],
+ cache_dir=str(self.cache), local_files_only=True,
+ )
+ expected = self.cache / "models--convaiinnovations--laya" / "snapshots" / WEIGHTS_COMMIT
+ self.assertEqual(Path(os.path.realpath(resolved)), Path(os.path.realpath(str(expected))))
+
+ def test_the_commit_the_bake_keyed_resolves_as_a_pin(self):
+ """`revision=` reaching the weights' own commit must hit the same snapshot."""
+ from huggingface_hub import snapshot_download
+
+ self.bake()
+ resolved = snapshot_download("convaiinnovations/laya", revision=WEIGHTS_COMMIT,
+ cache_dir=str(self.cache), local_files_only=True)
+ self.assertEqual(Path(os.path.realpath(resolved)).name, WEIGHTS_COMMIT)
+
+ def test_a_commit_the_bake_did_not_key_misses(self):
+ """A commit that was not baked must miss, not silently serve the weights' snapshot."""
+ from huggingface_hub import snapshot_download
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ self.bake()
+ with self.assertRaises(LocalEntryNotFoundError):
+ snapshot_download("convaiinnovations/laya", revision=TIP_COMMIT,
+ cache_dir=str(self.cache), local_files_only=True)
+
+ def test_a_dropped_connection_is_retried_and_resumed(self):
+ """A mid-body drop must resume byte-aligned, not restart or duplicate the file."""
+ repo_dir = self.bake(Dropping)
+ snapshot = repo_dir / "snapshots" / WEIGHTS_COMMIT
+ self.assertEqual(snapshot.joinpath("multilingual", "model.safetensors").read_bytes(),
+ CONTENTS["multilingual/model.safetensors"])
+
+ def test_truncated_download_refuses_to_publish(self):
+ with patch.object(prefetch, "RETRIES", 2), patch.object(prefetch, "BACKOFF", 0):
+ with self.assertRaises(SystemExit):
+ self.bake(ShortBody)
+ self.assertEqual(self.published(self.folder()), [])
+
+ def test_digest_mismatch_refuses_to_publish(self):
+ with self.assertRaises(SystemExit):
+ self.bake(WrongDigest)
+ self.assertEqual(self.published(self.folder()), [])
+
+ def test_empty_selection_is_refused(self):
+ with self.assertRaises(SystemExit):
+ self.bake(EmptyListing)
+ self.assertEqual(self.published(self.folder()), [])
+
+ def test_rebaking_the_same_commit_replaces_the_snapshot(self):
+ self.bake()
+ repo_dir = self.bake()
+ self.assertEqual(self.published(repo_dir), [WEIGHTS_COMMIT])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_packaging.py b/tests/test_packaging.py
index 656a2cfa..2f356cc4 100644
--- a/tests/test_packaging.py
+++ b/tests/test_packaging.py
@@ -303,9 +303,49 @@ check_true("compose.cuda/no stale reference to a missing file",
# Every file the Docker workflow validates must exist.
for name in ("compose.yaml", "compose.example.yml", "compose.cuda.yaml", "compose.http.yaml",
- "compose.spark.yaml"):
+ "compose.spark.yaml", "compose.modelscope.yaml"):
check_true("compose/%s exists" % name, os.path.exists(name))
+# The ModelScope bake is a build argument, so a deployment that wants it has to carry the
+# arguments into *both* services -- `laya-serve` is its own service and an override for `laya`
+# never reaches it, which is the same trap compose.cuda.yaml documents. And the image has to stay
+# offline: a baked snapshot is keyed by a ModelScope commit the Hub cannot confirm, so going online
+# downloads the same weights again instead of serving the baked copy.
+ms = read("compose.modelscope.yaml")
+check_true("compose.modelscope/covers laya-serve too",
+ re.search(r"^\s{2}laya-serve:", ms, re.M) is not None,
+ "compose.modelscope.yaml does not mention laya-serve, so served checkpoints come from "
+ "the Hub and a host without Hugging Face access cannot serve at all")
+# One argument selects the checkpoint, and it defaults to the multilingual one, the checkpoint
+# every Laya caller routes to without being asked.
+check("compose.modelscope/repeats the prefetch args for both services",
+ len(re.findall(r'MODELSCOPE_MODEL: "\$\{MODELSCOPE_MODEL:-multilingual\}"', ms)), 2)
+check("compose.modelscope/repeats the revision for both services",
+ len(re.findall(r'MODELSCOPE_REVISION: "\$\{MODELSCOPE_REVISION:-master\}"', ms)), 2)
+check("compose.modelscope/keeps every service on the baked cache",
+ len(re.findall(r'HF_HUB_OFFLINE: "\$\{HF_HUB_OFFLINE:-1\}"', ms)), 2)
+check_true("compose.modelscope/preloads only what can load offline",
+ 'LAYA_MODELS: "${LAYA_MODELS:-multilingual}"' in ms,
+ "LAYA_PRELOAD=1 with the family-wide default would fail on the first checkpoint that "
+ "was not baked")
+
+# The default build must stay exactly what it was: no prefetch, so a plain
+# `docker build .` hits the Hub as before, and the RUN is a no-op for the empty argument.
+dockerfile_runtime = dockerfile.partition("AS runtime")[2]
+check_true("Dockerfile/prefetch args default to off",
+ re.search(r'^ARG MODELSCOPE_MODEL=""', dockerfile_runtime, re.M) is not None,
+ "a non-empty default would change every existing build")
+check_true("Dockerfile/copies the prefetch script",
+ "docker/prefetch_modelscope.py" in dockerfile, "the RUN below references a missing file")
+check_true("Dockerfile/prefetch step is conditional",
+ re.search(r'^\s*RUN if \[ -n "\$MODELSCOPE_MODEL" \]; then', dockerfile_runtime, re.M) is not None,
+ "an unconditional RUN would make every build depend on modelscope.cn")
+check_true("Dockerfile/prefetch hands the cache to the runtime user",
+ "chown -R laya:laya /home/laya/.cache" in dockerfile_runtime,
+ "the tokenizer-compatibility fix writes into the snapshot on first load, and the image "
+ "runs as UID 10001")
+check_true("docker/prefetch_modelscope.py exists", os.path.exists("docker/prefetch_modelscope.py"))
+
# --------------------------------------------------------------- nix: the deployment layer
# Nix builds and evaluates nothing here in CI (`grep -rn nix .github/workflows/` is empty), so