Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
build(docker): add ARM64 and DGX Spark container builds (#263)
Maxfield Allison21d16bae36ea
diff --git a/.dockerignore b/.dockerignoreindex 5c01c0e0..0df3df0e 100644--- a/.dockerignore+++ b/.dockerignore@@ -8,6 +8,7 @@!LICENSE!docker/!docker/entrypoint.py+!docker/check_torch.py!laya/!laya/**!examples/diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.ymlindex 0def775e..afc23e7e 100644--- a/.github/workflows/docker.yml+++ b/.github/workflows/docker.yml@@ -9,13 +9,42 @@ permissions:contents: readjobs:+ spark-build:+ # Builds the DGX Spark image (ARM64, CUDA 13.0) and checks its libraries load.+ # No GPU here, so this proves installation, not inference.+ runs-on: ubuntu-24.04-arm+ timeout-minutes: 30+ steps:+ - uses: actions/checkout@v4+ - name: Build ARM64 CUDA 13.0 image+ run: docker build --pull --build-arg TORCH_INDEX=cu130 -t laya:spark .+ - name: Check CUDA libraries and dependencies without a GPU+ run: |+ docker run --rm --network none laya:spark python -c '+ import platform, torch+ from torch.backends import cusparselt+ assert platform.machine() == "aarch64"+ assert torch.version.cuda == "13.0"+ assert cusparselt.is_available()+ print(torch.__version__, cusparselt.version())+ '+ docker run --rm --network none laya:spark pip check+smoke:- runs-on: ubuntu-latest+ # Native runners for both architectures; torch wheels install for the target, so+ # an emulated cross-build would be slow and would not prove the ARM64 wheels.+ strategy:+ fail-fast: false+ matrix:+ runner: [ubuntu-latest, ubuntu-24.04-arm]+ runs-on: ${{ matrix.runner }}timeout-minutes: 20steps:- uses: actions/checkout@v4- name: Validate Compose and build CPU imagerun: |+ docker compose -f compose.yaml -f compose.spark.yaml config --quiet+ docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml config --quietdocker compose config --quietdocker compose -f compose.yaml -f compose.example.yml config --quietdocker compose -f compose.yaml -f compose.example.yml -f compose.cuda.yaml config --quiet@@ -55,6 +84,8 @@ jobs:run: |docker run --rm --network none -v "$PWD:/workspace:ro" -w /workspace \laya:smoke python tests/test_docker_entrypoint.py+ docker run --rm --network none -v "$PWD:/workspace:ro" -w /workspace \+ laya:smoke python tests/test_docker_torch.py- name: Reject unavailable CUDA before downloading modelsrun: |docker run --rm --network none -e LAYA_DEVICE=cuda laya:smoke python -c 'diff --git a/Dockerfile b/Dockerfileindex 7cc08fb1..caa02f40 100644--- a/Dockerfile+++ b/Dockerfile@@ -1,4 +1,7 @@-FROM python:3.11-slim-bookworm AS build+# Build and runtime share one base so the copied virtualenv matches its interpreter.+ARG PYTHON_IMAGE=python:3.11-slim-bookworm++FROM ${PYTHON_IMAGE} AS buildENV PIP_NO_CACHE_DIR=1 \PIP_DISABLE_PIP_VERSION_CHECK=1@@ -6,9 +9,15 @@ ENV PIP_NO_CACHE_DIR=1 \RUN python -m venv /opt/venvENV PATH="/opt/venv/bin:$PATH"-# CPU by default; the CUDA Compose override selects cu128.+# CPU by default on AMD64 and ARM64. The CUDA override selects cu128 and the+# DGX Spark override selects cu130. Bump TORCH_VERSION deliberately; the check+# below fails the build if the wheel does not match the requested index.ARG TORCH_INDEX=cpu-RUN pip install torch --index-url https://download.pytorch.org/whl/${TORCH_INDEX}+ARG TORCH_VERSION=2.14.0+COPY docker/check_torch.py /opt/check_torch.py+RUN pip install "torch==${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/${TORCH_INDEX} \+ && python /opt/check_torch.py "${TORCH_INDEX}" \+ && pip checkWORKDIR /srcCOPY pyproject.toml setup.py README.md LICENSE ./@@ -18,7 +27,7 @@ COPY laya/ ./laya/# fastapi and uvicorn only; torch was installed above.RUN pip install ".[serve]" && pip check-FROM python:3.11-slim-bookworm AS runtime+FROM ${PYTHON_IMAGE} AS runtimeLABEL org.opencontainers.image.title="Laya Docker quickstart" \org.opencontainers.image.source="https://github.com/NandhaKishorM/laya" \diff --git a/compose.http.yaml b/compose.http.yamlindex a3eb8272..2c5c82aa 100644--- a/compose.http.yaml+++ b/compose.http.yaml@@ -23,6 +23,7 @@ services:context: .args:TORCH_INDEX: "${LAYA_TORCH_INDEX:-cpu}"+ TORCH_VERSION: "${LAYA_TORCH_VERSION:-2.14.0}"# Overrides the image's one-shot quickstart CMD; the ENTRYPOINT that loads *_FILE# secrets still runs first.command: ["laya-serve"]diff --git a/compose.spark.yaml b/compose.spark.yamlnew file mode 100644index 00000000..a147f8f2--- /dev/null+++ b/compose.spark.yaml@@ -0,0 +1,38 @@+# DGX Spark is Linux ARM64 with NVIDIA CUDA 13.0, not Apple MPS or Jetson.+# Use instead of compose.cuda.yaml, with compose.yaml for the SDK quickstart and+# with compose.http.yaml as well for the HTTP server:+# docker compose -f compose.yaml -f compose.spark.yaml run --build --rm laya+# docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml up --build laya-serve+# `laya-serve` repeats the settings because it is a separate service; overrides+# for `laya` never reach it.+services:+ laya:+ platform: linux/arm64+ build:+ args:+ TORCH_INDEX: "${LAYA_TORCH_INDEX:-cu130}"+ environment:+ LAYA_DEVICE: "${LAYA_DEVICE:-cuda}"+ deploy:+ resources:+ reservations:+ devices:+ - driver: nvidia+ device_ids: ["${LAYA_GPU_ID:-0}"]+ capabilities: [gpu]++ laya-serve:+ platform: linux/arm64+ build:+ context: .+ args:+ TORCH_INDEX: "${LAYA_TORCH_INDEX:-cu130}"+ environment:+ LAYA_DEVICE: "${LAYA_DEVICE:-cuda}"+ deploy:+ resources:+ reservations:+ devices:+ - driver: nvidia+ device_ids: ["${LAYA_GPU_ID:-0}"]+ capabilities: [gpu]diff --git a/compose.yaml b/compose.yamlindex 1dd8573f..a53523c8 100644--- a/compose.yaml+++ b/compose.yaml@@ -4,6 +4,7 @@ services:context: .args:TORCH_INDEX: "${LAYA_TORCH_INDEX:-cpu}"+ TORCH_VERSION: "${LAYA_TORCH_VERSION:-2.14.0}"environment:LAYA_DEVICE: "${LAYA_DEVICE:-cpu}"LAYA_MODEL: "${LAYA_MODEL:-auto}"diff --git a/docker/check_torch.py b/docker/check_torch.pynew file mode 100644index 00000000..b41a52f9--- /dev/null+++ b/docker/check_torch.py@@ -0,0 +1,73 @@+"""Verify the installed wheel and repair one known NVIDIA ARM64 metadata defect.++cuSPARSELt 0.8.0 and 0.8.1 ship aarch64 wheels whose internal WHEEL file declares+an SBSA platform tag that pip rejects. PyTorch 2.11 (0.8.0) and 2.14 (0.8.1) pin them+for CUDA 13.0; NVIDIA corrected the tag in 0.9.0. Validate the actual native library+before correcting that tag. This does not change the library or relax dependency+checks for other packages.+"""+import base64+import csv+import ctypes+import hashlib+from importlib.metadata import distribution, PackageNotFoundError+import io+import platform+from pathlib import Path+import sys++# Versions whose wheels were inspected and carry only the tag defect.+REVIEWED = ("0.8.0", "0.8.1")+++def repair_cusparselt(dist):+ old = "Tag: py3-none-manylinux2014_sbsa"+ wheel = next(p for p in dist.files if str(p).endswith(".dist-info/WHEEL"))+ path = Path(dist.locate_file(wheel))+ data = path.read_text(encoding="utf-8")+ if old not in data.splitlines():+ return False+ if dist.metadata["Name"] != "nvidia-cusparselt-cu13" or dist.version not in REVIEWED:+ raise RuntimeError("Unreviewed cuSPARSELt SBSA wheel; verify its version and library first")+ library = Path(dist.locate_file("nvidia/cusparselt/lib/libcusparseLt.so.0"))+ with library.open("rb") as handle:+ header = handle.read(20)+ if header[:6] != b"\x7fELF\x02\x01" or int.from_bytes(header[18:20], "little") != 183:+ raise RuntimeError("cuSPARSELt library is not ELF64 AArch64")+ ctypes.CDLL(str(library))+ updated = data.replace(old, "Tag: py3-none-manylinux2014_aarch64").encode()+ record = path.with_name("RECORD")+ rows = list(csv.reader(io.StringIO(record.read_text(encoding="utf-8"))))+ entry = next(row for row in rows if row[0] == str(wheel))+ digest = base64.urlsafe_b64encode(hashlib.sha256(updated).digest()).rstrip(b"=").decode()+ entry[1:] = ["sha256=" + digest, str(len(updated))]+ path.write_bytes(updated)+ with record.open("w", encoding="utf-8", newline="") as handle:+ csv.writer(handle).writerows(rows)+ print(f"Corrected cuSPARSELt {dist.version} ARM64 wheel metadata")+ return True+++def main():+ if platform.system() == "Linux" and platform.machine() in ("aarch64", "arm64"):+ try:+ dist = distribution("nvidia-cusparselt-cu13")+ except PackageNotFoundError:+ pass+ else:+ repair_cusparselt(dist)+ import torch++ expected = sys.argv[1]+ actual = "cpu" if torch.version.cuda is None else "cu" + torch.version.cuda.replace(".", "")+ if actual != expected:+ raise RuntimeError(f"Expected {expected} PyTorch, installed {actual}")+ if actual != "cpu":+ from torch.backends import cusparselt+ if not cusparselt.is_available():+ raise RuntimeError("CUDA build is missing cuSPARSELt")+ print(f"PyTorch {torch.__version__} imported on {platform.machine()} ({actual})")+++if __name__ == "__main__":+ main()diff --git a/docs/docker-platforms.md b/docs/docker-platforms.mdnew file mode 100644index 00000000..36c2d86d--- /dev/null+++ b/docs/docker-platforms.md@@ -0,0 +1,96 @@+# ARM64 and DGX Spark containers++The Dockerfile builds for Linux AMD64 and ARM64. CPU remains the default on both;+an ARM64 CPU does not imply an NVIDIA GPU.++| Host | Configuration | Validation |+| --- | --- | --- |+| Linux AMD64 CPU | `compose.yaml` | CI build and checks; CPU inference |+| Linux ARM64 CPU | `compose.yaml`, built on the ARM64 host | CI build and checks on a native ARM64 runner; CPU inference |+| Linux AMD64 NVIDIA | add `compose.cuda.yaml` (CUDA 12.8) | CUDA inference on an RTX 4070 Ti with the base quickstart |+| DGX Spark | add `compose.spark.yaml` (ARM64, CUDA 13.0), with or without `compose.http.yaml` | CI builds and loads the CUDA libraries without a GPU; Spark inference not yet reported |+| Apple Silicon | Linux ARM64 container on CPU | See [Apple Silicon](#apple-silicon) |++## ARM64 CPU++Build on the target host. Docker selects its native architecture:++```bash+docker compose run --build --rm laya+```++Cross-builds with `docker buildx build --platform linux/arm64 --load -t laya:arm64 .`+need an ARM64 builder or configured emulation. An emulated build does not show+native inference performance.++## DGX Spark++Use the Spark's Linux host with its supported NVIDIA driver and the+[NVIDIA Container Toolkit](https://docs.nvidia.com/dgx/dgx-spark/nvidia-container-runtime-for-docker.html).+The override selects ARM64, CUDA 13.0 wheels and GPU 0. Use it instead of+`compose.cuda.yaml`, not with it:++```bash+docker compose -f compose.yaml -f compose.spark.yaml run --build --rm laya+```++To serve the HTTP API on the Spark, add `compose.http.yaml`. The+[HTTP serving](docker.md#http-serving) settings apply unchanged:++```bash+docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml up --build laya-serve+```++Set `LAYA_GPU_ID` to select another device.++`TORCH_VERSION` pins PyTorch for every build, CPU and CUDA alike. Compose reads+`LAYA_TORCH_VERSION`; direct builds take+`--build-arg TORCH_VERSION=2.14.0 --build-arg TORCH_INDEX=cu130`. Changing either+needs a rebuild, because a runtime environment variable cannot replace the+installed wheel.++PyTorch's CUDA 13.0 builds for ARM64 depend on cuSPARSELt 0.8.0 (PyTorch 2.11)+or 0.8.1 (PyTorch 2.14). NVIDIA's AArch64 wheels for those two versions declare+`manylinux2014_sbsa` inside their `WHEEL` file, which `pip check` rejects;+0.9.0 corrects it. The build checks that the library is ELF64 AArch64 and loads,+then corrects that tag and its `RECORD` hash. Any other version with the same+defect fails the build instead of receiving the repair, and `pip check` still+runs. This follows @TheIrritainer's compatibility finding in+[FastLaya](https://github.com/emtay-com/fastlaya).++### Reporting Spark results++CI has no GPU, so Spark inference needs a report from real hardware. Run this on+the Spark and include its output with `nvidia-smi`, the OS and driver versions,+and the image revision:++```bash+docker compose -f compose.yaml -f compose.spark.yaml run --build --rm laya python -c '+import json, platform, torch+from pathlib import Path+from laya import load+assert platform.machine() == "aarch64"+assert torch.cuda.is_available()+print(torch.__version__, torch.version.cuda, torch.cuda.get_device_name(0))+print(torch.cuda.get_device_capability(0), torch.cuda.get_arch_list())+agent = load("convaiinnovations/laya", device="cuda")+request = json.loads(Path("/opt/laya/examples/request.json").read_text())+result = agent.predict(request["state"], request["questions"])+assert next(agent.model.parameters()).device.type == "cuda", "fell back to CPU"+assert set(result["answers"]) == set(request["questions"])+print("CUDA inference passed", torch.cuda.max_memory_allocated())+'+```++Repeat with each checkpoint you intend to run. A native ARM64 CPU test does not+establish Blackwell kernel or GPU inference support. Jetson's platform-specific+CUDA stack is not covered by the Spark override.++## Apple Silicon++Apple GPU acceleration needs native macOS PyTorch with MPS. Docker Desktop runs+a Linux container, which has no MPS backend, so the container uses CPU. Laya+already has an MPS device path, and+[PR #51](https://github.com/NandhaKishorM/laya/pull/51) and+[PR #109](https://github.com/NandhaKishorM/laya/pull/109) address MPS+compatibility and performance outside Docker.diff --git a/docs/docker.md b/docs/docker.mdindex 37489043..48622078 100644--- a/docs/docker.md+++ b/docs/docker.md@@ -19,6 +19,9 @@ Weights stay in a named volume. Subsequent runs use `docker compose run --rm layPredictions and confidence still need evaluation on your workload. See the[benchmark limits](../BENCHMARKS.md).+For ARM64 hosts, DGX Spark and Apple Silicon, see+[ARM64 and DGX Spark containers](docker-platforms.md).+## NVIDIA GPU / CUDAInstall a compatible NVIDIA driver and configure Docker with the@@ -70,7 +73,8 @@ work with `docker run -e`; Compose-only settings are identified below.| `HF_HOME` | `/home/laya/.cache/huggingface` | Cache path; see mount requirement below || `LAYA_CACHE_VOLUME` | project model cache | **Compose only:** named cache volume || `LAYA_GPU_ID` | `0` | **Compose only:** NVIDIA device index or UUID |-| `LAYA_TORCH_INDEX` | `cpu` / `cu128` | **Compose build:** PyTorch wheel index |+| `LAYA_TORCH_INDEX` | `cpu` / `cu128` / `cu130` | **Compose build:** PyTorch wheel index |+| `LAYA_TORCH_VERSION` | `2.14.0` | **Compose build:** pinned PyTorch version |Compose forwards the runtime variables except `HF_HOME`, which stays alignedwith its fixed cache mount. If overriding `HF_HOME` in `docker run` or your owndiff --git a/tests/test_docker_torch.py b/tests/test_docker_torch.pynew file mode 100644index 00000000..5d298bef--- /dev/null+++ b/tests/test_docker_torch.py@@ -0,0 +1,82 @@+"""Check the narrow NVIDIA metadata repair without installing CUDA."""+import base64+import csv+import hashlib+import importlib.util+from pathlib import Path+import tempfile+import unittest+from unittest.mock import patch++SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "check_torch.py"+spec = importlib.util.spec_from_file_location("check_torch", SCRIPT)+check = importlib.util.module_from_spec(spec)+spec.loader.exec_module(check)+++class WheelFixture:+ metadata = {"Name": "nvidia-cusparselt-cu13"}+ version = "0.8.0"+ files = [Path("nvidia_cusparselt_cu13-0.8.0.dist-info/WHEEL")]++ def __init__(self, root):+ self.root = Path(root)+ self.wheel = self.locate_file(self.files[0])+ self.wheel.parent.mkdir()+ self.wheel.write_text("Wheel-Version: 1.0\nTag: py3-none-manylinux2014_sbsa\n")+ self.record = self.wheel.with_name("RECORD")+ self.record.write_text(f"{self.files[0]},sha256=old,0\nother-file,sha256=unchanged,12\n")+ self.library = self.locate_file("nvidia/cusparselt/lib/libcusparseLt.so.0")+ self.library.parent.mkdir(parents=True)+ self.library.write_bytes(b"\x7fELF\x02\x01" + bytes(12) + (183).to_bytes(2, "little"))++ def locate_file(self, path):+ return self.root / path+++class WheelRepairTests(unittest.TestCase):+ def test_repair_rehashes_record_and_is_idempotent(self):+ for version in check.REVIEWED:+ with self.subTest(version=version):+ self._repair(version)++ def _repair(self, version):+ with tempfile.TemporaryDirectory() as root:+ dist = WheelFixture(root)+ dist.version = version+ with patch.object(check.ctypes, "CDLL") as loader:+ self.assertTrue(check.repair_cusparselt(dist))+ self.assertFalse(check.repair_cusparselt(dist))+ loader.assert_called_once_with(str(dist.library))+ data = dist.wheel.read_bytes()+ self.assertIn(b"Tag: py3-none-manylinux2014_aarch64", data)+ with dist.record.open() as handle:+ rows = list(csv.reader(handle))+ digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode()+ self.assertEqual(rows[0][1:], ["sha256=" + digest, str(len(data))])+ self.assertEqual(rows[1], ["other-file", "sha256=unchanged", "12"])++ def test_unreviewed_version_is_untouched(self):+ with tempfile.TemporaryDirectory() as root:+ dist = WheelFixture(root)+ before = dist.wheel.read_bytes(), dist.record.read_bytes()+ dist.version = "0.8.2"+ with self.assertRaisesRegex(RuntimeError, "Unreviewed"):+ check.repair_cusparselt(dist)+ self.assertEqual(before, (dist.wheel.read_bytes(), dist.record.read_bytes()))++ def test_wrong_architecture_or_linkage_is_untouched(self):+ for error in ("architecture", "linkage"):+ with self.subTest(error=error), tempfile.TemporaryDirectory() as root:+ dist = WheelFixture(root)+ before = dist.wheel.read_bytes(), dist.record.read_bytes()+ if error == "architecture":+ dist.library.write_bytes(b"not an ARM64 library")+ with patch.object(check.ctypes, "CDLL", side_effect=OSError("cannot link")):+ with self.assertRaises((RuntimeError, OSError)):+ check.repair_cusparselt(dist)+ self.assertEqual(before, (dist.wheel.read_bytes(), dist.record.read_bytes()))+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_packaging.py b/tests/test_packaging.pyindex e3995249..5a133cb9 100644--- a/tests/test_packaging.py+++ b/tests/test_packaging.py@@ -198,7 +198,8 @@ check_true("compose.cuda/no stale reference to a missing file","compose.cuda.yaml still describes compose.http.yaml as living on another branch")# Every file the Docker workflow validates must exist.-for name in ("compose.yaml", "compose.example.yml", "compose.cuda.yaml", "compose.http.yaml"):+for name in ("compose.yaml", "compose.example.yml", "compose.cuda.yaml", "compose.http.yaml",+ "compose.spark.yaml"):check_true("compose/%s exists" % name, os.path.exists(name))print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))