Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

build(docker): add ARM64 and DGX Spark container builds (#263)

Maxfield Allison21d16bae36ea
diff --git a/.dockerignore b/.dockerignore
index 5c01c0e0..0df3df0e 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -8,6 +8,7 @@
!LICENSE
!docker/
!docker/entrypoint.py
+!docker/check_torch.py
!laya/
!laya/**
!examples/
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 0def775e..afc23e7e 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -9,13 +9,42 @@ permissions:
contents: read
jobs:
+ spark-build:
+ # Builds the DGX Spark image (ARM64, CUDA 13.0) and checks its libraries load.
+ # No GPU here, so this proves installation, not inference.
+ runs-on: ubuntu-24.04-arm
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@v4
+ - name: Build ARM64 CUDA 13.0 image
+ run: docker build --pull --build-arg TORCH_INDEX=cu130 -t laya:spark .
+ - name: Check CUDA libraries and dependencies without a GPU
+ run: |
+ docker run --rm --network none laya:spark python -c '
+ import platform, torch
+ from torch.backends import cusparselt
+ assert platform.machine() == "aarch64"
+ assert torch.version.cuda == "13.0"
+ assert cusparselt.is_available()
+ print(torch.__version__, cusparselt.version())
+ '
+ docker run --rm --network none laya:spark pip check
+
smoke:
- runs-on: ubuntu-latest
+ # Native runners for both architectures; torch wheels install for the target, so
+ # an emulated cross-build would be slow and would not prove the ARM64 wheels.
+ strategy:
+ fail-fast: false
+ matrix:
+ runner: [ubuntu-latest, ubuntu-24.04-arm]
+ runs-on: ${{ matrix.runner }}
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- name: Validate Compose and build CPU image
run: |
+ docker compose -f compose.yaml -f compose.spark.yaml config --quiet
+ docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml config --quiet
docker compose config --quiet
docker compose -f compose.yaml -f compose.example.yml config --quiet
docker compose -f compose.yaml -f compose.example.yml -f compose.cuda.yaml config --quiet
@@ -55,6 +84,8 @@ jobs:
run: |
docker run --rm --network none -v "$PWD:/workspace:ro" -w /workspace \
laya:smoke python tests/test_docker_entrypoint.py
+ docker run --rm --network none -v "$PWD:/workspace:ro" -w /workspace \
+ laya:smoke python tests/test_docker_torch.py
- name: Reject unavailable CUDA before downloading models
run: |
docker run --rm --network none -e LAYA_DEVICE=cuda laya:smoke python -c '
diff --git a/Dockerfile b/Dockerfile
index 7cc08fb1..caa02f40 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,7 @@
-FROM python:3.11-slim-bookworm AS build
+# Build and runtime share one base so the copied virtualenv matches its interpreter.
+ARG PYTHON_IMAGE=python:3.11-slim-bookworm
+
+FROM ${PYTHON_IMAGE} AS build
ENV PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
@@ -6,9 +9,15 @@ ENV PIP_NO_CACHE_DIR=1 \
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"
-# CPU by default; the CUDA Compose override selects cu128.
+# CPU by default on AMD64 and ARM64. The CUDA override selects cu128 and the
+# DGX Spark override selects cu130. Bump TORCH_VERSION deliberately; the check
+# below fails the build if the wheel does not match the requested index.
ARG TORCH_INDEX=cpu
-RUN pip install torch --index-url https://download.pytorch.org/whl/${TORCH_INDEX}
+ARG TORCH_VERSION=2.14.0
+COPY docker/check_torch.py /opt/check_torch.py
+RUN pip install "torch==${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/${TORCH_INDEX} \
+ && python /opt/check_torch.py "${TORCH_INDEX}" \
+ && pip check
WORKDIR /src
COPY pyproject.toml setup.py README.md LICENSE ./
@@ -18,7 +27,7 @@ COPY laya/ ./laya/
# fastapi and uvicorn only; torch was installed above.
RUN pip install ".[serve]" && pip check
-FROM python:3.11-slim-bookworm AS runtime
+FROM ${PYTHON_IMAGE} AS runtime
LABEL org.opencontainers.image.title="Laya Docker quickstart" \
org.opencontainers.image.source="https://github.com/NandhaKishorM/laya" \
diff --git a/compose.http.yaml b/compose.http.yaml
index a3eb8272..2c5c82aa 100644
--- a/compose.http.yaml
+++ b/compose.http.yaml
@@ -23,6 +23,7 @@ services:
context: .
args:
TORCH_INDEX: "${LAYA_TORCH_INDEX:-cpu}"
+ TORCH_VERSION: "${LAYA_TORCH_VERSION:-2.14.0}"
# Overrides the image's one-shot quickstart CMD; the ENTRYPOINT that loads *_FILE
# secrets still runs first.
command: ["laya-serve"]
diff --git a/compose.spark.yaml b/compose.spark.yaml
new file mode 100644
index 00000000..a147f8f2
--- /dev/null
+++ b/compose.spark.yaml
@@ -0,0 +1,38 @@
+# DGX Spark is Linux ARM64 with NVIDIA CUDA 13.0, not Apple MPS or Jetson.
+# Use instead of compose.cuda.yaml, with compose.yaml for the SDK quickstart and
+# with compose.http.yaml as well for the HTTP server:
+# docker compose -f compose.yaml -f compose.spark.yaml run --build --rm laya
+# docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml up --build laya-serve
+# `laya-serve` repeats the settings because it is a separate service; overrides
+# for `laya` never reach it.
+services:
+ laya:
+ platform: linux/arm64
+ build:
+ args:
+ TORCH_INDEX: "${LAYA_TORCH_INDEX:-cu130}"
+ environment:
+ LAYA_DEVICE: "${LAYA_DEVICE:-cuda}"
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ device_ids: ["${LAYA_GPU_ID:-0}"]
+ capabilities: [gpu]
+
+ laya-serve:
+ platform: linux/arm64
+ build:
+ context: .
+ args:
+ TORCH_INDEX: "${LAYA_TORCH_INDEX:-cu130}"
+ environment:
+ LAYA_DEVICE: "${LAYA_DEVICE:-cuda}"
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ device_ids: ["${LAYA_GPU_ID:-0}"]
+ capabilities: [gpu]
diff --git a/compose.yaml b/compose.yaml
index 1dd8573f..a53523c8 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -4,6 +4,7 @@ services:
context: .
args:
TORCH_INDEX: "${LAYA_TORCH_INDEX:-cpu}"
+ TORCH_VERSION: "${LAYA_TORCH_VERSION:-2.14.0}"
environment:
LAYA_DEVICE: "${LAYA_DEVICE:-cpu}"
LAYA_MODEL: "${LAYA_MODEL:-auto}"
diff --git a/docker/check_torch.py b/docker/check_torch.py
new file mode 100644
index 00000000..b41a52f9
--- /dev/null
+++ b/docker/check_torch.py
@@ -0,0 +1,73 @@
+"""Verify the installed wheel and repair one known NVIDIA ARM64 metadata defect.
+
+cuSPARSELt 0.8.0 and 0.8.1 ship aarch64 wheels whose internal WHEEL file declares
+an SBSA platform tag that pip rejects. PyTorch 2.11 (0.8.0) and 2.14 (0.8.1) pin them
+for CUDA 13.0; NVIDIA corrected the tag in 0.9.0. Validate the actual native library
+before correcting that tag. This does not change the library or relax dependency
+checks for other packages.
+"""
+import base64
+import csv
+import ctypes
+import hashlib
+from importlib.metadata import distribution, PackageNotFoundError
+import io
+import platform
+from pathlib import Path
+import sys
+
+# Versions whose wheels were inspected and carry only the tag defect.
+REVIEWED = ("0.8.0", "0.8.1")
+
+
+def repair_cusparselt(dist):
+ old = "Tag: py3-none-manylinux2014_sbsa"
+ wheel = next(p for p in dist.files if str(p).endswith(".dist-info/WHEEL"))
+ path = Path(dist.locate_file(wheel))
+ data = path.read_text(encoding="utf-8")
+ if old not in data.splitlines():
+ return False
+ if dist.metadata["Name"] != "nvidia-cusparselt-cu13" or dist.version not in REVIEWED:
+ raise RuntimeError("Unreviewed cuSPARSELt SBSA wheel; verify its version and library first")
+ library = Path(dist.locate_file("nvidia/cusparselt/lib/libcusparseLt.so.0"))
+ with library.open("rb") as handle:
+ header = handle.read(20)
+ if header[:6] != b"\x7fELF\x02\x01" or int.from_bytes(header[18:20], "little") != 183:
+ raise RuntimeError("cuSPARSELt library is not ELF64 AArch64")
+ ctypes.CDLL(str(library))
+ updated = data.replace(old, "Tag: py3-none-manylinux2014_aarch64").encode()
+ record = path.with_name("RECORD")
+ rows = list(csv.reader(io.StringIO(record.read_text(encoding="utf-8"))))
+ entry = next(row for row in rows if row[0] == str(wheel))
+ digest = base64.urlsafe_b64encode(hashlib.sha256(updated).digest()).rstrip(b"=").decode()
+ entry[1:] = ["sha256=" + digest, str(len(updated))]
+ path.write_bytes(updated)
+ with record.open("w", encoding="utf-8", newline="") as handle:
+ csv.writer(handle).writerows(rows)
+ print(f"Corrected cuSPARSELt {dist.version} ARM64 wheel metadata")
+ return True
+
+
+def main():
+ if platform.system() == "Linux" and platform.machine() in ("aarch64", "arm64"):
+ try:
+ dist = distribution("nvidia-cusparselt-cu13")
+ except PackageNotFoundError:
+ pass
+ else:
+ repair_cusparselt(dist)
+ import torch
+
+ expected = sys.argv[1]
+ actual = "cpu" if torch.version.cuda is None else "cu" + torch.version.cuda.replace(".", "")
+ if actual != expected:
+ raise RuntimeError(f"Expected {expected} PyTorch, installed {actual}")
+ if actual != "cpu":
+ from torch.backends import cusparselt
+ if not cusparselt.is_available():
+ raise RuntimeError("CUDA build is missing cuSPARSELt")
+ print(f"PyTorch {torch.__version__} imported on {platform.machine()} ({actual})")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/docker-platforms.md b/docs/docker-platforms.md
new file mode 100644
index 00000000..36c2d86d
--- /dev/null
+++ b/docs/docker-platforms.md
@@ -0,0 +1,96 @@
+# ARM64 and DGX Spark containers
+
+The Dockerfile builds for Linux AMD64 and ARM64. CPU remains the default on both;
+an ARM64 CPU does not imply an NVIDIA GPU.
+
+| Host | Configuration | Validation |
+| --- | --- | --- |
+| Linux AMD64 CPU | `compose.yaml` | CI build and checks; CPU inference |
+| Linux ARM64 CPU | `compose.yaml`, built on the ARM64 host | CI build and checks on a native ARM64 runner; CPU inference |
+| Linux AMD64 NVIDIA | add `compose.cuda.yaml` (CUDA 12.8) | CUDA inference on an RTX 4070 Ti with the base quickstart |
+| DGX Spark | add `compose.spark.yaml` (ARM64, CUDA 13.0), with or without `compose.http.yaml` | CI builds and loads the CUDA libraries without a GPU; Spark inference not yet reported |
+| Apple Silicon | Linux ARM64 container on CPU | See [Apple Silicon](#apple-silicon) |
+
+## ARM64 CPU
+
+Build on the target host. Docker selects its native architecture:
+
+```bash
+docker compose run --build --rm laya
+```
+
+Cross-builds with `docker buildx build --platform linux/arm64 --load -t laya:arm64 .`
+need an ARM64 builder or configured emulation. An emulated build does not show
+native inference performance.
+
+## DGX Spark
+
+Use the Spark's Linux host with its supported NVIDIA driver and the
+[NVIDIA Container Toolkit](https://docs.nvidia.com/dgx/dgx-spark/nvidia-container-runtime-for-docker.html).
+The override selects ARM64, CUDA 13.0 wheels and GPU 0. Use it instead of
+`compose.cuda.yaml`, not with it:
+
+```bash
+docker compose -f compose.yaml -f compose.spark.yaml run --build --rm laya
+```
+
+To serve the HTTP API on the Spark, add `compose.http.yaml`. The
+[HTTP serving](docker.md#http-serving) settings apply unchanged:
+
+```bash
+docker compose -f compose.yaml -f compose.http.yaml -f compose.spark.yaml up --build laya-serve
+```
+
+Set `LAYA_GPU_ID` to select another device.
+
+`TORCH_VERSION` pins PyTorch for every build, CPU and CUDA alike. Compose reads
+`LAYA_TORCH_VERSION`; direct builds take
+`--build-arg TORCH_VERSION=2.14.0 --build-arg TORCH_INDEX=cu130`. Changing either
+needs a rebuild, because a runtime environment variable cannot replace the
+installed wheel.
+
+PyTorch's CUDA 13.0 builds for ARM64 depend on cuSPARSELt 0.8.0 (PyTorch 2.11)
+or 0.8.1 (PyTorch 2.14). NVIDIA's AArch64 wheels for those two versions declare
+`manylinux2014_sbsa` inside their `WHEEL` file, which `pip check` rejects;
+0.9.0 corrects it. The build checks that the library is ELF64 AArch64 and loads,
+then corrects that tag and its `RECORD` hash. Any other version with the same
+defect fails the build instead of receiving the repair, and `pip check` still
+runs. This follows @TheIrritainer's compatibility finding in
+[FastLaya](https://github.com/emtay-com/fastlaya).
+
+### Reporting Spark results
+
+CI has no GPU, so Spark inference needs a report from real hardware. Run this on
+the Spark and include its output with `nvidia-smi`, the OS and driver versions,
+and the image revision:
+
+```bash
+docker compose -f compose.yaml -f compose.spark.yaml run --build --rm laya python -c '
+import json, platform, torch
+from pathlib import Path
+from laya import load
+assert platform.machine() == "aarch64"
+assert torch.cuda.is_available()
+print(torch.__version__, torch.version.cuda, torch.cuda.get_device_name(0))
+print(torch.cuda.get_device_capability(0), torch.cuda.get_arch_list())
+agent = load("convaiinnovations/laya", device="cuda")
+request = json.loads(Path("/opt/laya/examples/request.json").read_text())
+result = agent.predict(request["state"], request["questions"])
+assert next(agent.model.parameters()).device.type == "cuda", "fell back to CPU"
+assert set(result["answers"]) == set(request["questions"])
+print("CUDA inference passed", torch.cuda.max_memory_allocated())
+'
+```
+
+Repeat with each checkpoint you intend to run. A native ARM64 CPU test does not
+establish Blackwell kernel or GPU inference support. Jetson's platform-specific
+CUDA stack is not covered by the Spark override.
+
+## Apple Silicon
+
+Apple GPU acceleration needs native macOS PyTorch with MPS. Docker Desktop runs
+a Linux container, which has no MPS backend, so the container uses CPU. Laya
+already has an MPS device path, and
+[PR #51](https://github.com/NandhaKishorM/laya/pull/51) and
+[PR #109](https://github.com/NandhaKishorM/laya/pull/109) address MPS
+compatibility and performance outside Docker.
diff --git a/docs/docker.md b/docs/docker.md
index 37489043..48622078 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -19,6 +19,9 @@ Weights stay in a named volume. Subsequent runs use `docker compose run --rm lay
Predictions and confidence still need evaluation on your workload. See the
[benchmark limits](../BENCHMARKS.md).
+For ARM64 hosts, DGX Spark and Apple Silicon, see
+[ARM64 and DGX Spark containers](docker-platforms.md).
+
## NVIDIA GPU / CUDA
Install a compatible NVIDIA driver and configure Docker with the
@@ -70,7 +73,8 @@ work with `docker run -e`; Compose-only settings are identified below.
| `HF_HOME` | `/home/laya/.cache/huggingface` | Cache path; see mount requirement below |
| `LAYA_CACHE_VOLUME` | project model cache | **Compose only:** named cache volume |
| `LAYA_GPU_ID` | `0` | **Compose only:** NVIDIA device index or UUID |
-| `LAYA_TORCH_INDEX` | `cpu` / `cu128` | **Compose build:** PyTorch wheel index |
+| `LAYA_TORCH_INDEX` | `cpu` / `cu128` / `cu130` | **Compose build:** PyTorch wheel index |
+| `LAYA_TORCH_VERSION` | `2.14.0` | **Compose build:** pinned PyTorch version |
Compose forwards the runtime variables except `HF_HOME`, which stays aligned
with its fixed cache mount. If overriding `HF_HOME` in `docker run` or your own
diff --git a/tests/test_docker_torch.py b/tests/test_docker_torch.py
new file mode 100644
index 00000000..5d298bef
--- /dev/null
+++ b/tests/test_docker_torch.py
@@ -0,0 +1,82 @@
+"""Check the narrow NVIDIA metadata repair without installing CUDA."""
+import base64
+import csv
+import hashlib
+import importlib.util
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+SCRIPT = Path(__file__).resolve().parents[1] / "docker" / "check_torch.py"
+spec = importlib.util.spec_from_file_location("check_torch", SCRIPT)
+check = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(check)
+
+
+class WheelFixture:
+ metadata = {"Name": "nvidia-cusparselt-cu13"}
+ version = "0.8.0"
+ files = [Path("nvidia_cusparselt_cu13-0.8.0.dist-info/WHEEL")]
+
+ def __init__(self, root):
+ self.root = Path(root)
+ self.wheel = self.locate_file(self.files[0])
+ self.wheel.parent.mkdir()
+ self.wheel.write_text("Wheel-Version: 1.0\nTag: py3-none-manylinux2014_sbsa\n")
+ self.record = self.wheel.with_name("RECORD")
+ self.record.write_text(f"{self.files[0]},sha256=old,0\nother-file,sha256=unchanged,12\n")
+ self.library = self.locate_file("nvidia/cusparselt/lib/libcusparseLt.so.0")
+ self.library.parent.mkdir(parents=True)
+ self.library.write_bytes(b"\x7fELF\x02\x01" + bytes(12) + (183).to_bytes(2, "little"))
+
+ def locate_file(self, path):
+ return self.root / path
+
+
+class WheelRepairTests(unittest.TestCase):
+ def test_repair_rehashes_record_and_is_idempotent(self):
+ for version in check.REVIEWED:
+ with self.subTest(version=version):
+ self._repair(version)
+
+ def _repair(self, version):
+ with tempfile.TemporaryDirectory() as root:
+ dist = WheelFixture(root)
+ dist.version = version
+ with patch.object(check.ctypes, "CDLL") as loader:
+ self.assertTrue(check.repair_cusparselt(dist))
+ self.assertFalse(check.repair_cusparselt(dist))
+ loader.assert_called_once_with(str(dist.library))
+ data = dist.wheel.read_bytes()
+ self.assertIn(b"Tag: py3-none-manylinux2014_aarch64", data)
+ with dist.record.open() as handle:
+ rows = list(csv.reader(handle))
+ digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode()
+ self.assertEqual(rows[0][1:], ["sha256=" + digest, str(len(data))])
+ self.assertEqual(rows[1], ["other-file", "sha256=unchanged", "12"])
+
+ def test_unreviewed_version_is_untouched(self):
+ with tempfile.TemporaryDirectory() as root:
+ dist = WheelFixture(root)
+ before = dist.wheel.read_bytes(), dist.record.read_bytes()
+ dist.version = "0.8.2"
+ with self.assertRaisesRegex(RuntimeError, "Unreviewed"):
+ check.repair_cusparselt(dist)
+ self.assertEqual(before, (dist.wheel.read_bytes(), dist.record.read_bytes()))
+
+ def test_wrong_architecture_or_linkage_is_untouched(self):
+ for error in ("architecture", "linkage"):
+ with self.subTest(error=error), tempfile.TemporaryDirectory() as root:
+ dist = WheelFixture(root)
+ before = dist.wheel.read_bytes(), dist.record.read_bytes()
+ if error == "architecture":
+ dist.library.write_bytes(b"not an ARM64 library")
+ with patch.object(check.ctypes, "CDLL", side_effect=OSError("cannot link")):
+ with self.assertRaises((RuntimeError, OSError)):
+ check.repair_cusparselt(dist)
+ self.assertEqual(before, (dist.wheel.read_bytes(), dist.record.read_bytes()))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_packaging.py b/tests/test_packaging.py
index e3995249..5a133cb9 100644
--- a/tests/test_packaging.py
+++ b/tests/test_packaging.py
@@ -198,7 +198,8 @@ check_true("compose.cuda/no stale reference to a missing file",
"compose.cuda.yaml still describes compose.http.yaml as living on another branch")
# Every file the Docker workflow validates must exist.
-for name in ("compose.yaml", "compose.example.yml", "compose.cuda.yaml", "compose.http.yaml"):
+for name in ("compose.yaml", "compose.example.yml", "compose.cuda.yaml", "compose.http.yaml",
+ "compose.spark.yaml"):
check_true("compose/%s exists" % name, os.path.exists(name))
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))