Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
build: add Docker and Compose quickstart with optional CUDA (#45)
Maxfield Allison1addbb9ab8ff
diff --git a/.dockerignore b/.dockerignorenew file mode 100644index 00000000..5c01c0e0--- /dev/null+++ b/.dockerignore@@ -0,0 +1,20 @@+# Only package sources, the runnable example and build metadata belong here.+**+!Dockerfile+!.dockerignore+!pyproject.toml+!setup.py+!README.md+!LICENSE+!docker/+!docker/entrypoint.py+!laya/+!laya/**+!examples/+!examples/docker/+!examples/docker/quickstart.py+!examples/docker/request.json+**/__pycache__/+**/*.pyc+**/.env+**/.env.*diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.ymlnew file mode 100644index 00000000..f0233af4--- /dev/null+++ b/.github/workflows/docker.yml@@ -0,0 +1,69 @@+name: Docker++on:+ pull_request:+ push:+ branches: [main]++permissions:+ contents: read++jobs:+ smoke:+ runs-on: ubuntu-latest+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@v4+ - name: Validate Compose and build CPU image+ run: |+ docker compose config --quiet+ docker compose -f compose.yaml -f compose.example.yml config --quiet+ docker compose -f compose.yaml -f compose.example.yml -f compose.cuda.yaml config --quiet+ docker compose -f compose.yaml -f compose.cuda.yaml config --quiet+ docker build --pull -t laya:smoke .+ - name: Check installed runtime and fresh cache permissions+ run: |+ docker run --rm --network none -v laya-smoke-cache:/home/laya/.cache/huggingface \+ laya:smoke python -c '+ import json, os, pathlib, torch+ from laya import Router+ assert os.getuid() == 10001+ assert torch.version.cuda is None+ cache = pathlib.Path(os.environ["HF_HOME"])+ (cache / "write-test").write_text("cache survives container replacement")+ request = json.loads(pathlib.Path("/opt/laya/examples/request.json").read_text())+ assert Router().route(request["state"], request["questions"]).model == "english"+ '+ - name: Check cache survives container replacement+ run: |+ docker run --rm --network none -v laya-smoke-cache:/home/laya/.cache/huggingface \+ laya:smoke python -c '+ import os, pathlib+ marker = pathlib.Path(os.environ["HF_HOME"]) / "write-test"+ assert marker.read_text() == "cache survives container replacement"+ marker.unlink()+ '+ - name: Run existing tests without model downloads+ run: |+ docker run --rm --network none -v "$PWD:/workspace:ro" -w /workspace \+ laya:smoke python tests/test_router.py+ docker run --rm --network none -v "$PWD:/workspace:ro" -w /workspace \+ laya:smoke python tests/test_criteria.py+ - name: Test secret-file configuration+ run: |+ docker run --rm --network none -v "$PWD:/workspace:ro" -w /workspace \+ laya:smoke python tests/test_docker_entrypoint.py+ - name: Reject unavailable CUDA before downloading models+ run: |+ docker run --rm --network none -e LAYA_DEVICE=cuda laya:smoke python -c '+ import runpy+ try:+ runpy.run_path("/opt/laya/examples/quickstart.py", run_name="__main__")+ except RuntimeError as error:+ assert "CUDA was requested but is unavailable" in str(error)+ else:+ raise AssertionError("CUDA request unexpectedly succeeded in the CPU image")+ '+ - name: Remove test cache+ if: always()+ run: docker volume rm laya-smoke-cache || truediff --git a/Dockerfile b/Dockerfilenew file mode 100644index 00000000..940e6a5e--- /dev/null+++ b/Dockerfile@@ -0,0 +1,47 @@+FROM python:3.11-slim-bookworm AS build++ENV PIP_NO_CACHE_DIR=1 \+ PIP_DISABLE_PIP_VERSION_CHECK=1++RUN python -m venv /opt/venv+ENV PATH="/opt/venv/bin:$PATH"++# CPU by default; the CUDA Compose override selects cu128.+ARG TORCH_INDEX=cpu+RUN pip install torch --index-url https://download.pytorch.org/whl/${TORCH_INDEX}++WORKDIR /src+COPY pyproject.toml setup.py README.md LICENSE ./+COPY laya/ ./laya/+RUN pip install . && pip check++FROM python:3.11-slim-bookworm AS runtime++LABEL org.opencontainers.image.title="Laya Docker quickstart" \+ org.opencontainers.image.source="https://github.com/NandhaKishorM/laya" \+ org.opencontainers.image.licenses="Apache-2.0"++ENV PATH="/opt/venv/bin:$PATH" \+ PYTHONUNBUFFERED=1 \+ PYTHONDONTWRITEBYTECODE=1 \+ USE_TF=0 \+ USE_TORCH=1 \+ TOKENIZERS_PARALLELISM=false \+ OMP_NUM_THREADS=4 \+ LAYA_DEVICE=cpu \+ HF_HOME=/home/laya/.cache/huggingface++RUN groupadd --gid 10001 laya \+ && useradd --uid 10001 --gid laya --create-home laya \+ && mkdir -p /home/laya/.cache/huggingface \+ && chown -R laya:laya /home/laya/.cache++COPY --from=build /opt/venv /opt/venv+COPY LICENSE /usr/share/doc/laya/LICENSE+COPY examples/docker/ /opt/laya/examples/+COPY docker/entrypoint.py /opt/laya/entrypoint.py+USER laya+WORKDIR /home/laya++ENTRYPOINT ["python", "/opt/laya/entrypoint.py"]+CMD ["python", "/opt/laya/examples/quickstart.py"]diff --git a/README.md b/README.mdindex 0575ef13..2c6ee88f 100644--- a/README.md+++ b/README.md@@ -130,6 +130,10 @@ own. Both ids run the checkpoints published here, unmodified, at `max_len` 8192.## Quickstart: Route Mode (Recommended)+To try the Python SDK in a CPU container, see the+[Docker Compose quickstart](docs/docker.md). It runs a sample request and keeps+downloaded models between runs.+Laya ships three checkpoints. The built-in **`Router`** is the recommended entry point: it evaluates any state in any language, automatically detects scripts and languages in sub-milliseconds, and dispatches to the optimal checkpoint in a single forward pass.```pythondiff --git a/compose.cuda.yaml b/compose.cuda.yamlnew file mode 100644index 00000000..f2857819--- /dev/null+++ b/compose.cuda.yaml@@ -0,0 +1,15 @@+# Use with compose.yaml, or with compose.http.yaml on the HTTP preview branch.+services:+ laya:+ build:+ args:+ TORCH_INDEX: "${LAYA_TORCH_INDEX:-cu128}"+ environment:+ LAYA_DEVICE: "${LAYA_DEVICE:-cuda}"+ deploy:+ resources:+ reservations:+ devices:+ - driver: nvidia+ device_ids: ["${LAYA_GPU_ID:-0}"]+ capabilities: [gpu]diff --git a/compose.example.yml b/compose.example.ymlnew file mode 100644index 00000000..6d0cb477--- /dev/null+++ b/compose.example.yml@@ -0,0 +1,26 @@+# Run from the repository root:+# docker compose -f compose.yaml -f compose.example.yml run --build --rm laya+# For NVIDIA, append -f compose.cuda.yaml before "run".+# This override inherits the image build and persistent cache from compose.yaml.+services:+ laya:+ environment:+ LAYA_DEVICE: "${LAYA_DEVICE:-cpu}"+ LAYA_MODEL: "${LAYA_MODEL:-english}"+ LAYA_REQUEST_FILE: /inputs/request.json+ OMP_NUM_THREADS: "${OMP_NUM_THREADS:-4}"+ HF_HUB_OFFLINE: "${HF_HUB_OFFLINE:-0}"++ # Local checkpoint: set LAYA_MODEL=auto above, then enable both this+ # setting and the checkpoint mount below. The directory must be writable+ # by UID 10001 for tokenizer compatibility updates.+ # LAYA_MODEL_PATH: /models/custom++ # Optional token: enable this setting and its read-only mount below.+ # The token file must be readable by UID 10001. Never put its value here.+ # HF_TOKEN_FILE: /run/secrets/hf_token+ volumes:+ # Replace the source path with your own request file.+ - ./examples/docker/request.json:/inputs/request.json:ro+ # - ${LAYA_CHECKPOINT_PATH:?Set LAYA_CHECKPOINT_PATH}:/models/custom+ # - ${HF_TOKEN_PATH:?Set HF_TOKEN_PATH}:/run/secrets/hf_token:rodiff --git a/compose.yaml b/compose.yamlnew file mode 100644index 00000000..1dd8573f--- /dev/null+++ b/compose.yaml@@ -0,0 +1,22 @@+services:+ laya:+ build:+ context: .+ args:+ TORCH_INDEX: "${LAYA_TORCH_INDEX:-cpu}"+ environment:+ LAYA_DEVICE: "${LAYA_DEVICE:-cpu}"+ LAYA_MODEL: "${LAYA_MODEL:-auto}"+ LAYA_MODEL_PATH: "${LAYA_MODEL_PATH:-}"+ LAYA_REQUEST_FILE: "${LAYA_REQUEST_FILE:-/opt/laya/examples/request.json}"+ HF_HUB_OFFLINE: "${HF_HUB_OFFLINE:-0}"+ HF_TOKEN: "${HF_TOKEN:-}"+ HF_TOKEN_FILE: "${HF_TOKEN_FILE:-}"+ OMP_NUM_THREADS: "${OMP_NUM_THREADS:-4}"+ volumes:+ - model-cache:/home/laya/.cache/huggingface+ init: true++volumes:+ model-cache:+ name: "${LAYA_CACHE_VOLUME:-${COMPOSE_PROJECT_NAME}_model-cache}"diff --git a/docker/entrypoint.py b/docker/entrypoint.pynew file mode 100644index 00000000..ca871747--- /dev/null+++ b/docker/entrypoint.py@@ -0,0 +1,28 @@+"""Load supported file-backed secrets, then replace this process with the command."""+import os+from pathlib import Path+import sys++SECRET_NAMES = ("HF_TOKEN",)+++def main():+ for name in SECRET_NAMES:+ filename = os.environ.get(name + "_FILE")+ if not filename:+ continue+ try:+ value = Path(filename).read_text(encoding="utf-8").strip()+ except (OSError, UnicodeError, ValueError):+ sys.exit(f"Cannot read {name}_FILE")+ if not value or "\0" in value:+ sys.exit(f"Invalid empty or NUL-containing secret in {name}_FILE")+ os.environ[name] = value+ os.environ.pop(name + "_FILE", None)+ if len(sys.argv) < 2:+ sys.exit("A container command is required")+ os.execvp(sys.argv[1], sys.argv[1:])+++if __name__ == "__main__":+ main()diff --git a/docs/docker.md b/docs/docker.mdnew file mode 100644index 00000000..b273795a--- /dev/null+++ b/docs/docker.md@@ -0,0 +1,171 @@+# Docker quickstart++Run the SDK without installing Python or PyTorch on your host. For the CPU+quickstart, allow 8 GB of RAM and 10 GB of free disk, with Docker Engine or+Docker Desktop and Compose v2 or newer.++From the repository root:++```bash+docker compose run --build --rm laya+```++This builds the checkout, runs the [sample request](../examples/docker/request.json)+on CPU and prints JSON covering `choice`, `score` and `noul`. The first request+downloads the selected public Hugging Face checkpoint; no account is needed.+Allow several minutes for its first download.+Weights stay in a named volume. Subsequent runs use `docker compose run --rm laya`.++Predictions and confidence still need evaluation on your workload. See the+[benchmark limits](../BENCHMARKS.md).++## NVIDIA GPU / CUDA++Install a compatible NVIDIA driver and configure Docker with the+[NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).+The GPU image uses PyTorch CUDA 12.8 wheels. Check your GPU's compute capability+and driver against [PyTorch's supported builds](https://pytorch.org/get-started/locally/);+older cards may require a different build. Allow additional disk space for CUDA+layers. VRAM needs depend on the checkpoint, batch size and input length.++```bash+docker compose -f compose.yaml -f compose.cuda.yaml run --build --rm laya+```++The override selects GPU `0` and defaults to `LAYA_DEVICE=cuda`. Set+`LAYA_GPU_ID` to another host index or UUID. That GPU appears as device `0`+inside the container. Check access without downloading weights:++```bash+docker compose -f compose.yaml -f compose.cuda.yaml run --rm laya python -c \+ 'import torch; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0)); print(torch.ones(1, device="cuda").cpu())'+```++The sample rejects unavailable CUDA before loading a checkpoint. Laya can still+fall back to CPU after a memory or inference error, so inspect its warnings.+Rebuild when switching between CPU and CUDA configurations.++This uses [Compose GPU reservations](https://docs.docker.com/compose/how-tos/gpu-support/).+Windows requires Docker Desktop's supported WSL2 GPU setup. Apple MPS,+AMD/ROCm and Intel GPU containers are outside this quickstart; use CPU unless+you configure and validate another backend.++## Configuration++Set Compose variables in your shell, a local `.env` file, or the service's+`environment` block. Don't commit secrets in `.env`. Runtime variables also+work with `docker run -e`; Compose-only settings are identified below.++| Variable | Default | Purpose |+| --- | --- | --- |+| `LAYA_DEVICE` | `cpu` / `cuda` | Device selected by the base / GPU configuration |+| `LAYA_MODEL` | `auto` | Router alias: `auto`, `english`, `multilingual`, `typed-decisions` |+| `LAYA_MODEL_PATH` | unset | Compatible checkpoint path inside the container |+| `LAYA_REQUEST_FILE` | bundled request | JSON request path inside the container |+| `OMP_NUM_THREADS` | `4` | CPU threads; keep within available cores |+| `HF_TOKEN` / `HF_TOKEN_FILE` | unset | Optional Hugging Face credential |+| `HF_HUB_OFFLINE` | `0` | `1` uses only cached checkpoints |+| `HF_HOME` | `/home/laya/.cache/huggingface` | Cache path; see mount requirement below |+| `LAYA_CACHE_VOLUME` | project model cache | **Compose only:** named cache volume |+| `LAYA_GPU_ID` | `0` | **Compose only:** NVIDIA device index or UUID |+| `LAYA_TORCH_INDEX` | `cpu` / `cu128` | **Compose build:** PyTorch wheel index |++Compose forwards the runtime variables except `HF_HOME`, which stays aligned+with its fixed cache mount. If overriding `HF_HOME` in `docker run` or your own+Compose file, provide a matching mount writable by UID 10001. Direct Docker+builds select PyTorch with `--build-arg TORCH_INDEX=cu128`; runtime `-e` cannot+change the installed wheel.++```bash+LAYA_MODEL=english OMP_NUM_THREADS=2 docker compose run --build --rm laya++docker build -t laya:local .+docker run --rm -e LAYA_MODEL=english -e OMP_NUM_THREADS=2 \+ -v laya-model-cache:/home/laya/.cache/huggingface laya:local+```++For your own request:++```bash+docker compose run --rm --volume "$PWD/request.json:/inputs/request.json:ro" \+ --env LAYA_REQUEST_FILE=/inputs/request.json laya+```++For a commented configuration with request, checkpoint and secret-file mounts,+see [`compose.example.yml`](../compose.example.yml):++```bash+docker compose -f compose.yaml -f compose.example.yml run --build --rm laya+```++Add `-f compose.cuda.yaml` before `run` for NVIDIA GPUs. The example is an+override of `compose.yaml`, so cache and image settings stay in one place.++## Secret files++`HF_TOKEN_FILE` reads a mounted UTF-8 file at startup, trims surrounding+whitespace and takes precedence over `HF_TOKEN`. Unreadable, empty or invalid+files stop startup without printing their contents. The file must be readable+by UID 10001. `_FILE` applies only to supported secrets, not every setting.++With `HF_TOKEN_PATH` pointing to an existing host file outside the checkout:++```bash+docker compose run --rm --volume "$HF_TOKEN_PATH:/run/secrets/hf_token:ro" \+ --env HF_TOKEN_FILE=/run/secrets/hf_token laya+```++Docker secrets or Kubernetes Secret volumes can supply the same file. Values+are loaded into the process environment at startup; restart after changing a+file. Never use tokens as build arguments or bake them into images. Public+checkpoints need no token.++## Fine-tuned checkpoints++This image runs inference. Training scripts are tracked in+[#4](https://github.com/NandhaKishorM/laya/issues/4) and+[#26](https://github.com/NandhaKishorM/laya/issues/26); training commands can follow+once that interface is available.++Point `LAYA_CHECKPOINT_PATH` to an absolute host directory containing+`rl_agent_config.json`, `model.safetensors` and matching tokenizer files:++```bash+docker compose run --rm --volume "$LAYA_CHECKPOINT_PATH:/models/custom" \+ --env LAYA_MODEL_PATH=/models/custom laya+```++Use a working copy writable by UID 10001 because the loader may update tokenizer+configuration. A LoRA adapter alone is not a complete checkpoint. Leave+`LAYA_MODEL=auto` when setting `LAYA_MODEL_PATH`; an explicit alias and local path+are mutually exclusive. The local-path response comes from the Agent and has no+Router `routing` metadata. These settings also work with the CUDA override.+Evaluate fine-tuned checkpoints on held-out examples before relying on them.++## Development and cleanup++Open a Python prompt with `docker compose run --rm laya python`. To run the+existing routing/criteria checks and secret-file tests against your checkout+without downloading weights:++```bash+docker compose run --rm --volume "$PWD:/workspace:ro" --workdir /workspace laya \+ sh -ec 'python tests/test_router.py; python tests/test_criteria.py; python tests/test_docker_entrypoint.py'+```++Rebuild with `--build` after changing source or the bundled example. The image+runs as UID/GID 10001. New named volumes inherit the image cache directory's+ownership; host directories must be writable by that UID. Keep model caches+writable for tokenizer compatibility updates.++`--rm` removes completed containers. `docker compose down` retains the cache.+To **delete downloaded weights**, run `docker compose down --volumes` using the+same Compose files and `LAYA_CACHE_VOLUME` setting. The next request downloads+them again; don't remove a cache shared with another project.++## HTTP serving++The base quickstart runs the SDK and publishes no ports. HTTP serving is pending+[#3](https://github.com/NandhaKishorM/laya/pull/3) or+[#31](https://github.com/NandhaKishorM/laya/pull/31). The server command and health+probe can follow the interface accepted upstream.diff --git a/examples/docker/quickstart.py b/examples/docker/quickstart.pynew file mode 100644index 00000000..a1b1b0c0--- /dev/null+++ b/examples/docker/quickstart.py@@ -0,0 +1,34 @@+"""Run one SDK request using the adjacent example input."""+import json+import os+from pathlib import Path++import torch++from laya import Router, load+++def main():+ device = os.environ.get("LAYA_DEVICE", "cpu")+ if torch.device(device).type == "cuda" and not torch.cuda.is_available():+ raise RuntimeError(+ "CUDA was requested but is unavailable. "+ "Check the GPU override, host driver and NVIDIA Container Toolkit."+ )+ request_file = os.environ.get("LAYA_REQUEST_FILE") or Path(__file__).with_name("request.json")+ request = json.loads(Path(request_file).read_text(encoding="utf-8"))+ model_path = os.environ.get("LAYA_MODEL_PATH")+ model = os.environ.get("LAYA_MODEL", "auto") or "auto"+ if model_path:+ if model != "auto":+ raise ValueError("Set either LAYA_MODEL or LAYA_MODEL_PATH, not both")+ result = load(model_path, device=device).predict(request["state"], request["questions"])+ else:+ result = Router(device=device).predict(+ request["state"], request["questions"], model=None if model == "auto" else model+ )+ print(json.dumps(result, ensure_ascii=False, indent=2))+++if __name__ == "__main__":+ main()diff --git a/examples/docker/request.json b/examples/docker/request.jsonnew file mode 100644index 00000000..f5cdc883--- /dev/null+++ b/examples/docker/request.json@@ -0,0 +1,25 @@+{+ "state": {+ "body": "I was charged twice for my subscription. Please refund the duplicate charge."+ },+ "questions": {+ "department": {+ "type": "choice",+ "instructions": "Which department should handle this request?",+ "criteria": {+ "billing": "invoices, payments, refunds",+ "technical": "bugs, outages, system errors",+ "sales": "pricing, new contracts"+ }+ },+ "urgency": {+ "type": "score",+ "instructions": "How urgent is this request?",+ "criteria": ["not urgent", "needs attention soon", "critical deadline or blocking issue"]+ },+ "refund_requested": {+ "type": "noul",+ "instructions": "Does the user explicitly request a refund?"+ }+ }+}diff --git a/tests/test_docker_entrypoint.py b/tests/test_docker_entrypoint.pynew file mode 100644index 00000000..ed2e1d28--- /dev/null+++ b/tests/test_docker_entrypoint.py@@ -0,0 +1,64 @@+"""Exercise secret-file handling across the container command boundary."""+import os+from pathlib import Path+import subprocess+import sys+import tempfile+import unittest++ENTRYPOINT = Path(__file__).resolve().parents[1] / "docker" / "entrypoint.py"+++class EntrypointTests(unittest.TestCase):+ def invoke(self, extra, expected="direct"):+ env = {key: value for key, value in os.environ.items()+ if key not in {"HF_TOKEN", "HF_TOKEN_FILE"}}+ env.update(extra)+ child = "import os, sys; assert os.environ.get('HF_TOKEN') == sys.argv[1]; print('child ran')"+ return subprocess.run(+ [sys.executable, str(ENTRYPOINT), sys.executable, "-c", child, expected],+ env=env, capture_output=True, text=True, check=False,+ )++ def test_direct_value(self):+ result = self.invoke({"HF_TOKEN": "direct"})+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(result.stdout.strip(), "child ran")++ def test_file_overrides_direct_and_trims_whitespace(self):+ with tempfile.TemporaryDirectory() as directory:+ secret = Path(directory) / "token"+ secret.write_text(" synthetic-file-value\n", encoding="utf-8")+ result = self.invoke({"HF_TOKEN": "direct", "HF_TOKEN_FILE": str(secret)},+ "synthetic-file-value")+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertNotIn("synthetic-file-value", result.stdout + result.stderr)++ def test_invalid_files_stop_before_child_without_disclosure(self):+ with tempfile.TemporaryDirectory() as directory:+ secret = Path(directory) / "private-filename"+ for contents in (None, b"", b" \n", b"synthetic-secret\x00", b"\xff"):+ with self.subTest(contents=contents):+ if contents is not None:+ secret.write_bytes(contents)+ result = self.invoke({"HF_TOKEN": "direct", "HF_TOKEN_FILE": str(secret)})+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(result.stdout, "")+ self.assertIn("HF_TOKEN_FILE", result.stderr)+ self.assertNotIn(str(secret), result.stderr)+ self.assertNotIn("synthetic-secret", result.stderr)+ self.assertNotIn("direct", result.stderr)++ def test_empty_file_setting_and_unknown_suffix_are_ignored(self):+ result = self.invoke({"HF_TOKEN": "direct", "HF_TOKEN_FILE": "", "UNKNOWN_FILE": "/missing"})+ self.assertEqual(result.returncode, 0, result.stderr)++ def test_missing_command_fails(self):+ result = subprocess.run([sys.executable, str(ENTRYPOINT)], env={},+ capture_output=True, text=True, check=False)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("command is required", result.stderr)+++if __name__ == "__main__":+ unittest.main()