Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Add self-hostable HTTP server (Jev-compatible /v1/systemone), Nix flake, and NixOS module (#31)

Ben Fordd531dc73f27f
diff --git a/.envrc b/.envrc
new file mode 100644
index 00000000..3550a30f
--- /dev/null
+++ b/.envrc
@@ -0,0 +1 @@
+use flake
diff --git a/.gitignore b/.gitignore
index 817b66df..2ca9b6f8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,11 @@ __pycache__/
*.pyo
*.pyd
.Python
+
+# Nix / direnv
+result
+result-*
+.direnv/
env/
venv/
.venv/
diff --git a/README.md b/README.md
index b12ce8ca..bdf69803 100644
--- a/README.md
+++ b/README.md
@@ -252,6 +252,68 @@ The hint only decides *English or not*: a code whose primary subtag is `en`, `en
---
+## Self-Hosting: HTTP Server (Jev-compatible)
+
+`laya.serve` exposes the `Router` over HTTP on the same `POST /v1/systemone`
+wire protocol as TypeSafe's hosted Jev API. Laya's answer payload is already
+schema-identical to what Jev returns (`choice`/`score`/`noul` answers and a
+`{input_tokens, output_tokens}` usage block), so an existing Jev client — e.g.
+the [`hs-jev`](https://github.com/getmissionctrl/hs-jev) Haskell client — just
+needs its `baseUrl` repointed; nothing else changes.
+
+```bash
+pip install "laya[serve]" # adds fastapi + uvicorn
+LAYA_DEVICE=cuda LAYA_PRELOAD=1 laya-serve # binds 0.0.0.0:8000, preloads all 3 checkpoints
+```
+
+```bash
+curl -s localhost:8000/v1/systemone -H 'content-type: application/json' -d '{
+ "state": {"body": "billed twice, refund please or we cancel"},
+ "questions": {"dept": {"type": "choice", "instructions": "which team?",
+ "criteria": {"billing": "refunds", "tech": "bugs"}}}
+}'
+```
+
+Configuration is by environment variable: `LAYA_HOST`, `LAYA_PORT`,
+`LAYA_DEVICE`, `LAYA_PRELOAD`, `LAYA_MODELS` (comma list to preload),
+`LAYA_THREADS` (cap torch intra-op threads for CPU inference — keep at or below
+physical cores), `LAYA_AUTO_TASK`, and `LAYA_API_KEY` (when set, clients must
+send `Authorization: Bearer <key>`). A client's `model` field is honoured when it
+names a Laya checkpoint (`english`/`multilingual`/`typed-decisions`), otherwise
+the router auto-selects by script/language.
+
+### Nix / NixOS
+
+This repo is a flake. On a machine with an NVIDIA GPU:
+
+```bash
+nix run .#laya-serve # build (prebuilt CUDA torch, no compile) and serve
+nix develop # dev shell: torch-bin, transformers, fastapi, pytest
+```
+
+For a NixOS host, import the module and enable the service:
+
+```nix
+# flake inputs: laya.url = "github:<you>/laya"; # or path:/… on the same host
+{
+ imports = [ laya.nixosModules.default ];
+ services.laya-serve = {
+ enable = true;
+ host = "0.0.0.0"; # or bind to the Tailscale/LAN address
+ openFirewall = true;
+ device = "cuda";
+ models = [ "english" "multilingual" "typed-decisions" ];
+ # apiKeyFile = config.age.secrets.laya-api-key.path; # optional bearer auth
+ };
+}
+```
+
+The module runs a hardened `DynamicUser` systemd unit with CUDA device access,
+caches weights under `/var/lib/laya-serve`, and reads the bearer token (if any)
+via `LoadCredential` so it never enters the store.
+
+---
+
## Single-Model Mode (Direct SDK)
If you only need a single checkpoint for a dedicated pipeline, you can load models directly:
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 00000000..126017ec
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,61 @@
+{
+ "nodes": {
+ "flake-utils": {
+ "inputs": {
+ "systems": "systems"
+ },
+ "locked": {
+ "lastModified": 1731533236,
+ "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+ "type": "github"
+ },
+ "original": {
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "type": "github"
+ }
+ },
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1766166645,
+ "narHash": "sha256-7urOsghhTfOv2d7+2il711ej2BY5LlFHlOA7IOCD4OY=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "ccad53cd79cf4cf3bc338805d007d68565e75bda",
+ "type": "github"
+ },
+ "original": {
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "ccad53cd79cf4cf3bc338805d007d68565e75bda",
+ "type": "github"
+ }
+ },
+ "root": {
+ "inputs": {
+ "flake-utils": "flake-utils",
+ "nixpkgs": "nixpkgs"
+ }
+ },
+ "systems": {
+ "locked": {
+ "lastModified": 1681028828,
+ "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+ "owner": "nix-systems",
+ "repo": "default",
+ "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+ "type": "github"
+ },
+ "original": {
+ "owner": "nix-systems",
+ "repo": "default",
+ "type": "github"
+ }
+ }
+ },
+ "root": "root",
+ "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 00000000..fc8a321c
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,65 @@
+{
+ description = "laya + laya-serve: a self-hosted, TypeSafe Jev-compatible System-1 decision server";
+
+ inputs = {
+ # Pinned to the same rev missionctrl-infra uses, so hq's binary cache
+ # (cache.nixos.org + missionctrl.cachix.org) hits instead of rebuilding torch.
+ nixpkgs.url = "github:NixOS/nixpkgs/ccad53cd79cf4cf3bc338805d007d68565e75bda";
+ flake-utils.url = "github:numtide/flake-utils";
+ };
+
+ outputs = { self, nixpkgs, flake-utils }:
+ let
+ # Overlay exposing `laya` + `laya-serve`, built from ./nix/package.nix.
+ overlay = final: prev:
+ let p = final.callPackage ./nix/package.nix { };
+ in { inherit (p) laya laya-serve; };
+ in
+ {
+ overlays.default = overlay;
+
+ # The module builds its package from the host's own `pkgs` (see
+ # nix/laya-serve.nix), so it works even on hosts that inject `pkgs` via
+ # specialArgs and ignore module-level `nixpkgs.overlays` — no overlay
+ # required here.
+ nixosModules.default = ./nix/laya-serve.nix;
+ nixosModules.laya-serve = ./nix/laya-serve.nix;
+ }
+ // flake-utils.lib.eachDefaultSystem (system:
+ let
+ pkgs = import nixpkgs {
+ inherit system;
+ config.allowUnfree = true; # torch-bin bundles CUDA (unfree)
+ overlays = [ overlay ];
+ };
+ in
+ {
+ packages = {
+ default = pkgs.laya-serve;
+ laya-serve = pkgs.laya-serve;
+ laya = pkgs.laya;
+ };
+
+ devShells.default = pkgs.mkShell {
+ packages = [
+ (pkgs.python3.withPackages (ps: [
+ ps.torch-bin
+ ps.transformers
+ ps.safetensors
+ ps.huggingface-hub
+ ps.numpy
+ ps.fastapi
+ ps.uvicorn
+ ps.pytest
+ ps.httpx # fastapi TestClient
+ ]))
+ ];
+ shellHook = ''
+ export PYTHONPATH="$PWD:$PYTHONPATH"
+ # torch-bin's CUDA needs the host NVIDIA userspace driver.
+ export LD_LIBRARY_PATH="/run/opengl-driver/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+ echo "laya dev shell — python $(python --version 2>&1 | cut -d' ' -f2), torch $(python -c 'import torch; print(torch.__version__)' 2>/dev/null)"
+ '';
+ };
+ });
+}
diff --git a/laya/serve.py b/laya/serve.py
new file mode 100644
index 00000000..e223c0ee
--- /dev/null
+++ b/laya/serve.py
@@ -0,0 +1,161 @@
+"""HTTP server exposing Laya over TypeSafe Jev's ``/v1/systemone`` wire protocol.
+
+Laya's ``predict()`` output is already schema-compatible with the Jev decision
+API -- ``choice`` / ``score`` / ``noul`` answers and a ``{input_tokens,
+output_tokens}`` usage block -- so a client written against Jev (for example the
+`hs-jev` Haskell client) can point its ``baseUrl`` at this server and keep
+working unchanged. All this module adds is the HTTP surface Laya itself does not
+ship: a ``POST /v1/systemone`` route, an optional bearer check, and a health
+probe.
+
+Configuration is entirely via environment variables so the same entry point
+serves a laptop dev run and a systemd unit:
+
+====================== ============================================ =========
+env var meaning default
+====================== ============================================ =========
+``LAYA_HOST`` bind address 0.0.0.0
+``LAYA_PORT`` bind port 8000
+``LAYA_DEVICE`` torch device for every checkpoint (auto)
+``LAYA_PRELOAD`` build the checkpoints at startup, not lazily 1
+``LAYA_MODELS`` comma list to preload (english,multilingual, (all)
+ typed-decisions); empty = every checkpoint
+``LAYA_THREADS`` cap torch intra-op threads (CPU inference). (torch
+ Keep <= physical cores; oversubscribing the default)
+ logical/hyperthread count is a large regression.
+``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0
+``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)
+``LAYA_LOG_LEVEL`` uvicorn log level info
+====================== ============================================ =========
+
+Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are all
+deferred into the functions that need them, so ``import laya.serve`` stays cheap
+and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.
+"""
+import os
+from typing import Any, Dict, Optional
+
+# The three checkpoint names the router understands; used to decide whether a
+# client's `model` field names a Laya checkpoint (honour it) or is some other
+# Jev model id (ignore it and let the router auto-select).
+_KNOWN_MODELS = {"english", "multilingual", "typed-decisions"}
+
+
+def _env_bool(name: str, default: bool) -> bool:
+ v = os.environ.get(name)
+ if v is None:
+ return default
+ return v.strip().lower() in ("1", "true", "yes", "on")
+
+
+def _resolve_model(model: Optional[str]) -> Optional[str]:
+ """Map a client's `model` field onto a Laya checkpoint, or None to auto-route."""
+ if not model:
+ return None
+ from .router import normalise_name
+
+ # normalise_name raises ValueError on anything that is not a known checkpoint
+ # or alias. A Jev client's `model` field (e.g. "jev-1") is expected to miss;
+ # treat that as "no explicit checkpoint" and let the router auto-select.
+ try:
+ key = normalise_name(model)
+ except Exception:
+ return None
+ return key if key in _KNOWN_MODELS else None
+
+
+def _apply_thread_limit():
+ """Honour LAYA_THREADS by capping torch's intra-op thread count for CPU
+ inference. Returns the value applied, or None if unset/invalid. torch is
+ imported only when a limit is actually requested."""
+ raw = os.environ.get("LAYA_THREADS")
+ if not raw:
+ return None
+ try:
+ n = int(raw)
+ except ValueError:
+ return None
+ if n <= 0:
+ return None
+ import torch
+
+ torch.set_num_threads(n)
+ return n
+
+
+def build_router():
+ """Build a Router from the environment, preloading unless told otherwise."""
+ from .router import Router
+
+ _apply_thread_limit()
+ device = os.environ.get("LAYA_DEVICE") or None
+ models_env = os.environ.get("LAYA_MODELS", "").strip()
+ preload_names = [m.strip() for m in models_env.split(",") if m.strip()] or None
+ router = Router(device=device, auto_task_detection=_env_bool("LAYA_AUTO_TASK", False))
+ if _env_bool("LAYA_PRELOAD", True):
+ router.preload(preload_names)
+ return router
+
+
+def create_app(router: Optional[Any] = None):
+ """Build the FastAPI app. Pass a Router to inject one (tests); otherwise one
+ is built from the environment (and preloaded) at app-creation time."""
+ from fastapi import FastAPI, Header, HTTPException, Request
+
+ if router is None:
+ router = build_router()
+ api_key = os.environ.get("LAYA_API_KEY") or None
+
+ app = FastAPI(
+ title="laya-serve",
+ summary="Laya System-1 decisions over the TypeSafe Jev /v1/systemone protocol",
+ )
+
+ def _check_auth(authorization: Optional[str]) -> None:
+ if api_key is None:
+ return
+ if authorization != "Bearer " + api_key:
+ raise HTTPException(status_code=401, detail="invalid or missing bearer token")
+
+ @app.get("/health")
+ def health() -> Dict[str, Any]:
+ return {
+ "status": "ok",
+ "loaded": router.loaded,
+ "device": os.environ.get("LAYA_DEVICE") or "auto",
+ }
+
+ @app.post("/v1/systemone")
+ async def systemone(request: Request, authorization: Optional[str] = Header(default=None)):
+ _check_auth(authorization)
+ body = await request.json()
+ if not isinstance(body, dict) or "questions" not in body:
+ raise HTTPException(status_code=400, detail="request body must be an object with a 'questions' field")
+ state = body.get("state")
+ questions = body["questions"]
+ model = _resolve_model(body.get("model"))
+ try:
+ # Laya's result is already Jev-shaped: {model, answers, usage, routing}.
+ # hs-jev decodes `answers` and `usage` and ignores the rest.
+ return router.predict(state, questions, model=model)
+ except HTTPException:
+ raise
+ except Exception as e: # noqa: BLE001 -- surface model/tokenizer errors as 422
+ raise HTTPException(status_code=422, detail=str(e))
+
+ return app
+
+
+def main() -> None:
+ import uvicorn
+
+ uvicorn.run(
+ create_app(),
+ host=os.environ.get("LAYA_HOST", "0.0.0.0"),
+ port=int(os.environ.get("LAYA_PORT", "8000")),
+ log_level=os.environ.get("LAYA_LOG_LEVEL", "info"),
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/nix/laya-serve.nix b/nix/laya-serve.nix
new file mode 100644
index 00000000..f13d24b1
--- /dev/null
+++ b/nix/laya-serve.nix
@@ -0,0 +1,175 @@
+# NixOS module: run laya-serve (Laya's Jev-compatible /v1/systemone HTTP API)
+# as a hardened systemd service with CUDA access.
+#
+# The package is built from the host's own `pkgs` via ./package.nix (which pulls
+# the prebuilt CUDA torch, so the host needs `allowUnfree`). This deliberately
+# avoids an overlay, so the module also works on hosts that inject `pkgs` via
+# specialArgs and ignore module-level `nixpkgs.overlays`.
+{ config, lib, pkgs, ... }:
+let
+ cfg = config.services.laya-serve;
+
+ startScript = pkgs.writeShellScript "laya-serve-start" ''
+ set -eu
+ ${lib.optionalString (cfg.apiKeyFile != null) ''
+ export LAYA_API_KEY="$(cat "$CREDENTIALS_DIRECTORY/apikey")"
+ ''}
+ exec ${cfg.package}/bin/laya-serve
+ '';
+in
+{
+ options.services.laya-serve = {
+ enable = lib.mkEnableOption "Laya System-1 decision server (TypeSafe Jev-compatible HTTP API)";
+
+ package = lib.mkOption {
+ type = lib.types.package;
+ default = (pkgs.callPackage ./package.nix { }).laya-serve;
+ defaultText = lib.literalExpression "(pkgs.callPackage ./package.nix { }).laya-serve";
+ description = ''
+ The laya-serve package to run. Built from the host's own `pkgs` (needs
+ `allowUnfree` for the prebuilt CUDA torch), so it works regardless of how
+ the host provides `pkgs`.
+ '';
+ };
+
+ host = lib.mkOption {
+ type = lib.types.str;
+ default = "127.0.0.1";
+ description = "Address to bind. Use 0.0.0.0 to serve the LAN/Tailscale net.";
+ };
+
+ port = lib.mkOption {
+ type = lib.types.port;
+ default = 8000;
+ description = "TCP port to listen on.";
+ };
+
+ device = lib.mkOption {
+ type = lib.types.str;
+ default = "cuda";
+ example = "cpu";
+ description = "torch device for every checkpoint (cuda, cpu, cuda:0, ...).";
+ };
+
+ models = lib.mkOption {
+ type = lib.types.listOf (lib.types.enum [ "english" "multilingual" "typed-decisions" ]);
+ default = [ "english" "multilingual" "typed-decisions" ];
+ description = ''
+ Checkpoints to preload at startup. All three fit comfortably in a 24 GB
+ card (~1.16B params total), so the default keeps every one hot and makes
+ language routing free.
+ '';
+ };
+
+ preload = lib.mkOption {
+ type = lib.types.bool;
+ default = true;
+ description = "Build the checkpoints at startup rather than lazily on first request.";
+ };
+
+ threads = lib.mkOption {
+ type = lib.types.nullOr lib.types.ints.positive;
+ default = null;
+ example = 16;
+ description = ''
+ Cap torch intra-op threads for CPU inference (sets LAYA_THREADS and
+ OMP_NUM_THREADS). Ignored in practice on CUDA. Keep this at or below the
+ host's *physical* core count — oversubscribing the logical/hyperthread
+ count is a large latency regression. For single-request latency, a value
+ below the core count (e.g. 8-16) is often fastest; for batched
+ throughput, the physical core count is best. null leaves torch's default.
+ '';
+ };
+
+ autoTaskDetection = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ description = "Let the router auto-select the typed-decisions checkpoint when question ids match its workflows.";
+ };
+
+ apiKeyFile = lib.mkOption {
+ type = lib.types.nullOr lib.types.path;
+ default = null;
+ example = lib.literalExpression "config.age.secrets.laya-api-key.path";
+ description = ''
+ Path to a file containing a bearer token. When set, clients must send
+ `Authorization: Bearer <token>`. Read via systemd LoadCredential, so it
+ never lands in the store or the unit's environment.
+ '';
+ };
+
+ openFirewall = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ description = "Open `port` in the firewall.";
+ };
+
+ stateDirectory = lib.mkOption {
+ type = lib.types.str;
+ default = "laya-serve";
+ description = "Name under /var/lib for the Hugging Face weight cache (HF_HOME).";
+ };
+ };
+
+ config = lib.mkIf cfg.enable {
+ systemd.services.laya-serve = {
+ description = "Laya System-1 decision server (Jev-compatible)";
+ wantedBy = [ "multi-user.target" ];
+ after = [ "network-online.target" ];
+ wants = [ "network-online.target" ];
+
+ environment = {
+ LAYA_HOST = cfg.host;
+ LAYA_PORT = toString cfg.port;
+ LAYA_DEVICE = cfg.device;
+ LAYA_PRELOAD = if cfg.preload then "1" else "0";
+ LAYA_MODELS = lib.concatStringsSep "," cfg.models;
+ LAYA_AUTO_TASK = if cfg.autoTaskDetection then "1" else "0";
+ } // lib.optionalAttrs (cfg.threads != null) {
+ LAYA_THREADS = toString cfg.threads;
+ OMP_NUM_THREADS = toString cfg.threads;
+ } // {
+ HF_HOME = "/var/lib/${cfg.stateDirectory}/huggingface";
+ # torch-bin bundles its own CUDA runtime but still needs the host
+ # driver's libcuda.so.1 / libnvidia-ml.so, which NixOS exposes here.
+ LD_LIBRARY_PATH = "/run/opengl-driver/lib";
+ };
+
+ serviceConfig = {
+ ExecStart = startScript;
+ Restart = "on-failure";
+ RestartSec = 5;
+ # First start downloads ~1.2 GB of weights before it listens.
+ TimeoutStartSec = "600";
+
+ DynamicUser = true;
+ StateDirectory = cfg.stateDirectory;
+
+ # GPU: keep the nvidia device nodes visible to the sandbox.
+ PrivateDevices = false;
+ DeviceAllow = [
+ "/dev/nvidia0 rw"
+ "/dev/nvidiactl rw"
+ "/dev/nvidia-uvm rw"
+ "/dev/nvidia-uvm-tools rw"
+ "/dev/nvidia-modeset rw"
+ ];
+
+ # Hardening (kept compatible with CUDA device access).
+ NoNewPrivileges = true;
+ ProtectSystem = "strict";
+ ProtectHome = true;
+ PrivateTmp = true;
+ ProtectControlGroups = true;
+ ProtectKernelModules = true;
+ RestrictNamespaces = true;
+ RestrictSUIDSGID = true;
+ LockPersonality = true;
+ } // lib.optionalAttrs (cfg.apiKeyFile != null) {
+ LoadCredential = [ "apikey:${cfg.apiKeyFile}" ];
+ };
+ };
+
+ networking.firewall.allowedTCPPorts = lib.optional cfg.openFirewall cfg.port;
+ };
+}
diff --git a/nix/package.nix b/nix/package.nix
new file mode 100644
index 00000000..b3a895f5
--- /dev/null
+++ b/nix/package.nix
@@ -0,0 +1,33 @@
+# The laya python package + a laya-serve runner, built from whatever `pkgs` is
+# in scope. Factored out so both the flake's packages/overlay AND the NixOS
+# module can build it the same way -- the module cannot rely on an overlay,
+# because hosts that inject `pkgs` via specialArgs ignore module-level
+# `nixpkgs.overlays`.
+{ lib, python3, python3Packages, writeShellScriptBin }:
+let
+ laya = python3Packages.buildPythonPackage {
+ pname = "laya";
+ version = "0.3.4";
+ src = ../.;
+ format = "setuptools";
+ propagatedBuildInputs = with python3Packages; [
+ torch-bin # prebuilt CUDA wheel -- no source build
+ transformers
+ safetensors
+ huggingface-hub
+ numpy
+ ];
+ # Every test loads a checkpoint from the Hub -> needs network + a GPU.
+ doCheck = false;
+ # serve.py defers its fastapi/uvicorn imports, so this stays honest without
+ # dragging the web stack into the base library.
+ pythonImportsCheck = [ "laya" "laya.serve" ];
+ };
+ pyEnv = python3.withPackages (ps: [ laya ps.fastapi ps.uvicorn ]);
+in
+{
+ inherit laya;
+ laya-serve = writeShellScriptBin "laya-serve" ''
+ exec ${pyEnv}/bin/python -m laya.serve "$@"
+ '';
+}
diff --git a/pyproject.toml b/pyproject.toml
index ef616783..ad0e9a14 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -36,6 +36,16 @@ dependencies = [
# assets/, research/ and notebooks/ sit in the root, so auto-discovery bails out.
packages = ["laya"]
+[project.optional-dependencies]
+# `pip install laya[serve]` adds the HTTP server (laya.serve / the laya-serve CLI).
+serve = [
+ "fastapi>=0.110.0",
+ "uvicorn>=0.27.0",
+]
+
+[project.scripts]
+laya-serve = "laya.serve:main"
+
[project.urls]
Homepage = "https://huggingface.co/convaiinnovations/laya"
Demo = "https://huggingface.co/spaces/convaiinnovations/laya-demo"
diff --git a/tests/test_serve.py b/tests/test_serve.py
new file mode 100644
index 00000000..8c78f4ca
--- /dev/null
+++ b/tests/test_serve.py
@@ -0,0 +1,108 @@
+"""Server-shim tests: verify the Jev /v1/systemone surface without a GPU.
+
+A fake Router is injected so nothing loads a checkpoint; we only assert that the
+HTTP layer maps requests/responses and enforces auth as hs-jev expects.
+"""
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+from fastapi.testclient import TestClient # noqa: E402
+
+from laya.serve import _apply_thread_limit, _env_bool, _resolve_model, create_app # noqa: E402
+
+
+class FakeRouter:
+ """Records the last predict() call and returns a Jev-shaped payload."""
+
+ loaded = ["english"]
+
+ def __init__(self):
+ self.calls = []
+
+ def predict(self, state, questions, model=None):
+ self.calls.append({"state": state, "questions": questions, "model": model})
+ return {
+ "model": "laya-rl-agent",
+ "answers": {
+ "dept": {"type": "choice", "choice": "billing",
+ "probabilities": {"billing": 0.94, "tech": 0.06}, "confidence": 0.94},
+ },
+ "usage": {"input_tokens": 42, "output_tokens": 0},
+ "routing": {"model": "english", "reason": "English Latin text"},
+ }
+
+
+def _client(monkeypatch, api_key=None):
+ if api_key is None:
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ else:
+ monkeypatch.setenv("LAYA_API_KEY", api_key)
+ fake = FakeRouter()
+ return TestClient(create_app(router=fake)), fake
+
+
+REQ = {
+ "model": "jev-1", # a non-Laya model id -> should be ignored, router auto-routes
+ "state": {"body": "billed twice, refund please"},
+ "questions": {"dept": {"type": "choice", "instructions": "which team?",
+ "criteria": {"billing": None, "tech": None}}},
+}
+
+
+def test_predict_passthrough_shape(monkeypatch):
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone", json=REQ)
+ assert r.status_code == 200
+ body = r.json()
+ # exactly the fields hs-jev's Response/Usage decoders require
+ assert set(["answers", "usage"]).issubset(body)
+ assert body["usage"] == {"input_tokens": 42, "output_tokens": 0}
+ assert body["answers"]["dept"]["choice"] == "billing"
+ # unknown model id was dropped -> router asked to auto-route
+ assert fake.calls[0]["model"] is None
+
+
+def test_known_model_is_honoured(monkeypatch):
+ client, fake = _client(monkeypatch)
+ client.post("/v1/systemone", json={**REQ, "model": "multilingual"})
+ assert fake.calls[0]["model"] == "multilingual"
+
+
+def test_missing_questions_is_400(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone", json={"state": "hi"})
+ assert r.status_code == 400
+
+
+def test_auth_required_when_key_set(monkeypatch):
+ client, _ = _client(monkeypatch, api_key="s3cret")
+ assert client.post("/v1/systemone", json=REQ).status_code == 401
+ ok = client.post("/v1/systemone", json=REQ, headers={"Authorization": "Bearer s3cret"})
+ assert ok.status_code == 200
+
+
+def test_health(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.get("/health")
+ assert r.status_code == 200 and r.json()["status"] == "ok"
+
+
+def test_helpers():
+ assert _resolve_model("multilingual") == "multilingual"
+ assert _resolve_model("jev-1") is None
+ assert _resolve_model(None) is None
+ import os
+ os.environ.pop("X_FLAG", None)
+ assert _env_bool("X_FLAG", True) is True
+
+
+def test_thread_limit(monkeypatch):
+ monkeypatch.delenv("LAYA_THREADS", raising=False)
+ assert _apply_thread_limit() is None # unset -> no-op, no torch import
+ for bad in ("0", "-4", "abc", ""):
+ monkeypatch.setenv("LAYA_THREADS", bad)
+ assert _apply_thread_limit() is None
+ monkeypatch.setenv("LAYA_THREADS", "8")
+ assert _apply_thread_limit() == 8
+ import torch
+ assert torch.get_num_threads() == 8