Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(serve): attach Server-Timing and X-Inference-Time-Ms headers to POST /v1/systemone

Peter Friese639dedeb96ab
diff --git a/laya/serve.py b/laya/serve.py
index c0fc1bc3..ec4fca2d 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -35,6 +35,7 @@ and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.
import hmac
import json
import os
+import time
from contextlib import asynccontextmanager
from typing import Any, Dict, Optional
@@ -272,8 +273,18 @@ def create_app(router: Optional[Any] = None):
# hs-jev decodes `answers` and `usage` and ignores the rest.
async with gate:
loop = asyncio.get_running_loop()
- return await loop.run_in_executor(
+ t0 = time.perf_counter()
+ result = await loop.run_in_executor(
pool, lambda: router.predict(state, questions, model=model))
+ infer_ms = (time.perf_counter() - t0) * 1000.0
+ from fastapi.responses import JSONResponse
+ return JSONResponse(
+ content=result,
+ headers={
+ "Server-Timing": f"inference;dur={infer_ms:.2f}",
+ "X-Inference-Time-Ms": f"{infer_ms:.2f}"
+ }
+ )
except HTTPException:
raise
except ValueError as e:
diff --git a/tests/test_serve.py b/tests/test_serve.py
index 566660bb..b3421da9 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -315,3 +315,19 @@ def test_health_stays_available_during_inference(monkeypatch):
assert seen["slow"] == 200
assert seen["health"] == 200 and seen["payload"]["status"] == "ok"
assert fake.threads and "MainThread" not in fake.threads, fake.threads
+
+
+def test_inference_timing_headers():
+ """POST /v1/systemone returns Server-Timing and X-Inference-Time-Ms headers."""
+ router = FakeRouter()
+ client = TestClient(create_app(router=router))
+ res = client.post("/v1/systemone", json={
+ "state": "test timing",
+ "questions": {"dept": {"type": "choice", "instructions": "which?", "criteria": {"billing": "invoices"}}}
+ })
+ assert res.status_code == 200
+ assert "Server-Timing" in res.headers
+ assert res.headers["Server-Timing"].startswith("inference;dur=")
+ assert "X-Inference-Time-Ms" in res.headers
+ dur = float(res.headers["X-Inference-Time-Ms"])
+ assert dur >= 0.0