Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

10.6 KB
Raw Blame
"""Run the installed SDK against a separate Python process with real offline inference."""
import argparse
import json
import os
from pathlib import Path
import queue
import subprocess
import sys
import tempfile
import threading
import time

from laya_client import LayaAPIError, LayaClient, LayaConnectionError


def check_batch(result, expected, strict=False):
    assert len(result["results"]) == len(expected)
    for item, direct in zip(result["results"], expected):
        if strict:
            fields = {"choice": ("type", "choice", "probabilities", "confidence"),
                      "score": ("type", "score", "probabilities", "confidence", "legend"),
                      "noul": ("type", "noul")}
            projected = {
                "model": direct["model"],
                "answers": {qid: {k: answer[k] for k in fields[answer["type"]]}
                            for qid, answer in direct["answers"].items()},
                "usage": {k: direct["usage"][k] for k in ("input_tokens", "output_tokens")},
            }
            assert item == projected, (item, projected)
        else:
            assert item["model"] == direct["model"]
            for qid, answer in direct["answers"].items():
                for key, value in answer.items():
                    assert item["answers"][qid][key] == value, (qid, key)
            for key, value in direct["usage"].items():
                assert item["usage"][key] == value, key
            assert item["routing"] == direct["routing"]
    for key in ("input_tokens", "output_tokens"):
        assert result["total_usage"][key] == sum(item["usage"][key] for item in expected), key


def check_batch_http(client, anonymous, fixture, strict, legacy):
    questions = fixture["questions"]
    if legacy:
        # v0.4.7 has a batch endpoint. One-state passes give its unchanged fixture a parity oracle.
        states = ["hello", fixture["state"], ["hello"]]
        expected = [client.predict(state, questions, model="english") for state in states]
        check_batch(client.predict_batch(states, questions, model="english", batch_size=1), expected)
        check_batch(client.predict_batch(states[:1], questions, model="english"), expected[:1])
    else:
        for case in fixture["batch_cases"]:
            result = client.predict_batch(case["states"], questions, model="english", **case["controls"])
            check_batch(result, case["expected"], strict)
        labels = {str(i): "Option %d" % i for i in range(5)}
        large = {"large": {"type": "choice", "instructions": "Pick one", "criteria": labels}}
        states = fixture["batch_cases"][0]["states"]
        # Tournament execution is per state on the server, even when batch shape controls are set.
        expected = [client.predict(state, large, model="english") for state in states]
        check_batch(client.predict_batch(states, large, model="english",
                                         batch_size=2, sort_by_length=True), expected, strict)

    small = {"refund": questions["refund"]}
    result = client.predict_batch(["hello"] * 64, small, model="english", batch_size=8)
    assert len(result["results"]) == 64
    for key in ("input_tokens", "output_tokens"):
        assert result["total_usage"][key] == sum(item["usage"][key] for item in result["results"])
    for caller, states, controls, status in (
        (client, ["hello"] * 65, {}, 413),
        (client, ["hello", "x" * 50_001], {}, 413),
        (client, ["hello"], {"model": "unpublished/checkpoint"}, 422),
        (anonymous, ["hello"], {}, 401),
    ):
        try:
            caller.predict_batch(states, small, **controls)
        except LayaAPIError as error:
            assert error.status_code == status, (error.status_code, status, error.detail)
        else:
            raise AssertionError("Batch request should have failed with HTTP %d" % status)


def run(server_python, fixture_path, strict=False, legacy=False):
    with tempfile.TemporaryDirectory(prefix="laya-sdk-live-") as directory:
        root = Path(directory)
        command = [str(server_python), "-u", str(fixture_path)]
        if not legacy:
            command.extend(["--tournament", "--batch"])
            if strict:
                command.append("--strict")
        env = dict(os.environ)
        env.pop("PYTHONPATH", None)
        env.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", LAYA_TOURNAMENT_GROUP="0",
                   LAYA_JEV_STRICT="0", LAYA_PRELOAD="0", LAYA_IDLE_UNLOAD_SECONDS="0")
        with (root / "stderr.txt").open("w+", encoding="utf-8", errors="replace") as stderr:
            server = subprocess.Popen(command, cwd=root, env=env, stdout=subprocess.PIPE,
                                      stderr=stderr, text=True, encoding="utf-8")
            lines = queue.Queue()

            def read_lines():
                for line in server.stdout:
                    if line.startswith("LAYA_TEST_SERVER="):
                        lines.put(json.loads(line.split("=", 1)[1]))
                        return
                lines.put(None)

            reader = threading.Thread(target=read_lines, daemon=True)
            reader.start()
            try:
                fixture = lines.get(timeout=90)
                if fixture is None:
                    raise RuntimeError("Server exited before producing its fixture")
                client = LayaClient(base_url=fixture["baseURL"], api_key="integration-test", timeout=10)
                deadline = time.monotonic() + 15
                while True:
                    try:
                        client.health(timeout=0.5)
                        break
                    except LayaConnectionError:
                        if time.monotonic() >= deadline or server.poll() is not None:
                            raise
                        time.sleep(0.05)
                result = client.predict(fixture["state"], fixture["questions"], model="english")
                expected = fixture["expected"]
                assert result["model"] == expected["model"]
                if strict:
                    fields = {"choice": ("type", "choice", "probabilities", "confidence"),
                              "score": ("type", "score", "probabilities", "confidence", "legend"),
                              "noul": ("type", "noul")}
                    projected = {
                        "model": expected["model"],
                        "answers": {qid: {k: answer[k] for k in fields[answer["type"]]}
                                    for qid, answer in expected["answers"].items()},
                        "usage": {k: expected["usage"][k] for k in ("input_tokens", "output_tokens")},
                    }
                    assert result == projected, (result, projected)
                    gated = client.predict(fixture["state"], fixture["questions"], min_confidence=0.99)
                    assert all("abstention" not in a for a in gated["answers"].values())
                else:
                    for qid, answer in expected["answers"].items():
                        for key, value in answer.items():
                            assert result["answers"][qid][key] == value, (qid, key)
                    for key, value in expected["usage"].items():
                        assert result["usage"][key] == value, key
                    assert result["routing"]["model"] == "english"
                    auto = client.predict(fixture["state"], fixture["questions"])
                    assert auto["routing"]["model"] == "multilingual"
                    gated = client.predict(fixture["state"], fixture["questions"], lang="en",
                                           max_len=128, head_max_len=64, min_confidence=0.0)
                    assert gated["routing"]["model"] == "english"
                    assert all(a["abstention"] == "passed" for a in gated["answers"].values())

                if not legacy:
                    labels = {str(i): "Option %d" % i for i in range(5)}
                    answer = client.predict("hello", {"large": {
                        "type": "choice", "instructions": "Pick one", "criteria": labels,
                    }})["answers"]["large"]
                    assert set(answer["probabilities"]) == set(labels)
                    assert answer["choice"] in labels
                    if strict:
                        assert "x_tournament" not in answer
                    else:
                        finalists = answer["x_tournament"]["finalists"]
                        assert 0 < len(finalists) <= 2
                        assert all(p == 0.0 for k, p in answer["probabilities"].items() if k not in finalists)

                anonymous = LayaClient(base_url=fixture["baseURL"], api_key="wrong")
                assert anonymous.health() == {"status": "ok"}
                try:
                    anonymous.predict("hello", fixture["questions"])
                except LayaAPIError as error:
                    assert error.status_code == 401
                else:
                    raise AssertionError("Unauthenticated prediction succeeded")
                try:
                    client.predict("x" * 50_001, fixture["questions"])
                except LayaAPIError as error:
                    assert error.status_code == 413
                else:
                    raise AssertionError("Server request limit was bypassed")
                check_batch_http(client, anonymous, fixture, strict, legacy)
                print("PASS real HTTP single and batch inference, order, controls, auth and limits: "
                      + ("legacy fixture" if legacy else "strict" if strict else "extended/tournament"), flush=True)
            except BaseException:
                stderr.flush()
                stderr.seek(0)
                print(stderr.read(), file=sys.stderr)
                raise
            finally:
                if server.poll() is None:
                    server.terminate()
                    try:
                        server.wait(timeout=5)
                    except subprocess.TimeoutExpired:
                        server.kill()
                        server.wait(timeout=5)
                server.stdout.close()
                reader.join(timeout=1)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--server-python", required=True, type=Path)
    parser.add_argument("--server-fixture", type=Path,
                        default=Path(__file__).resolve().parents[3] / "tests" / "sdk_server_fixture.py")
    parser.add_argument("--legacy-fixture", action="store_true")
    args = parser.parse_args()
    run(args.server_python.resolve(), args.server_fixture.resolve(), legacy=args.legacy_fixture)
    if not args.legacy_fixture:
        run(args.server_python.resolve(), args.server_fixture.resolve(), strict=True)