Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #624 from Jayanth-reflex/fix/serve-tcp-nodelay

Nandakishorca59ce2785fe
diff --cc tests/test_serve.py
index e985611e,2604bcf1..dab066ab
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@@ -836,72 -531,37 +836,108 @@@ def test_admission_slot_is_released_aft
assert client.post("/v1/systemone", json=REQ).status_code == 200
+ def test_accepted_connections_set_tcp_nodelay(monkeypatch):
+ """#620: asyncio skips TCP_NODELAY when an accepted socket reports proto 0, as it
+ does on macOS and Windows, so Nagle held back small responses by about 50 ms."""
+ import asyncio
+ import socket
+
+ import uvicorn
+
+ import laya.serve
+
+ captured = {}
+ monkeypatch.setattr(uvicorn, "run", lambda app, **kwargs: captured.update(kwargs))
+ monkeypatch.setattr(laya.serve, "create_app", lambda: None)
+ laya.serve.main()
+
+ async def drive():
+ config = uvicorn.Config(create_app(router=FakeRouter()), host="127.0.0.1", port=0,
+ http=captured["http"], log_level="warning")
+ server = uvicorn.Server(config)
+ serving = asyncio.ensure_future(server.serve())
+ while not server.started:
+ await asyncio.sleep(0.01)
+ port = server.servers[0].sockets[0].getsockname()[1]
+ _, writer = await asyncio.open_connection("127.0.0.1", port)
+ while not server.server_state.connections:
+ await asyncio.sleep(0.01)
+ (conn,) = server.server_state.connections
+ nodelay = conn.transport.get_extra_info("socket").getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
+ writer.close()
+ server.should_exit = True
+ await serving
+ return nodelay
+
+ assert asyncio.run(drive())
++
++
+def test_no_budget_keeps_the_call_unchanged(monkeypatch):
+ """An injected router whose predict() takes no kwargs continues to work when body sends no budget."""
+ client, fake = _client(monkeypatch)
+ for body in (REQ, dict(REQ, max_len=None, head_max_len=None)):
+ assert client.post("/v1/systemone", json=body).status_code == 200
+ assert len(fake.calls) == 2
+
+
+def test_token_budget_forwarded(monkeypatch):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 4096, "head_max_len": 256})
+ assert r.status_code == 200
+ assert fake.calls[0]["max_len"] == 4096
+ assert fake.calls[0]["head_max_len"] == 256
+
+
[email protected]("bad_budget", ["fast", True, 3.14])
+def test_token_budget_validation_type(monkeypatch, bad_budget):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": bad_budget})
+ assert r.status_code == 422
+ assert "must be an integer" in r.json()["detail"]
+
+
[email protected]("bad_val", [0, -10])
+def test_token_budget_validation_positive(monkeypatch, bad_val):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": bad_val})
+ assert r.status_code == 422
+ assert "must be a positive integer" in r.json()["detail"]
+
+
+def test_token_budget_exceeds_server_cap(monkeypatch):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 9000})
+ assert r.status_code == 422
+ assert "exceeds server limit" in r.json()["detail"]
+
+
+def test_token_budget_head_max_len_equal_to_max_len(monkeypatch):
+ """Core accepts head_max_len == max_len; serve forwards both without artificial restriction."""
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 512, "head_max_len": 512})
+ assert r.status_code == 200
+ assert fake.calls[0]["max_len"] == 512
+ assert fake.calls[0]["head_max_len"] == 512
+
+
+def test_token_budget_env_cap_override(monkeypatch):
+ monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", "2048")
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 4096})
+ assert r.status_code == 422
+ assert "exceeds server limit" in r.json()["detail"]
+
+ r2 = client.post("/v1/systemone", json={**REQ, "max_len": 2048})
+ assert r2.status_code == 200
+ assert fake.calls[0]["max_len"] == 2048
+
+
+def test_resolve_max_token_budget_fallback(monkeypatch, caplog):
+ monkeypatch.delenv("LAYA_MAX_TOKEN_BUDGET", raising=False)
+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET
+ for bad in ("abc", "-10", "0"):
+ monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", bad)
+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET
+ assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.text
+ assert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text
+