LLM inference in C/C++
C++ ggml
tests : speed-up server test suite 3x (#26734)
Georgi Gerganov1621a3d388b7
diff --git a/tools/server/tests/conftest.py b/tools/server/tests/conftest.pyindex c7ed775968..5dfde40796 100644--- a/tools/server/tests/conftest.py+++ b/tools/server/tests/conftest.py@@ -15,7 +15,7 @@ def stop_server_after_each_test():server.stop()[email protected](scope="module", autouse=True)-def do_something():[email protected](scope="session", autouse=True)+def load_server_presets():# this will be run once per test session, before any testsServerPreset.load_all()diff --git a/tools/server/tests/tests.sh b/tools/server/tests/tests.shindex 709b5841aa..8d6681193d 100755--- a/tools/server/tests/tests.sh+++ b/tools/server/tests/tests.sh@@ -14,10 +14,10 @@ fiif [ $# -lt 1 ]thenif [[ "${SLOW_TESTS:-0}" == 1 ]]; then- pytest -v -x+ pytest --durations=30 -v -xelse- pytest -v -x -m "not slow"+ pytest --durations=30 -v -x -m "not slow"fielse- pytest "$@"+ pytest --durations=30 "$@"fidiff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.pyindex c503ee3420..5ab62666ce 100644--- a/tools/server/tests/unit/test_router.py+++ b/tools/server/tests/unit/test_router.py@@ -85,7 +85,7 @@ def _wait_for_model_status(model_id: str, desired: set[str], timeout: int = 60)last_status = _get_model_status(model_id)if last_status in desired:return last_status- time.sleep(1)+ time.sleep(0.01)raise AssertionError(f"Timed out waiting for {model_id} to reach {desired}, last status: {last_status}")@@ -460,7 +460,7 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: iwhile time.time() < deadline:if any(e.get("event") == event_type and e.get("model") == model for e in collected):return True- time.sleep(0.5)+ time.sleep(0.01)return Falsediff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex 8f3a986424..3416f0bfde 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -306,6 +306,7 @@ class ServerProcess:# wait for server to startstart_time = time.time()+ last_print_time = start_timewhile time.time() - start_time < timeout_seconds:try:response = self.make_request("GET", "/health", headers={@@ -320,8 +321,10 @@ class ServerProcess:if self.process.poll() is not None:raise RuntimeError(f"Server process died with return code {self.process.returncode}")- print(f"Waiting for server to start...")- time.sleep(0.5)+ if time.time() - last_print_time >= 1.0:+ print(f"Waiting for server to start...")+ last_print_time = time.time()+ time.sleep(0.01)raise TimeoutError(f"Server did not start within {timeout_seconds} seconds")def stop(self) -> None: