Personal AI, On Personal Devices
Python
fix(evals): honor --base-url/--api-key for first-party eval backends (#535)
Jon Saad-Falcon50993dfa4dae
diff --git a/src/openjarvis/cli/compose_cmd.py b/src/openjarvis/cli/compose_cmd.pyindex e7af878c..4995fa50 100644--- a/src/openjarvis/cli/compose_cmd.py+++ b/src/openjarvis/cli/compose_cmd.py@@ -332,7 +332,7 @@ def compose_bench(for i, rc in enumerate(run_configs, 1):console.print(f"\n[bold]Run {i}/{len(run_configs)}:[/bold] {rc.benchmark}")try:- summary = _run_single(rc, console=console)+ summary = _run_single(rc, console=console, suite_mode=True)results_table.add_row(rc.benchmark,f"{summary.accuracy:.4f}",diff --git a/src/openjarvis/cli/eval_cmd.py b/src/openjarvis/cli/eval_cmd.pyindex f2e9cde6..ed84af4a 100644--- a/src/openjarvis/cli/eval_cmd.py+++ b/src/openjarvis/cli/eval_cmd.py@@ -61,6 +61,12 @@ KNOWN_BENCHMARKS = {KNOWN_BACKENDS = {"jarvis-direct": "Engine-level inference (local or cloud)","jarvis-agent": "Agent-level inference with tool calling",+ "hermes": "Real Hermes Agent (Nous Research) via subprocess",+ "openclaw": "Real OpenClaw via Node subprocess",+ "terminalbench-native": (+ "TerminalBench V2.1 via terminal-bench Harness "+ "(selected with -b terminalbench-native)"+ ),}@@ -146,7 +152,9 @@ def eval_list() -> None:"base_url",default=None,help=(- "OpenAI-compat endpoint URL for hermes/openclaw backends "+ "OpenAI-compatible endpoint for the model under eval. Required for "+ "hermes/openclaw; for jarvis-direct/jarvis-agent/terminalbench-native "+ "it bypasses engine discovery and targets this URL directly ""(env: JARVIS_BACKEND_BASE_URL)."),)@@ -154,7 +162,11 @@ def eval_list() -> None:"--api-key","api_key",default=None,- help=("API key for the hermes/openclaw endpoint (env: JARVIS_BACKEND_API_KEY)."),+ help=(+ "API key for the --base-url endpoint, sent as a Bearer token. "+ "Required for hermes/openclaw; optional for first-party backends "+ "(env: JARVIS_BACKEND_API_KEY)."+ ),)@click.option("--agent",@@ -347,7 +359,7 @@ def eval_run(f"{rc.benchmark} / {rc.model}")try:- summary = _run_single(rc, console=console)+ summary = _run_single(rc, console=console, suite_mode=True)console.print(f" [green]{summary.accuracy:.4f}[/green] "f"({summary.correct}/{summary.scored_samples})"@@ -399,8 +411,10 @@ def eval_run(sheets_spreadsheet_id=sheets_spreadsheet_id,sheets_worksheet=sheets_worksheet,sheets_credentials_path=sheets_credentials_path,- # Spec §6.2 — for hermes/openclaw external backends. Falls back to env vars- # so users can also set JARVIS_BACKEND_BASE_URL/JARVIS_BACKEND_API_KEY.+ # OpenAI-compatible endpoint for the model under eval. Required for+ # hermes/openclaw (Spec §6.2); honored by first-party backends too on+ # this CLI path. Falls back to env vars so users can also set+ # JARVIS_BACKEND_BASE_URL/JARVIS_BACKEND_API_KEY.base_url=base_url or os.environ.get("JARVIS_BACKEND_BASE_URL"),api_key=api_key or os.environ.get("JARVIS_BACKEND_API_KEY"),)diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.pyindex edbf6d44..5f767478 100644--- a/src/openjarvis/engine/_openai_compat.py+++ b/src/openjarvis/engine/_openai_compat.py@@ -28,12 +28,31 @@ class _OpenAICompatibleEngine(InferenceEngine):_default_host: str = "http://localhost:8000"_api_prefix: str = "/v1"- def __init__(self, host: str | None = None, *, timeout: float = 600.0) -> None:+ def __init__(+ self,+ host: str | None = None,+ *,+ api_key: str | None = None,+ timeout: float = 600.0,+ ) -> None:import os- env_key = f"{self.engine_id.upper()}_HOST"- self._host = (host or os.environ.get(env_key) or self._default_host).rstrip("/")- self._client = httpx.Client(base_url=self._host, timeout=timeout)+ # Sanitize the engine id for env-var lookup ("openai-compat" ->+ # "OPENAI_COMPAT_..."); shells cannot set hyphenated variable names.+ env_prefix = self.engine_id.upper().replace("-", "_")+ self._host = (+ host or os.environ.get(f"{env_prefix}_HOST") or self._default_host+ ).rstrip("/")+ # Bearer auth for endpoints started with e.g. ``vllm serve --api-key``.+ # Setting it on the client covers generate/stream/stream_full/+ # list_models/health alike; ``None`` keeps requests header-free.+ self._api_key = api_key or os.environ.get(f"{env_prefix}_API_KEY") or None+ headers = (+ {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None+ )+ self._client = httpx.Client(+ base_url=self._host, timeout=timeout, headers=headers+ )# -- InferenceEngine interface ------------------------------------------diff --git a/src/openjarvis/engine/openai_compat_engines.py b/src/openjarvis/engine/openai_compat_engines.pyindex 6d191f3d..1c2d178e 100644--- a/src/openjarvis/engine/openai_compat_engines.py+++ b/src/openjarvis/engine/openai_compat_engines.py@@ -1,5 +1,7 @@"""Data-driven registration of OpenAI-compatible inference engines."""+from __future__ import annotations+from openjarvis.core.registry import EngineRegistryfrom openjarvis.engine._openai_compat import _OpenAICompatibleEngine@@ -25,4 +27,35 @@ for _key, (_cls_name, _default_host, _api_prefix) in _ENGINES.items():EngineRegistry.register(_key)(_cls)globals()[_cls_name] = _cls-__all__ = [name for name, _, _ in _ENGINES.values()]++def normalize_openai_base_url(url: str) -> str:+ """Strip a single trailing ``/v1`` segment from a user-supplied base URL.++ Users habitually pass ``http://host:8000/v1`` (the full OpenAI-compatible+ prefix); the engine's ``_api_prefix`` re-appends ``/v1`` to every request+ path, so a trailing copy would double up as ``/v1/v1``. Only a literal+ trailing ``/v1`` is stripped — proxy/gateway path prefixes are preserved.+ """+ base = url.rstrip("/")+ if base.endswith("/v1"):+ base = base[: -len("/v1")]+ return base+++class OpenAICompatEngine(_OpenAICompatibleEngine):+ """Generic engine for an explicitly-provided OpenAI-compatible endpoint.++ Deliberately NOT registered in ``EngineRegistry``: it is only ever+ constructed with an explicit host (e.g. ``jarvis eval --base-url``), so+ registering it would just add a useless localhost discovery probe and+ interact with the per-test registry wipe.+ """++ engine_id = "openai-compat"+ _api_prefix = "/v1"+++__all__ = [name for name, _, _ in _ENGINES.values()] + [+ "OpenAICompatEngine",+ "normalize_openai_base_url",+]diff --git a/src/openjarvis/evals/backends/_endpoint_util.py b/src/openjarvis/evals/backends/_endpoint_util.pynew file mode 100644index 00000000..cf60c495--- /dev/null+++ b/src/openjarvis/evals/backends/_endpoint_util.py@@ -0,0 +1,52 @@+"""Shared helper for targeting an explicit OpenAI-compatible endpoint.++Used by the first-party eval backends (jarvis-direct, jarvis-agent) when+``--base-url`` is given: the eval must use exactly that endpoint, with no+silent fallback to whatever other engine discovery happens to find.+"""++from __future__ import annotations++import logging+from typing import Optional++logger = logging.getLogger(__name__)+++def build_endpoint_engine(+ base_url: str,+ api_key: Optional[str] = None,+ engine_key: Optional[str] = None,+):+ """Construct an :class:`OpenAICompatEngine` pinned to ``base_url``.++ Pre-flight health-checks the endpoint and raises a loud, actionable+ error when it is unreachable — engine discovery is never consulted.+ """+ from openjarvis.engine.openai_compat_engines import (+ OpenAICompatEngine,+ normalize_openai_base_url,+ )++ if engine_key:+ logger.warning(+ "Both an engine key (%r) and base_url (%r) were given; "+ "base_url wins — targeting the endpoint directly.",+ engine_key,+ base_url,+ )+ host = normalize_openai_base_url(base_url)+ engine = OpenAICompatEngine(host=host, api_key=api_key)+ if not engine.health():+ engine.close()+ raise RuntimeError(+ f"--base-url endpoint not reachable: {base_url} "+ f"(GET {host}/v1/models failed). Is an OpenAI-compatible server "+ "(e.g. `vllm serve`) running at that address? If it requires "+ "authentication (HTTP 401), pass --api-key or set "+ "JARVIS_BACKEND_API_KEY."+ )+ return engine+++__all__ = ["build_endpoint_engine"]diff --git a/src/openjarvis/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.pyindex 8f876bb4..ef5625da 100644--- a/src/openjarvis/evals/backends/jarvis_agent.py+++ b/src/openjarvis/evals/backends/jarvis_agent.py@@ -31,6 +31,8 @@ class JarvisAgentBackend(InferenceBackend):max_turns: Optional[int] = None,skills_enabled: bool = True,overlay_dir: Optional[Path] = None,+ base_url: Optional[str] = None,+ api_key: Optional[str] = None,) -> None:from openjarvis.system import SystemBuilder@@ -40,7 +42,17 @@ class JarvisAgentBackend(InferenceBackend):self._gpu_metrics = gpu_metricsbuilder = SystemBuilder()- if engine_key:+ if base_url:+ # Explicit endpoint targeting (--base-url): pin the eval to+ # exactly this OpenAI-compatible endpoint. Fails fast if it is+ # unreachable; never falls back to a discovered engine.+ from openjarvis.evals.backends._endpoint_util import (+ build_endpoint_engine,+ )++ engine = build_endpoint_engine(base_url, api_key, engine_key)+ builder.engine_instance(engine, key=engine_key or "openai-compat")+ elif engine_key:builder.engine(engine_key)if model:builder.model(model)diff --git a/src/openjarvis/evals/backends/jarvis_direct.py b/src/openjarvis/evals/backends/jarvis_direct.pyindex d0195882..b90ba5cd 100644--- a/src/openjarvis/evals/backends/jarvis_direct.py+++ b/src/openjarvis/evals/backends/jarvis_direct.py@@ -24,6 +24,8 @@ class JarvisDirectBackend(InferenceBackend):engine_key: Optional[str] = None,telemetry: bool = False,gpu_metrics: bool = False,+ base_url: Optional[str] = None,+ api_key: Optional[str] = None,) -> None:from openjarvis.system import SystemBuilder@@ -31,7 +33,17 @@ class JarvisDirectBackend(InferenceBackend):self._gpu_metrics = gpu_metricsbuilder = SystemBuilder()- if engine_key:+ if base_url:+ # Explicit endpoint targeting (--base-url): pin the eval to+ # exactly this OpenAI-compatible endpoint. Fails fast if it is+ # unreachable; never falls back to a discovered engine.+ from openjarvis.evals.backends._endpoint_util import (+ build_endpoint_engine,+ )++ engine = build_endpoint_engine(base_url, api_key, engine_key)+ builder.engine_instance(engine, key=engine_key or "openai-compat")+ elif engine_key:builder.engine(engine_key)# Propagate gpu_metrics to the runtime config so SystemBuilder# creates an EnergyMonitor / GpuMonitor for the InstrumentedEngine.diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.pyindex 84e804ce..94715db7 100644--- a/src/openjarvis/evals/cli.py+++ b/src/openjarvis/evals/cli.py@@ -183,14 +183,28 @@ def _build_backend(max_turns: Optional[int] = None,base_url: Optional[str] = None,api_key: Optional[str] = None,+ first_party_endpoint: bool = True,):"""Construct the appropriate backend.- For "hermes" and "openclaw" backends, ``base_url`` and ``api_key`` are- REQUIRED — these foreign frameworks need an OpenAI-compatible endpoint- to send model calls to. Pass them via the eval config's- ``[backend.external]`` section or env vars.+ ``base_url``/``api_key`` point at the OpenAI-compatible endpoint serving+ the model under eval:++ - For "hermes" and "openclaw" they are REQUIRED — these foreign+ frameworks always call out to an external endpoint.+ - "jarvis-direct" and "jarvis-agent" honor them when+ ``first_party_endpoint`` is True (the CLI ``--base-url`` path): the+ eval targets exactly that endpoint — no engine-discovery fallback —+ and fails fast if it is unreachable. Suite mode passes+ ``first_party_endpoint=False`` so the suite TOML's+ ``[backend.external]`` section stays scoped to hermes/openclaw+ (extending it to first-party backends is explicitly deferred)."""+ if not first_party_endpoint:+ fp_base_url = fp_api_key = None+ else:+ fp_base_url, fp_api_key = base_url, api_key+if backend_name == "jarvis-agent":from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend@@ -202,6 +216,8 @@ def _build_backend(gpu_metrics=gpu_metrics,model=model,max_turns=max_turns,+ base_url=fp_base_url,+ api_key=fp_api_key,)elif backend_name == "jarvis-direct":from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend@@ -210,6 +226,8 @@ def _build_backend(engine_key=engine_key,telemetry=telemetry,gpu_metrics=gpu_metrics,+ base_url=fp_base_url,+ api_key=fp_api_key,)elif backend_name == "hermes":from openjarvis.evals.backends.external import HermesBackend@@ -656,8 +674,20 @@ def _build_trackers(config) -> list:return trackers-def _run_terminalbench_native(config, console: Console) -> object:- """Run TerminalBench V2.1 natively via terminal-bench Harness."""+def _run_terminalbench_native(+ config,+ console: Console,+ *,+ base_url: Optional[str] = None,+ api_key: Optional[str] = None,+) -> object:+ """Run TerminalBench V2.1 natively via terminal-bench Harness.++ ``base_url`` (from ``--base-url`` / JARVIS_BACKEND_BASE_URL) targets an+ already-running OpenAI-compatible endpoint; when unset, the legacy local+ vLLM default (http://localhost:8000/v1) is used.+ """+ from openjarvis.engine.openai_compat_engines import normalize_openai_base_urlfrom openjarvis.evals.backends.terminalbench_native import (TerminalBenchNativeBackend,)@@ -668,9 +698,16 @@ def _run_terminalbench_native(config, console: Console) -> object:litellm_model = f"openai/{model}"output_dir = getattr(config, "output_path", None) or "results/terminalbench-native/"+ # Normalize to exactly one trailing "/v1" — LiteLLM's api_base wants the+ # full OpenAI-compatible prefix, and users pass both forms of the URL.+ if base_url:+ api_base = normalize_openai_base_url(base_url) + "/v1"+ else:+ api_base = "http://localhost:8000/v1"+backend = TerminalBenchNativeBackend(model=litellm_model,- api_base="http://localhost:8000/v1",+ api_base=api_base,temperature=config.temperature,max_samples=config.max_samples,output_dir=output_dir,@@ -683,9 +720,25 @@ def _run_terminalbench_native(config, console: Console) -> object:model_slug = re.sub(r"[^a-z0-9_-]", "-", model.lower().replace("/", "-"))run_id = f"tb21-{model_slug}"console.print(f" Running TerminalBench V2.1 natively: {model}")+ console.print(f" API base: {api_base}")console.print(f" Harness run_id: {run_id}")- results = backend.run_harness(run_id)+ if api_key:+ # terminus-2 routes model calls through LiteLLM with the "openai/"+ # prefix, which reads OPENAI_API_KEY from the environment. The+ # harness runs in-process, so set the var for the duration of the+ # run and restore the previous value afterwards.+ prev_key = os.environ.get("OPENAI_API_KEY")+ os.environ["OPENAI_API_KEY"] = api_key+ try:+ results = backend.run_harness(run_id)+ finally:+ if prev_key is None:+ os.environ.pop("OPENAI_API_KEY", None)+ else:+ os.environ["OPENAI_API_KEY"] = prev_key+ else:+ results = backend.run_harness(run_id)# Convert BenchmarkResults to RunSummarytotal = len(results.trial_results) if hasattr(results, "trial_results") else 0@@ -711,18 +764,47 @@ def _run_terminalbench_native(config, console: Console) -> object:)-def _run_single(config, console: Optional[Console] = None) -> object:- """Run a single eval from a RunConfig and return the summary."""+def _run_single(+ config,+ console: Optional[Console] = None,+ *,+ suite_mode: bool = False,+) -> object:+ """Run a single eval from a RunConfig and return the summary.++ ``suite_mode=True`` (TOML-suite drivers) scopes ``config.base_url`` /+ ``config.api_key`` — stamped from the suite's ``[backend.external]``+ section onto every RunConfig — to the hermes/openclaw backends only;+ extending suite-level endpoint targeting to first-party backends is+ explicitly deferred. The CLI single-run path (``suite_mode=False``)+ honors ``--base-url``/``--api-key`` for every backend.+ """from openjarvis.evals.core.runner import EvalRunnerif console is None:console = Console()+ _metadata = getattr(config, "metadata", None) or {}+ base_url = (+ getattr(config, "base_url", None)+ or _metadata.get("base_url")+ or os.environ.get("JARVIS_BACKEND_BASE_URL")+ )+ api_key = (+ getattr(config, "api_key", None)+ or _metadata.get("api_key")+ or os.environ.get("JARVIS_BACKEND_API_KEY")+ )+# TerminalBench V2.1 native: use terminal-bench Harness directlyif config.benchmark == "terminalbench-native":- return _run_terminalbench_native(config, console)+ return _run_terminalbench_native(+ config,+ console,+ base_url=None if suite_mode else base_url,+ api_key=None if suite_mode else api_key,+ )- _metadata = getattr(config, "metadata", None) or {}eval_backend = _build_backend(config.backend,config.engine_key,@@ -732,16 +814,9 @@ def _run_single(config, console: Optional[Console] = None) -> object:gpu_metrics=getattr(config, "gpu_metrics", False),model=config.model,max_turns=getattr(config, "max_turns", None),- base_url=(- getattr(config, "base_url", None)- or _metadata.get("base_url")- or os.environ.get("JARVIS_BACKEND_BASE_URL")- ),- api_key=(- getattr(config, "api_key", None)- or _metadata.get("api_key")- or os.environ.get("JARVIS_BACKEND_API_KEY")- ),+ base_url=base_url,+ api_key=api_key,+ first_party_endpoint=not suite_mode,)dataset = _build_dataset(config.benchmark)# Inject engine config for benchmarks that run their own simulation@@ -1070,7 +1145,7 @@ def _run_from_config(f"Run {i}/{len(run_configs)}: {rc.benchmark} / {rc.model}",)try:- summary = _run_single(rc, console=console)+ summary = _run_single(rc, console=console, suite_mode=True)summaries.append(summary)console.print(f" [green]{summary.accuracy:.4f}[/green] "@@ -1115,12 +1190,21 @@ def main():@click.option("--base-url",default=None,- help="OpenAI-compat endpoint for hermes/openclaw",+ help=(+ "OpenAI-compatible endpoint for the model under eval. Required for "+ "hermes/openclaw; for jarvis-direct/jarvis-agent/terminalbench-native "+ "it bypasses engine discovery and targets this URL directly "+ "(env: JARVIS_BACKEND_BASE_URL)."+ ),)@click.option("--api-key",default=None,- help="API key for hermes/openclaw endpoint",+ help=(+ "API key for the --base-url endpoint, sent as a Bearer token. "+ "Required for hermes/openclaw; optional for first-party backends "+ "(env: JARVIS_BACKEND_API_KEY)."+ ),)@click.option("-m", "--model", default=None, help="Model identifier")@click.option(@@ -1526,8 +1610,7 @@ def summarize(jsonl_path):default=None,type=click.Path(),help=(- "Output JSONL path. Defaults to <jsonl>.reparsed when "- "--in-place is not set."+ "Output JSONL path. Defaults to <jsonl>.reparsed when --in-place is not set."),)@click.option(@@ -1642,9 +1725,7 @@ def reparse_judge(jsonl_path, out_path, in_place, summary_out):_json.dump(summary, f, indent=2)old_cont = [float(s) for s in old_scores if s is not None]- old_acc = (- sum(1 for s in old_cont if s >= 0.5) / len(old_cont) if old_cont else 0.0- )+ old_acc = sum(1 for s in old_cont if s >= 0.5) / len(old_cont) if old_cont else 0.0old_mean = sum(old_cont) / len(old_cont) if old_cont else 0.0new_mean = sum(cont) / len(cont) if cont else 0.0mean_shift = new_mean - old_meandiff --git a/src/openjarvis/evals/tests/test_backends.py b/src/openjarvis/evals/tests/test_backends.pyindex 7854d2fb..8c7fd25b 100644--- a/src/openjarvis/evals/tests/test_backends.py+++ b/src/openjarvis/evals/tests/test_backends.py@@ -4,6 +4,27 @@ from __future__ import annotationsfrom unittest.mock import MagicMock, patch+import httpx+import pytest+import respx+++def _mock_builder() -> MagicMock:+ """A SystemBuilder mock whose fluent methods chain like the real one."""+ builder = MagicMock()+ for method in (+ "engine",+ "engine_instance",+ "model",+ "agent",+ "tools",+ "telemetry",+ "traces",+ ):+ getattr(builder, method).return_value = builder+ builder.build.return_value = MagicMock()+ return builder+class TestJarvisDirectBackend:@patch("openjarvis.system.SystemBuilder")@@ -137,3 +158,130 @@ class TestJarvisAgentBackend:assert result["content"] == "The answer is 4."assert result["turns"] == 2assert len(result["tool_results"]) == 1+++class TestJarvisDirectBackendBaseUrl:+ """--base-url targeting for the jarvis-direct backend."""++ @patch("openjarvis.system.SystemBuilder")+ def test_base_url_injects_pinned_openai_compat_engine(self, mock_builder_cls):+ from openjarvis.engine.openai_compat_engines import OpenAICompatEngine+ from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend++ mock_builder = _mock_builder()+ mock_builder_cls.return_value = mock_builder++ with respx.mock:+ respx.get("http://127.0.0.1:18999/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ JarvisDirectBackend(base_url="http://127.0.0.1:18999/v1", api_key="sk-x")++ mock_builder.engine_instance.assert_called_once()+ injected = mock_builder.engine_instance.call_args[0][0]+ assert isinstance(injected, OpenAICompatEngine)+ # Trailing /v1 is normalized away so request paths don't double up.+ assert injected._host == "http://127.0.0.1:18999"+ assert injected._api_key == "sk-x"+ # The discovery path must not be engaged at all.+ mock_builder.engine.assert_not_called()++ @patch("openjarvis.system.SystemBuilder")+ def test_unreachable_base_url_fails_fast_naming_url(self, mock_builder_cls):+ from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend++ mock_builder = _mock_builder()+ mock_builder_cls.return_value = mock_builder++ with respx.mock:+ respx.get("http://127.0.0.1:18998/v1/models").mock(+ side_effect=httpx.ConnectError("connection refused")+ )+ with pytest.raises(RuntimeError, match=r"http://127\.0\.0\.1:18998"):+ JarvisDirectBackend(base_url="http://127.0.0.1:18998")++ # No silent engine substitution: the system is never built.+ mock_builder.engine_instance.assert_not_called()+ mock_builder.build.assert_not_called()++ @patch("openjarvis.system.SystemBuilder")+ def test_no_base_url_keeps_engine_key_path(self, mock_builder_cls):+ from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend++ mock_builder = _mock_builder()+ mock_builder_cls.return_value = mock_builder++ JarvisDirectBackend(engine_key="vllm")+ mock_builder.engine.assert_called_with("vllm")+ mock_builder.engine_instance.assert_not_called()++ @patch("openjarvis.system.SystemBuilder")+ def test_base_url_wins_over_engine_key(self, mock_builder_cls):+ from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend++ mock_builder = _mock_builder()+ mock_builder_cls.return_value = mock_builder++ with respx.mock:+ respx.get("http://127.0.0.1:18999/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ JarvisDirectBackend(engine_key="vllm", base_url="http://127.0.0.1:18999")++ mock_builder.engine.assert_not_called()+ mock_builder.engine_instance.assert_called_once()+ # The engine key is kept as the label for the injected engine.+ assert mock_builder.engine_instance.call_args.kwargs["key"] == "vllm"+++class TestJarvisAgentBackendBaseUrl:+ """--base-url targeting for the jarvis-agent backend."""++ @patch("openjarvis.system.SystemBuilder")+ def test_base_url_injects_pinned_openai_compat_engine(self, mock_builder_cls):+ from openjarvis.engine.openai_compat_engines import OpenAICompatEngine+ from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend++ mock_builder = _mock_builder()+ mock_builder_cls.return_value = mock_builder++ with respx.mock:+ respx.get("http://127.0.0.1:18999/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ JarvisAgentBackend(base_url="http://127.0.0.1:18999/v1", api_key="sk-x")++ mock_builder.engine_instance.assert_called_once()+ injected = mock_builder.engine_instance.call_args[0][0]+ assert isinstance(injected, OpenAICompatEngine)+ assert injected._host == "http://127.0.0.1:18999"+ assert injected._api_key == "sk-x"+ mock_builder.engine.assert_not_called()++ @patch("openjarvis.system.SystemBuilder")+ def test_unreachable_base_url_fails_fast_naming_url(self, mock_builder_cls):+ from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend++ mock_builder = _mock_builder()+ mock_builder_cls.return_value = mock_builder++ with respx.mock:+ respx.get("http://127.0.0.1:18998/v1/models").mock(+ side_effect=httpx.ConnectError("connection refused")+ )+ with pytest.raises(RuntimeError, match=r"http://127\.0\.0\.1:18998"):+ JarvisAgentBackend(base_url="http://127.0.0.1:18998")++ mock_builder.engine_instance.assert_not_called()+ mock_builder.build.assert_not_called()++ @patch("openjarvis.system.SystemBuilder")+ def test_no_base_url_keeps_engine_key_path(self, mock_builder_cls):+ from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend++ mock_builder = _mock_builder()+ mock_builder_cls.return_value = mock_builder++ JarvisAgentBackend(engine_key="vllm")+ mock_builder.engine.assert_called_with("vllm")+ mock_builder.engine_instance.assert_not_called()diff --git a/src/openjarvis/evals/tests/test_cli_endpoint.py b/src/openjarvis/evals/tests/test_cli_endpoint.pynew file mode 100644index 00000000..8257e61f--- /dev/null+++ b/src/openjarvis/evals/tests/test_cli_endpoint.py@@ -0,0 +1,190 @@+"""--base-url/--api-key forwarding through the eval CLI plumbing.++Covers the fix for the eval-CLI endpoint gap: the flags used to be silently+dropped for jarvis-direct/jarvis-agent and ignored by terminalbench-native+(which hardcoded api_base="http://localhost:8000/v1").+"""++from __future__ import annotations++import io+import os+from types import SimpleNamespace+from unittest.mock import MagicMock, patch++import click+import pytest+from rich.console import Console++from openjarvis.evals.cli import _build_backend, _run_terminalbench_native+from openjarvis.evals.core.types import RunConfig+++def _quiet_console() -> Console:+ return Console(file=io.StringIO())+++def _tb_config(**overrides) -> RunConfig:+ defaults = dict(+ benchmark="terminalbench-native",+ backend="jarvis-direct",+ model="my-model",+ max_samples=1,+ max_workers=1,+ temperature=0.2,+ )+ defaults.update(overrides)+ return RunConfig(**defaults)+++class TestBuildBackendForwardsEndpoint:+ @patch("openjarvis.evals.backends.jarvis_direct.JarvisDirectBackend")+ def test_jarvis_direct_receives_base_url_and_api_key(self, mock_cls):+ _build_backend(+ "jarvis-direct",+ "vllm",+ "orchestrator",+ [],+ base_url="http://node7:8123/v1",+ api_key="sk-k",+ )+ kwargs = mock_cls.call_args.kwargs+ assert kwargs["base_url"] == "http://node7:8123/v1"+ assert kwargs["api_key"] == "sk-k"++ @patch("openjarvis.evals.backends.jarvis_agent.JarvisAgentBackend")+ def test_jarvis_agent_receives_base_url_and_api_key(self, mock_cls):+ _build_backend(+ "jarvis-agent",+ "vllm",+ "orchestrator",+ ["calculator"],+ base_url="http://node7:8123/v1",+ api_key="sk-k",+ )+ kwargs = mock_cls.call_args.kwargs+ assert kwargs["base_url"] == "http://node7:8123/v1"+ assert kwargs["api_key"] == "sk-k"++ @patch("openjarvis.evals.backends.jarvis_direct.JarvisDirectBackend")+ def test_suite_mode_scopes_endpoint_to_external_backends(self, mock_cls):+ """[backend.external] suite semantics stay hermes/openclaw-only:+ first_party_endpoint=False must not forward to first-party."""+ _build_backend(+ "jarvis-direct",+ "vllm",+ "orchestrator",+ [],+ base_url="http://node7:8123/v1",+ api_key="sk-k",+ first_party_endpoint=False,+ )+ kwargs = mock_cls.call_args.kwargs+ assert kwargs["base_url"] is None+ assert kwargs["api_key"] is None++ def test_hermes_still_requires_base_url_and_api_key(self):+ with pytest.raises(click.UsageError, match="hermes"):+ _build_backend("hermes", None, "orchestrator", [])++ def test_openclaw_still_requires_base_url_and_api_key(self):+ with pytest.raises(click.UsageError, match="openclaw"):+ _build_backend("openclaw", None, "orchestrator", [])+++class TestTerminalBenchNativeApiBase:+ @patch("openjarvis.evals.backends.terminalbench_native.TerminalBenchNativeBackend")+ def test_base_url_passed_through_as_api_base(self, mock_cls):+ mock_backend = MagicMock()+ mock_backend.run_harness.return_value = SimpleNamespace(trial_results=[])+ mock_cls.return_value = mock_backend++ _run_terminalbench_native(+ _tb_config(),+ _quiet_console(),+ base_url="http://node7:8123/v1",+ )+ assert mock_cls.call_args.kwargs["api_base"] == "http://node7:8123/v1"++ @patch("openjarvis.evals.backends.terminalbench_native.TerminalBenchNativeBackend")+ def test_base_url_without_v1_gets_single_v1_suffix(self, mock_cls):+ mock_backend = MagicMock()+ mock_backend.run_harness.return_value = SimpleNamespace(trial_results=[])+ mock_cls.return_value = mock_backend++ _run_terminalbench_native(+ _tb_config(),+ _quiet_console(),+ base_url="http://node7:8123",+ )+ assert mock_cls.call_args.kwargs["api_base"] == "http://node7:8123/v1"++ @patch("openjarvis.evals.backends.terminalbench_native.TerminalBenchNativeBackend")+ def test_default_api_base_unchanged_without_base_url(self, mock_cls):+ mock_backend = MagicMock()+ mock_backend.run_harness.return_value = SimpleNamespace(trial_results=[])+ mock_cls.return_value = mock_backend++ _run_terminalbench_native(_tb_config(), _quiet_console())+ assert mock_cls.call_args.kwargs["api_base"] == "http://localhost:8000/v1"++ @patch("openjarvis.evals.backends.terminalbench_native.TerminalBenchNativeBackend")+ def test_api_key_exported_as_openai_api_key_during_run(self, mock_cls, monkeypatch):+ """terminus-2 reads OPENAI_API_KEY via LiteLLM; the var must be set+ during harness.run() and restored afterwards."""+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ seen: dict = {}++ def fake_run_harness(run_id):+ seen["openai_api_key"] = os.environ.get("OPENAI_API_KEY")+ return SimpleNamespace(trial_results=[])++ mock_backend = MagicMock()+ mock_backend.run_harness.side_effect = fake_run_harness+ mock_cls.return_value = mock_backend++ _run_terminalbench_native(+ _tb_config(),+ _quiet_console(),+ base_url="http://node7:8123/v1",+ api_key="sk-tb",+ )+ assert seen["openai_api_key"] == "sk-tb"+ assert "OPENAI_API_KEY" not in os.environ # restored++ @patch("openjarvis.evals.backends.terminalbench_native.TerminalBenchNativeBackend")+ def test_preexisting_openai_api_key_restored(self, mock_cls, monkeypatch):+ monkeypatch.setenv("OPENAI_API_KEY", "sk-original")+ mock_backend = MagicMock()+ mock_backend.run_harness.return_value = SimpleNamespace(trial_results=[])+ mock_cls.return_value = mock_backend++ _run_terminalbench_native(+ _tb_config(),+ _quiet_console(),+ base_url="http://node7:8123/v1",+ api_key="sk-tb",+ )+ assert os.environ["OPENAI_API_KEY"] == "sk-original"+++class TestRunSingleSuiteModeGating:+ @patch("openjarvis.evals.cli._run_terminalbench_native")+ def test_suite_mode_drops_endpoint_for_terminalbench(self, mock_tb):+ from openjarvis.evals.cli import _run_single++ mock_tb.return_value = SimpleNamespace(accuracy=0.0)+ config = _tb_config(base_url="http://node7:8123/v1", api_key="sk-k")+ _run_single(config, console=_quiet_console(), suite_mode=True)+ assert mock_tb.call_args.kwargs["base_url"] is None+ assert mock_tb.call_args.kwargs["api_key"] is None++ @patch("openjarvis.evals.cli._run_terminalbench_native")+ def test_cli_mode_forwards_endpoint_for_terminalbench(self, mock_tb):+ from openjarvis.evals.cli import _run_single++ mock_tb.return_value = SimpleNamespace(accuracy=0.0)+ config = _tb_config(base_url="http://node7:8123/v1", api_key="sk-k")+ _run_single(config, console=_quiet_console())+ assert mock_tb.call_args.kwargs["base_url"] == "http://node7:8123/v1"+ assert mock_tb.call_args.kwargs["api_key"] == "sk-k"diff --git a/src/openjarvis/system/builder.py b/src/openjarvis/system/builder.pyindex a5a317c3..c686c504 100644--- a/src/openjarvis/system/builder.py+++ b/src/openjarvis/system/builder.py@@ -33,6 +33,8 @@ class SystemBuilder:self._config = load_config()self._engine_key: Optional[str] = None+ self._engine_instance: Optional[InferenceEngine] = None+ self._engine_instance_key: Optional[str] = Noneself._model: Optional[str] = Noneself._agent_name: Optional[str] = Noneself._tool_names: Optional[List[str]] = None@@ -50,6 +52,20 @@ class SystemBuilder:self._engine_key = keyreturn self+ def engine_instance(+ self, engine: InferenceEngine, key: str = "openai-compat"+ ) -> SystemBuilder:+ """Inject a pre-built engine instance, bypassing engine discovery.++ Used by callers that must target one exact endpoint (e.g.+ ``jarvis eval --base-url``). ``build()`` health-checks the instance+ and raises a loud error if it is unreachable — it never silently+ substitutes a different discovered engine.+ """+ self._engine_instance = engine+ self._engine_instance_key = key+ return self+def model(self, name: str) -> SystemBuilder:self._model = namereturn self@@ -303,6 +319,23 @@ class SystemBuilder:return systemdef _resolve_engine(self, config: JarvisConfig):+ # An explicitly injected engine instance always wins and is never+ # silently replaced: when the caller pinned an endpoint (e.g.+ # ``jarvis eval --base-url``) and it is down, substituting whatever+ # other engine discovery finds would silently run against the wrong+ # model server. Fail loudly instead.+ if self._engine_instance is not None:+ engine = self._engine_instance+ key = self._engine_instance_key or "openai-compat"+ if not engine.health():+ host = getattr(engine, "_host", "<unknown host>")+ raise RuntimeError(+ f"Injected engine {key!r} is not reachable at {host} — "+ "is the endpoint running and serving GET /v1/models? "+ "Refusing to fall back to engine discovery."+ )+ return engine, key+from openjarvis.engine._discovery import get_enginepref = config.intelligence.preferred_engine@@ -313,7 +346,18 @@ class SystemBuilder:"No inference engine available. ""Make sure an engine is running (e.g. ollama serve).")- return resolved[1], resolved[0]+ resolved_key, engine = resolved+ if self._engine_key and resolved_key != self._engine_key:+ # get_engine() falls back to any healthy discovered engine; make+ # the substitution visible when the caller asked for a specific+ # engine (observed: requested vllm, silently got ollama@11434).+ logger.warning(+ "Requested engine %r is unavailable; using %r at %s instead",+ self._engine_key,+ resolved_key,+ getattr(engine, "_host", "<unknown host>"),+ )+ return engine, resolved_keydef _resolve_model(self, config: JarvisConfig, engine: InferenceEngine) -> str:if self._model:diff --git a/tests/engine/test_openai_compat_api_key.py b/tests/engine/test_openai_compat_api_key.pynew file mode 100644index 00000000..dc541358--- /dev/null+++ b/tests/engine/test_openai_compat_api_key.py@@ -0,0 +1,109 @@+"""API-key (Authorization header) support in the OpenAI-compat engine base."""++from __future__ import annotations++import httpx+import pytest+import respx++from openjarvis.core.types import Message, Role+from openjarvis.engine.openai_compat_engines import (+ OpenAICompatEngine,+ VLLMEngine,+ normalize_openai_base_url,+)++_CHAT_RESPONSE = {+ "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},+ "model": "m",+}+++class TestAuthorizationHeader:+ def test_bearer_header_sent_when_api_key_set(self) -> None:+ engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-test")+ with respx.mock:+ route = respx.post("http://testhost:9000/v1/chat/completions").mock(+ return_value=httpx.Response(200, json=_CHAT_RESPONSE)+ )+ engine.generate([Message(role=Role.USER, content="hi")], model="m")+ assert route.calls.last.request.headers["Authorization"] == "Bearer sk-test"++ def test_no_authorization_header_without_api_key(self) -> None:+ engine = OpenAICompatEngine(host="http://testhost:9000")+ with respx.mock:+ route = respx.post("http://testhost:9000/v1/chat/completions").mock(+ return_value=httpx.Response(200, json=_CHAT_RESPONSE)+ )+ engine.generate([Message(role=Role.USER, content="hi")], model="m")+ assert "authorization" not in route.calls.last.request.headers++ def test_health_check_sends_bearer_header(self) -> None:+ engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-test")+ with respx.mock:+ route = respx.get("http://testhost:9000/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ assert engine.health() is True+ assert route.calls.last.request.headers["Authorization"] == "Bearer sk-test"++ def test_env_var_fallback_sanitizes_hyphen(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ # engine_id "openai-compat" must map to OPENAI_COMPAT_API_KEY —+ # shells cannot set hyphenated env-var names.+ monkeypatch.setenv("OPENAI_COMPAT_API_KEY", "sk-env")+ engine = OpenAICompatEngine(host="http://testhost:9000")+ with respx.mock:+ route = respx.get("http://testhost:9000/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ engine.health()+ assert route.calls.last.request.headers["Authorization"] == "Bearer sk-env"++ def test_vllm_env_var_fallback(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("VLLM_API_KEY", "sk-vllm")+ engine = VLLMEngine(host="http://testhost:8000")+ with respx.mock:+ route = respx.get("http://testhost:8000/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ engine.health()+ assert route.calls.last.request.headers["Authorization"] == "Bearer sk-vllm"++ def test_explicit_api_key_beats_env_var(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("OPENAI_COMPAT_API_KEY", "sk-env")+ engine = OpenAICompatEngine(host="http://testhost:9000", api_key="sk-explicit")+ assert engine._api_key == "sk-explicit"+++class TestNormalizeOpenAIBaseUrl:+ @pytest.mark.parametrize(+ ("url", "expected"),+ [+ ("http://h:8000", "http://h:8000"),+ ("http://h:8000/", "http://h:8000"),+ ("http://h:8000/v1", "http://h:8000"),+ ("http://h:8000/v1/", "http://h:8000"),+ ("http://h:8000/gateway/v1", "http://h:8000/gateway"),+ # Only a literal trailing "/v1" is stripped — never other paths.+ ("http://h:8000/v1x", "http://h:8000/v1x"),+ ("http://h:8000/v2", "http://h:8000/v2"),+ ],+ )+ def test_normalization(self, url: str, expected: str) -> None:+ assert normalize_openai_base_url(url) == expected++ def test_engine_requests_have_single_v1_prefix(self) -> None:+ """End to end: a user-supplied .../v1 URL must not produce /v1/v1."""+ host = normalize_openai_base_url("http://testhost:9000/v1")+ engine = OpenAICompatEngine(host=host)+ with respx.mock:+ route = respx.get("http://testhost:9000/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ assert engine.health() is True+ assert route.calls.last.request.url.path == "/v1/models"diff --git a/tests/sdk/test_system.py b/tests/sdk/test_system.pyindex bb1c9b0c..4502ea47 100644--- a/tests/sdk/test_system.py+++ b/tests/sdk/test_system.py@@ -2,7 +2,7 @@from __future__ import annotations-from unittest.mock import MagicMock+from unittest.mock import MagicMock, patchimport pytest@@ -327,6 +327,85 @@ class TestSystemBuilder:assert builder._engine_key == "ollama"+class TestSystemBuilderEngineInstance:+ """Explicit engine injection (jarvis eval --base-url path)."""++ @staticmethod+ def _fake_engine(healthy: bool = True) -> MagicMock:+ engine = MagicMock(+ spec=["health", "can_serve", "generate", "list_models", "close"]+ )+ engine.health.return_value = healthy+ engine._host = "http://127.0.0.1:18999"+ return engine++ def test_engine_instance_is_fluent(self):+ builder = SystemBuilder(JarvisConfig())+ engine = self._fake_engine()+ result = builder.engine_instance(engine, key="my-endpoint")+ assert result is builder+ assert builder._engine_instance is engine+ assert builder._engine_instance_key == "my-endpoint"++ def test_resolve_engine_returns_injected_instance(self):+ config = JarvisConfig()+ engine = self._fake_engine(healthy=True)+ builder = SystemBuilder(config).engine_instance(engine, key="endpoint")+ resolved_engine, resolved_key = builder._resolve_engine(config)+ assert resolved_engine is engine+ assert resolved_key == "endpoint"++ def test_unhealthy_injected_instance_raises_naming_host(self):+ config = JarvisConfig()+ engine = self._fake_engine(healthy=False)+ builder = SystemBuilder(config).engine_instance(engine, key="endpoint")+ with pytest.raises(RuntimeError, match=r"http://127\.0\.0\.1:18999"):+ builder._resolve_engine(config)++ def test_unhealthy_injected_instance_never_consults_discovery(self):+ """The observed failure mode: an explicit endpoint must NOT be+ silently replaced by whatever other engine discovery finds."""+ config = JarvisConfig()+ engine = self._fake_engine(healthy=False)+ builder = SystemBuilder(config).engine_instance(engine)+ with patch("openjarvis.engine._discovery.get_engine") as mock_get_engine:+ with pytest.raises(RuntimeError, match="Refusing to fall back"):+ builder._resolve_engine(config)+ mock_get_engine.assert_not_called()++ def test_healthy_injected_instance_never_consults_discovery(self):+ config = JarvisConfig()+ engine = self._fake_engine(healthy=True)+ builder = SystemBuilder(config).engine_instance(engine, key="endpoint")+ with patch("openjarvis.engine._discovery.get_engine") as mock_get_engine:+ resolved_engine, _ = builder._resolve_engine(config)+ assert resolved_engine is engine+ mock_get_engine.assert_not_called()++ def test_build_wires_injected_engine(self):+ """build() must use the injected engine (possibly behind security+ wrappers) instead of running discovery."""+ config = JarvisConfig()+ engine = self._fake_engine(healthy=True)+ engine.list_models.return_value = ["stub-model"]+ builder = (+ SystemBuilder(config)+ .engine_instance(engine, key="endpoint")+ .model("stub-model")+ .telemetry(False)+ .traces(False)+ )+ system = builder.build()+ try:+ inner = system.engine+ while hasattr(inner, "_engine"):+ inner = inner._engine+ assert inner is engine+ assert system.engine_key == "endpoint"+ finally:+ system.close()++class TestJarvisSystemClose:def test_close_with_scheduler_store(self):engine = MagicMock()