Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

Fix: propagate custom endpoint to judge in self-eval mode (#17)

Aliciawque2e56c4f5cd8b
diff --git a/ifixai/cli/orchestrator.py b/ifixai/cli/orchestrator.py
index c80b3fb..0393d85 100644
--- a/ifixai/cli/orchestrator.py
+++ b/ifixai/cli/orchestrator.py
@@ -158,11 +158,12 @@ def _build_judge_config(
sut_provider: str | None,
sut_api_key: str,
sut_model: str | None,
- judge_providers: tuple[str, ...],
- judge_api_keys: tuple[str, ...],
- judge_models: tuple[str, ...],
- max_calls: int,
- timeout: int,
+ sut_endpoint: str | None = None,
+ judge_providers: tuple[str, ...] = (),
+ judge_api_keys: tuple[str, ...] = (),
+ judge_models: tuple[str, ...] = (),
+ max_calls: int = 200,
+ timeout: int = 30,
) -> JudgeConfig | None:
if eval_mode == "deterministic":
return None
@@ -171,6 +172,7 @@ def _build_judge_config(
provider=sut_provider or "",
api_key=sut_api_key,
model=sut_model,
+ endpoint=sut_endpoint,
max_calls_per_run=max_calls,
timeout=timeout,
)
diff --git a/ifixai/cli/run.py b/ifixai/cli/run.py
index 1184bb6..16f2447 100644
--- a/ifixai/cli/run.py
+++ b/ifixai/cli/run.py
@@ -899,6 +899,7 @@ def run(
sut_provider=provider,
sut_api_key=api_key or "",
sut_model=model,
+ sut_endpoint=endpoint,
judge_providers=judge_provider,
judge_api_keys=judge_api_key,
judge_models=judge_model,
diff --git a/ifixai/judge/config.py b/ifixai/judge/config.py
index 2ef75fe..f4fd16b 100644
--- a/ifixai/judge/config.py
+++ b/ifixai/judge/config.py
@@ -15,6 +15,7 @@ class JudgeConfig(BaseModel):
provider: str = ""
model: Optional[str] = None
api_key: str = ""
+ endpoint: Optional[str] = None
providers: Optional[list[JudgeProviderSpec]] = None
diff --git a/ifixai/judge/evaluator.py b/ifixai/judge/evaluator.py
index ad7a2e5..2660b5c 100644
--- a/ifixai/judge/evaluator.py
+++ b/ifixai/judge/evaluator.py
@@ -12,6 +12,7 @@ class JudgeEvaluator:
provider=config.provider,
api_key=config.api_key,
model=config.model,
+ endpoint=config.endpoint,
timeout=config.timeout,
)
self._call_count = 0
diff --git a/ifixai/providers/http.py b/ifixai/providers/http.py
index f7f7d27..6f630b9 100644
--- a/ifixai/providers/http.py
+++ b/ifixai/providers/http.py
@@ -19,7 +19,7 @@ from ifixai.providers.base import (
from ifixai.providers.secrets import scrub_secrets
from ifixai.core.types import ChatMessage, ProviderConfig, RetrievedSource
-DEFAULT_ENDPOINT = "http://localhost:8000/v1"
+DEFAULT_ENDPOINT = os.environ.get("IFIXAI_HTTP_ENDPOINT", "http://localhost:8000/v1")
EXTRA_HEADERS_ENV_VAR = "IFIXAI_EXTRA_HEADERS"