Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): a retired Jev pin runs on jev-latest and logs an ERROR (#15580)

Reinier van der Leerd5fc889d6293
diff --git a/autogpt_platform/backend/backend/blocks/typesafe/_client.py b/autogpt_platform/backend/backend/blocks/typesafe/_client.py
index 8c7b9bd985..447eed35db 100644
--- a/autogpt_platform/backend/backend/blocks/typesafe/_client.py
+++ b/autogpt_platform/backend/backend/blocks/typesafe/_client.py
@@ -1,5 +1,7 @@
import base64
import json
+import logging
+import re
from time import perf_counter
from typing import Any
@@ -9,14 +11,20 @@ from typesafe_sdk._core.errors import (
TypeSafeAPIConnectionError,
TypeSafeAPIError,
TypeSafeAPIResponseValidationError,
+ extract_message,
)
from typesafe_sdk._core.json import serialize
from typesafe_sdk._core.response_types import SystemOneResponse
-from typesafe_sdk.constants import DEFAULT_BASE_URL
+from typesafe_sdk.constants import DEFAULT_BASE_URL, DEFAULT_MODEL
from ._budget import PreparedState, prepare_state
from ._wire import WireCapture, capture_wire
+logger = logging.getLogger(__name__)
+# Pins TypeSafe has answered "Unknown model" for; later calls skip straight to the alias.
+_retired_models: set[str] = set()
+_PINNED_VERSION = re.compile(r"jev-\d+\.\d+\.\d+")
+
class JevCallResult(BaseModel):
answers: dict[str, dict[str, Any]]
@@ -38,6 +46,29 @@ async def call_jev(
*,
model: str | None = None,
timeout: float | None = None,
+) -> JevCallResult:
+ """A pinned ``model`` that TypeSafe has retired runs on ``jev-latest``
+ instead, with one ERROR per process naming the pin to update."""
+ if model in _retired_models:
+ model = DEFAULT_MODEL
+ try:
+ return await _call_jev(api_key, state, questions, model, timeout)
+ except _RetiredModel as retired:
+ if retired.model not in _retired_models:
+ _retired_models.add(retired.model)
+ logger.error(
+ f"TypeSafe no longer serves Jev model {retired.model}; calls pinned "
+ f"to it now run on {DEFAULT_MODEL}. Pin a current Jev version."
+ )
+ return await _call_jev(api_key, state, questions, DEFAULT_MODEL, timeout)
+
+
+async def _call_jev(
+ api_key: str,
+ state: Any,
+ questions: dict[str, Choice | Score | Noul],
+ model: str | None,
+ timeout: float | None,
) -> JevCallResult:
prepared = prepare_state(state, questions)
async with AsyncTypeSafeClient(
@@ -61,6 +92,8 @@ async def call_jev(
error.request_id or "",
)
except TypeSafeAPIError as error:
+ if model and _is_retired(model, error):
+ raise _RetiredModel(model) from error
return _failure(
wire,
prepared,
@@ -88,6 +121,21 @@ async def call_jev(
return _success(result, wire, prepared, started)
+class _RetiredModel(Exception):
+ def __init__(self, model: str) -> None:
+ super().__init__(model)
+ self.model = model
+
+
+def _is_retired(model: str, error: TypeSafeAPIError) -> bool:
+ # A concrete version only: an alias such as jev-preview fails as before.
+ return (
+ _PINNED_VERSION.fullmatch(model) is not None
+ and error.status == 400
+ and extract_message(error.body) == f"Unknown model: {model}"
+ )
+
+
def _success(
result: SystemOneResponse,
wire: WireCapture,
diff --git a/autogpt_platform/backend/backend/blocks/typesafe/_client_test.py b/autogpt_platform/backend/backend/blocks/typesafe/_client_test.py
index 4e8f732505..8e9340ac70 100644
--- a/autogpt_platform/backend/backend/blocks/typesafe/_client_test.py
+++ b/autogpt_platform/backend/backend/blocks/typesafe/_client_test.py
@@ -288,3 +288,105 @@ async def test_missing_request_id_preserves_response(monkeypatch: pytest.MonkeyP
assert result.response == RESPONSE
assert result.request_id == ""
assert result.error == "Jev response did not include a request ID."
+
+
+async def test_retired_pin_runs_on_jev_latest_and_reports_it(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+):
+ sent: list[str] = []
+ mock_transport(
+ monkeypatch, refusing(sent, "jev-1.13.0", "Unknown model: jev-1.13.0")
+ )
+ monkeypatch.setattr(_client, "_retired_models", set())
+ questions = {"q": Noul(instructions="Yes?")}
+ with caplog.at_level(logging.ERROR, logger=_client.__name__):
+ first = await _client.call_jev("test", "s", questions, model="jev-1.13.0")
+ second = await _client.call_jev("test", "s", questions, model="jev-1.13.0")
+ assert sent == ["jev-1.13.0", "jev-latest", "jev-latest"]
+ for result in (first, second):
+ assert result.error == ""
+ assert result.answers == json.loads(RESPONSE)["answers"]
+ assert json.loads(result.request)["model"] == "jev-latest"
+ errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR]
+ assert len(errors) == 1 and "jev-1.13.0" in errors[0]
+
+
+async def test_concurrent_calls_on_a_retired_pin_report_it_once(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+):
+ sent: list[str] = []
+ refuse = refusing(sent, "jev-1.13.0", "Unknown model: jev-1.13.0")
+ both_sent = asyncio.Event()
+
+ async def handler(request: httpx2.Request) -> httpx2.Response:
+ response = await refuse(request)
+ if sent.count("jev-1.13.0") == 2:
+ both_sent.set()
+ await asyncio.wait_for(both_sent.wait(), timeout=2)
+ return response
+
+ mock_transport(monkeypatch, handler)
+ monkeypatch.setattr(_client, "_retired_models", set())
+ questions = {"q": Noul(instructions="Yes?")}
+ with caplog.at_level(logging.ERROR, logger=_client.__name__):
+ results = await asyncio.gather(
+ _client.call_jev("test", "a", questions, model="jev-1.13.0"),
+ _client.call_jev("test", "b", questions, model="jev-1.13.0"),
+ )
+ assert sorted(sent) == ["jev-1.13.0", "jev-1.13.0", "jev-latest", "jev-latest"]
+ assert [result.error for result in results] == ["", ""]
+ assert len([r for r in caplog.records if r.levelno == logging.ERROR]) == 1
+
+
+ "model, refused, message",
+ [
+ ("jev-1.13.0", "jev-1.13.0", "Too many questions"),
+ ("jev-1.13.0", "jev-1.13.0", "Unknown model: jev-other"),
+ ("jev-preview", "jev-preview", "Unknown model: jev-preview"),
+ ("jev-latest", "jev-latest", "Unknown model: jev-latest"),
+ (None, "jev-latest", "Unknown model: jev-latest"),
+ ],
+ ids=[
+ "other-400-on-a-pin",
+ "names-another-model",
+ "unknown-preview-alias",
+ "unknown-latest-alias",
+ "unknown-sdk-default",
+ ],
+)
+async def test_other_refusals_are_not_retried(
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ model: str | None,
+ refused: str,
+ message: str,
+):
+ sent: list[str] = []
+ mock_transport(monkeypatch, refusing(sent, refused, message))
+ monkeypatch.setattr(_client, "_retired_models", set())
+ with caplog.at_level(logging.ERROR, logger=_client.__name__):
+ result = await _client.call_jev(
+ "test", "s", {"q": Noul(instructions="Yes?")}, model=model
+ )
+ assert result.error == "Jev API request failed (HTTP 400)."
+ assert sent == [refused]
+ assert not [r for r in caplog.records if r.levelno == logging.ERROR]
+
+
+def refusing(
+ sent: list[str], model: str, message: str
+) -> Callable[[httpx2.Request], Awaitable[httpx2.Response]]:
+ """A TypeSafe that answers 400 ``message`` for ``model`` and RESPONSE otherwise."""
+
+ async def handler(request: httpx2.Request) -> httpx2.Response:
+ asked = json.loads(request.content)["model"]
+ sent.append(asked)
+ if asked == model:
+ body = {"detail": {"error_type": "api_usage_error", "message": message}}
+ return httpx2.Response(400, json=body)
+ return httpx2.Response(
+ 200, content=RESPONSE, headers={"x-typesafe-request-id": "req"}
+ )
+
+ return handler
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index a205087970..28a8e7bfdb 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -299,7 +299,11 @@ class ChatConfig(BaseSettings):
"judged call and the LLM (``gate_model``) runs only on an ask, to write "
"the reason. Off without ``TYPESAFE_API_KEY``.",
)
- gate_jev_model: str = Field(default="jev-1.13.0", description="Jev model id.")
+ gate_jev_model: str = Field(
+ default="jev-1.13.0",
+ description="Pinned Jev version. Once TypeSafe retires it, calls run on "
+ "``jev-latest`` and log an ERROR naming it.",
+ )
gate_jev_ask_threshold: float | None = Field(
default=None,
description="Unset: Jev's allow/ask choice decides. Set: the call also "