AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): warm-context reranker 400s — raise max_tokens to OpenAI's minimum (#13337)
Nicholas Tindle512fb1e3ec14
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/client.py b/autogpt_platform/backend/backend/copilot/graphiti/client.pyindex 3e9c1f3208..54d231d849 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/client.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/client.py@@ -141,11 +141,11 @@ def _build_graphiti(group_id: str, llm_client):flex) without disturbing the embedder + cross-encoder defaults."""from graphiti_core import Graphiti- from graphiti_core.cross_encoder.openai_reranker_client import OpenAIRerankerClientfrom graphiti_core.embedder import OpenAIEmbedder, OpenAIEmbedderConfigfrom graphiti_core.llm_client import LLMConfigfrom .falkordb_driver import AutoGPTFalkorDriver+ from .reranker import CompatOpenAIRerankerClientembedder_config = OpenAIEmbedderConfig(api_key=graphiti_config.resolve_embedder_api_key(),@@ -155,16 +155,18 @@ def _build_graphiti(group_id: str, llm_client):embedder = OpenAIEmbedder(config=embedder_config)# P-1.4: cross-encoder reranker for warm-context retrieval.- # OpenAIRerankerClient runs concurrent boolean-classifier prompts- # (one per candidate edge) and uses log-probabilities to rank.- # Cheap because the reranker model defaults to gpt-4.1-nano —- # the cost is one batch of small calls per session-start search.+ # Runs concurrent boolean-classifier prompts (one per candidate+ # edge) and uses log-probabilities to rank. Cheap because the+ # reranker model defaults to gpt-4.1-nano — the cost is one batch+ # of small calls per session-start search. The Compat subclass+ # fixes the stock client's max_tokens=1, which OpenAI-compatible+ # upstreams now reject with a 400 (minimum is 16).reranker_config = LLMConfig(api_key=graphiti_config.resolve_llm_api_key(),model=graphiti_config.reranker_model,base_url=graphiti_config.resolve_llm_base_url(),)- cross_encoder = OpenAIRerankerClient(config=reranker_config)+ cross_encoder = CompatOpenAIRerankerClient(config=reranker_config)graph_driver = AutoGPTFalkorDriver(host=graphiti_config.falkordb_host,diff --git a/autogpt_platform/backend/backend/copilot/graphiti/reranker.py b/autogpt_platform/backend/backend/copilot/graphiti/reranker.pynew file mode 100644index 0000000000..cb17ab5ceb--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/reranker.py@@ -0,0 +1,128 @@+"""OpenAI-compat cross-encoder reranker for warm-context retrieval.++graphiti-core 0.28.2's stock ``OpenAIRerankerClient.rank`` sends+``max_tokens=1`` on each boolean-classifier call. OpenAI (and+OpenRouter's OpenAI/Azure upstreams) now reject completions below 16+output tokens ("Invalid 'max_output_tokens': integer below minimum+value. Expected a value >= 16, but got 1"), which turned every+warm-context fetch into a 400 — and, because the ratification hit+hook runs after the search gather that raised, also starved the+hit-time ratification loop.++``CompatOpenAIRerankerClient.rank`` is a faithful copy of the+upstream method (pinned at graphiti-core 0.28.2) with ``max_tokens``+raised to a compliant floor. Scoring is unchanged: the ``logit_bias``+pins the output to True/False, and only the FIRST token's+top-logprobs are read, so the extra token budget never affects the+ranking (same seam-level patch pattern as+``communities._bounded_label_propagation``).+"""++from __future__ import annotations++import logging++import numpy as np+import openai+from graphiti_core.cross_encoder.openai_reranker_client import OpenAIRerankerClient+from graphiti_core.helpers import semaphore_gather+from graphiti_core.llm_client import LLMConfig, RateLimitError+from openai.types.chat import (+ ChatCompletionMessageParam,+ ChatCompletionSystemMessageParam,+ ChatCompletionUserMessageParam,+)++logger = logging.getLogger(__name__)++# OpenAI's documented minimum for max output tokens on logprob+# completions. The classifier still only ever emits one token (the+# logit_bias restricts the vocabulary to "True"/"False").+MIN_COMPLETION_TOKENS = 16++DEFAULT_MODEL = "gpt-4.1-nano"+++class CompatOpenAIRerankerClient(OpenAIRerankerClient):+ """Stock reranker with an OpenAI-compliant ``max_tokens`` floor."""++ async def rank(self, query: str, passages: list[str]) -> list[tuple[str, float]]:+ openai_messages_list: list[list[ChatCompletionMessageParam]] = [+ [+ ChatCompletionSystemMessageParam(+ role="system",+ content=(+ "You are an expert tasked with determining whether the "+ "passage is relevant to the query"+ ),+ ),+ ChatCompletionUserMessageParam(+ role="user",+ content=f"""+ Respond with "True" if PASSAGE is relevant to QUERY and "False" otherwise.+ <PASSAGE>+ {passage}+ </PASSAGE>+ <QUERY>+ {query}+ </QUERY>+ """,+ ),+ ]+ for passage in passages+ ]+ try:+ responses = await semaphore_gather(+ *[+ self.client.chat.completions.create(+ model=self.config.model or DEFAULT_MODEL,+ messages=openai_messages,+ temperature=0,+ max_tokens=MIN_COMPLETION_TOKENS,+ logit_bias={"6432": 1, "7983": 1},+ logprobs=True,+ top_logprobs=2,+ )+ for openai_messages in openai_messages_list+ ]+ )++ responses_top_logprobs = [+ (+ response.choices[0].logprobs.content[0].top_logprobs+ if response.choices[0].logprobs is not None+ and response.choices[0].logprobs.content is not None+ else []+ )+ for response in responses+ ]+ scores: list[float] = []+ for top_logprobs in responses_top_logprobs:+ if len(top_logprobs) == 0:+ # Keep scores aligned 1:1 with passages — a skipped+ # entry desyncs the lists and makes the+ # ``zip(..., strict=True)`` below raise ``ValueError``,+ # aborting the whole rerank. Treat a missing classifier+ # response as least-relevant (0.0).+ scores.append(0.0)+ continue+ norm_logprobs = np.exp(top_logprobs[0].logprob)+ if top_logprobs[0].token.strip().split(" ")[0].lower() == "true":+ scores.append(norm_logprobs)+ else:+ scores.append(1 - norm_logprobs)++ results = [+ (passage, score)+ for passage, score in zip(passages, scores, strict=True)+ ]+ results.sort(reverse=True, key=lambda x: x[1])+ return results+ except openai.RateLimitError as e:+ raise RateLimitError from e+ except Exception as e:+ logger.error(f"Error in generating LLM response: {e}")+ raise+++__all__ = ["CompatOpenAIRerankerClient", "LLMConfig", "MIN_COMPLETION_TOKENS"]diff --git a/autogpt_platform/backend/backend/copilot/graphiti/reranker_test.py b/autogpt_platform/backend/backend/copilot/graphiti/reranker_test.pynew file mode 100644index 0000000000..b87eaf305f--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/reranker_test.py@@ -0,0 +1,127 @@+"""Tests for the OpenAI-compat cross-encoder reranker patch.++Dev outage: graphiti-core 0.28.2's stock ``OpenAIRerankerClient.rank``+sends ``max_tokens=1`` per boolean-classifier call; OpenAI/OpenRouter+upstreams now reject anything below 16 with a 400, which crashed every+warm-context fetch AND starved the hit-time ratification loop (the+hit hook runs after the search gather that raised).+"""++from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest+from graphiti_core.llm_client import LLMConfig++from backend.copilot.graphiti.reranker import CompatOpenAIRerankerClient+++def _logprob_response(token: str, logprob: float) -> SimpleNamespace:+ top = SimpleNamespace(token=token, logprob=logprob)+ content = SimpleNamespace(top_logprobs=[top])+ logprobs = SimpleNamespace(content=[content])+ choice = SimpleNamespace(logprobs=logprobs)+ return SimpleNamespace(choices=[choice])+++def _empty_logprob_response() -> SimpleNamespace:+ """A response whose first token has no top_logprobs entries — the+ classifier returned nothing usable for that passage."""+ content = SimpleNamespace(top_logprobs=[])+ logprobs = SimpleNamespace(content=[content])+ choice = SimpleNamespace(logprobs=logprobs)+ return SimpleNamespace(choices=[choice])+++def _client_with(responses: list[SimpleNamespace]) -> tuple[MagicMock, list[dict]]:+ captured: list[dict] = []++ async def fake_create(**kwargs):+ captured.append(kwargs)+ return responses[len(captured) - 1]++ client = MagicMock()+ client.chat.completions.create = AsyncMock(side_effect=fake_create)+ return client, captured+++async def test_rank_requests_at_least_sixteen_max_tokens():+ """The whole point of the subclass: OpenAI rejects max_tokens < 16+ on logprob calls, so every classifier request must send >= 16."""+ client, captured = _client_with(+ [_logprob_response("True", -0.1), _logprob_response("False", -0.2)]+ )+ reranker = CompatOpenAIRerankerClient(+ config=LLMConfig(api_key="k", model="gpt-4.1-nano"), client=client+ )++ await reranker.rank("query", ["passage a", "passage b"])++ assert len(captured) == 2+ for kwargs in captured:+ assert kwargs["max_tokens"] >= 16+ # The boolean-classifier mechanics must survive the patch —+ # logit_bias pins output to True/False and logprobs drive scores.+ assert kwargs["logit_bias"] == {"6432": 1, "7983": 1}+ assert kwargs["logprobs"] is True+++async def test_rank_scores_true_above_false_and_sorts():+ """Scoring parity with upstream: 'True' top-token scores exp(lp),+ 'False' scores 1-exp(lp); results sorted descending."""+ client, _ = _client_with(+ [_logprob_response("False", -0.05), _logprob_response("True", -0.05)]+ )+ reranker = CompatOpenAIRerankerClient(+ config=LLMConfig(api_key="k", model="gpt-4.1-nano"), client=client+ )++ ranked = await reranker.rank("query", ["irrelevant", "relevant"])++ assert [passage for passage, _ in ranked] == ["relevant", "irrelevant"]+ assert ranked[0][1] > ranked[1][1]+++async def test_rank_handles_empty_logprobs_without_desync():+ """An empty ``top_logprobs`` must not be silently skipped — doing so+ desyncs scores from passages and makes ``zip(..., strict=True)`` raise+ ``ValueError``, aborting the entire rerank. The passage with a missing+ classifier response should score lowest and sort last instead."""+ client, _ = _client_with(+ [_logprob_response("True", -0.05), _empty_logprob_response()]+ )+ reranker = CompatOpenAIRerankerClient(+ config=LLMConfig(api_key="k", model="gpt-4.1-nano"), client=client+ )++ ranked = await reranker.rank("query", ["relevant", "no_response"])++ assert [passage for passage, _ in ranked] == ["relevant", "no_response"]+ assert dict(ranked)["no_response"] == 0.0+++def test_build_graphiti_uses_compat_reranker(mocker):+ """client.py must construct the patched reranker, not the stock+ one — otherwise the 400s come back on the next refactor."""+ from backend.copilot.graphiti import client as client_mod++ captured: dict = {}++ class _FakeGraphiti:+ def __init__(self, **kwargs):+ captured.update(kwargs)++ mocker.patch("graphiti_core.Graphiti", _FakeGraphiti)+ mocker.patch(+ "backend.copilot.graphiti.falkordb_driver.AutoGPTFalkorDriver",+ MagicMock(),+ )++ client_mod._build_graphiti("user_test", llm_client=MagicMock())++ assert isinstance(captured["cross_encoder"], CompatOpenAIRerankerClient)