Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): migrate Graphiti extraction and reranking to GPT-6 Luna (#15656)

Nicholas Tindlecb768a60f11a
diff --git a/autogpt_platform/backend/.env.default b/autogpt_platform/backend/.env.default
index 578cbbb6db..424e3c070e 100644
--- a/autogpt_platform/backend/.env.default
+++ b/autogpt_platform/backend/.env.default
@@ -148,7 +148,7 @@ CHAT_USE_LOCAL=false
GRAPHITI_FALKORDB_HOST=localhost
GRAPHITI_FALKORDB_PORT=6380
GRAPHITI_FALKORDB_PASSWORD=local-dev-password
-GRAPHITI_LLM_MODEL=gpt-4.1-mini
+GRAPHITI_LLM_MODEL=gpt-6-luna
GRAPHITI_EMBEDDER_MODEL=text-embedding-3-small
GRAPHITI_SEMAPHORE_LIMIT=5
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/client.py b/autogpt_platform/backend/backend/copilot/graphiti/client.py
index c1217732b5..8795be813a 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/client.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/client.py
@@ -224,7 +224,7 @@ def _build_graphiti(
# Runs concurrent boolean-classifier prompts (one per candidate
# edge) and uses log-probabilities to rank. Costs one small call per
# candidate (up to 2 * context_max_facts) on the reranker model
- # (gpt-4.1-mini by default) per session-start search. The Compat subclass
+ # (gpt-6-luna by default) per session-start search. The Compat subclass
# fixes the stock client's max_tokens=1, which OpenAI-compatible
# upstreams now reject with a 400 (minimum is 16).
if cross_encoder is None:
@@ -261,7 +261,7 @@ async def get_graphiti_client(group_id: str):
Returns a ``graphiti_core.Graphiti`` instance.
"""
- from graphiti_core.llm_client import OpenAIClient
+ from .llm_client import GraphitiOpenAIClient
state = _get_loop_state()
cache = state.cache
@@ -270,7 +270,7 @@ async def get_graphiti_client(group_id: str):
if group_id in cache:
return cache[group_id]
- llm_client = OpenAIClient(config=_build_llm_config())
+ llm_client = GraphitiOpenAIClient(config=_build_llm_config())
client = _build_graphiti(group_id, llm_client)
cache[group_id] = client
return client
@@ -337,14 +337,14 @@ async def make_flex_graphiti_client(group_id: str):
from .flex_client import FlexOpenAIClient
if not chat_cfg.transport.supports_flex_tier:
- from graphiti_core.llm_client import OpenAIClient
+ from .llm_client import GraphitiOpenAIClient
logger.info(
"Graphiti community rebuild: flex tier not available on "
"transport=%s; falling back to sync OpenAIClient.",
chat_cfg.transport.name,
)
- llm_client = OpenAIClient(config=_build_llm_config())
+ llm_client = GraphitiOpenAIClient(config=_build_llm_config())
else:
llm_client = FlexOpenAIClient(config=_build_llm_config())
return _build_graphiti(group_id, llm_client)
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/client_test.py b/autogpt_platform/backend/backend/copilot/graphiti/client_test.py
index ad8a2e5123..1860634dce 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/client_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/client_test.py
@@ -221,12 +221,11 @@ class TestMakeFlexGraphitiClient:
# ``make_flex_graphiti_client`` imports both classes lazily;
# patch them on their actual modules so the import inside the
# function picks up the sentinel.
- import graphiti_core.llm_client
-
from . import flex_client as flex_module
+ from . import llm_client as llm_module
monkeypatch.setattr(flex_module, "FlexOpenAIClient", flex_sentinel)
- monkeypatch.setattr(graphiti_core.llm_client, "OpenAIClient", regular_sentinel)
+ monkeypatch.setattr(llm_module, "GraphitiOpenAIClient", regular_sentinel)
await make_flex_graphiti_client("user_abc")
@@ -265,12 +264,11 @@ class TestMakeFlexGraphitiClient:
client_mod, "_build_llm_config", lambda: MagicMock(name="LLMConfig")
)
- import graphiti_core.llm_client
-
from . import flex_client as flex_module
+ from . import llm_client as llm_module
monkeypatch.setattr(flex_module, "FlexOpenAIClient", flex_sentinel)
- monkeypatch.setattr(graphiti_core.llm_client, "OpenAIClient", regular_sentinel)
+ monkeypatch.setattr(llm_module, "GraphitiOpenAIClient", regular_sentinel)
await make_flex_graphiti_client("user_abc")
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/config.py b/autogpt_platform/backend/backend/copilot/graphiti/config.py
index 791cf2a309..e9ccbe8979 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/config.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/config.py
@@ -13,6 +13,8 @@ from pydantic_settings import (
from backend.util.clients import OPENROUTER_BASE_URL
+from .models import DEFAULT_LLM_MODEL
+
_BACKEND_ROOT = Path(__file__).resolve().parents[3]
@@ -20,8 +22,8 @@ _BACKEND_ROOT = Path(__file__).resolve().parents[3]
# local transport. Pinned as module constants so the field defaults and
# the validator can't drift — same pattern as
# ``copilot/config.py::_DEFAULT_TITLE_MODEL`` etc.
-_DEFAULT_LLM_MODEL = "gpt-4.1-mini"
-_DEFAULT_RERANKER_MODEL = "gpt-4.1-mini"
+_DEFAULT_LLM_MODEL = DEFAULT_LLM_MODEL
+_DEFAULT_RERANKER_MODEL = DEFAULT_LLM_MODEL
_DEFAULT_EMBEDDER_MODEL = "text-embedding-3-small"
# Local-transport defaults. Mirrors dev's chat-side ``--with-ollama``
@@ -92,7 +94,7 @@ class GraphitiConfig(BaseSettings):
# rerank top edges from BM25 + cosine + BFS. Graphiti's built-in
# OpenAIRerankerClient runs concurrent boolean-classifier prompts
# (one prompt per candidate; log-probabilities decide the score).
- # This config defaults the classifier to gpt-4.1-mini — the same
+ # This config defaults the classifier to gpt-6-luna — the same
# cloud slug as ``llm_model`` — so the reranker can ship with no
# env config, at the cost of up to 2 * context_max_facts classifier
# calls per session start.
@@ -275,10 +277,10 @@ class GraphitiConfig(BaseSettings):
still at their cloud defaults and the operator hasn't pinned
a ``GRAPHITI_*_MODEL`` override:
- - ``llm_model`` (``gpt-4.1-mini``) → ``hf.co/ornith-ai/Ornith-1.5-9B-GGUF:Q4_K_M``
+ - ``llm_model`` (``gpt-6-luna``) → ``hf.co/ornith-ai/Ornith-1.5-9B-GGUF:Q4_K_M``
(matches dev's chat default — one Ollama pull powers both
surfaces; vetted for structured output / tool-calling shape).
- - ``reranker_model`` (``gpt-4.1-mini``) → same Ornith slug.
+ - ``reranker_model`` (``gpt-6-luna``) → same Ornith slug.
Reranker prompts are simpler than extraction; reusing the
chat model avoids pulling a second model just for reranking.
- ``embedder_model`` (``text-embedding-3-small``) →
@@ -289,7 +291,8 @@ class GraphitiConfig(BaseSettings):
Operator overrides (custom slugs like ``qwen3:8b`` or
``hf.co/...``) pass through untouched — only the literal cloud
- defaults trigger rewriting. Lazy-imports ``chat_cfg`` to avoid
+ defaults and the legacy ``gpt-4.1-mini`` default trigger rewriting.
+ Lazy-imports ``chat_cfg`` to avoid
the ``copilot.sdk.env`` ↔ ``util/clients`` cycle dev's PR
established.
"""
@@ -307,9 +310,9 @@ class GraphitiConfig(BaseSettings):
if chat_cfg.transport.name != "local":
return self
- if self.llm_model == _DEFAULT_LLM_MODEL:
+ if self.llm_model in (_DEFAULT_LLM_MODEL, "gpt-4.1-mini"):
object.__setattr__(self, "llm_model", _LOCAL_LLM_MODEL)
- if self.reranker_model == _DEFAULT_RERANKER_MODEL:
+ if self.reranker_model in (_DEFAULT_RERANKER_MODEL, "gpt-4.1-mini"):
object.__setattr__(self, "reranker_model", _LOCAL_RERANKER_MODEL)
if self.embedder_model == _DEFAULT_EMBEDDER_MODEL:
object.__setattr__(self, "embedder_model", _LOCAL_EMBEDDER_MODEL)
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/config_test.py b/autogpt_platform/backend/backend/copilot/graphiti/config_test.py
index 355d78e836..ee3378f296 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/config_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/config_test.py
@@ -191,15 +191,19 @@ class TestApplyLocalGraphitiModels:
``GRAPHITI_*_MODEL`` env var per slot. Mirrors dev's
``_apply_local_aux_models`` pattern on the chat side."""
- @pytest.mark.parametrize("reranker_override", [None, "gpt-4.1-mini"])
+ @pytest.mark.parametrize(
+ "model_override",
+ [None, "gpt-6-luna", "gpt-4.1-mini"],
+ )
def test_rewrites_cloud_defaults_under_local(
- self, monkeypatch: pytest.MonkeyPatch, reranker_override: str | None
+ self, monkeypatch: pytest.MonkeyPatch, model_override: str | None
) -> None:
_patch_chat_cfg(monkeypatch, _local_chat_cfg())
- if reranker_override is not None:
- monkeypatch.setenv("GRAPHITI_RERANKER_MODEL", reranker_override)
+ if model_override is not None:
+ monkeypatch.setenv("GRAPHITI_LLM_MODEL", model_override)
+ monkeypatch.setenv("GRAPHITI_RERANKER_MODEL", model_override)
cfg = GraphitiConfig()
- # llm_model: gpt-4.1-mini → Ornith-1.5-9B GGUF
+ # llm_model: gpt-6-luna → Ornith-1.5-9B GGUF
assert cfg.llm_model == "hf.co/ornith-ai/Ornith-1.5-9B-GGUF:Q4_K_M"
# reranker reuses the same model (simpler prompts, avoids a
# second Ollama pull).
@@ -226,7 +230,7 @@ class TestApplyLocalGraphitiModels:
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""OpenRouter / direct-Anthropic transports leave the cloud
- defaults alone — graphiti's ``gpt-4.1-mini`` stays put for
+ defaults alone — graphiti's ``gpt-6-luna`` stays put for
anyone who isn't on a self-hosted backend."""
_patch_chat_cfg(
monkeypatch,
@@ -237,12 +241,13 @@ class TestApplyLocalGraphitiModels:
),
)
cfg = GraphitiConfig()
- assert cfg.llm_model == "gpt-4.1-mini"
- assert cfg.reranker_model == "gpt-4.1-mini"
+ assert cfg.llm_model == "gpt-6-luna"
+ assert cfg.reranker_model == "gpt-6-luna"
assert cfg.embedder_model == "text-embedding-3-small"
- def test_reranker_env_override_under_cloud_transport(
- self, monkeypatch: pytest.MonkeyPatch
+ @pytest.mark.parametrize("model", ["gpt-4.1-mini", "gpt-5.6-luna"])
+ def test_model_env_overrides_under_cloud_transport(
+ self, monkeypatch: pytest.MonkeyPatch, model: str
) -> None:
_patch_chat_cfg(
monkeypatch,
@@ -252,9 +257,11 @@ class TestApplyLocalGraphitiModels:
base_url="https://openrouter.ai/api/v1",
),
)
- monkeypatch.setenv("GRAPHITI_RERANKER_MODEL", "gpt-5.6-luna")
+ monkeypatch.setenv("GRAPHITI_LLM_MODEL", model)
+ monkeypatch.setenv("GRAPHITI_RERANKER_MODEL", model)
cfg = GraphitiConfig()
- assert cfg.reranker_model == "gpt-5.6-luna"
+ assert cfg.llm_model == model
+ assert cfg.reranker_model == model
class TestIsEnabledForUser:
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/flex_client.py b/autogpt_platform/backend/backend/copilot/graphiti/flex_client.py
index 37ebff7755..5aad893645 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/flex_client.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/flex_client.py
@@ -1,6 +1,6 @@
"""Graphiti LLM client that runs on OpenAI's flex tier.
-Subclasses graphiti-core's ``OpenAIClient`` to inject
+Extends AutoGPT's ``GraphitiOpenAIClient`` compatibility adapter to inject
``service_tier="flex"`` on every underlying LLM call. Used by the
nightly community-rebuild path: ~50% discount, best-effort latency
(worst-case ~15min queue) — acceptable for scheduled non-interactive
@@ -24,16 +24,18 @@ does).
from typing import Any
-from graphiti_core.llm_client import OpenAIClient
from openai.types.chat import ChatCompletionMessageParam
from pydantic import BaseModel
+from .llm_client import GraphitiOpenAIClient
+from .models import is_luna_model
-class FlexOpenAIClient(OpenAIClient):
- """Same as graphiti's ``OpenAIClient`` with ``service_tier`` injected.
- Constructor mirrors ``OpenAIClient`` exactly so it slots into
- graphiti's ``Graphiti(llm_client=...)`` without surprises.
+class FlexOpenAIClient(GraphitiOpenAIClient):
+ """Add flex-tier routing to ``GraphitiOpenAIClient``.
+
+ Luna requests delegate to the parent adapter; other models retain the
+ upstream request shape with ``service_tier`` injected here.
"""
SERVICE_TIER = "flex"
@@ -48,6 +50,16 @@ class FlexOpenAIClient(OpenAIClient):
reasoning: str | None = None,
verbosity: str | None = None,
):
+ if is_luna_model(model):
+ return await super()._create_structured_completion(
+ model,
+ messages,
+ temperature,
+ max_tokens,
+ response_model,
+ reasoning,
+ verbosity,
+ )
is_reasoning_model = (
model.startswith("gpt-5")
or model.startswith("o1")
@@ -84,6 +96,16 @@ class FlexOpenAIClient(OpenAIClient):
reasoning: str | None = None,
verbosity: str | None = None,
):
+ if is_luna_model(model):
+ return await super()._create_completion(
+ model,
+ messages,
+ temperature,
+ max_tokens,
+ response_model,
+ reasoning,
+ verbosity,
+ )
is_reasoning_model = (
model.startswith("gpt-5")
or model.startswith("o1")
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/llm_client.py b/autogpt_platform/backend/backend/copilot/graphiti/llm_client.py
new file mode 100644
index 0000000000..220e8dcec0
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/graphiti/llm_client.py
@@ -0,0 +1,81 @@
+from typing import Any
+
+from graphiti_core.llm_client import OpenAIClient
+from openai.types.chat import ChatCompletionMessageParam
+from pydantic import BaseModel
+
+from .models import is_luna_model
+
+
+class GraphitiOpenAIClient(OpenAIClient):
+ """Preserve non-reasoning extraction on Luna with graphiti-core 0.30.2."""
+
+ SERVICE_TIER: str | None = None
+
+ async def _create_structured_completion(
+ self,
+ model: str,
+ messages: list[ChatCompletionMessageParam],
+ temperature: float | None,
+ max_tokens: int,
+ response_model: type[BaseModel],
+ reasoning: str | None = None,
+ verbosity: str | None = None,
+ ):
+ if not is_luna_model(model):
+ return await super()._create_structured_completion(
+ model,
+ messages,
+ temperature,
+ max_tokens,
+ response_model,
+ reasoning,
+ verbosity,
+ )
+
+ kwargs: dict[str, Any] = {
+ "model": model,
+ "input": messages,
+ "max_output_tokens": max_tokens,
+ "text_format": response_model,
+ "reasoning": {"effort": "none"},
+ }
+ if temperature is not None:
+ kwargs["temperature"] = temperature
+ if self.SERVICE_TIER:
+ kwargs["extra_body"] = {"service_tier": self.SERVICE_TIER}
+ return await self.client.responses.parse(**kwargs)
+
+ async def _create_completion(
+ self,
+ model: str,
+ messages: list[ChatCompletionMessageParam],
+ temperature: float | None,
+ max_tokens: int,
+ response_model: type[BaseModel] | None = None,
+ reasoning: str | None = None,
+ verbosity: str | None = None,
+ ):
+ if not is_luna_model(model):
+ return await super()._create_completion(
+ model,
+ messages,
+ temperature,
+ max_tokens,
+ response_model,
+ reasoning,
+ verbosity,
+ )
+
+ kwargs: dict[str, Any] = {
+ "model": model,
+ "messages": messages,
+ "max_completion_tokens": max_tokens,
+ "response_format": {"type": "json_object"},
+ "reasoning_effort": "none",
+ }
+ if temperature is not None:
+ kwargs["temperature"] = temperature
+ if self.SERVICE_TIER:
+ kwargs["extra_body"] = {"service_tier": self.SERVICE_TIER}
+ return await self.client.chat.completions.create(**kwargs)
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/llm_client_test.py b/autogpt_platform/backend/backend/copilot/graphiti/llm_client_test.py
new file mode 100644
index 0000000000..36529c140e
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/graphiti/llm_client_test.py
@@ -0,0 +1,134 @@
+import json
+
+import httpx
+import pytest
+from graphiti_core.llm_client import LLMConfig
+from graphiti_core.llm_client.config import ModelSize
+from graphiti_core.prompts import Message
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+
+from backend.copilot.graphiti.flex_client import FlexOpenAIClient
+from backend.copilot.graphiti.llm_client import GraphitiOpenAIClient
+
+
+class ExtractedFact(BaseModel):
+ fact: str
+
+
[email protected]("client_type", [GraphitiOpenAIClient, FlexOpenAIClient])
[email protected]("structured", [True, False])
[email protected]("model_size", [ModelSize.small, ModelSize.medium])
+ "model",
+ [
+ "gpt-6-luna",
+ "openai/gpt-6-luna",
+ "gpt-4.1-mini",
+ "qwen3:8b",
+ ],
+)
+async def test_extraction_request_contract(
+ client_type, structured, model_size, model, request_boundary
+):
+ captured, transport = request_boundary
+
+ async with AsyncOpenAI(
+ api_key="test",
+ base_url="https://provider.invalid/v1",
+ max_retries=0,
+ http_client=httpx.AsyncClient(transport=transport),
+ ) as sdk:
+ client = client_type(
+ config=LLMConfig(model=model, small_model=model), client=sdk
+ )
+ result = await client.generate_response(
+ [Message(role="user", content="Extract JSON facts: Mira lives in Oslo.")],
+ response_model=ExtractedFact if structured else None,
+ max_tokens=256,
+ model_size=model_size,
+ )
+
+ assert result == {"fact": "Mira lives in Oslo"}
+ assert len(captured) == 1
+ _assert_request(captured[0], model, structured, client_type is FlexOpenAIClient)
+
+
+def _assert_request(body, model, structured, flex):
+ assert body["model"] == model
+ assert body["temperature"] == 1
+ if "gpt-6-luna" in model:
+ if structured:
+ assert body["reasoning"] == {"effort": "none"}
+ else:
+ assert body["reasoning_effort"] == "none"
+ assert body["max_completion_tokens"] == 256
+ assert "max_tokens" not in body
+ else:
+ assert "reasoning" not in body
+ assert "reasoning_effort" not in body
+ if not structured:
+ assert body["max_tokens"] == 256
+ if structured:
+ assert body["max_output_tokens"] == 256
+ assert body["text"]["format"]["type"] == "json_schema"
+ assert body["text"]["format"]["strict"] is True
+ else:
+ assert body["response_format"] == {"type": "json_object"}
+ if flex:
+ assert body["service_tier"] == "flex"
+ else:
+ assert "service_tier" not in body
+
+
+def request_boundary():
+ captured = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ body = json.loads(request.content)
+ captured.append(body)
+ content = '{"fact":"Mira lives in Oslo"}'
+ if request.url.path.endswith("/responses"):
+ payload = {
+ "id": "resp_test",
+ "object": "response",
+ "created_at": 0,
+ "model": body["model"],
+ "status": "completed",
+ "output": [
+ {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [
+ {"type": "output_text", "text": content, "annotations": []}
+ ],
+ }
+ ],
+ "usage": {"input_tokens": 20, "output_tokens": 10, "total_tokens": 30},
+ }
+ else:
+ payload = {
+ "id": "chat_test",
+ "object": "chat.completion",
+ "created": 0,
+ "model": body["model"],
+ "choices": [
+ {
+ "index": 0,
+ "finish_reason": "stop",
+ "message": {"role": "assistant", "content": content},
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 20,
+ "completion_tokens": 10,
+ "total_tokens": 30,
+ },
+ }
+ return httpx.Response(200, json=payload)
+
+ return captured, httpx.MockTransport(respond)
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/models.py b/autogpt_platform/backend/backend/copilot/graphiti/models.py
new file mode 100644
index 0000000000..f9e324bbd5
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/graphiti/models.py
@@ -0,0 +1,5 @@
+DEFAULT_LLM_MODEL = "gpt-6-luna"
+
+
+def is_luna_model(model: str) -> bool:
+ return model.removeprefix("openai/") == DEFAULT_LLM_MODEL
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/reranker.py b/autogpt_platform/backend/backend/copilot/graphiti/reranker.py
index 4bef93cf4b..d99553f6ad 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/reranker.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/reranker.py
@@ -1,26 +1,13 @@
-"""OpenAI-compat cross-encoder reranker for warm-context retrieval.
-
-graphiti-core 0.28.2's stock ``OpenAIRerankerClient.rank`` sends
-``max_tokens=1`` on each boolean-classifier call. OpenAI (and
-OpenRouter's OpenAI/Azure upstreams) now reject completions below 16
-output tokens ("Invalid 'max_output_tokens': integer below minimum
-value. Expected a value >= 16, but got 1"), which turned every
-warm-context fetch into a 400 — and, because the ratification hit
-hook runs after the search gather that raised, also starved the
-hit-time ratification loop.
-
-``CompatOpenAIRerankerClient.rank`` is a faithful copy of the
-upstream method (pinned at graphiti-core 0.28.2) with ``max_tokens``
-raised to a compliant floor. Scoring is unchanged: the ``logit_bias``
-pins the output to True/False, and only the FIRST token's
-top-logprobs are read, so the extra token budget never affects the
-ranking (same seam-level patch pattern as
-``communities._bounded_label_propagation``).
+"""Boolean/logprob reranking with a 16-token floor and non-reasoning Luna.
+
+Luna omits the legacy tokenizer's logit bias; only True/False first-token
+probabilities supply relevance scores. Unusable responses score zero.
"""
from __future__ import annotations
import logging
+from typing import Any
import numpy as np
import openai
@@ -33,20 +20,28 @@ from openai.types.chat import (
ChatCompletionUserMessageParam,
)
+from .models import DEFAULT_LLM_MODEL, is_luna_model
+
logger = logging.getLogger(__name__)
-# OpenAI's documented minimum for max output tokens on logprob
-# completions. The classifier still only ever emits one token (the
-# logit_bias restricts the vocabulary to "True"/"False").
MIN_COMPLETION_TOKENS = 16
-DEFAULT_MODEL = "gpt-4.1-mini"
+DEFAULT_MODEL = DEFAULT_LLM_MODEL
class CompatOpenAIRerankerClient(OpenAIRerankerClient):
- """Stock reranker with an OpenAI-compliant ``max_tokens`` floor."""
+ """Graphiti reranker with compatible request parameters per model."""
async def rank(self, query: str, passages: list[str]) -> list[tuple[str, float]]:
+ model = self.config.model or DEFAULT_MODEL
+ options: dict[str, Any] = (
+ {"reasoning_effort": "none", "max_completion_tokens": MIN_COMPLETION_TOKENS}
+ if is_luna_model(model)
+ else {
+ "max_tokens": MIN_COMPLETION_TOKENS,
+ "logit_bias": {"6432": 1, "7983": 1},
+ }
+ )
openai_messages_list: list[list[ChatCompletionMessageParam]] = [
[
ChatCompletionSystemMessageParam(
@@ -75,13 +70,12 @@ class CompatOpenAIRerankerClient(OpenAIRerankerClient):
responses = await semaphore_gather(
*[
self.client.chat.completions.create(
- model=self.config.model or DEFAULT_MODEL,
+ model=model,
messages=openai_messages,
temperature=0,
- max_tokens=MIN_COMPLETION_TOKENS,
- logit_bias={"6432": 1, "7983": 1},
logprobs=True,
top_logprobs=2,
+ **options,
)
for openai_messages in openai_messages_list
]
@@ -91,12 +85,13 @@ class CompatOpenAIRerankerClient(OpenAIRerankerClient):
(
response.choices[0].logprobs.content[0].top_logprobs
if response.choices[0].logprobs is not None
- and response.choices[0].logprobs.content is not None
+ and response.choices[0].logprobs.content
else []
)
for response in responses
]
scores: list[float] = []
+ non_boolean_count = 0
for top_logprobs in responses_top_logprobs:
if len(top_logprobs) == 0:
# Keep scores aligned 1:1 with passages — a skipped
@@ -107,10 +102,20 @@ class CompatOpenAIRerankerClient(OpenAIRerankerClient):
scores.append(0.0)
continue
norm_logprobs = np.exp(top_logprobs[0].logprob)
- if top_logprobs[0].token.strip().split(" ")[0].lower() == "true":
+ token = top_logprobs[0].token.strip().lower()
+ if token == "true":
scores.append(norm_logprobs)
- else:
+ elif token == "false":
scores.append(1 - norm_logprobs)
+ else:
+ non_boolean_count += 1
+ scores.append(0.0)
+
+ if non_boolean_count:
+ logger.warning(
+ f"Reranker model {model} returned {non_boolean_count} "
+ "non-boolean responses; scoring zero"
+ )
results = [
(passage, score)
@@ -125,4 +130,4 @@ class CompatOpenAIRerankerClient(OpenAIRerankerClient):
raise
-__all__ = ["CompatOpenAIRerankerClient", "LLMConfig", "MIN_COMPLETION_TOKENS"]
+__all__ = ["MIN_COMPLETION_TOKENS", "CompatOpenAIRerankerClient", "LLMConfig"]
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/reranker_test.py b/autogpt_platform/backend/backend/copilot/graphiti/reranker_test.py
index 0b2cfb0705..b9f2b1a98d 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/reranker_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/reranker_test.py
@@ -20,6 +20,25 @@ from backend.copilot.graphiti.config import GraphitiConfig
from backend.copilot.graphiti.reranker import CompatOpenAIRerankerClient
[email protected]("model", ["gpt-6-luna", "openai/gpt-6-luna"])
+async def test_luna_reranker_disables_reasoning_without_legacy_token_bias(model):
+ client, captured = _client_with([_logprob_response("True", -0.1)])
+ reranker = CompatOpenAIRerankerClient(
+ config=LLMConfig(api_key="k", model=model), client=client
+ )
+
+ ranked = await reranker.rank("query", ["passage"])
+
+ assert ranked[0][1] > 0.9
+ assert captured[0]["reasoning_effort"] == "none"
+ assert captured[0]["max_completion_tokens"] == 16
+ assert captured[0]["logprobs"] is True
+ assert captured[0]["top_logprobs"] == 2
+ assert "max_tokens" not in captured[0]
+ assert "logit_bias" not in captured[0]
+
+
def _logprob_response(token: str, logprob: float) -> SimpleNamespace:
top = SimpleNamespace(token=token, logprob=logprob)
content = SimpleNamespace(top_logprobs=[top])
@@ -66,13 +85,13 @@ async def test_rank_requests_at_least_sixteen_max_tokens():
for kwargs in captured:
assert kwargs["max_tokens"] >= 16
# The boolean-classifier mechanics must survive the patch —
- # logit_bias pins output to True/False and logprobs drive scores.
+ # Legacy logit bias nudges True/False and logprobs drive scores.
assert kwargs["logit_bias"] == {"6432": 1, "7983": 1}
assert kwargs["logprobs"] is True
@pytest.mark.asyncio
-async def test_rank_scores_true_above_false_and_sorts():
+async def test_rank_scores_true_above_false_and_sorts(caplog):
"""Scoring parity with upstream: 'True' top-token scores exp(lp),
'False' scores 1-exp(lp); results sorted descending."""
client, _ = _client_with(
@@ -86,6 +105,9 @@ async def test_rank_scores_true_above_false_and_sorts():
assert [passage for passage, _ in ranked] == ["relevant", "irrelevant"]
assert ranked[0][1] > ranked[1][1]
+ assert not any(
+ record.name == "backend.copilot.graphiti.reranker" for record in caplog.records
+ )
@pytest.mark.asyncio
@@ -107,12 +129,84 @@ async def test_rank_handles_empty_logprobs_without_desync():
assert dict(ranked)["no_response"] == 0.0
[email protected]("model", ["gpt-6-luna", "gpt-4.1-mini"])
[email protected]("token", ["Yes", "No", "\n", '"', "True.", "True enough"])
+async def test_rank_rejects_non_boolean_tokens(model, token):
+ client, _ = _client_with(
+ [
+ _logprob_response(token, -0.7),
+ _logprob_response("True", -0.05),
+ _logprob_response("False", -0.05),
+ ]
+ )
+ reranker = CompatOpenAIRerankerClient(
+ config=LLMConfig(api_key="k", model=model), client=client
+ )
+
+ ranked = await reranker.rank("query", ["unscorable", "relevant", "irrelevant"])
+
+ assert [passage for passage, _ in ranked] == [
+ "relevant",
+ "irrelevant",
+ "unscorable",
+ ]
+ assert dict(ranked)["unscorable"] == 0.0
+ assert dict(ranked)["relevant"] > 0.9
+ assert 0 < dict(ranked)["irrelevant"] < 0.1
+
+
+async def test_rank_handles_empty_token_content():
+ empty_response = _empty_logprob_response()
+ empty_response.choices[0].logprobs.content = []
+ client, _ = _client_with([empty_response, _logprob_response("True", -0.05)])
+ reranker = CompatOpenAIRerankerClient(
+ config=LLMConfig(api_key="k", model="gpt-6-luna"), client=client
+ )
+
+ ranked = await reranker.rank("query", ["unscorable", "relevant"])
+
+ assert [passage for passage, _ in ranked] == ["relevant", "unscorable"]
+ assert dict(ranked)["unscorable"] == 0.0
+
+
+async def test_rank_aggregates_warnings_without_logging_token_content(caplog):
+ tokens = ["private-output-one", "private-output-two"]
+ client, _ = _client_with(
+ [_logprob_response(token, -0.1) for token in tokens]
+ + [_logprob_response("True", -0.05)]
+ )
+ reranker = CompatOpenAIRerankerClient(
+ config=LLMConfig(api_key="k", model="gpt-6-luna"), client=client
+ )
+
+ ranked = await reranker.rank("query", ["unscorable1", "unscorable2", "relevant"])
+
+ warnings = [
+ record
+ for record in caplog.records
+ if record.name == "backend.copilot.graphiti.reranker"
+ ]
+ assert len(warnings) == 1
+ assert "2 non-boolean" in warnings[0].message
+ assert "gpt-6-luna" in warnings[0].message
+ assert all(token not in warnings[0].message for token in tokens)
+ assert [passage for passage, _ in ranked] == [
+ "relevant",
+ "unscorable1",
+ "unscorable2",
+ ]
+ assert dict(ranked)["unscorable1"] == dict(ranked)["unscorable2"] == 0.0
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model, expected_model",
[
- (None, "gpt-4.1-mini"),
- ("", "gpt-4.1-mini"),
+ (None, "gpt-6-luna"),
+ ("", "gpt-6-luna"),
("custom-reranker", "custom-reranker"),
],
)
@@ -160,4 +254,4 @@ async def test_build_graphiti_uses_compat_reranker(mocker):
await reranker.rank("query", ["passage"])
- assert requests[0]["model"] == "gpt-4.1-mini"
+ assert requests[0]["model"] == "gpt-6-luna"
diff --git a/docs/platform/copilot-local-llm.md b/docs/platform/copilot-local-llm.md
index 03953c0dc4..81a4e75f0a 100644
--- a/docs/platform/copilot-local-llm.md
+++ b/docs/platform/copilot-local-llm.md
@@ -390,8 +390,8 @@ rewrites the cloud OpenAI defaults to local Ollama equivalents:
| Setting | Cloud default | Local default |
|---|---|---|
-| `GRAPHITI_LLM_MODEL` | `gpt-4.1-mini` | `hf.co/ornith-ai/Ornith-1.5-9B-GGUF:Q4_K_M` |
-| `GRAPHITI_RERANKER_MODEL` | `gpt-4.1-mini` | `hf.co/ornith-ai/Ornith-1.5-9B-GGUF:Q4_K_M` |
+| `GRAPHITI_LLM_MODEL` | `gpt-6-luna` | `hf.co/ornith-ai/Ornith-1.5-9B-GGUF:Q4_K_M` |
+| `GRAPHITI_RERANKER_MODEL` | `gpt-6-luna` | `hf.co/ornith-ai/Ornith-1.5-9B-GGUF:Q4_K_M` |
| `GRAPHITI_EMBEDDER_MODEL` | `text-embedding-3-small` | `nomic-embed-text` |
The LLM + reranker reuse the same Ornith 1.5 9B model the `--with-ollama`
@@ -401,7 +401,8 @@ the next section.
You can pin your own slugs at any time by setting the matching
`GRAPHITI_*_MODEL` env var; the validator only touches slots still at
-their cloud default. A custom slug (`qwen3:8b`, `hf.co/...`,
+their cloud default, including the previous `gpt-4.1-mini` default in
+existing `.env` files. A custom slug (`qwen3:8b`, `hf.co/...`,
`my-registry.io/model:tag`) passes through untouched.
### Embeddings require an embedding model pulled into Ollama