Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix(agents): preserve Gemini tool-call thought signatures (#889)

Vítor Alves2a207c1312dd
diff --git a/backend/app/agents/models/conversation.py b/backend/app/agents/models/conversation.py
index ce4dd9ae..d8c34e76 100644
--- a/backend/app/agents/models/conversation.py
+++ b/backend/app/agents/models/conversation.py
@@ -50,7 +50,7 @@ class Message(Base):
role: Mapped[str] = mapped_column(String(20))
content: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
- # When role="assistant" with tool calls, list of {id,name,arguments}.
+ # When role="assistant", list of {id,name,arguments,thought_signature?}.
tool_calls: Mapped[Optional[list]] = mapped_column(JSON, nullable=True)
# When role="tool", payload is {tool_call_id, server, name, result, error?}.
tool_result: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)
diff --git a/backend/app/agents/providers/base.py b/backend/app/agents/providers/base.py
index a2d8fec3..ef23f581 100644
--- a/backend/app/agents/providers/base.py
+++ b/backend/app/agents/providers/base.py
@@ -51,6 +51,8 @@ class ToolCall:
id: str
name: str
arguments: dict[str, Any]
+ # Opaque Gemini signature; replay unchanged, never pass to the tool itself.
+ thought_signature: Optional[str] = None
@dataclass
@@ -87,6 +89,8 @@ class ChatChunk:
args_delta: Optional[str] = None # incremental JSON args while streaming
finish_reason: Optional[str] = None
usage: Optional[Usage] = None
+ # Delivered on tool_call_end, after all streaming deltas have arrived.
+ thought_signature: Optional[str] = None
@dataclass
@@ -147,6 +151,8 @@ class LLMProvider(ABC):
elif chunk.type == "tool_call_args_delta" and chunk.tool_call_id:
tc = tool_calls.setdefault(chunk.tool_call_id, {"id": chunk.tool_call_id, "name": chunk.tool_name or "", "args_buf": ""})
tc["args_buf"] += chunk.args_delta or ""
+ elif chunk.type == "tool_call_end" and chunk.tool_call_id in tool_calls:
+ tool_calls[chunk.tool_call_id]["thought_signature"] = chunk.thought_signature
elif chunk.type == "finish" and chunk.finish_reason:
finish = chunk.finish_reason
elif chunk.type == "usage" and chunk.usage:
@@ -158,7 +164,10 @@ class LLMProvider(ABC):
args = json.loads(tc["args_buf"]) if tc["args_buf"] else {}
except json.JSONDecodeError:
args = {"_raw": tc["args_buf"]}
- parsed_calls.append(ToolCall(id=tc["id"], name=tc["name"], arguments=args))
+ parsed_calls.append(ToolCall(
+ id=tc["id"], name=tc["name"], arguments=args,
+ thought_signature=tc.get("thought_signature"),
+ ))
return ChatResponse(content="".join(text_parts), tool_calls=parsed_calls, finish_reason=finish, usage=usage)
@abstractmethod
diff --git a/backend/app/agents/providers/openai.py b/backend/app/agents/providers/openai.py
index b7e57078..3551e980 100644
--- a/backend/app/agents/providers/openai.py
+++ b/backend/app/agents/providers/openai.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import re
from typing import AsyncIterator, Optional
from urllib.parse import urlparse
@@ -30,7 +31,7 @@ def normalize_openai_base_url(url: str) -> str:
Rules:
- empty → returned as-is
- trailing slashes stripped
- - if any path segment looks like a version (`v1`, `v2`, `beta`,
+ - if any path segment looks like a version (`v1`, `v2`, `v1beta`, `beta`,
`latest`, `api/v3`, etc.), leave the URL alone
- otherwise, append `/v1`
"""
@@ -40,7 +41,7 @@ def normalize_openai_base_url(url: str) -> str:
path_segments = [s for s in (urlparse(url).path or "").split("/") if s]
for seg in path_segments:
if (
- (seg.startswith("v") and seg[1:].isdigit())
+ re.fullmatch(r"v\d+(?:(?:alpha|beta)\d*)?", seg)
or seg in {"beta", "latest"}
):
return url
@@ -59,6 +60,8 @@ def _serialize_messages(messages: list[ChatMessage]) -> list[dict]:
"id": tc.id,
"type": "function",
"function": {"name": tc.name, "arguments": json.dumps(tc.arguments)},
+ **({"extra_content": {"google": {"thought_signature": tc.thought_signature}}}
+ if tc.thought_signature is not None else {}),
}
for tc in m.tool_calls
]
@@ -174,6 +177,13 @@ class OpenAIProvider(LLMProvider):
"started": False,
"pending_args": "",
})
+ # Gemini's OpenAI endpoint puts the opaque signature
+ # beside function, potentially in a later delta.
+ extra = tc.get("extra_content")
+ google = extra.get("google") if isinstance(extra, dict) else None
+ signature = google.get("thought_signature") if isinstance(google, dict) else None
+ if isinstance(signature, str):
+ state["thought_signature"] = signature
# First chunk usually carries both id and name;
# some servers split the name across chunks.
if tc.get("id") and not state["id"]:
@@ -212,7 +222,10 @@ class OpenAIProvider(LLMProvider):
finish_reason = choice["finish_reason"]
for state in idx_state.values():
if state["started"]:
- yield ChatChunk(type="tool_call_end", tool_call_id=state["id"])
+ yield ChatChunk(
+ type="tool_call_end", tool_call_id=state["id"],
+ thought_signature=state.get("thought_signature"),
+ )
if usage:
yield ChatChunk(type="usage", usage=usage)
yield ChatChunk(type="finish", finish_reason=finish_reason or "stop")
diff --git a/backend/app/agents/runtime/executor.py b/backend/app/agents/runtime/executor.py
index 418e6438..fb27f7fe 100644
--- a/backend/app/agents/runtime/executor.py
+++ b/backend/app/agents/runtime/executor.py
@@ -386,7 +386,10 @@ class AgentExecutor:
for m in history:
tcs = []
for raw in (m.tool_calls or []):
- tcs.append(ToolCall(id=raw.get("id"), name=raw.get("name"), arguments=raw.get("arguments") or {}))
+ tcs.append(ToolCall(
+ id=raw.get("id"), name=raw.get("name"), arguments=raw.get("arguments") or {},
+ thought_signature=raw.get("thought_signature"),
+ ))
tool_call_id = (m.tool_result or {}).get("tool_call_id") if m.role == "tool" else None
content = m.content
if m.role == "tool":
@@ -477,7 +480,10 @@ class AgentExecutor:
args = json.loads(tc["args_buf"]) if tc["args_buf"] else {}
except json.JSONDecodeError:
args = {"_raw": tc["args_buf"]}
- assembled_calls.append(ToolCall(id=tc["id"], name=tc["name"], arguments=args))
+ assembled_calls.append(ToolCall(
+ id=tc["id"], name=tc["name"], arguments=args,
+ thought_signature=tc.get("thought_signature"),
+ ))
# Persist assistant turn.
assistant_msg = await conversation_service.append_message(
@@ -485,7 +491,11 @@ class AgentExecutor:
conversation_id=conversation_id,
role="assistant",
content=assistant_text or None,
- tool_calls=[{"id": c.id, "name": c.name, "arguments": c.arguments} for c in assembled_calls] or None,
+ tool_calls=[{
+ "id": c.id, "name": c.name, "arguments": c.arguments,
+ **({"thought_signature": c.thought_signature}
+ if c.thought_signature is not None else {}),
+ } for c in assembled_calls] or None,
input_tokens=usage_input or None,
output_tokens=usage_output or None,
)
@@ -599,6 +609,8 @@ async def _process_chunk(chunk: ChatChunk, text_buf: list[str], open_calls: dict
elif chunk.type == "tool_call_args_delta" and chunk.tool_call_id:
tc = open_calls.setdefault(chunk.tool_call_id, {"id": chunk.tool_call_id, "name": "", "args_buf": ""})
tc["args_buf"] += chunk.args_delta or ""
+ elif chunk.type == "tool_call_end" and chunk.tool_call_id in open_calls:
+ open_calls[chunk.tool_call_id]["thought_signature"] = chunk.thought_signature
async def _safe_call_tool(
diff --git a/backend/tests/test_agents_executor.py b/backend/tests/test_agents_executor.py
index 9771f8b2..1f9a8785 100644
--- a/backend/tests/test_agents_executor.py
+++ b/backend/tests/test_agents_executor.py
@@ -10,6 +10,7 @@ Pattern:
- We patch `_provider_for` and pass our fake MCP into AgentExecutor.
"""
import uuid
+from copy import deepcopy
from typing import AsyncIterator
from unittest.mock import patch
@@ -45,10 +46,12 @@ class _ScriptedProvider(LLMProvider):
def __init__(self, turns: list[list[ChatChunk]]):
super().__init__(api_key="x")
self._turns = list(turns)
+ self.requests = []
async def chat_stream( # type: ignore[override]
self, messages, *, model, tools=None, temperature=0.4, max_tokens=None
) -> AsyncIterator[ChatChunk]:
+ self.requests.append(deepcopy(messages))
if not self._turns:
# No more scripted turns — emit a generic finish.
yield ChatChunk(type="finish", finish_reason="stop")
@@ -160,7 +163,8 @@ async def test_usage_row_recorded(session, test_user, test_agent, test_conversat
assert float(rows[0].cost_usd) > 0
-async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_agent, test_conversation):
[email protected]("signature", [None, "opaque-signature+/=="])
+async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_agent, test_conversation, signature):
"""LLM emits a tool call; executor runs it via fake MCP and feeds the
result back; second turn returns a plain answer."""
tools = [ToolHandle(server="securo", name="list_accounts", description="d", parameters={"type": "object"})]
@@ -173,7 +177,7 @@ async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_
[
ChatChunk(type="tool_call_start", tool_call_id="t1", tool_name="securo__list_accounts"),
ChatChunk(type="tool_call_args_delta", tool_call_id="t1", args_delta="{}"),
- ChatChunk(type="tool_call_end", tool_call_id="t1"),
+ ChatChunk(type="tool_call_end", tool_call_id="t1", thought_signature=signature),
ChatChunk(type="usage", usage=Usage(input_tokens=20, output_tokens=5)),
ChatChunk(type="finish", finish_reason="tool_calls"),
],
@@ -220,6 +224,24 @@ async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_
)).scalars().all()
assert len(usage_rows) == 2
+ saved_call = msgs[1].tool_calls[0]
+ if signature is None:
+ assert "thought_signature" not in saved_call
+ else:
+ assert saved_call["thought_signature"] == signature
+ replayed = [tc for m in provider.requests[1] for tc in m.tool_calls]
+ assert replayed[0].thought_signature == signature
+
+ # Start a separate request so history is reconstructed from persisted JSON.
+ session.expire(msgs[1])
+ with _patch_provider(provider):
+ await _drain(
+ executor, session=session, agent=test_agent, user_id=test_user.id,
+ conversation_id=test_conversation.id, user_message="thanks",
+ )
+ restored = [tc for m in provider.requests[2] for tc in m.tool_calls]
+ assert restored[0].thought_signature == signature
+
async def test_provider_auth_error_surfaced_as_friendly_message(
session, test_user, test_agent, test_conversation
diff --git a/backend/tests/test_agents_providers.py b/backend/tests/test_agents_providers.py
index 551cffe8..7334b564 100644
--- a/backend/tests/test_agents_providers.py
+++ b/backend/tests/test_agents_providers.py
@@ -120,6 +120,7 @@ def test_anthropic_tool_result_block():
("http://lmstudio:1234/v1/", "http://lmstudio:1234/v1"),
("https://api.openai.com/v1", "https://api.openai.com/v1"),
("https://api.groq.com/openai/v1", "https://api.groq.com/openai/v1"),
+ ("https://generativelanguage.googleapis.com/v1beta/openai/", "https://generativelanguage.googleapis.com/v1beta/openai"),
("https://api.example.com/v2", "https://api.example.com/v2"),
("https://api.example.com/beta", "https://api.example.com/beta"),
("https://api.example.com/", "https://api.example.com/v1"),
diff --git a/backend/tests/test_agents_providers_streaming.py b/backend/tests/test_agents_providers_streaming.py
index 352073c7..80d7bde2 100644
--- a/backend/tests/test_agents_providers_streaming.py
+++ b/backend/tests/test_agents_providers_streaming.py
@@ -1,5 +1,5 @@
"""Coverage for the streaming chat + embed paths in
-app/agents/providers/{ollama,anthropic}.py. Both providers open
+app/agents/providers/{openai,ollama,anthropic}.py. These providers open
httpx.AsyncClient.stream(...) inline, so we replace AsyncClient with a
fake whose stream() yields a pre-baked SSE/JSONL response.
"""
@@ -23,6 +23,8 @@ from app.agents.providers.base import (
ToolDefinition,
)
from app.agents.providers.ollama import OllamaProvider
+from app.agents.providers.openai import OpenAIProvider
+from app.agents.providers.openai_compatible import OpenAICompatibleProvider
# --------------------------------------------------------------------- httpx fake
@@ -89,6 +91,62 @@ def _fake_httpx(monkeypatch):
yield
+# --------------------------------------------------------------------- OpenAI-compatible: chat_stream
+
[email protected]("provider_class", [OpenAICompatibleProvider, OpenAIProvider])
[email protected]("signature_timing", ["first", "late", "absent"])
+async def test_openai_tool_signatures_survive_multiple_rounds(provider_class, signature_timing):
+ """Keep signatures on their own calls across parallel and sequential tools."""
+ def response(deltas):
+ return _FakeStreamResponse(status_code=200, lines=[
+ "data: " + json.dumps({"choices": [{"delta": delta}]}) for delta in deltas
+ ] + ['data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}', "data: [DONE]"])
+
+ signature_a = "opaque-A+/=="
+ first = {"index": 0, "id": "a", "function": {"name": "lookup", "arguments": '{"city":'}}
+ extra_a = {"google": {"thought_signature": signature_a}}
+ if signature_timing == "first":
+ first["extra_content"] = extra_a
+ deltas = [
+ {"tool_calls": [first]},
+ {"tool_calls": [{"index": 1, "id": "b", "function": {"name": "lookup", "arguments": "{}"}}]},
+ {"tool_calls": [{"index": 0, "function": {"arguments": '"Paris"}'}}]},
+ ]
+ if signature_timing == "late":
+ deltas.append({"tool_calls": [{"index": 0, "extra_content": extra_a}]})
+ second_call = {"index": 0, "id": "c", "function": {"name": "lookup", "arguments": "{}"}}
+ if signature_timing != "absent":
+ second_call["extra_content"] = {"google": {"thought_signature": "opaque-B=="}}
+ _FakeAsyncClient.queue_stream.extend([
+ response(deltas), response([{"tool_calls": [second_call]}]),
+ response([{"content": "Done"}]),
+ ])
+ provider = provider_class(base_url="https://generativelanguage.googleapis.com/v1beta/openai")
+ messages = [ChatMessage(role="user", content="Look up cities")]
+ for round_index in range(2):
+ result = await provider.chat(messages, model="gemini-test")
+ if round_index == 0:
+ assert [tc.id for tc in result.tool_calls] == ["a", "b"]
+ assert result.tool_calls[0].arguments == {"city": "Paris"}
+ messages.append(ChatMessage(role="assistant", tool_calls=result.tool_calls))
+ messages.extend(ChatMessage(role="tool", tool_call_id=tc.id, content="ok") for tc in result.tool_calls)
+ await provider.chat(messages, model="gemini-test")
+
+ assert len(_FakeAsyncClient.posted) == 3
+ for url, payload in _FakeAsyncClient.posted:
+ assert url == "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"
+ for message in payload["messages"]:
+ for call in message.get("tool_calls", []):
+ if signature_timing == "absent" or call["id"] == "b":
+ assert set(call) == {"id", "type", "function"}
+ else:
+ expected = signature_a if call["id"] == "a" else "opaque-B=="
+ assert call["extra_content"] == {"google": {"thought_signature": expected}}
+ final_calls = [tc for m in _FakeAsyncClient.posted[-1][1]["messages"] for tc in m.get("tool_calls", [])]
+ assert [tc["id"] for tc in final_calls] == ["a", "b", "c"]
+
+
# --------------------------------------------------------------------- Ollama: chat_stream
@pytest.mark.asyncio