Open-source personal finance manager. Self-hosted, privacy-first.
fix(agents): preserve Gemini tool-call thought signatures (#889)
Vítor Alves2a207c1312dd
diff --git a/backend/app/agents/models/conversation.py b/backend/app/agents/models/conversation.pyindex ce4dd9ae..d8c34e76 100644--- a/backend/app/agents/models/conversation.py+++ b/backend/app/agents/models/conversation.py@@ -50,7 +50,7 @@ class Message(Base):role: Mapped[str] = mapped_column(String(20))content: Mapped[Optional[str]] = mapped_column(Text, nullable=True)- # When role="assistant" with tool calls, list of {id,name,arguments}.+ # When role="assistant", list of {id,name,arguments,thought_signature?}.tool_calls: Mapped[Optional[list]] = mapped_column(JSON, nullable=True)# When role="tool", payload is {tool_call_id, server, name, result, error?}.tool_result: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)diff --git a/backend/app/agents/providers/base.py b/backend/app/agents/providers/base.pyindex a2d8fec3..ef23f581 100644--- a/backend/app/agents/providers/base.py+++ b/backend/app/agents/providers/base.py@@ -51,6 +51,8 @@ class ToolCall:id: strname: strarguments: dict[str, Any]+ # Opaque Gemini signature; replay unchanged, never pass to the tool itself.+ thought_signature: Optional[str] = None@dataclass@@ -87,6 +89,8 @@ class ChatChunk:args_delta: Optional[str] = None # incremental JSON args while streamingfinish_reason: Optional[str] = Noneusage: Optional[Usage] = None+ # Delivered on tool_call_end, after all streaming deltas have arrived.+ thought_signature: Optional[str] = None@dataclass@@ -147,6 +151,8 @@ class LLMProvider(ABC):elif chunk.type == "tool_call_args_delta" and chunk.tool_call_id:tc = tool_calls.setdefault(chunk.tool_call_id, {"id": chunk.tool_call_id, "name": chunk.tool_name or "", "args_buf": ""})tc["args_buf"] += chunk.args_delta or ""+ elif chunk.type == "tool_call_end" and chunk.tool_call_id in tool_calls:+ tool_calls[chunk.tool_call_id]["thought_signature"] = chunk.thought_signatureelif chunk.type == "finish" and chunk.finish_reason:finish = chunk.finish_reasonelif chunk.type == "usage" and chunk.usage:@@ -158,7 +164,10 @@ class LLMProvider(ABC):args = json.loads(tc["args_buf"]) if tc["args_buf"] else {}except json.JSONDecodeError:args = {"_raw": tc["args_buf"]}- parsed_calls.append(ToolCall(id=tc["id"], name=tc["name"], arguments=args))+ parsed_calls.append(ToolCall(+ id=tc["id"], name=tc["name"], arguments=args,+ thought_signature=tc.get("thought_signature"),+ ))return ChatResponse(content="".join(text_parts), tool_calls=parsed_calls, finish_reason=finish, usage=usage)@abstractmethoddiff --git a/backend/app/agents/providers/openai.py b/backend/app/agents/providers/openai.pyindex b7e57078..3551e980 100644--- a/backend/app/agents/providers/openai.py+++ b/backend/app/agents/providers/openai.py@@ -1,6 +1,7 @@from __future__ import annotationsimport json+import refrom typing import AsyncIterator, Optionalfrom urllib.parse import urlparse@@ -30,7 +31,7 @@ def normalize_openai_base_url(url: str) -> str:Rules:- empty → returned as-is- trailing slashes stripped- - if any path segment looks like a version (`v1`, `v2`, `beta`,+ - if any path segment looks like a version (`v1`, `v2`, `v1beta`, `beta`,`latest`, `api/v3`, etc.), leave the URL alone- otherwise, append `/v1`"""@@ -40,7 +41,7 @@ def normalize_openai_base_url(url: str) -> str:path_segments = [s for s in (urlparse(url).path or "").split("/") if s]for seg in path_segments:if (- (seg.startswith("v") and seg[1:].isdigit())+ re.fullmatch(r"v\d+(?:(?:alpha|beta)\d*)?", seg)or seg in {"beta", "latest"}):return url@@ -59,6 +60,8 @@ def _serialize_messages(messages: list[ChatMessage]) -> list[dict]:"id": tc.id,"type": "function","function": {"name": tc.name, "arguments": json.dumps(tc.arguments)},+ **({"extra_content": {"google": {"thought_signature": tc.thought_signature}}}+ if tc.thought_signature is not None else {}),}for tc in m.tool_calls]@@ -174,6 +177,13 @@ class OpenAIProvider(LLMProvider):"started": False,"pending_args": "",})+ # Gemini's OpenAI endpoint puts the opaque signature+ # beside function, potentially in a later delta.+ extra = tc.get("extra_content")+ google = extra.get("google") if isinstance(extra, dict) else None+ signature = google.get("thought_signature") if isinstance(google, dict) else None+ if isinstance(signature, str):+ state["thought_signature"] = signature# First chunk usually carries both id and name;# some servers split the name across chunks.if tc.get("id") and not state["id"]:@@ -212,7 +222,10 @@ class OpenAIProvider(LLMProvider):finish_reason = choice["finish_reason"]for state in idx_state.values():if state["started"]:- yield ChatChunk(type="tool_call_end", tool_call_id=state["id"])+ yield ChatChunk(+ type="tool_call_end", tool_call_id=state["id"],+ thought_signature=state.get("thought_signature"),+ )if usage:yield ChatChunk(type="usage", usage=usage)yield ChatChunk(type="finish", finish_reason=finish_reason or "stop")diff --git a/backend/app/agents/runtime/executor.py b/backend/app/agents/runtime/executor.pyindex 418e6438..fb27f7fe 100644--- a/backend/app/agents/runtime/executor.py+++ b/backend/app/agents/runtime/executor.py@@ -386,7 +386,10 @@ class AgentExecutor:for m in history:tcs = []for raw in (m.tool_calls or []):- tcs.append(ToolCall(id=raw.get("id"), name=raw.get("name"), arguments=raw.get("arguments") or {}))+ tcs.append(ToolCall(+ id=raw.get("id"), name=raw.get("name"), arguments=raw.get("arguments") or {},+ thought_signature=raw.get("thought_signature"),+ ))tool_call_id = (m.tool_result or {}).get("tool_call_id") if m.role == "tool" else Nonecontent = m.contentif m.role == "tool":@@ -477,7 +480,10 @@ class AgentExecutor:args = json.loads(tc["args_buf"]) if tc["args_buf"] else {}except json.JSONDecodeError:args = {"_raw": tc["args_buf"]}- assembled_calls.append(ToolCall(id=tc["id"], name=tc["name"], arguments=args))+ assembled_calls.append(ToolCall(+ id=tc["id"], name=tc["name"], arguments=args,+ thought_signature=tc.get("thought_signature"),+ ))# Persist assistant turn.assistant_msg = await conversation_service.append_message(@@ -485,7 +491,11 @@ class AgentExecutor:conversation_id=conversation_id,role="assistant",content=assistant_text or None,- tool_calls=[{"id": c.id, "name": c.name, "arguments": c.arguments} for c in assembled_calls] or None,+ tool_calls=[{+ "id": c.id, "name": c.name, "arguments": c.arguments,+ **({"thought_signature": c.thought_signature}+ if c.thought_signature is not None else {}),+ } for c in assembled_calls] or None,input_tokens=usage_input or None,output_tokens=usage_output or None,)@@ -599,6 +609,8 @@ async def _process_chunk(chunk: ChatChunk, text_buf: list[str], open_calls: dictelif chunk.type == "tool_call_args_delta" and chunk.tool_call_id:tc = open_calls.setdefault(chunk.tool_call_id, {"id": chunk.tool_call_id, "name": "", "args_buf": ""})tc["args_buf"] += chunk.args_delta or ""+ elif chunk.type == "tool_call_end" and chunk.tool_call_id in open_calls:+ open_calls[chunk.tool_call_id]["thought_signature"] = chunk.thought_signatureasync def _safe_call_tool(diff --git a/backend/tests/test_agents_executor.py b/backend/tests/test_agents_executor.pyindex 9771f8b2..1f9a8785 100644--- a/backend/tests/test_agents_executor.py+++ b/backend/tests/test_agents_executor.py@@ -10,6 +10,7 @@ Pattern:- We patch `_provider_for` and pass our fake MCP into AgentExecutor."""import uuid+from copy import deepcopyfrom typing import AsyncIteratorfrom unittest.mock import patch@@ -45,10 +46,12 @@ class _ScriptedProvider(LLMProvider):def __init__(self, turns: list[list[ChatChunk]]):super().__init__(api_key="x")self._turns = list(turns)+ self.requests = []async def chat_stream( # type: ignore[override]self, messages, *, model, tools=None, temperature=0.4, max_tokens=None) -> AsyncIterator[ChatChunk]:+ self.requests.append(deepcopy(messages))if not self._turns:# No more scripted turns — emit a generic finish.yield ChatChunk(type="finish", finish_reason="stop")@@ -160,7 +163,8 @@ async def test_usage_row_recorded(session, test_user, test_agent, test_conversatassert float(rows[0].cost_usd) > 0-async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_agent, test_conversation):[email protected]("signature", [None, "opaque-signature+/=="])+async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_agent, test_conversation, signature):"""LLM emits a tool call; executor runs it via fake MCP and feeds theresult back; second turn returns a plain answer."""tools = [ToolHandle(server="securo", name="list_accounts", description="d", parameters={"type": "object"})]@@ -173,7 +177,7 @@ async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_[ChatChunk(type="tool_call_start", tool_call_id="t1", tool_name="securo__list_accounts"),ChatChunk(type="tool_call_args_delta", tool_call_id="t1", args_delta="{}"),- ChatChunk(type="tool_call_end", tool_call_id="t1"),+ ChatChunk(type="tool_call_end", tool_call_id="t1", thought_signature=signature),ChatChunk(type="usage", usage=Usage(input_tokens=20, output_tokens=5)),ChatChunk(type="finish", finish_reason="tool_calls"),],@@ -220,6 +224,24 @@ async def test_tool_call_dispatch_and_result_persisted(session, test_user, test_)).scalars().all()assert len(usage_rows) == 2+ saved_call = msgs[1].tool_calls[0]+ if signature is None:+ assert "thought_signature" not in saved_call+ else:+ assert saved_call["thought_signature"] == signature+ replayed = [tc for m in provider.requests[1] for tc in m.tool_calls]+ assert replayed[0].thought_signature == signature++ # Start a separate request so history is reconstructed from persisted JSON.+ session.expire(msgs[1])+ with _patch_provider(provider):+ await _drain(+ executor, session=session, agent=test_agent, user_id=test_user.id,+ conversation_id=test_conversation.id, user_message="thanks",+ )+ restored = [tc for m in provider.requests[2] for tc in m.tool_calls]+ assert restored[0].thought_signature == signature+async def test_provider_auth_error_surfaced_as_friendly_message(session, test_user, test_agent, test_conversationdiff --git a/backend/tests/test_agents_providers.py b/backend/tests/test_agents_providers.pyindex 551cffe8..7334b564 100644--- a/backend/tests/test_agents_providers.py+++ b/backend/tests/test_agents_providers.py@@ -120,6 +120,7 @@ def test_anthropic_tool_result_block():("http://lmstudio:1234/v1/", "http://lmstudio:1234/v1"),("https://api.openai.com/v1", "https://api.openai.com/v1"),("https://api.groq.com/openai/v1", "https://api.groq.com/openai/v1"),+ ("https://generativelanguage.googleapis.com/v1beta/openai/", "https://generativelanguage.googleapis.com/v1beta/openai"),("https://api.example.com/v2", "https://api.example.com/v2"),("https://api.example.com/beta", "https://api.example.com/beta"),("https://api.example.com/", "https://api.example.com/v1"),diff --git a/backend/tests/test_agents_providers_streaming.py b/backend/tests/test_agents_providers_streaming.pyindex 352073c7..80d7bde2 100644--- a/backend/tests/test_agents_providers_streaming.py+++ b/backend/tests/test_agents_providers_streaming.py@@ -1,5 +1,5 @@"""Coverage for the streaming chat + embed paths in-app/agents/providers/{ollama,anthropic}.py. Both providers open+app/agents/providers/{openai,ollama,anthropic}.py. These providers openhttpx.AsyncClient.stream(...) inline, so we replace AsyncClient with afake whose stream() yields a pre-baked SSE/JSONL response."""@@ -23,6 +23,8 @@ from app.agents.providers.base import (ToolDefinition,)from app.agents.providers.ollama import OllamaProvider+from app.agents.providers.openai import OpenAIProvider+from app.agents.providers.openai_compatible import OpenAICompatibleProvider# --------------------------------------------------------------------- httpx fake@@ -89,6 +91,62 @@ def _fake_httpx(monkeypatch):yield+# --------------------------------------------------------------------- OpenAI-compatible: chat_stream+[email protected]("provider_class", [OpenAICompatibleProvider, OpenAIProvider])[email protected]("signature_timing", ["first", "late", "absent"])+async def test_openai_tool_signatures_survive_multiple_rounds(provider_class, signature_timing):+ """Keep signatures on their own calls across parallel and sequential tools."""+ def response(deltas):+ return _FakeStreamResponse(status_code=200, lines=[+ "data: " + json.dumps({"choices": [{"delta": delta}]}) for delta in deltas+ ] + ['data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}', "data: [DONE]"])++ signature_a = "opaque-A+/=="+ first = {"index": 0, "id": "a", "function": {"name": "lookup", "arguments": '{"city":'}}+ extra_a = {"google": {"thought_signature": signature_a}}+ if signature_timing == "first":+ first["extra_content"] = extra_a+ deltas = [+ {"tool_calls": [first]},+ {"tool_calls": [{"index": 1, "id": "b", "function": {"name": "lookup", "arguments": "{}"}}]},+ {"tool_calls": [{"index": 0, "function": {"arguments": '"Paris"}'}}]},+ ]+ if signature_timing == "late":+ deltas.append({"tool_calls": [{"index": 0, "extra_content": extra_a}]})+ second_call = {"index": 0, "id": "c", "function": {"name": "lookup", "arguments": "{}"}}+ if signature_timing != "absent":+ second_call["extra_content"] = {"google": {"thought_signature": "opaque-B=="}}+ _FakeAsyncClient.queue_stream.extend([+ response(deltas), response([{"tool_calls": [second_call]}]),+ response([{"content": "Done"}]),+ ])+ provider = provider_class(base_url="https://generativelanguage.googleapis.com/v1beta/openai")+ messages = [ChatMessage(role="user", content="Look up cities")]+ for round_index in range(2):+ result = await provider.chat(messages, model="gemini-test")+ if round_index == 0:+ assert [tc.id for tc in result.tool_calls] == ["a", "b"]+ assert result.tool_calls[0].arguments == {"city": "Paris"}+ messages.append(ChatMessage(role="assistant", tool_calls=result.tool_calls))+ messages.extend(ChatMessage(role="tool", tool_call_id=tc.id, content="ok") for tc in result.tool_calls)+ await provider.chat(messages, model="gemini-test")++ assert len(_FakeAsyncClient.posted) == 3+ for url, payload in _FakeAsyncClient.posted:+ assert url == "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"+ for message in payload["messages"]:+ for call in message.get("tool_calls", []):+ if signature_timing == "absent" or call["id"] == "b":+ assert set(call) == {"id", "type", "function"}+ else:+ expected = signature_a if call["id"] == "a" else "opaque-B=="+ assert call["extra_content"] == {"google": {"thought_signature": expected}}+ final_calls = [tc for m in _FakeAsyncClient.posted[-1][1]["messages"] for tc in m.get("tool_calls", [])]+ assert [tc["id"] for tc in final_calls] == ["a", "b", "c"]++# --------------------------------------------------------------------- Ollama: chat_stream@pytest.mark.asyncio