Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): keep the copilot system prompt fresh after a Langfuse publish (#14628)

Reinier van der Leer7973688fdb81
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index f90630f4e7..65ca85276b 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -339,7 +339,9 @@ class ChatConfig(BaseSettings):
)
langfuse_prompt_cache_ttl: int = Field(
default=300,
- description="Cache TTL in seconds for Langfuse prompt (0 to disable caching)",
+ ge=0,
+ description="How long a process may serve a cached Langfuse prompt before "
+ "re-fetching it (0 to disable caching)",
)
# Rate limiting — cost-based limits per day and per week, stored in
diff --git a/autogpt_platform/backend/backend/copilot/config_test.py b/autogpt_platform/backend/backend/copilot/config_test.py
index 538d33e728..576f4cf33a 100644
--- a/autogpt_platform/backend/backend/copilot/config_test.py
+++ b/autogpt_platform/backend/backend/copilot/config_test.py
@@ -1,6 +1,7 @@
"""Unit tests for ChatConfig."""
import pytest
+from pydantic import ValidationError
from backend.util.clients import OPENROUTER_BASE_URL
@@ -1203,3 +1204,16 @@ class TestHostMatches:
def test_case_insensitive(self):
assert _host_matches("https://API.ANTHROPIC.COM/", "anthropic.com")
+
+
+class TestLangfusePromptCacheTTL:
+ def test_default_is_five_minutes(self):
+ # Read the field default, not an instance: backend/.env can set
+ # CHAT_LANGFUSE_PROMPT_CACHE_TTL and mask it.
+ assert ChatConfig.model_fields["langfuse_prompt_cache_ttl"].default == 300
+
+ def test_a_negative_ttl_is_rejected(self):
+ # A negative TTL would skip our revalidation and expire the SDK entry
+ # at once, which is the unbounded staleness this field exists to avoid.
+ with pytest.raises(ValidationError, match="langfuse_prompt_cache_ttl"):
+ ChatConfig(langfuse_prompt_cache_ttl=-1)
diff --git a/autogpt_platform/backend/backend/copilot/prompt_cache_test.py b/autogpt_platform/backend/backend/copilot/prompt_cache_test.py
index 5f34e4ce62..e518a9391d 100644
--- a/autogpt_platform/backend/backend/copilot/prompt_cache_test.py
+++ b/autogpt_platform/backend/backend/copilot/prompt_cache_test.py
@@ -8,6 +8,8 @@ These tests verify that _build_system_prompt:
- Handles DB errors and Langfuse errors gracefully
"""
+import inspect
+import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -15,6 +17,19 @@ import pytest
_SVC = "backend.copilot.service"
[email protected](autouse=True)
+def cold_prompt_cache(monkeypatch):
+ """Start every test from a process that has never fetched the prompt.
+
+ ``_cached_prompt`` is per-process state by design, so without this it leaks
+ between tests in collection order.
+ """
+ from backend.copilot import service
+
+ monkeypatch.setattr(service, "_cached_prompt", None)
+ monkeypatch.setattr(service, "_last_prompt_revalidation", 0.0)
+
+
class TestBuildSystemPrompt:
@pytest.mark.asyncio
async def test_no_user_id_returns_static_prompt(self):
@@ -1122,3 +1137,161 @@ class TestInjectUserContextSessionCtx:
assert "session_context" not in stripped
assert "pending_followups" not in stripped
assert "user query" in stripped
+
+
+class _StuckRefreshLangfuse:
+ """A Langfuse client whose background prompt refresh never runs.
+
+ langfuse 3.14.1 answers an expired cache entry with the stale value and
+ queues a refresh on a background thread (``_client/client.py:3650-3674``).
+ When that thread is not running the queued key is never cleared
+ (``_utils/prompt_cache.py:92-115``), so the cached path keeps returning the
+ same version for the life of the process. Only ``cache_ttl_seconds=0``
+ reaches the server (``_client/client.py:3607``), which is why the service
+ passes nothing else.
+ """
+
+ def __init__(self, cached: str, live: str):
+ self._cached = cached
+ self._live = live
+ self.server_fetches = 0
+ self.fetch_error: Exception | None = None
+ self.calls: list[dict] = []
+
+ def get_prompt(self, name, **kwargs):
+ self.calls.append(kwargs)
+ if kwargs.get("cache_ttl_seconds") != 0:
+ return self._as_prompt(self._cached)
+ self.server_fetches += 1
+ if self.fetch_error is not None:
+ raise self.fetch_error
+ return self._as_prompt(self._live)
+
+ @staticmethod
+ def _as_prompt(text: str):
+ prompt = MagicMock()
+ prompt.compile.return_value = text
+ return prompt
+
+
+_TTL = 60
+
+
+def stuck_langfuse(monkeypatch):
+ from backend.copilot import service
+
+ client = _StuckRefreshLangfuse(cached="v35 prompt", live="v36 prompt")
+ monkeypatch.setattr(service.config, "langfuse_prompt_cache_ttl", _TTL)
+ monkeypatch.setattr(service, "_is_langfuse_configured", lambda: True)
+ monkeypatch.setattr(service, "_get_langfuse", lambda: client)
+ monkeypatch.setattr(service, "_cached_prompt", "v35 prompt")
+ monkeypatch.setattr(service, "_last_prompt_revalidation", time.monotonic())
+ return client
+
+
+def open_window(monkeypatch):
+ """Put the last revalidation far enough back that the next call owns it."""
+ from backend.copilot import service
+
+ monkeypatch.setattr(
+ service, "_last_prompt_revalidation", time.monotonic() - _TTL - 1
+ )
+
+
+class TestPromptRevalidation:
+ """No pod may serve one prompt version for longer than the TTL."""
+
+ @pytest.mark.asyncio
+ async def test_serves_the_cached_copy_inside_the_window(self, stuck_langfuse):
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ assert await _fetch_langfuse_prompt() == "v35 prompt"
+ assert stuck_langfuse.server_fetches == 0
+
+ @pytest.mark.asyncio
+ async def test_revalidates_past_the_ttl_although_the_refresh_is_stuck(
+ self, stuck_langfuse, open_window
+ ):
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ assert await _fetch_langfuse_prompt() == "v36 prompt"
+ assert stuck_langfuse.server_fetches == 1
+
+ @pytest.mark.asyncio
+ async def test_the_turns_after_a_revalidation_cost_nothing(
+ self, stuck_langfuse, open_window
+ ):
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ await _fetch_langfuse_prompt()
+
+ for _ in range(5):
+ assert await _fetch_langfuse_prompt() == "v36 prompt"
+ assert stuck_langfuse.server_fetches == 1
+
+ @pytest.mark.asyncio
+ async def test_a_failed_revalidation_serves_the_cached_copy(
+ self, stuck_langfuse, open_window
+ ):
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ stuck_langfuse.fetch_error = RuntimeError("langfuse unreachable")
+
+ # Not the bundled prompt: a stale copy beats no copy.
+ assert await _fetch_langfuse_prompt() == "v35 prompt"
+ assert stuck_langfuse.server_fetches == 1
+
+ @pytest.mark.asyncio
+ async def test_an_outage_costs_one_fetch_per_window_not_per_turn(
+ self, stuck_langfuse, open_window
+ ):
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ stuck_langfuse.fetch_error = RuntimeError("langfuse unreachable")
+
+ for _ in range(5):
+ assert await _fetch_langfuse_prompt() == "v35 prompt"
+ assert stuck_langfuse.server_fetches == 1
+
+ @pytest.mark.asyncio
+ async def test_a_zero_ttl_fetches_every_turn(self, stuck_langfuse, monkeypatch):
+ from backend.copilot import service
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ monkeypatch.setattr(service.config, "langfuse_prompt_cache_ttl", 0)
+
+ for _ in range(3):
+ await _fetch_langfuse_prompt()
+ assert stuck_langfuse.server_fetches == 3
+
+ @pytest.mark.asyncio
+ async def test_the_sdk_cache_is_never_relied_on(self, stuck_langfuse, open_window):
+ """Any call with a non-zero TTL would put us back on the wedged path."""
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ await _fetch_langfuse_prompt()
+ await _fetch_langfuse_prompt()
+
+ assert stuck_langfuse.calls
+ assert all(c.get("cache_ttl_seconds") == 0 for c in stuck_langfuse.calls)
+
+ @pytest.mark.asyncio
+ async def test_every_call_binds_to_the_sdk_signature(
+ self, stuck_langfuse, open_window
+ ):
+ """A renamed SDK parameter would disable the revalidation silently.
+
+ The fake accepts anything, so without this the fix could stop reaching
+ the server on a Langfuse bump and every test here would still pass.
+ """
+ from langfuse._client.client import Langfuse
+
+ from backend.copilot.service import _fetch_langfuse_prompt
+
+ await _fetch_langfuse_prompt()
+
+ assert stuck_langfuse.calls
+ for call in stuck_langfuse.calls:
+ inspect.signature(Langfuse.get_prompt).bind(None, "CoPilot Prompt", **call)
diff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.py
index 30afc65335..372a5eca29 100644
--- a/autogpt_platform/backend/backend/copilot/service.py
+++ b/autogpt_platform/backend/backend/copilot/service.py
@@ -11,6 +11,8 @@ This module contains:
import asyncio
import logging
import re
+import threading
+import time
from typing import Any
from langfuse import get_client
@@ -75,6 +77,15 @@ _main_client: LangfuseAsyncOpenAI | None = None
_aux_client: LangfuseAsyncOpenAI | None = None
_langfuse = None
+# The system prompt this process last fetched, the monotonic timestamp of that
+# fetch, and the lock that hands the next window to one caller.
+# See _get_prompt_bounded_stale().
+_cached_prompt: str | None = None
+_last_prompt_revalidation = 0.0
+_prompt_revalidation_lock = threading.Lock()
+
+_PROMPT_REVALIDATION_TIMEOUT_SECONDS = 5
+
def _get_main_client() -> LangfuseAsyncOpenAI:
"""Main OpenAI-compat client used by the baseline path.
@@ -558,36 +569,100 @@ async def _fetch_langfuse_prompt() -> str | None:
if not _is_langfuse_configured():
return None
try:
- label = (
- None if settings.config.app_env == AppEnvironment.PRODUCTION else "latest"
- )
- prompt = await asyncio.to_thread(
- _get_langfuse().get_prompt,
- config.langfuse_prompt_name,
- label=label,
- cache_ttl_seconds=config.langfuse_prompt_cache_ttl,
- )
- compiled = prompt.compile(users_information="")
- # Guard the caching contract: if the Langfuse template is ever updated
- # to re-embed the {users_information} placeholder, the compiled text
- # will contain a literal "{users_information}" (because we passed an
- # empty string). That would mean user-specific text is back in the
- # system prompt, defeating cross-session caching. Log an error so the
- # regression is immediately visible in production observability.
- if "{users_information}" in compiled:
- logger.error(
- "Langfuse prompt still contains {users_information} placeholder — "
- "user context has been re-embedded in the system prompt, which "
- "breaks cross-session LLM prompt caching. Remove the placeholder "
- "from the Langfuse template and inject user context via "
- "inject_user_context() instead."
- )
- return compiled
+ return await _get_prompt_bounded_stale()
except Exception as e:
logger.warning(f"Failed to fetch prompt from Langfuse, using default: {e}")
return None
+async def _get_prompt_bounded_stale() -> str:
+ """Return the prompt, never a copy this process has held longer than the TTL.
+
+ The SDK's own cache cannot give that bound. It answers an expired entry with
+ the stale value and queues a refresh on a background thread
+ (``langfuse/_client/client.py:3650-3674``), and that refresh can stop for the
+ life of the process: a queued key is cleared only by its task running, so a
+ consumer that is not running wedges the key and nothing is ever queued again
+ (``langfuse/_utils/prompt_cache.py:92-115``), while a refresh that fails every
+ time leaves the entry expired forever. Both are silent to us, because the call
+ still returns a value, and both made Dev pods serve one prompt version for as
+ long as they lived (2026-09-11).
+
+ So the copy and the clock are ours and the SDK cache is bypassed entirely.
+ A revalidation that fails keeps the window and serves the copy we hold: it is
+ the freshest thing available while Langfuse is unreachable, and retrying on
+ every turn would hammer an endpoint that is already failing.
+ """
+ claimed = _claim_prompt_revalidation()
+ cached = _cached_prompt
+ if not claimed and cached is not None:
+ return cached
+ try:
+ return await _revalidate_prompt()
+ except Exception as e:
+ cached = _cached_prompt
+ if cached is None:
+ raise
+ logger.warning(f"Langfuse prompt revalidation failed, serving cached: {e}")
+ return cached
+
+
+def _claim_prompt_revalidation() -> bool:
+ """Whether this caller should re-fetch rather than serve the cached copy.
+
+ The window is marked used before the fetch, so concurrent turns serve the
+ cached copy instead of stampeding Langfuse. A TTL of 0 disables caching, so
+ every caller re-fetches.
+ """
+ global _last_prompt_revalidation
+ ttl = config.langfuse_prompt_cache_ttl
+ if ttl == 0:
+ return True
+ with _prompt_revalidation_lock:
+ now = time.monotonic()
+ if now - _last_prompt_revalidation < ttl:
+ return False
+ _last_prompt_revalidation = now
+ return True
+
+
+async def _revalidate_prompt() -> str:
+ """Fetch the prompt from Langfuse past the SDK cache, and keep the result.
+
+ ``cache_ttl_seconds=0`` makes the SDK skip its cache and its background
+ refresh altogether (``langfuse/_client/client.py:3607``), so neither failure
+ above can reach us. One attempt, because the copy we hold covers a failure
+ and a chat turn should not wait out a retry chain.
+ """
+ global _cached_prompt
+ label = None if settings.config.app_env == AppEnvironment.PRODUCTION else "latest"
+ prompt = await asyncio.to_thread(
+ _get_langfuse().get_prompt,
+ config.langfuse_prompt_name,
+ label=label,
+ cache_ttl_seconds=0,
+ max_retries=0,
+ fetch_timeout_seconds=_PROMPT_REVALIDATION_TIMEOUT_SECONDS,
+ )
+ compiled = prompt.compile(users_information="")
+ # Guard the caching contract: if the Langfuse template is ever updated
+ # to re-embed the {users_information} placeholder, the compiled text
+ # will contain a literal "{users_information}" (because we passed an
+ # empty string). That would mean user-specific text is back in the
+ # system prompt, defeating cross-session caching. Log an error so the
+ # regression is immediately visible in production observability.
+ if "{users_information}" in compiled:
+ logger.error(
+ "Langfuse prompt still contains {users_information} placeholder — "
+ "user context has been re-embedded in the system prompt, which "
+ "breaks cross-session LLM prompt caching. Remove the placeholder "
+ "from the Langfuse template and inject user context via "
+ "inject_user_context() instead."
+ )
+ _cached_prompt = compiled
+ return compiled
+
+
async def _build_system_prompt(
user_id: str | None,
) -> tuple[str, BusinessUnderstanding | None]: