Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(prompts): skip managed prompt fetches with a project secret key (#121652)

posthog[bot]a71fce996f14
diff --git a/posthog/llm/managed_decision_model.py b/posthog/llm/managed_decision_model.py
index 9b7b33b30cb9..521ff218d130 100644
--- a/posthog/llm/managed_decision_model.py
+++ b/posthog/llm/managed_decision_model.py
@@ -9,6 +9,8 @@ import structlog
import posthoganalytics
from posthoganalytics.ai.prompts import PromptResult, Prompts
+from posthog.models.utils import SECRET_API_TOKEN_PREFIX
+
DEFAULT_DECISION_MODEL = "posthog/hogference/jeeves-0.1"
PROMPT_LABEL = "production"
PROMPT_REFRESH_SECONDS = 60
@@ -25,12 +27,35 @@ def app_prompts() -> Prompts:
return Prompts(posthoganalytics, capture_errors=True, default_cache_ttl_seconds=0)
+def _warn_project_secret_key() -> None:
+ logger.warning(
+ "managed_prompt_fetch_skipped",
+ reason="POSTHOG_PERSONAL_API_KEY holds a project secret key, and prompt fetches need a personal API key",
+ )
+
+
+def prompt_key_configured() -> bool:
+ """True when the SDK has a key that the prompts API can accept."""
+ # Without a key (tests, local dev, self-hosted) the SDK still sends the request and gets a 401.
+ key = posthoganalytics.personal_api_key
+ if not key:
+ return False
+ # The SDK accepts a project secret key for flags, but the prompts API rejects it. Each rejected
+ # fetch captures an exception, so skip the fetch and keep the bundled prompt.
+ if key.startswith(SECRET_API_TOKEN_PREFIX):
+ _warn_project_secret_key()
+ return False
+ return True
+
+
def get_app_prompt(prompt_name: str, *, version: int | None = None) -> PromptResult | None:
"""The `production` version, or `version` when given. None without a personal API key."""
- # Without a key (tests, local dev, self-hosted) the SDK still sends the request and gets a 401.
- if not posthoganalytics.personal_api_key:
+ if not prompt_key_configured():
if version is not None:
- raise RuntimeError(f"Reading {prompt_name} version {version} needs POSTHOG_PERSONAL_API_KEY")
+ raise RuntimeError(
+ f"Reading {prompt_name} version {version} needs a personal API key (phx_) in POSTHOG_PERSONAL_API_KEY"
+ )
return None
label = PROMPT_LABEL if version is None else None
return app_prompts().get(prompt_name, with_metadata=True, label=label, version=version)
diff --git a/posthog/llm/test_managed_decision_model.py b/posthog/llm/test_managed_decision_model.py
index 2b18ea32f045..b2809a26b96d 100644
--- a/posthog/llm/test_managed_decision_model.py
+++ b/posthog/llm/test_managed_decision_model.py
@@ -102,14 +102,17 @@ class TestManagedDecisionModel(SimpleTestCase):
get.assert_any_call("emoji-search-suggestions", with_metadata=True, label="production", version=None)
get.assert_any_call("emoji-search-suggestions", with_metadata=True, label=None, version=3)
- @patch("posthog.llm.managed_decision_model.posthoganalytics.personal_api_key", None)
+ @parameterized.expand([("no_key", None), ("project_secret_key", "phs_test")])
@patch("posthog.llm.managed_decision_model.Prompts")
- def test_without_a_personal_key_the_bundled_model_stays(self, prompts_class) -> None:
+ def test_without_a_personal_key_the_bundled_model_stays(self, _name: str, key: str | None, prompts_class) -> None:
managed = ManagedDecisionModel("emoji-search-suggestions")
- managed._refresher._refresh()
+ with patch("posthog.llm.managed_decision_model.posthoganalytics.personal_api_key", key):
+ managed._refresher._refresh()
+ with self.assertRaisesRegex(
+ RuntimeError, r"version 3 needs a personal API key \(phx_\) in POSTHOG_PERSONAL_API_KEY"
+ ):
+ managed.fetch(version=3)
assert managed.current() == DEFAULT_DECISION_MODEL
- with self.assertRaisesRegex(RuntimeError, "version 3 needs POSTHOG_PERSONAL_API_KEY"):
- managed.fetch(version=3)
prompts_class.assert_not_called()
diff --git a/products/signals/backend/system_one_prompts.py b/products/signals/backend/system_one_prompts.py
index 0f1c947dad10..7bef8300d181 100644
--- a/products/signals/backend/system_one_prompts.py
+++ b/products/signals/backend/system_one_prompts.py
@@ -10,6 +10,7 @@ import posthoganalytics
from posthoganalytics.ai.prompts import PromptResult, Prompts
from posthog.dataclasses import frozen
+from posthog.llm.managed_decision_model import prompt_key_configured
logger = structlog.get_logger(__name__)
@@ -111,7 +112,7 @@ def _parse_prompt(result: PromptResult, fallback: SystemOnePrompt) -> SystemOneP
def fetch_prompt(fallback: SystemOnePrompt, *, version: int | None = None) -> SystemOnePrompt | None:
- if not posthoganalytics.personal_api_key:
+ if not prompt_key_configured():
return None
result = Prompts(posthoganalytics, capture_errors=True).get(
fallback.name,
diff --git a/products/signals/backend/test/test_system_one_prompts.py b/products/signals/backend/test/test_system_one_prompts.py
index 8fdf47d0d97e..01a9c366c68d 100644
--- a/products/signals/backend/test/test_system_one_prompts.py
+++ b/products/signals/backend/test/test_system_one_prompts.py
@@ -219,3 +219,13 @@ def test_managed_jevk_prompt_is_rejected(fallback: SystemOnePrompt) -> None:
prompts.return_value.get.return_value = result
assert fetch_prompt(fallback) is None
assert fallback.model == DEFAULT_SYSTEM_ONE_MODEL
+
+
[email protected]("fallback", [SIGNAL_SAFETY_SYSTEM_ONE_PROMPT, REPORT_SAFETY_SYSTEM_ONE_PROMPT])
+def test_project_secret_key_skips_the_production_prompt_fetch(fallback: SystemOnePrompt) -> None:
+ with (
+ patch("products.signals.backend.system_one_prompts.posthoganalytics.personal_api_key", "phs_test"),
+ patch("products.signals.backend.system_one_prompts.Prompts") as prompts,
+ ):
+ assert fetch_prompt(fallback) is None
+ prompts.assert_not_called()
diff --git a/products/tasks/backend/facade/onboarding_prompt.py b/products/tasks/backend/facade/onboarding_prompt.py
index c35fc31066e3..f81641f4d8d0 100644
--- a/products/tasks/backend/facade/onboarding_prompt.py
+++ b/products/tasks/backend/facade/onboarding_prompt.py
@@ -1,6 +1,8 @@
import posthoganalytics
from posthoganalytics.ai.prompts import PromptResult, Prompts
+from posthog.llm.managed_decision_model import prompt_key_configured
+
ONBOARDING_PROMPT_NAME = "posthog-desktop-onboarding-run"
ONBOARDING_PROMPT_LABEL = "production"
REQUIRED_ONBOARDING_PROMPT_PLACEHOLDERS = frozenset({"brief", "channel_id", "followup", "homepage"})
@@ -148,6 +150,8 @@ _PROMPTS = Prompts(posthoganalytics, capture_errors=True)
def load_onboarding_prompt() -> PromptResult:
+ if not prompt_key_configured():
+ return PromptResult(source="code_fallback", prompt=BUNDLED_ONBOARDING_PROMPT)
return _PROMPTS.get(
ONBOARDING_PROMPT_NAME,
with_metadata=True,
diff --git a/products/tasks/backend/tests/test_onboarding_brief.py b/products/tasks/backend/tests/test_onboarding_brief.py
index cef4e1799644..1f2623b29d21 100644
--- a/products/tasks/backend/tests/test_onboarding_brief.py
+++ b/products/tasks/backend/tests/test_onboarding_brief.py
@@ -1,6 +1,8 @@
import json
from uuid import UUID
+from unittest.mock import patch
+
from django.test import SimpleTestCase
from parameterized import parameterized
@@ -21,6 +23,7 @@ from products.tasks.backend.facade.onboarding_brief import (
from products.tasks.backend.facade.onboarding_canvas import TeachingCanvas
from products.tasks.backend.facade.onboarding_prompt import (
BUNDLED_ONBOARDING_PROMPT,
+ load_onboarding_prompt,
missing_onboarding_prompt_placeholders,
render_onboarding_prompt,
)
@@ -381,6 +384,17 @@ class TestWhereFindingsLive(SimpleTestCase):
class TestBundledPromptRendering(SimpleTestCase):
+ @parameterized.expand([("no_key", None), ("project_secret_key", "phs_test")])
+ def test_without_a_personal_key_the_bundled_prompt_loads_without_a_fetch(self, _name: str, key: str | None) -> None:
+ with (
+ patch("posthog.llm.managed_decision_model.posthoganalytics.personal_api_key", key),
+ patch("products.tasks.backend.facade.onboarding_prompt._PROMPTS.get") as get,
+ ):
+ result = load_onboarding_prompt()
+
+ assert (result.source, result.prompt) == ("code_fallback", BUNDLED_ONBOARDING_PROMPT)
+ get.assert_not_called()
+
def test_the_managed_prompt_requires_every_runtime_value(self) -> None:
assert missing_onboarding_prompt_placeholders("{{brief}} {{homepage}}") == ("channel_id", "followup")