Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): ask system one models from the playground (#128137)

Bernat Torres1091b39c7c6b
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index 34ea84fa05b8..5ad1a22cff75 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -658,6 +658,8 @@ SPECTACULAR_SETTINGS = {
"LogsAlertConfigurationStateEnum": "products.logs.backend.models.LogsAlertConfiguration.State",
# Matches the shared alerts skeleton's PlatformAlert.State.
"BillingAlertConfigurationStateEnum": "products.billing_alerts.backend.models.BillingAlertConfiguration.State",
+ # The ai_observability playground asks the same System One question kinds, so both share one name.
+ "DecisionQuestionTypeEnum": "products.ml_inference.backend.facade.enums.DecisionQuestionType.choices",
"LogsPatternsSourceEnum": ["stored_patterns", "body_mining"],
# Dashboard filters and data catalog metric runs both take the query schema's IntervalType,
# a plain enum no class can carry.
diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.py
index 993922756821..0584fdcce228 100644
--- a/products/ai_observability/backend/api/proxy.py
+++ b/products/ai_observability/backend/api/proxy.py
@@ -4,6 +4,7 @@ ViewSet for AI observability proxy
Endpoints:
- GET /api/llm_proxy/models
- POST /api/llm_proxy/completion
+- POST /api/llm_proxy/decision
"""
import json
@@ -19,17 +20,30 @@ from django.http.response import HttpResponseBase
from django.utils import timezone
import structlog
-from drf_spectacular.utils import OpenApiParameter, extend_schema
+from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema
from rest_framework import serializers, viewsets
from rest_framework.decorators import action
from rest_framework.permissions import BasePermission, IsAuthenticated
from rest_framework.request import Request
from rest_framework.response import Response
+from posthog.api.mixins import ValidatedRequest, validated_request
from posthog.api.monitoring import monitor
from posthog.api.streaming import sse_streaming_response
from posthog.auth import SessionAuthentication
from posthog.event_usage import groups, report_user_action
+from posthog.llm.system_one import (
+ MAX_CHOICE_OPTIONS,
+ MAX_SCORE_LEVELS,
+ Answer,
+ ChoiceAnswer,
+ ChoiceQuestion,
+ NoulAnswer,
+ NoulQuestion,
+ Question,
+ ScoreAnswer,
+ ScoreQuestion,
+)
from posthog.rate_limit import (
LLMProxyBurstRateThrottle,
LLMProxyBYOKBurstRateThrottle,
@@ -58,7 +72,9 @@ from products.ai_observability.backend.llm import (
get_playground_models,
supports_reasoning_level,
)
+from products.ai_observability.backend.llm.decisions import DecisionClient
from products.ai_observability.backend.llm.errors import (
+ LLMError,
ProviderConfigurationError,
ProviderHostUnresolvedError,
UnsupportedProviderError,
@@ -153,6 +169,170 @@ class LLMProxyModelInfoSerializer(serializers.Serializer):
)
+MAX_DECISION_QUESTIONS = 20
+MAX_DECISION_STATE_CHARS = 200_000
+MAX_DECISION_INSTRUCTIONS_CHARS = 4_000
+MAX_DECISION_OPTION_CHARS = 500
+
+
+class LLMProxyDecisionQuestionType(models.TextChoices):
+ NOUL = "noul", "Yes or no"
+ CHOICE = "choice", "Multiple choice"
+ SCORE = "score", "Rating scale"
+
+
+class LLMProxyDecisionAnswerType(models.TextChoices):
+ NOUL = "noul", "Yes or no"
+ CHOICE = "choice", "Multiple choice"
+ SCORE = "score", "Rating scale"
+ REFUSAL = "refusal", "Refusal"
+
+
+class LLMProxyDecisionOptionSerializer(serializers.Serializer):
+ name = serializers.CharField(
+ max_length=MAX_DECISION_OPTION_CHARS,
+ help_text="The option name for a multiple choice question, or the level label for a rating question.",
+ )
+ description = serializers.CharField(
+ max_length=MAX_DECISION_OPTION_CHARS,
+ required=False,
+ allow_blank=True,
+ help_text="What a multiple choice option means. Ignored for a rating question.",
+ )
+
+
+class LLMProxyDecisionQuestionSerializer(serializers.Serializer):
+ type = serializers.ChoiceField(
+ choices=LLMProxyDecisionQuestionType.choices,
+ help_text="The answer to produce: a yes/no probability, one of the options, or a level on the scale.",
+ )
+ instructions = serializers.CharField(
+ max_length=MAX_DECISION_INSTRUCTIONS_CHARS,
+ help_text="The question to ask about the state, phrased for the model.",
+ )
+ options = LLMProxyDecisionOptionSerializer(
+ many=True,
+ required=False,
+ help_text=(
+ f"The options of a multiple choice question (1 to {MAX_CHOICE_OPTIONS}), or the levels of a rating "
+ f"question from lowest to highest (2 to {MAX_SCORE_LEVELS}). Omitted for a yes/no question."
+ ),
+ )
+
+ def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
+ options = attrs.get("options") or []
+ names = [option["name"] for option in options]
+ if len(set(names)) != len(names):
+ raise serializers.ValidationError({"options": "Give each option a different name."})
+ match attrs["type"]:
+ case LLMProxyDecisionQuestionType.CHOICE if not 1 <= len(options) <= MAX_CHOICE_OPTIONS:
+ raise serializers.ValidationError(
+ {"options": f"A multiple choice question needs 1 to {MAX_CHOICE_OPTIONS} options."}
+ )
+ case LLMProxyDecisionQuestionType.SCORE if not 2 <= len(options) <= MAX_SCORE_LEVELS:
+ raise serializers.ValidationError(
+ {"options": f"A rating question needs 2 to {MAX_SCORE_LEVELS} levels."}
+ )
+ return attrs
+
+
+class LLMProxyDecisionSerializer(serializers.Serializer):
+ provider_key_id = serializers.UUIDField(help_text="The team's System One provider key that answers the questions.")
+ state = serializers.CharField(
+ max_length=MAX_DECISION_STATE_CHARS,
+ trim_whitespace=False,
+ help_text="The text the questions are about, for example a conversation. Sent unchanged.",
+ )
+ questions = serializers.DictField(
+ child=LLMProxyDecisionQuestionSerializer(),
+ help_text=(
+ f"The questions to ask, keyed by an id of your choice, at most {MAX_DECISION_QUESTIONS}. "
+ "Answers come back under the same ids."
+ ),
+ )
+
+ def validate_questions(self, questions: dict[str, Any]) -> dict[str, Any]:
+ if not 1 <= len(questions) <= MAX_DECISION_QUESTIONS:
+ raise serializers.ValidationError(f"Ask between 1 and {MAX_DECISION_QUESTIONS} questions.")
+ return questions
+
+
+class LLMProxyDecisionAnswerSerializer(serializers.Serializer):
+ type = serializers.ChoiceField(
+ choices=LLMProxyDecisionAnswerType.choices,
+ help_text="The question type answered, or `refusal` when the model declined to answer.",
+ )
+ probability = serializers.FloatField(allow_null=True, help_text="For a yes/no question, the probability of yes.")
+ choice = serializers.CharField(allow_null=True, help_text="For a multiple choice question, the option chosen.")
+ score = serializers.FloatField(
+ allow_null=True, help_text="For a rating question, the expected level as a fractional index into the levels."
+ )
+ confidence = serializers.FloatField(
+ allow_null=True, help_text="For multiple choice and rating questions, how sure the model is, from 0 to 1."
+ )
+ probabilities = serializers.DictField(
+ child=serializers.FloatField(),
+ allow_null=True,
+ help_text="For multiple choice and rating questions, the probability of each option or level index.",
+ )
+
+
+class LLMProxyDecisionResponseSerializer(serializers.Serializer):
+ model = serializers.CharField(help_text="The versioned model that answered.")
+ answers = serializers.DictField(
+ child=LLMProxyDecisionAnswerSerializer(), help_text="One answer per question, under the ids the request used."
+ )
+ input_tokens = serializers.IntegerField(allow_null=True, help_text="Tokens the model read, if reported.")
+ output_tokens = serializers.IntegerField(allow_null=True, help_text="Tokens the model wrote, if reported.")
+
+
+class LLMProxyErrorSerializer(serializers.Serializer):
+ error = serializers.CharField(help_text="What went wrong and what to change.")
+
+
+def build_decision_question(question: dict[str, Any]) -> Question:
+ options = question.get("options") or []
+ match question["type"]:
+ case LLMProxyDecisionQuestionType.CHOICE:
+ return ChoiceQuestion(
+ instructions=question["instructions"],
+ criteria={option["name"]: option.get("description") or None for option in options},
+ )
+ case LLMProxyDecisionQuestionType.SCORE:
+ return ScoreQuestion(instructions=question["instructions"], criteria=[option["name"] for option in options])
+ case _:
+ return NoulQuestion(instructions=question["instructions"])
+
+
+def serialize_decision_answer(answer: Answer) -> dict[str, Any]:
+ body: dict[str, Any] = {
+ "type": LLMProxyDecisionAnswerType.REFUSAL,
+ "probability": None,
+ "choice": None,
+ "score": None,
+ "confidence": None,
+ "probabilities": None,
+ }
+ match answer:
+ case NoulAnswer():
+ body.update(type=LLMProxyDecisionAnswerType.NOUL, probability=answer.probability)
+ case ChoiceAnswer():
+ body.update(
+ type=LLMProxyDecisionAnswerType.CHOICE,
+ choice=answer.choice,
+ confidence=answer.confidence,
+ probabilities=dict(answer.probabilities),
+ )
+ case ScoreAnswer():
+ body.update(
+ type=LLMProxyDecisionAnswerType.SCORE,
+ score=answer.score,
+ confidence=answer.confidence,
+ probabilities=dict(answer.probabilities),
+ )
+ return body
+
+
class PlaygroundAccessPermission(BasePermission):
"""Resource-level RBAC for the playground.
@@ -164,7 +344,11 @@ class PlaygroundAccessPermission(BasePermission):
"""
# Listing models is a read; running a completion spends money, so it's a write.
- ACTION_REQUIRED_LEVELS: dict[str, AccessControlLevel] = {"models": "viewer", "completion": "editor"}
+ ACTION_REQUIRED_LEVELS: dict[str, AccessControlLevel] = {
+ "models": "viewer",
+ "completion": "editor",
+ "decision": "editor",
+ }
def has_permission(self, request: Request, view) -> bool:
user = request.user
@@ -212,7 +396,7 @@ class LLMProxyViewSet(viewsets.ViewSet):
# BYOK requests should not count against shared playground limits.
# Check the provider key directly from request data to avoid running the
# full serializer just for throttle selection.
- if self.action == "completion":
+ if self.action in ("completion", "decision"):
data = getattr(self.request, "data", {})
provider_key_id = data.get("provider_key_id")
if provider_key_id:
@@ -586,3 +770,62 @@ class LLMProxyViewSet(viewsets.ViewSet):
@monitor(feature=None, endpoint="llma_proxy_completion", method="POST")
def completion(self, request, *args, **kwargs):
return self._handle_completion_request(request)
+
+ @validated_request(
+ request_serializer=LLMProxyDecisionSerializer,
+ responses={
+ 200: OpenApiResponse(response=LLMProxyDecisionResponseSerializer, description="One answer per question."),
+ 400: OpenApiResponse(
+ response=LLMProxyErrorSerializer, description="The key or the endpoint rejected the request."
+ ),
+ 402: OpenApiResponse(response=LLMProxyErrorSerializer, description="No payment method on file."),
+ },
+ summary="Ask a System One model",
+ description="Ask the team's System One connection typed questions about a state, from the playground.",
+ )
+ @action(detail=False, methods=["POST"])
+ @llma_track_latency("llma_proxy_decision")
+ @monitor(feature=None, endpoint="llma_proxy_decision", method="POST")
+ def decision(self, request: ValidatedRequest, *args: Any, **kwargs: Any) -> Response:
+ organization = getattr(request.user, "organization", None)
+ if not organization or not organization.customer_id:
+ return Response(
+ {"error": "The playground requires a valid payment method on file to prevent abuse."}, status=402
+ )
+
+ data = request.validated_data
+ try:
+ provider_key = self._get_provider_key(str(data["provider_key_id"]), request.user)
+ except ValueError:
+ provider_key = None
+ # TypeSafe is not a subprocessor, so this path only reaches the team's own System One connection.
+ if provider_key is None or provider_key.provider != LLMProvider.SYSTEM_ONE:
+ return Response({"error": "Pick a System One connection to ask questions."}, status=400)
+
+ config = provider_key.encrypted_config
+ try:
+ result = DecisionClient.evaluate(
+ api_key=config.get("api_key", ""),
+ base_url=config.get("base_url", ""),
+ model=config.get("model", ""),
+ state=data["state"],
+ questions={
+ question_id: build_decision_question(question)
+ for question_id, question in data["questions"].items()
+ },
+ )
+ except LLMError as error:
+ return Response({"error": str(error)}, status=400)
+
+ return Response(
+ LLMProxyDecisionResponseSerializer(
+ {
+ "model": result.model,
+ "answers": {
+ question_id: serialize_decision_answer(answer) for question_id, answer in result.answers.items()
+ },
+ "input_tokens": result.input_tokens,
+ "output_tokens": result.output_tokens,
+ }
+ ).data
+ )
diff --git a/products/ai_observability/backend/api/test/test_playground_access_control.py b/products/ai_observability/backend/api/test/test_playground_access_control.py
index 2d3c61d0e8ab..af86cbd996c8 100644
--- a/products/ai_observability/backend/api/test/test_playground_access_control.py
+++ b/products/ai_observability/backend/api/test/test_playground_access_control.py
@@ -62,16 +62,18 @@ class TestPlaygroundAccessControl(APIBaseTest):
@parameterized.expand(
[
- ("none", status.HTTP_403_FORBIDDEN, status.HTTP_403_FORBIDDEN),
- ("viewer", status.HTTP_200_OK, status.HTTP_403_FORBIDDEN),
- ("editor", status.HTTP_200_OK, status.HTTP_200_OK),
+ ("none", status.HTTP_403_FORBIDDEN, status.HTTP_403_FORBIDDEN, status.HTTP_403_FORBIDDEN),
+ ("viewer", status.HTTP_200_OK, status.HTTP_403_FORBIDDEN, status.HTTP_403_FORBIDDEN),
+ ("editor", status.HTTP_200_OK, status.HTTP_200_OK, status.HTTP_400_BAD_REQUEST),
]
)
- def test_access_level_gates_each_action(self, access_level, models_status, completion_status):
+ def test_access_level_gates_each_action(self, access_level, models_status, completion_status, decision_status):
self._login_with_playground_level(access_level)
assert self.client.get("/api/llm_proxy/models/").status_code == models_status
assert self._post_completion().status_code == completion_status
+ # An empty body passes the gate and then fails validation, so 400 means the gate let it through.
+ assert self.client.post("/api/llm_proxy/decision/", {}, format="json").status_code == decision_status
def test_denied_without_a_current_team(self):
user = self._login_with_playground_level("editor")
diff --git a/products/ai_observability/backend/api/test/test_proxy.py b/products/ai_observability/backend/api/test/test_proxy.py
index efb3fe6ebb66..fa5f8deeb557 100644
--- a/products/ai_observability/backend/api/test/test_proxy.py
+++ b/products/ai_observability/backend/api/test/test_proxy.py
@@ -16,6 +16,16 @@ import httpx
from parameterized import parameterized
from rest_framework.request import Request
+from posthog.llm.system_one import (
+ ChoiceAnswer,
+ ChoiceQuestion,
+ NoulAnswer,
+ NoulQuestion,
+ RefusalAnswer,
+ ScoreAnswer,
+ ScoreQuestion,
+ SystemOneResult,
+)
from posthog.rate_limit import (
LLMProxyBurstRateThrottle,
LLMProxyBYOKBurstRateThrottle,
@@ -25,7 +35,11 @@ from posthog.rate_limit import (
LLMProxySustainedRateThrottle,
)
-from products.ai_observability.backend.api.proxy import LLMProxyCompletionSerializer, LLMProxyViewSet
+from products.ai_observability.backend.api.proxy import (
+ LLMProxyCompletionSerializer,
+ LLMProxyDecisionQuestionSerializer,
+ LLMProxyViewSet,
+)
from products.ai_observability.backend.llm import (
PLAYGROUND_MODEL_IDS,
PROVIDERS,
@@ -34,6 +48,7 @@ from products.ai_observability.backend.llm import (
get_default_models,
get_playground_models,
)
+from products.ai_observability.backend.llm.errors import AuthenticationError
from products.ai_observability.backend.llm.types import OutputSchema
from products.ai_observability.backend.models.provider_keys import LLMProviderKey
@@ -417,3 +432,128 @@ class TestPlaygroundModelEnforcement(APIBaseTest):
assert models["typesafe/jev-1.13"]["provider"] == "OpenRouter"
assert models["typesafe/jev-1.13"]["supports_decisions"] is True
assert models["typesafe/jev-1.13"]["supports_chat"] is False
+
+
+class TestLLMProxyDecisionQuestionSerializer(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("choice_without_options", {"type": "choice", "instructions": "Topic?"}),
+ ("score_with_one_level", {"type": "score", "instructions": "Tone?", "options": [{"name": "calm"}]}),
+ (
+ "duplicate_option_names",
+ {"type": "choice", "instructions": "Topic?", "options": [{"name": "billing"}, {"name": "billing"}]},
+ ),
+ ]
+ )
+ def test_rejects_questions_system_one_cannot_take(self, _name: str, question: dict) -> None:
+ serializer = LLMProxyDecisionQuestionSerializer(data=question)
+ assert not serializer.is_valid()
+ assert "options" in serializer.errors
+
+
+class TestLLMProxyDecision(APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.organization.customer_id = "cus_test"
+ self.organization.save()
+ self.system_one_key = LLMProviderKey.objects.create(
+ team=self.team,
+ provider="system_one",
+ name="System One",
+ encrypted_config={"api_key": "s1-key", "base_url": "https://decisions.example.com", "model": "jev-1"},
+ created_by=self.user,
+ )
+
+ def _payload(self, provider_key_id: str) -> dict:
+ return {
+ "provider_key_id": provider_key_id,
+ "state": "user: My invoice is wrong",
+ "questions": {
+ "urgent": {"type": "noul", "instructions": "Is this urgent?"},
+ "topic": {
+ "type": "choice",
+ "instructions": "What is it about?",
+ "options": [{"name": "billing", "description": "invoices"}, {"name": "bugs"}],
+ },
+ "tone": {
+ "type": "score",
+ "instructions": "How upset?",
+ "options": [{"name": "calm"}, {"name": "angry"}],
+ },
+ "language": {"type": "noul", "instructions": "Is it in French?"},
+ },
+ }
+
+ @patch("products.ai_observability.backend.api.proxy.DecisionClient.evaluate")
+ def test_asks_the_system_one_connection_and_returns_every_answer(self, mock_evaluate) -> None:
+ mock_evaluate.return_value = SystemOneResult(
+ model="jev-1.13",
+ answers={
+ "urgent": NoulAnswer(probability=0.9),
+ "topic": ChoiceAnswer(choice="billing", confidence=0.8, probabilities={"billing": 0.9, "bugs": 0.1}),
+ "tone": ScoreAnswer(score=0.4, confidence=0.6, probabilities={"0": 0.6, "1": 0.4}),
+ "language": RefusalAnswer(),
+ },
+ input_tokens=42,
+ )
+
+ response = self.client.post(
+ "/api/llm_proxy/decision/", self._payload(str(self.system_one_key.id)), format="json"
+ )
+
+ assert response.status_code == 200, response.json()
+ call = mock_evaluate.call_args.kwargs
+ assert (call["api_key"], call["base_url"], call["model"]) == (
+ "s1-key",
+ "https://decisions.example.com",
+ "jev-1",
+ )
+ assert call["state"] == "user: My invoice is wrong"
+ assert call["questions"] == {
+ "urgent": NoulQuestion(instructions="Is this urgent?"),
+ "topic": ChoiceQuestion(instructions="What is it about?", criteria={"billing": "invoices", "bugs": None}),
+ "tone": ScoreQuestion(instructions="How upset?", criteria=["calm", "angry"]),
+ "language": NoulQuestion(instructions="Is it in French?"),
+ }
+ body = response.json()
+ assert body["model"] == "jev-1.13"
+ assert body["input_tokens"] == 42
+ assert body["answers"]["urgent"]["type"] == "noul"
+ assert body["answers"]["urgent"]["probability"] == 0.9
+ assert body["answers"]["topic"]["choice"] == "billing"
+ assert body["answers"]["tone"]["score"] == 0.4
+ assert body["answers"]["tone"]["probabilities"] == {"0": 0.6, "1": 0.4}
+ assert body["answers"]["language"]["type"] == "refusal"
+
+ @parameterized.expand([("other_provider_key", "openai"), ("unknown_key", None)])
+ @patch("products.ai_observability.backend.api.proxy.DecisionClient.evaluate")
+ def test_only_a_system_one_connection_can_answer(self, _name: str, provider: str | None, mock_evaluate) -> None:
+ key_id = str(uuid4())
+ if provider:
+ key_id = str(
+ LLMProviderKey.objects.create(
+ team=self.team,
+ provider=provider,
+ name="Other",
+ encrypted_config={"api_key": "sk-test", "base_url": "https://api.typesafe.ai"},
+ created_by=self.user,
+ ).id
+ )
+
+ response = self.client.post("/api/llm_proxy/decision/", self._payload(key_id), format="json")
+
+ assert response.status_code == 400
+ mock_evaluate.assert_not_called()
+
+ @patch("products.ai_observability.backend.api.proxy.DecisionClient.evaluate")
+ def test_endpoint_errors_reach_the_user(self, mock_evaluate) -> None:
+ mock_evaluate.side_effect = AuthenticationError(
+ "The endpoint rejected this credential. Check the bearer token."
+ )
+
+ response = self.client.post(
+ "/api/llm_proxy/decision/", self._payload(str(self.system_one_key.id)), format="json"
+ )
+
+ assert response.status_code == 400
+ assert response.json() == {"error": "The endpoint rejected this credential. Check the bearer token."}
diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.ts
index 1ada56d37351..6c3b7e0b1632 100644
--- a/products/ai_observability/frontend/modelPickerLogic.test.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts
@@ -162,6 +162,8 @@ describe('modelPickerLogic', () => {
'typesafe/jev-router',
'example/dual-model',
])
+ // The playground asks decisions only through a System One connection, never through OpenRouter.
+ expect(logic.values.playgroundSystemOneModelGroups).toEqual([])
})
it('should map is_recommended correctly for both true and false values', async () => {
diff --git a/products/ai_observability/frontend/modelPickerLogic.ts b/products/ai_observability/frontend/modelPickerLogic.ts
index de80f4ea91ad..e77e957a2aad 100644
--- a/products/ai_observability/frontend/modelPickerLogic.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.ts
@@ -144,7 +144,9 @@ export interface modelPickerLogicValues {
playgroundModels: ModelOption[]
playgroundModelsLoading: boolean
playgroundProviderModelGroups: ProviderModelGroup[]
+ playgroundSystemOneModelGroups: ProviderModelGroup[]
providerModelGroups: ProviderModelGroup[]
+ systemOneModels: ModelOption[]
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
@@ -231,6 +233,8 @@ export interface modelPickerLogicMeta {
failedByokProviderKeyIds: string[]
) => ProviderModelGroup[]
providerModelGroups: (evaluationProviderModelGroups: ProviderModelGroup[]) => ProviderModelGroup[]
+ systemOneModels: (byokModels: ModelOption[], providerKeys: LLMProviderKey[]) => ModelOption[]
+ playgroundSystemOneModelGroups: (evaluationProviderModelGroups: ProviderModelGroup[]) => ProviderModelGroup[]
playgroundByokProviderModelGroups: (providerModelGroups: ProviderModelGroup[]) => ProviderModelGroup[]
evaluationModelNotice: (
providerKeys: LLMProviderKey[],
@@ -462,6 +466,23 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
}))
.filter((group) => group.models.length > 0 || group.disabledReason || group.allowsCustomModel),
],
+ systemOneModels: [
+ (s) => [s.byokModels, s.providerKeys],
+ (models: ModelOption[], keys: LLMProviderKey[]): ModelOption[] =>
+ models.filter((model) =>
+ keys.some((key) => key.id === model.providerKeyId && key.provider === 'system_one')
+ ),
+ ],
+ // System One answers typed questions instead of chatting, so the playground lists it apart from the chat keys.
+ playgroundSystemOneModelGroups: [
+ (s) => [s.evaluationProviderModelGroups],
+ (groups: ProviderModelGroup[]): ProviderModelGroup[] =>
+ groups
+ .filter(
+ (group) => group.provider === 'system_one' && (group.models.length > 0 || group.disabledReason)
+ )
+ .map((group) => (group.allowsCustomModel ? { ...group, allowsCustomModel: false } : group)),
+ ],
// The playground runs only listed models, so it does not offer typed deployment names.
playgroundByokProviderModelGroups: [
(s) => [s.providerModelGroups],
diff --git a/products/ai_observability/frontend/playground/AIObservabilityPlaygroundScene.tsx b/products/ai_observability/frontend/playground/AIObservabilityPlaygroundScene.tsx
index d88214122d25..c87c783ad5eb 100644
--- a/products/ai_observability/frontend/playground/AIObservabilityPlaygroundScene.tsx
+++ b/products/ai_observability/frontend/playground/AIObservabilityPlaygroundScene.tsx
@@ -62,9 +62,11 @@ import {
type MessageRole,
type PromptConfig,
} from './llmPlaygroundPromptsLogic'
-import { llmPlaygroundRunLogic, type ComparisonItem, type UsageSummary } from './llmPlaygroundRunLogic'
+import { findPromptModel, llmPlaygroundRunLogic, type ComparisonItem, type UsageSummary } from './llmPlaygroundRunLogic'
import { llmPlaygroundVariablesLogic } from './llmPlaygroundVariablesLogic'
import { MessageToolCallsEditor } from './MessageToolCallsEditor'
+import { PlaygroundDecisionAnswers } from './PlaygroundDecisionAnswers'
+import { PlaygroundDecisionEditor } from './PlaygroundDecisionEditor'
import { PlaygroundSaveMenu } from './PlaygroundSaveMenu'
import { PlaygroundVariablesPanel } from './PlaygroundVariablesPanel'
import { TemplateVariableTextArea } from './TemplateVariableTextArea'
@@ -136,9 +138,9 @@ export function AIObservabilityPlaygroundScene(): JSX.Element {
}
function PlaygroundHeaderActions(): JSX.Element {
- const { hasRunnablePrompts, promptConfigs } = useValues(llmPlaygroundPromptsLogic)
+ const { promptConfigs } = useValues(llmPlaygroundPromptsLogic)
const { addPromptConfig, resetPlayground } = useActions(llmPlaygroundPromptsLogic)
- const { submitting: playgroundSubmitting } = useValues(llmPlaygroundRunLogic)
+ const { submitting: playgroundSubmitting, hasRunnablePrompts } = useValues(llmPlaygroundRunLogic)
const { submitPrompt, abortRun } = useActions(llmPlaygroundRunLogic)
const todayRailNav = useFeatureFlag('TODAY_RAIL_NAV')
const firstPromptId = promptConfigs[0]?.id
@@ -373,6 +375,8 @@ function PromptCard({
onRemove: () => void
}): JSX.Element {
const { submitting } = useValues(llmPlaygroundRunLogic)
+ const { systemOnePromptIds } = useValues(llmPlaygroundModelLogic)
+ const isSystemOne = systemOnePromptIds.includes(prompt.id)
const showHeaderRow = promptCount > 1 || canRemove
const messagesRef = React.useRef<HTMLDivElement>(null)
@@ -409,10 +413,14 @@ function PromptCard({
</div>
<div ref={messagesRef} className="min-h-0 flex-1 overflow-y-auto pr-1">
- <MessagesSection promptId={prompt.id} />
+ {isSystemOne ? (
+ <PlaygroundDecisionEditor promptId={prompt.id} />
+ ) : (
+ <MessagesSection promptId={prompt.id} />
+ )}
</div>
- <MessageActions promptId={prompt.id} scrollContainerRef={messagesRef} />
+ {!isSystemOne && <MessageActions promptId={prompt.id} scrollContainerRef={messagesRef} />}
</div>
)
}
@@ -429,6 +437,7 @@ function PromptResultCard({ item }: { item?: ComparisonItem }): JSX.Element {
const { addResultToConversation } = useActions(llmPlaygroundPromptsLogic)
const hasToolCalls = !!item?.toolCalls?.length
const canAddToConversation = (!!item?.response || hasToolCalls) && !item?.error && !isStreaming
+ const decision = item?.decision
const handleAddToConversation = (): void => {
if (!item || (!item.response && !item.toolCalls?.length)) {
@@ -443,7 +452,7 @@ function PromptResultCard({ item }: { item?: ComparisonItem }): JSX.Element {
<LemonTag type="default" size="small">
Result
</LemonTag>
- {item ? (
+ {item && item.provider !== 'system_one' ? (
<LemonButton
type="secondary"
size="small"
@@ -482,7 +491,9 @@ function PromptResultCard({ item }: { item?: ComparisonItem }): JSX.Element {
item.error ? 'text-danger' : ''
}`}
>
- {item.response ? (
+ {decision ? (
+ <PlaygroundDecisionAnswers questions={decision.questions} result={decision.result} />
+ ) : item.response ? (
<LemonMarkdown className="whitespace-pre-wrap break-words [&_img]:max-w-full [&_img]:h-auto [&_img]:rounded [&_img]:my-2">
{item.response}
</LemonMarkdown>
@@ -504,7 +515,7 @@ function PromptResultCard({ item }: { item?: ComparisonItem }): JSX.Element {
</div>
)}
</div>
- {(!!item.response || hasToolCalls || hasUsage(item.usage)) && (
+ {(!!item.response || hasToolCalls || !!decision || hasUsage(item.usage)) && (
<MetadataHeader
className="mt-2 pt-2"
isError={item.error}
@@ -535,11 +546,13 @@ function getPlaygroundModelsErrorMessage(errorStatus: number | null): string | n
function PlaygroundModelPicker({ promptId }: { promptId: string }): JSX.Element {
const prompt = usePromptConfig(promptId)
- const { effectiveModelOptions, playgroundModelsErrorStatus } = useValues(llmPlaygroundModelLogic)
+ const { effectiveModelOptions, runnableModelOptions, playgroundModelsErrorStatus } =
+ useValues(llmPlaygroundModelLogic)
const {
hasByokKeys,
playgroundByokProviderModelGroups,
playgroundProviderModelGroups,
+ playgroundSystemOneModelGroups,
byokModelsLoading,
playgroundModelsLoading,
providerKeysLoading,
@@ -552,8 +565,11 @@ function PlaygroundModelPicker({ promptId }: { promptId: string }): JSX.Element
return <LemonSkeleton className="h-10" />
}
- const selectedModel = effectiveModelOptions.find((m) => m.id === prompt.model)
- const groups = hasByokKeys ? playgroundByokProviderModelGroups : playgroundProviderModelGroups
+ const selectedModel = findPromptModel(prompt, runnableModelOptions)
+ const groups = [
+ ...(hasByokKeys ? playgroundByokProviderModelGroups : playgroundProviderModelGroups),
+ ...playgroundSystemOneModelGroups,
+ ]
const loading = hasByokKeys ? byokModelsLoading || providerKeysLoading : playgroundModelsLoading
const errorMessage = !hasByokKeys ? getPlaygroundModelsErrorMessage(playgroundModelsErrorStatus) : null
const showError = !hasByokKeys && effectiveModelOptions.length === 0 && !playgroundModelsLoading
@@ -680,6 +696,7 @@ function SettingsDropdownOverlay({ promptId }: { promptId: string }): JSX.Elemen
function ModelConfigBar({ promptId }: { promptId: string }): JSX.Element {
const prompt = usePromptConfig(promptId)
+ const { systemOnePromptIds } = useValues(llmPlaygroundModelLogic)
if (!prompt) {
return <LemonSkeleton className="h-8" />
@@ -698,32 +715,35 @@ function ModelConfigBar({ promptId }: { promptId: string }): JSX.Element {
<PlaygroundModelPicker promptId={promptId} />
</div>
- <LemonDropdown
- overlay={<SettingsDropdownOverlay promptId={promptId} />}
- closeOnClickInside={false}
- placement="bottom-end"
- onVisibilityChange={(visible) => {
- if (!visible && prompt) {
- posthog.capture('llma playground parameters configured', {
- max_tokens: prompt.maxTokens,
- temperature: prompt.temperature,
- top_p: prompt.topP,
- reasoning_level: prompt.reasoningLevel,
- thinking: prompt.thinking,
- })
- }
- }}
- >
- <LemonButton
- type="secondary"
- size="small"
- icon={<IconGear />}
- tooltip="Max tokens, thinking, reasoning"
- active={hasNonDefaultSettings}
+ {/* System One takes no sampling settings. */}
+ {!systemOnePromptIds.includes(promptId) && (
+ <LemonDropdown
+ overlay={<SettingsDropdownOverlay promptId={promptId} />}
+ closeOnClickInside={false}
+ placement="bottom-end"
+ onVisibilityChange={(visible) => {
+ if (!visible && prompt) {
+ posthog.capture('llma playground parameters configured', {
+ max_tokens: prompt.maxTokens,
+ temperature: prompt.temperature,
+ top_p: prompt.topP,
+ reasoning_level: prompt.reasoningLevel,
+ thinking: prompt.thinking,
+ })
+ }
+ }}
>
- Settings
- </LemonButton>
- </LemonDropdown>
+ <LemonButton
+ type="secondary"
+ size="small"
+ icon={<IconGear />}
+ tooltip="Max tokens, thinking, reasoning"
+ active={hasNonDefaultSettings}
+ >
+ Settings
+ </LemonButton>
+ </LemonDropdown>
+ )}
</div>
)
}
diff --git a/products/ai_observability/frontend/playground/DecisionQuestionEditor.tsx b/products/ai_observability/frontend/playground/DecisionQuestionEditor.tsx
new file mode 100644
index 000000000000..f68bd8ba26a3
--- /dev/null
+++ b/products/ai_observability/frontend/playground/DecisionQuestionEditor.tsx
@@ -0,0 +1,142 @@
+import { IconPlus, IconTrash } from '@posthog/icons'
+import { LemonButton, LemonInput, LemonSelect } from '@posthog/lemon-ui'
+
+import {
+ MAX_SCORE_LEVELS,
+ MIN_OPTIONS,
+ type PlaygroundQuestion,
+ type PlaygroundQuestionOption,
+ type PlaygroundQuestionType,
+ createPlaygroundQuestionOption,
+} from './playgroundDecisions'
+
+const QUESTION_TYPE_OPTIONS: { value: PlaygroundQuestionType; label: string }[] = [
+ { value: 'noul', label: 'Yes or no' },
+ { value: 'choice', label: 'Multiple choice' },
+ { value: 'score', label: 'Rating scale' },
+]
+
+export function DecisionQuestionEditor({
+ question,
+ disabledReason,
+ onChange,
+ onRemove,
+}: {
+ question: PlaygroundQuestion
+ disabledReason?: string
+ onChange: (question: PlaygroundQuestion) => void
+ onRemove: () => void
+}): JSX.Element {
+ const isChoice = question.type === 'choice'
+ const updateOption = (optionId: string, patch: Partial<PlaygroundQuestionOption>): void =>
+ onChange({
+ ...question,
+ options: question.options.map((option) => (option.id === optionId ? { ...option, ...patch } : option)),
+ })
+ const changeType = (type: PlaygroundQuestionType): void => {
+ const options = [...question.options]
+ while (options.length < MIN_OPTIONS[type]) {
+ options.push(createPlaygroundQuestionOption())
+ }
+ onChange({ ...question, type, options })
+ }
+
+ return (
+ <div className="flex flex-col gap-2 border rounded p-2 bg-surface-primary">
+ <div className="flex gap-2 items-center">
+ <LemonSelect
+ size="small"
+ value={question.type}
+ onChange={(type) => type && changeType(type)}
+ options={QUESTION_TYPE_OPTIONS}
+ disabledReason={disabledReason}
+ data-attr="llma-playground-decision-question-type"
+ />
+ <LemonInput
+ className="flex-1 min-w-0"
+ size="small"
+ value={question.instructions}
+ onChange={(instructions) => onChange({ ...question, instructions })}
+ placeholder="Does the assistant answer the user's question?"
+ disabled={!!disabledReason}
+ />
+ <LemonButton
+ size="small"
+ icon={<IconTrash />}
+ status="danger"
+ noPadding
+ onClick={onRemove}
+ tooltip="Remove question"
+ disabledReason={disabledReason}
+ data-attr="llma-playground-decision-remove-question"
+ />
+ </div>
+ {question.type !== 'noul' && (
+ <div className="flex flex-col gap-1 pl-2">
+ <span className="text-xs text-secondary">
+ {isChoice ? 'Options' : 'Scale, from lowest to highest'}
+ </span>
+ {question.options.map((option, index) => (
+ <div key={option.id} className="flex gap-2 items-center">
+ {!isChoice && <span className="text-xs text-secondary w-4 text-right">{index}</span>}
+ <LemonInput
+ className={isChoice ? 'w-40' : 'flex-1 min-w-0'}
+ size="small"
+ value={option.name}
+ onChange={(name) => updateOption(option.id, { name })}
+ placeholder={isChoice ? 'Resolved' : 'Not helpful'}
+ disabled={!!disabledReason}
+ />
+ {isChoice && (
+ <LemonInput
+ className="flex-1 min-w-0"
+ size="small"
+ value={option.description}
+ onChange={(description) => updateOption(option.id, { description })}
+ placeholder="What this option means (optional)"
+ disabled={!!disabledReason}
+ />
+ )}
+ <LemonButton
+ size="small"
+ icon={<IconTrash />}
+ status="danger"
+ noPadding
+ onClick={() =>
+ onChange({
+ ...question,
+ options: question.options.filter((candidate) => candidate.id !== option.id),
+ })
+ }
+ tooltip={isChoice ? 'Remove option' : 'Remove level'}
+ disabledReason={disabledReason}
+ />
+ </div>
+ ))}
+ <div>
+ <LemonButton
+ type="secondary"
+ size="xsmall"
+ icon={<IconPlus />}
+ onClick={() =>
+ onChange({
+ ...question,
+ options: [...question.options, createPlaygroundQuestionOption()],
+ })
+ }
+ disabledReason={
+ disabledReason ??
+ (!isChoice && question.options.length >= MAX_SCORE_LEVELS
+ ? `A rating scale takes at most ${MAX_SCORE_LEVELS} levels`
+ : undefined)
+ }
+ data-attr="llma-playground-decision-add-option"
+ >
+ {isChoice ? 'Option' : 'Level'}
+ </LemonButton>
+ </div>
+ </div>
+ )}
+ </div>
+ )
+}
diff --git a/products/ai_observability/frontend/playground/PlaygroundDecisionAnswers.stories.tsx b/products/ai_observability/frontend/playground/PlaygroundDecisionAnswers.stories.tsx
new file mode 100644
index 000000000000..fa603fad84aa
--- /dev/null
+++ b/products/ai_observability/frontend/playground/PlaygroundDecisionAnswers.stories.tsx
@@ -0,0 +1,84 @@
+import { Meta, StoryObj } from '@storybook/react'
+
+import { PlaygroundDecisionAnswers } from './PlaygroundDecisionAnswers'
+
+const meta: Meta<typeof PlaygroundDecisionAnswers> = {
+ title: 'Scenes-App/AI Observability/Playground Decision Answers',
+ component: PlaygroundDecisionAnswers,
+ parameters: { testOptions: { snapshotBrowsers: [] } },
+}
+export default meta
+type Story = StoryObj<typeof PlaygroundDecisionAnswers>
+
+export const EveryAnswerType: Story = {
+ render: function Render() {
+ return (
+ <div className="max-w-150 text-xs">
+ <PlaygroundDecisionAnswers
+ questions={[
+ { id: 'upset', type: 'noul', instructions: 'Is the user upset?', options: [] },
+ {
+ id: 'topic',
+ type: 'choice',
+ instructions: 'What is the message about?',
+ options: [
+ { id: 'a', name: 'billing', description: 'Invoices and charges' },
+ { id: 'b', name: 'bug', description: '' },
+ ],
+ },
+ {
+ id: 'urgency',
+ type: 'score',
+ instructions: 'How urgent is it?',
+ options: [
+ { id: 'c', name: 'Low', description: '' },
+ { id: 'd', name: 'Medium', description: '' },
+ { id: 'e', name: 'High', description: '' },
+ ],
+ },
+ { id: 'language', type: 'noul', instructions: 'Is it written in French?', options: [] },
+ ]}
+ result={{
+ model: 'jev-1.13',
+ answers: {
+ upset: {
+ type: 'noul',
+ probability: 0.83,
+ choice: null,
+ score: null,
+ confidence: null,
+ probabilities: null,
+ },
+ topic: {
+ type: 'choice',
+ probability: null,
+ choice: 'billing',
+ score: null,
+ confidence: 0.7,
+ probabilities: { billing: 0.67, bug: 0.33 },
+ },
+ urgency: {
+ type: 'score',
+ probability: null,
+ choice: null,
+ score: 0.6,
+ confidence: 0.55,
+ probabilities: { '0': 0.57, '1': 0.29, '2': 0.14 },
+ },
+ language: {
+ type: 'refusal',
+ probability: null,
+ choice: null,
+ score: null,
+ confidence: null,
+ probabilities: null,
+ },
+ },
+ input_tokens: 37,
+ output_tokens: null,
+ }}
+ />
+ </div>
+ )
+ },
+}
diff --git a/products/ai_observability/frontend/playground/PlaygroundDecisionAnswers.tsx b/products/ai_observability/frontend/playground/PlaygroundDecisionAnswers.tsx
new file mode 100644
index 000000000000..300e4a194b11
--- /dev/null
+++ b/products/ai_observability/frontend/playground/PlaygroundDecisionAnswers.tsx
@@ -0,0 +1,71 @@
+import { LemonTag } from '@posthog/lemon-ui'
+
+import type { PlaygroundDecisionAnswer, PlaygroundDecisionResult, PlaygroundQuestion } from './playgroundDecisions'
+
+function percent(value: number | null): string {
+ return value === null ? '' : `${Math.round(value * 100)}%`
+}
+
+function AnswerSummary({
+ answer,
+ question,
+}: {
+ answer: PlaygroundDecisionAnswer | undefined
+ question: PlaygroundQuestion
+}): JSX.Element {
+ if (!answer || answer.type === 'refusal') {
+ return <LemonTag type="muted">Declined to answer</LemonTag>
+ }
+ if (answer.type === 'noul') {
+ const yes = answer.probability ?? 0
+ return (
+ <div className="flex items-center gap-2">
+ <LemonTag type={yes >= 0.5 ? 'success' : 'danger'}>{yes >= 0.5 ? 'Yes' : 'No'}</LemonTag>
+ <span className="text-secondary">{percent(yes)} yes</span>
+ </div>
+ )
+ }
+ // Score probabilities are keyed by level index, so read them back as the level names.
+ const labelOf = (key: string): string =>
+ answer.type === 'score' ? (question.options[Number(key)]?.name.trim() ?? key) : key
+ const ranked = Object.entries(answer.probabilities ?? {}).sort(([, a], [, b]) => b - a)
+ const score = answer.score ?? 0
+ return (
+ <div className="flex flex-col gap-1">
+ <div className="flex flex-wrap items-center gap-2">
+ <LemonTag type="highlight">
+ {answer.type === 'choice' ? answer.choice : labelOf(String(Math.round(score)))}
+ </LemonTag>
+ {answer.type === 'score' && (
+ <span className="text-secondary">
+ {score.toFixed(2)} on a 0 to {question.options.length - 1} scale
+ </span>
+ )}
+ <span className="text-secondary">{percent(answer.confidence)} confidence</span>
+ </div>
+ <div className="text-secondary">
+ {ranked.map(([key, probability]) => `${labelOf(key)} ${percent(probability)}`).join(' · ')}
+ </div>
+ </div>
+ )
+}
+
+export function PlaygroundDecisionAnswers({
+ questions,
+ result,
+}: {
+ questions: PlaygroundQuestion[]
+ result: PlaygroundDecisionResult
+}): JSX.Element {
+ return (
+ <div className="flex flex-col gap-3">
+ {questions.map((question) => (
+ <div key={question.id} className="flex flex-col gap-1">
+ <div className="font-semibold">{question.instructions}</div>
+ <AnswerSummary answer={result.answers[question.id]} question={question} />
+ </div>
+ ))}
+ <div className="text-secondary">Answered by {result.model}</div>
+ </div>
+ )
+}
diff --git a/products/ai_observability/frontend/playground/PlaygroundDecisionEditor.tsx b/products/ai_observability/frontend/playground/PlaygroundDecisionEditor.tsx
new file mode 100644
index 000000000000..de0ea5aee4d5
--- /dev/null
+++ b/products/ai_observability/frontend/playground/PlaygroundDecisionEditor.tsx
@@ -0,0 +1,87 @@
+import { useActions, useValues } from 'kea'
+
+import { IconPlus } from '@posthog/icons'
+import { LemonButton } from '@posthog/lemon-ui'
+
+import { DecisionQuestionEditor } from './DecisionQuestionEditor'
+import { llmPlaygroundPromptsLogic } from './llmPlaygroundPromptsLogic'
+import { llmPlaygroundRunLogic } from './llmPlaygroundRunLogic'
+import { llmPlaygroundVariablesLogic } from './llmPlaygroundVariablesLogic'
+import { MAX_DECISION_QUESTIONS, createPlaygroundQuestion } from './playgroundDecisions'
+import { TemplateVariableTextArea } from './TemplateVariableTextArea'
+
+export function PlaygroundDecisionEditor({ promptId }: { promptId: string }): JSX.Element | null {
+ const { promptConfigs } = useValues(llmPlaygroundPromptsLogic)
+ const { setDecisionInput, setDecisionQuestions } = useActions(llmPlaygroundPromptsLogic)
+ const { submitting } = useValues(llmPlaygroundRunLogic)
+ const { submitPrompt } = useActions(llmPlaygroundRunLogic)
+ const { unfilledVariables } = useValues(llmPlaygroundVariablesLogic)
+ const prompt = promptConfigs.find((candidate) => candidate.id === promptId)
+
+ if (!prompt) {
+ return null
+ }
+
+ const questions = prompt.decisionQuestions
+ const disabledReason = submitting ? 'Wait for the run to finish' : undefined
+
+ return (
+ <div className="flex flex-col gap-4">
+ <div className="flex flex-col gap-1">
+ <div className="text-xs font-semibold">Input</div>
+ <TemplateVariableTextArea
+ value={prompt.decisionInput}
+ onChange={(decisionInput) => setDecisionInput(decisionInput, promptId)}
+ unfilledVariables={unfilledVariables}
+ placeholder="Paste the text to judge, for example a conversation or a support ticket"
+ minRows={6}
+ onPressCmdEnter={() => submitPrompt()}
+ />
+ </div>
+ <div className="flex flex-col gap-2">
+ <div>
+ <div className="text-xs font-semibold">Questions</div>
+ <div className="text-xs text-secondary">
+ The model answers each question about the input, with a probability for every answer.
+ </div>
+ </div>
+ {questions.map((question) => (
+ <DecisionQuestionEditor
+ key={question.id}
+ question={question}
+ disabledReason={disabledReason}
+ onChange={(updated) =>
+ setDecisionQuestions(
+ questions.map((candidate) => (candidate.id === updated.id ? updated : candidate)),
+ promptId
+ )
+ }
+ onRemove={() =>
+ setDecisionQuestions(
+ questions.filter((candidate) => candidate.id !== question.id),
+ promptId
+ )
+ }
+ />
+ ))}
+ <div>
+ <LemonButton
+ type="secondary"
+ size="small"
+ icon={<IconPlus />}
+ onClick={() => setDecisionQuestions([...questions, createPlaygroundQuestion()], promptId)}
+ disabledReason={
+ disabledReason ??
+ (questions.length >= MAX_DECISION_QUESTIONS
+ ? `Ask at most ${MAX_DECISION_QUESTIONS} questions at once`
+ : undefined)
+ }
+ data-attr="llma-playground-decision-add-question"
+ >
+ Question
+ </LemonButton>
+ </div>
+ </div>
+ </div>
+ )
+}
diff --git a/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts b/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts
index 133a9a9cc4e2..fb6cd7e07fd5 100644
--- a/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts
+++ b/products/ai_observability/frontend/playground/llmPlaygroundModelLogic.ts
@@ -9,6 +9,7 @@ import type { ProviderModelGroup } from '../modelPickerLogic'
import { LLMProviderKey, llmProviderKeysLogic, normalizeLLMProvider } from '../settings/llmProviderKeysLogic'
import { llmPlaygroundPromptsLogic, type PromptConfig } from './llmPlaygroundPromptsLogic'
import type { PlaygroundSetupPayload } from './llmPlaygroundPromptsLogic'
+import { messagesToDecisionInput } from './playgroundDecisions'
import {
isTraceLikeSelection,
matchClosestModel,
@@ -23,9 +24,15 @@ function normalizePromptsToAvailableModels(
promptConfigs: PromptConfig[],
pendingTargetModel: string | null,
availableModels: MatchModelOption[],
- providerKeys: LLMProviderKey[]
+ providerKeys: LLMProviderKey[],
+ systemOnePromptIds: string[]
): PromptConfig[] {
return promptConfigs.map((prompt: PromptConfig, index: number) => {
+ const isPendingTarget = index === 0 && !!pendingTargetModel
+ // A System One model is not in the chat-only list, so matching would swap in a chat model and keep the System One key.
+ if (!isPendingTarget && systemOnePromptIds.includes(prompt.id)) {
+ return prompt
+ }
const targetModel = index === 0 && pendingTargetModel ? pendingTargetModel : prompt.model
const matchedModel = matchClosestModelOption(targetModel, availableModels, providerKeys)
return {
@@ -57,6 +64,7 @@ export interface llmPlaygroundModelLogicValues {
playgroundModels: ModelOption[] // modelPickerLogic
playgroundModelsLoading: boolean // modelPickerLogic
playgroundProviderModelGroups: ProviderModelGroup[] // modelPickerLogic
+ systemOneModels: ModelOption[] // modelPickerLogic
activeProviderKeyId: string | null
allModelOptions: ModelOption[]
byokModelsSettled: boolean
@@ -72,6 +80,8 @@ export interface llmPlaygroundModelLogicValues {
playgroundModelsSettled: boolean
providerKeyForCurrentModel: LLMProviderKey | null
providerKeysSettled: boolean
+ runnableModelOptions: ModelOption[]
+ systemOnePromptIds: string[]
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
@@ -86,6 +96,13 @@ export interface llmPlaygroundModelLogicActions {
clearPendingTargetModel: () => {
value: true
} // llmPlaygroundPromptsLogic
+ setDecisionInput: (
+ decisionInput: string,
+ promptId?: string | undefined
+ ) => {
+ decisionInput: string
+ promptId: string | undefined
+ } // llmPlaygroundPromptsLogic
setModel: (
model: string,
providerKeyId?: string | undefined,
@@ -180,6 +197,8 @@ export interface llmPlaygroundModelLogicMeta {
byokModels: ModelOption[],
playgroundModels: ModelOption[]
) => ModelOption[]
+ runnableModelOptions: (effectiveModelOptions: ModelOption[], systemOneModels: ModelOption[]) => ModelOption[]
+ systemOnePromptIds: (promptConfigs: PromptConfig[], providerKeys: LLMProviderKey[]) => string[]
allModelOptions: (playgroundModels: ModelOption[], byokModels: ModelOption[]) => ModelOption[]
providerKeyForCurrentModel: (
activePromptConfig: PromptConfig,
@@ -220,6 +239,7 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
'playgroundModelsLoading',
'hasByokKeys',
'playgroundProviderModelGroups',
+ 'systemOneModels',
],
llmProviderKeysLogic,
['providerKeys', 'providerKeysLoading'],
@@ -232,6 +252,7 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
'setModel',
'setPromptConfigs',
'clearPendingTargetModel',
+ 'setDecisionInput',
],
modelPickerLogic,
[
@@ -322,6 +343,26 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
(hasByokKeys: boolean, byokModels: ModelOption[], playgroundModels: ModelOption[]): ModelOption[] =>
hasByokKeys && byokModels.length > 0 ? byokModels : playgroundModels,
],
+ // Chat models plus System One models, which the normalizers above leave out because they cannot chat.
+ runnableModelOptions: [
+ (s) => [s.effectiveModelOptions, s.systemOneModels],
+ (effectiveModelOptions: ModelOption[], systemOneModels: ModelOption[]): ModelOption[] => [
+ ...effectiveModelOptions,
+ ...systemOneModels,
+ ],
+ ],
+ // Read from the selected key, not the loaded models, so a failed model load keeps the panel on System One.
+ systemOnePromptIds: [
+ (s) => [s.promptConfigs, s.providerKeys],
+ (promptConfigs: PromptConfig[], providerKeys: LLMProviderKey[]): string[] =>
+ promptConfigs
+ .filter((prompt) =>
+ providerKeys.some(
+ (key) => key.id === prompt.selectedProviderKeyId && key.provider === 'system_one'
+ )
+ )
+ .map((prompt) => prompt.id),
+ ],
allModelOptions: [
(s) => [s.playgroundModels, s.byokModels],
(playgroundModels: ModelOption[], byokModels: ModelOption[]): ModelOption[] => [
@@ -381,13 +422,22 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
values.promptConfigs,
pendingTargetModel,
values.effectiveModelOptions,
- values.providerKeys
+ values.providerKeys,
+ values.systemOnePromptIds
)
actions.setPromptConfigs(normalizedPromptsWithTarget)
actions.clearPendingTargetModel()
}
return {
+ setModel: ({ promptId }: { promptId?: string }) => {
+ const prompt = promptId
+ ? values.promptConfigs.find((candidate) => candidate.id === promptId)
+ : values.promptConfigs[0]
+ if (prompt && !prompt.decisionInput.trim() && values.systemOnePromptIds.includes(prompt.id)) {
+ actions.setDecisionInput(messagesToDecisionInput(prompt.messages), prompt.id)
+ }
+ },
loadEvaluationConfigSuccess: ({
evaluationConfig,
}: {
@@ -407,7 +457,10 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
if (index === 0 && pendingTargetModel) {
return prompt
}
- if (values.hasByokKeys && prompt.selectedProviderKeyId) {
+ if (
+ (values.hasByokKeys && prompt.selectedProviderKeyId) ||
+ values.systemOnePromptIds.includes(prompt.id)
+ ) {
return prompt
}
@@ -452,7 +505,8 @@ export const llmPlaygroundModelLogic = kea<llmPlaygroundModelLogicType>([
values.promptConfigs,
targetModelForFirstPrompt,
byokModels,
- values.providerKeys
+ values.providerKeys,
+ values.systemOnePromptIds
)
actions.setPromptConfigs(normalizedPrompts)
diff --git a/products/ai_observability/frontend/playground/llmPlaygroundPromptsLogic.ts b/products/ai_observability/frontend/playground/llmPlaygroundPromptsLogic.ts
index d4b951d1f9f1..03dead69332c 100644
--- a/products/ai_observability/frontend/playground/llmPlaygroundPromptsLogic.ts
+++ b/products/ai_observability/frontend/playground/llmPlaygroundPromptsLogic.ts
@@ -23,7 +23,7 @@ import { llmPromptLogic } from '../prompts/llmPromptLogic'
import { getApiErrorDetail } from '../prompts/utils'
import { normalizeLLMProvider } from '../settings/llmProviderKeysLogic'
import { isOTelPartsMessage, normalizeRole } from '../utils'
-import { isMessageSendable } from './playgroundMessageMapping'
+import { type PlaygroundQuestion, createPlaygroundQuestion } from './playgroundDecisions'
import { isTraceLikeSelection } from './playgroundModelMatching'
import { type PlaygroundModelConfig, parsePlaygroundConfig, serializePlaygroundConfig } from './playgroundPromptConfig'
import { type RawMessage, appendRawMessage, flattenOutputMessages } from './playgroundTraceImport'
@@ -88,6 +88,8 @@ export interface PromptConfig {
sourceEvaluationId: string | null
sourceEvaluationName: string | null
messages: Message[]
+ decisionInput: string
+ decisionQuestions: PlaygroundQuestion[]
}
export interface PlaygroundSetupPayload {
@@ -187,6 +189,8 @@ export function createPromptConfig(partial: Partial<PromptConfig> = {}): PromptC
sourceEvaluationId: partial.sourceEvaluationId ?? null,
sourceEvaluationName: partial.sourceEvaluationName ?? null,
messages: partial.messages ?? [],
+ decisionInput: partial.decisionInput ?? '',
+ decisionQuestions: partial.decisionQuestions ?? [createPlaygroundQuestion()],
}
}
@@ -223,7 +227,6 @@ export interface llmPlaygroundPromptsLogicValues {
promptId: string
type: 'message' | 'response_format' | 'system' | 'tools'
} | null
- hasRunnablePrompts: boolean
linkedSource: {
evaluationId: string | null
evaluationName: string | null
@@ -369,6 +372,20 @@ export interface llmPlaygroundPromptsLogicActions {
setActivePromptId: (promptId: string | null) => {
promptId: string | null
}
+ setDecisionInput: (
+ decisionInput: string,
+ promptId?: string
+ ) => {
+ decisionInput: string
+ promptId: string | undefined
+ }
+ setDecisionQuestions: (
+ decisionQuestions: PlaygroundQuestion[],
+ promptId?: string
+ ) => {
+ decisionQuestions: PlaygroundQuestion[]
+ promptId: string | undefined
+ }
setEditModal: (
target: {
messageIndex?: number
@@ -536,7 +553,6 @@ export interface llmPlaygroundPromptsLogicMeta {
promptVersion: number | null
type: 'evaluation' | 'prompt' | null
}
- hasRunnablePrompts: (promptConfigs: PromptConfig[]) => boolean
}
}
@@ -564,6 +580,11 @@ export const llmPlaygroundPromptsLogic = kea<llmPlaygroundPromptsLogicType>([
setThinking: (thinking: boolean, promptId?: string) => ({ thinking, promptId }),
setReasoningLevel: (reasoningLevel: ReasoningLevel, promptId?: string) => ({ reasoningLevel, promptId }),
setTools: (tools: Record<string, unknown>[] | null, promptId?: string) => ({ tools, promptId }),
+ setDecisionInput: (decisionInput: string, promptId?: string) => ({ decisionInput, promptId }),
+ setDecisionQuestions: (decisionQuestions: PlaygroundQuestion[], promptId?: string) => ({
+ decisionQuestions,
+ promptId,
+ }),
setResponseFormat: (responseFormat: PlaygroundResponseFormat | null, promptId?: string) => ({
responseFormat,
promptId,
@@ -683,6 +704,14 @@ export const llmPlaygroundPromptsLogic = kea<llmPlaygroundPromptsLogicType>([
state: PromptConfig[],
{ tools, promptId }: { tools: Record<string, unknown>[] | null; promptId?: string }
) => updatePromptConfigs(state, promptId, (prompt) => ({ ...prompt, tools })),
+ setDecisionInput: (
+ state: PromptConfig[],
+ { decisionInput, promptId }: { decisionInput: string; promptId?: string }
+ ) => updatePromptConfigs(state, promptId, (prompt) => ({ ...prompt, decisionInput })),
+ setDecisionQuestions: (
+ state: PromptConfig[],
+ { decisionQuestions, promptId }: { decisionQuestions: PlaygroundQuestion[]; promptId?: string }
+ ) => updatePromptConfigs(state, promptId, (prompt) => ({ ...prompt, decisionQuestions })),
setResponseFormat: (
state: PromptConfig[],
{ responseFormat, promptId }: { responseFormat: PlaygroundResponseFormat | null; promptId?: string }
@@ -807,6 +836,7 @@ export const llmPlaygroundPromptsLogic = kea<llmPlaygroundPromptsLogicType>([
sourcePromptVersion: payload.sourcePromptVersion ?? null,
sourceEvaluationId: payload.sourceEvaluationId ?? null,
sourceEvaluationName: null,
+ decisionInput: '',
},
]
},
@@ -1083,11 +1113,6 @@ export const llmPlaygroundPromptsLogic = kea<llmPlaygroundPromptsLogicType>([
}
},
],
- hasRunnablePrompts: [
- (s) => [s.promptConfigs],
- (promptConfigs: PromptConfig[]): boolean =>
- promptConfigs.some((prompt) => prompt.messages.some(isMessageSendable)),
- ],
}),
listeners(({ actions, values }) => ({
diff --git a/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.test.ts b/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.test.ts
index 5fee1529195d..c67a566db61b 100644
--- a/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.test.ts
+++ b/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.test.ts
@@ -9,11 +9,13 @@ import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
import { AccessControlLevel } from '~/types'
+import { modelPickerLogic } from '../modelPickerLogic'
import { createPromptConfig, llmPlaygroundPromptsLogic } from './llmPlaygroundPromptsLogic'
import {
appendToolCallChunk,
describeError,
escapeMarkdownInline,
+ findPromptModel,
llmPlaygroundRunLogic,
mergeUsage,
} from './llmPlaygroundRunLogic'
@@ -36,6 +38,30 @@ function mockStreamWithRetryContract(
}
}
+function useSystemOneKeyMocks(): void {
+ useMocks({
+ get: {
+ '/api/environments/:team_id/llm_analytics/provider_keys/': {
+ results: [{ id: 'key-s1', provider: 'system_one', name: 'System One', state: 'ok' }],
+ },
+ '/api/llm_proxy/models/': ({ request }) => [
+ 200,
+ new URL(request.url).searchParams.get('provider_key_id')
+ ? [
+ {
+ id: 'jev-1',
+ name: 'jev-1',
+ provider: 'System One',
+ supports_decisions: true,
+ supports_chat: false,
+ },
+ ]
+ : [{ id: 'gpt-5-mini', name: 'GPT-5 Mini', provider: 'OpenAI', description: '' }],
+ ],
+ },
+ })
+}
+
function setPlaygroundAccessLevel(level: AccessControlLevel): void {
window.POSTHOG_APP_CONTEXT = {
...window.POSTHOG_APP_CONTEXT,
@@ -130,6 +156,156 @@ describe('llmPlaygroundRunLogic', () => {
streamSpy.mockRestore()
})
+ it('carries the conversation into the System One input and sends it to the decision endpoint', async () => {
+ useSystemOneKeyMocks()
+ const result = {
+ model: 'jev-1.13',
+ answers: {
+ q1: {
+ type: 'noul',
+ probability: 0.9,
+ choice: null,
+ score: null,
+ confidence: null,
+ probabilities: null,
+ },
+ },
+ input_tokens: 12,
+ output_tokens: null,
+ }
+ const createSpy = jest.spyOn(api, 'create').mockResolvedValue(result)
+ const streamSpy = jest.spyOn(api, 'stream').mockImplementation(async () => {})
+
+ const logic = llmPlaygroundRunLogic()
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+
+ const question = { id: 'q1', type: 'noul' as const, instructions: 'Is {{topic}} discussed?', options: [] }
+ llmPlaygroundPromptsLogic.actions.setMessages([
+ { role: 'user', content: 'Tell me about {{topic}}' },
+ { role: 'assistant', content: 'They swim.' },
+ ])
+ llmPlaygroundPromptsLogic.actions.setModel('jev-1', 'key-s1')
+ await expectLogic(logic).toFinishAllListeners()
+ expect(llmPlaygroundPromptsLogic.values.promptConfigs[0].decisionInput).toBe(
+ 'user: Tell me about {{topic}}\n\nassistant: They swim.'
+ )
+ // The input alone makes a System One panel runnable, so the Run button must not wait for messages.
+ llmPlaygroundPromptsLogic.actions.setMessages([])
+ expect(logic.values.hasRunnablePrompts).toBe(true)
+ llmPlaygroundPromptsLogic.actions.setDecisionQuestions([question])
+ llmPlaygroundVariablesLogic.actions.setVariableValue('topic', 'penguins')
+ llmPlaygroundRunLogic.actions.submitPrompt()
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(streamSpy).not.toHaveBeenCalled()
+ expect(createSpy).toHaveBeenCalledWith(
+ '/api/llm_proxy/decision/',
+ {
+ provider_key_id: 'key-s1',
+ state: 'user: Tell me about penguins\n\nassistant: They swim.',
+ questions: { q1: { type: 'noul', instructions: 'Is penguins discussed?' } },
+ },
+ expect.anything()
+ )
+ expect(logic.values.comparisonItems[0]).toMatchObject({
+ error: false,
+ decision: { questions: [question], result },
+ usage: { prompt_tokens: 12 },
+ })
+
+ logic.unmount()
+ createSpy.mockRestore()
+ streamSpy.mockRestore()
+ })
+
+ it.each([
+ ['its models load', 200],
+ ['its model load fails', 500],
+ ])('keeps a System One panel on its model when the key list reloads and %s', async (_case, systemOneStatus) => {
+ // A chat key too, so the reload has chat models to match a System One panel against.
+ useMocks({
+ get: {
+ '/api/environments/:team_id/llm_analytics/provider_keys/': {
+ results: [
+ { id: 'key-s1', provider: 'system_one', name: 'System One', state: 'ok' },
+ { id: 'key-oa', provider: 'openai', name: 'OpenAI', state: 'ok' },
+ ],
+ },
+ '/api/llm_proxy/models/': ({ request }) => {
+ const keyId = new URL(request.url).searchParams.get('provider_key_id')
+ if (keyId === 'key-s1') {
+ return systemOneStatus === 200
+ ? [200, [{ id: 'jev-1', name: 'jev-1', provider: 'System One', supports_chat: false }]]
+ : [systemOneStatus, { detail: 'Could not list models' }]
+ }
+ return [200, [{ id: 'gpt-5-mini', name: 'gpt-5-mini', provider: 'OpenAI', description: '' }]]
+ },
+ },
+ })
+ const logic = llmPlaygroundRunLogic()
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+
+ llmPlaygroundPromptsLogic.actions.setModel('jev-1', 'key-s1')
+ modelPickerLogic.actions.loadByokModels()
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(llmPlaygroundPromptsLogic.values.promptConfigs[0]).toMatchObject({
+ model: 'jev-1',
+ selectedProviderKeyId: 'key-s1',
+ })
+ logic.unmount()
+ })
+
+ it('starts a new trace with an empty System One input, so the next prefill uses the new conversation', async () => {
+ useSystemOneKeyMocks()
+ const logic = llmPlaygroundRunLogic()
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+
+ llmPlaygroundPromptsLogic.actions.setDecisionInput('user: the first trace')
+ llmPlaygroundPromptsLogic.actions.setupPlaygroundFromEvent({ input: 'the second trace' })
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(llmPlaygroundPromptsLogic.values.promptConfigs[0].decisionInput).toBe('')
+ logic.unmount()
+ })
+
+ it('shows the rate limit banner when the decision endpoint throttles', async () => {
+ useSystemOneKeyMocks()
+ const createSpy = jest
+ .spyOn(api, 'create')
+ .mockRejectedValue(new ApiError('Throttled', 429, new Headers({ 'Retry-After': '30' })))
+
+ const logic = llmPlaygroundRunLogic()
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+
+ llmPlaygroundPromptsLogic.actions.setModel('jev-1', 'key-s1')
+ llmPlaygroundPromptsLogic.actions.setDecisionInput('user: hello')
+ llmPlaygroundPromptsLogic.actions.setDecisionQuestions([
+ { id: 'q1', type: 'noul', instructions: 'Is it a greeting?', options: [] },
+ ])
+ llmPlaygroundRunLogic.actions.submitPrompt()
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(logic.values.rateLimitedUntil).not.toBeNull()
+
+ logic.unmount()
+ createSpy.mockRestore()
+ })
+
+ it('picks the model entry of the selected key when two connections share a model id', () => {
+ const prompt = createPromptConfig({ model: 'jev-1', selectedProviderKeyId: 'key-s1' })
+ const options = [
+ { id: 'jev-1', name: 'jev-1', provider: 'OpenRouter', description: '', providerKeyId: 'key-or' },
+ { id: 'jev-1', name: 'jev-1', provider: 'System One', description: '', providerKeyId: 'key-s1' },
+ ]
+
+ expect(findPromptModel(prompt, options)?.provider).toBe('System One')
+ })
+
it('normalizes a display-name provider to its enum value in the completion payload', async () => {
useMocks({
get: {
diff --git a/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.ts b/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.ts
index a948ab7acd35..f33e3aa5ae82 100644
--- a/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.ts
+++ b/products/ai_observability/frontend/playground/llmPlaygroundRunLogic.ts
@@ -16,9 +16,20 @@ import type { LLMProviderKey } from '../settings/llmProviderKeysLogic'
import { llmPlaygroundModelLogic } from './llmPlaygroundModelLogic'
import { llmPlaygroundPromptsLogic, type Message, type PromptConfig } from './llmPlaygroundPromptsLogic'
import { llmPlaygroundVariablesLogic } from './llmPlaygroundVariablesLogic'
+import {
+ type PlaygroundDecisionResult,
+ type PlaygroundQuestion,
+ decisionQuestionsDisabledReason,
+ toDecisionQuestionsPayload,
+} from './playgroundDecisions'
import { isMessageSendable, toProviderMessages } from './playgroundMessageMapping'
import { resolveProviderKeyForPrompt } from './playgroundModelMatching'
-import { extractVariablesFromTexts, getVariableValue, substituteVariables } from './playgroundTemplating'
+import {
+ extractVariablesFromTexts,
+ getVariableValue,
+ promptTemplateTexts,
+ substituteVariables,
+} from './playgroundTemplating'
interface ToolCallChunk {
id?: string
@@ -154,6 +165,70 @@ export interface ComparisonItem {
usage?: UsageSummary
ttftMs?: number | null
latencyMs?: number | null
+ /** Set for a System One run, with the questions as asked so the answers keep their labels after edits. */
+ decision?: { questions: PlaygroundQuestion[]; result: PlaygroundDecisionResult } | null
+}
+
+function buildCompletionRequest(
+ prompt: PromptConfig,
+ model: string,
+ provider: string,
+ providerKeyId: string | null,
+ system: string,
+ messages: Message[]
+): Record<string, unknown> {
+ return {
+ system,
+ messages: toProviderMessages(messages),
+ model,
+ provider,
+ thinking: prompt.thinking,
+ ...(providerKeyId ? { provider_key_id: providerKeyId } : {}),
+ ...(prompt.tools ? { tools: prompt.tools } : {}),
+ ...(prompt.responseFormat ? { response_format: prompt.responseFormat } : {}),
+ ...(prompt.maxTokens !== null && prompt.maxTokens > 0 ? { max_tokens: prompt.maxTokens } : {}),
+ ...(typeof prompt.temperature === 'number' ? { temperature: prompt.temperature } : {}),
+ ...(typeof prompt.topP === 'number' ? { top_p: prompt.topP } : {}),
+ ...(prompt.reasoningLevel ? { reasoning_level: prompt.reasoningLevel } : {}),
+ }
+}
+
+/** Throws RateLimitError for a throttled request, so it reaches the same banner as a throttled stream. */
+async function requestDecision(
+ prompt: PromptConfig,
+ providerKeyId: string | null,
+ variableValues: Record<string, string>,
+ signal: AbortSignal
+): Promise<PlaygroundDecisionResult> {
+ const resolve = (text: string): string => substituteVariables(text, variableValues)
+ try {
+ // nosemgrep: prefer-codegen-api -- the llm_proxy routes sit on the root router, which the generated client does not cover.
+ return await api.create<PlaygroundDecisionResult>(
+ '/api/llm_proxy/decision/',
+ {
+ provider_key_id: providerKeyId,
+ state: resolve(prompt.decisionInput),
+ questions: toDecisionQuestionsPayload(prompt.decisionQuestions, resolve),
+ },
+ { signal }
+ )
+ } catch (err) {
+ const retryAfter =
+ err instanceof ApiError && err.status === 429 ? parseInt(err.headers?.get('Retry-After') ?? '', 10) : NaN
+ throw Number.isFinite(retryAfter) ? new RateLimitError(retryAfter) : err
+ }
+}
+
+/** Two connections can serve the same model id, so the selected key decides which entry the prompt means. */
+export function findPromptModel(prompt: PromptConfig, options: ModelOption[]): ModelOption | undefined {
+ return (
+ options.find((option) => option.id === prompt.model && option.providerKeyId === prompt.selectedProviderKeyId) ??
+ options.find((option) => option.id === prompt.model)
+ )
+}
+
+export function isPromptRunnable(prompt: PromptConfig, isSystemOne: boolean): boolean {
+ return isSystemOne ? !!prompt.decisionInput.trim() : prompt.messages.some(isMessageSendable)
}
// Module-level abort controller so an in-flight run can be cancelled. Using a ref instead of
@@ -171,11 +246,13 @@ export type LLMPlaygroundRunLogicProps = Record<string, never>
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface llmPlaygroundRunLogicValues {
activeProviderKeyId: string | null // llmPlaygroundModelLogic
- effectiveModelOptions: ModelOption[] // llmPlaygroundModelLogic
+ runnableModelOptions: ModelOption[] // llmPlaygroundModelLogic
+ systemOnePromptIds: string[] // llmPlaygroundModelLogic
promptConfigs: PromptConfig[] // llmPlaygroundPromptsLogic
variableValues: Record<string, string> // llmPlaygroundVariablesLogic
providerKeys: LLMProviderKey[] // llmProviderKeysLogic
comparisonItems: ComparisonItem[]
+ hasRunnablePrompts: boolean
rateLimitedUntil: number | null
runAccessDeniedReason: string | null
submitting: boolean
@@ -214,10 +291,18 @@ export interface llmPlaygroundRunLogicActions {
}
}
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface llmPlaygroundRunLogicMeta {
+ __keaTypeGenInternalSelectorTypes: {
+ hasRunnablePrompts: (promptConfigs: PromptConfig[], systemOnePromptIds: string[]) => boolean
+ }
+}
+
export type llmPlaygroundRunLogicType = MakeLogicType<
llmPlaygroundRunLogicValues,
llmPlaygroundRunLogicActions,
- LLMPlaygroundRunLogicProps
+ LLMPlaygroundRunLogicProps,
+ llmPlaygroundRunLogicMeta
>
export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
@@ -229,7 +314,7 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
llmPlaygroundPromptsLogic,
['promptConfigs'],
llmPlaygroundModelLogic,
- ['effectiveModelOptions', 'activeProviderKeyId'],
+ ['runnableModelOptions', 'activeProviderKeyId', 'systemOnePromptIds'],
llmProviderKeysLogic,
['providerKeys'],
llmPlaygroundVariablesLogic,
@@ -284,6 +369,11 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
}),
selectors({
+ hasRunnablePrompts: [
+ (s) => [s.promptConfigs, s.systemOnePromptIds],
+ (promptConfigs: PromptConfig[], systemOnePromptIds: string[]): boolean =>
+ promptConfigs.some((prompt) => isPromptRunnable(prompt, systemOnePromptIds.includes(prompt.id))),
+ ],
// Resource-level only, so this comes from the server-rendered app context rather than the API.
// A selector keeps it readable from the scene and overridable in tests.
runAccessDeniedReason: [
@@ -324,10 +414,12 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
index,
messagesToSend: prompt.messages.filter(isMessageSendable),
}))
- .filter((item: { messagesToSend: Message[] }) => item.messagesToSend.length > 0)
+ .filter(({ prompt }: { prompt: PromptConfig }) =>
+ isPromptRunnable(prompt, values.systemOnePromptIds.includes(prompt.id))
+ )
if (runnablePrompts.length === 0) {
- lemonToast.error('Please add some messages before running prompts')
+ lemonToast.error('Add some messages, or the input for a System One model, before running prompts')
actions.finishSubmitPrompt()
return
}
@@ -335,11 +427,9 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
// Count only the prompts actually submitted; a skipped panel's variables are not sent,
// so they belong in neither the warning nor the event.
const runVariables = extractVariablesFromTexts(
- runnablePrompts.flatMap(({ prompt, messagesToSend }) => [
- prompt.systemPrompt,
- ...messagesToSend.map((m) => m.content),
- ...messagesToSend.flatMap((m) => m.toolCalls?.map((toolCall) => toolCall.arguments) ?? []),
- ])
+ runnablePrompts.flatMap(({ prompt }) =>
+ promptTemplateTexts(prompt, values.systemOnePromptIds.includes(prompt.id))
+ )
)
const runUnfilledVariables = runVariables.filter((name) => !getVariableValue(values.variableValues, name))
@@ -398,6 +488,7 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
let toolCalls: AggregatedToolCall[] = []
let providerKeyId: string | null = null
let selectedModelProvider = ''
+ let decision: ComparisonItem['decision'] = null
let itemAdded = false
let reportedStreamError: unknown = undefined
@@ -419,6 +510,7 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
usage: responseUsage,
ttftMs,
latencyMs,
+ decision,
}
if (!itemAdded) {
actions.addToComparison(payload)
@@ -459,9 +551,7 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
try {
startTime = performance.now()
- const selectedModel = (values.effectiveModelOptions as ModelOption[]).find(
- (m) => m.id === prompt.model
- )
+ const selectedModel = findPromptModel(prompt, values.runnableModelOptions)
if (!selectedModel?.provider) {
// Reachable without the model ever being picked here: opening a trace in
// the playground, or loading a saved prompt, can carry a model the current
@@ -490,96 +580,120 @@ export const llmPlaygroundRunLogic = kea<llmPlaygroundRunLogicType>([
}
providerKeyId =
- resolveProviderKeyForPrompt(prompt, values.effectiveModelOptions, values.providerKeys)
- ?.id ?? values.activeProviderKeyId
+ resolveProviderKeyForPrompt(prompt, values.runnableModelOptions, values.providerKeys)?.id ??
+ values.activeProviderKeyId
selectedModelProvider = normalizedProvider
- const requestData: Record<string, unknown> = {
- system: resolvedSystemPrompt,
- messages: toProviderMessages(resolvedMessages),
- model: selectedModel.id,
- provider: selectedModelProvider,
- thinking: prompt.thinking,
- ...(providerKeyId ? { provider_key_id: providerKeyId } : {}),
- ...(prompt.tools ? { tools: prompt.tools } : {}),
- ...(prompt.responseFormat ? { response_format: prompt.responseFormat } : {}),
- ...(prompt.maxTokens !== null && prompt.maxTokens > 0
- ? { max_tokens: prompt.maxTokens }
- : {}),
- ...(typeof prompt.temperature === 'number' ? { temperature: prompt.temperature } : {}),
- ...(typeof prompt.topP === 'number' ? { top_p: prompt.topP } : {}),
- ...(prompt.reasoningLevel ? { reasoning_level: prompt.reasoningLevel } : {}),
- }
-
- // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. No generated function covers this endpoint yet. Find out why the generated client skips it (no schema, no product tag, or excluded from the spec) and fix that first.
- await api.stream('/api/llm_proxy/completion', {
- method: 'POST',
- data: requestData,
- headers: { 'Content-Type': 'application/json' },
- signal: abortController.signal,
- onMessage: (event) => {
- if (abortController.signal.aborted || !event.data) {
- return
+ if (normalizedProvider === 'system_one') {
+ const questionsError = decisionQuestionsDisabledReason(prompt.decisionQuestions)
+ if (questionsError) {
+ lemonToast.error(questionsError)
+ responseText = `**Error:** ${questionsError}`
+ responseHasError = true
+ upsertLiveItem()
+ return
+ }
+ const askedQuestions = prompt.decisionQuestions
+ try {
+ const result = await requestDecision(
+ prompt,
+ providerKeyId,
+ values.variableValues,
+ abortController.signal
+ )
+ decision = { questions: askedQuestions, result }
+ responseUsage = {
+ prompt_tokens: result.input_tokens,
+ completion_tokens: result.output_tokens,
}
+ upsertLiveItem()
+ } catch (err) {
+ if (!isAbandoned() && !abortController.signal.aborted) {
+ reportedStreamError = err
+ reportStreamError(err)
+ }
+ }
+ } else {
+ // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. No generated function covers this endpoint yet. Find out why the generated client skips it (no schema, no product tag, or excluded from the spec) and fix that first.
+ await api.stream('/api/llm_proxy/completion', {
+ method: 'POST',
+ data: buildCompletionRequest(
+ prompt,
+ selectedModel.id,
+ selectedModelProvider,
+ providerKeyId,
+ resolvedSystemPrompt,
+ resolvedMessages
+ ),
+ headers: { 'Content-Type': 'application/json' },
+ signal: abortController.signal,
+ onMessage: (event) => {
+ if (abortController.signal.aborted || !event.data) {
+ return
+ }
- try {
- const data = JSON.parse(event.data)
- if (data.type === 'text') {
- if (firstTokenTime === null && startTime !== null) {
- firstTokenTime = performance.now()
- ttftMs = firstTokenTime - startTime
- }
- responseText += data.text
- upsertLiveItem()
- } else if (data.type === 'tool_call') {
- if (firstTokenTime === null && startTime !== null) {
- firstTokenTime = performance.now()
- ttftMs = firstTokenTime - startTime
+ try {
+ const data = JSON.parse(event.data)
+ if (data.type === 'text') {
+ if (firstTokenTime === null && startTime !== null) {
+ firstTokenTime = performance.now()
+ ttftMs = firstTokenTime - startTime
+ }
+ responseText += data.text
+ upsertLiveItem()
+ } else if (data.type === 'tool_call') {
+ if (firstTokenTime === null && startTime !== null) {
+ firstTokenTime = performance.now()
+ ttftMs = firstTokenTime - startTime
+ }
+ toolCalls = appendToolCallChunk(toolCalls, data)
+ actions.updateComparisonItem(liveItemId, {
+ toolCalls,
+ ttftMs,
+ })
+ } else if (data.type === 'reasoning') {
+ const reasoningChunk =
+ typeof data.reasoning === 'string'
+ ? data.reasoning
+ : typeof data.text === 'string'
+ ? data.text
+ : ''
+ responseReasoning += reasoningChunk
+ actions.updateComparisonItem(liveItemId, {
+ reasoning: responseReasoning,
+ ttftMs,
+ })
+ } else if (data.type === 'usage') {
+ responseUsage = mergeUsage(
+ responseUsage,
+ normalizeUsageFromStreamChunk(data)
+ )
+ actions.updateComparisonItem(liveItemId, { usage: responseUsage })
+ } else if (data.error) {
+ responseText += `\n\n**LLM Error:** ${data.error}`
+ responseHasError = true
+ upsertLiveItem()
}
- toolCalls = appendToolCallChunk(toolCalls, data)
- actions.updateComparisonItem(liveItemId, {
- toolCalls,
- ttftMs,
- })
- } else if (data.type === 'reasoning') {
- const reasoningChunk =
- typeof data.reasoning === 'string'
- ? data.reasoning
- : typeof data.text === 'string'
- ? data.text
- : ''
- responseReasoning += reasoningChunk
- actions.updateComparisonItem(liveItemId, {
- reasoning: responseReasoning,
- ttftMs,
- })
- } else if (data.type === 'usage') {
- responseUsage = mergeUsage(responseUsage, normalizeUsageFromStreamChunk(data))
- actions.updateComparisonItem(liveItemId, { usage: responseUsage })
- } else if (data.error) {
- responseText += `\n\n**LLM Error:** ${data.error}`
+ } catch (e) {
+ console.error('Error parsing stream message:', e, 'Data:', event.data)
+ responseText += `\n\n**Stream Error:** Could not parse response chunk.`
responseHasError = true
upsertLiveItem()
}
- } catch (e) {
- console.error('Error parsing stream message:', e, 'Data:', event.data)
- responseText += `\n\n**Stream Error:** Could not parse response chunk.`
- responseHasError = true
- upsertLiveItem()
- }
- },
- onError: (err) => {
- // api.stream can pass the same error to onError twice: once from its onopen
- // handler, and again when fetch-event-source catches the rethrow below.
- if (!abortController.signal.aborted && err !== reportedStreamError) {
- reportedStreamError = err
- reportStreamError(err)
- }
- // fetch-event-source sends the request again whenever onerror returns, which
- // would charge the user for a duplicate completion. Throwing ends the stream.
- throw err
- },
- })
+ },
+ onError: (err) => {
+ // api.stream can pass the same error to onError twice: once from its onopen
+ // handler, and again when fetch-event-source catches the rethrow below.
+ if (!abortController.signal.aborted && err !== reportedStreamError) {
+ reportedStreamError = err
+ reportStreamError(err)
+ }
+ // fetch-event-source sends the request again whenever onerror returns, which
+ // would charge the user for a duplicate completion. Throwing ends the stream.
+ throw err
+ },
+ })
+ }
globalSetupLogic.findMounted()?.actions.markTaskAsCompleted(SetupTaskId.RunAiPlayground)
} catch (error) {
diff --git a/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.test.ts b/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.test.ts
index d40ef53c2019..8c0ec9887624 100644
--- a/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.test.ts
+++ b/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.test.ts
@@ -1,5 +1,6 @@
import { expectLogic } from 'kea-test-utils'
+import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
import { createPromptConfig, llmPlaygroundPromptsLogic } from './llmPlaygroundPromptsLogic'
@@ -8,6 +9,38 @@ import { llmPlaygroundVariablesLogic } from './llmPlaygroundVariablesLogic'
describe('llmPlaygroundVariablesLogic', () => {
beforeEach(() => {
initKeaTests()
+ useMocks({
+ get: {
+ '/api/environments/:team_id/llm_analytics/provider_keys/': {
+ results: [{ id: 'key-s1', provider: 'system_one', name: 'System One', state: 'ok' }],
+ },
+ '/api/environments/:team_id/llm_analytics/evaluation_config/': { active_provider_key: null },
+ '/api/llm_proxy/models/': ({ request }) => [
+ 200,
+ new URL(request.url).searchParams.get('provider_key_id')
+ ? [{ id: 'jev-1', name: 'jev-1', provider: 'System One', supports_chat: false }]
+ : [{ id: 'gpt-5-mini', name: 'GPT-5 Mini', provider: 'OpenAI', description: '' }],
+ ],
+ },
+ })
+ })
+
+ it('detects only the texts a System One panel sends', async () => {
+ const logic = llmPlaygroundVariablesLogic()
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+
+ llmPlaygroundPromptsLogic.actions.setMessages([{ role: 'user', content: 'Hidden {{message_only}}' }])
+ llmPlaygroundPromptsLogic.actions.setModel('jev-1', 'key-s1')
+ llmPlaygroundPromptsLogic.actions.setDecisionInput('About {{topic}}')
+ llmPlaygroundPromptsLogic.actions.setDecisionQuestions([
+ { id: 'q1', type: 'noul', instructions: 'Is {{tone}} kind?', options: [] },
+ ])
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(logic.values.detectedVariables).toEqual(['topic', 'tone'])
+
+ logic.unmount()
})
it('detects variables across all comparison panels and shares values by name', async () => {
diff --git a/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.ts b/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.ts
index b760b931573a..68714924a607 100644
--- a/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.ts
+++ b/products/ai_observability/frontend/playground/llmPlaygroundVariablesLogic.ts
@@ -1,12 +1,14 @@
import { MakeLogicType, actions, connect, kea, path, props, reducers, selectors } from 'kea'
+import { llmPlaygroundModelLogic } from './llmPlaygroundModelLogic'
import { llmPlaygroundPromptsLogic, type PromptConfig } from './llmPlaygroundPromptsLogic'
-import { extractVariablesFromTexts, getVariableValue } from './playgroundTemplating'
+import { extractVariablesFromTexts, getVariableValue, promptTemplateTexts } from './playgroundTemplating'
export type LLMPlaygroundVariablesLogicProps = Record<string, never>
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface llmPlaygroundVariablesLogicValues {
+ systemOnePromptIds: string[] // llmPlaygroundModelLogic
promptConfigs: PromptConfig[] // llmPlaygroundPromptsLogic
detectedVariables: string[]
unfilledVariables: string[]
@@ -33,7 +35,7 @@ export interface llmPlaygroundVariablesLogicActions {
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface llmPlaygroundVariablesLogicMeta {
__keaTypeGenInternalSelectorTypes: {
- detectedVariables: (promptConfigs: PromptConfig[]) => string[]
+ detectedVariables: (promptConfigs: PromptConfig[], systemOnePromptIds: string[]) => string[]
unfilledVariables: (detectedVariables: string[], variableValues: Record<string, string>) => string[]
}
}
@@ -50,7 +52,7 @@ export const llmPlaygroundVariablesLogic = kea<llmPlaygroundVariablesLogicType>(
props({} as LLMPlaygroundVariablesLogicProps),
connect(() => ({
- values: [llmPlaygroundPromptsLogic, ['promptConfigs']],
+ values: [llmPlaygroundPromptsLogic, ['promptConfigs'], llmPlaygroundModelLogic, ['systemOnePromptIds']],
actions: [llmPlaygroundPromptsLogic, ['resetPlayground', 'setupPlaygroundFromEvent']],
})),
@@ -81,10 +83,12 @@ export const llmPlaygroundVariablesLogic = kea<llmPlaygroundVariablesLogicType>(
selectors({
detectedVariables: [
- (s) => [s.promptConfigs],
- (promptConfigs: PromptConfig[]): string[] =>
+ (s) => [s.promptConfigs, s.systemOnePromptIds],
+ (promptConfigs: PromptConfig[], systemOnePromptIds: string[]): string[] =>
extractVariablesFromTexts(
- promptConfigs.flatMap((prompt) => [prompt.systemPrompt, ...prompt.messages.map((m) => m.content)])
+ promptConfigs.flatMap((prompt) =>
+ promptTemplateTexts(prompt, systemOnePromptIds.includes(prompt.id))
+ )
),
],
unfilledVariables: [
diff --git a/products/ai_observability/frontend/playground/playgroundDecisions.test.ts b/products/ai_observability/frontend/playground/playgroundDecisions.test.ts
new file mode 100644
index 000000000000..def18d07a843
--- /dev/null
+++ b/products/ai_observability/frontend/playground/playgroundDecisions.test.ts
@@ -0,0 +1,88 @@
+import {
+ type PlaygroundQuestion,
+ decisionQuestionsDisabledReason,
+ messagesToDecisionInput,
+ toDecisionQuestionsPayload,
+} from './playgroundDecisions'
+
+function question(partial: Partial<PlaygroundQuestion>): PlaygroundQuestion {
+ return { id: 'q', type: 'noul', instructions: 'Is it urgent?', options: [], ...partial }
+}
+
+function options(...names: string[]): PlaygroundQuestion['options'] {
+ return names.map((name, index) => ({ id: `o${index}`, name, description: '' }))
+}
+
+describe('playgroundDecisions', () => {
+ it.each([
+ ['no questions', [], 'Add a question to ask the model'],
+ ['a blank question', [question({ instructions: ' ' })], 'Write every question before running'],
+ ['an unnamed option', [question({ type: 'choice', options: options('billing', ' ') })], 'Name every option'],
+ [
+ 'duplicate option names',
+ [question({ type: 'choice', options: options('billing', 'billing ') })],
+ 'Give each option of a question a different name',
+ ],
+ [
+ 'a one-level scale',
+ [question({ type: 'score', options: options('calm') })],
+ 'A rating question needs 2 to 10 levels',
+ ],
+ ['a yes/no question with hidden blank options', [question({ options: options('', '') })], null],
+ ['valid questions', [question({}), question({ type: 'score', options: options('calm', 'angry') })], null],
+ ])('blocks a run with %s', (_name, questions, reason) => {
+ expect(decisionQuestionsDisabledReason(questions)).toBe(reason)
+ })
+
+ it('keeps tool calls and tool results when a conversation becomes input', () => {
+ expect(
+ messagesToDecisionInput([
+ { role: 'user', content: 'Weather in Paris?' },
+ {
+ role: 'assistant',
+ content: '',
+ toolCalls: [{ id: 'call_1', name: 'weather', arguments: '{"city":"Paris"}' }],
+ },
+ { role: 'tool', content: 'Sunny', toolCallId: 'call_1' },
+ { role: 'assistant', content: ' ' },
+ ])
+ ).toBe(
+ 'user: Weather in Paris?\n\nassistant: tool call call_1: weather({"city":"Paris"})\n\ntool (result of call_1): Sunny'
+ )
+ })
+
+ it('keys questions by id and sends option descriptions only for multiple choice', () => {
+ const payload = toDecisionQuestionsPayload(
+ [
+ question({ id: 'urgent', instructions: ' Is {{x}} urgent? ' }),
+ question({
+ id: 'topic',
+ type: 'choice',
+ options: [
+ { id: 'a', name: ' billing ', description: 'invoices' },
+ { id: 'b', name: 'bugs', description: ' ' },
+ ],
+ }),
+ question({
+ id: 'tone',
+ type: 'score',
+ options: [
+ { id: 'c', name: 'calm', description: 'ignored' },
+ { id: 'd', name: 'angry', description: '' },
+ ],
+ }),
+ ],
+ (text) => text.replace('{{x}}', 'this')
+ )
+
+ expect(payload).toEqual({
+ urgent: { type: 'noul', instructions: 'Is this urgent?' },
+ topic: {
+ type: 'choice',
+ instructions: 'Is it urgent?',
+ options: [{ name: 'billing', description: 'invoices' }, { name: 'bugs' }],
+ },
+ tone: { type: 'score', instructions: 'Is it urgent?', options: [{ name: 'calm' }, { name: 'angry' }] },
+ })
+ })
+})
diff --git a/products/ai_observability/frontend/playground/playgroundDecisions.ts b/products/ai_observability/frontend/playground/playgroundDecisions.ts
new file mode 100644
index 000000000000..afbebd246610
--- /dev/null
+++ b/products/ai_observability/frontend/playground/playgroundDecisions.ts
@@ -0,0 +1,138 @@
+import { uuid } from 'lib/utils/dom'
+
+import type { Message } from './llmPlaygroundPromptsLogic'
+
+// The API caps mirror `MAX_DECISION_QUESTIONS` and `MAX_SCORE_LEVELS` in the proxy serializer.
+export const MAX_DECISION_QUESTIONS = 20
+export const MAX_SCORE_LEVELS = 10
+
+export type PlaygroundQuestionType = 'noul' | 'choice' | 'score'
+
+export const MIN_OPTIONS: Record<PlaygroundQuestionType, number> = { noul: 0, choice: 1, score: 2 }
+
+export interface PlaygroundQuestionOption {
+ id: string
+ name: string
+ description: string
+}
+
+export interface PlaygroundQuestion {
+ id: string
+ type: PlaygroundQuestionType
+ instructions: string
+ options: PlaygroundQuestionOption[]
+}
+
+export interface PlaygroundDecisionAnswer {
+ type: PlaygroundQuestionType | 'refusal'
+ probability: number | null
+ choice: string | null
+ score: number | null
+ confidence: number | null
+ probabilities: Record<string, number> | null
+}
+
+export interface PlaygroundDecisionResult {
+ model: string
+ answers: Record<string, PlaygroundDecisionAnswer>
+ input_tokens: number | null
+ output_tokens: number | null
+}
+
+export interface DecisionQuestionPayload {
+ type: PlaygroundQuestionType
+ instructions: string
+ options?: { name: string; description?: string }[]
+}
+
+/** Turns a conversation into System One input text, so a trace opened in the playground carries over. */
+export function messagesToDecisionInput(messages: Message[]): string {
+ return messages
+ .map((message) => {
+ const role = message.toolCallId ? `${message.role} (result of ${message.toolCallId})` : message.role
+ const lines = [
+ ...(message.content.trim() ? [message.content.trim()] : []),
+ ...(message.toolCalls ?? []).map(
+ (toolCall) => `tool call ${toolCall.id}: ${toolCall.name}(${toolCall.arguments})`
+ ),
+ ]
+ return lines.length > 0 ? `${role}: ${lines.join('\n')}` : null
+ })
+ .filter((block): block is string => block !== null)
+ .join('\n\n')
+}
+
+export function createPlaygroundQuestionOption(): PlaygroundQuestionOption {
+ return { id: uuid(), name: '', description: '' }
+}
+
+export function createPlaygroundQuestion(type: PlaygroundQuestionType = 'noul'): PlaygroundQuestion {
+ return {
+ id: uuid(),
+ type,
+ instructions: '',
+ options: Array.from({ length: MIN_OPTIONS[type] }, createPlaygroundQuestionOption),
+ }
+}
+
+function optionsDisabledReason(question: PlaygroundQuestion): string | null {
+ const names = question.options.map((option) => option.name.trim())
+ if (names.some((name) => !name)) {
+ return question.type === 'choice' ? 'Name every option' : 'Name every level of the scale'
+ }
+ if (new Set(names).size !== names.length) {
+ return 'Give each option of a question a different name'
+ }
+ if (question.type === 'choice' && names.length < MIN_OPTIONS.choice) {
+ return 'A multiple choice question needs at least one option'
+ }
+ if (question.type === 'score' && (names.length < MIN_OPTIONS.score || names.length > MAX_SCORE_LEVELS)) {
+ return `A rating question needs 2 to ${MAX_SCORE_LEVELS} levels`
+ }
+ return null
+}
+
+export function decisionQuestionsDisabledReason(questions: PlaygroundQuestion[]): string | null {
+ if (questions.length === 0) {
+ return 'Add a question to ask the model'
+ }
+ if (questions.length > MAX_DECISION_QUESTIONS) {
+ return `Ask at most ${MAX_DECISION_QUESTIONS} questions at once`
+ }
+ for (const question of questions) {
+ if (!question.instructions.trim()) {
+ return 'Write every question before running'
+ }
+ // A yes/no question sends no options, so options left over from another type do not matter.
+ const reason = question.type === 'noul' ? null : optionsDisabledReason(question)
+ if (reason) {
+ return reason
+ }
+ }
+ return null
+}
+
+export function toDecisionQuestionsPayload(
+ questions: PlaygroundQuestion[],
+ resolve: (text: string) => string
+): Record<string, DecisionQuestionPayload> {
+ return Object.fromEntries(
+ questions.map((question) => [
+ question.id,
+ {
+ type: question.type,
+ instructions: resolve(question.instructions.trim()),
+ ...(question.type === 'noul'
+ ? {}
+ : {
+ options: question.options.map((option) => ({
+ name: option.name.trim(),
+ ...(question.type === 'choice' && option.description.trim()
+ ? { description: option.description.trim() }
+ : {}),
+ })),
+ }),
+ },
+ ])
+ )
+}
diff --git a/products/ai_observability/frontend/playground/playgroundTemplating.ts b/products/ai_observability/frontend/playground/playgroundTemplating.ts
index 3354cf6026ac..5e342da0bf39 100644
--- a/products/ai_observability/frontend/playground/playgroundTemplating.ts
+++ b/products/ai_observability/frontend/playground/playgroundTemplating.ts
@@ -1,3 +1,6 @@
+import type { PromptConfig } from './llmPlaygroundPromptsLogic'
+import { isMessageSendable } from './playgroundMessageMapping'
+
/**
* Template variable syntax shared with the SDK `compile` helpers
* (posthog-js `packages/ai/src/prompts.ts`, posthog-python `posthog/ai/prompts.py`).
@@ -78,3 +81,16 @@ export function substituteVariables(text: string, values: Record<string, string>
// Function replacer so values containing `$&`-style patterns substitute literally
return text.replace(TEMPLATE_VARIABLE_REGEX, (match, name: string) => getVariableValue(values, name) || match)
}
+
+/** The texts a run sends for a panel. A System One panel sends its input and questions, a chat panel its prompt and messages. */
+export function promptTemplateTexts(prompt: PromptConfig, isSystemOne: boolean): string[] {
+ if (isSystemOne) {
+ return [prompt.decisionInput, ...prompt.decisionQuestions.map((question) => question.instructions)]
+ }
+ const messages = prompt.messages.filter(isMessageSendable)
+ return [
+ prompt.systemPrompt,
+ ...messages.map((message) => message.content),
+ ...messages.flatMap((message) => message.toolCalls?.map((toolCall) => toolCall.arguments) ?? []),
+ ]
+}