Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): retry invalid research follow-up replies (#125210)

Mikayla Thompsonedf688c9e528
diff --git a/docs/internal/signals-pr-lifecycle.md b/docs/internal/signals-pr-lifecycle.md
index 25f879ff565d..85e039beb5ce 100644
--- a/docs/internal/signals-pr-lifecycle.md
+++ b/docs/internal/signals-pr-lifecycle.md
@@ -69,6 +69,9 @@ Qualitative findings can omit metrics.
Research presentation drops invalid observation metrics individually and keeps the report prose and valid metrics.
A snapshot series without a paired value and measurement time is cleared while its live query remains available.
+Each structured research follow-up gets one JSON correction turn in the same session if its reply cannot be parsed or fails schema validation.
+The correction uses the same schema and validation context. A second invalid reply propagates to the caller's existing failure handling; execution failures and cancellation propagate without a JSON retry.
+
Observation metrics support HogQL filters that compare direct event fields or event properties with literal values.
Boolean operations, arithmetic, and literal lists are supported; other function calls and subqueries remain unreadable.
Report reads, research selection, and snapshot refresh use the same filter check.
diff --git a/products/signals/backend/report_generation/research.py b/products/signals/backend/report_generation/research.py
index e26fa5fc9d60..75c54688f596 100644
--- a/products/signals/backend/report_generation/research.py
+++ b/products/signals/backend/report_generation/research.py
@@ -1883,6 +1883,7 @@ async def run_multi_turn_research(
followup_prompt,
finding_model,
label=f"signal_{i}_of_{total}",
+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,
)
finding, is_new = _resolve_finding_response(response, previous_finding, signal.signal_id)
(new_artefacts if is_new else old_artefacts).append(finding)
@@ -1908,6 +1909,7 @@ async def run_multi_turn_research(
actionability_prompt,
actionability_schema,
label="actionability",
+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,
)
actionability_result, actionability_is_new = _resolve_actionability_response(
actionability_response, previous_actionability
@@ -1934,6 +1936,7 @@ async def run_multi_turn_research(
priority_prompt,
priority_schema,
label="priority",
+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,
)
priority_result, priority_is_new = _resolve_priority_response(priority_response, previous_priority)
(new_artefacts if priority_is_new else old_artefacts).append(priority_result)
@@ -1960,6 +1963,7 @@ async def run_multi_turn_research(
presentation_prompt,
ReportPresentationOutput,
label="presentation",
+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,
)
if output_fn:
output_fn(f"Report title: {presentation_result.title}")
@@ -1986,6 +1990,7 @@ async def run_multi_turn_research(
verification_prompt,
FixVerificationOutput,
label="fix_verification",
+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,
validation_context={
"report_id": signal_report_id,
"team_id": context.team_id,
@@ -2035,6 +2040,7 @@ async def run_multi_turn_research(
build_supersede_prompt(own_pr_url, previous_report_research.summary),
ImplementationAssessment,
label="supersede",
+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,
)
candidates = {target.pr_url: target for target in implementation_context.candidates}
selected = list(dict.fromkeys(assessment.obsolete_pr_urls))
diff --git a/products/signals/backend/test/test_agentic_report_activity.py b/products/signals/backend/test/test_agentic_report_activity.py
index 9987cdff6b4b..2ee77dace29a 100644
--- a/products/signals/backend/test/test_agentic_report_activity.py
+++ b/products/signals/backend/test/test_agentic_report_activity.py
@@ -45,6 +45,7 @@ from products.signals.backend.models import (
from products.signals.backend.repo_corrections import SCOUT_REPOSITORY_REASON
from products.signals.backend.report_charts import ReportChart
from products.signals.backend.report_generation.research import (
+ RESEARCH_JSON_RETRY_PROMPT,
ActionabilityAssessment,
ActionabilityChoice,
ActionabilityUpdate,
@@ -2045,7 +2046,8 @@ async def test_run_multi_turn_research_survives_a_failed_supersede_turn(supersed
"supersede": supersede_outcome,
}
- async def fake_send_followup(message, model, *, label="", validation_context=None):
+ async def fake_send_followup(message, model, *, label="", validation_context=None, json_retry_prompt=None):
+ assert json_retry_prompt == RESEARCH_JSON_RETRY_PROMPT
outcome = by_label[label]
if isinstance(outcome, Exception):
raise outcome
@@ -2139,7 +2141,8 @@ async def test_run_multi_turn_research_only_asks_about_the_pr_when_actionable(ac
}
asked_labels: list[str] = []
- async def fake_send_followup(message, model, *, label="", validation_context=None):
+ async def fake_send_followup(message, model, *, label="", validation_context=None, json_retry_prompt=None):
+ assert json_retry_prompt == RESEARCH_JSON_RETRY_PROMPT
asked_labels.append(label)
return by_label[label]
diff --git a/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py b/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py
index 975009385b21..d89dc4a6b489 100644
--- a/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py
+++ b/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py
@@ -348,13 +348,27 @@ class MultiTurnSession:
*,
label: str = "",
validation_context: dict[str, object] | None = None,
+ json_retry_prompt: str | None = None,
) -> _ModelT:
- """Send a follow-up message and wait for the agent's next structured response."""
+ """Send a structured follow-up, optionally asking once for corrected JSON on parse or validation failure."""
last_message = await self.send_followup_raw(message, label=label)
- parsed = self._parse_and_validate(
- last_message, model, label=label or "followup", validation_context=validation_context
- )
- return parsed
+ try:
+ return self._parse_and_validate(
+ last_message, model, label=label or "followup", validation_context=validation_context
+ )
+ except ValueError as error:
+ if json_retry_prompt is None:
+ raise
+ logger.warning(
+ "multi_turn: followup did not validate against %s for run=%s label=%s, asking for JSON once (%s)",
+ model.__name__,
+ self.task_run.id,
+ label,
+ type(error).__name__,
+ )
+ retry_label = f"{label or 'followup'} json retry"
+ retry_message = await self.send_followup_raw(json_retry_prompt, label=retry_label)
+ return self._parse_and_validate(retry_message, model, label=retry_label, validation_context=validation_context)
async def send_followup_raw(
self,
diff --git a/products/tasks/backend/tests/test_multi_turn_session.py b/products/tasks/backend/tests/test_multi_turn_session.py
index c8642e6a4be4..11fb59c5e845 100644
--- a/products/tasks/backend/tests/test_multi_turn_session.py
+++ b/products/tasks/backend/tests/test_multi_turn_session.py
@@ -1,5 +1,6 @@
import json
import asyncio
+import logging
from pathlib import Path
from types import SimpleNamespace
from typing import cast
@@ -12,7 +13,7 @@ from django.db import OperationalError, transaction
from asgiref.sync import sync_to_async
from parameterized import parameterized
-from pydantic import BaseModel, Field, JsonValue
+from pydantic import BaseModel, Field, JsonValue, ValidationInfo, field_validator
from temporalio.service import RPCError, RPCStatusCode
from posthog.models import Integration, Organization, Team
@@ -68,6 +69,15 @@ class _DefaultedResp(BaseModel):
values: list[str] = Field(default_factory=list)
+class _ContextResp(_Resp):
+ @field_validator("value")
+ @classmethod
+ def value_matches_context(cls, value: str, info: ValidationInfo) -> str:
+ if info.context is None or value != info.context["expected_value"]:
+ raise ValueError("value must match the validation context")
+ return value
+
+
class TestPollForTurnEmptyEndTurn:
@pytest.mark.asyncio
async def test_raises_empty_agent_turn_error_with_offsets(self):
@@ -1366,9 +1376,74 @@ class TestMultiTurnSessionRetry:
)
return session
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("first_reply", ["private-reply-text", '{"value":', '{"value": "private-value"}'])
+ @pytest.mark.parametrize("retry_reply", [None, '{"value": "ok"}', '{"value": "still-invalid"}'])
+ async def test_json_retry_preserves_schema_and_context(
+ self, first_reply: str, retry_reply: str | None, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ session = self._make_session()
+ replies = [first_reply] + ([retry_reply] if retry_reply is not None else [])
+ poll = AsyncMock(
+ side_effect=[
+ TurnPollResult(last_message=reply, full_log=None, total_lines=index * 10, printed_lines=index * 5)
+ for index, reply in enumerate(replies, start=1)
+ ]
+ )
+ with (
+ caplog.at_level(logging.WARNING),
+ patch("products.tasks.backend.logic.services.custom_prompt_multi_turn_runner.poll_for_turn", new=poll),
+ ):
+ validation_context: dict[str, object] = {"expected_value": "ok"}
+ if retry_reply == '{"value": "ok"}':
+ result = await session.send_followup(
+ "question",
+ _ContextResp,
+ json_retry_prompt="Return valid JSON",
+ validation_context=validation_context,
+ )
+ assert result.value == "ok"
+ else:
+ with pytest.raises(ValueError):
+ await session.send_followup(
+ "question",
+ _ContextResp,
+ json_retry_prompt="Return valid JSON" if retry_reply is not None else None,
+ validation_context=validation_context,
+ )
+
+ assert poll.await_count == len(replies)
+ if retry_reply is not None:
+ assert poll.await_args_list[1].kwargs["skip_lines"] == 10
+ assert cast(AsyncMock, session._workflow_handle).signal.await_args_list[1].args[1] == "Return valid JSON"
+ assert "asking for JSON once" in caplog.text
+ assert "private-reply-text" not in caplog.text
+ assert "private-value" not in caplog.text
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("during_retry", [False, True])
+ @pytest.mark.parametrize(
+ "failure", [RuntimeError("poll failed"), ValueError("poll failed"), asyncio.CancelledError()]
+ )
+ async def test_json_retry_propagates_execution_failures(self, during_retry: bool, failure: BaseException) -> None:
+ session = self._make_session()
+ responses = (
+ [TurnPollResult(last_message="prose", full_log=None, total_lines=10, printed_lines=5), failure]
+ if during_retry
+ else [failure]
+ )
+ poll = AsyncMock(side_effect=responses)
+ with patch("products.tasks.backend.logic.services.custom_prompt_multi_turn_runner.poll_for_turn", new=poll):
+ with pytest.raises(type(failure)) as raised:
+ await session.send_followup("question", _Resp, json_retry_prompt="Return valid JSON")
+
+ assert raised.value is failure
+ assert poll.await_count == len(responses)
+
@pytest.mark.asyncio
@pytest.mark.parametrize("deadline,turns", [(None, 1), (float("inf"), 2), (0.0, 1)])
- async def test_happy_path_no_retry(self, deadline: float | None, turns: int) -> None:
+ @pytest.mark.parametrize("json_retry_prompt", [None, "Return valid JSON"])
+ async def test_happy_path_no_retry(self, deadline: float | None, turns: int, json_retry_prompt: str | None) -> None:
session = self._make_session()
session.poll_deadline = deadline
agent_response = json.dumps({"value": "ok"})
@@ -1381,12 +1456,12 @@ class TestMultiTurnSessionRetry:
) as poll:
if deadline == 0.0:
with pytest.raises(TimeoutError):
- await session.send_followup("hello", _Resp, label="unit")
+ await session.send_followup("hello", _Resp, label="unit", json_retry_prompt=json_retry_prompt)
cast(AsyncMock, session._workflow_handle).signal.assert_not_awaited()
poll.assert_not_awaited()
return
for _ in range(turns):
- result = await session.send_followup("hello", _Resp, label="unit")
+ result = await session.send_followup("hello", _Resp, label="unit", json_retry_prompt=json_retry_prompt)
assert result == _Resp(value="ok")
assert all(call.kwargs["poll_deadline"] == deadline for call in poll.await_args_list)