:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): retry invalid research follow-up replies (#125210)
Mikayla Thompsonedf688c9e528
diff --git a/docs/internal/signals-pr-lifecycle.md b/docs/internal/signals-pr-lifecycle.mdindex 25f879ff565d..85e039beb5ce 100644--- a/docs/internal/signals-pr-lifecycle.md+++ b/docs/internal/signals-pr-lifecycle.md@@ -69,6 +69,9 @@ Qualitative findings can omit metrics.Research presentation drops invalid observation metrics individually and keeps the report prose and valid metrics.A snapshot series without a paired value and measurement time is cleared while its live query remains available.+Each structured research follow-up gets one JSON correction turn in the same session if its reply cannot be parsed or fails schema validation.+The correction uses the same schema and validation context. A second invalid reply propagates to the caller's existing failure handling; execution failures and cancellation propagate without a JSON retry.+Observation metrics support HogQL filters that compare direct event fields or event properties with literal values.Boolean operations, arithmetic, and literal lists are supported; other function calls and subqueries remain unreadable.Report reads, research selection, and snapshot refresh use the same filter check.diff --git a/products/signals/backend/report_generation/research.py b/products/signals/backend/report_generation/research.pyindex e26fa5fc9d60..75c54688f596 100644--- a/products/signals/backend/report_generation/research.py+++ b/products/signals/backend/report_generation/research.py@@ -1883,6 +1883,7 @@ async def run_multi_turn_research(followup_prompt,finding_model,label=f"signal_{i}_of_{total}",+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,)finding, is_new = _resolve_finding_response(response, previous_finding, signal.signal_id)(new_artefacts if is_new else old_artefacts).append(finding)@@ -1908,6 +1909,7 @@ async def run_multi_turn_research(actionability_prompt,actionability_schema,label="actionability",+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,)actionability_result, actionability_is_new = _resolve_actionability_response(actionability_response, previous_actionability@@ -1934,6 +1936,7 @@ async def run_multi_turn_research(priority_prompt,priority_schema,label="priority",+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,)priority_result, priority_is_new = _resolve_priority_response(priority_response, previous_priority)(new_artefacts if priority_is_new else old_artefacts).append(priority_result)@@ -1960,6 +1963,7 @@ async def run_multi_turn_research(presentation_prompt,ReportPresentationOutput,label="presentation",+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,)if output_fn:output_fn(f"Report title: {presentation_result.title}")@@ -1986,6 +1990,7 @@ async def run_multi_turn_research(verification_prompt,FixVerificationOutput,label="fix_verification",+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,validation_context={"report_id": signal_report_id,"team_id": context.team_id,@@ -2035,6 +2040,7 @@ async def run_multi_turn_research(build_supersede_prompt(own_pr_url, previous_report_research.summary),ImplementationAssessment,label="supersede",+ json_retry_prompt=RESEARCH_JSON_RETRY_PROMPT,)candidates = {target.pr_url: target for target in implementation_context.candidates}selected = list(dict.fromkeys(assessment.obsolete_pr_urls))diff --git a/products/signals/backend/test/test_agentic_report_activity.py b/products/signals/backend/test/test_agentic_report_activity.pyindex 9987cdff6b4b..2ee77dace29a 100644--- a/products/signals/backend/test/test_agentic_report_activity.py+++ b/products/signals/backend/test/test_agentic_report_activity.py@@ -45,6 +45,7 @@ from products.signals.backend.models import (from products.signals.backend.repo_corrections import SCOUT_REPOSITORY_REASONfrom products.signals.backend.report_charts import ReportChartfrom products.signals.backend.report_generation.research import (+ RESEARCH_JSON_RETRY_PROMPT,ActionabilityAssessment,ActionabilityChoice,ActionabilityUpdate,@@ -2045,7 +2046,8 @@ async def test_run_multi_turn_research_survives_a_failed_supersede_turn(supersed"supersede": supersede_outcome,}- async def fake_send_followup(message, model, *, label="", validation_context=None):+ async def fake_send_followup(message, model, *, label="", validation_context=None, json_retry_prompt=None):+ assert json_retry_prompt == RESEARCH_JSON_RETRY_PROMPToutcome = by_label[label]if isinstance(outcome, Exception):raise outcome@@ -2139,7 +2141,8 @@ async def test_run_multi_turn_research_only_asks_about_the_pr_when_actionable(ac}asked_labels: list[str] = []- async def fake_send_followup(message, model, *, label="", validation_context=None):+ async def fake_send_followup(message, model, *, label="", validation_context=None, json_retry_prompt=None):+ assert json_retry_prompt == RESEARCH_JSON_RETRY_PROMPTasked_labels.append(label)return by_label[label]diff --git a/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py b/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.pyindex 975009385b21..d89dc4a6b489 100644--- a/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py+++ b/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py@@ -348,13 +348,27 @@ class MultiTurnSession:*,label: str = "",validation_context: dict[str, object] | None = None,+ json_retry_prompt: str | None = None,) -> _ModelT:- """Send a follow-up message and wait for the agent's next structured response."""+ """Send a structured follow-up, optionally asking once for corrected JSON on parse or validation failure."""last_message = await self.send_followup_raw(message, label=label)- parsed = self._parse_and_validate(- last_message, model, label=label or "followup", validation_context=validation_context- )- return parsed+ try:+ return self._parse_and_validate(+ last_message, model, label=label or "followup", validation_context=validation_context+ )+ except ValueError as error:+ if json_retry_prompt is None:+ raise+ logger.warning(+ "multi_turn: followup did not validate against %s for run=%s label=%s, asking for JSON once (%s)",+ model.__name__,+ self.task_run.id,+ label,+ type(error).__name__,+ )+ retry_label = f"{label or 'followup'} json retry"+ retry_message = await self.send_followup_raw(json_retry_prompt, label=retry_label)+ return self._parse_and_validate(retry_message, model, label=retry_label, validation_context=validation_context)async def send_followup_raw(self,diff --git a/products/tasks/backend/tests/test_multi_turn_session.py b/products/tasks/backend/tests/test_multi_turn_session.pyindex c8642e6a4be4..11fb59c5e845 100644--- a/products/tasks/backend/tests/test_multi_turn_session.py+++ b/products/tasks/backend/tests/test_multi_turn_session.py@@ -1,5 +1,6 @@import jsonimport asyncio+import loggingfrom pathlib import Pathfrom types import SimpleNamespacefrom typing import cast@@ -12,7 +13,7 @@ from django.db import OperationalError, transactionfrom asgiref.sync import sync_to_asyncfrom parameterized import parameterized-from pydantic import BaseModel, Field, JsonValue+from pydantic import BaseModel, Field, JsonValue, ValidationInfo, field_validatorfrom temporalio.service import RPCError, RPCStatusCodefrom posthog.models import Integration, Organization, Team@@ -68,6 +69,15 @@ class _DefaultedResp(BaseModel):values: list[str] = Field(default_factory=list)+class _ContextResp(_Resp):+ @field_validator("value")+ @classmethod+ def value_matches_context(cls, value: str, info: ValidationInfo) -> str:+ if info.context is None or value != info.context["expected_value"]:+ raise ValueError("value must match the validation context")+ return value++class TestPollForTurnEmptyEndTurn:@pytest.mark.asyncioasync def test_raises_empty_agent_turn_error_with_offsets(self):@@ -1366,9 +1376,74 @@ class TestMultiTurnSessionRetry:)return session+ @pytest.mark.asyncio+ @pytest.mark.parametrize("first_reply", ["private-reply-text", '{"value":', '{"value": "private-value"}'])+ @pytest.mark.parametrize("retry_reply", [None, '{"value": "ok"}', '{"value": "still-invalid"}'])+ async def test_json_retry_preserves_schema_and_context(+ self, first_reply: str, retry_reply: str | None, caplog: pytest.LogCaptureFixture+ ) -> None:+ session = self._make_session()+ replies = [first_reply] + ([retry_reply] if retry_reply is not None else [])+ poll = AsyncMock(+ side_effect=[+ TurnPollResult(last_message=reply, full_log=None, total_lines=index * 10, printed_lines=index * 5)+ for index, reply in enumerate(replies, start=1)+ ]+ )+ with (+ caplog.at_level(logging.WARNING),+ patch("products.tasks.backend.logic.services.custom_prompt_multi_turn_runner.poll_for_turn", new=poll),+ ):+ validation_context: dict[str, object] = {"expected_value": "ok"}+ if retry_reply == '{"value": "ok"}':+ result = await session.send_followup(+ "question",+ _ContextResp,+ json_retry_prompt="Return valid JSON",+ validation_context=validation_context,+ )+ assert result.value == "ok"+ else:+ with pytest.raises(ValueError):+ await session.send_followup(+ "question",+ _ContextResp,+ json_retry_prompt="Return valid JSON" if retry_reply is not None else None,+ validation_context=validation_context,+ )++ assert poll.await_count == len(replies)+ if retry_reply is not None:+ assert poll.await_args_list[1].kwargs["skip_lines"] == 10+ assert cast(AsyncMock, session._workflow_handle).signal.await_args_list[1].args[1] == "Return valid JSON"+ assert "asking for JSON once" in caplog.text+ assert "private-reply-text" not in caplog.text+ assert "private-value" not in caplog.text++ @pytest.mark.asyncio+ @pytest.mark.parametrize("during_retry", [False, True])+ @pytest.mark.parametrize(+ "failure", [RuntimeError("poll failed"), ValueError("poll failed"), asyncio.CancelledError()]+ )+ async def test_json_retry_propagates_execution_failures(self, during_retry: bool, failure: BaseException) -> None:+ session = self._make_session()+ responses = (+ [TurnPollResult(last_message="prose", full_log=None, total_lines=10, printed_lines=5), failure]+ if during_retry+ else [failure]+ )+ poll = AsyncMock(side_effect=responses)+ with patch("products.tasks.backend.logic.services.custom_prompt_multi_turn_runner.poll_for_turn", new=poll):+ with pytest.raises(type(failure)) as raised:+ await session.send_followup("question", _Resp, json_retry_prompt="Return valid JSON")++ assert raised.value is failure+ assert poll.await_count == len(responses)+@pytest.mark.asyncio@pytest.mark.parametrize("deadline,turns", [(None, 1), (float("inf"), 2), (0.0, 1)])- async def test_happy_path_no_retry(self, deadline: float | None, turns: int) -> None:+ @pytest.mark.parametrize("json_retry_prompt", [None, "Return valid JSON"])+ async def test_happy_path_no_retry(self, deadline: float | None, turns: int, json_retry_prompt: str | None) -> None:session = self._make_session()session.poll_deadline = deadlineagent_response = json.dumps({"value": "ok"})@@ -1381,12 +1456,12 @@ class TestMultiTurnSessionRetry:) as poll:if deadline == 0.0:with pytest.raises(TimeoutError):- await session.send_followup("hello", _Resp, label="unit")+ await session.send_followup("hello", _Resp, label="unit", json_retry_prompt=json_retry_prompt)cast(AsyncMock, session._workflow_handle).signal.assert_not_awaited()poll.assert_not_awaited()returnfor _ in range(turns):- result = await session.send_followup("hello", _Resp, label="unit")+ result = await session.send_followup("hello", _Resp, label="unit", json_retry_prompt=json_retry_prompt)assert result == _Resp(value="ok")assert all(call.kwargs["poll_deadline"] == deadline for call in poll.await_args_list)