Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): ground eval report truncation warnings in full inputs (#122556)

Bernat Torres4ad04bc099ac
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 52e6e3efb816..0894ce0c6b07 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -289,6 +289,10 @@ The report agent's `list_all_eval_results` and `sample_eval_results` tools cap e
Large results reduce the number of examples returned; each returned example keeps its complete category list.
The tools indicate when examples are omitted, and aggregate counts and pass rates still cover the full period.
+The report agent's `sample_generation_details` tool keeps the beginning and end of input and input-state previews within 500 characters, with an elision marker.
+Both sampled and full generation details include input, output, cache-read, and cache-creation token counts.
+The agent must inspect full generation details before claiming captured input is truncated or instrumentation is broken. Previews and low input token counts alone are insufficient evidence.
+
## Browser compatibility
The evaluations list keeps supported rows visible if the API returns an output type the browser cannot display.
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
index 98c911e1778b..e97b1b0ab2f2 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/prompts.py
@@ -263,8 +263,8 @@ If a session cannot be resolved, try another example. If none resolve, report th
"generation and get its `trace_id`."
)
detail_tools_section = (
- "- **`sample_generation_details(generation_ids)`**: generation data including input, output, model, "
- "tokens, and trace ID. Call this before citing.\n"
+ "- **`sample_generation_details(generation_ids)`**: 500-character head-and-tail input previews, "
+ "output previews, model, token counts including cache usage, and trace ID. Call this before citing.\n"
"- **`get_generation_detail(generation_id)`**: complete data for one generation and its evaluation "
"results.\n"
"- **`get_generation_text_repr(generation_id)`**: canonical rendering for a generation."
@@ -276,7 +276,9 @@ If a session cannot be resolved, try another example. If none resolve, report th
3. Call `add_citation(generation_id=generation_id, trace_id=trace_id, reason=reason)` for each example you use.
4. Reference the generation ID inline with single backticks so the renderer can link it.
-If a generation cannot be resolved, try another example. If none resolve, report the data-quality limitation."""
+If a generation cannot be resolved, try another example. If none resolve, report the data-quality limitation.
+
+Sample previews omit content. You must call `get_generation_detail` before claiming that captured input is truncated or instrumentation is broken. Low `input_tokens` alone is not evidence of missing input: inspect `cache_read_input_tokens` and `cache_creation_input_tokens` as well as the complete input."""
detail_step = "Call `sample_generation_details(...)` on 3-5 useful examples."
else:
raise ValueError(f"Unsupported evaluation target: {evaluation_target}")
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_graph.py b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_graph.py
index 9e754442201e..a409ee505c00 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_graph.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_graph.py
@@ -124,7 +124,14 @@ class TestSystemPromptFormat(SimpleTestCase):
),
(
"generation",
- ["sample_generation_details", "get_generation_detail", "add_citation(generation_id=generation_id"],
+ [
+ "sample_generation_details",
+ "get_generation_detail",
+ "add_citation(generation_id=generation_id",
+ "must call `get_generation_detail` before claiming",
+ "cache_read_input_tokens",
+ "cache_creation_input_tokens",
+ ],
["sample_trace_details", "sample_session_details"],
),
(
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools_ai_events_routing.py b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools_ai_events_routing.py
index 8dc1be32f425..dd72bf3e0a57 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools_ai_events_routing.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/test/test_tools_ai_events_routing.py
@@ -14,6 +14,8 @@ from typing import cast
from posthog.test.base import BaseTest
from unittest.mock import MagicMock, patch
+from parameterized import parameterized
+
from posthog.hogql import ast
from posthog.temporal.ai_observability.eval_reports.report_agent.schema import EvalReportContent
@@ -51,26 +53,43 @@ def _resolver_response(rows: list[list]) -> MagicMock:
class TestSampleGenerationDetailsRoutesThroughResolver(BaseTest):
+ @parameterized.expand(
+ [
+ ("short", [{"role": "user", "content": "hi"}], None, None),
+ ("at_limit", "x" * 500, 0, 0),
+ ("long", "BEGIN_INPUT " + "x" * 800 + " END_INPUT", 1000, 50),
+ ]
+ )
@patch("posthog.hogql_queries.ai.ai_table_resolver.query_ai_events")
@patch(_RESOLVE_PATH, return_value={_VALID_GEN_ID: "trace-1"})
- def test_routes_through_resolver_against_ai_events(self, _mock_lookup, mock_resolver):
+ def test_routes_through_resolver_against_ai_events(
+ self,
+ _name: str,
+ input_value: str | list[dict[str, str]],
+ cache_read_tokens: int | None,
+ cache_creation_tokens: int | None,
+ _mock_lookup: MagicMock,
+ mock_resolver: MagicMock,
+ ) -> None:
"""The heavy-prop tool must hit the resolver, not raw `execute_hogql_query`."""
mock_resolver.return_value = _resolver_response(
[
[
_VALID_GEN_ID,
"gpt-4o", # model
- [{"role": "user", "content": "hi"}], # input (heavy, native column)
+ input_value,
[{"role": "assistant", "content": "hello"}], # output
- 10,
+ 3,
5,
"trace-1",
False,
None,
None,
None,
+ input_value,
None,
- None,
+ cache_read_tokens,
+ cache_creation_tokens,
]
]
)
@@ -79,9 +98,26 @@ class TestSampleGenerationDetailsRoutesThroughResolver(BaseTest):
# Returns rendered JSON with input/output previews from heavy columns.
assert result
+ generation = json.loads(result)[0]
+ input_text = str(input_value)
+ for preview in (generation["input_preview"], generation["input_state_preview"]):
+ if len(input_text) <= 500:
+ assert preview == input_text
+ else:
+ assert len(preview) == 500
+ assert preview.startswith(input_text[:250])
+ assert preview.endswith(input_text[-245:])
+ assert "\n...\n" in preview
+ assert generation["input_tokens"] == 3
+ assert generation["cache_read_input_tokens"] == cache_read_tokens
+ assert generation["cache_creation_input_tokens"] == cache_creation_tokens
assert mock_resolver.call_count == 1
kwargs = mock_resolver.call_args.kwargs
select = cast(ast.SelectQuery, kwargs["query"])
+ assert [cast(ast.Field, field).chain for field in select.select[-2:]] == [
+ ["cache_read_input_tokens"],
+ ["cache_creation_input_tokens"],
+ ]
from_chain = select.select_from.table.chain # type: ignore[union-attr]
# nosemgrep: hogql-no-string-table-chain
assert from_chain == ["posthog", "ai_events"]
@@ -137,6 +173,8 @@ class TestGetGenerationDetailRoutesThroughResolver(BaseTest):
None,
None,
None,
+ 1000,
+ 50,
]
]
)
@@ -148,6 +186,8 @@ class TestGetGenerationDetailRoutesThroughResolver(BaseTest):
result = json.loads(_get_detail_fn(state=_state(self.team.id), generation_id=_VALID_GEN_ID))
assert result["eval_results"][0]["outcome"] == "skipped"
+ assert result["cache_read_input_tokens"] == 1000
+ assert result["cache_creation_input_tokens"] == 50
eval_query = cast(ast.SelectQuery, mock_events.call_args.kwargs["query"])
skipped = cast(ast.Alias, eval_query.select[-1])
assert skipped.alias == "skipped"
@@ -161,6 +201,11 @@ class TestGetGenerationDetailRoutesThroughResolver(BaseTest):
# Heavy fetch carries the resolved trace_id — proves the two-query
# pattern wires through, not just that the WHERE has uuid.
kwargs = mock_resolver.call_args.kwargs
+ generation_query = cast(ast.SelectQuery, kwargs["query"])
+ assert [cast(ast.Field, field).chain for field in generation_query.select[-2:]] == [
+ ["cache_read_input_tokens"],
+ ["cache_creation_input_tokens"],
+ ]
placeholders = kwargs["placeholders"]
assert "trace_id" in placeholders
assert placeholders["trace_id"].value == "trace-1"
diff --git a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
index 5695735159d1..2ad393c7000c 100644
--- a/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
+++ b/posthog/temporal/ai_observability/eval_reports/report_agent/tools.py
@@ -48,6 +48,7 @@ from posthog.temporal.ai_observability.eval_reports.targets import (
resolve_evaluation_target,
target_event_predicate,
)
+from posthog.temporal.ai_observability.sentiment.extraction import truncate_to_head_tail
from posthog.temporal.ai_observability.trace_summarization.constants import (
MAX_TRACE_EVENTS_LIMIT,
MAX_TRACE_PROPERTIES_SIZE,
@@ -747,11 +748,12 @@ def sample_generation_details(
state: Annotated[dict, InjectedState],
generation_ids: list[str],
) -> str:
- """Get full $ai_generation event data for specific generations.
+ """Get $ai_generation previews and token usage for specific generations.
- Returns input, output, model, tokens, AND trace_id for verification before
- citing in the report. **Always call this before add_citation** so you have
- the trace_id to pass in.
+ Input previews keep the head and tail within 500 characters. Output previews
+ keep the first 500 characters. Returns model, token counts including cache
+ usage, and trace_id for verification before citing in the report.
+ **Always call this before add_citation** so you have the trace_id to pass in.
Args:
generation_ids: List of generation IDs to look up (max 20)
@@ -811,7 +813,9 @@ def sample_generation_details(
properties.$ai_tools_called as tools_called,
properties.$ai_tool_call_count as tool_call_count,
input_state,
- output_state
+ output_state,
+ cache_read_input_tokens,
+ cache_creation_input_tokens
FROM posthog.ai_events AS ai_events
WHERE event = '$ai_generation'
AND trace_id IN {trace_ids}
@@ -827,7 +831,7 @@ def sample_generation_details(
result = []
for row in rows:
- input_text = str(row[2])[:500] if row[2] else ""
+ input_text = truncate_to_head_tail(str(row[2]), max_chars=500, head_chars=250) if row[2] else ""
output_text = str(row[3])[:500] if row[3] else ""
entry: dict = {
"generation_id": str(row[0]) if row[0] else "",
@@ -836,6 +840,8 @@ def sample_generation_details(
"output_preview": output_text,
"input_tokens": row[4],
"output_tokens": row[5],
+ "cache_read_input_tokens": row[13],
+ "cache_creation_input_tokens": row[14],
"trace_id": str(row[6]) if row[6] else "",
}
if row[7]: # is_error
@@ -846,7 +852,7 @@ def sample_generation_details(
entry["tools_called"] = str(row[9])
entry["tool_call_count"] = row[10]
if row[11]: # input_state
- entry["input_state_preview"] = str(row[11])[:500]
+ entry["input_state_preview"] = truncate_to_head_tail(str(row[11]), max_chars=500, head_chars=250)
if row[12]: # output_state
entry["output_state_preview"] = str(row[12])[:500]
result.append(entry)
@@ -963,7 +969,9 @@ def get_generation_detail(
properties.$ai_tool_call_count as tool_call_count,
tools as tools_available,
input_state,
- output_state
+ output_state,
+ cache_read_input_tokens,
+ cache_creation_input_tokens
FROM posthog.ai_events AS ai_events
WHERE event = '$ai_generation'
AND trace_id = {trace_id}
@@ -1016,6 +1024,8 @@ def get_generation_detail(
"output": str(row[4]) if row[4] else "",
"input_tokens": row[5],
"output_tokens": row[6],
+ "cache_read_input_tokens": row[19],
+ "cache_creation_input_tokens": row[20],
"cost": row[7],
"latency": row[8],
"trace_id": str(row[9]) if row[9] else "",