:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): report per-item models in the offline eval harness (#124039)
Radu Raicea8dd35fc04d79
diff --git a/docs/internal/ai-offline-evaluation-reporting.md b/docs/internal/ai-offline-evaluation-reporting.mdindex 9afc6c3ed976..31521cacf264 100644--- a/docs/internal/ai-offline-evaluation-reporting.md+++ b/docs/internal/ai-offline-evaluation-reporting.md@@ -248,6 +248,9 @@ Numeric scores can use the domain of their configured scorer version. The API va`OfflineEvalSuite.experiment_options` accepts the SDK's `ExperimentOptions` for external dataset identifiers, `prompt_version`, model/application versions, and `run_source`.The harness owns the experiment ID, start time, suite key, and expected counts.+Each case's `output["model"]` becomes the item's `model_version`, so one-shot suites can report the model they actually used and compare different models within one experiment.+Cases without a model, or with `model=None`, inherit the experiment's model version.+Invalid model identifiers are omitted and recorded in the item's `omitted_fields`; the case's scores still upload.For external datasets, case metadata can supply `dataset_item_identifier` and `dataset_item_version_identifier`.The harness publisher does not support hosted datasets. A non-null `dataset_revision_id` stops preparation with a local error before any upload request.diff --git a/products/posthog_ai/eval_harness/offline_results.py b/products/posthog_ai/eval_harness/offline_results.pyindex 3a505069fa51..582323878bc0 100644--- a/products/posthog_ai/eval_harness/offline_results.py+++ b/products/posthog_ai/eval_harness/offline_results.py@@ -282,6 +282,7 @@ class OfflineEvalPublisher:omitted: list[str] = []for key, value in {"case_key": name,+ "model_version": case.output.get("model") if isinstance(case.output, dict) else None,"application_trace_id": case.metadata.get("trace_id") if self.application_trace_ids_enabled else None,"dataset_item_identifier": case.metadata.get("dataset_item_identifier"),"dataset_item_version_identifier": case.metadata.get("dataset_item_version_identifier"),diff --git a/products/posthog_ai/eval_harness/test/test_offline_results.py b/products/posthog_ai/eval_harness/test/test_offline_results.pyindex f7e10b51c954..5fc9b85ad92c 100644--- a/products/posthog_ai/eval_harness/test/test_offline_results.py+++ b/products/posthog_ai/eval_harness/test/test_offline_results.py@@ -162,6 +162,35 @@ def test_prepare_assigns_stable_trial_ids_and_uses_captured_revision() -> None:assert items[0]["payload"]["metadata"]["braintrust_url"] == "https://braintrust.example.com/experiment/test"+ [+ ("missing", {}, None, []),+ ("null", {"model": None}, None, []),+ ("no_output", None, None, []),+ ("model", {"model": "case-model"}, "case-model", []),+ ("invalid_type", {"model": 42}, None, ["model_version"]),+ ("empty_identifier", {"model": ""}, None, ["model_version"]),+ ]+)+def test_prepare_preserves_per_item_models(+ _name: str, output: dict[str, object] | None, model: str | None, omitted: list[str]+) -> None:+ case = _case()+ case.output = output+ other_case = _case("other")+ other_case.output = {"model": "other-model"}++ upload = _prepare(_publisher(), [case, other_case])+ items = [item.to_dict() for item in _items(upload)]++ assert upload.experiment["model_version"] == "test-model"+ assert items[0].get("model_version") == model+ assert items[1]["model_version"] == "other-model"+ assert items[0]["payload"]["metadata"]["omitted_fields"] == omitted+ assert items[0]["payload"]["output"] == output+ assert all(result.to_dict()["status"] == "ok" for result in upload.results)++def test_prepare_omits_raw_logs_and_oversized_fields_without_mutating_results() -> None:case = _case()original_output = {"raw_log": "local trace", "tool_result": "x" * (1100 * 1024), "answer": "There are 3 events"}diff --git a/pyproject.toml b/pyproject.tomlindex 137ea3e0d064..6a9f394af6f5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -105,7 +105,7 @@ dependencies = ["paramiko~=3.5.0","pillow==12.3.0","protobuf~=6.31.1",- "posthoganalytics==7.75.11",+ "posthoganalytics==7.76.0","polars==1.37.1","psycopg2-binary==2.9.11","psycopg[binary]==3.2.10",diff --git a/uv.lock b/uv.lockindex 5edc05ace7eb..ad1757b49ffb 100644--- a/uv.lock+++ b/uv.lock@@ -6329,7 +6329,7 @@ requires-dist = [{ name = "playwright", specifier = "~=1.60.0" },{ name = "polars", specifier = "==1.37.1" },{ name = "posthog-hogland", specifier = "==0.4.1" },- { name = "posthoganalytics", specifier = "==7.75.11" },+ { name = "posthoganalytics", specifier = "==7.76.0" },{ name = "protobuf", specifier = "~=6.31.1" },{ name = "psycopg", extras = ["binary"], specifier = "==3.2.10" },{ name = "psycopg2-binary", specifier = "==2.9.11" },@@ -6496,7 +6496,7 @@ wheels = [[[package]]name = "posthoganalytics"-version = "7.75.11"+version = "7.76.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "backoff" },@@ -6504,9 +6504,9 @@ dependencies = [{ name = "requests" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/69/92/df593753bd8d9599fede0ed9b5a61fd6db2543f53b3a0c88af01e8b6afb5/posthoganalytics-7.75.11.tar.gz", hash = "sha256:bb98529eae8c6be34bef14bd9ec53b3bbbd8286f08df0d6971f395faa1a56bb1", size = 819852, upload-time = "2026-10-23T15:27:53.543Z" }+sdist = { url = "https://files.pythonhosted.org/packages/cc/52/717c6d4ea646ad62a9f9f64e531397811549758e08b6a5f6d5aadbf065b0/posthoganalytics-7.76.0.tar.gz", hash = "sha256:ac46889d4957f6edddbe6ffe5a0857aa696a13ba204490a4090604b3c0f36b0d", size = 822783, upload-time = "2026-10-27T13:06:52.861Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/da/76/0532e4db9c08f41ca6008948f8b5fef65a090a9411b58d73b27ded6d2ec1/posthoganalytics-7.75.11-py3-none-any.whl", hash = "sha256:4c5f750cebe3f52d4bb2fb603ccb2220a04a4afa125d7fe9a70ee5fc7abbc248", size = 963793, upload-time = "2026-10-23T15:27:51.413Z" },+ { url = "https://files.pythonhosted.org/packages/68/dc/b0abea89d488203836613f4a33cbb381d49c43004c4383e2020e700de91c/posthoganalytics-7.76.0-py3-none-any.whl", hash = "sha256:b5e13eb025a93e82c0b7d12184e9aae7f8dd420c66b7a71b93210eef19033cdb", size = 966059, upload-time = "2026-10-27T13:06:50.842Z" },][[package]]