:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add retell ai agent versions and test case tables (#127368)
Tom Owersd7e75798e515
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex f518f6973029..0c648475e300 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10260,11 +10260,11 @@ Today (5): `agents`, `calls`, `chats`, `phone_numbers`, `voices`Diffed against: <https://docs.retellai.com/llms.txt>-- [ ] `Agent versions (GET /list-agent-versions/{agent_id})` — resolve agent_version values on calls and compare outcomes between published versions (high)+- [x] `Agent versions (GET /list-agent-versions/{agent_id})` — resolve agent_version values on calls and compare outcomes between published versions (high)- [ ] `Contacts (POST /list-contacts)` — connect call and chat outcomes to customer profiles, external CRM IDs, and contact history (medium)- [ ] `Batch tests (GET /v2/list-batch-tests)` — track evaluation jobs and compare agent quality across test batches (medium)- [ ] `Test runs (GET /v2/list-test-runs/{test_case_batch_job_id})` — analyze individual evaluation results, failure explanations, and transcript snapshots (medium)-- [ ] `Test case definitions (GET /v2/list-test-case-definitions)` — interpret evaluation results using scenario definitions and configured scoring criteria (medium)+- [x] `Test case definitions (GET /v2/list-test-case-definitions)` — interpret evaluation results using scenario definitions and configured scoring criteria (medium)Note: The official index and embedded OpenAPI definitions were fetched with curl for collection reads and nested version lists. Contacts use POST for a read-only list operation, as do existing call and agent imports. The existing agents endpoint lists both voice and chat agents without a channel filter, so chat agents are covered. Version and test-run tables require requests for each agent or batch; write operations and agent configuration resources are excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.pyindex 5e133aa9a9ca..99248ae5e06f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.py@@ -51,4 +51,33 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"provider": "Provider of the voice.",},},+ "agent_versions": {+ "description": "Stored versions of every voice and chat agent, with publish state and version lineage.",+ "docs_url": "https://docs.retellai.com/api-references/list-agent-versions",+ "columns": {+ "agent_id": "Identifier of the agent the version belongs to.",+ "version": "Version number of the agent.",+ "is_published": "Whether the agent version is published.",+ "version_title": "Optional title of the agent version.",+ "version_description": "Optional description of the agent version.",+ "base_version": "Version that this agent version was based on.",+ "last_modification_timestamp": "Last modification time in epoch milliseconds.",+ },+ },+ "test_case_definitions": {+ "description": "Simulation test case definitions for every Retell LLM and conversation flow, with prompts, metrics, and tool mocks.",+ "docs_url": "https://docs.retellai.com/api-references/list-test-case-definitions",+ "columns": {+ "test_case_definition_id": "Unique identifier of the test case definition.",+ "name": "Name of the test case definition.",+ "response_engine": "Retell LLM or conversation flow the test case runs against.",+ "user_prompt": "User prompt to simulate in the test case.",+ "metrics": "Metric names the test case evaluates.",+ "dynamic_variables": "Dynamic variables injected into the response engine.",+ "tool_mocks": "Mock tool calls used during the test.",+ "llm_model": "LLM model used for the simulation.",+ "creation_timestamp": "Creation time in epoch milliseconds.",+ "user_modified_timestamp": "Last modification time in epoch milliseconds.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.pyindex a8d0edfe987c..76bfb3cb91f5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.py@@ -1,6 +1,7 @@from collections.abc import Iteratorfrom datetime import UTC, datetime-from typing import Any+from typing import Any, Literal+from urllib.parse import quotefrom requests import Responsefrom requests.exceptions import HTTPError@@ -8,6 +9,7 @@ from requests.exceptions import HTTPErrorfrom posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,@@ -28,6 +30,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.retell_ai.ENDPOINTS,INCREMENTAL_FIELDS,PAGE_SIZE,+ RetellAIEndpoint,+ RetellAIParent,)@@ -52,6 +56,10 @@ class RetellAIPaginator(JSONResponseCursorPaginator):super().update_state(response, data)+def _paginator(location: Literal["query", "json"]) -> RetellAIPaginator:+ return RetellAIPaginator(cursor_path="pagination_key", cursor_param="pagination_key", param_location=location)++def _client_config(api_key: str) -> ClientConfig:return {"base_url": BASE_URL,@@ -86,17 +94,15 @@ def get_resource(probe: bool = False,) -> EndpointResource:settings = schema_for_resource(ENDPOINTS, name)- endpoint: Endpoint = {"path": settings.path.format(api_version=api_version), "method": settings.method}+ # A fan-out table has no listing of its own, so it reads (and probes) its first parent listing.+ path, method = (+ (settings.parents[0].path, settings.parents[0].method) if settings.parents else (settings.path, settings.method)+ )+ endpoint: Endpoint = {"path": path.format(api_version=api_version), "method": method}if settings.paginated:endpoint["data_selector"] = "items"endpoint["data_selector_required"] = True- endpoint["paginator"] = (- "single_page"- if probe- else RetellAIPaginator(- cursor_path="pagination_key", cursor_param="pagination_key", param_location=settings.pagination_location- )- )+ endpoint["paginator"] = "single_page" if probe else _paginator(settings.pagination_location)pagination: dict[str, Any] = {"limit": 1 if probe else PAGE_SIZE, "sort_order": "ascending"}if settings.pagination_location == "json":if should_use_incremental_field and name in INCREMENTAL_FIELDS and watermark is not None:@@ -112,7 +118,7 @@ def get_resource(else:endpoint["paginator"] = "single_page"- resource: EndpointResource = {"name": name, "endpoint": endpoint, "primary_key": settings.primary_key}+ resource: EndpointResource = {"name": name, "endpoint": endpoint, "primary_key": settings.primary_keys}if name == "calls":resource["data_map"] = _normalize_callelif name in INCREMENTAL_FIELDS:@@ -120,6 +126,76 @@ def get_resource(return resource+def _parent_resource(parent: RetellAIParent) -> EndpointResource:+ return {+ "name": parent.path,+ "endpoint": {+ "path": parent.path,+ "method": parent.method,+ "params": {"limit": PAGE_SIZE, "sort_order": "ascending"},+ "data_selector": "items",+ "data_selector_required": True,+ "paginator": _paginator("query"),+ },+ }+++def _child_resource(name: str, settings: RetellAIEndpoint, parent: RetellAIParent, parent_id: str) -> EndpointResource:+ params: dict[str, Any] = {"limit": PAGE_SIZE}+ if settings.sortable:+ params["sort_order"] = "ascending"+ params.update({key: value.replace("{id}", parent_id) for key, value in parent.child_params})+ resource: EndpointResource = {+ "name": name,+ "endpoint": {+ "path": settings.path.replace("{id}", quote(parent_id, safe="")),+ "method": settings.method,+ "params": params,+ "data_selector": "items",+ "data_selector_required": True,+ "paginator": _paginator("query"),+ # A parent deleted between the parent listing and this request has no children left to read.+ "response_actions": [{"status_code": 404, "action": "ignore"}],+ },+ "primary_key": settings.primary_keys,+ }+ if settings.include_parent_id:+ resource["data_map"] = lambda row: {**row, parent.id_field: parent_id}+ return resource+++def _fan_out(+ api_key: str,+ inputs: SourceInputs,+ manager: ResumableSourceManager[RetellAIResumeConfig],+ settings: RetellAIEndpoint,+) -> Iterator[list[dict[str, Any]]]:+ # One session for every parent and child request, so the fan-out reuses its connections.+ client: ClientConfig = {+ **_client_config(api_key),+ "session": make_tracked_session(redact_values=(api_key,), allow_redirects=False),+ }+ for parent in settings.parents:+ # Parent listings can return one row per version of the same parent.+ seen: set[str] = set()+ parent_pages = rest_api_resource(+ {"client": client, "resources": [_parent_resource(parent)]}, inputs.team_id, inputs.job_id, None+ )+ for parent_page in parent_pages:+ for parent_row in parent_page:+ parent_id = parent_row.get(parent.id_field)+ if not isinstance(parent_id, str) or not parent_id or parent_id in seen:+ continue+ seen.add(parent_id)+ child_config: RESTAPIConfig = {+ "client": client,+ "resources": [_child_resource(inputs.schema_name, settings, parent, parent_id)],+ }+ yield from rest_api_resource(child_config, inputs.team_id, inputs.job_id, None)+ manager.safe_point()+ manager.safe_point()++def validate_credentials(api_key: str, schema_name: str | None, api_version: str) -> tuple[bool, str | None]:if not api_key or not api_key.isascii() or any(character.isspace() for character in api_key):return False, "Enter a valid Retell AI API key without spaces."@@ -148,6 +224,13 @@ def retell_ai_source(api_version: str,) -> SourceResponse:settings = schema_for_resource(ENDPOINTS, inputs.schema_name)+ if settings.parents:+ return SourceResponse(+ name=inputs.schema_name,+ items=lambda: _fan_out(api_key, inputs, manager, settings),+ primary_keys=settings.primary_keys,+ supports_resume=False,+ )resume = manager.load_state() if settings.paginated and manager.can_resume() else Nonedef save_state(state: dict[str, Any] | None) -> None:@@ -179,7 +262,7 @@ def retell_ai_source(return SourceResponse(name=inputs.schema_name,items=items,- primary_keys=[settings.primary_key],+ primary_keys=settings.primary_keys,partition_keys=[settings.partition_key] if settings.partition_key else None,partition_mode="datetime" if settings.partition_key else None,partition_format="month" if settings.partition_key else None,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.pyindex 1c05b75e0f59..e86d59dbfb2a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.py@@ -10,14 +10,31 @@ PAGE_SIZE = 100INCREMENTAL_LOOKBACK_SECONDS = 3600+@frozen+class RetellAIParent:+ path: str+ id_field: str+ method: Literal["GET", "POST"] = "GET"+ # Query params naming the parent on the child request. A `{id}` value is replaced by the parent id.+ child_params: tuple[tuple[str, str], ...] = ()++@frozenclass RetellAIEndpoint:path: str- primary_key: str+ primary_key: str | tuple[str, ...]method: Literal["GET", "POST"] = "GET"pagination_location: Literal["query", "json"] = "query"paginated: bool = Truepartition_key: str | None = None+ sortable: bool = True+ parents: tuple[RetellAIParent, ...] = ()+ # Copies the parent id onto each child row under the parent's id field.+ include_parent_id: bool = False++ @property+ def primary_keys(self) -> list[str]:+ return [self.primary_key] if isinstance(self.primary_key, str) else list(self.primary_key)ENDPOINTS: dict[str, RetellAIEndpoint] = {@@ -38,6 +55,30 @@ ENDPOINTS: dict[str, RetellAIEndpoint] = {),"phone_numbers": RetellAIEndpoint(path="/v2/list-phone-numbers", primary_key="phone_number"),"voices": RetellAIEndpoint(path="/list-voices", primary_key="voice_id", paginated=False),+ "agent_versions": RetellAIEndpoint(+ path="/list-agent-versions/{id}",+ primary_key=("agent_id", "version"),+ parents=(RetellAIParent(path="/v2/list-agents", id_field="agent_id", method="POST"),),+ include_parent_id=True,+ ),+ # Retell lists test case definitions per response engine, so the source walks every Retell LLM and conversation flow.+ "test_case_definitions": RetellAIEndpoint(+ path="/v2/list-test-case-definitions",+ primary_key="test_case_definition_id",+ sortable=False,+ parents=(+ RetellAIParent(+ path="/v2/list-retell-llms",+ id_field="llm_id",+ child_params=(("type", "retell-llm"), ("llm_id", "{id}")),+ ),+ RetellAIParent(+ path="/v2/list-conversation-flows",+ id_field="conversation_flow_id",+ child_params=(("type", "conversation-flow"), ("conversation_flow_id", "{id}")),+ ),+ ),+ ),}INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.pyindex 6f48b7176843..9c1d7339583b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.py@@ -8,6 +8,7 @@ from urllib.parse import parse_qs, urlsplitimport pytestfrom unittest.mock import MagicMock+from requests import PreparedRequest, Responsefrom requests.exceptions import HTTPErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs@@ -212,3 +213,94 @@ def test_invalid_watermark_is_not_silently_ignored(with pytest.raises(ValueError, match="watermark"):list(cast(Iterable[Any], result.items()))http.assert_not_called()+++FAN_OUT_RESPONSES: dict[tuple[str, str | None], list[dict[str, Any]]] = {+ ("/v2/list-agents", None): [+ {"items": [{"agent_id": "agent-a"}, {"agent_id": "agent-b"}], "has_more": True, "pagination_key": "agents-2"},+ {"items": [{"agent_id": "agent-a"}, {"agent_id": "agent-gone"}], "has_more": False},+ ],+ ("/list-agent-versions/agent-a", None): [+ {"items": [{"version": 0}], "has_more": True, "pagination_key": "versions-2"},+ {"items": [{"version": 1}], "has_more": False},+ ],+ ("/list-agent-versions/agent-b", None): [{"items": [{"version": 0}], "has_more": False}],+ ("/v2/list-retell-llms", None): [+ {"items": [{"llm_id": "llm-a", "version": 0}, {"llm_id": "llm-a", "version": 1}], "has_more": False}+ ],+ ("/v2/list-conversation-flows", None): [{"items": [{"conversation_flow_id": "flow-a"}], "has_more": False}],+ ("/v2/list-test-case-definitions", "llm-a"): [+ {"items": [{"test_case_definition_id": "test-llm"}], "has_more": False}+ ],+ ("/v2/list-test-case-definitions", "flow-a"): [+ {"items": [{"test_case_definition_id": "test-flow"}], "has_more": False}+ ],+}+++ "name,expected_rows,expected_keys",+ [+ (+ "agent_versions",+ [+ {"version": 0, "agent_id": "agent-a"},+ {"version": 1, "agent_id": "agent-a"},+ {"version": 0, "agent_id": "agent-b"},+ ],+ ["agent_id", "version"],+ ),+ (+ "test_case_definitions",+ [{"test_case_definition_id": "test-llm"}, {"test_case_definition_id": "test-flow"}],+ ["test_case_definition_id"],+ ),+ ],+)+def test_fan_out_walks_each_parent_once(+ name: str,+ expected_rows: list[dict[str, Any]],+ expected_keys: list[str],+ inputs: SourceInputs,+ http: MagicMock,+ redis_client: MagicMock,+) -> None:+ served: dict[tuple[str, str | None], int] = {}++ def send(_session: object, request: PreparedRequest, **_kwargs: Any) -> Response:+ url = urlsplit(request.url or "")+ query = parse_qs(url.query)+ engines = query.get("llm_id") or query.get("conversation_flow_id")+ key = (url.path, engines[0] if engines else None)+ if key not in FAN_OUT_RESPONSES:+ return response({"status": "error"}, 404)+ page = served.get(key, 0)+ served[key] = page + 1+ return response(FAN_OUT_RESPONSES[key][page])++ http.side_effect = send+ source = RetellAISource()+ inputs = replace(inputs, schema_name=name)+ result = source.source_for_pipeline(+ RetellAISourceConfig(api_key="fake-key"), source.get_resumable_source_manager(inputs), inputs+ )+ rows = [row for page in cast(Iterable[Any], result.items()) for row in page]+ assert rows == expected_rows+ assert result.primary_keys == expected_keys+ child_queries = [+ parse_qs(urlsplit(call.args[1].url).query)+ for call in http.call_args_list+ if not urlsplit(call.args[1].url).path.startswith(("/v2/list-agents", "/v2/list-retell", "/v2/list-conv"))+ ]+ if name == "test_case_definitions":+ assert [(query["type"], "sort_order" in query) for query in child_queries] == [+ (["retell-llm"], False),+ (["conversation-flow"], False),+ ]+ else:+ assert [path for path, _ in served] == [+ "/v2/list-agents",+ "/list-agent-versions/agent-a",+ "/list-agent-versions/agent-b",+ ]+ assert http.call_count == 6diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.pyindex 93a7245083dd..ba0e7eacd338 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py@@ -14,7 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.retell_ai.@pytest.mark.parametrize("status", [200, 401, 403])[email protected]("name", [None, "agents", "chats", "phone_numbers", "voices"])+ "name", [None, "agents", "chats", "phone_numbers", "voices", "agent_versions", "test_case_definitions"]+)def test_scoped_credential_probes(status: int, name: str | None, http: MagicMock) -> None:http.return_value = response([] if name == "voices" else {"items": [], "has_more": True, "pagination_key": "unused"}, status