Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add retell ai agent versions and test case tables (#127368)

Tom Owersd7e75798e515
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index f518f6973029..0c648475e300 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10260,11 +10260,11 @@ Today (5): `agents`, `calls`, `chats`, `phone_numbers`, `voices`
Diffed against: <https://docs.retellai.com/llms.txt>
-- [ ] `Agent versions (GET /list-agent-versions/{agent_id})` — resolve agent_version values on calls and compare outcomes between published versions (high)
+- [x] `Agent versions (GET /list-agent-versions/{agent_id})` — resolve agent_version values on calls and compare outcomes between published versions (high)
- [ ] `Contacts (POST /list-contacts)` — connect call and chat outcomes to customer profiles, external CRM IDs, and contact history (medium)
- [ ] `Batch tests (GET /v2/list-batch-tests)` — track evaluation jobs and compare agent quality across test batches (medium)
- [ ] `Test runs (GET /v2/list-test-runs/{test_case_batch_job_id})` — analyze individual evaluation results, failure explanations, and transcript snapshots (medium)
-- [ ] `Test case definitions (GET /v2/list-test-case-definitions)` — interpret evaluation results using scenario definitions and configured scoring criteria (medium)
+- [x] `Test case definitions (GET /v2/list-test-case-definitions)` — interpret evaluation results using scenario definitions and configured scoring criteria (medium)
Note: The official index and embedded OpenAPI definitions were fetched with curl for collection reads and nested version lists. Contacts use POST for a read-only list operation, as do existing call and agent imports. The existing agents endpoint lists both voice and chat agents without a channel filter, so chat agents are covered. Version and test-run tables require requests for each agent or batch; write operations and agent configuration resources are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.py
index 5e133aa9a9ca..99248ae5e06f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/canonical_descriptions.py
@@ -51,4 +51,33 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"provider": "Provider of the voice.",
},
},
+ "agent_versions": {
+ "description": "Stored versions of every voice and chat agent, with publish state and version lineage.",
+ "docs_url": "https://docs.retellai.com/api-references/list-agent-versions",
+ "columns": {
+ "agent_id": "Identifier of the agent the version belongs to.",
+ "version": "Version number of the agent.",
+ "is_published": "Whether the agent version is published.",
+ "version_title": "Optional title of the agent version.",
+ "version_description": "Optional description of the agent version.",
+ "base_version": "Version that this agent version was based on.",
+ "last_modification_timestamp": "Last modification time in epoch milliseconds.",
+ },
+ },
+ "test_case_definitions": {
+ "description": "Simulation test case definitions for every Retell LLM and conversation flow, with prompts, metrics, and tool mocks.",
+ "docs_url": "https://docs.retellai.com/api-references/list-test-case-definitions",
+ "columns": {
+ "test_case_definition_id": "Unique identifier of the test case definition.",
+ "name": "Name of the test case definition.",
+ "response_engine": "Retell LLM or conversation flow the test case runs against.",
+ "user_prompt": "User prompt to simulate in the test case.",
+ "metrics": "Metric names the test case evaluates.",
+ "dynamic_variables": "Dynamic variables injected into the response engine.",
+ "tool_mocks": "Mock tool calls used during the test.",
+ "llm_model": "LLM model used for the simulation.",
+ "creation_timestamp": "Creation time in epoch milliseconds.",
+ "user_modified_timestamp": "Last modification time in epoch milliseconds.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.py
index a8d0edfe987c..76bfb3cb91f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/retell_ai.py
@@ -1,6 +1,7 @@
from collections.abc import Iterator
from datetime import UTC, datetime
-from typing import Any
+from typing import Any, Literal
+from urllib.parse import quote
from requests import Response
from requests.exceptions import HTTPError
@@ -8,6 +9,7 @@ from requests.exceptions import HTTPError
from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
@@ -28,6 +30,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.retell_ai.
ENDPOINTS,
INCREMENTAL_FIELDS,
PAGE_SIZE,
+ RetellAIEndpoint,
+ RetellAIParent,
)
@@ -52,6 +56,10 @@ class RetellAIPaginator(JSONResponseCursorPaginator):
super().update_state(response, data)
+def _paginator(location: Literal["query", "json"]) -> RetellAIPaginator:
+ return RetellAIPaginator(cursor_path="pagination_key", cursor_param="pagination_key", param_location=location)
+
+
def _client_config(api_key: str) -> ClientConfig:
return {
"base_url": BASE_URL,
@@ -86,17 +94,15 @@ def get_resource(
probe: bool = False,
) -> EndpointResource:
settings = schema_for_resource(ENDPOINTS, name)
- endpoint: Endpoint = {"path": settings.path.format(api_version=api_version), "method": settings.method}
+ # A fan-out table has no listing of its own, so it reads (and probes) its first parent listing.
+ path, method = (
+ (settings.parents[0].path, settings.parents[0].method) if settings.parents else (settings.path, settings.method)
+ )
+ endpoint: Endpoint = {"path": path.format(api_version=api_version), "method": method}
if settings.paginated:
endpoint["data_selector"] = "items"
endpoint["data_selector_required"] = True
- endpoint["paginator"] = (
- "single_page"
- if probe
- else RetellAIPaginator(
- cursor_path="pagination_key", cursor_param="pagination_key", param_location=settings.pagination_location
- )
- )
+ endpoint["paginator"] = "single_page" if probe else _paginator(settings.pagination_location)
pagination: dict[str, Any] = {"limit": 1 if probe else PAGE_SIZE, "sort_order": "ascending"}
if settings.pagination_location == "json":
if should_use_incremental_field and name in INCREMENTAL_FIELDS and watermark is not None:
@@ -112,7 +118,7 @@ def get_resource(
else:
endpoint["paginator"] = "single_page"
- resource: EndpointResource = {"name": name, "endpoint": endpoint, "primary_key": settings.primary_key}
+ resource: EndpointResource = {"name": name, "endpoint": endpoint, "primary_key": settings.primary_keys}
if name == "calls":
resource["data_map"] = _normalize_call
elif name in INCREMENTAL_FIELDS:
@@ -120,6 +126,76 @@ def get_resource(
return resource
+def _parent_resource(parent: RetellAIParent) -> EndpointResource:
+ return {
+ "name": parent.path,
+ "endpoint": {
+ "path": parent.path,
+ "method": parent.method,
+ "params": {"limit": PAGE_SIZE, "sort_order": "ascending"},
+ "data_selector": "items",
+ "data_selector_required": True,
+ "paginator": _paginator("query"),
+ },
+ }
+
+
+def _child_resource(name: str, settings: RetellAIEndpoint, parent: RetellAIParent, parent_id: str) -> EndpointResource:
+ params: dict[str, Any] = {"limit": PAGE_SIZE}
+ if settings.sortable:
+ params["sort_order"] = "ascending"
+ params.update({key: value.replace("{id}", parent_id) for key, value in parent.child_params})
+ resource: EndpointResource = {
+ "name": name,
+ "endpoint": {
+ "path": settings.path.replace("{id}", quote(parent_id, safe="")),
+ "method": settings.method,
+ "params": params,
+ "data_selector": "items",
+ "data_selector_required": True,
+ "paginator": _paginator("query"),
+ # A parent deleted between the parent listing and this request has no children left to read.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ "primary_key": settings.primary_keys,
+ }
+ if settings.include_parent_id:
+ resource["data_map"] = lambda row: {**row, parent.id_field: parent_id}
+ return resource
+
+
+def _fan_out(
+ api_key: str,
+ inputs: SourceInputs,
+ manager: ResumableSourceManager[RetellAIResumeConfig],
+ settings: RetellAIEndpoint,
+) -> Iterator[list[dict[str, Any]]]:
+ # One session for every parent and child request, so the fan-out reuses its connections.
+ client: ClientConfig = {
+ **_client_config(api_key),
+ "session": make_tracked_session(redact_values=(api_key,), allow_redirects=False),
+ }
+ for parent in settings.parents:
+ # Parent listings can return one row per version of the same parent.
+ seen: set[str] = set()
+ parent_pages = rest_api_resource(
+ {"client": client, "resources": [_parent_resource(parent)]}, inputs.team_id, inputs.job_id, None
+ )
+ for parent_page in parent_pages:
+ for parent_row in parent_page:
+ parent_id = parent_row.get(parent.id_field)
+ if not isinstance(parent_id, str) or not parent_id or parent_id in seen:
+ continue
+ seen.add(parent_id)
+ child_config: RESTAPIConfig = {
+ "client": client,
+ "resources": [_child_resource(inputs.schema_name, settings, parent, parent_id)],
+ }
+ yield from rest_api_resource(child_config, inputs.team_id, inputs.job_id, None)
+ manager.safe_point()
+ manager.safe_point()
+
+
def validate_credentials(api_key: str, schema_name: str | None, api_version: str) -> tuple[bool, str | None]:
if not api_key or not api_key.isascii() or any(character.isspace() for character in api_key):
return False, "Enter a valid Retell AI API key without spaces."
@@ -148,6 +224,13 @@ def retell_ai_source(
api_version: str,
) -> SourceResponse:
settings = schema_for_resource(ENDPOINTS, inputs.schema_name)
+ if settings.parents:
+ return SourceResponse(
+ name=inputs.schema_name,
+ items=lambda: _fan_out(api_key, inputs, manager, settings),
+ primary_keys=settings.primary_keys,
+ supports_resume=False,
+ )
resume = manager.load_state() if settings.paginated and manager.can_resume() else None
def save_state(state: dict[str, Any] | None) -> None:
@@ -179,7 +262,7 @@ def retell_ai_source(
return SourceResponse(
name=inputs.schema_name,
items=items,
- primary_keys=[settings.primary_key],
+ primary_keys=settings.primary_keys,
partition_keys=[settings.partition_key] if settings.partition_key else None,
partition_mode="datetime" if settings.partition_key else None,
partition_format="month" if settings.partition_key else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.py
index 1c05b75e0f59..e86d59dbfb2a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/settings.py
@@ -10,14 +10,31 @@ PAGE_SIZE = 100
INCREMENTAL_LOOKBACK_SECONDS = 3600
+@frozen
+class RetellAIParent:
+ path: str
+ id_field: str
+ method: Literal["GET", "POST"] = "GET"
+ # Query params naming the parent on the child request. A `{id}` value is replaced by the parent id.
+ child_params: tuple[tuple[str, str], ...] = ()
+
+
@frozen
class RetellAIEndpoint:
path: str
- primary_key: str
+ primary_key: str | tuple[str, ...]
method: Literal["GET", "POST"] = "GET"
pagination_location: Literal["query", "json"] = "query"
paginated: bool = True
partition_key: str | None = None
+ sortable: bool = True
+ parents: tuple[RetellAIParent, ...] = ()
+ # Copies the parent id onto each child row under the parent's id field.
+ include_parent_id: bool = False
+
+ @property
+ def primary_keys(self) -> list[str]:
+ return [self.primary_key] if isinstance(self.primary_key, str) else list(self.primary_key)
ENDPOINTS: dict[str, RetellAIEndpoint] = {
@@ -38,6 +55,30 @@ ENDPOINTS: dict[str, RetellAIEndpoint] = {
),
"phone_numbers": RetellAIEndpoint(path="/v2/list-phone-numbers", primary_key="phone_number"),
"voices": RetellAIEndpoint(path="/list-voices", primary_key="voice_id", paginated=False),
+ "agent_versions": RetellAIEndpoint(
+ path="/list-agent-versions/{id}",
+ primary_key=("agent_id", "version"),
+ parents=(RetellAIParent(path="/v2/list-agents", id_field="agent_id", method="POST"),),
+ include_parent_id=True,
+ ),
+ # Retell lists test case definitions per response engine, so the source walks every Retell LLM and conversation flow.
+ "test_case_definitions": RetellAIEndpoint(
+ path="/v2/list-test-case-definitions",
+ primary_key="test_case_definition_id",
+ sortable=False,
+ parents=(
+ RetellAIParent(
+ path="/v2/list-retell-llms",
+ id_field="llm_id",
+ child_params=(("type", "retell-llm"), ("llm_id", "{id}")),
+ ),
+ RetellAIParent(
+ path="/v2/list-conversation-flows",
+ id_field="conversation_flow_id",
+ child_params=(("type", "conversation-flow"), ("conversation_flow_id", "{id}")),
+ ),
+ ),
+ ),
}
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.py
index 6f48b7176843..9c1d7339583b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai.py
@@ -8,6 +8,7 @@ from urllib.parse import parse_qs, urlsplit
import pytest
from unittest.mock import MagicMock
+from requests import PreparedRequest, Response
from requests.exceptions import HTTPError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
@@ -212,3 +213,94 @@ def test_invalid_watermark_is_not_silently_ignored(
with pytest.raises(ValueError, match="watermark"):
list(cast(Iterable[Any], result.items()))
http.assert_not_called()
+
+
+FAN_OUT_RESPONSES: dict[tuple[str, str | None], list[dict[str, Any]]] = {
+ ("/v2/list-agents", None): [
+ {"items": [{"agent_id": "agent-a"}, {"agent_id": "agent-b"}], "has_more": True, "pagination_key": "agents-2"},
+ {"items": [{"agent_id": "agent-a"}, {"agent_id": "agent-gone"}], "has_more": False},
+ ],
+ ("/list-agent-versions/agent-a", None): [
+ {"items": [{"version": 0}], "has_more": True, "pagination_key": "versions-2"},
+ {"items": [{"version": 1}], "has_more": False},
+ ],
+ ("/list-agent-versions/agent-b", None): [{"items": [{"version": 0}], "has_more": False}],
+ ("/v2/list-retell-llms", None): [
+ {"items": [{"llm_id": "llm-a", "version": 0}, {"llm_id": "llm-a", "version": 1}], "has_more": False}
+ ],
+ ("/v2/list-conversation-flows", None): [{"items": [{"conversation_flow_id": "flow-a"}], "has_more": False}],
+ ("/v2/list-test-case-definitions", "llm-a"): [
+ {"items": [{"test_case_definition_id": "test-llm"}], "has_more": False}
+ ],
+ ("/v2/list-test-case-definitions", "flow-a"): [
+ {"items": [{"test_case_definition_id": "test-flow"}], "has_more": False}
+ ],
+}
+
+
+ "name,expected_rows,expected_keys",
+ [
+ (
+ "agent_versions",
+ [
+ {"version": 0, "agent_id": "agent-a"},
+ {"version": 1, "agent_id": "agent-a"},
+ {"version": 0, "agent_id": "agent-b"},
+ ],
+ ["agent_id", "version"],
+ ),
+ (
+ "test_case_definitions",
+ [{"test_case_definition_id": "test-llm"}, {"test_case_definition_id": "test-flow"}],
+ ["test_case_definition_id"],
+ ),
+ ],
+)
+def test_fan_out_walks_each_parent_once(
+ name: str,
+ expected_rows: list[dict[str, Any]],
+ expected_keys: list[str],
+ inputs: SourceInputs,
+ http: MagicMock,
+ redis_client: MagicMock,
+) -> None:
+ served: dict[tuple[str, str | None], int] = {}
+
+ def send(_session: object, request: PreparedRequest, **_kwargs: Any) -> Response:
+ url = urlsplit(request.url or "")
+ query = parse_qs(url.query)
+ engines = query.get("llm_id") or query.get("conversation_flow_id")
+ key = (url.path, engines[0] if engines else None)
+ if key not in FAN_OUT_RESPONSES:
+ return response({"status": "error"}, 404)
+ page = served.get(key, 0)
+ served[key] = page + 1
+ return response(FAN_OUT_RESPONSES[key][page])
+
+ http.side_effect = send
+ source = RetellAISource()
+ inputs = replace(inputs, schema_name=name)
+ result = source.source_for_pipeline(
+ RetellAISourceConfig(api_key="fake-key"), source.get_resumable_source_manager(inputs), inputs
+ )
+ rows = [row for page in cast(Iterable[Any], result.items()) for row in page]
+ assert rows == expected_rows
+ assert result.primary_keys == expected_keys
+ child_queries = [
+ parse_qs(urlsplit(call.args[1].url).query)
+ for call in http.call_args_list
+ if not urlsplit(call.args[1].url).path.startswith(("/v2/list-agents", "/v2/list-retell", "/v2/list-conv"))
+ ]
+ if name == "test_case_definitions":
+ assert [(query["type"], "sort_order" in query) for query in child_queries] == [
+ (["retell-llm"], False),
+ (["conversation-flow"], False),
+ ]
+ else:
+ assert [path for path, _ in served] == [
+ "/v2/list-agents",
+ "/list-agent-versions/agent-a",
+ "/list-agent-versions/agent-b",
+ ]
+ assert http.call_count == 6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py
index 93a7245083dd..ba0e7eacd338 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retell_ai/tests/test_retell_ai_source.py
@@ -14,7 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.retell_ai.
@pytest.mark.parametrize("status", [200, 401, 403])
[email protected]("name", [None, "agents", "chats", "phone_numbers", "voices"])
+ "name", [None, "agents", "chats", "phone_numbers", "voices", "agent_versions", "test_case_definitions"]
+)
def test_scoped_credential_probes(status: int, name: str | None, http: MagicMock) -> None:
http.return_value = response(
[] if name == "voices" else {"items": [], "has_more": True, "pagination_key": "unused"}, status