:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add elevenlabs deployments, flows runs and voice collections tables (#119238)
Tom Owers7b41b003f551
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex fa689e51789b..b2f317980bf8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -4218,6 +4218,9 @@ Diffed against: <https://api.elevenlabs.io/openapi.json>- [ ] `/v1/dubbing (list dubs)` — dubbing jobs are a separate billable workload not represented by history or conversations (low)- [ ] `/v1/convai/agent-testing and /v1/convai/test-invocations` — agent test definitions and their invocation results for quality tracking (low)- [x] `/v1/convai/triage-tickets` — workspace-wide Conversational AI triage tickets raised about agent performance on conversations (medium)+- [x] `/v1/convai/agents/{agent_id}/deployments` — current and historical traffic splits across agent branches (medium)+- [x] `/v1/flows/templates and /v1/flows/templates/{template_id}/runs` — Flows templates plus the runs created through the API (low)+- [x] `/v1/voices/collections` — voice collections lookup (low)Note: Official spec is large (277 paths); the ConvAI surface is the bulk of it. Static endpoint list in settings.py, no dynamic discovery.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/canonical_descriptions.pyindex 490710377651..a741dd06be17 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/canonical_descriptions.py@@ -192,6 +192,51 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"retry_count": "Number of times the job was retried.",},},+ "agent_deployments": {+ "description": "Traffic split history of each Conversational AI agent across its branches. One row per traffic split, including the live one.",+ "docs_url": "https://elevenlabs.io/docs/api-reference/agents/deployments/list",+ "columns": {+ "id": "Identifier of the deployment.",+ "agent_id": "Identifier of the agent the deployment belongs to.",+ "traffic_percentage_branch_id_map": "Map of branch IDs to the percentage of traffic each branch receives.",+ "deployed_at_unix_secs": "Time the traffic split was applied, in Unix seconds.",+ "is_active": "Whether this is the live traffic split.",+ "source": "What caused the traffic split change (manual or branch_merge). Null for older deployments.",+ "access_info": "Access metadata for the deployment, including who created it.",+ },+ },+ "flow_templates": {+ "description": "Published Flows templates in your workspace, with their runnable versions. One row per template.",+ "docs_url": "https://elevenlabs.io/docs/api-reference/flows/templates/list",+ "columns": {+ "id": "Unique identifier for the template.",+ "name": "Name of the template.",+ "description": "Description of the template, if it has one.",+ "versions": "Up to three of the most recent published versions you can run, with their inputs and outputs.",+ "has_more_versions": "Whether the template has more published versions than the ones returned.",+ },+ },+ "flow_template_runs": {+ "description": "Runs of Flows templates that were created through the API. One row per run.",+ "docs_url": "https://elevenlabs.io/docs/api-reference/flows/templates/runs/list",+ "columns": {+ "id": "Unique identifier for the run.",+ "template_id": "Identifier of the template the run executed.",+ "version_id": "Identifier of the template version the run executed.",+ "status": "Status of the run (pending, generating, completed, or failed).",+ "outputs": "Outputs of the run, keyed by output port ID.",+ },+ },+ "voice_collections": {+ "description": "Voice collections you can access. One row per collection.",+ "docs_url": "https://elevenlabs.io/docs/api-reference/voices/collections/list",+ "columns": {+ "collection_id": "Unique identifier for the voice collection.",+ "title": "Title of the voice collection.",+ "icon": "Icon of the voice collection.",+ "permission_on_resource": "Your access role on the voice collection.",+ },+ },"subscription": {"description": "Subscription plan, quota, and current billing period usage. A single row that each sync replaces.","docs_url": "https://elevenlabs.io/docs/api-reference/user/subscription/get",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/elevenlabs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/elevenlabs.pyindex 7f39f3501c1b..9f26f423a3a1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/elevenlabs.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/elevenlabs.py@@ -16,6 +16,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (BasePaginator,+ PageNumberPaginator,SinglePagePaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource@@ -179,6 +180,8 @@ class ElevenLabsCursorPaginator(BasePaginator):def _paginator(config: ElevenLabsEndpointConfig) -> BasePaginator:+ if config.page_number_param is not None:+ return PageNumberPaginator(base_page=1, page_param=config.page_number_param)if config.cursor_param is None or config.cursor_response_key is None:return SinglePagePaginator()return ElevenLabsCursorPaginator(config.cursor_param, config.cursor_response_key)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/settings.pyindex 5ab790724198..80c15e27e9f2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/settings.py@@ -24,6 +24,8 @@ class ElevenLabsEndpointConfig:cursor_response_key: Optional[str] = Nonepage_size: Optional[int] = Nonepage_size_param: str = "page_size"+ # Set for endpoints paged by a 1-based page number instead of a cursor.+ page_number_param: Optional[str] = None# None for tables without a stable creation timestamp; those small tables stay unpartitioned.partition_key: Optional[str] = None# Stable, monotonic incremental column and the server-side filter param that bounds it. Both None@@ -218,6 +220,64 @@ ELEVENLABS_ENDPOINTS: dict[str, ElevenLabsEndpointConfig] = {page_size_param="limit",partition_key="created_at_unix",),+ # Traffic split history per agent, fanned out over `agents` and returned newest first. Rows carry no+ # agent id, so the parent id is copied in. No updated-since filter, so full refresh only.+ "agent_deployments": ElevenLabsEndpointConfig(+ name="agent_deployments",+ path="/v1/convai/agents/{agent_id}/deployments",+ items_key="results",+ page_number_param="page",+ primary_keys=["agent_id", "id"],+ page_size=100,+ partition_key="deployed_at_unix_secs",+ sort_mode="desc",+ fanout=DependentEndpointConfig(+ parent_name="agents",+ resolve_param="agent_id",+ resolve_field="agent_id",+ include_from_parent=["agent_id"],+ parent_field_renames={"agent_id": "agent_id"},+ parent_params={"sort_by": "created_at", "sort_direction": "asc"},+ ),+ ),+ # Published Flows templates. The API caps the page size at 25 because each row expands its versions'+ # input and output schemas. No creation timestamp, so the table stays unpartitioned.+ "flow_templates": ElevenLabsEndpointConfig(+ name="flow_templates",+ path="/v1/flows/templates",+ items_key="templates",+ cursor_param="cursor",+ cursor_response_key="next_cursor",+ primary_keys=["id"],+ page_size=25,+ ),+ # Runs of each Flows template, fanned out over `flow_templates`. The API lists only runs created+ # through the API, newest first, with no timestamp or time filter, so full refresh and unpartitioned.+ "flow_template_runs": ElevenLabsEndpointConfig(+ name="flow_template_runs",+ path="/v1/flows/templates/{template_id}/runs",+ items_key="runs",+ cursor_param="cursor",+ cursor_response_key="next_cursor",+ primary_keys=["template_id", "id"],+ page_size=100,+ sort_mode="desc",+ fanout=DependentEndpointConfig(+ parent_name="flow_templates",+ resolve_param="template_id",+ resolve_field="id",+ include_from_parent=[],+ # The fan-out needs only template ids, so skip the extra version schemas.+ parent_params={"versions_per_template": 1},+ ),+ ),+ # Voice collections the key can access. Returned in one body sorted by title.+ "voice_collections": ElevenLabsEndpointConfig(+ name="voice_collections",+ path="/v1/voices/collections",+ items_key="collections",+ primary_keys=["collection_id"],+ ),# Plan quota and current-period usage, returned as one object that becomes a single row."subscription": ElevenLabsEndpointConfig(name="subscription",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/source.pyindex ac02c6a27189..7f8f1179e0d1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/source.py@@ -67,8 +67,8 @@ You can create an API key in your [ElevenLabs account settings](https://elevenlaGrant the following read permissions when creating the key:- **History** (text-to-speech generation history)-- **Conversational AI** (conversations, agents, triage tickets, tools, tool executions, phone numbers, and batch calls)-- **Voices**+- **Conversational AI** (conversations, agents, agent deployments, triage tickets, tools, tool executions, phone numbers, and batch calls)+- **Voices** (voices and voice collections)- **Models**- **Workspace** (members and groups)- **User** (subscription)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/tests/test_elevenlabs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/tests/test_elevenlabs.pyindex 7c1937597ddb..643e5ec9dc30 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/tests/test_elevenlabs.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/elevenlabs/tests/test_elevenlabs.py@@ -172,6 +172,49 @@ class TestFanOut:expected_start = ["1700000000"] * 3 if use_incremental else [None] * 3assert result.params("start_time")[1:] == expected_start+ def test_agent_deployments_walk_page_numbers_and_carry_the_agent_id(self) -> None:+ result = _driver().run(+ "agent_deployments",+ [+ ScriptedResponse(json={"agents": [{"agent_id": "a1"}], "has_more": False, "next_cursor": None}),+ ScriptedResponse(json={"results": [{"id": "d1"}], "meta": {"page": 1, "page_size": 100}}),+ ScriptedResponse(json={"results": [{"id": "d2"}], "meta": {"page": 2, "page_size": 100}}),+ ScriptedResponse(json={"results": [], "meta": {"page": 3, "page_size": 100}}),+ ],+ )++ assert result.raised is None+ # Deployment rows carry no agent id; without the parent id the primary key is not table-unique.+ assert result.rows == [{"id": "d1", "agent_id": "a1"}, {"id": "d2", "agent_id": "a1"}]+ assert result.params("page")[1:] == ["1", "2", "3"]+ assert result.params("page_size")[1:] == ["100"] * 3++ def test_flow_template_runs_walk_every_template_and_page(self) -> None:+ result = _driver().run(+ "flow_template_runs",+ [+ ScriptedResponse(+ json={"templates": [{"id": "t1"}, {"id": "t2"}], "has_more": False, "next_cursor": None}+ ),+ ScriptedResponse(+ json={"runs": [{"id": "r1", "template_id": "t1"}], "has_more": True, "next_cursor": "c1"}+ ),+ ScriptedResponse(+ json={"runs": [{"id": "r2", "template_id": "t1"}], "has_more": False, "next_cursor": None}+ ),+ ScriptedResponse(+ json={"runs": [{"id": "r3", "template_id": "t2"}], "has_more": False, "next_cursor": None}+ ),+ ],+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["r1", "r2", "r3"]+ # The template list rejects a page size above 25.+ assert result.requests[0].param("page_size") == "25"+ assert result.requests[0].param("versions_per_template") == "1"+ assert result.requests[2].param("cursor") == "c1"+class TestSecretRedaction:@parameterized.expand(@@ -272,6 +315,13 @@ class TestUnpaginatedEndpoints:["p1", "p2"],"phone_number_id",),+ (+ "voice_collections_wrapped",+ "voice_collections",+ {"collections": [{"collection_id": "c1"}, {"collection_id": "c2"}]},+ ["c1", "c2"],+ "collection_id",+ ),# The subscription is one object, not a list, and must land as a single snapshot row.("subscription_single_object",@@ -370,6 +420,11 @@ class TestSourceResponse:("tool_executions", ["tool_id", "id"], "desc", None),("phone_numbers", ["phone_number_id"], "asc", None),("batch_calls", ["id"], "asc", "created_at_unix"),+ ("agent_deployments", ["agent_id", "id"], "desc", "deployed_at_unix_secs"),+ ("flow_templates", ["id"], "asc", None),+ # Runs from different templates interleave, and run ids are only documented per template.+ ("flow_template_runs", ["template_id", "id"], "desc", None),+ ("voice_collections", ["collection_id"], "asc", None),# A single snapshot row has no unique column to merge on; full refresh replaces it.("subscription", None, "asc", None),]