:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add recall ai usage table (#126975)
Tom Owersf7bb9f047efe
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 9c81d950f50c..4adcbaa5dae0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10081,7 +10081,7 @@ Today (7): `bots`, `calendar_events`, `calendars`, `meeting_metadata`, `participDiffed against: <https://docs.recall.ai/llms.txt>- [x] `Desktop SDK uploads (GET /api/v1/sdk_upload/)` — resolve desktop_sdk_upload references on recordings and measure upload success or failure (high)-- [ ] `Usage (GET /api/v1/billing/usage/)` — compare bot usage totals with recording activity and allocated costs (high)+- [x] `Usage (GET /api/v1/billing/usage/)` — compare bot usage totals with recording activity and allocated costs (high)- [x] `Meeting Direct Connects (GET /api/v1/meeting_direct_connect/)` — analyze connection status and recording activity for meetings captured without bots (medium)Note: The official documentation index was fetched with curl, followed by the embedded OpenAPI definitions on the missing resource pages. The recording schema confirms desktop_sdk_upload references, and usage returns workspace bot totals for a requested period. Older calendar-user and calendar-meeting endpoints overlap the imported v2 calendar resources. Media downloads, OAuth credentials, and realtime delivery configuration are excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.pyindex 91a7338951ea..6b6a496025a8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.py@@ -128,4 +128,12 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "When the Meeting Direct Connect was created.",},},+ "usage": {+ "description": "Bot usage for the workspace, one row per UTC day. Recent days can change as bots finish and their usage is billed.",+ "docs_url": "https://docs.recall.ai/reference/billing_usage_retrieve",+ "columns": {+ "date": "The UTC day the usage covers.",+ "bot_total": "Total bot usage in seconds for the workspace on that day.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.pyindex 1ecdca9645ad..11d71fd3bc1f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.py@@ -1,9 +1,13 @@-from datetime import UTC, date, datetime+from collections.abc import Iterator+from datetime import UTC, date, datetime, timedeltafrom typing import Any, Optionalfrom posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,@@ -16,7 +20,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import RECALL_AI_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import (+ RECALL_AI_ENDPOINTS,+ USAGE_BACKFILL_DAYS,+ USAGE_ENDPOINT,+ USAGE_REQUEST_INTERVAL_SECONDS,+ USAGE_RESTATEMENT_DAYS,+)# API keys are region-scoped and each region is its own host. The allowlist pins outbound# traffic to *.recall.ai even if a stored config carries an unexpected region value.@@ -25,7 +35,9 @@ REGIONS = ("us-east-1", "us-west-2", "eu-central-1", "ap-northeast-1")@frozenclass RecallAIResumeConfig:- next_url: str+ next_url: Optional[str] = None+ # The next UTC day the usage walk requests, as an ISO date.+ next_date: Optional[str] = Nonedef base_url_for_region(region: str) -> str:@@ -102,6 +114,70 @@ def get_resource(}+def _utc_today() -> date:+ return datetime.now(UTC).date()+++def _to_date(value: Any) -> Optional[date]:+ if isinstance(value, datetime):+ return value.date()+ if isinstance(value, date):+ return value+ if isinstance(value, str):+ try:+ return date.fromisoformat(value[:10])+ except ValueError:+ return None+ return None+++def _usage_rows(+ api_key: str,+ region: str,+ resumable_source_manager: ResumableSourceManager[RecallAIResumeConfig],+ db_incremental_field_last_value: Optional[Any],+) -> Iterator[dict[str, Any]]:+ today = _utc_today()+ day = today - timedelta(days=USAGE_BACKFILL_DAYS)+ watermark = _to_date(db_incremental_field_last_value)+ if watermark is not None:+ day = watermark - timedelta(days=USAGE_RESTATEMENT_DAYS)++ if resumable_source_manager.can_resume():+ resume_config = resumable_source_manager.load_state()+ resumed_day = _to_date(resume_config.next_date) if resume_config is not None else None+ if resumed_day is not None and resumed_day > day:+ day = resumed_day++ url = f"{base_url_for_region(region)}{RECALL_AI_ENDPOINTS[USAGE_ENDPOINT].path}"+ session = make_tracked_session(+ headers={"Authorization": f"Token {api_key}", "Accept": "application/json"},+ redact_values=(api_key,),+ )++ first_request = True+ while day <= today:+ if not first_request:+ interruptible_wait(USAGE_REQUEST_INTERVAL_SECONDS, safe_point=resumable_source_manager.safe_point)+ first_request = False++ # Both bounds round up to UTC midnight, so [day, day + 1) covers exactly one UTC day.+ next_day = day + timedelta(days=1)+ response = session.get(+ url,+ params={"start": _to_iso8601(day), "end": _to_iso8601(next_day)},+ timeout=60,+ allow_redirects=False,+ )+ response.raise_for_status()+ body = response.json()++ if next_day <= today:+ resumable_source_manager.save_state(RecallAIResumeConfig(next_date=next_day.isoformat()))+ yield {"date": day.isoformat(), "bot_total": body.get("bot_total") if isinstance(body, dict) else None}+ day = next_day++def recall_ai_source(api_key: str,region: str,@@ -114,6 +190,25 @@ def recall_ai_source() -> SourceResponse:endpoint_config = RECALL_AI_ENDPOINTS[endpoint]+ if endpoint == USAGE_ENDPOINT:+ return SourceResponse(+ name=endpoint,+ items=lambda: _usage_rows(+ api_key,+ region,+ resumable_source_manager,+ db_incremental_field_last_value if should_use_incremental_field else None,+ ),+ primary_keys=["date"],+ partition_count=1,+ partition_size=1,+ partition_mode="datetime",+ partition_format="month",+ partition_keys=[endpoint_config.partition_key],+ # Days are requested oldest first, one ordered series per run.+ sort_mode="asc",+ )+config: RESTAPIConfig = {"client": {"base_url": base_url_for_region(region),@@ -142,7 +237,7 @@ def recall_ai_source(initial_paginator_state: Optional[dict[str, Any]] = Noneif resumable_source_manager.can_resume():resume_config = resumable_source_manager.load_state()- if resume_config is not None:+ if resume_config is not None and resume_config.next_url:initial_paginator_state = {"next_url": resume_config.next_url}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.pyindex 7d2144686b33..c96c7482b7a1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.py@@ -18,8 +18,15 @@ class RecallAIEndpointConfig:extra_params: tuple[tuple[str, str], ...] = ()# False for endpoints that return a bare JSON array instead of a cursor-paginated envelope.paginated: bool = True+ incremental_field_type: IncrementalFieldType = IncrementalFieldType.DateTime+USAGE_ENDPOINT = "usage"+# The usage endpoint allows 5 requests per minute per workspace.+USAGE_REQUEST_INTERVAL_SECONDS = 12.0+USAGE_BACKFILL_DAYS = 90+USAGE_RESTATEMENT_DAYS = 3+RECALL_AI_ENDPOINTS: dict[str, RecallAIEndpointConfig] = {# The bot list only filters on join_at (date granularity), and join_at is the scheduled# join time: a bot scheduled for next week advances a join_at watermark past bots created@@ -85,6 +92,17 @@ RECALL_AI_ENDPOINTS: dict[str, RecallAIEndpointConfig] = {scrub_fields=("zoom_rtms.signature", "google_meet_media_api.access_token"),paginated=False,),+ # One row per UTC day, built from one request per day. The endpoint only returns a bot_total+ # for the requested range, so `date` and `bot_total` are the whole row. Recent days restate+ # as late bot completions are billed, so incremental syncs re-read a trailing window.+ USAGE_ENDPOINT: RecallAIEndpointConfig(+ path="/api/v1/billing/usage/",+ partition_key="date",+ incremental_param="start",+ incremental_field="date",+ paginated=False,+ incremental_field_type=IncrementalFieldType.Date,+ ),}ENDPOINTS = tuple(RECALL_AI_ENDPOINTS.keys())@@ -94,9 +112,9 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {[{"label": config.incremental_field,- "type": IncrementalFieldType.DateTime,+ "type": config.incremental_field_type,"field": config.incremental_field,- "field_type": IncrementalFieldType.DateTime,+ "field_type": config.incremental_field_type,}]if config.incremental_param and config.incremental_fielddiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.pyindex 7f6aafbdfdf9..6dcec0b2f1e7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.py@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import (ENDPOINTS,INCREMENTAL_FIELDS,+ USAGE_ENDPOINT,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -73,7 +74,8 @@ class RecallAISource(ResumableSource[RecallAISourceConfig, RecallAIResumeConfig]force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)+ # Usage re-reads restated days, which only a merge on `date` keeps unique.+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=(USAGE_ENDPOINT,))def validate_credentials(self,@@ -125,7 +127,7 @@ class RecallAISource(ResumableSource[RecallAISourceConfig, RecallAIResumeConfig]name=ExternalDataSourceType.RECALLAI,category=DataWarehouseSourceCategory.COMMUNICATION,label="Recall.ai",- caption="Sync meeting bots, recordings, transcripts, participant events, and calendar data from Recall.ai.",+ caption="Sync meeting bots, recordings, transcripts, participant events, calendar data, and bot usage from Recall.ai.",docsUrl="https://posthog.com/docs/cdp/sources/recall-ai",iconPath="/static/services/recall_ai.png",keywords=["meetings", "call recording", "transcripts", "bots"],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.pyindex ef4945a2bdfc..808d20c40f36 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.py@@ -2,6 +2,7 @@ from datetime import UTC, date, datetimefrom typing import Anyimport pytest+from unittest import mockfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (DriveResult,@@ -17,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai._to_iso8601,base_url_for_region,)+from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import USAGE_BACKFILL_DAYSfrom products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.source import RecallAISourceBASE_URL = "https://us-east-1.recall.ai"@@ -179,6 +181,51 @@ class TestRecallAICredentialScrubbing:assert result.rows == expected_rows+class TestRecallAIUsage:+ TODAY = date(2026, 3, 10)++ def _drive_usage(self, days: int, **inputs: Any) -> DriveResult:+ script = [_page({"bot_total": float(index * 60)}) for index in range(days)]+ with mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.recall_ai._utc_today",+ return_value=self.TODAY,+ ):+ return _drive("usage", script, **inputs)++ def test_incremental_run_rereads_restated_days_and_requests_one_utc_day_each(self) -> None:+ result = self._drive_usage(5, incremental_field="date", db_incremental_field_last_value=date(2026, 3, 9))++ assert result.params("start") == [+ "2026-03-06T00:00:00Z",+ "2026-03-07T00:00:00Z",+ "2026-03-08T00:00:00Z",+ "2026-03-09T00:00:00Z",+ "2026-03-10T00:00:00Z",+ ]+ assert result.params("end") == [+ "2026-03-07T00:00:00Z",+ "2026-03-08T00:00:00Z",+ "2026-03-09T00:00:00Z",+ "2026-03-10T00:00:00Z",+ "2026-03-11T00:00:00Z",+ ]+ assert result.rows[0] == {"date": "2026-03-06", "bot_total": 0.0}+ assert result.rows[-1] == {"date": "2026-03-10", "bot_total": 240.0}+ assert result.saved_states[-1] == RecallAIResumeConfig(next_date="2026-03-10")+ assert result.response is not None and result.response.sort_mode == "asc"++ def test_first_run_backfills_a_bounded_window(self) -> None:+ result = self._drive_usage(USAGE_BACKFILL_DAYS + 1)++ assert len(result.requests) == USAGE_BACKFILL_DAYS + 1+ assert result.rows[-1]["date"] == "2026-03-10"++ def test_resume_starts_at_saved_day(self) -> None:+ result = self._drive_usage(2, resume_state=RecallAIResumeConfig(next_date="2026-03-09"))++ assert [row["date"] for row in result.rows] == ["2026-03-09", "2026-03-10"]++class TestRecallAIHelpers:@pytest.mark.parametrize(("value", "expected"),