Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add recall ai usage table (#126975)

Tom Owersf7bb9f047efe
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 9c81d950f50c..4adcbaa5dae0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10081,7 +10081,7 @@ Today (7): `bots`, `calendar_events`, `calendars`, `meeting_metadata`, `particip
Diffed against: <https://docs.recall.ai/llms.txt>
- [x] `Desktop SDK uploads (GET /api/v1/sdk_upload/)` — resolve desktop_sdk_upload references on recordings and measure upload success or failure (high)
-- [ ] `Usage (GET /api/v1/billing/usage/)` — compare bot usage totals with recording activity and allocated costs (high)
+- [x] `Usage (GET /api/v1/billing/usage/)` — compare bot usage totals with recording activity and allocated costs (high)
- [x] `Meeting Direct Connects (GET /api/v1/meeting_direct_connect/)` — analyze connection status and recording activity for meetings captured without bots (medium)
Note: The official documentation index was fetched with curl, followed by the embedded OpenAPI definitions on the missing resource pages. The recording schema confirms desktop_sdk_upload references, and usage returns workspace bot totals for a requested period. Older calendar-user and calendar-meeting endpoints overlap the imported v2 calendar resources. Media downloads, OAuth credentials, and realtime delivery configuration are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.py
index 91a7338951ea..6b6a496025a8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/canonical_descriptions.py
@@ -128,4 +128,12 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "When the Meeting Direct Connect was created.",
},
},
+ "usage": {
+ "description": "Bot usage for the workspace, one row per UTC day. Recent days can change as bots finish and their usage is billed.",
+ "docs_url": "https://docs.recall.ai/reference/billing_usage_retrieve",
+ "columns": {
+ "date": "The UTC day the usage covers.",
+ "bot_total": "Total bot usage in seconds for the workspace on that day.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.py
index 1ecdca9645ad..11d71fd3bc1f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/recall_ai.py
@@ -1,9 +1,13 @@
-from datetime import UTC, date, datetime
+from collections.abc import Iterator
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
+ interruptible_wait,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
@@ -16,7 +20,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import RECALL_AI_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import (
+ RECALL_AI_ENDPOINTS,
+ USAGE_BACKFILL_DAYS,
+ USAGE_ENDPOINT,
+ USAGE_REQUEST_INTERVAL_SECONDS,
+ USAGE_RESTATEMENT_DAYS,
+)
# API keys are region-scoped and each region is its own host. The allowlist pins outbound
# traffic to *.recall.ai even if a stored config carries an unexpected region value.
@@ -25,7 +35,9 @@ REGIONS = ("us-east-1", "us-west-2", "eu-central-1", "ap-northeast-1")
@frozen
class RecallAIResumeConfig:
- next_url: str
+ next_url: Optional[str] = None
+ # The next UTC day the usage walk requests, as an ISO date.
+ next_date: Optional[str] = None
def base_url_for_region(region: str) -> str:
@@ -102,6 +114,70 @@ def get_resource(
}
+def _utc_today() -> date:
+ return datetime.now(UTC).date()
+
+
+def _to_date(value: Any) -> Optional[date]:
+ if isinstance(value, datetime):
+ return value.date()
+ if isinstance(value, date):
+ return value
+ if isinstance(value, str):
+ try:
+ return date.fromisoformat(value[:10])
+ except ValueError:
+ return None
+ return None
+
+
+def _usage_rows(
+ api_key: str,
+ region: str,
+ resumable_source_manager: ResumableSourceManager[RecallAIResumeConfig],
+ db_incremental_field_last_value: Optional[Any],
+) -> Iterator[dict[str, Any]]:
+ today = _utc_today()
+ day = today - timedelta(days=USAGE_BACKFILL_DAYS)
+ watermark = _to_date(db_incremental_field_last_value)
+ if watermark is not None:
+ day = watermark - timedelta(days=USAGE_RESTATEMENT_DAYS)
+
+ if resumable_source_manager.can_resume():
+ resume_config = resumable_source_manager.load_state()
+ resumed_day = _to_date(resume_config.next_date) if resume_config is not None else None
+ if resumed_day is not None and resumed_day > day:
+ day = resumed_day
+
+ url = f"{base_url_for_region(region)}{RECALL_AI_ENDPOINTS[USAGE_ENDPOINT].path}"
+ session = make_tracked_session(
+ headers={"Authorization": f"Token {api_key}", "Accept": "application/json"},
+ redact_values=(api_key,),
+ )
+
+ first_request = True
+ while day <= today:
+ if not first_request:
+ interruptible_wait(USAGE_REQUEST_INTERVAL_SECONDS, safe_point=resumable_source_manager.safe_point)
+ first_request = False
+
+ # Both bounds round up to UTC midnight, so [day, day + 1) covers exactly one UTC day.
+ next_day = day + timedelta(days=1)
+ response = session.get(
+ url,
+ params={"start": _to_iso8601(day), "end": _to_iso8601(next_day)},
+ timeout=60,
+ allow_redirects=False,
+ )
+ response.raise_for_status()
+ body = response.json()
+
+ if next_day <= today:
+ resumable_source_manager.save_state(RecallAIResumeConfig(next_date=next_day.isoformat()))
+ yield {"date": day.isoformat(), "bot_total": body.get("bot_total") if isinstance(body, dict) else None}
+ day = next_day
+
+
def recall_ai_source(
api_key: str,
region: str,
@@ -114,6 +190,25 @@ def recall_ai_source(
) -> SourceResponse:
endpoint_config = RECALL_AI_ENDPOINTS[endpoint]
+ if endpoint == USAGE_ENDPOINT:
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: _usage_rows(
+ api_key,
+ region,
+ resumable_source_manager,
+ db_incremental_field_last_value if should_use_incremental_field else None,
+ ),
+ primary_keys=["date"],
+ partition_count=1,
+ partition_size=1,
+ partition_mode="datetime",
+ partition_format="month",
+ partition_keys=[endpoint_config.partition_key],
+ # Days are requested oldest first, one ordered series per run.
+ sort_mode="asc",
+ )
+
config: RESTAPIConfig = {
"client": {
"base_url": base_url_for_region(region),
@@ -142,7 +237,7 @@ def recall_ai_source(
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume_config = resumable_source_manager.load_state()
- if resume_config is not None:
+ if resume_config is not None and resume_config.next_url:
initial_paginator_state = {"next_url": resume_config.next_url}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.py
index 7d2144686b33..c96c7482b7a1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/settings.py
@@ -18,8 +18,15 @@ class RecallAIEndpointConfig:
extra_params: tuple[tuple[str, str], ...] = ()
# False for endpoints that return a bare JSON array instead of a cursor-paginated envelope.
paginated: bool = True
+ incremental_field_type: IncrementalFieldType = IncrementalFieldType.DateTime
+USAGE_ENDPOINT = "usage"
+# The usage endpoint allows 5 requests per minute per workspace.
+USAGE_REQUEST_INTERVAL_SECONDS = 12.0
+USAGE_BACKFILL_DAYS = 90
+USAGE_RESTATEMENT_DAYS = 3
+
RECALL_AI_ENDPOINTS: dict[str, RecallAIEndpointConfig] = {
# The bot list only filters on join_at (date granularity), and join_at is the scheduled
# join time: a bot scheduled for next week advances a join_at watermark past bots created
@@ -85,6 +92,17 @@ RECALL_AI_ENDPOINTS: dict[str, RecallAIEndpointConfig] = {
scrub_fields=("zoom_rtms.signature", "google_meet_media_api.access_token"),
paginated=False,
),
+ # One row per UTC day, built from one request per day. The endpoint only returns a bot_total
+ # for the requested range, so `date` and `bot_total` are the whole row. Recent days restate
+ # as late bot completions are billed, so incremental syncs re-read a trailing window.
+ USAGE_ENDPOINT: RecallAIEndpointConfig(
+ path="/api/v1/billing/usage/",
+ partition_key="date",
+ incremental_param="start",
+ incremental_field="date",
+ paginated=False,
+ incremental_field_type=IncrementalFieldType.Date,
+ ),
}
ENDPOINTS = tuple(RECALL_AI_ENDPOINTS.keys())
@@ -94,9 +112,9 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
[
{
"label": config.incremental_field,
- "type": IncrementalFieldType.DateTime,
+ "type": config.incremental_field_type,
"field": config.incremental_field,
- "field_type": IncrementalFieldType.DateTime,
+ "field_type": config.incremental_field_type,
}
]
if config.incremental_param and config.incremental_field
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.py
index 7f6aafbdfdf9..6dcec0b2f1e7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/source.py
@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.
from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ USAGE_ENDPOINT,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -73,7 +74,8 @@ class RecallAISource(ResumableSource[RecallAISourceConfig, RecallAIResumeConfig]
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ # Usage re-reads restated days, which only a merge on `date` keeps unique.
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=(USAGE_ENDPOINT,))
def validate_credentials(
self,
@@ -125,7 +127,7 @@ class RecallAISource(ResumableSource[RecallAISourceConfig, RecallAIResumeConfig]
name=ExternalDataSourceType.RECALLAI,
category=DataWarehouseSourceCategory.COMMUNICATION,
label="Recall.ai",
- caption="Sync meeting bots, recordings, transcripts, participant events, and calendar data from Recall.ai.",
+ caption="Sync meeting bots, recordings, transcripts, participant events, calendar data, and bot usage from Recall.ai.",
docsUrl="https://posthog.com/docs/cdp/sources/recall-ai",
iconPath="/static/services/recall_ai.png",
keywords=["meetings", "call recording", "transcripts", "bots"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.py
index ef4945a2bdfc..808d20c40f36 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recall_ai/tests/test_recall_ai.py
@@ -2,6 +2,7 @@ from datetime import UTC, date, datetime
from typing import Any
import pytest
+from unittest import mock
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
@@ -17,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.
_to_iso8601,
base_url_for_region,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.settings import USAGE_BACKFILL_DAYS
from products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.source import RecallAISource
BASE_URL = "https://us-east-1.recall.ai"
@@ -179,6 +181,51 @@ class TestRecallAICredentialScrubbing:
assert result.rows == expected_rows
+class TestRecallAIUsage:
+ TODAY = date(2026, 3, 10)
+
+ def _drive_usage(self, days: int, **inputs: Any) -> DriveResult:
+ script = [_page({"bot_total": float(index * 60)}) for index in range(days)]
+ with mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.recall_ai.recall_ai._utc_today",
+ return_value=self.TODAY,
+ ):
+ return _drive("usage", script, **inputs)
+
+ def test_incremental_run_rereads_restated_days_and_requests_one_utc_day_each(self) -> None:
+ result = self._drive_usage(5, incremental_field="date", db_incremental_field_last_value=date(2026, 3, 9))
+
+ assert result.params("start") == [
+ "2026-03-06T00:00:00Z",
+ "2026-03-07T00:00:00Z",
+ "2026-03-08T00:00:00Z",
+ "2026-03-09T00:00:00Z",
+ "2026-03-10T00:00:00Z",
+ ]
+ assert result.params("end") == [
+ "2026-03-07T00:00:00Z",
+ "2026-03-08T00:00:00Z",
+ "2026-03-09T00:00:00Z",
+ "2026-03-10T00:00:00Z",
+ "2026-03-11T00:00:00Z",
+ ]
+ assert result.rows[0] == {"date": "2026-03-06", "bot_total": 0.0}
+ assert result.rows[-1] == {"date": "2026-03-10", "bot_total": 240.0}
+ assert result.saved_states[-1] == RecallAIResumeConfig(next_date="2026-03-10")
+ assert result.response is not None and result.response.sort_mode == "asc"
+
+ def test_first_run_backfills_a_bounded_window(self) -> None:
+ result = self._drive_usage(USAGE_BACKFILL_DAYS + 1)
+
+ assert len(result.requests) == USAGE_BACKFILL_DAYS + 1
+ assert result.rows[-1]["date"] == "2026-03-10"
+
+ def test_resume_starts_at_saved_day(self) -> None:
+ result = self._drive_usage(2, resume_state=RecallAIResumeConfig(next_date="2026-03-09"))
+
+ assert [row["date"] for row in result.rows] == ["2026-03-09", "2026-03-10"]
+
+
class TestRecallAIHelpers:
@pytest.mark.parametrize(
("value", "expected"),