:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Zoom source (#124349)
Tom Owers0120b78245ed
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 89ca409c83b9..c74d904e7d80 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1181,7 +1181,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/zep/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/zep/zep.py1 products/warehouse_sources/backend/temporal/data_imports/sources/zoho_crm/zoho_crm.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/zoom/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/zoom/zoom.py1 products/warehouse_sources/backend/temporal/data_imports/sources/zuora/zuora.py1 products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_google_ads_source.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 82307369cf23..8875fa0ac1e7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -13679,10 +13679,10 @@ Today (3): `meetings`, `users`, `webinars`Diffed against: <https://developers.zoom.us/api-hub/meetings/methods/endpoints.json>-- [ ] `/report/meetings/{meetingId}/participants (and /past_meetings/{meetingId}/participants)` — per-attendee join/leave records - the core meeting-attendance fact table, and nothing in the current 3 tables carries attendance (high)-- [ ] `/report/webinars/{webinarId}/participants (and /past_webinars/{webinarId}/participants)` — webinar attendance rows; webinars are synced but have zero attendance data (high)-- [ ] `/webinars/{webinarId}/registrants` — registration records with approval status, the denominator for webinar show-up rate (high)-- [ ] `/meetings/{meetingId}/registrants` — registration records for registration-enabled meetings we already sync (high)+- [x] `/report/meetings/{meetingId}/participants (and /past_meetings/{meetingId}/participants)` — per-attendee join/leave records - the core meeting-attendance fact table, and nothing in the current 3 tables carries attendance (high)+- [x] `/report/webinars/{webinarId}/participants (and /past_webinars/{webinarId}/participants)` — webinar attendance rows; webinars are synced but have zero attendance data (high)+- [x] `/webinars/{webinarId}/registrants` — registration records with approval status, the denominator for webinar show-up rate (high)+- [x] `/meetings/{meetingId}/registrants` — registration records for registration-enabled meetings we already sync (high)- [ ] `/past_meetings/{meetingId}/instances (and /past_webinars/{webinarId}/instances)` — lookup resolving each recurring meeting/webinar ID we sync to its per-occurrence UUIDs, which every report endpoint keys on (high)- [ ] `/groups and /groups/{groupId}/members` — lookup table resolving the group IDs carried on the users rows we already sync (high)- [ ] `/report/users` — active/inactive host report with per-user meeting counts, minutes and participant totals - Zoom's headline usage metric (high)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/canonical_descriptions.pyindex e8d11eeadc38..66285fb61fa9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/canonical_descriptions.py@@ -64,4 +64,72 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "Time the webinar was created.",},},+ "meeting_registrants": {+ "description": "An approved registration for a registration-enabled meeting.",+ "docs_url": "https://developers.zoom.us/docs/api/meetings/#tag/meetings/GET/meetings/{meetingId}/registrants",+ "columns": {+ "meeting_id": "Numeric identifier of the meeting the registration is for.",+ "id": "Unique identifier of the registrant.",+ "email": "The registrant's email address.",+ "first_name": "The registrant's first name.",+ "last_name": "The registrant's last name.",+ "status": "The registrant's status: approved, pending, or denied. Only approved registrants are synced.",+ "create_time": "Time the registrant registered.",+ "join_url": "URL an approved registrant uses to join.",+ "custom_questions": "The registrant's answers to the custom registration questions.",+ },+ },+ "webinar_registrants": {+ "description": "An approved registration for a webinar.",+ "docs_url": "https://developers.zoom.us/docs/api/meetings/#tag/webinars/GET/webinars/{webinarId}/registrants",+ "columns": {+ "webinar_id": "Numeric identifier of the webinar the registration is for.",+ "id": "Unique identifier of the registrant.",+ "email": "The registrant's email address.",+ "first_name": "The registrant's first name.",+ "last_name": "The registrant's last name.",+ "status": "The registrant's status: approved, pending, or denied. Only approved registrants are synced.",+ "create_time": "Time the registrant registered.",+ "join_url": "URL an approved registrant uses to join.",+ "custom_questions": "The registrant's answers to the custom registration questions.",+ },+ },+ "meeting_participants": {+ "description": "One join session of a participant in an ended meeting instance. A participant who rejoins has one row per join.",+ "docs_url": "https://developers.zoom.us/docs/api/meetings/#tag/meetings/GET/past_meetings/{meetingId}/participants",+ "columns": {+ "meeting_id": "Numeric identifier of the meeting.",+ "meeting_uuid": "Unique identifier of the meeting instance the participant attended.",+ "id": "The participant's universally unique identifier. Empty for participants who are not signed in to Zoom.",+ "user_id": "Participant ID that Zoom assigns when the participant joins. Valid only for this instance.",+ "name": "The participant's display name.",+ "user_email": "The participant's email address.",+ "registrant_id": "The participant's registrant ID, when they registered.",+ "join_time": "Time the participant joined.",+ "leave_time": "Time the participant left.",+ "duration": "Time the participant spent in this session, in seconds.",+ "failover": "Whether a failover occurred during the session.",+ "status": "The participant's status: in_meeting or in_waiting_room.",+ "internal_user": "Whether the participant is a user in the host's account.",+ },+ },+ "webinar_participants": {+ "description": "One join session of a participant in an ended webinar instance. A participant who rejoins has one row per join.",+ "docs_url": "https://developers.zoom.us/docs/api/meetings/#tag/webinars/GET/past_webinars/{webinarId}/participants",+ "columns": {+ "webinar_id": "Numeric identifier of the webinar.",+ "webinar_uuid": "Unique identifier of the webinar instance the participant attended.",+ "id": "The participant's universally unique identifier. Empty for participants who are not signed in to Zoom.",+ "user_id": "Participant ID that Zoom assigns when the participant joins. Valid only for this instance.",+ "name": "The participant's display name.",+ "user_email": "The participant's email address.",+ "registrant_id": "The participant's registrant ID, when they registered.",+ "join_time": "Time the participant joined.",+ "leave_time": "Time the participant left.",+ "duration": "Time the participant spent in this session, in seconds.",+ "failover": "Whether a failover occurred during the session.",+ "status": "The participant's status: in_meeting or in_waiting_room.",+ "internal_user": "Whether the participant is a user in the host's account.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/settings.pyindex 6c8c020ce198..78753394f3b3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/settings.py@@ -1,29 +1,39 @@-from dataclasses import dataclass, field+from dataclasses import fieldfrom typing import Optional+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.types import IncrementalField-@dataclass+@frozenclass ZoomEndpointConfig:name: str- # Path template relative to the Zoom API base. Fan-out endpoints contain a- # ``{user_id}`` placeholder that is filled in per user (see ``fan_out``).+ # Path template relative to the Zoom API base. Fan-out endpoints contain one placeholder+ # (``parent_param``) that is filled in per row of the ``parent`` endpoint.path: str# Key in the JSON response body that holds the list of records# (e.g. ``{"users": [...], "next_page_token": "..."}``).data_key: str- primary_key: str = "id"+ primary_keys: list[str] = field(default_factory=lambda: ["id"])# A STABLE datetime field used for partitioning. Never use a field that# mutates over time (e.g. ``updated_at``) — partitions would be rewritten# on every sync.partition_key: Optional[str] = "created_at"- # When True the endpoint is queried once per user returned by ``/users``.- fan_out: bool = False+ # Name of the endpoint (in ``ZOOM_ENDPOINTS`` or ``ZOOM_PARENT_ENDPOINTS``) whose rows fill the+ # path placeholder. Parents can chain, e.g. participants <- meeting instances <- meetings <- users.+ parent: Optional[str] = None+ parent_param: str = ""+ parent_field: str = "id"+ # Parent row fields copied onto each row, as parent field -> column name.+ parent_columns: dict[str, str] = field(default_factory=dict)# Static query params sent on every request to this endpoint.params: dict[str, str] = field(default_factory=dict)# Zoom caps ``page_size`` at 300 for these list endpoints.page_size: int = 300+ paginated: bool = True+ # Adds ``_uuid_path``: the row's ``uuid`` encoded the way Zoom requires in a path segment.+ adds_uuid_path: bool = False# Zoom's list endpoints expose no server-side timestamp filter (no ``since`` /@@ -35,16 +45,15 @@ ZOOM_ENDPOINTS: dict[str, ZoomEndpointConfig] = {name="users",path="/users",data_key="users",- primary_key="id",partition_key="created_at",),"meetings": ZoomEndpointConfig(name="meetings",path="/users/{user_id}/meetings",data_key="meetings",- primary_key="id",partition_key="created_at",- fan_out=True,+ parent="users",+ parent_param="user_id",# ``scheduled`` returns all of a user's stored scheduled meetings,# including recurring ones — the most complete list for ingestion.params={"type": "scheduled"},@@ -53,9 +62,78 @@ ZOOM_ENDPOINTS: dict[str, ZoomEndpointConfig] = {name="webinars",path="/users/{user_id}/webinars",data_key="webinars",- primary_key="id",partition_key="created_at",- fan_out=True,+ parent="users",+ parent_param="user_id",+ ),+ "meeting_registrants": ZoomEndpointConfig(+ name="meeting_registrants",+ path="/meetings/{meeting_id}/registrants",+ data_key="registrants",+ # Registrant IDs are only documented as unique within a meeting.+ primary_keys=["meeting_id", "id"],+ partition_key="create_time",+ parent="meetings",+ parent_param="meeting_id",+ parent_columns={"id": "meeting_id"},+ ),+ "webinar_registrants": ZoomEndpointConfig(+ name="webinar_registrants",+ path="/webinars/{webinar_id}/registrants",+ data_key="registrants",+ primary_keys=["webinar_id", "id"],+ partition_key="create_time",+ parent="webinars",+ parent_param="webinar_id",+ parent_columns={"id": "webinar_id"},+ ),+ "meeting_participants": ZoomEndpointConfig(+ name="meeting_participants",+ path="/past_meetings/{meeting_uuid}/participants",+ data_key="participants",+ # A participant gets one row per join, so a rejoin shares the participant IDs.+ primary_keys=["meeting_uuid", "user_id", "join_time"],+ partition_key="join_time",+ parent="meeting_instances",+ parent_param="meeting_uuid",+ parent_field="_uuid_path",+ parent_columns={"meeting_id": "meeting_id", "uuid": "meeting_uuid"},+ ),+ "webinar_participants": ZoomEndpointConfig(+ name="webinar_participants",+ path="/past_webinars/{webinar_uuid}/participants",+ data_key="participants",+ primary_keys=["webinar_uuid", "user_id", "join_time"],+ partition_key="join_time",+ parent="webinar_instances",+ parent_param="webinar_uuid",+ parent_field="_uuid_path",+ parent_columns={"webinar_id": "webinar_id", "uuid": "webinar_uuid"},+ ),+}++# Intermediate fan-out levels that are not synced as tables of their own. A meeting ID only+# resolves to its latest instance, so participants are read per ended instance (UUID) instead.+ZOOM_PARENT_ENDPOINTS: dict[str, ZoomEndpointConfig] = {+ "meeting_instances": ZoomEndpointConfig(+ name="meeting_instances",+ path="/past_meetings/{meeting_id}/instances",+ data_key="meetings",+ parent="meetings",+ parent_param="meeting_id",+ parent_columns={"id": "meeting_id"},+ paginated=False,+ adds_uuid_path=True,+ ),+ "webinar_instances": ZoomEndpointConfig(+ name="webinar_instances",+ path="/past_webinars/{webinar_id}/instances",+ data_key="webinars",+ parent="webinars",+ parent_param="webinar_id",+ parent_columns={"id": "webinar_id"},+ paginated=False,+ adds_uuid_path=True,),}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.pyindex 8d0a8ead4442..be156ab4df93 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.py@@ -23,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_from products.warehouse_sources.backend.temporal.data_imports.sources.zoom.settings import ENDPOINTS, INCREMENTAL_FIELDSfrom products.warehouse_sources.backend.temporal.data_imports.sources.zoom.zoom import (ZoomResumeConfig,+ supports_resume,validate_credentials as validate_zoom_credentials,zoom_source,)@@ -156,6 +157,9 @@ class ZoomSource(ResumableSource[ZoomSourceConfig, ZoomResumeConfig]):schema_name=schema_name,)+ def resume_covers_run(self, *, incremental_or_append: bool, schema_name: str | None = None) -> bool:+ return schema_name is None or schema_name not in ENDPOINTS or supports_resume(schema_name)+def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[ZoomResumeConfig]:return ResumableSourceManager[ZoomResumeConfig](inputs, ZoomResumeConfig)@@ -182,7 +186,7 @@ class ZoomSource(ResumableSource[ZoomSourceConfig, ZoomResumeConfig]):category=DataWarehouseSourceCategory.COMMUNICATION,label="Zoom",releaseStatus=ReleaseStatus.GA,- caption="""Sync your Zoom users, meetings, and webinars into the PostHog Data warehouse.+ caption="""Sync your Zoom users, meetings, webinars, registrants, and attendance into the PostHog Data warehouse.Create a **Server-to-Server OAuth** app in the [Zoom App Marketplace](https://marketplace.zoom.us/develop/create) and copy its Account ID, Client ID, and Client Secret below.@@ -190,6 +194,8 @@ Grant the following read scopes:- `user:read:list_users:admin`- `meeting:read:list_meetings:admin`- `webinar:read:list_webinars:admin`+- `meeting:read:list_registrants:admin` and `webinar:read:list_registrants:admin` for registrants+- `meeting:read:list_past_instances:admin`, `meeting:read:list_past_participants:admin`, `webinar:read:list_past_instances:admin` and `webinar:read:list_past_participants:admin` for participants""",iconPath="/static/services/zoom.png",docsUrl="https://posthog.com/docs/cdp/sources/zoom",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/tests/test_zoom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/tests/test_zoom.pyindex a52b1973848b..bb093be7e1e9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/tests/test_zoom.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/tests/test_zoom.py@@ -1,7 +1,9 @@from typing import Anyimport pytest+from unittest.mock import patch+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import RESTClientRetryableErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (RecordedRequest,ScriptedResponse,@@ -102,8 +104,150 @@ class TestFanOutRows:assert "/v2/users/u1/meetings" not in result.pathsassert "/v2/users/u2/meetings" in result.paths+ @pytest.mark.parametrize(+ "endpoint,parent_key,parent_path,instances_path,participants_root,parent_column",+ [+ (+ "meeting_participants",+ "meetings",+ "/users/u1/meetings",+ "/past_meetings/{id}/instances",+ "/past_meetings",+ "meeting",+ ),+ (+ "webinar_participants",+ "webinars",+ "/users/u1/webinars",+ "/past_webinars/{id}/instances",+ "/past_webinars",+ "webinar",+ ),+ ],+ )+ @pytest.mark.parametrize(+ "uuid,encoded",+ [+ ("abcDEF12==", "abcDEF12%3D%3D"),+ ("ab/c+d==", "ab%2Fc%2Bd%3D%3D"),+ ("/abcd==", "%252Fabcd%253D%253D"),+ ("ab//cd==", "ab%252F%252Fcd%253D%253D"),+ ],+ )+ def test_participants_read_each_ended_instance_by_encoded_uuid(+ self,+ endpoint: str,+ parent_key: str,+ parent_path: str,+ instances_path: str,+ participants_root: str,+ parent_column: str,+ uuid: str,+ encoded: str,+ ) -> None:+ participant = {"id": "p1", "user_id": "16778240", "join_time": "2026-01-05T10:00:00Z"}+ responses = {+ "/oauth/token": _TOKEN,+ "/v2/users": ScriptedResponse(json={"users": [{"id": "u1"}], "next_page_token": ""}),+ f"/v2{parent_path}": ScriptedResponse(json={parent_key: [{"id": 111}, {"id": 222}], "next_page_token": ""}),+ # 222 never took place: Zoom answers 404 and that parent is skipped.+ "/v2" + instances_path.format(id=111): ScriptedResponse(+ json={parent_key: [{"uuid": uuid, "start_time": "2026-01-05T10:00:00Z"}]}+ ),+ "/v2" + instances_path.format(id=222): ScriptedResponse(status=404, json={"code": 3001}),+ f"/v2{participants_root}/{encoded}/participants": ScriptedResponse(+ json={"participants": [participant], "next_page_token": ""}+ ),+ }++ def respond(request: RecordedRequest) -> ScriptedResponse:+ assert request.path in responses, f"unexpected request {request.path}"+ return responses[request.path]++ result = _driver().run(endpoint, respond)++ assert result.rows == [{**participant, f"{parent_column}_id": 111, f"{parent_column}_uuid": uuid}]++ @pytest.mark.parametrize(+ "endpoint,parent_key,parent_path,registrants_root,parent_column",+ [+ ("meeting_registrants", "meetings", "/users/u1/meetings", "/meetings", "meeting_id"),+ ("webinar_registrants", "webinars", "/users/u1/webinars", "/webinars", "webinar_id"),+ ],+ )+ def test_registrants_carry_parent_id_and_skip_unreadable_parent(+ self, endpoint: str, parent_key: str, parent_path: str, registrants_root: str, parent_column: str+ ) -> None:+ result = _driver().run(+ endpoint,+ route(+ {+ "/oauth/token": [_TOKEN],+ "/v2/users": [ScriptedResponse(json={"users": [{"id": "u1"}], "next_page_token": ""})],+ parent_path: [+ ScriptedResponse(json={parent_key: [{"id": 111}, {"id": 222}], "next_page_token": ""})+ ],+ f"{registrants_root}/111/registrants": [+ ScriptedResponse(status=400, json={"code": 200, "message": "No permission."})+ ],+ f"{registrants_root}/222/registrants": [+ ScriptedResponse(json={"registrants": [{"id": "r1"}], "next_page_token": "t1"}),+ ScriptedResponse(json={"registrants": [{"id": "r2"}], "next_page_token": ""}),+ ],+ }+ ),+ )++ assert result.rows == [{"id": "r1", parent_column: 222}, {"id": "r2", parent_column: 222}]+ registrant_requests = [r for r in result.requests if r.path == f"/v2{registrants_root}/222/registrants"]+ assert [r.param("next_page_token") for r in registrant_requests] == [None, "t1"]++ def test_expired_page_token_fails_instead_of_truncating(self) -> None:+ responses = {+ "/oauth/token": _TOKEN,+ "/v2/users": ScriptedResponse(json={"users": [{"id": "u1"}], "next_page_token": ""}),+ "/v2/users/u1/meetings": ScriptedResponse(json={"meetings": [{"id": 111}], "next_page_token": ""}),+ }++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/v2/meetings/111/registrants":+ if request.param("next_page_token") is None:+ return ScriptedResponse(json={"registrants": [{"id": "r1"}], "next_page_token": "t1"})+ return ScriptedResponse(status=400, json={"code": 300, "message": "next_page_token expired"})+ return responses[request.path]++ with patch("time.sleep"):+ result = _driver().run("meeting_registrants", respond)++ assert isinstance(result.raised, RESTClientRetryableError)+class TestZoomSourceResponse:+ @pytest.mark.parametrize(+ "endpoint,resumable",+ [+ ("users", True),+ ("meetings", True),+ ("webinars", True),+ ("meeting_registrants", False),+ ("webinar_registrants", False),+ ("meeting_participants", False),+ ("webinar_participants", False),+ ],+ )+ def test_chained_tables_are_not_resumable(self, endpoint: str, resumable: bool) -> None:+ assert ZoomSource().resume_covers_run(incremental_or_append=False, schema_name=endpoint) is resumable++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/oauth/token":+ return _TOKEN+ return ScriptedResponse(json={"users": [], "next_page_token": ""})++ result = _driver().run(endpoint, respond)++ assert result.response is not None+ assert result.response.supports_resume is resumable+@pytest.mark.parametrize("endpoint", ["users", "meetings", "webinars"])def test_primary_keys_per_endpoint(self, endpoint: str) -> None:def respond(request: RecordedRequest) -> ScriptedResponse:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/zoom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/zoom.pyindex 6e379c550a65..82c1534081b8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/zoom.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/zoom.py@@ -1,5 +1,7 @@import dataclasses+from collections.abc import Callablefrom typing import Any, Optional, cast+from urllib.parse import quotefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (@@ -11,16 +13,24 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resOAuth2Auth,OAuth2GrantType,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (+ make_parent_key_name,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponseCursorPaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ Endpoint,+ EndpointResource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.zoom.settings import (ZOOM_ENDPOINTS,+ ZOOM_PARENT_ENDPOINTS,ZoomEndpointConfig,)@@ -113,6 +123,85 @@ def _top_level_resource()+def _encode_uuid(uuid: str) -> str:+ # Zoom UUIDs are base64 and can hold `/`, `+` and `=`. Zoom asks for a double-encoded UUID when it+ # starts with `/` or contains `//`, and a single-encoded one otherwise.+ encoded = quote(uuid, safe="")+ if uuid.startswith("/") or "//" in uuid:+ encoded = quote(encoded, safe="")+ return encoded+++def _row_map(config: ZoomEndpointConfig) -> Optional[Callable[[dict[str, Any]], dict[str, Any]]]:+ parent = config.parent+ renames = (+ {make_parent_key_name(parent, field): column for field, column in config.parent_columns.items()}+ if parent is not None+ else {}+ )+ if not renames and not config.adds_uuid_path:+ return None++ def row_map(row: dict[str, Any]) -> dict[str, Any]:+ for parent_key, column in renames.items():+ row[column] = row.pop(parent_key, None)+ if config.adds_uuid_path and row.get("uuid"):+ row["_uuid_path"] = _encode_uuid(row["uuid"])+ return row++ return row_map+++def _endpoint_config(name: str) -> ZoomEndpointConfig:+ return ZOOM_ENDPOINTS.get(name) or ZOOM_PARENT_ENDPOINTS[name]+++def _resource_chain(config: ZoomEndpointConfig) -> list[ZoomEndpointConfig]:+ chain = [config]+ while chain[0].parent is not None:+ chain.insert(0, _endpoint_config(chain[0].parent))+ return chain+++def _resource_config(config: ZoomEndpointConfig) -> EndpointResource:+ params: dict[str, Any] = {**config.params}+ if config.paginated:+ params["page_size"] = config.page_size+ endpoint: Endpoint = {"path": config.path, "params": params, "data_selector": config.data_key}+ if not config.paginated:+ endpoint["paginator"] = "single_page"++ resource: EndpointResource = {"name": config.name, "endpoint": endpoint}+ if config.parent is not None:+ params[config.parent_param] = {"type": "resolve", "resource": config.parent, "field": config.parent_field}+ resource["include_from_parent"] = list(config.parent_columns)+ # A parent this endpoint can't serve (a user without the relevant license or plan, a meeting+ # that never took place) returns 400/404; skip that parent without aborting the whole sync+ # (an empty page that stops this parent's pagination). Any other non-2xx still raises.+ # Code 300 is an invalid or expired `next_page_token`: ignoring it would silently drop the+ # rest of that parent's pages, so fail the request instead and let the run retry.+ endpoint["response_actions"] = [+ {+ "status_code": 400,+ "json_field": "code",+ "json_values": [300],+ "action": "retry",+ "message": "Zoom next_page_token is invalid or expired",+ },+ {"status_code": 400, "action": "ignore"},+ {"status_code": 404, "action": "ignore"},+ ]+ row_map = _row_map(config)+ if row_map is not None:+ resource["data_map"] = row_map+ return resource+++def supports_resume(endpoint: str) -> bool:+ # The REST framework checkpoints a fan-out with one dependent level only.+ return len(_resource_chain(ZOOM_ENDPOINTS[endpoint])) <= 2++def _fan_out_resource(client_config: ClientConfig,config: ZoomEndpointConfig,@@ -120,8 +209,8 @@ def _fan_out_resource(job_id: str,resumable_source_manager: ResumableSourceManager[ZoomResumeConfig],) -> Resource:- users_config = ZOOM_ENDPOINTS["users"]-+ # The framework resumes a fan-out with one dependent level only. A chained fan-out (e.g.+ # participants <- instances <- meetings <- users) gets no resume hook and re-reads on retry.initial_paginator_state: Optional[dict[str, Any]] = Noneif resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()@@ -132,41 +221,10 @@ def _fan_out_resource(if state is not None:resumable_source_manager.save_state(ZoomResumeConfig(fanout_state=state))- child_params: dict[str, Any] = {- "user_id": {"type": "resolve", "resource": "users", "field": "id"},- "page_size": config.page_size,- **config.params,- }-rest_config: RESTAPIConfig = {"client": client_config,"resource_defaults": {},- "resources": [- {- "name": "users",- "endpoint": {- "path": users_config.path,- "params": {"page_size": users_config.page_size},- "data_selector": users_config.data_key,- },- },- {- "name": config.name,- "include_from_parent": [],- "endpoint": {- "path": config.path,- "params": child_params,- "data_selector": config.data_key,- # A user without the relevant license returns 400/404 for that feature; skip that- # user without aborting the whole sync (an empty page that stops this user's- # pagination). Any other non-2xx still raises.- "response_actions": [- {"status_code": 400, "action": "ignore"},- {"status_code": 404, "action": "ignore"},- ],- },- },- ],+ "resources": [_resource_config(level) for level in _resource_chain(config)],}resources = rest_api_resources(@@ -192,7 +250,7 @@ def zoom_source(config = ZOOM_ENDPOINTS[endpoint]client_config = _client_config(account_id, client_id, client_secret)- if config.fan_out:+ if config.parent is not None:resource = _fan_out_resource(client_config, config, team_id, job_id, resumable_source_manager)else:resource = _top_level_resource(client_config, config, team_id, job_id, resumable_source_manager)@@ -200,13 +258,14 @@ def zoom_source(return SourceResponse(name=endpoint,items=lambda: resource,- primary_keys=[config.primary_key],+ primary_keys=config.primary_keys,partition_count=1,partition_size=1,partition_mode="datetime" if config.partition_key else None,partition_format="week" if config.partition_key else None,partition_keys=[config.partition_key] if config.partition_key else None,column_hints=resource.column_hints,+ supports_resume=supports_resume(endpoint),)