:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): sync gong call media urls (#129685)
Tyler Crandall33ac0a3da409
diff --git a/products/warehouse_sources/sources/gong/canonical_descriptions.py b/products/warehouse_sources/sources/gong/canonical_descriptions.pyindex 211209495a4e..47fa00995ae7 100644--- a/products/warehouse_sources/sources/gong/canonical_descriptions.py+++ b/products/warehouse_sources/sources/gong/canonical_descriptions.py@@ -65,6 +65,25 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"content": "Gong's AI-generated summary of the call — the Spotlight `brief`, `keyPoints`, `highlights`, `callOutcome`, and `outline`. For the verbatim conversation rather than the summary, see `transcripts`.",},},+ "calls_media": {+ "description": "Download URLs for the audio and video recording of a Gong call, sourced from the /v2/calls/extensive endpoint. Gong keeps each URL valid for 8 hours, so the table is rewritten on every sync and only URLs from the latest sync work.",+ "docs_url": "https://gong.app.gong.io/settings/api/documentation#post-/v2/calls/extensive",+ "columns": {+ "id": "Unique identifier for the call. Join to `calls` or `calls_extensive` on this.",+ "title": "Title of the call.",+ "url": "URL of the call within the Gong web app.",+ "started": "Time at which the call started (ISO 8601).",+ "duration": "Duration of the call in seconds.",+ "direction": "Direction of the call (Inbound, Outbound, Conference, or Unknown).",+ "scope": "Whether the call was internal or external.",+ "media": "Media type of the call (Audio or Video).",+ "language": "Detected primary language of the call.",+ "workspaceId": "Identifier of the workspace the call belongs to.",+ "primaryUserId": "Identifier of the main Gong user (host) on the call.",+ "scheduled": "Scheduled start time of the call, if applicable.",+ "mediaUrls": "Download URLs for the call recording: `audioUrl`, and `videoUrl` when the call has video. Each URL is valid for 8 hours after the sync that wrote it.",+ },+ },"transcripts": {"description": "The transcript of a recorded Gong call — what each speaker said, grouped into monologues and sentences.","docs_url": "https://gong.app.gong.io/settings/api/documentation#post-/v2/calls/transcript",diff --git a/products/warehouse_sources/sources/gong/gong.py b/products/warehouse_sources/sources/gong/gong.pyindex 44ac943a2911..471d655f673b 100644--- a/products/warehouse_sources/sources/gong/gong.py+++ b/products/warehouse_sources/sources/gong/gong.py@@ -136,18 +136,22 @@ def _extensive_body(return body-def _flatten_extensive_call(call: dict[str, Any], row_keys: tuple[str, ...]) -> dict[str, Any]:+def _flatten_extensive_call(+ call: dict[str, Any], row_keys: tuple[str, ...], row_columns: Optional[dict[str, str]] = None+) -> dict[str, Any]:"""Lift the `metaData` block of an extensive call row to the top level.`/v2/calls/extensive` nests the core call fields under `metaData` and returns the requestedenrichment blocks as siblings. Flattening `metaData` up keeps `id` and `started` available atthe top level for primary-key merge and datetime partitioning, while preserving each- enrichment block as a nested column.+ enrichment block as a nested column, renamed through `row_columns` where its key clashes with+ a `metaData` field."""meta = call.get("metaData") or {}flattened = dict(meta)+ columns = row_columns or {}for key in row_keys:- flattened[key] = call.get(key)+ flattened[columns.get(key, key)] = call.get(key)return flattened@@ -341,7 +345,8 @@ def _iter_call_rows(json_body=_extensive_body(config, window_start, window_end, cursor),)rows = [- _flatten_extensive_call(row, config.extensive_row_keys) for row in data.get(config.response_key, [])+ _flatten_extensive_call(row, config.extensive_row_keys, config.extensive_row_columns)+ for row in data.get(config.response_key, [])]else:params: dict[str, Any] = {diff --git a/products/warehouse_sources/sources/gong/settings.py b/products/warehouse_sources/sources/gong/settings.pyindex a7e0062adc20..89857030a698 100644--- a/products/warehouse_sources/sources/gong/settings.py+++ b/products/warehouse_sources/sources/gong/settings.py@@ -31,6 +31,10 @@ class GongEndpointConfig:# Sibling keys of `metaData` to keep as nested columns on the flattened row. Must line up with# what `extensive_content_selector` requests, or the columns arrive permanently null.extensive_row_keys: tuple[str, ...] = ()+ # Column to store an enrichment block under, for blocks whose key clashes with a `metaData`+ # field — the `media` URLs block would otherwise overwrite the `media` type string. Keys not+ # listed keep their own name.+ extensive_row_columns: dict[str, str] = field(default_factory=dict)# `POST /v2/calls/transcript` answers with `callId` and `transcript` and nothing else. Endpoints# that set this are driven off the `/v2/calls` list for the same window instead: each page of# calls supplies the `callIds` filter for one transcript request, and the call's `started`@@ -168,6 +172,30 @@ GONG_ENDPOINTS: dict[str, GongEndpointConfig] = {},],),+ # Download URLs for each call's audio and video recording, from `/v2/calls/extensive` with the+ # `media` block of the selector requested. Gong signs these URLs for 8 hours only, so the table+ # is full refresh only: an incremental sync would fetch each call once and keep its URL long+ # after it stopped working, while a full refresh rewrites every URL each run. Kept as its own+ # default-off table because the URLs open the recording to anyone who reads them, and because+ # asking for media needs the `api:calls:read:media-url` scope: a key without it must not break+ # `calls_extensive`.+ "calls_media": GongEndpointConfig(+ name="calls_media",+ path="/v2/calls/extensive",+ response_key="calls",+ primary_key="id",+ partition_key="started",+ uses_date_window=True,+ uses_extensive=True,+ capture_http_samples=False,+ should_sync_default=False,+ extensive_content_selector={+ "context": "None",+ "exposedFields": {"media": True},+ },+ extensive_row_keys=("media",),+ extensive_row_columns={"media": "mediaUrls"},+ ),# Transcript text for the same calls, from `POST /v2/calls/transcript`, which returns only# `callId` and `transcript`. Each row is stamped with the `started` of the call it belongs to,# read from the `/v2/calls` page that drove the request — without it a transcript carries nodiff --git a/products/warehouse_sources/sources/gong/source.py b/products/warehouse_sources/sources/gong/source.pyindex 9283e102a962..6eb1633a3941 100644--- a/products/warehouse_sources/sources/gong/source.py+++ b/products/warehouse_sources/sources/gong/source.py@@ -63,6 +63,10 @@ To also sync the `calls_extensive` table (call participants and CRM associationsTo also sync the `transcripts` table (what was said on each call), additionally grant:- `api:calls:read:transcript`+To also sync the `calls_media` table (audio and video download URLs for each call, which Gong keeps valid for 8 hours), additionally grant:+- `api:calls:read:extensive`+- `api:calls:read:media-url`+To also sync these tables, additionally grant the scope listed next to each:- `trackers` (keyword tracker definitions): `api:settings:trackers:read`- `answered_scorecards` (scorecard reviews of calls): `api:stats:scorecards`@@ -119,7 +123,11 @@ To also sync these tables, additionally grant the scope listed next to each:incremental_fields=endpoint_config.incremental_fields,default_incremental_lookback_seconds=endpoint_config.default_incremental_lookback_seconds,should_sync_default=endpoint_config.should_sync_default,- description="Only syncs the last 365 days on initial sync"+ description=(+ "Only syncs the last 365 days on initial sync"+ if endpoint_config.supports_incremental+ else "Syncs the last 365 days on every sync"+ )if endpoint_config.uses_date_windowelse None,)diff --git a/products/warehouse_sources/sources/gong/tests/test_gong.py b/products/warehouse_sources/sources/gong/tests/test_gong.pyindex ae6b9d63dae5..4f4338534355 100644--- a/products/warehouse_sources/sources/gong/tests/test_gong.py+++ b/products/warehouse_sources/sources/gong/tests/test_gong.py@@ -259,6 +259,32 @@ class TestExtensiveCalls:},{"content": {"brief": "Buyer wants SSO.", "keyPoints": [{"text": "Asked about SSO"}]}},),+ (+ "calls_media_keeps_media_type_and_stores_urls_in_their_own_column",+ "calls_media",+ {+ # `metaData` also has a `media` key, so the URLs must not land on it.+ "metaData": {+ "id": "c1",+ "title": "Discovery",+ "started": "2026-03-01T00:00:00Z",+ "media": "Video",+ },+ "media": {+ "audioUrl": "https://media.example.com/c1/audio.mp3",+ "videoUrl": "https://media.example.com/c1/video.mp4",+ },+ "parties": [{"emailAddress": "[email protected]"}],+ },+ {"context": "None", "exposedFields": {"media": True}},+ {+ "media": "Video",+ "mediaUrls": {+ "audioUrl": "https://media.example.com/c1/audio.mp3",+ "videoUrl": "https://media.example.com/c1/video.mp4",+ },+ },+ ),])def test_posts_selector_and_flattens_metadata(