Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add genesys cloud lookup tables (#121101)

Tom Owers48f2609866da
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 13717db466b5..c334952419ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5185,13 +5185,13 @@ Today (5): `calls`, `conversations`, `participants`, `queues`, `users`
Diffed against: <https://api.mypurecloud.com/api/v2/docs/swagger>
-- [ ] `Get list of wrapup codes (GET /api/v2/routing/wrapupcodes)` — resolve wrapUpCode values into conversation outcome labels (high)
-- [ ] `Get the list of routing skills (GET /api/v2/routing/skills)` — resolve requestedRoutingSkillIds and activeSkillIds on conversation sessions (high)
-- [ ] `Retrieve all divisions (GET /api/v2/authorization/divisions)` — resolve divisionIds on conversations and group performance by business unit (high)
+- [x] `Get list of wrapup codes (GET /api/v2/routing/wrapupcodes)` — resolve wrapUpCode values into conversation outcome labels (high)
+- [x] `Get the list of routing skills (GET /api/v2/routing/skills)` — resolve requestedRoutingSkillIds and activeSkillIds on conversation sessions (high)
+- [x] `Retrieve all divisions (GET /api/v2/authorization/divisions)` — resolve divisionIds on conversations and group performance by business unit (high)
- [ ] `Get Team listing (GET /api/v2/teams)` — resolve participant teamId values for agent team reporting (high)
- [ ] `Get a group list (GET /api/v2/groups)` — resolve groupId and agentGroups references in conversation routing data (high)
- [ ] `Get external contacts (GET /api/v2/externalcontacts/contacts)` — resolve externalContactId on participants into customer identities (high)
-- [ ] `Get external organizations (GET /api/v2/externalcontacts/organizations)` — resolve externalOrganizationId on participants into customer organizations (high)
+- [x] `Get external organizations (GET /api/v2/externalcontacts/organizations)` — resolve externalOrganizationId on participants into customer organizations (high)
- [ ] `Query for user details (POST /api/v2/analytics/users/details/query)` — measure agent presence and routing-state history, including time available and offline (high)
- [ ] `Queries Evaluations (GET /api/v2/quality/evaluations/query)` — analyze detailed quality evaluations beyond evaluation summaries embedded in conversations (medium)
- [ ] `Get the members of this queue (GET /api/v2/routing/queues/{queueId}/members)` — connect agents to queues and report queue staffing and membership (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.py
index 84f765ce94f4..4c56ccbd0b1a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.py
@@ -83,4 +83,62 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"scoringMethod": "The scoring method for the queue.",
},
},
+ "wrapup_codes": {
+ "description": "Wrap-up codes agents assign at the end of a conversation to record its outcome.",
+ "docs_url": "https://developer.genesys.cloud/routing/routing/",
+ "columns": {
+ "id": "The globally unique identifier for the wrap-up code. Matches wrapUpCode on conversation segments.",
+ "name": "The wrap-up code name.",
+ "description": "The wrap-up code description.",
+ "division": "The division to which this wrap-up code belongs.",
+ "dateCreated": "Date when the wrap-up code was created.",
+ "dateModified": "Date when the wrap-up code was last modified.",
+ "createdBy": "The ID of the user that created the wrap-up code.",
+ "modifiedBy": "The ID of the user that modified the wrap-up code.",
+ },
+ },
+ "skills": {
+ "description": "ACD routing skills in the Genesys Cloud organization.",
+ "docs_url": "https://developer.genesys.cloud/routing/routing/",
+ "columns": {
+ "id": "The globally unique identifier for the skill. Matches requestedRoutingSkillIds and activeSkillIds on conversation segments.",
+ "name": "The name of the skill.",
+ "division": "The division this skill belongs to.",
+ "dateModified": "Date the skill was last modified.",
+ "state": "The current state for this skill.",
+ "version": "The version of the skill.",
+ },
+ },
+ "divisions": {
+ "description": "Divisions (business units) that partition objects in the Genesys Cloud organization.",
+ "docs_url": "https://developer.genesys.cloud/authorization/platform-auth/",
+ "columns": {
+ "id": "The globally unique identifier for the division. Matches divisionIds on conversations.",
+ "name": "The division name.",
+ "description": "A helpful description for the division.",
+ "homeDivision": "Whether this division is the Home (default) division.",
+ },
+ },
+ "external_organizations": {
+ "description": "External organizations (customer companies) stored in Genesys Cloud external contacts.",
+ "docs_url": "https://developer.genesys.cloud/commdigital/externalcontacts/",
+ "columns": {
+ "id": "The globally unique identifier for the external organization. Matches externalOrganizationId on participants.",
+ "name": "The name of the company.",
+ "division": "The division to which this external organization belongs.",
+ "companyType": "The type of company.",
+ "industry": "The industry of the company.",
+ "address": "The address of the company.",
+ "phoneNumber": "The phone number of the company.",
+ "employeeCount": "The number of employees.",
+ "revenue": "The revenue of the company.",
+ "tags": "Tags on the external organization.",
+ "websites": "Websites of the company.",
+ "externalSystemUrl": "A link to the record of this organization in an external system of record.",
+ "createDate": "Date the external organization was created.",
+ "modifyDate": "Date the external organization was last modified.",
+ "customFields": "Custom fields defined in the schema attached to this external organization.",
+ "externalIds": "Identifiers for this external organization in external systems.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.py
index aa23a06d2a6c..21e422cda48a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.py
@@ -50,6 +50,8 @@ class GenesysCloudResumeConfig:
window_start: Optional[str] = None
# Listing endpoints: next page number to read.
page_number: Optional[int] = None
+ # Scan endpoints: cursor of the next page to read. Genesys Cloud keeps a cursor for 24 hours.
+ cursor: Optional[str] = None
def _validated_region(region: str) -> str:
@@ -233,6 +235,42 @@ def _iter_listing_rows(
page_number += 1
+def _iter_scan_rows(
+ session: requests.Session,
+ base_url: str,
+ config: GenesysCloudEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[GenesysCloudResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ cursor = resume.cursor if resume is not None else None
+ sends_resumed_cursor = cursor is not None
+
+ url = f"{base_url}{config.path}"
+ while True:
+ params: dict[str, Any] = {**config.params, "limit": LISTING_PAGE_SIZE}
+ if cursor:
+ params["cursor"] = cursor
+ response = session.get(url, params=params, timeout=REQUEST_TIMEOUT_SECONDS)
+ if sends_resumed_cursor and response.status_code in (400, 422):
+ # The saved cursor expired. Restarting here would append rows this run already loaded, so drop the
+ # cursor and fail: the retry then runs as a fresh full refresh.
+ resumable_source_manager.clear_state()
+ sends_resumed_cursor = False
+ response.raise_for_status()
+ data = response.json() or {}
+ entities = data.get("entities") or []
+ next_cursor = (data.get("cursors") or {}).get("after")
+ has_next_page = bool(entities) and bool(next_cursor)
+ if has_next_page:
+ resumable_source_manager.save_state(GenesysCloudResumeConfig(cursor=next_cursor))
+ if entities:
+ yield entities
+ if not has_next_page:
+ resumable_source_manager.clear_state()
+ return
+ cursor = next_cursor
+
+
def genesys_cloud_source(
region: str,
client_id: str,
@@ -250,6 +288,8 @@ def genesys_cloud_source(
yield from _iter_analytics_rows(
session, base_url, config, resumable_source_manager, db_incremental_field_last_value
)
+ elif config.kind == "scan":
+ yield from _iter_scan_rows(session, base_url, config, resumable_source_manager)
else:
yield from _iter_listing_rows(session, base_url, config, resumable_source_manager)
@@ -286,7 +326,8 @@ def validate_credentials(
else:
response = session.get(
f"{base_url}{config.path}",
- params={"pageSize": 1, "pageNumber": 1},
+ # The scan listings reject a limit below 10.
+ params={"limit": 10} if config.kind == "scan" else {"pageSize": 1, "pageNumber": 1},
timeout=VALIDATION_TIMEOUT_SECONDS,
)
except OAuth2AuthRequestError as e:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.py
index db4f654c1e7b..d127a061cf93 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.py
@@ -46,8 +46,9 @@ class GenesysCloudEndpointConfig:
name: str
primary_keys: list[str]
# "analytics" endpoints read the conversation details query in time windows;
- # "listing" endpoints page through a plain GET entity listing.
- kind: Literal["analytics", "listing"]
+ # "listing" endpoints page through a plain GET entity listing by page number;
+ # "scan" endpoints page through a GET listing by cursor.
+ kind: Literal["analytics", "listing", "scan"]
path: str
# Permission a Genesys Cloud role must grant for the OAuth client to read this endpoint.
permission: str | None = None
@@ -118,6 +119,34 @@ GENESYS_CLOUD_ENDPOINTS: dict[str, GenesysCloudEndpointConfig] = {
permission="routing:queue:view",
params={"sortOrder": "asc"},
),
+ "wrapup_codes": GenesysCloudEndpointConfig(
+ name="wrapup_codes",
+ primary_keys=["id"],
+ kind="listing",
+ path="/api/v2/routing/wrapupcodes",
+ permission="routing:wrapupCode:view",
+ ),
+ "skills": GenesysCloudEndpointConfig(
+ name="skills",
+ primary_keys=["id"],
+ kind="listing",
+ path="/api/v2/routing/skills",
+ permission="routing:skill:view",
+ ),
+ "divisions": GenesysCloudEndpointConfig(
+ name="divisions",
+ primary_keys=["id"],
+ kind="listing",
+ path="/api/v2/authorization/divisions",
+ ),
+ # The plain /externalcontacts/organizations listing stops at the first 1,000 records, so read the scan listing.
+ "external_organizations": GenesysCloudEndpointConfig(
+ name="external_organizations",
+ primary_keys=["id"],
+ kind="scan",
+ path="/api/v2/externalcontacts/scan/organizations",
+ permission="externalContacts:externalOrganization:view",
+ ),
}
ENDPOINTS = tuple(GENESYS_CLOUD_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.py
index 0476c74ac970..233c7ccb6b37 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.py
@@ -98,12 +98,15 @@ class GenesysCloudSource(ResumableSource[GenesysCloudSourceConfig, GenesysCloudR
category=DataWarehouseSourceCategory.CUSTOMER_SUPPORT,
keywords=["genesys", "purecloud", "contact center"],
label="Genesys Cloud",
- caption="""Connect your Genesys Cloud organization to import calls, conversations, participants, users, and queues.
+ caption="""Connect your Genesys Cloud organization to import calls, conversations, participants, users, queues, wrap-up codes, skills, divisions, and external organizations.
In Genesys Cloud, go to **Admin > Integrations > OAuth** and add a client with the **Client Credentials** grant type. Assign it a role with these permissions:
- `analytics:conversationDetail:view` for calls, conversations, and participants
- `routing:queue:view` for queues
+- `routing:wrapupCode:view` for wrap-up codes
+- `routing:skill:view` for skills
+- `externalContacts:externalOrganization:view` for external organizations
Then select the region your organization is hosted in.""",
iconPath="/static/services/genesys_cloud.png",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.py
index ddc6259de4bd..2891b2e45bcf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.py
@@ -4,6 +4,8 @@ from typing import Any
import pytest
import time_machine
+import requests
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
OAuth2AuthRequestError,
)
@@ -191,12 +193,50 @@ class TestListingEndpoints:
assert result.saved_states == ([GenesysCloudResumeConfig(page_number=2)] if resume is None else [])
assert result.final_state is None
+ @pytest.mark.parametrize(
+ "resume,expected_cursors",
+ [
+ (None, [None, "after-1"]),
+ (GenesysCloudResumeConfig(cursor="after-1"), ["after-1"]),
+ ],
+ )
+ def test_scan_pages_until_cursor_runs_out(self, resume, expected_cursors):
+ pages = {
+ None: ScriptedResponse(json={"entities": [{"id": "o1"}], "cursors": {"after": "after-1"}}),
+ "after-1": ScriptedResponse(json={"entities": [{"id": "o2"}], "cursors": {}}),
+ }
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/oauth/token":
+ return TOKEN
+ return pages[request.param("cursor")]
+
+ result = _driver().run("external_organizations", respond, resume_state=resume)
+
+ assert result.raised is None
+ assert result.params("cursor")[1:] == expected_cursors
+ assert [row["id"] for row in result.rows] == ["o1", "o2"][-len(expected_cursors) :]
+ assert result.saved_states == ([GenesysCloudResumeConfig(cursor="after-1")] if resume is None else [])
+ assert result.final_state is None
+
+ def test_scan_drops_an_expired_resumed_cursor(self):
+ result = _driver().run(
+ "external_organizations",
+ [TOKEN, ScriptedResponse(status=422, json={"code": "bad.request"})],
+ resume_state=GenesysCloudResumeConfig(cursor="expired"),
+ )
+
+ assert isinstance(result.raised, requests.HTTPError)
+ assert result.rows == []
+ assert result.final_state is None
+
class TestValidateCredentials:
@pytest.mark.parametrize(
"endpoint,status_code,expected_valid,expected_message",
[
(None, 200, True, None),
+ ("external_organizations", 200, True, None),
],
)
def test_maps_probe_status(self, endpoint, status_code, expected_valid, expected_message):
@@ -217,6 +257,7 @@ class TestValidateCredentials:
"conversations": "/api/v2/analytics/conversations/details/query",
"users": "/api/v2/users",
"queues": "/api/v2/routing/queues",
+ "external_organizations": "/api/v2/externalcontacts/scan/organizations",
}[endpoint]
)