:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add genesys cloud lookup tables (#121101)
Tom Owers48f2609866da
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 13717db466b5..c334952419ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -5185,13 +5185,13 @@ Today (5): `calls`, `conversations`, `participants`, `queues`, `users`Diffed against: <https://api.mypurecloud.com/api/v2/docs/swagger>-- [ ] `Get list of wrapup codes (GET /api/v2/routing/wrapupcodes)` — resolve wrapUpCode values into conversation outcome labels (high)-- [ ] `Get the list of routing skills (GET /api/v2/routing/skills)` — resolve requestedRoutingSkillIds and activeSkillIds on conversation sessions (high)-- [ ] `Retrieve all divisions (GET /api/v2/authorization/divisions)` — resolve divisionIds on conversations and group performance by business unit (high)+- [x] `Get list of wrapup codes (GET /api/v2/routing/wrapupcodes)` — resolve wrapUpCode values into conversation outcome labels (high)+- [x] `Get the list of routing skills (GET /api/v2/routing/skills)` — resolve requestedRoutingSkillIds and activeSkillIds on conversation sessions (high)+- [x] `Retrieve all divisions (GET /api/v2/authorization/divisions)` — resolve divisionIds on conversations and group performance by business unit (high)- [ ] `Get Team listing (GET /api/v2/teams)` — resolve participant teamId values for agent team reporting (high)- [ ] `Get a group list (GET /api/v2/groups)` — resolve groupId and agentGroups references in conversation routing data (high)- [ ] `Get external contacts (GET /api/v2/externalcontacts/contacts)` — resolve externalContactId on participants into customer identities (high)-- [ ] `Get external organizations (GET /api/v2/externalcontacts/organizations)` — resolve externalOrganizationId on participants into customer organizations (high)+- [x] `Get external organizations (GET /api/v2/externalcontacts/organizations)` — resolve externalOrganizationId on participants into customer organizations (high)- [ ] `Query for user details (POST /api/v2/analytics/users/details/query)` — measure agent presence and routing-state history, including time available and offline (high)- [ ] `Queries Evaluations (GET /api/v2/quality/evaluations/query)` — analyze detailed quality evaluations beyond evaluation summaries embedded in conversations (medium)- [ ] `Get the members of this queue (GET /api/v2/routing/queues/{queueId}/members)` — connect agents to queues and report queue staffing and membership (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.pyindex 84f765ce94f4..4c56ccbd0b1a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/canonical_descriptions.py@@ -83,4 +83,62 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"scoringMethod": "The scoring method for the queue.",},},+ "wrapup_codes": {+ "description": "Wrap-up codes agents assign at the end of a conversation to record its outcome.",+ "docs_url": "https://developer.genesys.cloud/routing/routing/",+ "columns": {+ "id": "The globally unique identifier for the wrap-up code. Matches wrapUpCode on conversation segments.",+ "name": "The wrap-up code name.",+ "description": "The wrap-up code description.",+ "division": "The division to which this wrap-up code belongs.",+ "dateCreated": "Date when the wrap-up code was created.",+ "dateModified": "Date when the wrap-up code was last modified.",+ "createdBy": "The ID of the user that created the wrap-up code.",+ "modifiedBy": "The ID of the user that modified the wrap-up code.",+ },+ },+ "skills": {+ "description": "ACD routing skills in the Genesys Cloud organization.",+ "docs_url": "https://developer.genesys.cloud/routing/routing/",+ "columns": {+ "id": "The globally unique identifier for the skill. Matches requestedRoutingSkillIds and activeSkillIds on conversation segments.",+ "name": "The name of the skill.",+ "division": "The division this skill belongs to.",+ "dateModified": "Date the skill was last modified.",+ "state": "The current state for this skill.",+ "version": "The version of the skill.",+ },+ },+ "divisions": {+ "description": "Divisions (business units) that partition objects in the Genesys Cloud organization.",+ "docs_url": "https://developer.genesys.cloud/authorization/platform-auth/",+ "columns": {+ "id": "The globally unique identifier for the division. Matches divisionIds on conversations.",+ "name": "The division name.",+ "description": "A helpful description for the division.",+ "homeDivision": "Whether this division is the Home (default) division.",+ },+ },+ "external_organizations": {+ "description": "External organizations (customer companies) stored in Genesys Cloud external contacts.",+ "docs_url": "https://developer.genesys.cloud/commdigital/externalcontacts/",+ "columns": {+ "id": "The globally unique identifier for the external organization. Matches externalOrganizationId on participants.",+ "name": "The name of the company.",+ "division": "The division to which this external organization belongs.",+ "companyType": "The type of company.",+ "industry": "The industry of the company.",+ "address": "The address of the company.",+ "phoneNumber": "The phone number of the company.",+ "employeeCount": "The number of employees.",+ "revenue": "The revenue of the company.",+ "tags": "Tags on the external organization.",+ "websites": "Websites of the company.",+ "externalSystemUrl": "A link to the record of this organization in an external system of record.",+ "createDate": "Date the external organization was created.",+ "modifyDate": "Date the external organization was last modified.",+ "customFields": "Custom fields defined in the schema attached to this external organization.",+ "externalIds": "Identifiers for this external organization in external systems.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.pyindex aa23a06d2a6c..21e422cda48a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/genesys_cloud.py@@ -50,6 +50,8 @@ class GenesysCloudResumeConfig:window_start: Optional[str] = None# Listing endpoints: next page number to read.page_number: Optional[int] = None+ # Scan endpoints: cursor of the next page to read. Genesys Cloud keeps a cursor for 24 hours.+ cursor: Optional[str] = Nonedef _validated_region(region: str) -> str:@@ -233,6 +235,42 @@ def _iter_listing_rows(page_number += 1+def _iter_scan_rows(+ session: requests.Session,+ base_url: str,+ config: GenesysCloudEndpointConfig,+ resumable_source_manager: ResumableSourceManager[GenesysCloudResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ cursor = resume.cursor if resume is not None else None+ sends_resumed_cursor = cursor is not None++ url = f"{base_url}{config.path}"+ while True:+ params: dict[str, Any] = {**config.params, "limit": LISTING_PAGE_SIZE}+ if cursor:+ params["cursor"] = cursor+ response = session.get(url, params=params, timeout=REQUEST_TIMEOUT_SECONDS)+ if sends_resumed_cursor and response.status_code in (400, 422):+ # The saved cursor expired. Restarting here would append rows this run already loaded, so drop the+ # cursor and fail: the retry then runs as a fresh full refresh.+ resumable_source_manager.clear_state()+ sends_resumed_cursor = False+ response.raise_for_status()+ data = response.json() or {}+ entities = data.get("entities") or []+ next_cursor = (data.get("cursors") or {}).get("after")+ has_next_page = bool(entities) and bool(next_cursor)+ if has_next_page:+ resumable_source_manager.save_state(GenesysCloudResumeConfig(cursor=next_cursor))+ if entities:+ yield entities+ if not has_next_page:+ resumable_source_manager.clear_state()+ return+ cursor = next_cursor++def genesys_cloud_source(region: str,client_id: str,@@ -250,6 +288,8 @@ def genesys_cloud_source(yield from _iter_analytics_rows(session, base_url, config, resumable_source_manager, db_incremental_field_last_value)+ elif config.kind == "scan":+ yield from _iter_scan_rows(session, base_url, config, resumable_source_manager)else:yield from _iter_listing_rows(session, base_url, config, resumable_source_manager)@@ -286,7 +326,8 @@ def validate_credentials(else:response = session.get(f"{base_url}{config.path}",- params={"pageSize": 1, "pageNumber": 1},+ # The scan listings reject a limit below 10.+ params={"limit": 10} if config.kind == "scan" else {"pageSize": 1, "pageNumber": 1},timeout=VALIDATION_TIMEOUT_SECONDS,)except OAuth2AuthRequestError as e:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.pyindex db4f654c1e7b..d127a061cf93 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/settings.py@@ -46,8 +46,9 @@ class GenesysCloudEndpointConfig:name: strprimary_keys: list[str]# "analytics" endpoints read the conversation details query in time windows;- # "listing" endpoints page through a plain GET entity listing.- kind: Literal["analytics", "listing"]+ # "listing" endpoints page through a plain GET entity listing by page number;+ # "scan" endpoints page through a GET listing by cursor.+ kind: Literal["analytics", "listing", "scan"]path: str# Permission a Genesys Cloud role must grant for the OAuth client to read this endpoint.permission: str | None = None@@ -118,6 +119,34 @@ GENESYS_CLOUD_ENDPOINTS: dict[str, GenesysCloudEndpointConfig] = {permission="routing:queue:view",params={"sortOrder": "asc"},),+ "wrapup_codes": GenesysCloudEndpointConfig(+ name="wrapup_codes",+ primary_keys=["id"],+ kind="listing",+ path="/api/v2/routing/wrapupcodes",+ permission="routing:wrapupCode:view",+ ),+ "skills": GenesysCloudEndpointConfig(+ name="skills",+ primary_keys=["id"],+ kind="listing",+ path="/api/v2/routing/skills",+ permission="routing:skill:view",+ ),+ "divisions": GenesysCloudEndpointConfig(+ name="divisions",+ primary_keys=["id"],+ kind="listing",+ path="/api/v2/authorization/divisions",+ ),+ # The plain /externalcontacts/organizations listing stops at the first 1,000 records, so read the scan listing.+ "external_organizations": GenesysCloudEndpointConfig(+ name="external_organizations",+ primary_keys=["id"],+ kind="scan",+ path="/api/v2/externalcontacts/scan/organizations",+ permission="externalContacts:externalOrganization:view",+ ),}ENDPOINTS = tuple(GENESYS_CLOUD_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.pyindex 0476c74ac970..233c7ccb6b37 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/source.py@@ -98,12 +98,15 @@ class GenesysCloudSource(ResumableSource[GenesysCloudSourceConfig, GenesysCloudRcategory=DataWarehouseSourceCategory.CUSTOMER_SUPPORT,keywords=["genesys", "purecloud", "contact center"],label="Genesys Cloud",- caption="""Connect your Genesys Cloud organization to import calls, conversations, participants, users, and queues.+ caption="""Connect your Genesys Cloud organization to import calls, conversations, participants, users, queues, wrap-up codes, skills, divisions, and external organizations.In Genesys Cloud, go to **Admin > Integrations > OAuth** and add a client with the **Client Credentials** grant type. Assign it a role with these permissions:- `analytics:conversationDetail:view` for calls, conversations, and participants- `routing:queue:view` for queues+- `routing:wrapupCode:view` for wrap-up codes+- `routing:skill:view` for skills+- `externalContacts:externalOrganization:view` for external organizationsThen select the region your organization is hosted in.""",iconPath="/static/services/genesys_cloud.png",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.pyindex ddc6259de4bd..2891b2e45bcf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/genesys_cloud/tests/test_genesys_cloud.py@@ -4,6 +4,8 @@ from typing import Anyimport pytestimport time_machine+import requests+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (OAuth2AuthRequestError,)@@ -191,12 +193,50 @@ class TestListingEndpoints:assert result.saved_states == ([GenesysCloudResumeConfig(page_number=2)] if resume is None else [])assert result.final_state is None+ @pytest.mark.parametrize(+ "resume,expected_cursors",+ [+ (None, [None, "after-1"]),+ (GenesysCloudResumeConfig(cursor="after-1"), ["after-1"]),+ ],+ )+ def test_scan_pages_until_cursor_runs_out(self, resume, expected_cursors):+ pages = {+ None: ScriptedResponse(json={"entities": [{"id": "o1"}], "cursors": {"after": "after-1"}}),+ "after-1": ScriptedResponse(json={"entities": [{"id": "o2"}], "cursors": {}}),+ }++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/oauth/token":+ return TOKEN+ return pages[request.param("cursor")]++ result = _driver().run("external_organizations", respond, resume_state=resume)++ assert result.raised is None+ assert result.params("cursor")[1:] == expected_cursors+ assert [row["id"] for row in result.rows] == ["o1", "o2"][-len(expected_cursors) :]+ assert result.saved_states == ([GenesysCloudResumeConfig(cursor="after-1")] if resume is None else [])+ assert result.final_state is None++ def test_scan_drops_an_expired_resumed_cursor(self):+ result = _driver().run(+ "external_organizations",+ [TOKEN, ScriptedResponse(status=422, json={"code": "bad.request"})],+ resume_state=GenesysCloudResumeConfig(cursor="expired"),+ )++ assert isinstance(result.raised, requests.HTTPError)+ assert result.rows == []+ assert result.final_state is None+class TestValidateCredentials:@pytest.mark.parametrize("endpoint,status_code,expected_valid,expected_message",[(None, 200, True, None),+ ("external_organizations", 200, True, None),],)def test_maps_probe_status(self, endpoint, status_code, expected_valid, expected_message):@@ -217,6 +257,7 @@ class TestValidateCredentials:"conversations": "/api/v2/analytics/conversations/details/query","users": "/api/v2/users","queues": "/api/v2/routing/queues",+ "external_organizations": "/api/v2/externalcontacts/scan/organizations",}[endpoint])