:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add onepagecrm lookup tables (#123354)
Tom Owersd67add066ad5
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 3f1d570bc974..2fac4dc7c2ce 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -814,7 +814,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/oncehub/oncehub.py1 products/warehouse_sources/backend/temporal/data_imports/sources/oncehub/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/onepagecrm.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/onepassword.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 01c64d5a9261..7ae6dc017d9f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8656,10 +8656,10 @@ Today (10): `actions`, `calls`, `companies`, `contacts`, `deals`, `lead_sources`Diffed against: <https://developer.onepagecrm.com/api/reference/>-- [ ] `pipelines` — lookup that resolves the pipeline/stage IDs carried on the deals we already sync (high)-- [ ] `call_results` — lookup resolving the call outcome IDs on synced calls (high)-- [ ] `custom_fields` — field definitions needed to interpret custom field IDs on synced contacts (high)-- [ ] `deal_fields` — definitions for custom deal fields, needed to name deal custom values (medium)+- [x] `pipelines` — lookup that resolves the pipeline/stage IDs carried on the deals we already sync (high)+- [x] `call_results` — lookup resolving the call outcome IDs on synced calls (high)+- [x] `custom_fields` — field definitions needed to interpret custom field IDs on synced contacts (high)+- [x] `deal_fields` — definitions for custom deal fields, needed to name deal custom values (medium)- [ ] `company_fields` — definitions for custom company fields (medium)- [ ] `action_stream` — chronological activity/state-change feed across records (medium)- [ ] `team_stream` — team-wide activity history for rep productivity analysis (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.pyindex 94406b9a0202..c0bc808418f8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.py@@ -171,4 +171,50 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"total_count": "Number of contacts across the account with this lead source.",},},+ "call_results": {+ "description": "Call outcomes configured on the account, with per-result call counts.",+ "docs_url": "https://developer.onepagecrm.com/api/reference/calls/get-call_results/",+ "columns": {+ "result": "Identifier of the call result, referenced by calls.",+ "text": "Display text of the call result.",+ "counts": "Number of the API user's calls with this result.",+ "total_counts": "Number of calls across the account with this result.",+ "team_counts": "Number of calls with this result for each user.",+ },+ },+ "pipelines": {+ "description": "Deal pipelines on the account, each with its ordered deal stages.",+ "docs_url": "https://developer.onepagecrm.com/api/reference/pipelines/get-pipelines/",+ "columns": {+ "id": "Unique identifier of the pipeline.",+ "name": "Name of the pipeline.",+ "type": "Type of the pipeline (sales or delivery).",+ "default": "Whether this pipeline is the default for the API user.",+ "won_column_enabled": "Whether the WON column is enabled for the pipeline.",+ "won_column_name": "Name of the WON column for the pipeline.",+ "stages": "Deal stages of the pipeline, each with a numeric stage (0 to 100) and a label.",+ },+ },+ "custom_fields": {+ "description": "Definitions of the custom fields configured for contacts.",+ "docs_url": "https://developer.onepagecrm.com/api/reference/custom-fields/get-custom_fields/",+ "columns": {+ "id": "Unique identifier of the custom field, referenced by contact custom field values.",+ "name": "Name of the custom field.",+ "type": "Data type of the custom field (for example date, number, select_box, or section_divider).",+ "position": "Position of the custom field in the list.",+ "choices": "Possible choices for multiple_choice and select_box fields.",+ },+ },+ "deal_fields": {+ "description": "Definitions of the custom fields configured for deals.",+ "docs_url": "https://developer.onepagecrm.com/api/reference/deal-fields/get-deal_fields/",+ "columns": {+ "id": "Unique identifier of the deal field, referenced by deal custom field values.",+ "name": "Name of the deal field.",+ "type": "Data type of the deal field (for example date, number, select_box, or section_divider).",+ "position": "Position of the deal field in the list.",+ "choices": "Possible choices for multiple_choice and select_box fields.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.pyindex b53935ff8a1a..4ba36a2e320c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py@@ -1,6 +1,8 @@-from dataclasses import dataclass, field+from dataclasses import fieldfrom typing import Optional+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType# OnePageCRM timestamps are ISO 8601 strings (e.g. "2018-05-16T11:52:09Z"); the `modified_since`@@ -13,13 +15,13 @@ MODIFIED_AT_INCREMENTAL_FIELD: IncrementalField = {}-@dataclass+@frozenclass OnepagecrmEndpointConfig:name: strpath: str# Key the record list is nested under in the response's `data` object (e.g.# {"data": {"contacts": [...]}}). None means `data` itself is the list (users, statuses,- # lead_sources).+ # lead_sources, call_results).data_key: Optional[str]# Singular key each record is wrapped under (e.g. {"contact": {...}}); None if records are# returned unwrapped (lead_sources).@@ -27,7 +29,7 @@ class OnepagecrmEndpointConfig:# OnePageCRM object IDs are BSON ids unique across the account, so `id` is a safe primary key.primary_keys: list[str] = field(default_factory=lambda: ["id"])# Whether the endpoint accepts page/per_page params. Config endpoints (users, statuses,- # lead_sources) return the full list in one response.+ # lead_sources, call_results) return the full list in one response.paginated: bool = True# Whether the endpoint accepts sort_by/order params (every sortable endpoint here allows both# created_at and modified_at).@@ -42,7 +44,8 @@ class OnepagecrmEndpointConfig:# OnePageCRM API v3 list endpoints (https://developer.onepagecrm.com/api/). Contacts, deals,# actions, notes, calls, and meetings support incremental sync via the server-side# `modified_since` UNIX-timestamp filter; companies documents no `modified_since`, and the config-# endpoints (users, statuses, lead_sources) are small unpaginated lists — all full refresh only.+# endpoints (users, statuses, lead_sources, call_results, pipelines, custom_fields, deal_fields)+# are small lookup lists — all full refresh only.ONEPAGECRM_ENDPOINTS: dict[str, OnepagecrmEndpointConfig] = {"contacts": OnepagecrmEndpointConfig(name="contacts",@@ -127,6 +130,34 @@ ONEPAGECRM_ENDPOINTS: dict[str, OnepagecrmEndpointConfig] = {item_key=None,paginated=False,),+ "call_results": OnepagecrmEndpointConfig(+ name="call_results",+ path="/call_results",+ data_key=None,+ item_key="call_result",+ # Call results carry no `id`; `result` is the identifier calls reference.+ primary_keys=["result"],+ paginated=False,+ ),+ "pipelines": OnepagecrmEndpointConfig(+ name="pipelines",+ path="/pipelines",+ data_key="pipelines",+ item_key="pipeline",+ ),+ "custom_fields": OnepagecrmEndpointConfig(+ name="custom_fields",+ path="/custom_fields",+ data_key="custom_fields",+ item_key="custom_field",+ supports_sort=True,+ ),+ "deal_fields": OnepagecrmEndpointConfig(+ name="deal_fields",+ path="/deal_fields",+ data_key="deal_fields",+ item_key="deal_field",+ ),}ENDPOINTS = tuple(ONEPAGECRM_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.pyindex 15005ff652a0..5c3d445cead1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.py@@ -57,7 +57,7 @@ class OnepagecrmSource(ResumableSource[OnepagecrmSourceConfig, OnepagecrmResumeCreleaseStatus=ReleaseStatus.BETA,caption="""Enter your OnePageCRM API credentials to pull your CRM data into the PostHog Data warehouse.-You can find your User ID and API key in [OnePageCRM](https://app.onepagecrm.com) under **Apps & Integrations → API**. The key grants read access to your contacts, companies, deals, actions, notes, calls, meetings, users, statuses, and lead sources.+You can find your User ID and API key in [OnePageCRM](https://app.onepagecrm.com) under **Apps & Integrations → API**. The key grants read access to your contacts, companies, deals, actions, notes, calls, meetings, users, statuses, lead sources, call results, pipelines, and custom field definitions.""",iconPath="/static/services/onepagecrm.png",docsUrl="https://posthog.com/docs/cdp/sources/onepagecrm",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.pyindex 46edebb3afd7..584c9ef616fb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.py@@ -124,17 +124,46 @@ class TestPagination:("users", {"data": [{"user": {"id": "u1"}}, {"user": {"id": "u2"}}]}, ["u1", "u2"]),("statuses", {"data": [{"status": {"id": "s1"}}]}, ["s1"]),("lead_sources", {"data": [{"id": "advertisement"}, {"id": "web"}]}, ["advertisement", "web"]),+ (+ "call_results",+ {"data": [{"call_result": {"result": "interested"}}, {"call_result": {"result": "no_answer"}}]},+ ["interested", "no_answer"],+ ),])def test_config_endpoints_yield_bare_array_records(self, endpoint: str, body: dict[str, Any], expected_ids: list[str]) -> None:result = _driver().run(endpoint, [ScriptedResponse(json=body)])+ primary_key = ONEPAGECRM_ENDPOINTS[endpoint].primary_keys[0]assert result.raised is None- assert [row["id"] for row in result.rows] == expected_ids+ assert [row[primary_key] for row in result.rows] == expected_idsassert len(result.requests) == 1+ @parameterized.expand(+ [+ ("pipelines", "pipeline", None),+ ("custom_fields", "custom_field", "created_at"),+ ("deal_fields", "deal_field", None),+ ]+ )+ def test_lookup_endpoints_walk_pages_and_only_sort_where_documented(+ self, endpoint: str, item_key: str, expected_sort_by: str | None+ ) -> None:+ result = _driver().run(+ endpoint,+ [+ _list_page(endpoint, item_key, [{"id": "f1"}], page=1, max_page=2),+ _list_page(endpoint, item_key, [{"id": "f2"}], page=2, max_page=2),+ ],+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["f1", "f2"]+ assert result.params("page") == ["1", "2"]+ assert result.params("sort_by") == [expected_sort_by, expected_sort_by]+def test_missing_data_key_raises_loudly(self) -> None:result = _driver().run("contacts", [ScriptedResponse(json={"status": 0, "message": "OK"})])