Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add onepagecrm lookup tables (#123354)

Tom Owersd67add066ad5
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 3f1d570bc974..2fac4dc7c2ce 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -814,7 +814,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/oncehub/oncehub.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/oncehub/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/onepagecrm.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/onepassword.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 01c64d5a9261..7ae6dc017d9f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8656,10 +8656,10 @@ Today (10): `actions`, `calls`, `companies`, `contacts`, `deals`, `lead_sources`
Diffed against: <https://developer.onepagecrm.com/api/reference/>
-- [ ] `pipelines` — lookup that resolves the pipeline/stage IDs carried on the deals we already sync (high)
-- [ ] `call_results` — lookup resolving the call outcome IDs on synced calls (high)
-- [ ] `custom_fields` — field definitions needed to interpret custom field IDs on synced contacts (high)
-- [ ] `deal_fields` — definitions for custom deal fields, needed to name deal custom values (medium)
+- [x] `pipelines` — lookup that resolves the pipeline/stage IDs carried on the deals we already sync (high)
+- [x] `call_results` — lookup resolving the call outcome IDs on synced calls (high)
+- [x] `custom_fields` — field definitions needed to interpret custom field IDs on synced contacts (high)
+- [x] `deal_fields` — definitions for custom deal fields, needed to name deal custom values (medium)
- [ ] `company_fields` — definitions for custom company fields (medium)
- [ ] `action_stream` — chronological activity/state-change feed across records (medium)
- [ ] `team_stream` — team-wide activity history for rep productivity analysis (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.py
index 94406b9a0202..c0bc808418f8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/canonical_descriptions.py
@@ -171,4 +171,50 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"total_count": "Number of contacts across the account with this lead source.",
},
},
+ "call_results": {
+ "description": "Call outcomes configured on the account, with per-result call counts.",
+ "docs_url": "https://developer.onepagecrm.com/api/reference/calls/get-call_results/",
+ "columns": {
+ "result": "Identifier of the call result, referenced by calls.",
+ "text": "Display text of the call result.",
+ "counts": "Number of the API user's calls with this result.",
+ "total_counts": "Number of calls across the account with this result.",
+ "team_counts": "Number of calls with this result for each user.",
+ },
+ },
+ "pipelines": {
+ "description": "Deal pipelines on the account, each with its ordered deal stages.",
+ "docs_url": "https://developer.onepagecrm.com/api/reference/pipelines/get-pipelines/",
+ "columns": {
+ "id": "Unique identifier of the pipeline.",
+ "name": "Name of the pipeline.",
+ "type": "Type of the pipeline (sales or delivery).",
+ "default": "Whether this pipeline is the default for the API user.",
+ "won_column_enabled": "Whether the WON column is enabled for the pipeline.",
+ "won_column_name": "Name of the WON column for the pipeline.",
+ "stages": "Deal stages of the pipeline, each with a numeric stage (0 to 100) and a label.",
+ },
+ },
+ "custom_fields": {
+ "description": "Definitions of the custom fields configured for contacts.",
+ "docs_url": "https://developer.onepagecrm.com/api/reference/custom-fields/get-custom_fields/",
+ "columns": {
+ "id": "Unique identifier of the custom field, referenced by contact custom field values.",
+ "name": "Name of the custom field.",
+ "type": "Data type of the custom field (for example date, number, select_box, or section_divider).",
+ "position": "Position of the custom field in the list.",
+ "choices": "Possible choices for multiple_choice and select_box fields.",
+ },
+ },
+ "deal_fields": {
+ "description": "Definitions of the custom fields configured for deals.",
+ "docs_url": "https://developer.onepagecrm.com/api/reference/deal-fields/get-deal_fields/",
+ "columns": {
+ "id": "Unique identifier of the deal field, referenced by deal custom field values.",
+ "name": "Name of the deal field.",
+ "type": "Data type of the deal field (for example date, number, select_box, or section_divider).",
+ "position": "Position of the deal field in the list.",
+ "choices": "Possible choices for multiple_choice and select_box fields.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py
index b53935ff8a1a..4ba36a2e320c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/settings.py
@@ -1,6 +1,8 @@
-from dataclasses import dataclass, field
+from dataclasses import field
from typing import Optional
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# OnePageCRM timestamps are ISO 8601 strings (e.g. "2018-05-16T11:52:09Z"); the `modified_since`
@@ -13,13 +15,13 @@ MODIFIED_AT_INCREMENTAL_FIELD: IncrementalField = {
}
-@dataclass
+@frozen
class OnepagecrmEndpointConfig:
name: str
path: str
# Key the record list is nested under in the response's `data` object (e.g.
# {"data": {"contacts": [...]}}). None means `data` itself is the list (users, statuses,
- # lead_sources).
+ # lead_sources, call_results).
data_key: Optional[str]
# Singular key each record is wrapped under (e.g. {"contact": {...}}); None if records are
# returned unwrapped (lead_sources).
@@ -27,7 +29,7 @@ class OnepagecrmEndpointConfig:
# OnePageCRM object IDs are BSON ids unique across the account, so `id` is a safe primary key.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
# Whether the endpoint accepts page/per_page params. Config endpoints (users, statuses,
- # lead_sources) return the full list in one response.
+ # lead_sources, call_results) return the full list in one response.
paginated: bool = True
# Whether the endpoint accepts sort_by/order params (every sortable endpoint here allows both
# created_at and modified_at).
@@ -42,7 +44,8 @@ class OnepagecrmEndpointConfig:
# OnePageCRM API v3 list endpoints (https://developer.onepagecrm.com/api/). Contacts, deals,
# actions, notes, calls, and meetings support incremental sync via the server-side
# `modified_since` UNIX-timestamp filter; companies documents no `modified_since`, and the config
-# endpoints (users, statuses, lead_sources) are small unpaginated lists — all full refresh only.
+# endpoints (users, statuses, lead_sources, call_results, pipelines, custom_fields, deal_fields)
+# are small lookup lists — all full refresh only.
ONEPAGECRM_ENDPOINTS: dict[str, OnepagecrmEndpointConfig] = {
"contacts": OnepagecrmEndpointConfig(
name="contacts",
@@ -127,6 +130,34 @@ ONEPAGECRM_ENDPOINTS: dict[str, OnepagecrmEndpointConfig] = {
item_key=None,
paginated=False,
),
+ "call_results": OnepagecrmEndpointConfig(
+ name="call_results",
+ path="/call_results",
+ data_key=None,
+ item_key="call_result",
+ # Call results carry no `id`; `result` is the identifier calls reference.
+ primary_keys=["result"],
+ paginated=False,
+ ),
+ "pipelines": OnepagecrmEndpointConfig(
+ name="pipelines",
+ path="/pipelines",
+ data_key="pipelines",
+ item_key="pipeline",
+ ),
+ "custom_fields": OnepagecrmEndpointConfig(
+ name="custom_fields",
+ path="/custom_fields",
+ data_key="custom_fields",
+ item_key="custom_field",
+ supports_sort=True,
+ ),
+ "deal_fields": OnepagecrmEndpointConfig(
+ name="deal_fields",
+ path="/deal_fields",
+ data_key="deal_fields",
+ item_key="deal_field",
+ ),
}
ENDPOINTS = tuple(ONEPAGECRM_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.py
index 15005ff652a0..5c3d445cead1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/source.py
@@ -57,7 +57,7 @@ class OnepagecrmSource(ResumableSource[OnepagecrmSourceConfig, OnepagecrmResumeC
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your OnePageCRM API credentials to pull your CRM data into the PostHog Data warehouse.
-You can find your User ID and API key in [OnePageCRM](https://app.onepagecrm.com) under **Apps & Integrations → API**. The key grants read access to your contacts, companies, deals, actions, notes, calls, meetings, users, statuses, and lead sources.
+You can find your User ID and API key in [OnePageCRM](https://app.onepagecrm.com) under **Apps & Integrations → API**. The key grants read access to your contacts, companies, deals, actions, notes, calls, meetings, users, statuses, lead sources, call results, pipelines, and custom field definitions.
""",
iconPath="/static/services/onepagecrm.png",
docsUrl="https://posthog.com/docs/cdp/sources/onepagecrm",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.py
index 46edebb3afd7..584c9ef616fb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/tests/test_onepagecrm.py
@@ -124,17 +124,46 @@ class TestPagination:
("users", {"data": [{"user": {"id": "u1"}}, {"user": {"id": "u2"}}]}, ["u1", "u2"]),
("statuses", {"data": [{"status": {"id": "s1"}}]}, ["s1"]),
("lead_sources", {"data": [{"id": "advertisement"}, {"id": "web"}]}, ["advertisement", "web"]),
+ (
+ "call_results",
+ {"data": [{"call_result": {"result": "interested"}}, {"call_result": {"result": "no_answer"}}]},
+ ["interested", "no_answer"],
+ ),
]
)
def test_config_endpoints_yield_bare_array_records(
self, endpoint: str, body: dict[str, Any], expected_ids: list[str]
) -> None:
result = _driver().run(endpoint, [ScriptedResponse(json=body)])
+ primary_key = ONEPAGECRM_ENDPOINTS[endpoint].primary_keys[0]
assert result.raised is None
- assert [row["id"] for row in result.rows] == expected_ids
+ assert [row[primary_key] for row in result.rows] == expected_ids
assert len(result.requests) == 1
+ @parameterized.expand(
+ [
+ ("pipelines", "pipeline", None),
+ ("custom_fields", "custom_field", "created_at"),
+ ("deal_fields", "deal_field", None),
+ ]
+ )
+ def test_lookup_endpoints_walk_pages_and_only_sort_where_documented(
+ self, endpoint: str, item_key: str, expected_sort_by: str | None
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ [
+ _list_page(endpoint, item_key, [{"id": "f1"}], page=1, max_page=2),
+ _list_page(endpoint, item_key, [{"id": "f2"}], page=2, max_page=2),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["f1", "f2"]
+ assert result.params("page") == ["1", "2"]
+ assert result.params("sort_by") == [expected_sort_by, expected_sort_by]
+
def test_missing_data_key_raises_loudly(self) -> None:
result = _driver().run("contacts", [ScriptedResponse(json={"status": 0, "message": "OK"})])