:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): sync every help scout conversation (#123458)
Daniel RC37aac89987fe
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.pyindex 6f28952e752c..006a58f56a08 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.py@@ -49,7 +49,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"lastName": "Customer's last name.","photoUrl": "URL of the customer's photo.","createdAt": "Time the customer was created, in ISO 8601 format.",- "modifiedAt": "Time the customer was last modified, in ISO 8601 format.",+ "updatedAt": "Time the customer was last updated, in ISO 8601 format.",},},"mailboxes": {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.pyindex f24b7b49dc47..4b99b4610958 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.py@@ -101,7 +101,6 @@ def _list_params(config: HelpScoutEndpointConfig,use_v3_route: bool,should_use_incremental_field: bool,- incremental_field: Optional[str],db_incremental_field_last_value: Any,) -> dict[str, Any]:if use_v3_route:@@ -114,10 +113,10 @@ def _list_params(return {}if should_use_incremental_field and config.updated_since_param and db_incremental_field_last_value is not None:- sort_field = incremental_field or config.default_incremental_field or "modifiedAt"return {config.updated_since_param: _format_since(db_incremental_field_last_value),- "sortField": sort_field,+ # `modifiedSince` filters on `modifiedAt`, which is a valid sort but not a returned field.+ "sortField": "modifiedAt","sortOrder": "asc",}@@ -178,7 +177,6 @@ def _top_level_resource(job_id: str,resumable_source_manager: ResumableSourceManager[HelpScoutResumeConfig],should_use_incremental_field: bool,- incremental_field: Optional[str],db_incremental_field_last_value: Any,) -> Resource:rest_config: RESTAPIConfig = {@@ -194,7 +192,6 @@ def _top_level_resource(config,use_v3_route,should_use_incremental_field,- incremental_field,db_incremental_field_last_value,),},@@ -281,7 +278,6 @@ def helpscout_source(resumable_source_manager: ResumableSourceManager[HelpScoutResumeConfig],api_version: str,should_use_incremental_field: bool = False,- incremental_field: Optional[str] = None,db_incremental_field_last_value: Optional[Any] = None,) -> SourceResponse:config = HELP_SCOUT_ENDPOINTS[endpoint]@@ -301,7 +297,6 @@ def helpscout_source(job_id,resumable_source_manager,should_use_incremental_field,- incremental_field,db_incremental_field_last_value,)@@ -314,9 +309,10 @@ def helpscout_source(partition_mode="datetime" if config.partition_key else None,partition_format="week" if config.partition_key else None,partition_keys=[config.partition_key] if config.partition_key else None,- # v3 lists come back newest-first by createdAt, not ordered by the modifiedAt watermark,- # so the pipeline must only persist the watermark once a sync completes.- sort_mode="desc" if use_v3_route and config.updated_since_param else "asc",+ # Neither route reads in watermark order: v3 lists newest-first by createdAt, and v2 sorts by+ # modifiedAt, which rows don't carry. So the pipeline must only persist the watermark once a+ # sync completes.+ sort_mode="desc" if config.updated_since_param else "asc",)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.pyindex c43ade578102..9b9e4c2212a1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.py@@ -47,23 +47,29 @@ class HelpScoutEndpointConfig:is_report: bool = False+# The conversations list returns only active conversations unless a status is sent.+_ALL_CONVERSATION_STATUSES: dict[str, Any] = {"status": "all"}+HELP_SCOUT_ENDPOINTS: dict[str, HelpScoutEndpointConfig] = {"conversations": HelpScoutEndpointConfig(name="conversations",path="/conversations",embedded_key="conversations",- incremental_fields=[incremental_field("modifiedAt")],- default_incremental_field="modifiedAt",+ # Rows carry no `modifiedAt`. Only some changes move `userUpdatedAt`, so a watermark on it+ # lags `modifiedSince` and a sync re-reads rows rather than skipping them.+ incremental_fields=[incremental_field("userUpdatedAt")],+ default_incremental_field="userUpdatedAt",updated_since_param="modifiedSince",supports_sort=True,has_v3_route=True,+ params=_ALL_CONVERSATION_STATUSES,),"customers": HelpScoutEndpointConfig(name="customers",path="/customers",embedded_key="customers",- incremental_fields=[incremental_field("modifiedAt")],- default_incremental_field="modifiedAt",+ incremental_fields=[incremental_field("updatedAt")],+ default_incremental_field="updatedAt",updated_since_param="modifiedSince",supports_sort=True,has_v3_route=True,@@ -102,6 +108,7 @@ HELP_SCOUT_ENDPOINTS: dict[str, HelpScoutEndpointConfig] = {resolve_field="id",include_from_parent=["id"],parent_field_renames={"id": "conversation_id"},+ parent_params=_ALL_CONVERSATION_STATUSES,),),"mailbox_fields": HelpScoutEndpointConfig(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.pyindex f2635c418f23..c7f5216ddfc3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py@@ -162,7 +162,6 @@ class HelpScoutSource(OAuthMixin, ResumableSource[HelpScoutSourceConfig, HelpScoresumable_source_manager=resumable_source_manager,api_version=self.resolve_api_version(inputs.api_version),should_use_incremental_field=inputs.should_use_incremental_field,- incremental_field=inputs.incremental_field,db_incremental_field_last_value=inputs.db_incremental_field_last_valueif inputs.should_use_incremental_fieldelse None,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.pyindex f62539578d46..89a4dbe976d6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.py@@ -1,4 +1,5 @@from typing import Any+from urllib.parse import urlsplitimport pytestfrom unittest import mock@@ -176,11 +177,12 @@ class TestTopLevelPagination:class TestIncrementalParams:def test_conversations_incremental_sends_modified_since_and_sort(self) -> None:+ # The incremental field is not a valid v2 sortField, so the sort must stay on modifiedAt.result = _run("conversations",[_page("conversations", [])],api_version="v2",- incremental_field="modifiedAt",+ incremental_field="userUpdatedAt",db_incremental_field_last_value="2024-01-01T00:00:00Z",)@@ -192,6 +194,7 @@ class TestIncrementalParams:result = _run("conversations", [_page("conversations", [])], api_version="v2")assert result.requests[0].param("modifiedSince") is None+ assert result.requests[0].param("status") == "all"assert result.requests[0].param("sortField") == "createdAt"assert result.requests[0].param("sortOrder") == "asc"@@ -200,14 +203,19 @@ class TestIncrementalParams:"customers",[_page("customers", [])],api_version="v2",- incremental_field="modifiedAt",+ incremental_field="updatedAt",db_incremental_field_last_value="2024-06-01T00:00:00Z",)assert result.requests[0].param("modifiedSince") == "2024-06-01T00:00:00Z"- @pytest.mark.parametrize("endpoint", ["conversations", "customers"])- def test_v3_incremental_sends_only_modified_since(self, endpoint: str) -> None:+ @pytest.mark.parametrize(+ "endpoint,static_query",+ [("conversations", {"status": ("all",)}), ("customers", {})],+ )+ def test_v3_incremental_sends_only_modified_since(+ self, endpoint: str, static_query: dict[str, tuple[str, ...]]+ ) -> None:result = _run(endpoint,[_page(endpoint, [])],@@ -216,14 +224,23 @@ class TestIncrementalParams:db_incremental_field_last_value="2024-01-01T00:00:00Z",)- assert result.queries[0] == {"modifiedSince": ("2024-01-01T00:00:00Z",)}+ assert result.queries[0] == {**static_query, "modifiedSince": ("2024-01-01T00:00:00Z",)}- @pytest.mark.parametrize("endpoint", ["conversations", "customers"])- def test_v3_full_refresh_sends_no_sort_params(self, endpoint: str) -> None:+ @pytest.mark.parametrize(+ "endpoint,expected_query",+ [+ # The API returns only active conversations unless asked for every status.+ ("conversations", {"status": ("all",)}),+ ("customers", {}),+ ],+ )+ def test_v3_full_refresh_sends_no_sort_params(+ self, endpoint: str, expected_query: dict[str, tuple[str, ...]]+ ) -> None:# v3 rejects sortField/sortOrder with a 400, so the v2 full-refresh sort must not leak in.result = _run(endpoint, [_page(endpoint, [])], api_version="v3")- assert result.queries[0] == {}+ assert result.queries[0] == expected_query@pytest.mark.parametrize("endpoint,expected_query",@@ -266,17 +283,30 @@ class TestRetries:class TestFanout:@pytest.mark.parametrize(- "api_version,endpoint,parent,child_path,base",+ "api_version,endpoint,parent,child_path,base,parent_query",[- ("v2", "threads", "conversations", "conversations/42/threads", HELP_SCOUT_API_BASE),- ("v3", "threads", "conversations", "conversations/42/threads", HELP_SCOUT_API_BASE_V3),- ("v2", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE),+ ("v2", "threads", "conversations", "conversations/42/threads", HELP_SCOUT_API_BASE, {"status": ("all",)}),+ (+ "v3",+ "threads",+ "conversations",+ "conversations/42/threads",+ HELP_SCOUT_API_BASE_V3,+ {"status": ("all",)},+ ),+ ("v2", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE, {}),# Mailbox fields have no v3 route, so a v3 pin keeps parent and child on v2.- ("v3", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE),+ ("v3", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE, {}),],)def test_parent_and_child_endpoints(- self, api_version: str, endpoint: str, parent: str, child_path: str, base: str+ self,+ api_version: str,+ endpoint: str,+ parent: str,+ child_path: str,+ base: str,+ parent_query: dict[str, tuple[str, ...]],) -> None:result = _run(endpoint,@@ -285,8 +315,9 @@ class TestFanout:)assert result.raised is None- assert result.urls == [f"{base}/{parent}", f"{base}/{child_path}"]- assert result.queries == [{}, {}]+ base_path = urlsplit(base).path+ assert result.paths == [f"{base_path}/{parent}", f"{base_path}/{child_path}"]+ assert result.queries == [parent_query, {}]@pytest.mark.parametrize("endpoint,parent,parent_key",@@ -366,15 +397,10 @@ class TestHappinessRatingsReport:class TestSourceResponseShape:- @pytest.mark.parametrize(- "api_version,desc_endpoints",- [- ("v2", {}),- # v3 lists are newest-first, not ordered by the modifiedAt watermark.- ("v3", {"conversations": "desc", "customers": "desc"}),- ],- )- def test_response_metadata_per_top_level_endpoint(self, api_version: str, desc_endpoints: dict[str, str]) -> None:+ @pytest.mark.parametrize("api_version", ["v2", "v3"])+ def test_response_metadata_per_top_level_endpoint(self, api_version: str) -> None:+ # Neither route reads in watermark order, so an "asc" claim would let a resumed sync skip rows.+ desc_endpoints = {"conversations": "desc", "customers": "desc"}for endpoint in ENDPOINTS:config = HELP_SCOUT_ENDPOINTS[endpoint]if config.fanout or config.is_report:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.pyindex 9bd72be20c29..ba95dff00f9f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.py@@ -65,11 +65,11 @@ class TestHelpScoutSource:schemas = self.source.get_schemas(self.config, self.team_id)assert {schema.name for schema in schemas} == set(ENDPOINTS)- @pytest.mark.parametrize("endpoint", ["conversations", "customers"])- def test_get_schemas_supports_incremental_for_modified_since_endpoints(self, endpoint):+ @pytest.mark.parametrize("endpoint,field", [("conversations", "userUpdatedAt"), ("customers", "updatedAt")])+ def test_get_schemas_supports_incremental_for_modified_since_endpoints(self, endpoint, field):schemas = {s.name: s for s in self.source.get_schemas(self.config, self.team_id)}assert schemas[endpoint].supports_incremental is True- assert [f["field"] for f in schemas[endpoint].incremental_fields] == ["modifiedAt"]+ assert [f["field"] for f in schemas[endpoint].incremental_fields] == [field]@pytest.mark.parametrize("endpoint", ["mailboxes", "users", "tags", "workflows", "threads"])def test_get_schemas_full_refresh_only_for_remaining_endpoints(self, endpoint):@@ -156,7 +156,6 @@ class TestHelpScoutSource:inputs = mock.MagicMock()inputs.schema_name = "conversations"inputs.should_use_incremental_field = True- inputs.incremental_field = "modifiedAt"inputs.db_incremental_field_last_value = "2024-01-01T00:00:00Z"inputs.api_version = pinned_versionmanager = mock.MagicMock()@@ -169,7 +168,6 @@ class TestHelpScoutSource:assert kwargs["endpoint"] == "conversations"assert kwargs["resumable_source_manager"] is managerassert kwargs["should_use_incremental_field"] is True- assert kwargs["incremental_field"] == "modifiedAt"assert kwargs["db_incremental_field_last_value"] == "2024-01-01T00:00:00Z"assert kwargs["api_version"] == expected_version