Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): sync every help scout conversation (#123458)

Daniel RC37aac89987fe
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.py
index 6f28952e752c..006a58f56a08 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/canonical_descriptions.py
@@ -49,7 +49,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"lastName": "Customer's last name.",
"photoUrl": "URL of the customer's photo.",
"createdAt": "Time the customer was created, in ISO 8601 format.",
- "modifiedAt": "Time the customer was last modified, in ISO 8601 format.",
+ "updatedAt": "Time the customer was last updated, in ISO 8601 format.",
},
},
"mailboxes": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.py
index f24b7b49dc47..4b99b4610958 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/helpscout.py
@@ -101,7 +101,6 @@ def _list_params(
config: HelpScoutEndpointConfig,
use_v3_route: bool,
should_use_incremental_field: bool,
- incremental_field: Optional[str],
db_incremental_field_last_value: Any,
) -> dict[str, Any]:
if use_v3_route:
@@ -114,10 +113,10 @@ def _list_params(
return {}
if should_use_incremental_field and config.updated_since_param and db_incremental_field_last_value is not None:
- sort_field = incremental_field or config.default_incremental_field or "modifiedAt"
return {
config.updated_since_param: _format_since(db_incremental_field_last_value),
- "sortField": sort_field,
+ # `modifiedSince` filters on `modifiedAt`, which is a valid sort but not a returned field.
+ "sortField": "modifiedAt",
"sortOrder": "asc",
}
@@ -178,7 +177,6 @@ def _top_level_resource(
job_id: str,
resumable_source_manager: ResumableSourceManager[HelpScoutResumeConfig],
should_use_incremental_field: bool,
- incremental_field: Optional[str],
db_incremental_field_last_value: Any,
) -> Resource:
rest_config: RESTAPIConfig = {
@@ -194,7 +192,6 @@ def _top_level_resource(
config,
use_v3_route,
should_use_incremental_field,
- incremental_field,
db_incremental_field_last_value,
),
},
@@ -281,7 +278,6 @@ def helpscout_source(
resumable_source_manager: ResumableSourceManager[HelpScoutResumeConfig],
api_version: str,
should_use_incremental_field: bool = False,
- incremental_field: Optional[str] = None,
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = HELP_SCOUT_ENDPOINTS[endpoint]
@@ -301,7 +297,6 @@ def helpscout_source(
job_id,
resumable_source_manager,
should_use_incremental_field,
- incremental_field,
db_incremental_field_last_value,
)
@@ -314,9 +309,10 @@ def helpscout_source(
partition_mode="datetime" if config.partition_key else None,
partition_format="week" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
- # v3 lists come back newest-first by createdAt, not ordered by the modifiedAt watermark,
- # so the pipeline must only persist the watermark once a sync completes.
- sort_mode="desc" if use_v3_route and config.updated_since_param else "asc",
+ # Neither route reads in watermark order: v3 lists newest-first by createdAt, and v2 sorts by
+ # modifiedAt, which rows don't carry. So the pipeline must only persist the watermark once a
+ # sync completes.
+ sort_mode="desc" if config.updated_since_param else "asc",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.py
index c43ade578102..9b9e4c2212a1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/settings.py
@@ -47,23 +47,29 @@ class HelpScoutEndpointConfig:
is_report: bool = False
+# The conversations list returns only active conversations unless a status is sent.
+_ALL_CONVERSATION_STATUSES: dict[str, Any] = {"status": "all"}
+
HELP_SCOUT_ENDPOINTS: dict[str, HelpScoutEndpointConfig] = {
"conversations": HelpScoutEndpointConfig(
name="conversations",
path="/conversations",
embedded_key="conversations",
- incremental_fields=[incremental_field("modifiedAt")],
- default_incremental_field="modifiedAt",
+ # Rows carry no `modifiedAt`. Only some changes move `userUpdatedAt`, so a watermark on it
+ # lags `modifiedSince` and a sync re-reads rows rather than skipping them.
+ incremental_fields=[incremental_field("userUpdatedAt")],
+ default_incremental_field="userUpdatedAt",
updated_since_param="modifiedSince",
supports_sort=True,
has_v3_route=True,
+ params=_ALL_CONVERSATION_STATUSES,
),
"customers": HelpScoutEndpointConfig(
name="customers",
path="/customers",
embedded_key="customers",
- incremental_fields=[incremental_field("modifiedAt")],
- default_incremental_field="modifiedAt",
+ incremental_fields=[incremental_field("updatedAt")],
+ default_incremental_field="updatedAt",
updated_since_param="modifiedSince",
supports_sort=True,
has_v3_route=True,
@@ -102,6 +108,7 @@ HELP_SCOUT_ENDPOINTS: dict[str, HelpScoutEndpointConfig] = {
resolve_field="id",
include_from_parent=["id"],
parent_field_renames={"id": "conversation_id"},
+ parent_params=_ALL_CONVERSATION_STATUSES,
),
),
"mailbox_fields": HelpScoutEndpointConfig(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py
index f2635c418f23..c7f5216ddfc3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/source.py
@@ -162,7 +162,6 @@ class HelpScoutSource(OAuthMixin, ResumableSource[HelpScoutSourceConfig, HelpSco
resumable_source_manager=resumable_source_manager,
api_version=self.resolve_api_version(inputs.api_version),
should_use_incremental_field=inputs.should_use_incremental_field,
- incremental_field=inputs.incremental_field,
db_incremental_field_last_value=inputs.db_incremental_field_last_value
if inputs.should_use_incremental_field
else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.py
index f62539578d46..89a4dbe976d6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout.py
@@ -1,4 +1,5 @@
from typing import Any
+from urllib.parse import urlsplit
import pytest
from unittest import mock
@@ -176,11 +177,12 @@ class TestTopLevelPagination:
class TestIncrementalParams:
def test_conversations_incremental_sends_modified_since_and_sort(self) -> None:
+ # The incremental field is not a valid v2 sortField, so the sort must stay on modifiedAt.
result = _run(
"conversations",
[_page("conversations", [])],
api_version="v2",
- incremental_field="modifiedAt",
+ incremental_field="userUpdatedAt",
db_incremental_field_last_value="2024-01-01T00:00:00Z",
)
@@ -192,6 +194,7 @@ class TestIncrementalParams:
result = _run("conversations", [_page("conversations", [])], api_version="v2")
assert result.requests[0].param("modifiedSince") is None
+ assert result.requests[0].param("status") == "all"
assert result.requests[0].param("sortField") == "createdAt"
assert result.requests[0].param("sortOrder") == "asc"
@@ -200,14 +203,19 @@ class TestIncrementalParams:
"customers",
[_page("customers", [])],
api_version="v2",
- incremental_field="modifiedAt",
+ incremental_field="updatedAt",
db_incremental_field_last_value="2024-06-01T00:00:00Z",
)
assert result.requests[0].param("modifiedSince") == "2024-06-01T00:00:00Z"
- @pytest.mark.parametrize("endpoint", ["conversations", "customers"])
- def test_v3_incremental_sends_only_modified_since(self, endpoint: str) -> None:
+ @pytest.mark.parametrize(
+ "endpoint,static_query",
+ [("conversations", {"status": ("all",)}), ("customers", {})],
+ )
+ def test_v3_incremental_sends_only_modified_since(
+ self, endpoint: str, static_query: dict[str, tuple[str, ...]]
+ ) -> None:
result = _run(
endpoint,
[_page(endpoint, [])],
@@ -216,14 +224,23 @@ class TestIncrementalParams:
db_incremental_field_last_value="2024-01-01T00:00:00Z",
)
- assert result.queries[0] == {"modifiedSince": ("2024-01-01T00:00:00Z",)}
+ assert result.queries[0] == {**static_query, "modifiedSince": ("2024-01-01T00:00:00Z",)}
- @pytest.mark.parametrize("endpoint", ["conversations", "customers"])
- def test_v3_full_refresh_sends_no_sort_params(self, endpoint: str) -> None:
+ @pytest.mark.parametrize(
+ "endpoint,expected_query",
+ [
+ # The API returns only active conversations unless asked for every status.
+ ("conversations", {"status": ("all",)}),
+ ("customers", {}),
+ ],
+ )
+ def test_v3_full_refresh_sends_no_sort_params(
+ self, endpoint: str, expected_query: dict[str, tuple[str, ...]]
+ ) -> None:
# v3 rejects sortField/sortOrder with a 400, so the v2 full-refresh sort must not leak in.
result = _run(endpoint, [_page(endpoint, [])], api_version="v3")
- assert result.queries[0] == {}
+ assert result.queries[0] == expected_query
@pytest.mark.parametrize(
"endpoint,expected_query",
@@ -266,17 +283,30 @@ class TestRetries:
class TestFanout:
@pytest.mark.parametrize(
- "api_version,endpoint,parent,child_path,base",
+ "api_version,endpoint,parent,child_path,base,parent_query",
[
- ("v2", "threads", "conversations", "conversations/42/threads", HELP_SCOUT_API_BASE),
- ("v3", "threads", "conversations", "conversations/42/threads", HELP_SCOUT_API_BASE_V3),
- ("v2", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE),
+ ("v2", "threads", "conversations", "conversations/42/threads", HELP_SCOUT_API_BASE, {"status": ("all",)}),
+ (
+ "v3",
+ "threads",
+ "conversations",
+ "conversations/42/threads",
+ HELP_SCOUT_API_BASE_V3,
+ {"status": ("all",)},
+ ),
+ ("v2", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE, {}),
# Mailbox fields have no v3 route, so a v3 pin keeps parent and child on v2.
- ("v3", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE),
+ ("v3", "mailbox_fields", "mailboxes", "mailboxes/42/fields", HELP_SCOUT_API_BASE, {}),
],
)
def test_parent_and_child_endpoints(
- self, api_version: str, endpoint: str, parent: str, child_path: str, base: str
+ self,
+ api_version: str,
+ endpoint: str,
+ parent: str,
+ child_path: str,
+ base: str,
+ parent_query: dict[str, tuple[str, ...]],
) -> None:
result = _run(
endpoint,
@@ -285,8 +315,9 @@ class TestFanout:
)
assert result.raised is None
- assert result.urls == [f"{base}/{parent}", f"{base}/{child_path}"]
- assert result.queries == [{}, {}]
+ base_path = urlsplit(base).path
+ assert result.paths == [f"{base_path}/{parent}", f"{base_path}/{child_path}"]
+ assert result.queries == [parent_query, {}]
@pytest.mark.parametrize(
"endpoint,parent,parent_key",
@@ -366,15 +397,10 @@ class TestHappinessRatingsReport:
class TestSourceResponseShape:
- @pytest.mark.parametrize(
- "api_version,desc_endpoints",
- [
- ("v2", {}),
- # v3 lists are newest-first, not ordered by the modifiedAt watermark.
- ("v3", {"conversations": "desc", "customers": "desc"}),
- ],
- )
- def test_response_metadata_per_top_level_endpoint(self, api_version: str, desc_endpoints: dict[str, str]) -> None:
+ @pytest.mark.parametrize("api_version", ["v2", "v3"])
+ def test_response_metadata_per_top_level_endpoint(self, api_version: str) -> None:
+ # Neither route reads in watermark order, so an "asc" claim would let a resumed sync skip rows.
+ desc_endpoints = {"conversations": "desc", "customers": "desc"}
for endpoint in ENDPOINTS:
config = HELP_SCOUT_ENDPOINTS[endpoint]
if config.fanout or config.is_report:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.py
index 9bd72be20c29..ba95dff00f9f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helpscout/tests/test_helpscout_source.py
@@ -65,11 +65,11 @@ class TestHelpScoutSource:
schemas = self.source.get_schemas(self.config, self.team_id)
assert {schema.name for schema in schemas} == set(ENDPOINTS)
- @pytest.mark.parametrize("endpoint", ["conversations", "customers"])
- def test_get_schemas_supports_incremental_for_modified_since_endpoints(self, endpoint):
+ @pytest.mark.parametrize("endpoint,field", [("conversations", "userUpdatedAt"), ("customers", "updatedAt")])
+ def test_get_schemas_supports_incremental_for_modified_since_endpoints(self, endpoint, field):
schemas = {s.name: s for s in self.source.get_schemas(self.config, self.team_id)}
assert schemas[endpoint].supports_incremental is True
- assert [f["field"] for f in schemas[endpoint].incremental_fields] == ["modifiedAt"]
+ assert [f["field"] for f in schemas[endpoint].incremental_fields] == [field]
@pytest.mark.parametrize("endpoint", ["mailboxes", "users", "tags", "workflows", "threads"])
def test_get_schemas_full_refresh_only_for_remaining_endpoints(self, endpoint):
@@ -156,7 +156,6 @@ class TestHelpScoutSource:
inputs = mock.MagicMock()
inputs.schema_name = "conversations"
inputs.should_use_incremental_field = True
- inputs.incremental_field = "modifiedAt"
inputs.db_incremental_field_last_value = "2024-01-01T00:00:00Z"
inputs.api_version = pinned_version
manager = mock.MagicMock()
@@ -169,7 +168,6 @@ class TestHelpScoutSource:
assert kwargs["endpoint"] == "conversations"
assert kwargs["resumable_source_manager"] is manager
assert kwargs["should_use_incremental_field"] is True
- assert kwargs["incremental_field"] == "modifiedAt"
assert kwargs["db_incremental_field_last_value"] == "2024-01-01T00:00:00Z"
assert kwargs["api_version"] == expected_version