:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the SafetyCulture source (#129117)
Tom Owersf671d4f871b4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 3c3ad0cc35d7..36b8525ca94d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10522,10 +10522,10 @@ Today (11): `actions`, `assets`, `groups`, `inspection_items`, `inspections`, `iDiffed against: <https://developer.safetyculture.com/reference/thepubservice_feedactions>-- [ ] `feed/group_users` — membership table joining the users and groups we already sync (high)-- [ ] `feed/site_members` — membership table joining users to the sites we already sync (high)-- [ ] `feed/action_assignees` — resolves who each synced action is assigned to; actions are unattributable without it (high)-- [ ] `feed/issue_assignees` — same attribution join for the issues we already sync (high)+- [x] `feed/group_users` — membership table joining the users and groups we already sync (high)+- [x] `feed/site_members` — membership table joining users to the sites we already sync (high)+- [x] `feed/action_assignees` — resolves who each synced action is assigned to; actions are unattributable without it (high)+- [x] `feed/issue_assignees` — same attribution join for the issues we already sync (high)- [ ] `feed/schedule_occurrences` — actual scheduled occurrences, the only way to measure scheduled vs completed inspections (high)- [ ] `feed/action_timeline_items` — state-transition history for actions, the basis of cycle-time analysis (high)- [ ] `feed/training_course_progress` — training completion per user, a headline compliance metric (high)diff --git a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.pyindex 11a220c62aa4..2022644f0df2 100644--- a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py+++ b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py@@ -239,4 +239,46 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"creator_user_id": "ID of the creator user.",},},+ "group_users": {+ "description": "A membership row linking a user to a group in your SafetyCulture organization.",+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedgroupusers",+ "columns": {+ "user_id": "ID of the user.",+ "group_id": "ID of the group.",+ "organisation_id": "ID of the organisation.",+ },+ },+ "site_members": {+ "description": "A membership row linking a member to a site in your SafetyCulture organization.",+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedsitemembers",+ "columns": {+ "site_id": "ID of the site.",+ "member_id": "ID of the member.",+ },+ },+ "action_assignees": {+ "description": "An assignee of an action, one row per action and assignee.",+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedactionassignees",+ "columns": {+ "id": "Unique action assignee ID.",+ "action_id": "ID of the action.",+ "assignee_id": "ID of the assignee.",+ "name": "Name of the assignee.",+ "organisation_id": "Organisation that owns this action.",+ "type": "Type of assignee.",+ },+ },+ "issue_assignees": {+ "description": "An assignee of an issue, one row per issue and assignee.",+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedissueassignees",+ "columns": {+ "id": "Unique issue assignee ID.",+ "issue_id": "ID of the issue.",+ "assignee_id": "ID of the assignee.",+ "name": "Name of the assignee.",+ "organisation_id": "Organisation that owns this issue.",+ "modified_at": "The date this assignee was modified.",+ "type": "Type of assignee.",+ },+ },}diff --git a/products/warehouse_sources/sources/safetyculture/settings.py b/products/warehouse_sources/sources/safetyculture/settings.pyindex 916859c21a4e..5041052674ea 100644--- a/products/warehouse_sources/sources/safetyculture/settings.py+++ b/products/warehouse_sources/sources/safetyculture/settings.py@@ -116,6 +116,37 @@ SAFETYCULTURE_ENDPOINTS: dict[str, SafetyCultureEndpointConfig] = {# partition key.partition_key=None,),+ # Membership and assignee join feeds. None exposes `created_at`, and a full refresh is the only+ # way to drop a removed membership or assignment, so they all sync full refresh.+ "group_users": SafetyCultureEndpointConfig(+ name="group_users",+ path="/feed/group_users",+ supports_incremental=False,+ partition_key=None,+ primary_keys=["group_id", "user_id"],+ ),+ "site_members": SafetyCultureEndpointConfig(+ name="site_members",+ path="/feed/site_members",+ supports_incremental=False,+ partition_key=None,+ primary_keys=["site_id", "member_id"],+ ),+ "action_assignees": SafetyCultureEndpointConfig(+ name="action_assignees",+ path="/feed/action_assignees",+ # The feed accepts `modified_after` (on the parent action), but its rows carry no+ # `modified_at` to hold the watermark.+ supports_incremental=False,+ partition_key=None,+ ),+ "issue_assignees": SafetyCultureEndpointConfig(+ name="issue_assignees",+ path="/feed/issue_assignees",+ # The issue assignees feed documents no `modified_after` param, so it's full refresh only.+ supports_incremental=False,+ partition_key=None,+ ),}ENDPOINTS = tuple(SAFETYCULTURE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.pyindex c04d0b97271f..36edce233d53 100644--- a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py+++ b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py@@ -312,8 +312,32 @@ class TestSafetyCultureSourceResponse:assert response.partition_mode is Noneassert response.partition_keys is None+ @parameterized.expand(+ [+ ("group_users", {"user_id": "user_1", "group_id": "role_1", "organisation_id": "role_org"}),+ ("site_members", {"site_id": "location_1", "member_id": "user_1"}),+ (+ "action_assignees",+ {"id": "aa_1", "action_id": "audit_1", "assignee_id": "user_1", "name": "Ada", "type": "USER"},+ ),+ (+ "issue_assignees",+ {"id": "ia_1", "issue_id": "issue_1", "assignee_id": "user_1", "modified_at": "2024-03-01T00:00:00Z"},+ ),+ ]+ )+ def test_join_feed_primary_keys_exist_on_documented_rows(self, endpoint: str, row: dict[str, Any]) -> None:+ # The membership feeds have no `id`, so the default primary key would make every merge fail.+ result = _driver().run(endpoint, [_response([row])])++ assert result.response is not None+ assert result.rows == [row]+ assert result.response.primary_keys+ assert all(row.get(key) for key in result.response.primary_keys)+def test_incremental_endpoints_match_documented_modified_after_support(self) -> None:- # Only these four feeds document a server-side modified_after filter; flipping any other- # endpoint to incremental would silently ship a full-refresh-cost "incremental" sync.+ # Only these four feeds document a modified_after filter that keys off a `modified_at` column;+ # action_assignees accepts the param but its rows carry no `modified_at` for the watermark.+ # Flipping any other endpoint to incremental would ship a full-refresh-cost "incremental" sync.incremental = {name for name, config in SAFETYCULTURE_ENDPOINTS.items() if config.supports_incremental}assert incremental == {"inspections", "inspection_items", "templates", "actions"}