Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the SafetyCulture source (#129117)

Tom Owersf671d4f871b4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 3c3ad0cc35d7..36b8525ca94d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10522,10 +10522,10 @@ Today (11): `actions`, `assets`, `groups`, `inspection_items`, `inspections`, `i
Diffed against: <https://developer.safetyculture.com/reference/thepubservice_feedactions>
-- [ ] `feed/group_users` — membership table joining the users and groups we already sync (high)
-- [ ] `feed/site_members` — membership table joining users to the sites we already sync (high)
-- [ ] `feed/action_assignees` — resolves who each synced action is assigned to; actions are unattributable without it (high)
-- [ ] `feed/issue_assignees` — same attribution join for the issues we already sync (high)
+- [x] `feed/group_users` — membership table joining the users and groups we already sync (high)
+- [x] `feed/site_members` — membership table joining users to the sites we already sync (high)
+- [x] `feed/action_assignees` — resolves who each synced action is assigned to; actions are unattributable without it (high)
+- [x] `feed/issue_assignees` — same attribution join for the issues we already sync (high)
- [ ] `feed/schedule_occurrences` — actual scheduled occurrences, the only way to measure scheduled vs completed inspections (high)
- [ ] `feed/action_timeline_items` — state-transition history for actions, the basis of cycle-time analysis (high)
- [ ] `feed/training_course_progress` — training completion per user, a headline compliance metric (high)
diff --git a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py
index 11a220c62aa4..2022644f0df2 100644
--- a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py
@@ -239,4 +239,46 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"creator_user_id": "ID of the creator user.",
},
},
+ "group_users": {
+ "description": "A membership row linking a user to a group in your SafetyCulture organization.",
+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedgroupusers",
+ "columns": {
+ "user_id": "ID of the user.",
+ "group_id": "ID of the group.",
+ "organisation_id": "ID of the organisation.",
+ },
+ },
+ "site_members": {
+ "description": "A membership row linking a member to a site in your SafetyCulture organization.",
+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedsitemembers",
+ "columns": {
+ "site_id": "ID of the site.",
+ "member_id": "ID of the member.",
+ },
+ },
+ "action_assignees": {
+ "description": "An assignee of an action, one row per action and assignee.",
+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedactionassignees",
+ "columns": {
+ "id": "Unique action assignee ID.",
+ "action_id": "ID of the action.",
+ "assignee_id": "ID of the assignee.",
+ "name": "Name of the assignee.",
+ "organisation_id": "Organisation that owns this action.",
+ "type": "Type of assignee.",
+ },
+ },
+ "issue_assignees": {
+ "description": "An assignee of an issue, one row per issue and assignee.",
+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedissueassignees",
+ "columns": {
+ "id": "Unique issue assignee ID.",
+ "issue_id": "ID of the issue.",
+ "assignee_id": "ID of the assignee.",
+ "name": "Name of the assignee.",
+ "organisation_id": "Organisation that owns this issue.",
+ "modified_at": "The date this assignee was modified.",
+ "type": "Type of assignee.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/safetyculture/settings.py b/products/warehouse_sources/sources/safetyculture/settings.py
index 916859c21a4e..5041052674ea 100644
--- a/products/warehouse_sources/sources/safetyculture/settings.py
+++ b/products/warehouse_sources/sources/safetyculture/settings.py
@@ -116,6 +116,37 @@ SAFETYCULTURE_ENDPOINTS: dict[str, SafetyCultureEndpointConfig] = {
# partition key.
partition_key=None,
),
+ # Membership and assignee join feeds. None exposes `created_at`, and a full refresh is the only
+ # way to drop a removed membership or assignment, so they all sync full refresh.
+ "group_users": SafetyCultureEndpointConfig(
+ name="group_users",
+ path="/feed/group_users",
+ supports_incremental=False,
+ partition_key=None,
+ primary_keys=["group_id", "user_id"],
+ ),
+ "site_members": SafetyCultureEndpointConfig(
+ name="site_members",
+ path="/feed/site_members",
+ supports_incremental=False,
+ partition_key=None,
+ primary_keys=["site_id", "member_id"],
+ ),
+ "action_assignees": SafetyCultureEndpointConfig(
+ name="action_assignees",
+ path="/feed/action_assignees",
+ # The feed accepts `modified_after` (on the parent action), but its rows carry no
+ # `modified_at` to hold the watermark.
+ supports_incremental=False,
+ partition_key=None,
+ ),
+ "issue_assignees": SafetyCultureEndpointConfig(
+ name="issue_assignees",
+ path="/feed/issue_assignees",
+ # The issue assignees feed documents no `modified_after` param, so it's full refresh only.
+ supports_incremental=False,
+ partition_key=None,
+ ),
}
ENDPOINTS = tuple(SAFETYCULTURE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py
index c04d0b97271f..36edce233d53 100644
--- a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py
+++ b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py
@@ -312,8 +312,32 @@ class TestSafetyCultureSourceResponse:
assert response.partition_mode is None
assert response.partition_keys is None
+ @parameterized.expand(
+ [
+ ("group_users", {"user_id": "user_1", "group_id": "role_1", "organisation_id": "role_org"}),
+ ("site_members", {"site_id": "location_1", "member_id": "user_1"}),
+ (
+ "action_assignees",
+ {"id": "aa_1", "action_id": "audit_1", "assignee_id": "user_1", "name": "Ada", "type": "USER"},
+ ),
+ (
+ "issue_assignees",
+ {"id": "ia_1", "issue_id": "issue_1", "assignee_id": "user_1", "modified_at": "2024-03-01T00:00:00Z"},
+ ),
+ ]
+ )
+ def test_join_feed_primary_keys_exist_on_documented_rows(self, endpoint: str, row: dict[str, Any]) -> None:
+ # The membership feeds have no `id`, so the default primary key would make every merge fail.
+ result = _driver().run(endpoint, [_response([row])])
+
+ assert result.response is not None
+ assert result.rows == [row]
+ assert result.response.primary_keys
+ assert all(row.get(key) for key in result.response.primary_keys)
+
def test_incremental_endpoints_match_documented_modified_after_support(self) -> None:
- # Only these four feeds document a server-side modified_after filter; flipping any other
- # endpoint to incremental would silently ship a full-refresh-cost "incremental" sync.
+ # Only these four feeds document a modified_after filter that keys off a `modified_at` column;
+ # action_assignees accepts the param but its rows carry no `modified_at` for the watermark.
+ # Flipping any other endpoint to incremental would ship a full-refresh-cost "incremental" sync.
incremental = {name for name, config in SAFETYCULTURE_ENDPOINTS.items() if config.supports_incremental}
assert incremental == {"inspections", "inspection_items", "templates", "actions"}