:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(safetyculture): add 4 missing tables to the safetyculture source (#129999)
Tom Owers32b1ef816fd6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex eb491208c250..15c8d48ab553 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10526,10 +10526,10 @@ Diffed against: <https://developer.safetyculture.com/reference/thepubservice_fee- [x] `feed/site_members` — membership table joining users to the sites we already sync (high)- [x] `feed/action_assignees` — resolves who each synced action is assigned to; actions are unattributable without it (high)- [x] `feed/issue_assignees` — same attribution join for the issues we already sync (high)-- [ ] `feed/schedule_occurrences` — actual scheduled occurrences, the only way to measure scheduled vs completed inspections (high)-- [ ] `feed/action_timeline_items` — state-transition history for actions, the basis of cycle-time analysis (high)-- [ ] `feed/training_course_progress` — training completion per user, a headline compliance metric (high)-- [ ] `feed/issue_timeline_items` — state-transition history for issues (medium)+- [x] `feed/schedule_occurrences` — actual scheduled occurrences, the only way to measure scheduled vs completed inspections (high)+- [x] `feed/action_timeline_items` — state-transition history for actions, the basis of cycle-time analysis (high)+- [x] `feed/training_course_progress` — training completion per user, a headline compliance metric (high)+- [x] `feed/issue_timeline_items` — state-transition history for issues (medium)- [ ] `feed/activity_log_events` — org-wide audit event stream for usage and adoption analysis (medium)- [ ] `feed/investigations (plus investigation fields and relationships)` — incident investigation records linked to the issues and actions we sync (medium)- [ ] `feed/assets_maintenance, feed/maintenance_plans, feed/maintenance_programs` — maintenance schedule and status for the assets we already sync (medium)diff --git a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.pyindex 2022644f0df2..667372f54727 100644--- a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py+++ b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py@@ -281,4 +281,84 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"type": "Type of assignee.",},},+ "schedule_occurrences": {+ "description": "A scheduled inspection occurrence, one row per occurrence, assignee and site or asset. Compare scheduled against completed inspections with it.",+ "docs_url": "https://developer.safetyculture.com/reference/schedulingfeedservice_feedscheduleoccurrences",+ "columns": {+ "id": "ID of the schedule occurrence for one assignee. A schedule for several sites or assets repeats it once per site or asset.",+ "schedule_id": "ID of the schedule that generated this occurrence.",+ "occurrence_id": "Sequence number of the occurrence within the schedule, per site or asset.",+ "template_id": "ID of the template used for inspections created from this occurrence.",+ "start_time": "When the occurrence becomes available for inspection.",+ "due_time": "Final deadline. After this time the occurrence is missed and cannot be completed.",+ "miss_time": "When the occurrence becomes overdue. Empty when the schedule has no overdue period.",+ "occurrence_status": "Status of the occurrence as a whole: TODO, IN_PROGRESS, COMPLETED, LATE, OVERDUE or MISSED.",+ "audit_id": "ID of the inspection created from this occurrence.",+ "completed_at": "When this occurrence was completed.",+ "completion_rule": "Whether any assignee or all assignees must complete the occurrence (`any` or `all`).",+ "organisation_id": "ID of the organisation that owns the occurrence.",+ "wont_do_note": "Reason given when the occurrence was marked as won't do.",+ "assignee_id": "ID of the user assigned to complete this occurrence.",+ "assignee_from": "How the user was assigned, as `{type}_{id}` where the type is user, group, role or location.",+ "target_id": "ID of the site or asset this occurrence is for. Empty when the schedule has no sites or assets.",+ "target_type": "Type of `target_id`: SITE or ASSET.",+ "completed_by_user_id": "ID of the user who completed the inspection for this assignee.",+ "assignee_status": "Status of this assignee's own work on the occurrence. Uses the same values as `occurrence_status`, plus WONT_DO.",+ },+ },+ "action_timeline_items": {+ "description": "A timeline event on an action, such as a status change, a comment or a field update.",+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedactiontimelineitems",+ "columns": {+ "id": "ID of the timeline item.",+ "task_id": "ID of the action.",+ "organisation_id": "ID of the SafetyCulture organisation.",+ "task_creator_id": "ID of the user who created the action.",+ "task_creator_name": "Name of the user who created the action.",+ "timestamp": "When the timeline item was created, in UTC.",+ "creator_id": "ID of the user who created the timeline item.",+ "creator_name": "Name of the user who created the timeline item.",+ "item_type": "Type of the timeline item.",+ "item_data": "Data of the timeline item.",+ },+ },+ "issue_timeline_items": {+ "description": "A timeline event on an issue, such as a status change, a comment or a field update.",+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedissuetimelineitems",+ "columns": {+ "id": "ID of the timeline item.",+ "task_id": "ID of the issue.",+ "organisation_id": "ID of the SafetyCulture organisation.",+ "task_creator_id": "ID of the user who created the issue.",+ "task_creator_name": "Name of the user who created the issue.",+ "timestamp": "When the timeline item was created, in UTC.",+ "creator_id": "ID of the user who created the timeline item.",+ "creator_name": "Name of the user who created the timeline item.",+ "item_type": "Type of the timeline item.",+ "item_data": "Data of the timeline item.",+ },+ },+ "training_course_progress": {+ "description": "Training course progress, one row per user and course.",+ "docs_url": "https://developer.safetyculture.com/reference/feedservice_feedtrainingcourseprogress",+ "columns": {+ "courseId": "ID of the course.",+ "courseExternalId": "External ID of the course.",+ "courseTitle": "Title of the course.",+ "userId": "ID of the user.",+ "userExternalId": "External ID of the user.",+ "userEmail": "Email address of the user.",+ "userFirstName": "First name of the user.",+ "userLastName": "Last name of the user.",+ "openedAt": "When the user first opened the course.",+ "completedAt": "When the user completed the course.",+ "dueAt": "When the course is due.",+ "completionExpiryDate": "When the course completion expires.",+ "totalLessons": "Number of lessons in the course.",+ "completedLessons": "Number of lessons the user completed.",+ "progressPercent": "Percentage of the course the user completed.",+ "score": "Score of the user on the course.",+ "timeSpentSeconds": "Total time the user spent on the course, in seconds.",+ },+ },}diff --git a/products/warehouse_sources/sources/safetyculture/safetyculture.py b/products/warehouse_sources/sources/safetyculture/safetyculture.pyindex 9e1ab77d0c38..8567236cb2fb 100644--- a/products/warehouse_sources/sources/safetyculture/safetyculture.py+++ b/products/warehouse_sources/sources/safetyculture/safetyculture.py@@ -1,4 +1,5 @@import dataclasses+from collections.abc import Callablefrom datetime import UTC, date, datetimefrom typing import Any, Optional@@ -8,6 +9,7 @@ from products.warehouse_sources.sources.safetyculture.settings import SAFETYCULTfrom products.warehouse_sources.sources.sdk import (BasePaginator,Endpoint,+ EndpointResource,JSONResponsePaginator,RESTAPIConfig,ResumableSourceManager,@@ -134,6 +136,25 @@ class SafetyCultureStructuresPaginator(BasePaginator):request.json = body+def _parse_watermark(value: Any) -> datetime:+ if isinstance(value, datetime):+ return value if value.tzinfo is not None else value.replace(tzinfo=UTC)+ if isinstance(value, date):+ return datetime.combine(value, datetime.min.time(), tzinfo=UTC)+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)+++def _blank_missing_fields(fields: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:+ def blank(row: dict[str, Any]) -> dict[str, Any]:+ for name in fields:+ if row.get(name) is None:+ row[name] = ""+ return row++ return blank++def safetyculture_source(api_token: str,endpoint: str,@@ -150,7 +171,10 @@ def safetyculture_source(# incremental sync that actually has a watermark to filter from.params: dict[str, Any] = dict(config.params)if config.supports_incremental and should_use_incremental_field and db_incremental_field_last_value:- params["modified_after"] = _format_modified_after(db_incremental_field_last_value)+ watermark = db_incremental_field_last_value+ if config.incremental_rewind is not None:+ watermark = _parse_watermark(watermark) - config.incremental_rewind+ params[config.incremental_param] = _format_modified_after(watermark)endpoint_config: Endpointif config.structure_types:@@ -173,6 +197,10 @@ def safetyculture_source("data_selector_malformed_retryable": True,}+ resource_config: EndpointResource = {"name": endpoint, "endpoint": endpoint_config}+ if config.blank_when_missing:+ resource_config["data_map"] = _blank_missing_fields(config.blank_when_missing)+rest_config: RESTAPIConfig = {"client": {"base_url": SAFETYCULTURE_BASE_URL,@@ -188,12 +216,7 @@ def safetyculture_source("allow_redirects": False,},"resource_defaults": {},- "resources": [- {- "name": endpoint,- "endpoint": endpoint_config,- }- ],+ "resources": [resource_config],}initial_paginator_state: Optional[dict[str, Any]] = None@@ -243,7 +266,7 @@ def safetyculture_source(partition_keys=[config.partition_key] if config.partition_key else None,# Incremental feeds paginate by advancing `modified_after` in the server-issued `next_page`# path, so rows arrive oldest-modified-first — matching the pipeline's ascending watermark.- sort_mode="asc",+ sort_mode=config.sort_mode,column_hints=resource.column_hints,)diff --git a/products/warehouse_sources/sources/safetyculture/settings.py b/products/warehouse_sources/sources/safetyculture/settings.pyindex 5041052674ea..70a8cde461e1 100644--- a/products/warehouse_sources/sources/safetyculture/settings.py+++ b/products/warehouse_sources/sources/safetyculture/settings.py@@ -1,5 +1,6 @@from dataclasses import dataclass, field-from typing import Optional+from datetime import timedelta+from typing import Literal, Optionalfrom products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType@@ -17,6 +18,17 @@ def _modified_at_incremental_fields() -> list[IncrementalField]:]+def _due_time_incremental_fields() -> list[IncrementalField]:+ return [+ {+ "label": "due_time",+ "type": IncrementalFieldType.DateTime,+ "field": "due_time",+ "field_type": IncrementalFieldType.DateTime,+ },+ ]++@dataclass(frozen=True)class SafetyCultureEndpointConfig:name: str@@ -35,6 +47,14 @@ class SafetyCultureEndpointConfig:# Non-empty marks a POST search endpoint scoped to one structure type per request; the sync# walks each listed type in turn.structure_types: tuple[str, ...] = ()+ # Query param that carries the incremental watermark on the first request.+ incremental_param: str = "modified_after"+ # How far before the watermark an incremental sync starts reading, for a feed whose filter+ # field can sit ahead of the rows that still change.+ incremental_rewind: Optional[timedelta] = None+ sort_mode: Optional[Literal["asc"]] = "asc"+ # Primary key fields the feed omits on some rows. They are set to "" so a merge can match them.+ blank_when_missing: tuple[str, ...] = ()# SafetyCulture Data Feed endpoints (https://developer.safetyculture.com/reference/data-feeds) —@@ -147,6 +167,51 @@ SAFETYCULTURE_ENDPOINTS: dict[str, SafetyCultureEndpointConfig] = {supports_incremental=False,partition_key=None,),+ # Scheduling service feed, which replaces the legacy /feed/schedule_occurrences. Records are one+ # per occurrence, assignee and site or asset. `start_date` filters on `due_time`, and the+ # default window is now - 5 months to now + 1 month. A full refresh holds only that window, so+ # incremental merge is the way to keep older occurrences.+ "schedule_occurrences": SafetyCultureEndpointConfig(+ name="schedule_occurrences",+ path="/scheduling/v1/feed/schedule_occurrences",+ supports_incremental=True,+ params={"limit": "1000"},+ # `due_time` is part of the occurrence `id`, so it never changes.+ partition_key="due_time",+ incremental_fields=_due_time_incremental_fields(),+ primary_keys=["id", "target_id"],+ incremental_param="start_date",+ # The watermark is the latest due time, up to a month ahead of the last sync. An occurrence+ # keeps changing status until its due time, so re-read everything due since the last sync.+ incremental_rewind=timedelta(days=45),+ # The feed documents no row order.+ sort_mode=None,+ blank_when_missing=("target_id",),+ ),+ # Timeline feeds: `modified_after` filters on the parent action, and the rows carry only their+ # creation `timestamp`, so there is no watermark column. Full refresh only.+ "action_timeline_items": SafetyCultureEndpointConfig(+ name="action_timeline_items",+ path="/feed/action_timeline_items",+ supports_incremental=False,+ partition_key="timestamp",+ ),+ "issue_timeline_items": SafetyCultureEndpointConfig(+ name="issue_timeline_items",+ path="/feed/issue_timeline_items",+ supports_incremental=False,+ partition_key="timestamp",+ ),+ # Training feed: one row per user and course, in camelCase. `modifiedAfter` exists, but the rows+ # have no modified time to hold the watermark, so it's full refresh only.+ "training_course_progress": SafetyCultureEndpointConfig(+ name="training_course_progress",+ path="/training/v1/feed/training-course-progress",+ supports_incremental=False,+ params={"completionStatus": "COMPLETION_STATUS_ALL"},+ partition_key=None,+ primary_keys=["courseId", "userId"],+ ),}ENDPOINTS = tuple(SAFETYCULTURE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.pyindex 36edce233d53..0c46050fa045 100644--- a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py+++ b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py@@ -51,30 +51,77 @@ class TestFormatModifiedAfter:class TestInitialRequest:- def test_incremental_first_request_carries_modified_after(self) -> None:+ @parameterized.expand(+ [+ (+ "inspections_modified_after",+ "inspections",+ "modified_at",+ datetime(2024, 3, 1, tzinfo=UTC),+ {"archived": "both", "completed": "both", "modified_after": "2024-03-01T00:00:00.000Z"},+ ),+ # The watermark is the latest due time, up to a month ahead of the last sync, so the+ # filter rewinds to re-read occurrences whose status can still change.+ (+ "schedule_occurrences_rewinds_datetime_watermark",+ "schedule_occurrences",+ "due_time",+ datetime(2024, 3, 1, 12, 30, tzinfo=UTC),+ {"limit": "1000", "start_date": "2024-01-16T12:30:00.000Z"},+ ),+ (+ "schedule_occurrences_rewinds_string_watermark",+ "schedule_occurrences",+ "due_time",+ "2024-03-01T12:30:00Z",+ {"limit": "1000", "start_date": "2024-01-16T12:30:00.000Z"},+ ),+ (+ "schedule_occurrences_rewinds_naive_watermark_as_utc",+ "schedule_occurrences",+ "due_time",+ datetime(2024, 3, 1, 12, 30),+ {"limit": "1000", "start_date": "2024-01-16T12:30:00.000Z"},+ ),+ ]+ )+ def test_incremental_first_request_carries_the_watermark_filter(+ self, _name: str, endpoint: str, incremental_field: str, watermark: Any, expected: dict[str, str]+ ) -> None:result = _driver().run(- "inspections",+ endpoint,[_response([{"id": "a"}])],- incremental_field="modified_at",- db_incremental_field_last_value=datetime(2024, 3, 1, tzinfo=UTC),+ incremental_field=incremental_field,+ db_incremental_field_last_value=watermark,)- assert {name: values[0] for name, values in result.queries[0].items()} == {- "archived": "both",- "completed": "both",- "modified_after": "2024-03-01T00:00:00.000Z",- }+ assert {name: values[0] for name, values in result.queries[0].items()} == expected- def test_full_refresh_endpoint_never_sends_modified_after(self) -> None:- # The issues feed documents no modified_after — a stale watermark must never leak into it.+ @parameterized.expand(+ [+ # The issues feed documents no modified_after — a stale watermark must never leak into it.+ ("issues", "modified_at", {}),+ ("action_timeline_items", "modified_at", {}),+ ("training_course_progress", "modified_at", {"completionStatus": "COMPLETION_STATUS_ALL"}),+ ]+ )+ def test_full_refresh_endpoint_never_sends_a_watermark(+ self, endpoint: str, incremental_field: str, expected: dict[str, str]+ ) -> None:result = _driver().run(- "issues",+ endpoint,[_response([{"id": "a"}])],- incremental_field="modified_at",+ incremental_field=incremental_field,db_incremental_field_last_value=datetime(2024, 3, 1, tzinfo=UTC),)- assert "modified_after" not in result.queries[0]+ assert {name: values[0] for name, values in result.queries[0].items()} == expected++ def test_schedule_occurrences_without_a_watermark_uses_the_default_window(self) -> None:+ result = _driver().run("schedule_occurrences", [_response([{"id": "a", "target_id": "t"}])])++ assert result.urls[0] == f"{BASE_URL}/scheduling/v1/feed/schedule_occurrences?limit=1000"+ assert {name: values[0] for name, values in result.queries[0].items()} == {"limit": "1000"}class TestPagination:@@ -303,7 +350,8 @@ class TestSafetyCultureSourceResponse:config = SAFETYCULTURE_ENDPOINTS[endpoint]assert response.name == endpointassert response.primary_keys == config.primary_keys- assert response.sort_mode == "asc"+ # The scheduling feed documents no row order, so a resumed run must not skip below its cursor.+ assert response.sort_mode == (None if endpoint == "schedule_occurrences" else "asc")if config.partition_key:assert response.partition_mode == "datetime"assert response.partition_format == "month"@@ -324,6 +372,15 @@ class TestSafetyCultureSourceResponse:"issue_assignees",{"id": "ia_1", "issue_id": "issue_1", "assignee_id": "user_1", "modified_at": "2024-03-01T00:00:00Z"},),+ (+ "schedule_occurrences",+ {+ "id": "scheduleitem_1_occurrence_1705338000000_user_1",+ "target_id": "location_1",+ "due_time": "2024-01-15T17:00:00Z",+ },+ ),+ ("training_course_progress", {"courseId": "course_1", "userId": "user_1", "progressPercent": 50}),])def test_join_feed_primary_keys_exist_on_documented_rows(self, endpoint: str, row: dict[str, Any]) -> None:@@ -335,9 +392,24 @@ class TestSafetyCultureSourceResponse:assert result.response.primary_keysassert all(row.get(key) for key in result.response.primary_keys)+ @parameterized.expand(+ [+ ("missing", {"id": "occ_1"}),+ ("null", {"id": "occ_1", "target_id": None}),+ ("empty", {"id": "occ_1", "target_id": ""}),+ ]+ )+ def test_schedule_occurrence_without_a_target_gets_a_mergeable_key(self, _name: str, row: dict[str, Any]) -> None:+ # A null merge key never matches, so every incremental sync would add the row again.+ result = _driver().run("schedule_occurrences", [_response([row])])++ assert result.rows == [{"id": "occ_1", "target_id": ""}]+def test_incremental_endpoints_match_documented_modified_after_support(self) -> None:- # Only these four feeds document a modified_after filter that keys off a `modified_at` column;- # action_assignees accepts the param but its rows carry no `modified_at` for the watermark.+ # Only these feeds document a server-side filter on a column their rows carry: four+ # `modified_after` feeds on `modified_at`, and schedule occurrences' `start_date` on `due_time`.+ # action_assignees, action_timeline_items and training_course_progress accept a modified filter,+ # but their rows carry no modified time for the watermark.# Flipping any other endpoint to incremental would ship a full-refresh-cost "incremental" sync.incremental = {name for name, config in SAFETYCULTURE_ENDPOINTS.items() if config.supports_incremental}- assert incremental == {"inspections", "inspection_items", "templates", "actions"}+ assert incremental == {"inspections", "inspection_items", "templates", "actions", "schedule_occurrences"}