Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(safetyculture): add 4 missing tables to the safetyculture source (#129999)

Tom Owers32b1ef816fd6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index eb491208c250..15c8d48ab553 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10526,10 +10526,10 @@ Diffed against: <https://developer.safetyculture.com/reference/thepubservice_fee
- [x] `feed/site_members` — membership table joining users to the sites we already sync (high)
- [x] `feed/action_assignees` — resolves who each synced action is assigned to; actions are unattributable without it (high)
- [x] `feed/issue_assignees` — same attribution join for the issues we already sync (high)
-- [ ] `feed/schedule_occurrences` — actual scheduled occurrences, the only way to measure scheduled vs completed inspections (high)
-- [ ] `feed/action_timeline_items` — state-transition history for actions, the basis of cycle-time analysis (high)
-- [ ] `feed/training_course_progress` — training completion per user, a headline compliance metric (high)
-- [ ] `feed/issue_timeline_items` — state-transition history for issues (medium)
+- [x] `feed/schedule_occurrences` — actual scheduled occurrences, the only way to measure scheduled vs completed inspections (high)
+- [x] `feed/action_timeline_items` — state-transition history for actions, the basis of cycle-time analysis (high)
+- [x] `feed/training_course_progress` — training completion per user, a headline compliance metric (high)
+- [x] `feed/issue_timeline_items` — state-transition history for issues (medium)
- [ ] `feed/activity_log_events` — org-wide audit event stream for usage and adoption analysis (medium)
- [ ] `feed/investigations (plus investigation fields and relationships)` — incident investigation records linked to the issues and actions we sync (medium)
- [ ] `feed/assets_maintenance, feed/maintenance_plans, feed/maintenance_programs` — maintenance schedule and status for the assets we already sync (medium)
diff --git a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py
index 2022644f0df2..667372f54727 100644
--- a/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/safetyculture/canonical_descriptions.py
@@ -281,4 +281,84 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"type": "Type of assignee.",
},
},
+ "schedule_occurrences": {
+ "description": "A scheduled inspection occurrence, one row per occurrence, assignee and site or asset. Compare scheduled against completed inspections with it.",
+ "docs_url": "https://developer.safetyculture.com/reference/schedulingfeedservice_feedscheduleoccurrences",
+ "columns": {
+ "id": "ID of the schedule occurrence for one assignee. A schedule for several sites or assets repeats it once per site or asset.",
+ "schedule_id": "ID of the schedule that generated this occurrence.",
+ "occurrence_id": "Sequence number of the occurrence within the schedule, per site or asset.",
+ "template_id": "ID of the template used for inspections created from this occurrence.",
+ "start_time": "When the occurrence becomes available for inspection.",
+ "due_time": "Final deadline. After this time the occurrence is missed and cannot be completed.",
+ "miss_time": "When the occurrence becomes overdue. Empty when the schedule has no overdue period.",
+ "occurrence_status": "Status of the occurrence as a whole: TODO, IN_PROGRESS, COMPLETED, LATE, OVERDUE or MISSED.",
+ "audit_id": "ID of the inspection created from this occurrence.",
+ "completed_at": "When this occurrence was completed.",
+ "completion_rule": "Whether any assignee or all assignees must complete the occurrence (`any` or `all`).",
+ "organisation_id": "ID of the organisation that owns the occurrence.",
+ "wont_do_note": "Reason given when the occurrence was marked as won't do.",
+ "assignee_id": "ID of the user assigned to complete this occurrence.",
+ "assignee_from": "How the user was assigned, as `{type}_{id}` where the type is user, group, role or location.",
+ "target_id": "ID of the site or asset this occurrence is for. Empty when the schedule has no sites or assets.",
+ "target_type": "Type of `target_id`: SITE or ASSET.",
+ "completed_by_user_id": "ID of the user who completed the inspection for this assignee.",
+ "assignee_status": "Status of this assignee's own work on the occurrence. Uses the same values as `occurrence_status`, plus WONT_DO.",
+ },
+ },
+ "action_timeline_items": {
+ "description": "A timeline event on an action, such as a status change, a comment or a field update.",
+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedactiontimelineitems",
+ "columns": {
+ "id": "ID of the timeline item.",
+ "task_id": "ID of the action.",
+ "organisation_id": "ID of the SafetyCulture organisation.",
+ "task_creator_id": "ID of the user who created the action.",
+ "task_creator_name": "Name of the user who created the action.",
+ "timestamp": "When the timeline item was created, in UTC.",
+ "creator_id": "ID of the user who created the timeline item.",
+ "creator_name": "Name of the user who created the timeline item.",
+ "item_type": "Type of the timeline item.",
+ "item_data": "Data of the timeline item.",
+ },
+ },
+ "issue_timeline_items": {
+ "description": "A timeline event on an issue, such as a status change, a comment or a field update.",
+ "docs_url": "https://developer.safetyculture.com/reference/thepubservice_feedissuetimelineitems",
+ "columns": {
+ "id": "ID of the timeline item.",
+ "task_id": "ID of the issue.",
+ "organisation_id": "ID of the SafetyCulture organisation.",
+ "task_creator_id": "ID of the user who created the issue.",
+ "task_creator_name": "Name of the user who created the issue.",
+ "timestamp": "When the timeline item was created, in UTC.",
+ "creator_id": "ID of the user who created the timeline item.",
+ "creator_name": "Name of the user who created the timeline item.",
+ "item_type": "Type of the timeline item.",
+ "item_data": "Data of the timeline item.",
+ },
+ },
+ "training_course_progress": {
+ "description": "Training course progress, one row per user and course.",
+ "docs_url": "https://developer.safetyculture.com/reference/feedservice_feedtrainingcourseprogress",
+ "columns": {
+ "courseId": "ID of the course.",
+ "courseExternalId": "External ID of the course.",
+ "courseTitle": "Title of the course.",
+ "userId": "ID of the user.",
+ "userExternalId": "External ID of the user.",
+ "userEmail": "Email address of the user.",
+ "userFirstName": "First name of the user.",
+ "userLastName": "Last name of the user.",
+ "openedAt": "When the user first opened the course.",
+ "completedAt": "When the user completed the course.",
+ "dueAt": "When the course is due.",
+ "completionExpiryDate": "When the course completion expires.",
+ "totalLessons": "Number of lessons in the course.",
+ "completedLessons": "Number of lessons the user completed.",
+ "progressPercent": "Percentage of the course the user completed.",
+ "score": "Score of the user on the course.",
+ "timeSpentSeconds": "Total time the user spent on the course, in seconds.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/safetyculture/safetyculture.py b/products/warehouse_sources/sources/safetyculture/safetyculture.py
index 9e1ab77d0c38..8567236cb2fb 100644
--- a/products/warehouse_sources/sources/safetyculture/safetyculture.py
+++ b/products/warehouse_sources/sources/safetyculture/safetyculture.py
@@ -1,4 +1,5 @@
import dataclasses
+from collections.abc import Callable
from datetime import UTC, date, datetime
from typing import Any, Optional
@@ -8,6 +9,7 @@ from products.warehouse_sources.sources.safetyculture.settings import SAFETYCULT
from products.warehouse_sources.sources.sdk import (
BasePaginator,
Endpoint,
+ EndpointResource,
JSONResponsePaginator,
RESTAPIConfig,
ResumableSourceManager,
@@ -134,6 +136,25 @@ class SafetyCultureStructuresPaginator(BasePaginator):
request.json = body
+def _parse_watermark(value: Any) -> datetime:
+ if isinstance(value, datetime):
+ return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
+ if isinstance(value, date):
+ return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
+
+
+def _blank_missing_fields(fields: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:
+ def blank(row: dict[str, Any]) -> dict[str, Any]:
+ for name in fields:
+ if row.get(name) is None:
+ row[name] = ""
+ return row
+
+ return blank
+
+
def safetyculture_source(
api_token: str,
endpoint: str,
@@ -150,7 +171,10 @@ def safetyculture_source(
# incremental sync that actually has a watermark to filter from.
params: dict[str, Any] = dict(config.params)
if config.supports_incremental and should_use_incremental_field and db_incremental_field_last_value:
- params["modified_after"] = _format_modified_after(db_incremental_field_last_value)
+ watermark = db_incremental_field_last_value
+ if config.incremental_rewind is not None:
+ watermark = _parse_watermark(watermark) - config.incremental_rewind
+ params[config.incremental_param] = _format_modified_after(watermark)
endpoint_config: Endpoint
if config.structure_types:
@@ -173,6 +197,10 @@ def safetyculture_source(
"data_selector_malformed_retryable": True,
}
+ resource_config: EndpointResource = {"name": endpoint, "endpoint": endpoint_config}
+ if config.blank_when_missing:
+ resource_config["data_map"] = _blank_missing_fields(config.blank_when_missing)
+
rest_config: RESTAPIConfig = {
"client": {
"base_url": SAFETYCULTURE_BASE_URL,
@@ -188,12 +216,7 @@ def safetyculture_source(
"allow_redirects": False,
},
"resource_defaults": {},
- "resources": [
- {
- "name": endpoint,
- "endpoint": endpoint_config,
- }
- ],
+ "resources": [resource_config],
}
initial_paginator_state: Optional[dict[str, Any]] = None
@@ -243,7 +266,7 @@ def safetyculture_source(
partition_keys=[config.partition_key] if config.partition_key else None,
# Incremental feeds paginate by advancing `modified_after` in the server-issued `next_page`
# path, so rows arrive oldest-modified-first — matching the pipeline's ascending watermark.
- sort_mode="asc",
+ sort_mode=config.sort_mode,
column_hints=resource.column_hints,
)
diff --git a/products/warehouse_sources/sources/safetyculture/settings.py b/products/warehouse_sources/sources/safetyculture/settings.py
index 5041052674ea..70a8cde461e1 100644
--- a/products/warehouse_sources/sources/safetyculture/settings.py
+++ b/products/warehouse_sources/sources/safetyculture/settings.py
@@ -1,5 +1,6 @@
from dataclasses import dataclass, field
-from typing import Optional
+from datetime import timedelta
+from typing import Literal, Optional
from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
@@ -17,6 +18,17 @@ def _modified_at_incremental_fields() -> list[IncrementalField]:
]
+def _due_time_incremental_fields() -> list[IncrementalField]:
+ return [
+ {
+ "label": "due_time",
+ "type": IncrementalFieldType.DateTime,
+ "field": "due_time",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+ ]
+
+
@dataclass(frozen=True)
class SafetyCultureEndpointConfig:
name: str
@@ -35,6 +47,14 @@ class SafetyCultureEndpointConfig:
# Non-empty marks a POST search endpoint scoped to one structure type per request; the sync
# walks each listed type in turn.
structure_types: tuple[str, ...] = ()
+ # Query param that carries the incremental watermark on the first request.
+ incremental_param: str = "modified_after"
+ # How far before the watermark an incremental sync starts reading, for a feed whose filter
+ # field can sit ahead of the rows that still change.
+ incremental_rewind: Optional[timedelta] = None
+ sort_mode: Optional[Literal["asc"]] = "asc"
+ # Primary key fields the feed omits on some rows. They are set to "" so a merge can match them.
+ blank_when_missing: tuple[str, ...] = ()
# SafetyCulture Data Feed endpoints (https://developer.safetyculture.com/reference/data-feeds) —
@@ -147,6 +167,51 @@ SAFETYCULTURE_ENDPOINTS: dict[str, SafetyCultureEndpointConfig] = {
supports_incremental=False,
partition_key=None,
),
+ # Scheduling service feed, which replaces the legacy /feed/schedule_occurrences. Records are one
+ # per occurrence, assignee and site or asset. `start_date` filters on `due_time`, and the
+ # default window is now - 5 months to now + 1 month. A full refresh holds only that window, so
+ # incremental merge is the way to keep older occurrences.
+ "schedule_occurrences": SafetyCultureEndpointConfig(
+ name="schedule_occurrences",
+ path="/scheduling/v1/feed/schedule_occurrences",
+ supports_incremental=True,
+ params={"limit": "1000"},
+ # `due_time` is part of the occurrence `id`, so it never changes.
+ partition_key="due_time",
+ incremental_fields=_due_time_incremental_fields(),
+ primary_keys=["id", "target_id"],
+ incremental_param="start_date",
+ # The watermark is the latest due time, up to a month ahead of the last sync. An occurrence
+ # keeps changing status until its due time, so re-read everything due since the last sync.
+ incremental_rewind=timedelta(days=45),
+ # The feed documents no row order.
+ sort_mode=None,
+ blank_when_missing=("target_id",),
+ ),
+ # Timeline feeds: `modified_after` filters on the parent action, and the rows carry only their
+ # creation `timestamp`, so there is no watermark column. Full refresh only.
+ "action_timeline_items": SafetyCultureEndpointConfig(
+ name="action_timeline_items",
+ path="/feed/action_timeline_items",
+ supports_incremental=False,
+ partition_key="timestamp",
+ ),
+ "issue_timeline_items": SafetyCultureEndpointConfig(
+ name="issue_timeline_items",
+ path="/feed/issue_timeline_items",
+ supports_incremental=False,
+ partition_key="timestamp",
+ ),
+ # Training feed: one row per user and course, in camelCase. `modifiedAfter` exists, but the rows
+ # have no modified time to hold the watermark, so it's full refresh only.
+ "training_course_progress": SafetyCultureEndpointConfig(
+ name="training_course_progress",
+ path="/training/v1/feed/training-course-progress",
+ supports_incremental=False,
+ params={"completionStatus": "COMPLETION_STATUS_ALL"},
+ partition_key=None,
+ primary_keys=["courseId", "userId"],
+ ),
}
ENDPOINTS = tuple(SAFETYCULTURE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py
index 36edce233d53..0c46050fa045 100644
--- a/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py
+++ b/products/warehouse_sources/sources/safetyculture/tests/test_safetyculture.py
@@ -51,30 +51,77 @@ class TestFormatModifiedAfter:
class TestInitialRequest:
- def test_incremental_first_request_carries_modified_after(self) -> None:
+ @parameterized.expand(
+ [
+ (
+ "inspections_modified_after",
+ "inspections",
+ "modified_at",
+ datetime(2024, 3, 1, tzinfo=UTC),
+ {"archived": "both", "completed": "both", "modified_after": "2024-03-01T00:00:00.000Z"},
+ ),
+ # The watermark is the latest due time, up to a month ahead of the last sync, so the
+ # filter rewinds to re-read occurrences whose status can still change.
+ (
+ "schedule_occurrences_rewinds_datetime_watermark",
+ "schedule_occurrences",
+ "due_time",
+ datetime(2024, 3, 1, 12, 30, tzinfo=UTC),
+ {"limit": "1000", "start_date": "2024-01-16T12:30:00.000Z"},
+ ),
+ (
+ "schedule_occurrences_rewinds_string_watermark",
+ "schedule_occurrences",
+ "due_time",
+ "2024-03-01T12:30:00Z",
+ {"limit": "1000", "start_date": "2024-01-16T12:30:00.000Z"},
+ ),
+ (
+ "schedule_occurrences_rewinds_naive_watermark_as_utc",
+ "schedule_occurrences",
+ "due_time",
+ datetime(2024, 3, 1, 12, 30),
+ {"limit": "1000", "start_date": "2024-01-16T12:30:00.000Z"},
+ ),
+ ]
+ )
+ def test_incremental_first_request_carries_the_watermark_filter(
+ self, _name: str, endpoint: str, incremental_field: str, watermark: Any, expected: dict[str, str]
+ ) -> None:
result = _driver().run(
- "inspections",
+ endpoint,
[_response([{"id": "a"}])],
- incremental_field="modified_at",
- db_incremental_field_last_value=datetime(2024, 3, 1, tzinfo=UTC),
+ incremental_field=incremental_field,
+ db_incremental_field_last_value=watermark,
)
- assert {name: values[0] for name, values in result.queries[0].items()} == {
- "archived": "both",
- "completed": "both",
- "modified_after": "2024-03-01T00:00:00.000Z",
- }
+ assert {name: values[0] for name, values in result.queries[0].items()} == expected
- def test_full_refresh_endpoint_never_sends_modified_after(self) -> None:
- # The issues feed documents no modified_after — a stale watermark must never leak into it.
+ @parameterized.expand(
+ [
+ # The issues feed documents no modified_after — a stale watermark must never leak into it.
+ ("issues", "modified_at", {}),
+ ("action_timeline_items", "modified_at", {}),
+ ("training_course_progress", "modified_at", {"completionStatus": "COMPLETION_STATUS_ALL"}),
+ ]
+ )
+ def test_full_refresh_endpoint_never_sends_a_watermark(
+ self, endpoint: str, incremental_field: str, expected: dict[str, str]
+ ) -> None:
result = _driver().run(
- "issues",
+ endpoint,
[_response([{"id": "a"}])],
- incremental_field="modified_at",
+ incremental_field=incremental_field,
db_incremental_field_last_value=datetime(2024, 3, 1, tzinfo=UTC),
)
- assert "modified_after" not in result.queries[0]
+ assert {name: values[0] for name, values in result.queries[0].items()} == expected
+
+ def test_schedule_occurrences_without_a_watermark_uses_the_default_window(self) -> None:
+ result = _driver().run("schedule_occurrences", [_response([{"id": "a", "target_id": "t"}])])
+
+ assert result.urls[0] == f"{BASE_URL}/scheduling/v1/feed/schedule_occurrences?limit=1000"
+ assert {name: values[0] for name, values in result.queries[0].items()} == {"limit": "1000"}
class TestPagination:
@@ -303,7 +350,8 @@ class TestSafetyCultureSourceResponse:
config = SAFETYCULTURE_ENDPOINTS[endpoint]
assert response.name == endpoint
assert response.primary_keys == config.primary_keys
- assert response.sort_mode == "asc"
+ # The scheduling feed documents no row order, so a resumed run must not skip below its cursor.
+ assert response.sort_mode == (None if endpoint == "schedule_occurrences" else "asc")
if config.partition_key:
assert response.partition_mode == "datetime"
assert response.partition_format == "month"
@@ -324,6 +372,15 @@ class TestSafetyCultureSourceResponse:
"issue_assignees",
{"id": "ia_1", "issue_id": "issue_1", "assignee_id": "user_1", "modified_at": "2024-03-01T00:00:00Z"},
),
+ (
+ "schedule_occurrences",
+ {
+ "id": "scheduleitem_1_occurrence_1705338000000_user_1",
+ "target_id": "location_1",
+ "due_time": "2024-01-15T17:00:00Z",
+ },
+ ),
+ ("training_course_progress", {"courseId": "course_1", "userId": "user_1", "progressPercent": 50}),
]
)
def test_join_feed_primary_keys_exist_on_documented_rows(self, endpoint: str, row: dict[str, Any]) -> None:
@@ -335,9 +392,24 @@ class TestSafetyCultureSourceResponse:
assert result.response.primary_keys
assert all(row.get(key) for key in result.response.primary_keys)
+ @parameterized.expand(
+ [
+ ("missing", {"id": "occ_1"}),
+ ("null", {"id": "occ_1", "target_id": None}),
+ ("empty", {"id": "occ_1", "target_id": ""}),
+ ]
+ )
+ def test_schedule_occurrence_without_a_target_gets_a_mergeable_key(self, _name: str, row: dict[str, Any]) -> None:
+ # A null merge key never matches, so every incremental sync would add the row again.
+ result = _driver().run("schedule_occurrences", [_response([row])])
+
+ assert result.rows == [{"id": "occ_1", "target_id": ""}]
+
def test_incremental_endpoints_match_documented_modified_after_support(self) -> None:
- # Only these four feeds document a modified_after filter that keys off a `modified_at` column;
- # action_assignees accepts the param but its rows carry no `modified_at` for the watermark.
+ # Only these feeds document a server-side filter on a column their rows carry: four
+ # `modified_after` feeds on `modified_at`, and schedule occurrences' `start_date` on `due_time`.
+ # action_assignees, action_timeline_items and training_course_progress accept a modified filter,
+ # but their rows carry no modified time for the watermark.
# Flipping any other endpoint to incremental would ship a full-refresh-cost "incremental" sync.
incremental = {name for name, config in SAFETYCULTURE_ENDPOINTS.items() if config.supports_incremental}
- assert incremental == {"inspections", "inspection_items", "templates", "actions"}
+ assert incremental == {"inspections", "inspection_items", "templates", "actions", "schedule_occurrences"}