:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse_sources): add six rippling endpoint tables (#127353)
Tom Owersc1bceb8bcd21
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 7a91a239afb4..c75c14cc342f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10318,10 +10318,10 @@ Diffed against: <https://developer.rippling.com/sitemap.xml>- [x] `list-leave-requests` — time-off requests with status and dates, the core absence dataset (high)- [x] `list-worker-payroll-records` — per-worker payroll line items - actual pay, taxes and deductions, not just contracted compensation (high)- [x] `list-payroll-runs` — the payroll period header that payroll records roll up to (high)-- [ ] `list-worker-changes (and list-worker-change-fields)` — the transition history for workers - promotions, transfers, terminations; today only current state is synced (high)-- [ ] `list-leave-balances (and list-leave-accruals)` — accrued and remaining balance per worker per policy, the standard PTO liability metric (high)-- [ ] `list-titles` — lookup resolving the job title ID carried on workers (high)-- [ ] `list-tracks` — lookup grouping the levels table already synced into career tracks (high)+- [x] `list-worker-changes (and list-worker-change-fields)` — the transition history for workers - promotions, transfers, terminations; today only current state is synced (high)+- [x] `list-leave-balances (and list-leave-accruals)` — accrued and remaining balance per worker per policy, the standard PTO liability metric (high)+- [x] `list-titles` — lookup resolving the job title ID carried on workers (high)+- [x] `list-tracks` — lookup grouping the levels table already synced into career tracks (high)- [ ] `list-leave-types` — lookup naming the leave policy IDs on leave requests and balances (medium)- [ ] `list-legal-entities and list-company-legal-entities` — lookup resolving the legal entity on workers and payroll, required for multi-entity reporting (medium)- [ ] `list-job-codes and list-job-assignments` — lookup resolving the job code stamped on time entries and shift assignments (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/canonical_descriptions.pyindex b586d35f753e..6c912fecc946 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/canonical_descriptions.py@@ -233,4 +233,75 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"garnishments": "List of garnishment line items.",},},+ "tracks": {+ "description": "A career track that groups job levels, such as an individual contributor or a management track.",+ "docs_url": "https://developer.rippling.com/documentation/rest-api/reference/list-tracks",+ "columns": _columns(+ name="The name of the track. Unique within the company.",+ is_default="Whether this is the company's default track.",+ order="The one-based horizontal display order of the track.",+ start_global_level="The highest global level that this track occupies. Null for an empty track.",+ ),+ },+ "titles": {+ "description": "A job title defined for the company.",+ "docs_url": "https://developer.rippling.com/documentation/rest-api/reference/list-titles",+ "columns": _columns(+ name="The name of the title.",+ ),+ },+ "leave_balances": {+ "description": "A worker's current balance for one leave type, with and without future leave requests.",+ "docs_url": "https://developer.rippling.com/documentation/rest-api/reference/list-leave-balances",+ "columns": _columns(+ worker_id="ID of the worker associated with the leave balance.",+ leave_type_id="ID of the leave type associated with the leave balance.",+ is_balance_unlimited="Whether the leave balance is unlimited.",+ balance_including_future_requests="The leave balance including future leave requests. Null when the balance is unlimited.",+ balance_excluding_future_requests="The leave balance excluding future leave requests. Null when the balance is unlimited.",+ balance_including_future_requests_in_days="The leave balance in days including future leave requests. Null when the balance is unlimited.",+ balance_excluding_future_requests_in_days="The leave balance in days excluding future leave requests. Null when the balance is unlimited.",+ ),+ },+ "leave_accruals": {+ "description": "A single accrual or deduction of leave time for a worker under a leave type.",+ "docs_url": "https://developer.rippling.com/documentation/rest-api/reference/list-leave-accruals",+ "columns": _columns(+ worker_id="ID of the worker the accrual is for.",+ grantor_id="ID of the worker who granted the accrual, typically the time off admin.",+ leave_type_id="ID of the company leave type associated with the accrual.",+ accrual_type="The type of accrual (e.g. AUTO, LEAVE_GRANT, BALANCE_CORRECTION_ADJUSTMENT, PAID_OUT).",+ accrual_date="The date when the accrual becomes effective.",+ num_hours="The number of hours accrued. Negative for deductions. Set when the policy tracks hours.",+ num_days="The number of days accrued. Negative for deductions. Set when the policy tracks days.",+ monetary_amount_accrued="The monetary value of the accrual, for policies with monetary tracking.",+ expiration_date="The date when the accrual expires, if applicable.",+ comment="The comment for the accrual.",+ ),+ },+ "worker_changes": {+ "description": "An HR profile change event for a worker, such as a hire, field change, scheduled change or offboarding.",+ "docs_url": "https://developer.rippling.com/documentation/rest-api/guides/worker-changes",+ "columns": _columns(+ worker_id="ID of the worker affected by the change.",+ changed_by_id="ID of the user who made the change. Null for system-initiated events.",+ referenced_transaction_id="For amendments or undos, the ID of the worker change event being modified.",+ applied_on="Time at which the change was saved or approved, in UTC.",+ effective_on="Time at which the change takes effect, in UTC. Can be in the past or the future.",+ event_type="Lifecycle event type (e.g. HIRE, CHANGE, SCHEDULED_CHANGE, OFFBOARD, UNDO).",+ import_type="INITIAL or HISTORICAL. Set only when event_type is IMPORT.",+ reason_code="Structured transition reason, such as PROMOTION or MARKET_ADJUSTMENT.",+ changes="Field-level changes in the event, each with its field key and from and to values.",+ ),+ },+ "worker_change_fields": {+ "description": "A worker field that can appear in worker change events, with its label, category and value type.",+ "docs_url": "https://developer.rippling.com/documentation/rest-api/guides/worker-changes",+ "columns": {+ "field_key": "Key of the field, as it appears in the changes of a worker change event.",+ "field_display_name": "UI label for the field.",+ "category": "Group of the field (e.g. PERSONAL, JOB, COMPENSATION, BENEFITS, CUSTOM).",+ "value_type": "Value type of the field (e.g. STRING, NUMBER, DATE, CURRENCY, ENUM).",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/rippling.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/rippling.pyindex a991026c1855..a81cc9cb58a6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/rippling.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/rippling.py@@ -61,9 +61,10 @@ def _build_params(if should_use_incremental_field and config.incremental_fields and db_incremental_field_last_value is not None:cursor_field = incremental_field or config.default_incremental_fieldparams["filter"] = f"{cursor_field} ge {_format_filter_timestamp(db_incremental_field_last_value)}"- # Ascending order on the cursor field so the incremental watermark- # advances monotonically as pages are consumed (order_by defaults to asc).- params["order_by"] = cursor_field+ if config.order_by:+ # Ascending order on the cursor field so the incremental watermark+ # advances monotonically as pages are consumed (order_by defaults to asc).+ params["order_by"] = cursor_fieldelif config.order_by:# Full refresh: a stable creation-time sort prevents page-boundary# skips/duplicates if rows change mid-sync.@@ -217,6 +218,7 @@ def rippling_source(partition_mode="datetime" if config.partition_key else None,partition_format="month" if config.partition_key else None,partition_keys=[config.partition_key] if config.partition_key else None,- sort_mode="asc",+ # Endpoints that accept no order_by return rows in an undocumented order.+ sort_mode="asc" if config.order_by else None,column_hints=resource.column_hints,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/settings.pyindex 79ee5542218b..5ceb191b3308 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/settings.py@@ -121,6 +121,54 @@ RIPPLING_ENDPOINTS: dict[str, RipplingEndpointConfig] = {incremental_fields=[],order_by="check_date",),+ # Tracks and titles accept no filter, so full refresh only.+ "tracks": RipplingEndpointConfig(+ name="tracks",+ path="/tracks",+ incremental_fields=[],+ ),+ "titles": RipplingEndpointConfig(+ name="titles",+ path="/titles",+ incremental_fields=[],+ ),+ # Leave balances filter only on worker_id and leave_type_id, so full refresh only.+ "leave_balances": RipplingEndpointConfig(+ name="leave_balances",+ path="/leave-balances",+ incremental_fields=[],+ ),+ # Leave accruals filter on accrual_date, but accruals can be back- or forward-dated, so a+ # watermark on it would skip rows. Full refresh only.+ "leave_accruals": RipplingEndpointConfig(+ name="leave_accruals",+ path="/leave-accruals",+ incremental_fields=[],+ partition_key="created_at",+ ),+ # Rippling documents applied_on (when the change was saved) as the polling filter. It+ # documents no sortable fields, so requests send no order_by and rows arrive unordered.+ "worker_changes": RipplingEndpointConfig(+ name="worker_changes",+ path="/worker-changes",+ incremental_fields=[+ {+ "label": "applied_on",+ "type": IncrementalFieldType.DateTime,+ "field": "applied_on",+ "field_type": IncrementalFieldType.DateTime,+ }+ ],+ partition_key="created_at",+ order_by=None,+ ),+ "worker_change_fields": RipplingEndpointConfig(+ name="worker_change_fields",+ path="/worker-change-fields",+ primary_keys=["field_key"],+ incremental_fields=[],+ order_by=None,+ ),"worker_payroll_records": RipplingEndpointConfig(name="worker_payroll_records",path="/payroll-runs/{run_id}/worker-payroll-records",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/source.pyindex 721a55c96a65..604916bf738d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/source.py@@ -75,7 +75,7 @@ class RipplingSource(ResumableSource[RipplingSourceConfig, RipplingResumeConfig]label="Rippling",caption="""Enter your Rippling API token to pull your Rippling workforce data into the PostHog Data warehouse.-A Rippling admin can create a scoped API token under Settings > Company Settings > API Access. Enable the read scope for each dataset you want to sync (e.g. `companies.read`, `users.read`, `workers.read`, `departments.read`, `teams.read`, `levels.read`, `work-locations.read`, `employment-types.read`, `compensations.read`, `time-entries.read`, `leave-requests.read`, `payroll-runs.read`, `worker-payroll-records.read`). Payroll tables need Rippling Global Payroll. Note that tokens expire after 30 days of inactivity.""",+A Rippling admin can create a scoped API token under Settings > Company Settings > API Access. Enable the read scope for each dataset you want to sync (e.g. `companies.read`, `users.read`, `workers.read`, `departments.read`, `teams.read`, `levels.read`, `work-locations.read`, `employment-types.read`, `compensations.read`, `time-entries.read`, `leave-requests.read`, `payroll-runs.read`, `worker-payroll-records.read`, `tracks.read`, `titles.read`, `leave-balances.read`, `leave-accruals.read`, `worker-changes.read`, `worker-change-fields.read`). Payroll tables need Rippling Global Payroll. Worker change tables need the Worker Changes API entitlement, which your Rippling account team can turn on. Note that tokens expire after 30 days of inactivity.""",iconPath="/static/services/rippling.png",docsUrl="https://posthog.com/docs/cdp/sources/rippling",releaseStatus=ReleaseStatus.BETA,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/tests/test_rippling.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/tests/test_rippling.pyindex 83538af534d7..d8f6d659ac3a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/tests/test_rippling.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rippling/tests/test_rippling.py@@ -63,6 +63,10 @@ class TestBuildParams:# The payouts list rejects OData filters and accepts no order_by.("variable_compensation_payouts", True, "updated_at", None, None),("variable_compensation_payouts", False, None, None, None),+ # Worker changes filter on applied_on but document no sortable fields.+ ("worker_changes", True, "applied_on", "applied_on ge 2024-10-01T00:00:00", None),+ ("worker_changes", False, None, None, None),+ ("leave_accruals", True, "accrual_date", None, "created_at"),],)def test_request_params(@@ -126,6 +130,25 @@ class TestGetRows:assert result.urls[0] == resume_url+ @pytest.mark.parametrize(+ "endpoint, incremental_field, expected_sort_mode",+ [+ ("workers", "updated_at", "asc"),+ ("worker_changes", "applied_on", None),+ ],+ )+ def test_sort_mode_follows_request_order(self, endpoint, incremental_field, expected_sort_mode):+ result = _driver().run(+ endpoint,+ [_response([{"id": "1"}])],+ incremental_field=incremental_field,+ db_incremental_field_last_value=datetime(2024, 10, 1, tzinfo=UTC),+ )++ assert result.raised is None+ assert result.response is not None+ assert result.response.sort_mode == expected_sort_mode+def test_off_domain_next_link_raises(self):result = _driver().run("workers", [_response([{"id": "1"}], next_link="https://attacker.example/workers")])