:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add onfleet route plans table (#123362)
Tom Owers4240c19845a3
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 1efc084eccad..ba4d445ed822 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -814,8 +814,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/onepagecrm.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/onepassword.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/open_meteo/open_meteo.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 8e86e99fc77d..24de3231cdad 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8687,7 +8687,7 @@ Today (7): `administrators`, `hubs`, `organization`, `tasks`, `teams`, `webhooksDiffed against: <https://docs.onfleet.com/reference/administrators>-- [ ] `routePlans` — route plans group tasks into driver routes - core delivery-efficiency object (high)+- [x] `routePlans` — route plans group tasks into driver routes - core delivery-efficiency object (high)- [ ] `taskOrders (orders)` — order-level records that tasks are created from, with pricing/quote state (medium)- [ ] `destinations` — lookup resolving the destination IDs referenced by synced tasks (address/geo dimension) (medium)- [ ] `recipients` — lookup resolving recipient IDs on synced tasks (customer dimension) (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.pyindex dc52af2a56e8..92e58dc3b937 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.py@@ -123,6 +123,34 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"isEnabled": "Whether the webhook is currently enabled.",},},+ "route_plans": {+ "description": "Route plans that group a driver's tasks into an ordered route for a shift, day, or planned segment of work.",+ "docs_url": "https://docs.onfleet.com/reference/get-route-plan",+ "columns": {+ "id": "Unique identifier for the route plan.",+ "name": "The route plan's name.",+ "state": "Current state of the route plan, for example PENDING.",+ "tasks": "Identifiers of the tasks on the route, in route order.",+ "taskStates": "State of each task on the route, in the same order as tasks (0 unassigned, 1 assigned, 2 active, 3 completed).",+ "organization": "Identifier of the organization that owns the route plan.",+ "team": "Identifier of the team the route plan belongs to.",+ "worker": "Identifier of the worker assigned to the route plan, if any.",+ "vehicleType": "Type of vehicle used for the route.",+ "startTime": "Planned start time of the route, in UNIX epoch milliseconds.",+ "endTime": "Planned end time of the route, in UNIX epoch milliseconds.",+ "actualStartTime": "Time the route actually started, in UNIX epoch milliseconds.",+ "actualEndTime": "Time the route actually ended, in UNIX epoch milliseconds.",+ "startingHubId": "Identifier of the hub the route starts at.",+ "endingHubId": "Identifier of the hub the route ends at.",+ "timeCreated": "Time the route plan was created, in UNIX epoch milliseconds.",+ "timeLastModified": "Time the route plan was last modified, in UNIX epoch milliseconds.",+ "plannedDistance": "Planned distance of the route.",+ "plannedDuration": "Planned duration of the route.",+ "isOverdue": "Whether the route is overdue.",+ "status": "Actual performance against the planned times: EARLY, ON_TIME, LATE, or UNAVAILABLE.",+ "statusTimeDifference": "Difference between the actual and planned times, in milliseconds.",+ },+ },"organization": {"description": "The caller's own Onfleet organization: its name, timezone, country, and delegatee relationships.","docs_url": "https://docs.onfleet.com/reference/organizations",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.pyindex 0ef4fa294696..f0ed145f8c24 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py@@ -1,11 +1,15 @@import dataclasses+from collections.abc import Iteratorfrom datetime import UTC, date, datetimefrom typing import Any, Optionalfrom requests import Request, Responsefrom requests.auth import HTTPBasicAuth-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (+ default_request_timeout,+ make_tracked_session,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,@@ -28,15 +32,20 @@ DEFAULT_FROM_MS = 0# Key carrying the `lastId` cursor in the `/tasks/all` response body and the query param used to# continue after it on the next page._CURSOR_KEY = "lastId"+# `/routePlans` returns at most this many rows per request (its max `limit`) and has no cursor.+ROUTE_PLANS_LIMIT = 500[email protected](frozen=True)class OnfleetResumeConfig:- # The `lastId` cursor returned by the previous page; the next request continues after it.- last_id: str- # The `from` epoch-ms lower bound in effect for this sync, re-sent on every page so the- # server-side window stays applied across the whole paginated walk.- from_ms: int+ # `/tasks/all`: the `lastId` cursor returned by the previous page; the next request continues after it.+ last_id: Optional[str] = None+ # `/tasks/all`: the `from` epoch-ms lower bound in effect for this sync, re-sent on every page so+ # the server-side window stays applied across the whole paginated walk.+ # `/routePlans`: the epoch-ms start of the next created-time window.+ from_ms: int = 0+ # `/routePlans`: the width of the next created-time window, in milliseconds.+ window_ms: Optional[int] = Nonedef _to_epoch_ms(value: Any) -> Optional[int]:@@ -133,7 +142,7 @@ def _paginated_source(initial_paginator_state: Optional[dict[str, Any]] = Noneif resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()- if resume is not None:+ if resume is not None and resume.last_id is not None:initial_paginator_state = {"cursor": resume.last_id}# Re-apply the window the interrupted sync used so resumed pages stay on it.from_ms = resume.from_ms@@ -202,6 +211,82 @@ def _single_batch_source(return rest_api_resource(rest_config, team_id, job_id, db_incremental_field_last_value)+def _now_ms() -> int:+ return int(datetime.now(UTC).timestamp() * 1000)+++def _created_before(row: dict[str, Any], end_ms: int) -> bool:+ created_ms = _to_epoch_ms(row.get("timeCreated"))+ return created_ms is None or created_ms < end_ms+++def _iter_route_plans(+ api_key: str,+ config: OnfleetEndpointConfig,+ resumable_source_manager: ResumableSourceManager[OnfleetResumeConfig],+ from_ms: int,+) -> Iterator[list[dict[str, Any]]]:+ """Walk `/routePlans` in `createdTimeFrom`/`createdTimeTo` windows.++ A window that returns a full page can be truncated, so it is halved and fetched again. A window+ that returns less is complete, and the next window is twice as wide. The docs don't say whether+ `createdTimeTo` is inclusive, so each window keeps only the rows created before its end, and the+ next window starts at that end.+ """+ session = make_tracked_session(redact_values=(api_key,))+ auth = HTTPBasicAuth(api_key, "")+ until_ms = _now_ms()+ window_start = from_ms+ window_ms = max(until_ms - window_start, 1)++ if resumable_source_manager.can_resume():+ resume = resumable_source_manager.load_state()+ if resume is not None and resume.window_ms is not None:+ window_start = resume.from_ms+ window_ms = max(resume.window_ms, 1)++ while True:+ window_end = min(window_start + window_ms, until_ms)+ is_last_window = window_end >= until_ms+ response = session.get(+ f"{ONFLEET_BASE_URL}{config.path}",+ params={"createdTimeFrom": window_start, "createdTimeTo": window_end, "limit": ROUTE_PLANS_LIMIT},+ auth=auth,+ timeout=default_request_timeout(),+ )+ response.raise_for_status()+ rows = response.json()+ # A full refresh replaces the table, so a malformed response must fail the sync rather than+ # silently drop rows.+ if not isinstance(rows, list):+ raise ValueError(f"Onfleet returned a non-list response for {config.path}")++ if len(rows) >= ROUTE_PLANS_LIMIT:+ if window_end - window_start > 1:+ window_ms = max((window_end - window_start) // 2, 1)+ resumable_source_manager.safe_point()+ continue+ # A 1 ms window can't be split further, so advancing past it would silently drop rows.+ raise ValueError(+ f"Onfleet returned {ROUTE_PLANS_LIMIT} or more route plans created between {window_start} and "+ f"{window_end} (epoch ms), which can't be fetched completely"+ )++ if is_last_window:+ if rows:+ yield rows+ return++ rows = [row for row in rows if _created_before(row, window_end)]+ window_ms = (window_end - window_start) * 2+ window_start = window_end+ resumable_source_manager.save_state(OnfleetResumeConfig(from_ms=window_start, window_ms=window_ms))+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()++def onfleet_source(api_key: str,endpoint: str,@@ -213,6 +298,17 @@ def onfleet_source() -> SourceResponse:config = ONFLEET_ENDPOINTS[endpoint]+ if config.created_time_windowed:+ from_ms = _to_epoch_ms(db_incremental_field_last_value) if should_use_incremental_field else None+ windowed_from_ms = DEFAULT_FROM_MS if from_ms is None else from_ms+ return SourceResponse(+ name=endpoint,+ items=lambda: _iter_route_plans(api_key, config, resumable_source_manager, windowed_from_ms),+ primary_keys=config.primary_keys,+ # Windows are walked in ascending order, but rows inside a window are unordered.+ sort_mode=None,+ )+if config.paginated:resource = _paginated_source(api_key=api_key,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.pyindex 5283bf58c585..df091796d44e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py@@ -4,13 +4,16 @@ from typing import Optionalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class OnfleetEndpointConfig:name: strpath: str# Only `/tasks/all` is paginated (cursor via `lastId`, 64 rows/page). Every other list# endpoint returns the full collection as a bare JSON array in a single response.paginated: bool = False+ # `/routePlans` has no cursor and caps each response at 500 rows, so it is walked in+ # `createdTimeFrom`/`createdTimeTo` windows instead.+ created_time_windowed: bool = False# Key wrapping the row array on the paginated endpoint (e.g. {"lastId": ..., "tasks": [...]}).data_key: Optional[str] = None# `/organization` returns a single object rather than an array.@@ -23,10 +26,11 @@ class OnfleetEndpointConfig:# Onfleet timestamps are UNIX epoch **milliseconds** stored as integers, so candidate incremental# fields are integers even though the UI presents them as datetimes.#-# Only `/tasks/all` exposes a server-side time filter (`from`/`to`, epoch ms). `from` filters on a+# `/tasks/all` exposes a server-side time filter (`from`/`to`, epoch ms). `from` filters on a# task's creation time (completion time for tasks in the `completed` state) and results are always-# sorted ascending by creation time, so incremental sync anchors on `timeCreated`. Every other-# endpoint returns a bare array with no time filter and no pagination, so those are full refresh.+# sorted ascending by creation time, so incremental sync anchors on `timeCreated`. `/routePlans`+# filters on creation time too (`createdTimeFrom`/`createdTimeTo`). Every other endpoint returns a+# bare array with no time filter and no pagination, so those are full refresh.## Partitioning is intentionally left off: the datetime partitioner interprets an integer partition# key as epoch *seconds*, which would misbucket Onfleet's epoch-*millisecond* timestamps.@@ -45,6 +49,19 @@ ONFLEET_ENDPOINTS: dict[str, OnfleetEndpointConfig] = {},],),+ "route_plans": OnfleetEndpointConfig(+ name="route_plans",+ path="/routePlans",+ created_time_windowed=True,+ incremental_fields=[+ {+ "label": "timeCreated",+ "type": IncrementalFieldType.DateTime,+ "field": "timeCreated",+ "field_type": IncrementalFieldType.Integer,+ },+ ],+ ),"workers": OnfleetEndpointConfig(name="workers", path="/workers"),"teams": OnfleetEndpointConfig(name="teams", path="/teams"),"hubs": OnfleetEndpointConfig(name="hubs", path="/hubs"),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.pyindex 756d803cccb5..d298dd4ad4ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.py@@ -5,6 +5,7 @@ from typing import Anyimport pytestfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,ScriptedResponse,SourceDriver,always,@@ -13,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.onfleet import (OnfleetSourceConfig,)+from products.warehouse_sources.backend.temporal.data_imports.sources.onfleet import onfleet as onfleet_modulefrom products.warehouse_sources.backend.temporal.data_imports.sources.onfleet.onfleet import (OnfleetResumeConfig,_to_epoch_ms,@@ -98,6 +100,70 @@ class TestGetRowsPaginated:assert [row["id"] for row in result.rows] == ["1", "1"]+class TestRoutePlans:+ @pytest.mark.parametrize("to_inclusive", [True, False])+ def test_window_walk_yields_every_route_plan_once(+ self, monkeypatch: pytest.MonkeyPatch, to_inclusive: bool+ ) -> None:+ monkeypatch.setattr(onfleet_module, "ROUTE_PLANS_LIMIT", 3)+ monkeypatch.setattr(onfleet_module, "_now_ms", lambda: 128)+ # Rows sit on the power-of-two window boundaries the walk produces.+ created_times = {+ f"r{index}": created for index, created in enumerate([0, 0, 4, 6, 8, 10, 10, 12, 16, 30, 32, 47, 64, 100])+ }++ def respond(request: RecordedRequest) -> ScriptedResponse:+ start, end, limit = (+ int(request.param(name) or 0) for name in ("createdTimeFrom", "createdTimeTo", "limit")+ )+ matches = [+ {"id": plan_id, "timeCreated": created}+ for plan_id, created in reversed(created_times.items())+ if start <= created and (created <= end if to_inclusive else created < end)+ ]+ return ScriptedResponse(json=matches[:limit])++ result = _driver().run("route_plans", respond)++ assert result.raised is None+ assert sorted(row["id"] for row in result.rows) == sorted(created_times)++ def test_saturated_minimum_window_fails_the_sync(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(onfleet_module, "ROUTE_PLANS_LIMIT", 2)+ monkeypatch.setattr(onfleet_module, "_now_ms", lambda: 8)+ full_page = [{"id": "a", "timeCreated": 0}, {"id": "b", "timeCreated": 0}]++ result = _driver().run("route_plans", always(ScriptedResponse(json=full_page)))++ assert isinstance(result.raised, ValueError)+ assert result.rows == []++ @pytest.mark.parametrize(+ "inputs, resume_state, expected_from, expected_to",+ [+ ({}, None, "0", "2000"),+ ({"incremental_field": "timeCreated", "db_incremental_field_last_value": 1200}, None, "1200", "2000"),+ ({}, OnfleetResumeConfig(from_ms=1000, window_ms=500), "1000", "1500"),+ ],+ )+ def test_first_window(+ self,+ monkeypatch: pytest.MonkeyPatch,+ inputs: dict[str, Any],+ resume_state: OnfleetResumeConfig | None,+ expected_from: str,+ expected_to: str,+ ) -> None:+ monkeypatch.setattr(onfleet_module, "_now_ms", lambda: 2000)++ result = _driver().run("route_plans", always(ScriptedResponse(json=[])), resume_state=resume_state, **inputs)++ assert result.raised is None+ assert result.paths[0] == "/api/v2/routePlans"+ assert result.requests[0].param("createdTimeFrom") == expected_from+ assert result.requests[0].param("createdTimeTo") == expected_to++class TestGetCredentialsStatus:def test_returns_none_on_transport_error(self) -> None:with scripted_network(always(ConnectionError("boom"))) as network: