Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add onfleet route plans table (#123362)

Tom Owers4240c19845a3
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 1efc084eccad..ba4d445ed822 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -814,8 +814,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/onepagecrm.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/onepassword.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/open_meteo/open_meteo.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8e86e99fc77d..24de3231cdad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8687,7 +8687,7 @@ Today (7): `administrators`, `hubs`, `organization`, `tasks`, `teams`, `webhooks
Diffed against: <https://docs.onfleet.com/reference/administrators>
-- [ ] `routePlans` — route plans group tasks into driver routes - core delivery-efficiency object (high)
+- [x] `routePlans` — route plans group tasks into driver routes - core delivery-efficiency object (high)
- [ ] `taskOrders (orders)` — order-level records that tasks are created from, with pricing/quote state (medium)
- [ ] `destinations` — lookup resolving the destination IDs referenced by synced tasks (address/geo dimension) (medium)
- [ ] `recipients` — lookup resolving recipient IDs on synced tasks (customer dimension) (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.py
index dc52af2a56e8..92e58dc3b937 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/canonical_descriptions.py
@@ -123,6 +123,34 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"isEnabled": "Whether the webhook is currently enabled.",
},
},
+ "route_plans": {
+ "description": "Route plans that group a driver's tasks into an ordered route for a shift, day, or planned segment of work.",
+ "docs_url": "https://docs.onfleet.com/reference/get-route-plan",
+ "columns": {
+ "id": "Unique identifier for the route plan.",
+ "name": "The route plan's name.",
+ "state": "Current state of the route plan, for example PENDING.",
+ "tasks": "Identifiers of the tasks on the route, in route order.",
+ "taskStates": "State of each task on the route, in the same order as tasks (0 unassigned, 1 assigned, 2 active, 3 completed).",
+ "organization": "Identifier of the organization that owns the route plan.",
+ "team": "Identifier of the team the route plan belongs to.",
+ "worker": "Identifier of the worker assigned to the route plan, if any.",
+ "vehicleType": "Type of vehicle used for the route.",
+ "startTime": "Planned start time of the route, in UNIX epoch milliseconds.",
+ "endTime": "Planned end time of the route, in UNIX epoch milliseconds.",
+ "actualStartTime": "Time the route actually started, in UNIX epoch milliseconds.",
+ "actualEndTime": "Time the route actually ended, in UNIX epoch milliseconds.",
+ "startingHubId": "Identifier of the hub the route starts at.",
+ "endingHubId": "Identifier of the hub the route ends at.",
+ "timeCreated": "Time the route plan was created, in UNIX epoch milliseconds.",
+ "timeLastModified": "Time the route plan was last modified, in UNIX epoch milliseconds.",
+ "plannedDistance": "Planned distance of the route.",
+ "plannedDuration": "Planned duration of the route.",
+ "isOverdue": "Whether the route is overdue.",
+ "status": "Actual performance against the planned times: EARLY, ON_TIME, LATE, or UNAVAILABLE.",
+ "statusTimeDifference": "Difference between the actual and planned times, in milliseconds.",
+ },
+ },
"organization": {
"description": "The caller's own Onfleet organization: its name, timezone, country, and delegatee relationships.",
"docs_url": "https://docs.onfleet.com/reference/organizations",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py
index 0ef4fa294696..f0ed145f8c24 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/onfleet.py
@@ -1,11 +1,15 @@
import dataclasses
+from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
from requests import Request, Response
from requests.auth import HTTPBasicAuth
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ default_request_timeout,
+ make_tracked_session,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
@@ -28,15 +32,20 @@ DEFAULT_FROM_MS = 0
# Key carrying the `lastId` cursor in the `/tasks/all` response body and the query param used to
# continue after it on the next page.
_CURSOR_KEY = "lastId"
+# `/routePlans` returns at most this many rows per request (its max `limit`) and has no cursor.
+ROUTE_PLANS_LIMIT = 500
[email protected](frozen=True)
class OnfleetResumeConfig:
- # The `lastId` cursor returned by the previous page; the next request continues after it.
- last_id: str
- # The `from` epoch-ms lower bound in effect for this sync, re-sent on every page so the
- # server-side window stays applied across the whole paginated walk.
- from_ms: int
+ # `/tasks/all`: the `lastId` cursor returned by the previous page; the next request continues after it.
+ last_id: Optional[str] = None
+ # `/tasks/all`: the `from` epoch-ms lower bound in effect for this sync, re-sent on every page so
+ # the server-side window stays applied across the whole paginated walk.
+ # `/routePlans`: the epoch-ms start of the next created-time window.
+ from_ms: int = 0
+ # `/routePlans`: the width of the next created-time window, in milliseconds.
+ window_ms: Optional[int] = None
def _to_epoch_ms(value: Any) -> Optional[int]:
@@ -133,7 +142,7 @@ def _paginated_source(
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
- if resume is not None:
+ if resume is not None and resume.last_id is not None:
initial_paginator_state = {"cursor": resume.last_id}
# Re-apply the window the interrupted sync used so resumed pages stay on it.
from_ms = resume.from_ms
@@ -202,6 +211,82 @@ def _single_batch_source(
return rest_api_resource(rest_config, team_id, job_id, db_incremental_field_last_value)
+def _now_ms() -> int:
+ return int(datetime.now(UTC).timestamp() * 1000)
+
+
+def _created_before(row: dict[str, Any], end_ms: int) -> bool:
+ created_ms = _to_epoch_ms(row.get("timeCreated"))
+ return created_ms is None or created_ms < end_ms
+
+
+def _iter_route_plans(
+ api_key: str,
+ config: OnfleetEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[OnfleetResumeConfig],
+ from_ms: int,
+) -> Iterator[list[dict[str, Any]]]:
+ """Walk `/routePlans` in `createdTimeFrom`/`createdTimeTo` windows.
+
+ A window that returns a full page can be truncated, so it is halved and fetched again. A window
+ that returns less is complete, and the next window is twice as wide. The docs don't say whether
+ `createdTimeTo` is inclusive, so each window keeps only the rows created before its end, and the
+ next window starts at that end.
+ """
+ session = make_tracked_session(redact_values=(api_key,))
+ auth = HTTPBasicAuth(api_key, "")
+ until_ms = _now_ms()
+ window_start = from_ms
+ window_ms = max(until_ms - window_start, 1)
+
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None and resume.window_ms is not None:
+ window_start = resume.from_ms
+ window_ms = max(resume.window_ms, 1)
+
+ while True:
+ window_end = min(window_start + window_ms, until_ms)
+ is_last_window = window_end >= until_ms
+ response = session.get(
+ f"{ONFLEET_BASE_URL}{config.path}",
+ params={"createdTimeFrom": window_start, "createdTimeTo": window_end, "limit": ROUTE_PLANS_LIMIT},
+ auth=auth,
+ timeout=default_request_timeout(),
+ )
+ response.raise_for_status()
+ rows = response.json()
+ # A full refresh replaces the table, so a malformed response must fail the sync rather than
+ # silently drop rows.
+ if not isinstance(rows, list):
+ raise ValueError(f"Onfleet returned a non-list response for {config.path}")
+
+ if len(rows) >= ROUTE_PLANS_LIMIT:
+ if window_end - window_start > 1:
+ window_ms = max((window_end - window_start) // 2, 1)
+ resumable_source_manager.safe_point()
+ continue
+ # A 1 ms window can't be split further, so advancing past it would silently drop rows.
+ raise ValueError(
+ f"Onfleet returned {ROUTE_PLANS_LIMIT} or more route plans created between {window_start} and "
+ f"{window_end} (epoch ms), which can't be fetched completely"
+ )
+
+ if is_last_window:
+ if rows:
+ yield rows
+ return
+
+ rows = [row for row in rows if _created_before(row, window_end)]
+ window_ms = (window_end - window_start) * 2
+ window_start = window_end
+ resumable_source_manager.save_state(OnfleetResumeConfig(from_ms=window_start, window_ms=window_ms))
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
def onfleet_source(
api_key: str,
endpoint: str,
@@ -213,6 +298,17 @@ def onfleet_source(
) -> SourceResponse:
config = ONFLEET_ENDPOINTS[endpoint]
+ if config.created_time_windowed:
+ from_ms = _to_epoch_ms(db_incremental_field_last_value) if should_use_incremental_field else None
+ windowed_from_ms = DEFAULT_FROM_MS if from_ms is None else from_ms
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: _iter_route_plans(api_key, config, resumable_source_manager, windowed_from_ms),
+ primary_keys=config.primary_keys,
+ # Windows are walked in ascending order, but rows inside a window are unordered.
+ sort_mode=None,
+ )
+
if config.paginated:
resource = _paginated_source(
api_key=api_key,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py
index 5283bf58c585..df091796d44e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/settings.py
@@ -4,13 +4,16 @@ from typing import Optional
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
class OnfleetEndpointConfig:
name: str
path: str
# Only `/tasks/all` is paginated (cursor via `lastId`, 64 rows/page). Every other list
# endpoint returns the full collection as a bare JSON array in a single response.
paginated: bool = False
+ # `/routePlans` has no cursor and caps each response at 500 rows, so it is walked in
+ # `createdTimeFrom`/`createdTimeTo` windows instead.
+ created_time_windowed: bool = False
# Key wrapping the row array on the paginated endpoint (e.g. {"lastId": ..., "tasks": [...]}).
data_key: Optional[str] = None
# `/organization` returns a single object rather than an array.
@@ -23,10 +26,11 @@ class OnfleetEndpointConfig:
# Onfleet timestamps are UNIX epoch **milliseconds** stored as integers, so candidate incremental
# fields are integers even though the UI presents them as datetimes.
#
-# Only `/tasks/all` exposes a server-side time filter (`from`/`to`, epoch ms). `from` filters on a
+# `/tasks/all` exposes a server-side time filter (`from`/`to`, epoch ms). `from` filters on a
# task's creation time (completion time for tasks in the `completed` state) and results are always
-# sorted ascending by creation time, so incremental sync anchors on `timeCreated`. Every other
-# endpoint returns a bare array with no time filter and no pagination, so those are full refresh.
+# sorted ascending by creation time, so incremental sync anchors on `timeCreated`. `/routePlans`
+# filters on creation time too (`createdTimeFrom`/`createdTimeTo`). Every other endpoint returns a
+# bare array with no time filter and no pagination, so those are full refresh.
#
# Partitioning is intentionally left off: the datetime partitioner interprets an integer partition
# key as epoch *seconds*, which would misbucket Onfleet's epoch-*millisecond* timestamps.
@@ -45,6 +49,19 @@ ONFLEET_ENDPOINTS: dict[str, OnfleetEndpointConfig] = {
},
],
),
+ "route_plans": OnfleetEndpointConfig(
+ name="route_plans",
+ path="/routePlans",
+ created_time_windowed=True,
+ incremental_fields=[
+ {
+ "label": "timeCreated",
+ "type": IncrementalFieldType.DateTime,
+ "field": "timeCreated",
+ "field_type": IncrementalFieldType.Integer,
+ },
+ ],
+ ),
"workers": OnfleetEndpointConfig(name="workers", path="/workers"),
"teams": OnfleetEndpointConfig(name="teams", path="/teams"),
"hubs": OnfleetEndpointConfig(name="hubs", path="/hubs"),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.py b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.py
index 756d803cccb5..d298dd4ad4ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/onfleet/tests/test_onfleet.py
@@ -5,6 +5,7 @@ from typing import Any
import pytest
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
@@ -13,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.onfleet import (
OnfleetSourceConfig,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.onfleet import onfleet as onfleet_module
from products.warehouse_sources.backend.temporal.data_imports.sources.onfleet.onfleet import (
OnfleetResumeConfig,
_to_epoch_ms,
@@ -98,6 +100,70 @@ class TestGetRowsPaginated:
assert [row["id"] for row in result.rows] == ["1", "1"]
+class TestRoutePlans:
+ @pytest.mark.parametrize("to_inclusive", [True, False])
+ def test_window_walk_yields_every_route_plan_once(
+ self, monkeypatch: pytest.MonkeyPatch, to_inclusive: bool
+ ) -> None:
+ monkeypatch.setattr(onfleet_module, "ROUTE_PLANS_LIMIT", 3)
+ monkeypatch.setattr(onfleet_module, "_now_ms", lambda: 128)
+ # Rows sit on the power-of-two window boundaries the walk produces.
+ created_times = {
+ f"r{index}": created for index, created in enumerate([0, 0, 4, 6, 8, 10, 10, 12, 16, 30, 32, 47, 64, 100])
+ }
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ start, end, limit = (
+ int(request.param(name) or 0) for name in ("createdTimeFrom", "createdTimeTo", "limit")
+ )
+ matches = [
+ {"id": plan_id, "timeCreated": created}
+ for plan_id, created in reversed(created_times.items())
+ if start <= created and (created <= end if to_inclusive else created < end)
+ ]
+ return ScriptedResponse(json=matches[:limit])
+
+ result = _driver().run("route_plans", respond)
+
+ assert result.raised is None
+ assert sorted(row["id"] for row in result.rows) == sorted(created_times)
+
+ def test_saturated_minimum_window_fails_the_sync(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(onfleet_module, "ROUTE_PLANS_LIMIT", 2)
+ monkeypatch.setattr(onfleet_module, "_now_ms", lambda: 8)
+ full_page = [{"id": "a", "timeCreated": 0}, {"id": "b", "timeCreated": 0}]
+
+ result = _driver().run("route_plans", always(ScriptedResponse(json=full_page)))
+
+ assert isinstance(result.raised, ValueError)
+ assert result.rows == []
+
+ @pytest.mark.parametrize(
+ "inputs, resume_state, expected_from, expected_to",
+ [
+ ({}, None, "0", "2000"),
+ ({"incremental_field": "timeCreated", "db_incremental_field_last_value": 1200}, None, "1200", "2000"),
+ ({}, OnfleetResumeConfig(from_ms=1000, window_ms=500), "1000", "1500"),
+ ],
+ )
+ def test_first_window(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ inputs: dict[str, Any],
+ resume_state: OnfleetResumeConfig | None,
+ expected_from: str,
+ expected_to: str,
+ ) -> None:
+ monkeypatch.setattr(onfleet_module, "_now_ms", lambda: 2000)
+
+ result = _driver().run("route_plans", always(ScriptedResponse(json=[])), resume_state=resume_state, **inputs)
+
+ assert result.raised is None
+ assert result.paths[0] == "/api/v2/routePlans"
+ assert result.requests[0].param("createdTimeFrom") == expected_from
+ assert result.requests[0].param("createdTimeTo") == expected_to
+
+
class TestGetCredentialsStatus:
def test_returns_none_on_transport_error(self) -> None:
with scripted_network(always(ConnectionError("boom"))) as network: