Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add recreation reservations, attribute and zone tables (#126978)

Tom Owers08d24cad9f23
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 5a9ae7a4a16b..2411690e833e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10104,10 +10104,10 @@ Note: No OpenAPI spec is served (openapi.json 404s). The docs are a Nuxt SSR app
Diffed against: <https://ridb.recreation.gov/shared/swagger/ridb.yaml>
-- [ ] `/reservations` — The only transactional table in RIDB - actual bookings by date; everything currently synced is static reference data (high)
-- [ ] `/campsites/{id}/attributes` — Per-campsite attribute breakdown dimensions (hookups, max occupancy, site type) that make campsite data filterable (medium)
-- [ ] `/permitentrances/{id}/zones` — Lookup resolving the zones under permit entrances already synced (medium)
-- [ ] `/permitentrances/{id}/attributes and /tours/{id}/attributes` — Attribute breakdown dimensions for permits and tours, matching the campsite attributes pattern (low)
+- [x] `/reservations` — The only transactional table in RIDB - actual bookings by date; everything currently synced is static reference data (high)
+- [x] `/campsites/{id}/attributes` — Per-campsite attribute breakdown dimensions (hookups, max occupancy, site type) that make campsite data filterable (medium)
+- [x] `/permitentrances/{id}/zones` — Lookup resolving the zones under permit entrances already synced (medium)
+- [x] `/permitentrances/{id}/attributes and /tours/{id}/attributes` — Attribute breakdown dimensions for permits and tours, matching the campsite attributes pattern (low)
- [ ] `Availability API: /api/availability/camping/{facilityId} and /ticket/{facilityId}` — Site-level availability by month - the demand signal that pairs with the facility data already synced (separate spec, tiered key) (low)
Note: The documented spec path (ridb.recreation.gov/docs/ridb_swagger.json) is dead; the live OpenAPI 3.0 spec is at /shared/swagger/ridb.yaml, found by reading the docs SPA bundle. Coverage of the reference-data collections is essentially complete - every top-level RIDB collection except reservations is already synced. There is also a separate Availability API spec at /shared/swagger/availability.yaml (camping and ticket availability, per-facility per-month) behind a higher API-key tier.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/canonical_descriptions.py
index c6440eb165e1..21bd2a924228 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/canonical_descriptions.py
@@ -219,4 +219,81 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"MEMBERTOURS": "Member tours included within this tour.",
},
},
+ "CampsiteAttributes": {
+ "description": "One row per attribute of a campsite (e.g. max num of people, driveway length, site access), so campsites can be filtered by attribute.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "CampsiteID": "Identifier of the campsite the attribute belongs to.",
+ "AttributeID": "Identifier of the attribute.",
+ "AttributeName": "Name of the attribute.",
+ "AttributeValue": "Value of the attribute for the campsite.",
+ },
+ },
+ "PermitEntranceAttributes": {
+ "description": "One row per attribute of a permit entrance.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "PermitEntranceID": "Identifier of the permit entrance the attribute belongs to.",
+ "AttributeID": "Identifier of the attribute.",
+ "AttributeName": "Name of the attribute.",
+ "AttributeValue": "Value of the attribute for the permit entrance.",
+ },
+ },
+ "PermitEntranceZones": {
+ "description": "Zones that belong to a permit entrance.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "PermitEntranceID": "Identifier of the permit entrance the zone belongs to.",
+ "PermitEntranceZoneID": "Identifier of the zone.",
+ "Zone": "Name of the zone.",
+ },
+ },
+ "TourAttributes": {
+ "description": "One row per attribute of a tour.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "TourID": "Identifier of the tour the attribute belongs to.",
+ "AttributeID": "Identifier of the attribute.",
+ "AttributeName": "Name of the attribute.",
+ "AttributeValue": "Value of the attribute for the tour.",
+ },
+ },
+ "Reservations": {
+ "description": "Historical Recreation.gov reservations (orders) with location, product, fees, stay dates, and party size.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "HistoricalReservationID": "Unique identifier for the reservation record.",
+ "OrderNumber": "Recreation.gov order number of the reservation.",
+ "Agency": "Federal agency that manages the reserved location.",
+ "OrgID": "Identifier of the organization that manages the reserved location.",
+ "RegionCode": "Code of the agency region.",
+ "RegionDescription": "Name of the agency region.",
+ "ParentLocationID": "Identifier of the parent location (e.g. recreation area).",
+ "ParentLocation": "Name of the parent location.",
+ "LegacyFacilityID": "Facility identifier in the legacy reservation system.",
+ "Park": "Name of the reserved facility or park.",
+ "SiteType": "Type of the reserved site.",
+ "UseType": "Use type of the reservation (e.g. overnight, day).",
+ "ProductID": "Identifier of the reserved product (e.g. campsite).",
+ "InventoryType": "Inventory type of the reserved product (e.g. camping, ticket, permit).",
+ "FacilityID": "Identifier of the reserved facility.",
+ "FacilityZIP": "ZIP code of the reserved facility.",
+ "FacilityState": "State of the reserved facility.",
+ "CustomerZIP": "ZIP code of the customer.",
+ "Tax": "Tax paid for the reservation.",
+ "UseFee": "Use fee paid for the reservation.",
+ "TranFee": "Transaction fee paid for the reservation.",
+ "AttrFee": "Attribute fee paid for the reservation.",
+ "TotalBeforeTax": "Total amount before tax.",
+ "Discount": "Discount applied to the reservation.",
+ "TotalPaid": "Total amount paid for the reservation.",
+ "StartDate": "First day of the reserved stay or use.",
+ "EndDate": "Last day of the reserved stay or use.",
+ "OrderDate": "Date the reservation was ordered.",
+ "Nights": "Number of nights reserved.",
+ "NumberOfPeople": "Number of people in the reservation.",
+ "EquipmentDescription": "Equipment the customer brings (e.g. tent, RV).",
+ "EquipmentLength": "Length of the customer's equipment.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/recreation.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/recreation.py
index 1381291078df..28a5baaf15bf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/recreation.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/recreation.py
@@ -1,13 +1,19 @@
-import dataclasses
+from collections.abc import Callable
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
OffsetPaginator,
+ PageNumberPaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
@@ -15,15 +21,30 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.temporal.data_imports.sources.recreation.settings import (
PAGE_LIMIT,
RECREATION_ENDPOINTS,
+ RESERVATIONS_ENDPOINT,
+ RESERVATIONS_FORWARD_DAYS,
+ RESERVATIONS_LOOKBACK_DAYS,
RIDB_BASE_URL,
+ NestedRowsConfig,
)
VALIDATION_TIMEOUT_SECONDS = 10
+@frozen
class RecreationResumeConfig:
- offset: int
+ offset: int | None = None
+ # `/reservations` pages by page number instead of offset. A resumed run reuses the saved date
+ # window, because a moved window shifts the rows behind each page number.
+ page: int | None = None
+ date_from: str | None = None
+ date_to: str | None = None
+
+
+@frozen
+class ReservationsWindow:
+ date_from: date
+ date_to: date
def _build_paginator() -> OffsetPaginator:
@@ -32,19 +53,81 @@ def _build_paginator() -> OffsetPaginator:
return OffsetPaginator(limit=PAGE_LIMIT, total_path="METADATA.RESULTS.TOTAL_COUNT")
-def get_resource(endpoint: str) -> EndpointResource:
+def _nested_rows(nested: NestedRowsConfig) -> Callable[[dict[str, Any]], list[dict[str, Any]]]:
+ def _explode(row: dict[str, Any]) -> list[dict[str, Any]]:
+ parent_id = row.get(nested.parent_id_field)
+ return [
+ {nested.parent_id_field: parent_id, **item}
+ for item in row.get(nested.field) or []
+ if isinstance(item, dict)
+ ]
+
+ return _explode
+
+
+def _as_date(value: Any) -> date | None:
+ if isinstance(value, date) and not isinstance(value, datetime):
+ return value
+ parsed = parse_datetime_value(value)
+ return parsed.date() if parsed is not None else None
+
+
+def _reservations_window(
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+ resume_config: RecreationResumeConfig | None,
+ today: date,
+) -> ReservationsWindow:
+ if resume_config is not None and resume_config.date_from and resume_config.date_to:
+ return ReservationsWindow(
+ date_from=date.fromisoformat(resume_config.date_from),
+ date_to=date.fromisoformat(resume_config.date_to),
+ )
+ date_from = today - timedelta(days=RESERVATIONS_LOOKBACK_DAYS)
+ if should_use_incremental_field:
+ # Restart from the watermark day itself: the filter is day-grained, and merge drops the repeats.
+ last_value = _as_date(db_incremental_field_last_value)
+ if last_value is not None:
+ date_from = last_value
+ return ReservationsWindow(date_from=date_from, date_to=today + timedelta(days=RESERVATIONS_FORWARD_DAYS))
+
+
+def get_resource(
+ endpoint: str,
+ should_use_incremental_field: bool = False,
+ reservations_window: ReservationsWindow | None = None,
+) -> EndpointResource:
config = RECREATION_ENDPOINTS[endpoint]
- return {
+ params: dict[str, Any] = {}
+ data_selector = "RECDATA"
+ paginator: BasePaginator = _build_paginator()
+
+ if reservations_window is not None:
+ params = {
+ "dateFrom": reservations_window.date_from.isoformat(),
+ "dateTo": reservations_window.date_to.isoformat(),
+ "limit": PAGE_LIMIT,
+ }
+ data_selector = "data"
+ # `page` is the number of pages to skip, so the first page is 0. `total_results` counts rows,
+ # not pages, so the paginator stops on the first empty page.
+ paginator = PageNumberPaginator(base_page=0, page_param="page")
+
+ resource: EndpointResource = {
"name": config.name,
"table_name": config.name.lower(),
- "write_disposition": "replace",
+ "write_disposition": "merge" if should_use_incremental_field else "replace",
"endpoint": {
- "data_selector": "RECDATA",
+ "data_selector": data_selector,
"path": config.path,
- "paginator": _build_paginator(),
+ "params": params,
+ "paginator": paginator,
},
"table_format": "delta",
}
+ if config.nested is not None:
+ resource["data_map"] = _nested_rows(config.nested)
+ return resource
def recreation_source(
@@ -53,9 +136,18 @@ def recreation_source(
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[RecreationResumeConfig],
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
endpoint_config = RECREATION_ENDPOINTS[endpoint]
+ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ reservations_window: ReservationsWindow | None = None
+ if endpoint == RESERVATIONS_ENDPOINT:
+ reservations_window = _reservations_window(
+ should_use_incremental_field, db_incremental_field_last_value, resume_config, datetime.now(UTC).date()
+ )
+
config: RESTAPIConfig = {
"client": {
"base_url": RIDB_BASE_URL,
@@ -73,20 +165,32 @@ def recreation_source(
"resource_defaults": {
"write_disposition": "replace",
},
- "resources": [get_resource(endpoint)],
+ "resources": [get_resource(endpoint, should_use_incremental_field, reservations_window)],
}
initial_paginator_state: Optional[dict[str, Any]] = None
- if resumable_source_manager.can_resume():
- resume_config = resumable_source_manager.load_state()
- if resume_config is not None:
- initial_paginator_state = {"offset": resume_config.offset}
+ if resume_config is not None:
+ initial_paginator_state = {
+ key: value
+ for key, value in (("offset", resume_config.offset), ("page", resume_config.page))
+ if value is not None
+ } or None
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Only persist when there's a next page to resume to; the Redis TTL
# handles cleanup on completion.
- if state and state.get("offset") is not None:
- resumable_source_manager.save_state(RecreationResumeConfig(offset=int(state["offset"])))
+ if not state:
+ return
+ offset, page = state.get("offset"), state.get("page")
+ if offset is not None or page is not None:
+ resumable_source_manager.save_state(
+ RecreationResumeConfig(
+ offset=int(offset) if offset is not None else None,
+ page=int(page) if page is not None else None,
+ date_from=reservations_window.date_from.isoformat() if reservations_window else None,
+ date_to=reservations_window.date_to.isoformat() if reservations_window else None,
+ )
+ )
resource = rest_api_resource(
config,
@@ -107,10 +211,15 @@ def recreation_source(
"partition_keys": [endpoint_config.partition_key],
}
+ primary_keys = [endpoint_config.primary_key]
+ if endpoint_config.nested is not None:
+ # Attribute and zone rows are unique only within their parent.
+ primary_keys = [endpoint_config.nested.parent_id_field, endpoint_config.primary_key]
+
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=[endpoint_config.primary_key],
+ primary_keys=primary_keys,
**partition_kwargs,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/settings.py
index e012948389ce..2b67ab1d3530 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/settings.py
@@ -1,12 +1,27 @@
from dataclasses import dataclass
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
RIDB_BASE_URL = "https://ridb.recreation.gov/api/v1"
# RIDB caps the `limit` query param at 50 records per page.
PAGE_LIMIT = 50
+RESERVATIONS_ENDPOINT = "Reservations"
+# `/reservations` requires a date range. A full refresh, or the first incremental sync, reads this far back.
+RESERVATIONS_LOOKBACK_DAYS = 365
+# The spec does not say which reservation date `dateFrom`/`dateTo` filter on. An end date in the
+# future keeps new orders for future stays in range if the filter is on the stay dates.
+RESERVATIONS_FORWARD_DAYS = 2 * 365
+
+
+@dataclass(frozen=True)
+class NestedRowsConfig:
+ """Rows taken from an array that every parent row embeds, one table row per array item."""
+
+ field: str
+ parent_id_field: str
+
@dataclass(frozen=True)
class RecreationEndpointConfig:
@@ -16,6 +31,9 @@ class RecreationEndpointConfig:
# Stable creation timestamp for datetime partitioning. Most RIDB entities only expose
# LastUpdatedDate, which changes over time and is unsafe as a partition key.
partition_key: str | None = None
+ # The parent list embeds the same rows as the per-parent `/{id}/attributes` and `/{id}/zones`
+ # endpoints, so reading them from the list avoids one request per parent row.
+ nested: NestedRowsConfig | None = None
RECREATION_ENDPOINTS: dict[str, RecreationEndpointConfig] = {
@@ -62,7 +80,7 @@ RECREATION_ENDPOINTS: dict[str, RecreationEndpointConfig] = {
),
"PermitEntrances": RecreationEndpointConfig(
name="PermitEntrances",
- path="/permits",
+ path="/permitentrances",
primary_key="PermitEntranceID",
partition_key="CreatedDate",
),
@@ -82,10 +100,48 @@ RECREATION_ENDPOINTS: dict[str, RecreationEndpointConfig] = {
primary_key="TourID",
partition_key="CreatedDate",
),
+ "CampsiteAttributes": RecreationEndpointConfig(
+ name="CampsiteAttributes",
+ path="/campsites",
+ primary_key="AttributeName",
+ nested=NestedRowsConfig(field="ATTRIBUTES", parent_id_field="CampsiteID"),
+ ),
+ "PermitEntranceAttributes": RecreationEndpointConfig(
+ name="PermitEntranceAttributes",
+ path="/permitentrances",
+ primary_key="AttributeName",
+ nested=NestedRowsConfig(field="ATTRIBUTES", parent_id_field="PermitEntranceID"),
+ ),
+ "PermitEntranceZones": RecreationEndpointConfig(
+ name="PermitEntranceZones",
+ path="/permitentrances",
+ primary_key="PermitEntranceZoneID",
+ nested=NestedRowsConfig(field="ZONES", parent_id_field="PermitEntranceID"),
+ ),
+ "TourAttributes": RecreationEndpointConfig(
+ name="TourAttributes",
+ path="/tours",
+ primary_key="AttributeName",
+ nested=NestedRowsConfig(field="ATTRIBUTES", parent_id_field="TourID"),
+ ),
+ RESERVATIONS_ENDPOINT: RecreationEndpointConfig(
+ name=RESERVATIONS_ENDPOINT,
+ path="/reservations",
+ primary_key="HistoricalReservationID",
+ ),
}
ENDPOINTS = tuple(RECREATION_ENDPOINTS.keys())
-# RIDB has no server-side timestamp filter on any list endpoint (query params cover
-# state/activity/keyword/radius only), so every endpoint is full refresh.
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+# Apart from the `/reservations` date range, the synced RIDB list endpoints have no server-side
+# timestamp filter (query params cover state/activity/keyword/radius only), so they are full refresh.
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ RESERVATIONS_ENDPOINT: [
+ {
+ "label": "OrderDate",
+ "type": IncrementalFieldType.Date,
+ "field": "OrderDate",
+ "field_type": IncrementalFieldType.Date,
+ }
+ ],
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/source.py
index 421162598b07..22a9811e0cd1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/source.py
@@ -30,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.recreation
from products.warehouse_sources.backend.temporal.data_imports.sources.recreation.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ RESERVATIONS_ENDPOINT,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -67,7 +68,8 @@ class RecreationSource(ResumableSource[RecreationSourceConfig, RecreationResumeC
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ # The reservations window restarts at the watermark day, so append would duplicate that day's rows.
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=(RESERVATIONS_ENDPOINT,))
def validate_credentials(
self,
@@ -93,6 +95,8 @@ class RecreationSource(ResumableSource[RecreationSourceConfig, RecreationResumeC
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
@property
@@ -101,7 +105,7 @@ class RecreationSource(ResumableSource[RecreationSourceConfig, RecreationResumeC
name=ExternalDataSourceType.RECREATION,
category=DataWarehouseSourceCategory.ANALYTICS,
label="Recreation.gov",
- caption="""Import public US federal recreation data from the Recreation Information Database (RIDB) behind Recreation.gov: recreation areas, facilities, campsites, tours, permit entrances, and more.
+ caption="""Import public US federal recreation data from the Recreation Information Database (RIDB) behind Recreation.gov: recreation areas, facilities, campsites, tours, permit entrances, reservations, and more.
To get an API key, sign in at [ridb.recreation.gov](https://ridb.recreation.gov/), open your profile from the account menu, and copy the API key shown there.""",
keywords=["ridb", "recreation.gov", "campgrounds", "campsites"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation.py
index 1d57f66c134b..99f1fca811e3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation.py
@@ -1,5 +1,8 @@
+from datetime import UTC, date, datetime, timedelta
from typing import Any
+import pytest
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
ScriptedResponse,
@@ -11,7 +14,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.recreation.recreation import (
RecreationResumeConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.recreation.settings import PAGE_LIMIT
+from products.warehouse_sources.backend.temporal.data_imports.sources.recreation.settings import (
+ PAGE_LIMIT,
+ RESERVATIONS_FORWARD_DAYS,
+ RESERVATIONS_LOOKBACK_DAYS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.recreation.source import RecreationSource
@@ -31,6 +38,11 @@ def _facility_rows(start: int, count: int) -> list[dict[str, Any]]:
return [{"FacilityID": str(i), "FacilityName": f"Facility {i}"} for i in range(start, start + count)]
+def _reservations_response(start: int, count: int) -> ScriptedResponse:
+ rows = [{"HistoricalReservationID": str(i), "OrderDate": "2026-10-01"} for i in range(start, start + count)]
+ return ScriptedResponse(json={"data": rows, "total_results": 120})
+
+
class TestRecreationSourceTransport:
def _drive(self, endpoint: str, responses: list[ScriptedResponse | BaseException], **inputs: Any) -> DriveResult:
config = RecreationSourceConfig(api_key="test-key")
@@ -72,3 +84,111 @@ class TestRecreationSourceTransport:
result = self._drive("Facilities", responses)
assert result.params("offset") == ["0"]
+
+ @pytest.mark.parametrize(
+ ("endpoint", "expected_path", "parent_rows", "expected_rows"),
+ [
+ (
+ "CampsiteAttributes",
+ "/api/v1/campsites",
+ [
+ {
+ "CampsiteID": "10",
+ "ATTRIBUTES": [
+ {"AttributeName": "Max Num of People", "AttributeValue": "6"},
+ {"AttributeName": "Driveway Length", "AttributeValue": "40"},
+ ],
+ },
+ {"CampsiteID": "11", "ATTRIBUTES": []},
+ {"CampsiteID": "12", "ATTRIBUTES": None},
+ {"CampsiteID": "13", "ATTRIBUTES": [{"AttributeName": "Shade", "AttributeValue": "Yes"}]},
+ ],
+ [
+ {"CampsiteID": "10", "AttributeName": "Max Num of People", "AttributeValue": "6"},
+ {"CampsiteID": "10", "AttributeName": "Driveway Length", "AttributeValue": "40"},
+ {"CampsiteID": "13", "AttributeName": "Shade", "AttributeValue": "Yes"},
+ ],
+ ),
+ (
+ "PermitEntranceZones",
+ "/api/v1/permitentrances",
+ [
+ {
+ "PermitEntranceID": "200",
+ "ATTRIBUTES": [{"AttributeName": "Quota", "AttributeValue": "10"}],
+ "ZONES": [{"PermitEntranceZoneID": "1", "Zone": "North"}],
+ }
+ ],
+ [{"PermitEntranceID": "200", "PermitEntranceZoneID": "1", "Zone": "North"}],
+ ),
+ ],
+ )
+ def test_nested_tables_explode_the_parent_list_arrays(
+ self,
+ endpoint: str,
+ expected_path: str,
+ parent_rows: list[dict[str, Any]],
+ expected_rows: list[dict[str, Any]],
+ ) -> None:
+ result = self._drive(endpoint, [_ridb_response(parent_rows, total_count=len(parent_rows), offset=0)])
+
+ assert result.rows == expected_rows
+ assert result.requests[0].path == expected_path
+
+ def test_reservations_walk_pages_to_the_first_empty_page_and_checkpoint_the_page(self) -> None:
+ responses: list[ScriptedResponse | BaseException] = [
+ _reservations_response(0, PAGE_LIMIT),
+ _reservations_response(PAGE_LIMIT, PAGE_LIMIT),
+ _reservations_response(2 * PAGE_LIMIT, 0),
+ ]
+ result = self._drive("Reservations", responses)
+
+ assert len(result.rows) == 2 * PAGE_LIMIT
+ assert result.params("page") == ["0", "1", "2"]
+ assert result.params("limit") == [str(PAGE_LIMIT)] * 3
+ assert result.saved_states[0] == RecreationResumeConfig(
+ page=1, date_from=result.params("dateFrom")[0], date_to=result.params("dateTo")[0]
+ )
+
+ def test_reservations_resume_from_the_saved_page_and_date_window(self) -> None:
+ resume_state = RecreationResumeConfig(page=7, date_from="2025-01-01", date_to="2027-01-01")
+ result = self._drive(
+ "Reservations",
+ [_reservations_response(0, 0)],
+ resume_state=resume_state,
+ incremental_field="OrderDate",
+ db_incremental_field_last_value="2026-03-04",
+ )
+
+ assert result.params("page") == ["7"]
+ assert result.params("dateFrom") == ["2025-01-01"]
+ assert result.params("dateTo") == ["2027-01-01"]
+
+ @pytest.mark.parametrize(
+ ("inputs", "expected_from"),
+ [
+ ({}, None),
+ ({"incremental_field": "OrderDate"}, None),
+ ({"incremental_field": "OrderDate", "db_incremental_field_last_value": "2026-03-04"}, date(2026, 3, 4)),
+ (
+ {"incremental_field": "OrderDate", "db_incremental_field_last_value": datetime(2026, 3, 4, 15, 0)},
+ date(2026, 3, 4),
+ ),
+ (
+ {
+ "incremental_field": "OrderDate",
+ "should_use_incremental_field": False,
+ "db_incremental_field_last_value": "2026-03-04",
+ },
+ None,
+ ),
+ ],
+ )
+ def test_reservations_date_range(self, inputs: dict[str, Any], expected_from: date | None) -> None:
+ result = self._drive("Reservations", [_reservations_response(0, 0)], **inputs)
+
+ today = datetime.now(UTC).date()
+ if expected_from is None:
+ expected_from = today - timedelta(days=RESERVATIONS_LOOKBACK_DAYS)
+ assert result.params("dateFrom") == [expected_from.isoformat()]
+ assert result.params("dateTo") == [(today + timedelta(days=RESERVATIONS_FORWARD_DAYS)).isoformat()]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation_source.py
index abefeb4bd5c1..42a672e4321b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recreation/tests/test_recreation_source.py
@@ -51,7 +51,10 @@ class TestRecreationSource:
assert response is not None
endpoint_config = RECREATION_ENDPOINTS[endpoint]
assert response.name == endpoint
- assert response.primary_keys == [endpoint_config.primary_key]
+ if endpoint_config.nested is not None:
+ assert response.primary_keys == [endpoint_config.nested.parent_id_field, endpoint_config.primary_key]
+ else:
+ assert response.primary_keys == [endpoint_config.primary_key]
if endpoint_config.partition_key is not None:
assert response.partition_mode == "datetime"
assert response.partition_keys == [endpoint_config.partition_key]