Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Housecall Pro source (#122040)

Tom Owers36f2b8fe5599
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 08506093820c..c9ca10b0f705 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6050,14 +6050,14 @@ Note: Fetched the OpenAPI spec at https://api.hoorayhr.io/swagger.json (linked f
## Housecall Pro — gaps
-Today (6): `customers`, `employees`, `estimates`, `invoices`, `jobs`, `leads`
+Today (10): `customers`, `employees`, `estimates`, `invoices`, `job_appointments`, `job_line_items`, `job_types`, `jobs`, `leads`, `pipeline_statuses`
Diffed against: <https://stoplight.io/api/v1/projects/housecallpro/housecall-public-api/nodes/reference/housecall.v1.yaml?fromExportButton=true&snapshotType=http_service>
-- [ ] `Job Types (GET /job_fields/job_types)` — resolve job_type_uuid on leads and group service demand by job type (high)
-- [ ] `Job Appointments (GET /jobs/{job_id}/appointments)` — analyze appointment schedules, assigned technicians, and repeat visits for each job (high)
-- [ ] `Job line items (GET /jobs/{job_id}/line_items)` — break job revenue into services, materials, quantities, and unit prices (high)
-- [ ] `Pipeline Statuses (GET /pipeline/statuses)` — map lead pipeline statuses to names and status types for funnel analysis (medium)
+- [x] `Job Types (GET /job_fields/job_types)` — resolve job_type_uuid on leads and group service demand by job type (high)
+- [x] `Job Appointments (GET /jobs/{job_id}/appointments)` — analyze appointment schedules, assigned technicians, and repeat visits for each job (high)
+- [x] `Job line items (GET /jobs/{job_id}/line_items)` — break job revenue into services, materials, quantities, and unit prices (high)
+- [x] `Pipeline Statuses (GET /pipeline/statuses)` — map lead pipeline statuses to names and status types for funnel analysis (medium)
- [ ] `Lead Sources (GET /lead_sources)` — compare the available acquisition sources with source names on customers, leads, jobs, and estimates (medium)
- [ ] `Estimate option line items (GET /estimates/{estimate_id}/options/{option_id}/line_items)` — compare quoted services and prices across estimate options and completed jobs (medium)
- [ ] `Job input materials (GET /jobs/{job_id}/job_input_materials)` — measure materials consumed on jobs and compare usage with billed items (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.py
index bbc856cb7bb9..de40856ef5fd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.py
@@ -107,4 +107,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "When the lead was last updated.",
},
},
+ "job_types": {
+ "description": "Job types defined in the Housecall Pro account, used to categorize jobs and leads.",
+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",
+ "columns": {
+ "id": "Unique identifier for the job type.",
+ "name": "Display name of the job type.",
+ },
+ },
+ "pipeline_statuses": {
+ "description": "Pipeline statuses configured for leads, jobs, and estimates. Empty when the account has no pipeline enabled.",
+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",
+ "columns": {
+ "id": "Unique identifier of the pipeline status.",
+ "name": "Display name of the status.",
+ "status_type": "Type of the pipeline status.",
+ "resource_type": "Resource the status applies to: lead, job, or estimate.",
+ },
+ },
+ "job_appointments": {
+ "description": "Scheduled appointments (visits) for each job.",
+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",
+ "columns": {
+ "id": "Unique identifier for the appointment.",
+ "job_id": "Identifier of the job the appointment belongs to.",
+ "start_date": "Start date of the appointment.",
+ "start_time": "Start time of the appointment.",
+ "end_time": "End time of the appointment.",
+ "anytime": "Whether the appointment can happen at any time of day.",
+ "arrival_window_minutes": "Length of the arrival window, in minutes.",
+ "dispatched_employees_ids": "Identifiers of the employees dispatched to the appointment.",
+ },
+ },
+ "job_line_items": {
+ "description": "Services, materials, and discounts billed on each job.",
+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",
+ "columns": {
+ "id": "Unique identifier for the line item.",
+ "job_id": "Identifier of the job the line item belongs to.",
+ "name": "Name of the line item.",
+ "description": "Description of the line item.",
+ "unit_price": "Price per unit.",
+ "unit_cost": "Cost per unit.",
+ "unit_of_measure": "Unit the quantity is measured in.",
+ "quantity": "Number of units.",
+ "kind": "Kind of line item: materials, labor, fixed gratuity, fixed discount, or percent discount.",
+ "taxable": "Whether the line item is taxable.",
+ "amount": "Total amount of the line item.",
+ "order_index": "Position of the line item on the job.",
+ "service_item_id": "Identifier of the price book or marketplace item the line item comes from.",
+ "service_item_type": "Source of the service item: market_place, organizational, or pricebook_material.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.py
index 2f812dc12a2a..8d5c86023127 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.py
@@ -1,5 +1,5 @@
import dataclasses
-from collections.abc import Iterator
+from collections.abc import Callable, Iterable, Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional, cast
@@ -9,23 +9,24 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
PageNumberPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.housecall_pro.settings import (
HOUSECALL_PRO_ENDPOINTS,
+ PAGE_SIZE,
)
HOUSECALL_PRO_BASE_URL = "https://api.housecallpro.com"
-# The vendor's own default is 10; raising it cuts round trips against an undisclosed rate limit.
-# Pagination termination reads `total_pages` from the body, so a smaller honored page size still
-# walks every page correctly.
-PAGE_SIZE = 100
-
# Hard cap so a paginator that never sees `total_pages` drop below the current page can't loop forever.
MAX_PAGES = 100_000
@@ -33,7 +34,10 @@ MAX_PAGES = 100_000
@dataclasses.dataclass(frozen=False) # resume state is rebuilt wholesale each save, not mutated in place
class HousecallProResumeConfig:
# Next 1-indexed page to fetch.
- page: int
+ page: Optional[int] = None
+ # Per-job tables checkpoint the shape `build_dependent_resource` emits: which jobs finished
+ # and where the current one stopped.
+ fanout_state: Optional[dict[str, Any]] = None
def _format_created_at_min(value: Any) -> str | None:
@@ -58,6 +62,25 @@ def validate_credentials(api_key: str) -> bool:
return ok
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": HOUSECALL_PRO_BASE_URL,
+ "headers": {"Accept": "application/json"},
+ "auth": {"type": "bearer", "token": api_key},
+ }
+
+
+def _page_number_paginator() -> PageNumberPaginator:
+ # `total_pages` in the body is the number of PAGES, so pagination stops after the last page
+ # without paying an extra empty-page request.
+ return PageNumberPaginator(
+ base_page=1,
+ page_param="page",
+ total_path="total_pages",
+ maximum_page=MAX_PAGES,
+ )
+
+
def get_rows(
api_key: str,
endpoint: str,
@@ -70,7 +93,9 @@ def get_rows(
config = HOUSECALL_PRO_ENDPOINTS[endpoint]
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- initial_paginator_state: Optional[dict[str, Any]] = {"page": resume.page} if resume is not None else None
+ initial_paginator_state: Optional[dict[str, Any]] = (
+ {"page": resume.page} if resume is not None and resume.page is not None else None
+ )
params: dict[str, Any] = {"page_size": PAGE_SIZE}
if config.sort_field:
@@ -84,14 +109,7 @@ def get_rows(
"params": params,
# A 200 body without the response key reads as an empty page and ends pagination.
"data_selector": config.response_key,
- # `total_pages` in the body is the number of PAGES, so pagination stops after the last page
- # without paying an extra empty-page request.
- "paginator": PageNumberPaginator(
- base_page=1,
- page_param="page",
- total_path="total_pages",
- maximum_page=MAX_PAGES,
- ),
+ "paginator": _page_number_paginator(),
}
if config.supports_incremental and config.incremental_param and should_use_incremental_field:
endpoint_config["incremental"] = {
@@ -101,11 +119,7 @@ def get_rows(
}
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": HOUSECALL_PRO_BASE_URL,
- "headers": {"Accept": "application/json"},
- "auth": {"type": "bearer", "token": api_key},
- },
+ "client": _client_config(api_key),
"resources": [{"name": endpoint, "endpoint": endpoint_config}],
}
@@ -126,6 +140,71 @@ def get_rows(
)
+def get_job_child_rows(
+ api_key: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[HousecallProResumeConfig],
+) -> Iterable[Any]:
+ config = HOUSECALL_PRO_ENDPOINTS[endpoint]
+ fanout = cast("DependentEndpointConfig", config.fanout)
+ parent_config = HOUSECALL_PRO_ENDPOINTS[fanout.parent_name]
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(HousecallProResumeConfig(fanout_state=state))
+
+ return build_dependent_resource(
+ endpoint_configs=HOUSECALL_PRO_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=fanout,
+ client_config=_client_config(api_key),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ # Page size goes in `fanout.parent_params`: the per-job endpoints take no paging params.
+ page_size_param=None,
+ parent_endpoint_extra={"data_selector": parent_config.response_key, "paginator": _page_number_paginator()},
+ child_endpoint_extra={"data_selector": config.response_key, "paginator": "single_page"},
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume is not None else None,
+ )
+
+
+def get_rows_per_resource_type(
+ api_key: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[HousecallProResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ config = HOUSECALL_PRO_ENDPOINTS[endpoint]
+
+ for resource_type in config.resource_types:
+ rest_config: RESTAPIConfig = {
+ "client": _client_config(api_key),
+ "resources": [
+ {
+ "name": endpoint,
+ "endpoint": {
+ "path": config.path,
+ "params": {"page_size": PAGE_SIZE, "resource_type": resource_type},
+ "data_selector": config.response_key,
+ "paginator": _page_number_paginator(),
+ },
+ }
+ ],
+ }
+ for page in rest_api_resource(rest_config, team_id, job_id, None):
+ yield [{**row, "resource_type": resource_type} for row in page]
+ # An account without pipelines enabled returns no rows for any resource type.
+ resumable_source_manager.safe_point()
+
+
def housecall_pro_source(
api_key: str,
endpoint: str,
@@ -137,9 +216,13 @@ def housecall_pro_source(
) -> SourceResponse:
config = HOUSECALL_PRO_ENDPOINTS[endpoint]
- return SourceResponse(
- name=endpoint,
- items=lambda: get_rows(
+ items: Callable[[], Iterable[Any]]
+ if config.fanout is not None:
+ items = lambda: get_job_child_rows(api_key, endpoint, team_id, job_id, resumable_source_manager)
+ elif config.resource_types:
+ items = lambda: get_rows_per_resource_type(api_key, endpoint, team_id, job_id, resumable_source_manager)
+ else:
+ items = lambda: get_rows(
api_key=api_key,
endpoint=endpoint,
team_id=team_id,
@@ -147,7 +230,11 @@ def housecall_pro_source(
resumable_source_manager=resumable_source_manager,
should_use_incremental_field=should_use_incremental_field,
db_incremental_field_last_value=db_incremental_field_last_value,
- ),
+ )
+
+ return SourceResponse(
+ name=endpoint,
+ items=items,
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.py
index 173a3b5427ca..ef30f4efa623 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.py
@@ -1,8 +1,16 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+# The vendor's own default is 10; raising it cuts round trips against an undisclosed rate limit.
+# Pagination termination reads `total_pages` from the body, so a smaller honored page size still
+# walks every page correctly.
+PAGE_SIZE = 100
+
@dataclass(frozen=False) # static endpoint catalog, never mutated after construction
class HousecallProEndpointConfig:
@@ -24,10 +32,18 @@ class HousecallProEndpointConfig:
supports_incremental: bool = False
primary_keys: list[str] = field(default_factory=lambda: ["id"])
should_sync_default: bool = True
+ page_size: int = PAGE_SIZE
+ default_incremental_field: Optional[str] = None
+ # Set for resources only listed per job; the jobs list is re-read each sync.
+ fanout: Optional[DependentEndpointConfig] = None
+ # Values of the required `resource_type` query param. The table reads each value in turn and
+ # stores it on the row as `resource_type`.
+ resource_types: tuple[str, ...] = ()
# Housecall Pro REST API (https://api.housecallpro.com). Every list endpoint supports page-number
-# pagination (`page`, `page_size`) and reports `total_pages` in the body.
+# pagination (`page`, `page_size`) and reports `total_pages` in the body. The per-job endpoints
+# (appointments, line items) are the exception: they return the whole list in one body.
#
# Only Invoices documents a genuine server-side timestamp filter (`created_at_min`), so it's the
# only endpoint marked incremental. The others (Customers, Jobs, Estimates, Leads) accept
@@ -87,6 +103,54 @@ HOUSECALL_PRO_ENDPOINTS: dict[str, HousecallProEndpointConfig] = {
partition_key="created_at",
incremental_fields=[],
),
+ "job_types": HousecallProEndpointConfig(
+ name="job_types",
+ path="/job_fields/job_types",
+ response_key="job_types",
+ incremental_fields=[],
+ ),
+ "pipeline_statuses": HousecallProEndpointConfig(
+ name="pipeline_statuses",
+ path="/pipeline/statuses",
+ response_key="statuses",
+ incremental_fields=[],
+ resource_types=("lead", "job", "estimate"),
+ # The docs do not say that status ids are unique across resource types.
+ primary_keys=["resource_type", "id"],
+ ),
+ "job_appointments": HousecallProEndpointConfig(
+ name="job_appointments",
+ path="/jobs/{job_id}/appointments",
+ response_key="appointments",
+ incremental_fields=[],
+ primary_keys=["job_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="jobs",
+ resolve_param="job_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "job_id"},
+ parent_params={"page_size": PAGE_SIZE},
+ # A job deleted between the jobs listing and its child fetch must not fail the sync.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "job_line_items": HousecallProEndpointConfig(
+ name="job_line_items",
+ path="/jobs/{job_id}/line_items",
+ response_key="data",
+ incremental_fields=[],
+ primary_keys=["job_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="jobs",
+ resolve_param="job_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "job_id"},
+ parent_params={"page_size": PAGE_SIZE},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(HOUSECALL_PRO_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.py
index dafb5cc80a21..5466b4c96659 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.py
@@ -3,10 +3,13 @@ from typing import Any
import pytest
+from parameterized import parameterized
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.housecallpro import (
@@ -90,12 +93,54 @@ class TestGetRows:
assert result.raised is None
assert result.requests[0].param("created_at_min") is None
+ @parameterized.expand([("job_appointments", "appointments"), ("job_line_items", "data")])
+ def test_job_children_walk_every_jobs_page_and_carry_the_job_id(self, endpoint: str, response_key: str) -> None:
+ child_path = HOUSECALL_PRO_ENDPOINTS[endpoint].path
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ "/jobs": [
+ _page("jobs", [{"id": "job_1"}, {"id": "job_2"}], total_pages=2),
+ _page("jobs", [{"id": "job_3"}], total_pages=2),
+ ],
+ child_path.format(job_id="job_1"): [ScriptedResponse(json={response_key: [{"id": "a"}]})],
+ child_path.format(job_id="job_2"): [ScriptedResponse(status=404)],
+ child_path.format(job_id="job_3"): [ScriptedResponse(json={response_key: [{"id": "a"}]})],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert [(row["job_id"], row["id"]) for row in result.rows] == [("job_1", "a"), ("job_3", "a")]
+ assert result.params("page") == ["1", None, None, "2", None]
+ assert result.params("page_size") == ["100", None, None, "100", None]
+ assert result.saved_states
+ assert all(state.page is None and state.fanout_state for state in result.saved_states)
+
+ def test_pipeline_statuses_read_each_resource_type(self) -> None:
+ result = _driver().run(
+ "pipeline_statuses",
+ [
+ _page("statuses", [{"id": "s1", "name": "New", "status_type": "open"}], total_pages=1),
+ _page("statuses", [], total_pages=0),
+ _page("statuses", [{"id": "s1", "name": "Won", "status_type": "closed"}], total_pages=1),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.params("resource_type") == ["lead", "job", "estimate"]
+ assert [(row["resource_type"], row["id"], row["name"]) for row in result.rows] == [
+ ("lead", "s1", "New"),
+ ("estimate", "s1", "Won"),
+ ]
+
class TestHousecallProSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = HOUSECALL_PRO_ENDPOINTS[endpoint]
- result = _driver().run(endpoint, [_page(config.response_key, [], total_pages=1)])
+ result = _driver().run(endpoint, always(_page(config.response_key, [], total_pages=1)))
response = result.response
assert result.raised is None