:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Housecall Pro source (#122040)
Tom Owers36f2b8fe5599
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 08506093820c..c9ca10b0f705 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6050,14 +6050,14 @@ Note: Fetched the OpenAPI spec at https://api.hoorayhr.io/swagger.json (linked f## Housecall Pro — gaps-Today (6): `customers`, `employees`, `estimates`, `invoices`, `jobs`, `leads`+Today (10): `customers`, `employees`, `estimates`, `invoices`, `job_appointments`, `job_line_items`, `job_types`, `jobs`, `leads`, `pipeline_statuses`Diffed against: <https://stoplight.io/api/v1/projects/housecallpro/housecall-public-api/nodes/reference/housecall.v1.yaml?fromExportButton=true&snapshotType=http_service>-- [ ] `Job Types (GET /job_fields/job_types)` — resolve job_type_uuid on leads and group service demand by job type (high)-- [ ] `Job Appointments (GET /jobs/{job_id}/appointments)` — analyze appointment schedules, assigned technicians, and repeat visits for each job (high)-- [ ] `Job line items (GET /jobs/{job_id}/line_items)` — break job revenue into services, materials, quantities, and unit prices (high)-- [ ] `Pipeline Statuses (GET /pipeline/statuses)` — map lead pipeline statuses to names and status types for funnel analysis (medium)+- [x] `Job Types (GET /job_fields/job_types)` — resolve job_type_uuid on leads and group service demand by job type (high)+- [x] `Job Appointments (GET /jobs/{job_id}/appointments)` — analyze appointment schedules, assigned technicians, and repeat visits for each job (high)+- [x] `Job line items (GET /jobs/{job_id}/line_items)` — break job revenue into services, materials, quantities, and unit prices (high)+- [x] `Pipeline Statuses (GET /pipeline/statuses)` — map lead pipeline statuses to names and status types for funnel analysis (medium)- [ ] `Lead Sources (GET /lead_sources)` — compare the available acquisition sources with source names on customers, leads, jobs, and estimates (medium)- [ ] `Estimate option line items (GET /estimates/{estimate_id}/options/{option_id}/line_items)` — compare quoted services and prices across estimate options and completed jobs (medium)- [ ] `Job input materials (GET /jobs/{job_id}/job_input_materials)` — measure materials consumed on jobs and compare usage with billed items (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.pyindex bbc856cb7bb9..de40856ef5fd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/canonical_descriptions.py@@ -107,4 +107,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "When the lead was last updated.",},},+ "job_types": {+ "description": "Job types defined in the Housecall Pro account, used to categorize jobs and leads.",+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",+ "columns": {+ "id": "Unique identifier for the job type.",+ "name": "Display name of the job type.",+ },+ },+ "pipeline_statuses": {+ "description": "Pipeline statuses configured for leads, jobs, and estimates. Empty when the account has no pipeline enabled.",+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",+ "columns": {+ "id": "Unique identifier of the pipeline status.",+ "name": "Display name of the status.",+ "status_type": "Type of the pipeline status.",+ "resource_type": "Resource the status applies to: lead, job, or estimate.",+ },+ },+ "job_appointments": {+ "description": "Scheduled appointments (visits) for each job.",+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",+ "columns": {+ "id": "Unique identifier for the appointment.",+ "job_id": "Identifier of the job the appointment belongs to.",+ "start_date": "Start date of the appointment.",+ "start_time": "Start time of the appointment.",+ "end_time": "End time of the appointment.",+ "anytime": "Whether the appointment can happen at any time of day.",+ "arrival_window_minutes": "Length of the arrival window, in minutes.",+ "dispatched_employees_ids": "Identifiers of the employees dispatched to the appointment.",+ },+ },+ "job_line_items": {+ "description": "Services, materials, and discounts billed on each job.",+ "docs_url": "https://docs.housecallpro.com/docs/housecall-public-api",+ "columns": {+ "id": "Unique identifier for the line item.",+ "job_id": "Identifier of the job the line item belongs to.",+ "name": "Name of the line item.",+ "description": "Description of the line item.",+ "unit_price": "Price per unit.",+ "unit_cost": "Cost per unit.",+ "unit_of_measure": "Unit the quantity is measured in.",+ "quantity": "Number of units.",+ "kind": "Kind of line item: materials, labor, fixed gratuity, fixed discount, or percent discount.",+ "taxable": "Whether the line item is taxable.",+ "amount": "Total amount of the line item.",+ "order_index": "Position of the line item on the job.",+ "service_item_id": "Identifier of the price book or marketplace item the line item comes from.",+ "service_item_type": "Source of the service item: market_place, organizational, or pricebook_material.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.pyindex 2f812dc12a2a..8d5c86023127 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/housecall_pro.py@@ -1,5 +1,5 @@import dataclasses-from collections.abc import Iterator+from collections.abc import Callable, Iterable, Iteratorfrom datetime import UTC, date, datetimefrom typing import Any, Optional, cast@@ -9,23 +9,24 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (PageNumberPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.housecall_pro.settings import (HOUSECALL_PRO_ENDPOINTS,+ PAGE_SIZE,)HOUSECALL_PRO_BASE_URL = "https://api.housecallpro.com"-# The vendor's own default is 10; raising it cuts round trips against an undisclosed rate limit.-# Pagination termination reads `total_pages` from the body, so a smaller honored page size still-# walks every page correctly.-PAGE_SIZE = 100-# Hard cap so a paginator that never sees `total_pages` drop below the current page can't loop forever.MAX_PAGES = 100_000@@ -33,7 +34,10 @@ MAX_PAGES = 100_000@dataclasses.dataclass(frozen=False) # resume state is rebuilt wholesale each save, not mutated in placeclass HousecallProResumeConfig:# Next 1-indexed page to fetch.- page: int+ page: Optional[int] = None+ # Per-job tables checkpoint the shape `build_dependent_resource` emits: which jobs finished+ # and where the current one stopped.+ fanout_state: Optional[dict[str, Any]] = Nonedef _format_created_at_min(value: Any) -> str | None:@@ -58,6 +62,25 @@ def validate_credentials(api_key: str) -> bool:return ok+def _client_config(api_key: str) -> ClientConfig:+ return {+ "base_url": HOUSECALL_PRO_BASE_URL,+ "headers": {"Accept": "application/json"},+ "auth": {"type": "bearer", "token": api_key},+ }+++def _page_number_paginator() -> PageNumberPaginator:+ # `total_pages` in the body is the number of PAGES, so pagination stops after the last page+ # without paying an extra empty-page request.+ return PageNumberPaginator(+ base_page=1,+ page_param="page",+ total_path="total_pages",+ maximum_page=MAX_PAGES,+ )++def get_rows(api_key: str,endpoint: str,@@ -70,7 +93,9 @@ def get_rows(config = HOUSECALL_PRO_ENDPOINTS[endpoint]resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- initial_paginator_state: Optional[dict[str, Any]] = {"page": resume.page} if resume is not None else None+ initial_paginator_state: Optional[dict[str, Any]] = (+ {"page": resume.page} if resume is not None and resume.page is not None else None+ )params: dict[str, Any] = {"page_size": PAGE_SIZE}if config.sort_field:@@ -84,14 +109,7 @@ def get_rows("params": params,# A 200 body without the response key reads as an empty page and ends pagination."data_selector": config.response_key,- # `total_pages` in the body is the number of PAGES, so pagination stops after the last page- # without paying an extra empty-page request.- "paginator": PageNumberPaginator(- base_page=1,- page_param="page",- total_path="total_pages",- maximum_page=MAX_PAGES,- ),+ "paginator": _page_number_paginator(),}if config.supports_incremental and config.incremental_param and should_use_incremental_field:endpoint_config["incremental"] = {@@ -101,11 +119,7 @@ def get_rows(}rest_config: RESTAPIConfig = {- "client": {- "base_url": HOUSECALL_PRO_BASE_URL,- "headers": {"Accept": "application/json"},- "auth": {"type": "bearer", "token": api_key},- },+ "client": _client_config(api_key),"resources": [{"name": endpoint, "endpoint": endpoint_config}],}@@ -126,6 +140,71 @@ def get_rows()+def get_job_child_rows(+ api_key: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[HousecallProResumeConfig],+) -> Iterable[Any]:+ config = HOUSECALL_PRO_ENDPOINTS[endpoint]+ fanout = cast("DependentEndpointConfig", config.fanout)+ parent_config = HOUSECALL_PRO_ENDPOINTS[fanout.parent_name]++ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(HousecallProResumeConfig(fanout_state=state))++ return build_dependent_resource(+ endpoint_configs=HOUSECALL_PRO_ENDPOINTS,+ child_endpoint=endpoint,+ fanout=fanout,+ client_config=_client_config(api_key),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ # Page size goes in `fanout.parent_params`: the per-job endpoints take no paging params.+ page_size_param=None,+ parent_endpoint_extra={"data_selector": parent_config.response_key, "paginator": _page_number_paginator()},+ child_endpoint_extra={"data_selector": config.response_key, "paginator": "single_page"},+ resume_hook=save_fanout_checkpoint,+ initial_paginator_state=resume.fanout_state if resume is not None else None,+ )+++def get_rows_per_resource_type(+ api_key: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[HousecallProResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ config = HOUSECALL_PRO_ENDPOINTS[endpoint]++ for resource_type in config.resource_types:+ rest_config: RESTAPIConfig = {+ "client": _client_config(api_key),+ "resources": [+ {+ "name": endpoint,+ "endpoint": {+ "path": config.path,+ "params": {"page_size": PAGE_SIZE, "resource_type": resource_type},+ "data_selector": config.response_key,+ "paginator": _page_number_paginator(),+ },+ }+ ],+ }+ for page in rest_api_resource(rest_config, team_id, job_id, None):+ yield [{**row, "resource_type": resource_type} for row in page]+ # An account without pipelines enabled returns no rows for any resource type.+ resumable_source_manager.safe_point()++def housecall_pro_source(api_key: str,endpoint: str,@@ -137,9 +216,13 @@ def housecall_pro_source() -> SourceResponse:config = HOUSECALL_PRO_ENDPOINTS[endpoint]- return SourceResponse(- name=endpoint,- items=lambda: get_rows(+ items: Callable[[], Iterable[Any]]+ if config.fanout is not None:+ items = lambda: get_job_child_rows(api_key, endpoint, team_id, job_id, resumable_source_manager)+ elif config.resource_types:+ items = lambda: get_rows_per_resource_type(api_key, endpoint, team_id, job_id, resumable_source_manager)+ else:+ items = lambda: get_rows(api_key=api_key,endpoint=endpoint,team_id=team_id,@@ -147,7 +230,11 @@ def housecall_pro_source(resumable_source_manager=resumable_source_manager,should_use_incremental_field=should_use_incremental_field,db_incremental_field_last_value=db_incremental_field_last_value,- ),+ )++ return SourceResponse(+ name=endpoint,+ items=items,primary_keys=config.primary_keys,partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.pyindex 173a3b5427ca..ef30f4efa623 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/settings.py@@ -1,8 +1,16 @@from dataclasses import dataclass, fieldfrom typing import Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType+# The vendor's own default is 10; raising it cuts round trips against an undisclosed rate limit.+# Pagination termination reads `total_pages` from the body, so a smaller honored page size still+# walks every page correctly.+PAGE_SIZE = 100+@dataclass(frozen=False) # static endpoint catalog, never mutated after constructionclass HousecallProEndpointConfig:@@ -24,10 +32,18 @@ class HousecallProEndpointConfig:supports_incremental: bool = Falseprimary_keys: list[str] = field(default_factory=lambda: ["id"])should_sync_default: bool = True+ page_size: int = PAGE_SIZE+ default_incremental_field: Optional[str] = None+ # Set for resources only listed per job; the jobs list is re-read each sync.+ fanout: Optional[DependentEndpointConfig] = None+ # Values of the required `resource_type` query param. The table reads each value in turn and+ # stores it on the row as `resource_type`.+ resource_types: tuple[str, ...] = ()# Housecall Pro REST API (https://api.housecallpro.com). Every list endpoint supports page-number-# pagination (`page`, `page_size`) and reports `total_pages` in the body.+# pagination (`page`, `page_size`) and reports `total_pages` in the body. The per-job endpoints+# (appointments, line items) are the exception: they return the whole list in one body.## Only Invoices documents a genuine server-side timestamp filter (`created_at_min`), so it's the# only endpoint marked incremental. The others (Customers, Jobs, Estimates, Leads) accept@@ -87,6 +103,54 @@ HOUSECALL_PRO_ENDPOINTS: dict[str, HousecallProEndpointConfig] = {partition_key="created_at",incremental_fields=[],),+ "job_types": HousecallProEndpointConfig(+ name="job_types",+ path="/job_fields/job_types",+ response_key="job_types",+ incremental_fields=[],+ ),+ "pipeline_statuses": HousecallProEndpointConfig(+ name="pipeline_statuses",+ path="/pipeline/statuses",+ response_key="statuses",+ incremental_fields=[],+ resource_types=("lead", "job", "estimate"),+ # The docs do not say that status ids are unique across resource types.+ primary_keys=["resource_type", "id"],+ ),+ "job_appointments": HousecallProEndpointConfig(+ name="job_appointments",+ path="/jobs/{job_id}/appointments",+ response_key="appointments",+ incremental_fields=[],+ primary_keys=["job_id", "id"],+ fanout=DependentEndpointConfig(+ parent_name="jobs",+ resolve_param="job_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "job_id"},+ parent_params={"page_size": PAGE_SIZE},+ # A job deleted between the jobs listing and its child fetch must not fail the sync.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "job_line_items": HousecallProEndpointConfig(+ name="job_line_items",+ path="/jobs/{job_id}/line_items",+ response_key="data",+ incremental_fields=[],+ primary_keys=["job_id", "id"],+ fanout=DependentEndpointConfig(+ parent_name="jobs",+ resolve_param="job_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "job_id"},+ parent_params={"page_size": PAGE_SIZE},+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),}ENDPOINTS = tuple(HOUSECALL_PRO_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.py b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.pyindex dafb5cc80a21..5466b4c96659 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/housecall_pro/tests/test_housecall_pro.py@@ -3,10 +3,13 @@ from typing import Anyimport pytest+from parameterized import parameterized+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (ScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.housecallpro import (@@ -90,12 +93,54 @@ class TestGetRows:assert result.raised is Noneassert result.requests[0].param("created_at_min") is None+ @parameterized.expand([("job_appointments", "appointments"), ("job_line_items", "data")])+ def test_job_children_walk_every_jobs_page_and_carry_the_job_id(self, endpoint: str, response_key: str) -> None:+ child_path = HOUSECALL_PRO_ENDPOINTS[endpoint].path+ result = _driver().run(+ endpoint,+ route(+ {+ "/jobs": [+ _page("jobs", [{"id": "job_1"}, {"id": "job_2"}], total_pages=2),+ _page("jobs", [{"id": "job_3"}], total_pages=2),+ ],+ child_path.format(job_id="job_1"): [ScriptedResponse(json={response_key: [{"id": "a"}]})],+ child_path.format(job_id="job_2"): [ScriptedResponse(status=404)],+ child_path.format(job_id="job_3"): [ScriptedResponse(json={response_key: [{"id": "a"}]})],+ }+ ),+ )++ assert result.raised is None+ assert [(row["job_id"], row["id"]) for row in result.rows] == [("job_1", "a"), ("job_3", "a")]+ assert result.params("page") == ["1", None, None, "2", None]+ assert result.params("page_size") == ["100", None, None, "100", None]+ assert result.saved_states+ assert all(state.page is None and state.fanout_state for state in result.saved_states)++ def test_pipeline_statuses_read_each_resource_type(self) -> None:+ result = _driver().run(+ "pipeline_statuses",+ [+ _page("statuses", [{"id": "s1", "name": "New", "status_type": "open"}], total_pages=1),+ _page("statuses", [], total_pages=0),+ _page("statuses", [{"id": "s1", "name": "Won", "status_type": "closed"}], total_pages=1),+ ],+ )++ assert result.raised is None+ assert result.params("resource_type") == ["lead", "job", "estimate"]+ assert [(row["resource_type"], row["id"], row["name"]) for row in result.rows] == [+ ("lead", "s1", "New"),+ ("estimate", "s1", "Won"),+ ]+class TestHousecallProSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = HOUSECALL_PRO_ENDPOINTS[endpoint]- result = _driver().run(endpoint, [_page(config.response_key, [], total_pages=1)])+ result = _driver().run(endpoint, always(_page(config.response_key, [], total_pages=1)))response = result.responseassert result.raised is None