:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 2 missing tables to the Recruitee source (#124697)
Tom Owers067d04cc10d3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 992b2aa781e5..17637ba93525 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10117,9 +10117,9 @@ Today (4): `candidates`, `departments`, `offers`, `placements`Diffed against: <https://docs.recruitee.com/reference>-- [ ] `GET /locations` — Lookup table resolving the location IDs carried on offers we already sync (high)+- [x] `GET /locations` — Lookup table resolving the location IDs carried on offers we already sync (high)- [ ] `GET /timeline` — Hiring activity/event stream - the state-transition history behind candidates and placements (high)-- [ ] `GET /candidates/{id}/notes` — Recruiter notes per candidate, the qualitative record attached to candidates already synced (medium)+- [x] `GET /candidates/{id}/notes` — Recruiter notes per candidate, the qualitative record attached to candidates already synced (medium)- [ ] `POST /analytics (job campaign analytics)` — Sourcing and job-campaign performance breakdowns - the vendor's own reporting dimensions (medium)- [ ] `GET /candidates/{id}/reports` — Assessment and screening report results attached to candidates (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.pyindex 2be3d3e3c49a..6c0456eb112f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.py@@ -54,4 +54,46 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "When the placement was last updated.",},},+ "locations": {+ "description": "A physical location that job offers in Recruitee are assigned to.",+ "docs_url": "https://docs.recruitee.com/reference/locations",+ "columns": {+ "id": "The unique ID of the location, used by job offers to refer to it.",+ "name": "The name of the location.",+ "country_code": "The ISO 3166 country code of the location.",+ "state_name": "The name of the state or region.",+ "state_code": "The short code of the state or region.",+ "city": "The city of the location.",+ "street": "The street of the location.",+ "postal_code": "The postal code of the location.",+ "full_address": "The one-line address: country, state, city, postal code, and street.",+ "created_at": "When the location was created.",+ "updated_at": "When the location was last edited.",+ "archived_at": "When the location was archived, or null if it is active.",+ "lang_code": "The default language code of the location.",+ "active_offers_count": "The number of offers the location is assigned to.",+ "active_requisitions_count": "The number of requisitions the location is assigned to.",+ "is_deletable": "Whether the location can be deleted. It cannot while offers still use it.",+ "translations": "The translated fields of the location, one object for each language.",+ },+ },+ "candidate_notes": {+ "description": "A note that a recruiter or team member wrote on a candidate's profile.",+ "docs_url": "https://docs.recruitee.com/reference/candidatesidnotes",+ "columns": {+ "id": "The unique ID of the note.",+ "candidate_id": "The ID of the candidate the note belongs to.",+ "admin_id": "The ID of the team member who wrote the note.",+ "offer_id": "The ID of the offer the note relates to, if any.",+ "body": "The content of the note.",+ "body_html": "The content of the note as HTML.",+ "visibility": "Who can see the note: a level plus any allowed team member and role IDs.",+ "replies": "The replies to the note.",+ "reply_to_id": "The ID of the note this note replies to, if any.",+ "attachments": "The files attached to the note.",+ "pinned_at": "When the note was pinned, or null if it is not pinned.",+ "created_at": "When the note was created.",+ "updated_at": "When the note was last updated.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.pyindex 93fe700075f6..ceb424662142 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.py@@ -7,17 +7,25 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ BasePaginator,OffsetPaginator,+ PageNumberPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-from products.warehouse_sources.backend.temporal.data_imports.sources.recruitee.settings import RECRUITEE_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.recruitee.settings import (+ PAGE_SIZE,+ RECRUITEE_ENDPOINTS,+ RecruiteeEndpointConfig,+)RECRUITEE_HOST = "https://api.recruitee.com"-# The list endpoints accept a `limit`; the largest page minimises round trips.-PAGE_SIZE = 100# Cheap probe used to confirm credentials. Departments is the smallest company-level list, and the# token is company-wide, so one probe validates access to every list endpoint.DEFAULT_PROBE_PATH = "/departments"@@ -26,11 +34,13 @@ DEFAULT_PROBE_PATH = "/departments"COMPANY_ID_REGEX = re.compile(r"^[a-zA-Z0-9_-]+$")[email protected](frozen=True)class RecruiteeResumeConfig:# Offset of the next page to fetch. Offset pagination is deterministic, so a crashed# full-refresh sync resumes from the page after the last one yielded; merge dedupes on `id`.offset: int = 0+ # Next page number, for the endpoints that paginate by `page` instead of `offset`.+ page: Optional[int] = Nonedef base_url(company_id: str) -> str:@@ -45,6 +55,68 @@ def _headers() -> dict[str, str]:return {"Accept": "application/json"}+def _client_config(company_id: str, api_token: str) -> ClientConfig:+ return {+ "base_url": base_url(company_id),+ "headers": _headers(),+ "auth": {"type": "bearer", "token": api_token},+ # The token rides only to api.recruitee.com/c/<company_id>; pin every request (including+ # paginator follow-ups) to that host so a redirect or crafted next-page can't exfiltrate it.+ "allowed_hosts": [],+ # No top-level `total`; termination is a short/empty page (OffsetPaginator default).+ "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),+ }+++def _paginator(config: RecruiteeEndpointConfig) -> BasePaginator:+ if config.pagination == "page":+ # `meta.total_count` counts records, not pages, so the paginator stops on an empty page.+ return PageNumberPaginator(base_page=1)+ return OffsetPaginator(limit=PAGE_SIZE, total_path=None)+++def _candidate_notes_source(+ company_id: str,+ api_token: str,+ config: RecruiteeEndpointConfig,+ team_id: int,+ job_id: str,+) -> SourceResponse:+ assert config.fanout is not None+ resource = build_dependent_resource(+ endpoint_configs=RECRUITEE_ENDPOINTS,+ child_endpoint=config.name,+ fanout=config.fanout,+ client_config=_client_config(company_id, api_token),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ parent_endpoint_extra={+ "data_selector": RECRUITEE_ENDPOINTS[config.fanout.parent_name].data_key,+ "data_selector_malformed_retryable": True,+ },+ child_endpoint_extra={+ # The notes of one candidate come back in a single unpaginated response.+ "paginator": "single_page",+ "data_selector": config.data_key,+ "data_selector_malformed_retryable": True,+ },+ # The client paginator sends `limit` on the candidates listing; the notes call takes none.+ page_size_param=None,+ )++ return SourceResponse(+ name=config.name,+ items=lambda: resource,+ primary_keys=config.primary_keys,+ partition_count=1,+ partition_size=1,+ # No checkpoints are saved, so a shutdown handoff would repeat every candidate's notes call.+ supports_resume=False,+ )++def recruitee_source(company_id: str,api_token: str,@@ -56,17 +128,12 @@ def recruitee_source() -> SourceResponse:config = RECRUITEE_ENDPOINTS[endpoint]+ if config.fanout is not None:+ # The fan-out re-lists the candidates each run, so it keeps no resume state.+ return _candidate_notes_source(company_id, api_token, config, team_id, job_id)+rest_config: RESTAPIConfig = {- "client": {- "base_url": base_url(company_id),- "headers": _headers(),- "auth": {"type": "bearer", "token": api_token},- # The token rides only to api.recruitee.com/c/<company_id>; pin every request (including- # paginator follow-ups) to that host so a redirect or crafted next-page can't exfiltrate it.- "allowed_hosts": [],- # No top-level `total`; termination is a short/empty page (OffsetPaginator default).- "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),- },+ "client": _client_config(company_id, api_token),# Per-resource settings are fully specified below, so no shared defaults are needed."resource_defaults": {},"resources": [@@ -74,7 +141,8 @@ def recruitee_source("name": endpoint,"endpoint": {"path": config.path,- "params": {"limit": PAGE_SIZE},+ "params": {"limit": PAGE_SIZE, **config.params},+ "paginator": _paginator(config),# Recruitee wraps records under a key named after the resource, e.g.# {"candidates": [...]}. A 200 whose body isn't that list shape is treated as a# transient blip and retried, matching the old defensive retryable classification.@@ -89,12 +157,19 @@ def recruitee_source(if resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()if resume is not None:+ # Each paginator reads only its own key, so both are passed.initial_paginator_state = {"offset": resume.offset}+ if resume.page is not None:+ initial_paginator_state["page"] = resume.pagedef save_checkpoint(state: Optional[dict[str, Any]]) -> None:# Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields the# last page (merge dedupes on the primary key) rather than skipping it.- if state and state.get("offset") is not None:+ if not state:+ return+ if state.get("page") is not None:+ resumable_source_manager.save_state(RecruiteeResumeConfig(page=int(state["page"])))+ elif state.get("offset") is not None:resumable_source_manager.save_state(RecruiteeResumeConfig(offset=int(state["offset"])))resource = rest_api_resource(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.pyindex 583b4510c162..404beb85d57e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.py@@ -1,9 +1,16 @@from dataclasses import dataclass, field+from typing import Literal, Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField+# The list endpoints accept a `limit`; the largest page minimises round trips.+PAGE_SIZE = 100-@dataclass++@dataclass(frozen=True)class RecruiteeEndpointConfig:name: str# Path relative to https://api.recruitee.com/c/<company_id> (leading slash included).@@ -12,6 +19,18 @@ class RecruiteeEndpointConfig:data_key: str# Recruitee object IDs are unique within a company, so `id` is a safe primary key.primary_keys: list[str] = field(default_factory=lambda: ["id"])+ pagination: Literal["offset", "page"] = "offset"+ params: dict[str, str] = field(default_factory=dict)+ fanout: Optional[DependentEndpointConfig] = None+ incremental_fields: list[IncrementalField] = field(default_factory=list)++ @property+ def default_incremental_field(self) -> Optional[str]:+ return None++ @property+ def page_size(self) -> int:+ return PAGE_SIZE# Recruitee company-level list endpoints. All are full refresh only: Recruitee exposes no@@ -22,6 +41,32 @@ RECRUITEE_ENDPOINTS: dict[str, RecruiteeEndpointConfig] = {"offers": RecruiteeEndpointConfig(name="offers", path="/offers", data_key="offers"),"departments": RecruiteeEndpointConfig(name="departments", path="/departments", data_key="departments"),"placements": RecruiteeEndpointConfig(name="placements", path="/placements", data_key="placements"),+ "locations": RecruiteeEndpointConfig(+ name="locations",+ path="/locations",+ data_key="locations",+ # /locations documents `page` + `limit`, not `offset`.+ pagination="page",+ # `scope=all` keeps archived locations that older offers still reference; `view_mode=full`+ # adds the translations and the active offer counters.+ params={"scope": "all", "view_mode": "full"},+ ),+ "candidate_notes": RecruiteeEndpointConfig(+ name="candidate_notes",+ path="/candidates/{candidate_id}/notes",+ data_key="notes",+ # Notes are fetched per candidate, so the parent candidate ID is part of the key.+ primary_keys=["candidate_id", "id"],+ fanout=DependentEndpointConfig(+ parent_name="candidates",+ resolve_param="candidate_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "candidate_id"},+ # A candidate deleted between the candidates listing and its notes fetch returns 404.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),}ENDPOINTS = tuple(RECRUITEE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.pyindex 47651f0aacff..02e891fe90a6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.py@@ -61,7 +61,7 @@ class RecruiteeSource(ResumableSource[RecruiteeSourceConfig, RecruiteeResumeConfreleaseStatus=ReleaseStatus.BETA,caption="""Enter your Recruitee company ID and API token to pull your recruiting data into the PostHog Data warehouse.-Find your company ID and create a personal API token under **Settings → Apps and plugins → Personal API tokens** in [Recruitee](https://app.recruitee.com/). The token grants read access to your candidates, offers, departments, and placements. All tables sync via full refresh.+Find your company ID and create a personal API token under **Settings → Apps and plugins → Personal API tokens** in [Recruitee](https://app.recruitee.com/). The token grants read access to your candidates, candidate notes, offers, departments, locations, and placements. All tables sync via full refresh.""",iconPath="/static/services/recruitee.png",docsUrl="https://posthog.com/docs/cdp/sources/recruitee",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.pyindex 0512005be454..0e7810ac50b9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.py@@ -13,6 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.recruitee import (@@ -57,11 +58,42 @@ class TestPagination:# Checkpoint saved once after the first full page (points at the next page); short page ends it.assert result.saved_states == [RecruiteeResumeConfig(offset=PAGE_SIZE)]- def test_resumes_from_saved_offset(self) -> None:- result = _source(script=[_response([{"id": 5}])], resume_state=RecruiteeResumeConfig(offset=200))-- # Offset 0 must never be fetched on resume — the first request starts at the saved offset.- assert result.params("offset") == ["200"]+ @parameterized.expand(+ [+ ("offset", "candidates", RecruiteeResumeConfig(offset=200), [_response([{"id": 5}])], "200"),+ (+ "page",+ "locations",+ RecruiteeResumeConfig(page=3),+ [_response([{"id": 5}], data_key="locations"), _response([], data_key="locations")],+ "3",+ ),+ ]+ )+ def test_resumes_from_saved_position(+ self, param: str, endpoint: str, state: RecruiteeResumeConfig, script: Script, expected_first: str+ ) -> None:+ result = _source(endpoint=endpoint, script=script, resume_state=state)++ # The first page must never be fetched on resume — the first request starts at the saved position.+ assert result.params(param)[0] == expected_first++ def test_locations_paginate_by_page_until_an_empty_page(self) -> None:+ result = _source(+ endpoint="locations",+ script=[+ _response([{"id": 1}, {"id": 2}], data_key="locations"),+ _response([{"id": 3}], data_key="locations"),+ _response([], data_key="locations"),+ ],+ )++ assert [r["id"] for r in result.rows] == [1, 2, 3]+ assert result.params("page") == ["1", "2", "3"]+ assert result.params("offset") == [None, None, None]+ assert result.params("scope") == ["all"] * 3+ assert result.params("view_mode") == ["full"] * 3+ assert result.saved_states == [RecruiteeResumeConfig(page=2), RecruiteeResumeConfig(page=3)]def test_a_run_with_no_saved_state_starts_from_the_first_page(self) -> None:result = _source(script=[_response([{"id": 5}])])@@ -93,6 +125,38 @@ class TestDataExtraction:assert len(result.requests) > 1+class TestCandidateNotes:+ def test_fans_out_over_candidates_and_skips_a_deleted_candidate(self) -> None:+ result = _source(+ endpoint="candidate_notes",+ script=route(+ {+ "/c/acme/candidates": [_response([{"id": 11}, {"id": 12}, {"id": 13}])],+ "/c/acme/candidates/11/notes": [+ _response([{"id": 1, "body": "first"}, {"id": 2, "body": "second"}], data_key="notes")+ ],+ "/c/acme/candidates/12/notes": [ScriptedResponse(status=404, json={})],+ "/c/acme/candidates/13/notes": [_response([], data_key="notes")],+ }+ ),+ )++ assert result.raised is None+ assert [(r["candidate_id"], r["id"], r["body"]) for r in result.rows] == [+ (11, 1, "first"),+ (11, 2, "second"),+ ]+ assert all("_candidates_id" not in r for r in result.rows)+ assert result.paths == [+ "/c/acme/candidates",+ "/c/acme/candidates/11/notes",+ "/c/acme/candidates/12/notes",+ "/c/acme/candidates/13/notes",+ ]+ assert result.params("limit") == [str(PAGE_SIZE), None, None, None]+ assert result.params("offset") == ["0", None, None, None]++class TestValidateCredentials:@parameterized.expand([