Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 2 missing tables to the Recruitee source (#124697)

Tom Owers067d04cc10d3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 992b2aa781e5..17637ba93525 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10117,9 +10117,9 @@ Today (4): `candidates`, `departments`, `offers`, `placements`
Diffed against: <https://docs.recruitee.com/reference>
-- [ ] `GET /locations` — Lookup table resolving the location IDs carried on offers we already sync (high)
+- [x] `GET /locations` — Lookup table resolving the location IDs carried on offers we already sync (high)
- [ ] `GET /timeline` — Hiring activity/event stream - the state-transition history behind candidates and placements (high)
-- [ ] `GET /candidates/{id}/notes` — Recruiter notes per candidate, the qualitative record attached to candidates already synced (medium)
+- [x] `GET /candidates/{id}/notes` — Recruiter notes per candidate, the qualitative record attached to candidates already synced (medium)
- [ ] `POST /analytics (job campaign analytics)` — Sourcing and job-campaign performance breakdowns - the vendor's own reporting dimensions (medium)
- [ ] `GET /candidates/{id}/reports` — Assessment and screening report results attached to candidates (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.py
index 2be3d3e3c49a..6c0456eb112f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/canonical_descriptions.py
@@ -54,4 +54,46 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "When the placement was last updated.",
},
},
+ "locations": {
+ "description": "A physical location that job offers in Recruitee are assigned to.",
+ "docs_url": "https://docs.recruitee.com/reference/locations",
+ "columns": {
+ "id": "The unique ID of the location, used by job offers to refer to it.",
+ "name": "The name of the location.",
+ "country_code": "The ISO 3166 country code of the location.",
+ "state_name": "The name of the state or region.",
+ "state_code": "The short code of the state or region.",
+ "city": "The city of the location.",
+ "street": "The street of the location.",
+ "postal_code": "The postal code of the location.",
+ "full_address": "The one-line address: country, state, city, postal code, and street.",
+ "created_at": "When the location was created.",
+ "updated_at": "When the location was last edited.",
+ "archived_at": "When the location was archived, or null if it is active.",
+ "lang_code": "The default language code of the location.",
+ "active_offers_count": "The number of offers the location is assigned to.",
+ "active_requisitions_count": "The number of requisitions the location is assigned to.",
+ "is_deletable": "Whether the location can be deleted. It cannot while offers still use it.",
+ "translations": "The translated fields of the location, one object for each language.",
+ },
+ },
+ "candidate_notes": {
+ "description": "A note that a recruiter or team member wrote on a candidate's profile.",
+ "docs_url": "https://docs.recruitee.com/reference/candidatesidnotes",
+ "columns": {
+ "id": "The unique ID of the note.",
+ "candidate_id": "The ID of the candidate the note belongs to.",
+ "admin_id": "The ID of the team member who wrote the note.",
+ "offer_id": "The ID of the offer the note relates to, if any.",
+ "body": "The content of the note.",
+ "body_html": "The content of the note as HTML.",
+ "visibility": "Who can see the note: a level plus any allowed team member and role IDs.",
+ "replies": "The replies to the note.",
+ "reply_to_id": "The ID of the note this note replies to, if any.",
+ "attachments": "The files attached to the note.",
+ "pinned_at": "When the note was pinned, or null if it is not pinned.",
+ "created_at": "When the note was created.",
+ "updated_at": "When the note was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.py
index 93fe700075f6..ceb424662142 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/recruitee.py
@@ -7,17 +7,25 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
OffsetPaginator,
+ PageNumberPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.recruitee.settings import RECRUITEE_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.recruitee.settings import (
+ PAGE_SIZE,
+ RECRUITEE_ENDPOINTS,
+ RecruiteeEndpointConfig,
+)
RECRUITEE_HOST = "https://api.recruitee.com"
-# The list endpoints accept a `limit`; the largest page minimises round trips.
-PAGE_SIZE = 100
# Cheap probe used to confirm credentials. Departments is the smallest company-level list, and the
# token is company-wide, so one probe validates access to every list endpoint.
DEFAULT_PROBE_PATH = "/departments"
@@ -26,11 +34,13 @@ DEFAULT_PROBE_PATH = "/departments"
COMPANY_ID_REGEX = re.compile(r"^[a-zA-Z0-9_-]+$")
[email protected](frozen=True)
class RecruiteeResumeConfig:
# Offset of the next page to fetch. Offset pagination is deterministic, so a crashed
# full-refresh sync resumes from the page after the last one yielded; merge dedupes on `id`.
offset: int = 0
+ # Next page number, for the endpoints that paginate by `page` instead of `offset`.
+ page: Optional[int] = None
def base_url(company_id: str) -> str:
@@ -45,6 +55,68 @@ def _headers() -> dict[str, str]:
return {"Accept": "application/json"}
+def _client_config(company_id: str, api_token: str) -> ClientConfig:
+ return {
+ "base_url": base_url(company_id),
+ "headers": _headers(),
+ "auth": {"type": "bearer", "token": api_token},
+ # The token rides only to api.recruitee.com/c/<company_id>; pin every request (including
+ # paginator follow-ups) to that host so a redirect or crafted next-page can't exfiltrate it.
+ "allowed_hosts": [],
+ # No top-level `total`; termination is a short/empty page (OffsetPaginator default).
+ "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),
+ }
+
+
+def _paginator(config: RecruiteeEndpointConfig) -> BasePaginator:
+ if config.pagination == "page":
+ # `meta.total_count` counts records, not pages, so the paginator stops on an empty page.
+ return PageNumberPaginator(base_page=1)
+ return OffsetPaginator(limit=PAGE_SIZE, total_path=None)
+
+
+def _candidate_notes_source(
+ company_id: str,
+ api_token: str,
+ config: RecruiteeEndpointConfig,
+ team_id: int,
+ job_id: str,
+) -> SourceResponse:
+ assert config.fanout is not None
+ resource = build_dependent_resource(
+ endpoint_configs=RECRUITEE_ENDPOINTS,
+ child_endpoint=config.name,
+ fanout=config.fanout,
+ client_config=_client_config(company_id, api_token),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={
+ "data_selector": RECRUITEE_ENDPOINTS[config.fanout.parent_name].data_key,
+ "data_selector_malformed_retryable": True,
+ },
+ child_endpoint_extra={
+ # The notes of one candidate come back in a single unpaginated response.
+ "paginator": "single_page",
+ "data_selector": config.data_key,
+ "data_selector_malformed_retryable": True,
+ },
+ # The client paginator sends `limit` on the candidates listing; the notes call takes none.
+ page_size_param=None,
+ )
+
+ return SourceResponse(
+ name=config.name,
+ items=lambda: resource,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ # No checkpoints are saved, so a shutdown handoff would repeat every candidate's notes call.
+ supports_resume=False,
+ )
+
+
def recruitee_source(
company_id: str,
api_token: str,
@@ -56,17 +128,12 @@ def recruitee_source(
) -> SourceResponse:
config = RECRUITEE_ENDPOINTS[endpoint]
+ if config.fanout is not None:
+ # The fan-out re-lists the candidates each run, so it keeps no resume state.
+ return _candidate_notes_source(company_id, api_token, config, team_id, job_id)
+
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": base_url(company_id),
- "headers": _headers(),
- "auth": {"type": "bearer", "token": api_token},
- # The token rides only to api.recruitee.com/c/<company_id>; pin every request (including
- # paginator follow-ups) to that host so a redirect or crafted next-page can't exfiltrate it.
- "allowed_hosts": [],
- # No top-level `total`; termination is a short/empty page (OffsetPaginator default).
- "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),
- },
+ "client": _client_config(company_id, api_token),
# Per-resource settings are fully specified below, so no shared defaults are needed.
"resource_defaults": {},
"resources": [
@@ -74,7 +141,8 @@ def recruitee_source(
"name": endpoint,
"endpoint": {
"path": config.path,
- "params": {"limit": PAGE_SIZE},
+ "params": {"limit": PAGE_SIZE, **config.params},
+ "paginator": _paginator(config),
# Recruitee wraps records under a key named after the resource, e.g.
# {"candidates": [...]}. A 200 whose body isn't that list shape is treated as a
# transient blip and retried, matching the old defensive retryable classification.
@@ -89,12 +157,19 @@ def recruitee_source(
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None:
+ # Each paginator reads only its own key, so both are passed.
initial_paginator_state = {"offset": resume.offset}
+ if resume.page is not None:
+ initial_paginator_state["page"] = resume.page
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields the
# last page (merge dedupes on the primary key) rather than skipping it.
- if state and state.get("offset") is not None:
+ if not state:
+ return
+ if state.get("page") is not None:
+ resumable_source_manager.save_state(RecruiteeResumeConfig(page=int(state["page"])))
+ elif state.get("offset") is not None:
resumable_source_manager.save_state(RecruiteeResumeConfig(offset=int(state["offset"])))
resource = rest_api_resource(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.py
index 583b4510c162..404beb85d57e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/settings.py
@@ -1,9 +1,16 @@
from dataclasses import dataclass, field
+from typing import Literal, Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField
+# The list endpoints accept a `limit`; the largest page minimises round trips.
+PAGE_SIZE = 100
-@dataclass
+
+@dataclass(frozen=True)
class RecruiteeEndpointConfig:
name: str
# Path relative to https://api.recruitee.com/c/<company_id> (leading slash included).
@@ -12,6 +19,18 @@ class RecruiteeEndpointConfig:
data_key: str
# Recruitee object IDs are unique within a company, so `id` is a safe primary key.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ pagination: Literal["offset", "page"] = "offset"
+ params: dict[str, str] = field(default_factory=dict)
+ fanout: Optional[DependentEndpointConfig] = None
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+
+ @property
+ def default_incremental_field(self) -> Optional[str]:
+ return None
+
+ @property
+ def page_size(self) -> int:
+ return PAGE_SIZE
# Recruitee company-level list endpoints. All are full refresh only: Recruitee exposes no
@@ -22,6 +41,32 @@ RECRUITEE_ENDPOINTS: dict[str, RecruiteeEndpointConfig] = {
"offers": RecruiteeEndpointConfig(name="offers", path="/offers", data_key="offers"),
"departments": RecruiteeEndpointConfig(name="departments", path="/departments", data_key="departments"),
"placements": RecruiteeEndpointConfig(name="placements", path="/placements", data_key="placements"),
+ "locations": RecruiteeEndpointConfig(
+ name="locations",
+ path="/locations",
+ data_key="locations",
+ # /locations documents `page` + `limit`, not `offset`.
+ pagination="page",
+ # `scope=all` keeps archived locations that older offers still reference; `view_mode=full`
+ # adds the translations and the active offer counters.
+ params={"scope": "all", "view_mode": "full"},
+ ),
+ "candidate_notes": RecruiteeEndpointConfig(
+ name="candidate_notes",
+ path="/candidates/{candidate_id}/notes",
+ data_key="notes",
+ # Notes are fetched per candidate, so the parent candidate ID is part of the key.
+ primary_keys=["candidate_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="candidates",
+ resolve_param="candidate_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "candidate_id"},
+ # A candidate deleted between the candidates listing and its notes fetch returns 404.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(RECRUITEE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.py
index 47651f0aacff..02e891fe90a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/source.py
@@ -61,7 +61,7 @@ class RecruiteeSource(ResumableSource[RecruiteeSourceConfig, RecruiteeResumeConf
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your Recruitee company ID and API token to pull your recruiting data into the PostHog Data warehouse.
-Find your company ID and create a personal API token under **Settings → Apps and plugins → Personal API tokens** in [Recruitee](https://app.recruitee.com/). The token grants read access to your candidates, offers, departments, and placements. All tables sync via full refresh.
+Find your company ID and create a personal API token under **Settings → Apps and plugins → Personal API tokens** in [Recruitee](https://app.recruitee.com/). The token grants read access to your candidates, candidate notes, offers, departments, locations, and placements. All tables sync via full refresh.
""",
iconPath="/static/services/recruitee.png",
docsUrl="https://posthog.com/docs/cdp/sources/recruitee",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.py b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.py
index 0512005be454..0e7810ac50b9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/recruitee/tests/test_recruitee.py
@@ -13,6 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.recruitee import (
@@ -57,11 +58,42 @@ class TestPagination:
# Checkpoint saved once after the first full page (points at the next page); short page ends it.
assert result.saved_states == [RecruiteeResumeConfig(offset=PAGE_SIZE)]
- def test_resumes_from_saved_offset(self) -> None:
- result = _source(script=[_response([{"id": 5}])], resume_state=RecruiteeResumeConfig(offset=200))
-
- # Offset 0 must never be fetched on resume — the first request starts at the saved offset.
- assert result.params("offset") == ["200"]
+ @parameterized.expand(
+ [
+ ("offset", "candidates", RecruiteeResumeConfig(offset=200), [_response([{"id": 5}])], "200"),
+ (
+ "page",
+ "locations",
+ RecruiteeResumeConfig(page=3),
+ [_response([{"id": 5}], data_key="locations"), _response([], data_key="locations")],
+ "3",
+ ),
+ ]
+ )
+ def test_resumes_from_saved_position(
+ self, param: str, endpoint: str, state: RecruiteeResumeConfig, script: Script, expected_first: str
+ ) -> None:
+ result = _source(endpoint=endpoint, script=script, resume_state=state)
+
+ # The first page must never be fetched on resume — the first request starts at the saved position.
+ assert result.params(param)[0] == expected_first
+
+ def test_locations_paginate_by_page_until_an_empty_page(self) -> None:
+ result = _source(
+ endpoint="locations",
+ script=[
+ _response([{"id": 1}, {"id": 2}], data_key="locations"),
+ _response([{"id": 3}], data_key="locations"),
+ _response([], data_key="locations"),
+ ],
+ )
+
+ assert [r["id"] for r in result.rows] == [1, 2, 3]
+ assert result.params("page") == ["1", "2", "3"]
+ assert result.params("offset") == [None, None, None]
+ assert result.params("scope") == ["all"] * 3
+ assert result.params("view_mode") == ["full"] * 3
+ assert result.saved_states == [RecruiteeResumeConfig(page=2), RecruiteeResumeConfig(page=3)]
def test_a_run_with_no_saved_state_starts_from_the_first_page(self) -> None:
result = _source(script=[_response([{"id": 5}])])
@@ -93,6 +125,38 @@ class TestDataExtraction:
assert len(result.requests) > 1
+class TestCandidateNotes:
+ def test_fans_out_over_candidates_and_skips_a_deleted_candidate(self) -> None:
+ result = _source(
+ endpoint="candidate_notes",
+ script=route(
+ {
+ "/c/acme/candidates": [_response([{"id": 11}, {"id": 12}, {"id": 13}])],
+ "/c/acme/candidates/11/notes": [
+ _response([{"id": 1, "body": "first"}, {"id": 2, "body": "second"}], data_key="notes")
+ ],
+ "/c/acme/candidates/12/notes": [ScriptedResponse(status=404, json={})],
+ "/c/acme/candidates/13/notes": [_response([], data_key="notes")],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert [(r["candidate_id"], r["id"], r["body"]) for r in result.rows] == [
+ (11, 1, "first"),
+ (11, 2, "second"),
+ ]
+ assert all("_candidates_id" not in r for r in result.rows)
+ assert result.paths == [
+ "/c/acme/candidates",
+ "/c/acme/candidates/11/notes",
+ "/c/acme/candidates/12/notes",
+ "/c/acme/candidates/13/notes",
+ ]
+ assert result.params("limit") == [str(PAGE_SIZE), None, None, None]
+ assert result.params("offset") == ["0", None, None, None]
+
+
class TestValidateCredentials:
@parameterized.expand(
[