:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(sage-hr): add 3 missing tables to the Sage HR source (#129121)
Tom Owers74dc0ca661bb
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex f9f994c60c76..35abe8c08985 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10544,10 +10544,10 @@ Today (12): `document_categories`, `documents`, `employees`, `individual_allowanDiffed against: <https://jsapi.apiary.io/apis/sagehr.apib>-- [ ] `timesheets/workdays (GET /timesheets/workdays/)` — actual hours worked per employee per day — the core time-tracking fact table (high)-- [ ] `employees/{id}/compensations` — salary/compensation history per employee, needed for any pay or cost analysis (high)-- [ ] `employees/{id}/leave-management/balances` — remaining leave per employee per policy; leave_requests alone can't answer balance questions (high)-- [ ] `recruitment/positions/{id}/applicants` — hiring funnel — applicants per open role (high)+- [ ] `timesheets/workdays (GET /timesheets/workdays/)` — actual hours worked per employee per day — the core time-tracking fact table (high) — skipped for now: the spec documents no success response for the GET, and spells its required employee filter `employe_id`+- [x] `employees/{id}/compensations` — salary/compensation history per employee, needed for any pay or cost analysis (high)+- [x] `employees/{id}/leave-management/balances` — remaining leave per employee per policy; leave_requests alone can't answer balance questions (high)+- [x] `recruitment/positions/{id}/applicants` — hiring funnel — applicants per open role (high)- [ ] `recruitment/positions` — open job requisitions; also the lookup that resolves position ids on applicants (medium)- [ ] `timesheets/projects` — lookup resolving the project ids carried on workdays (medium)- [ ] `timesheets/working_packages` — lookup resolving work package ids on timesheet entries (medium)diff --git a/products/warehouse_sources/sources/sage_hr/canonical_descriptions.py b/products/warehouse_sources/sources/sage_hr/canonical_descriptions.pyindex 8d692409de53..e41eec1f09e8 100644--- a/products/warehouse_sources/sources/sage_hr/canonical_descriptions.py+++ b/products/warehouse_sources/sources/sage_hr/canonical_descriptions.py@@ -152,4 +152,48 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"title": "The category title.",},},+ "compensations": {+ "description": "An entry in an active employee's compensation history, such as a salary or a bonus.",+ "docs_url": "https://sagehr.docs.apiary.io",+ "columns": {+ "id": "Identifier PostHog derives from the row content, because Sage HR returns none.",+ "employee_id": "The ID of the employee the compensation belongs to.",+ "start_date": "The date the compensation starts.",+ "end_date": "The date the compensation ends, or null when it is current.",+ "currency": "Three-letter currency code of the amount.",+ "amount": "The compensation amount in `currency`.",+ "period": "How often the amount is paid (e.g. monthly).",+ "comment": "A free-text note about the compensation.",+ "category": "The compensation category (e.g. Salary).",+ },+ },+ "leave_balances": {+ "description": "An active employee's used and remaining time off under each leave policy.",+ "docs_url": "https://sagehr.docs.apiary.io",+ "columns": {+ "employee_id": "The ID of the employee the balance belongs to.",+ "policy_id": "The ID of the leave policy.",+ "used": "The time off the employee has used under the policy.",+ "available": "The time off the employee has left under the policy, or null when it is unlimited.",+ },+ },+ "recruitment_applicants": {+ "description": "An applicant for a recruitment position, with their current hiring stage and scores.",+ "docs_url": "https://sagehr.docs.apiary.io",+ "columns": {+ "id": "The ID of the applicant.",+ "position_id": "The ID of the recruitment position the applicant applied for.",+ "full_name": "The applicant's full name.",+ "first_name": "The applicant's first name.",+ "last_name": "The applicant's last name.",+ "email": "The applicant's email address.",+ "source": "Where the applicant came from (e.g. recruiters, referral).",+ "created_at": "When the applicant was added.",+ "disqualified_date": "The date the applicant was disqualified, or null.",+ "hired_date": "The date the applicant was hired, or null.",+ "stage": "The applicant's current hiring stage (id, title, created_at).",+ "tags": "Tags on the applicant.",+ "scores": "Scores that reviewers gave the applicant.",+ },+ },}diff --git a/products/warehouse_sources/sources/sage_hr/sage_hr.py b/products/warehouse_sources/sources/sage_hr/sage_hr.pyindex eb9674b90198..64ae3e66084c 100644--- a/products/warehouse_sources/sources/sage_hr/sage_hr.py+++ b/products/warehouse_sources/sources/sage_hr/sage_hr.py@@ -1,9 +1,12 @@import re+import json+import hashlibimport dataclasses+from collections import Counterfrom collections.abc import Iteratorfrom datetime import UTC, date, datetime, timedeltafrom typing import Any, Optional-from urllib.parse import urlencode+from urllib.parse import quote, urlencodeimport requestsfrom structlog.types import FilteringBoundLogger@@ -43,12 +46,18 @@ class SageHRRetryableError(Exception):pass[email protected](frozen=True)class SageHRResumeConfig:# Next 1-indexed page to fetch (from the response `meta.next_page`). None means "start over".next_page: int | None = None# For the windowed leave_requests endpoint: ISO date of the window `next_page` belongs to.window_from: str | None = None+ # For fan-out endpoints: the parent page, and the index and id on it of the parent `next_page` belongs to.+ parent_page: int | None = None+ parent_index: int | None = None+ parent_id: Any = None+ # For endpoints with derived row ids: how often each row hash already appeared under `parent_id`.+ row_occurrences: dict[str, int] | None = Nonedef normalize_subdomain(subdomain: str) -> str:@@ -204,6 +213,116 @@ def _get_windowed_rows(page = next_page+def _derive_row_id(parent_id: Any, row: dict[str, Any], occurrences: Counter[str]) -> str:+ payload = json.dumps({"parent": parent_id, "row": row}, sort_keys=True, default=str)+ row_hash = hashlib.sha256(payload.encode()).hexdigest()[:32]+ # Identical rows under one parent would share a hash, so each repeat adds its ordinal to stay unique.+ occurrence = occurrences[row_hash]+ occurrences[row_hash] += 1+ if not occurrence:+ return row_hash+ return hashlib.sha256(f"{row_hash}#{occurrence}".encode()).hexdigest()[:32]+++def _shape_child_row(+ config: SageHREndpointConfig, parent_key: str, parent_id: Any, row: dict[str, Any], occurrences: Counter[str]+) -> dict[str, Any]:+ shaped = {**row, parent_key: parent_id}+ if config.derive_row_id:+ shaped["id"] = _derive_row_id(parent_id, row, occurrences)+ return shaped+++def _get_fan_out_rows(+ session: requests.Session,+ base_url: str,+ config: SageHREndpointConfig,+ resume: SageHRResumeConfig | None,+ resumable_source_manager: ResumableSourceManager[SageHRResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ fan_out = config.fan_out+ if fan_out is None:+ raise ValueError(f"Sage HR endpoint {config.name} has no fan-out parent")++ parent_page = resume.parent_page if resume is not None and resume.parent_page is not None else 1+ resume_index = resume.parent_index if resume is not None and resume.parent_index is not None else 0+ resume_child_page = resume.next_page if resume is not None and resume.next_page is not None else 1+ is_resumed_page = resume is not None++ while True:+ parents, next_parent_page = _fetch_page(+ session, _build_url(base_url, fan_out.parent_path, {"page": parent_page}), logger+ )+ has_next_parent_page = bool(next_parent_page and parents and next_parent_page > parent_page)+ first_index, first_child_page = 0, 1+ if is_resumed_page and resume is not None:+ # The parent list can shift between attempts, so the saved child page applies only to the+ # parent with the saved id. A parent that is gone restarts at its saved index from page 1.+ first_index = resume_index+ for candidate_index, parent in enumerate(parents):+ if resume.parent_id is not None and parent.get("id") == resume.parent_id:+ first_index, first_child_page = candidate_index, resume_child_page+ break++ for index in range(first_index, len(parents)):+ parent_id = parents[index]["id"]+ child_path = config.path.format(id=quote(str(parent_id), safe=""))+ child_page = first_child_page if index == first_index else 1+ occurrences: Counter[str] = Counter()+ if is_resumed_page and resume is not None and child_page > 1 and index == first_index:+ occurrences.update(resume.row_occurrences or {})++ while True:+ params = {"page": child_page} if config.paginated else {}+ items, next_child_page = _fetch_page(session, _build_url(base_url, child_path, params), logger)+ has_next_child_page = config.paginated and bool(+ next_child_page and items and next_child_page > child_page+ )++ # Shape before staging the cursor, so the saved occurrences cover this page's rows.+ rows = [_shape_child_row(config, fan_out.parent_key, parent_id, row, occurrences) for row in items]++ if has_next_child_page:+ resumable_source_manager.save_state(+ SageHRResumeConfig(+ next_page=next_child_page,+ parent_page=parent_page,+ parent_index=index,+ parent_id=parent_id,+ row_occurrences=dict(occurrences) if config.derive_row_id else None,+ )+ )+ elif index + 1 < len(parents):+ resumable_source_manager.save_state(+ SageHRResumeConfig(+ next_page=1,+ parent_page=parent_page,+ parent_index=index + 1,+ parent_id=parents[index + 1]["id"],+ )+ )+ elif has_next_parent_page:+ resumable_source_manager.save_state(+ SageHRResumeConfig(next_page=1, parent_page=next_parent_page, parent_index=0)+ )++ if rows:+ yield rows+ else:+ # Many parents have no children, so the walk needs a stop point that does not depend on rows.+ resumable_source_manager.safe_point()++ if not has_next_child_page or next_child_page is None:+ break+ child_page = next_child_page++ is_resumed_page = False+ if not has_next_parent_page or next_parent_page is None:+ break+ parent_page = next_parent_page++def get_rows(subdomain: str,api_key: str,@@ -219,6 +338,10 @@ def get_rows(if resume is not None:logger.debug(f"Sage HR: resuming {endpoint} from page={resume.next_page}, window={resume.window_from}")+ if config.fan_out is not None:+ yield from _get_fan_out_rows(session, base_url, config, resume, resumable_source_manager, logger)+ return+if config.requires_date_window:yield from _get_windowed_rows(session, base_url, config, resume, resumable_source_manager, logger)returndiff --git a/products/warehouse_sources/sources/sage_hr/settings.py b/products/warehouse_sources/sources/sage_hr/settings.pyindex cfb335ce8d82..3c532962ab1d 100644--- a/products/warehouse_sources/sources/sage_hr/settings.py+++ b/products/warehouse_sources/sources/sage_hr/settings.py@@ -1,7 +1,15 @@from dataclasses import dataclass, field-@dataclass+@dataclass(frozen=True)+class SageHRFanOut:+ # Paginated parent collection whose rows' `id` fills the `{id}` placeholder of the child path.+ parent_path: str+ # Column added to every child row so rows from different parents stay distinct.+ parent_key: str+++@dataclass(frozen=True)class SageHREndpointConfig:name: strpath: str@@ -13,6 +21,13 @@ class SageHREndpointConfig:requires_date_window: bool = False# Sage HR resource identifiers are account-unique integers, so `id` is a safe primary key.primary_keys: list[str] = field(default_factory=lambda: ["id"])+ fan_out: SageHRFanOut | None = None+ # Compensation rows carry no identifier, so the source derives `id` from the row content.+ derive_row_id: bool = False+++_PER_EMPLOYEE = SageHRFanOut(parent_path="/employees", parent_key="employee_id")+_PER_RECRUITMENT_POSITION = SageHRFanOut(parent_path="/recruitment/positions", parent_key="position_id")# Sage HR API collection endpoints (https://sagehr.docs.apiary.io). All are full-refresh only:@@ -38,6 +53,27 @@ SAGE_HR_ENDPOINTS: dict[str, SageHREndpointConfig] = {),"onboarding_categories": SageHREndpointConfig(name="onboarding_categories", path="/onboarding/categories"),"offboarding_categories": SageHREndpointConfig(name="offboarding_categories", path="/offboarding/categories"),+ "compensations": SageHREndpointConfig(+ name="compensations",+ path="/employees/{id}/compensations",+ primary_keys=["employee_id", "id"],+ fan_out=_PER_EMPLOYEE,+ derive_row_id=True,+ ),+ "leave_balances": SageHREndpointConfig(+ name="leave_balances",+ path="/employees/{id}/leave-management/balances",+ paginated=False,+ primary_keys=["employee_id", "policy_id"],+ fan_out=_PER_EMPLOYEE,+ ),+ # The docs don't say whether applicant ids are unique across positions, so key on both.+ "recruitment_applicants": SageHREndpointConfig(+ name="recruitment_applicants",+ path="/recruitment/positions/{id}/applicants",+ primary_keys=["position_id", "id"],+ fan_out=_PER_RECRUITMENT_POSITION,+ ),}ENDPOINTS = tuple(SAGE_HR_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.pyindex 17013c74c8b0..a362ebde5f91 100644--- a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py+++ b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py@@ -216,6 +216,133 @@ class TestWindowedLeaveRequests:assert rows == [{"id": 2}]+class TestFanOutEndpoints:+ @staticmethod+ def _collect(+ manager: _FakeResumableManager,+ monkeypatch: Any,+ pages: Mapping[str, tuple[list[dict], int | None]],+ endpoint: str,+ requested: list[str] | None = None,+ ) -> list[dict]:+ def fake_fetch(session: Any, url: str, logger: Any) -> tuple[list[dict], int | None]:+ parsed = urlparse(url)+ key = parsed.path.removeprefix("/api") + ("?" + parsed.query if parsed.query else "")+ if requested is not None:+ requested.append(key)+ return pages[key]++ monkeypatch.setattr(sage_hr, "_fetch_page", fake_fetch)+ monkeypatch.setattr(sage_hr, "make_tracked_session", lambda **kwargs: MagicMock())++ rows: list[dict] = []+ for batch in get_rows(+ subdomain="acme",+ api_key="sage-key",+ endpoint=endpoint,+ logger=MagicMock(),+ resumable_source_manager=manager, # type: ignore[arg-type]+ ):+ rows.extend(batch)+ return rows++ def test_walks_parent_and_child_pages_and_tags_rows_with_parent(self, monkeypatch: Any) -> None:+ manager = _FakeResumableManager()+ pages = {+ "/recruitment/positions?page=1": ([{"id": 10}, {"id": 11}], 2),+ "/recruitment/positions?page=2": ([{"id": 12}], None),+ "/recruitment/positions/10/applicants?page=1": ([{"id": 1}], 2),+ "/recruitment/positions/10/applicants?page=2": ([{"id": 2}], None),+ "/recruitment/positions/11/applicants?page=1": ([], None),+ "/recruitment/positions/12/applicants?page=1": ([{"id": 1}], None),+ }+ rows = self._collect(manager, monkeypatch, pages, "recruitment_applicants")+ assert rows == [+ {"id": 1, "position_id": 10},+ {"id": 2, "position_id": 10},+ {"id": 1, "position_id": 12},+ ]+ assert [(s.parent_page, s.parent_index, s.parent_id, s.next_page) for s in manager.saved] == [+ (1, 0, 10, 2),+ (1, 1, 11, 1),+ (2, 0, None, 1),+ ]++ def test_unpaginated_child_is_fetched_once_per_parent_without_page_param(self, monkeypatch: Any) -> None:+ manager = _FakeResumableManager()+ requested: list[str] = []+ pages: dict[str, tuple[list[dict], int | None]] = {+ "/employees?page=1": ([{"id": 5}, {"id": 6}], None),+ "/employees/5/leave-management/balances": ([{"policy_id": 1, "used": 2, "available": 3}], None),+ "/employees/6/leave-management/balances": ([{"policy_id": 1, "used": 0, "available": None}], None),+ }+ rows = self._collect(manager, monkeypatch, pages, "leave_balances", requested)+ assert rows == [+ {"policy_id": 1, "used": 2, "available": 3, "employee_id": 5},+ {"policy_id": 1, "used": 0, "available": None, "employee_id": 6},+ ]+ assert requested == list(pages)++ @pytest.mark.parametrize(+ "parent_index, parent_id, first_child_key",+ [+ pytest.param(1, 21, "/recruitment/positions/21/applicants?page=3", id="saved_parent_at_saved_index"),+ pytest.param(2, 21, "/recruitment/positions/21/applicants?page=3", id="saved_parent_moved"),+ pytest.param(1, 99, "/recruitment/positions/21/applicants?page=1", id="saved_parent_gone"),+ ],+ )+ def test_resumes_from_saved_parent_and_child_page(+ self, parent_index: int, parent_id: int, first_child_key: str, monkeypatch: Any+ ) -> None:+ manager = _FakeResumableManager(+ SageHRResumeConfig(next_page=3, parent_page=2, parent_index=parent_index, parent_id=parent_id)+ )+ # Earlier parent pages, earlier parents on the page and earlier child pages must never be fetched.+ pages: dict[str, tuple[list[dict], int | None]] = {+ "/recruitment/positions?page=2": ([{"id": 20}, {"id": 21}, {"id": 22}], None),+ first_child_key: ([{"id": 7}], None),+ "/recruitment/positions/22/applicants?page=1": ([{"id": 8}], None),+ }+ rows = self._collect(manager, monkeypatch, pages, "recruitment_applicants")+ assert rows == [{"id": 7, "position_id": 21}, {"id": 8, "position_id": 22}]++ def test_derived_compensation_ids_are_stable_and_distinct(self, monkeypatch: Any) -> None:+ salary: dict[str, Any] = {+ "start_date": "2024-01-01",+ "end_date": None,+ "amount": 1000,+ "currency": "EUR",+ "category": "Salary",+ }+ bonus = {**salary, "amount": 200, "category": "Bonus"}+ pages: dict[str, tuple[list[dict], int | None]] = {+ "/employees?page=1": ([{"id": 5}, {"id": 6}], None),+ "/employees/5/compensations?page=1": ([salary, bonus, dict(bonus)], None),+ "/employees/6/compensations?page=1": ([salary], None),+ }+ first = self._collect(_FakeResumableManager(), monkeypatch, pages, "compensations")+ second = self._collect(_FakeResumableManager(), monkeypatch, pages, "compensations")++ assert [row["employee_id"] for row in first] == [5, 5, 5, 6]+ assert len({row["id"] for row in first}) == 4+ assert [row["id"] for row in first] == [row["id"] for row in second]++ def test_resumed_compensation_ids_match_an_uninterrupted_walk(self, monkeypatch: Any) -> None:+ salary: dict[str, Any] = {"start_date": "2024-01-01", "amount": 1000, "category": "Salary"}+ pages: dict[str, tuple[list[dict], int | None]] = {+ "/employees?page=1": ([{"id": 5}], None),+ "/employees/5/compensations?page=1": ([salary], 2),+ "/employees/5/compensations?page=2": ([dict(salary)], None),+ }+ uninterrupted = _FakeResumableManager()+ full_rows = self._collect(uninterrupted, monkeypatch, pages, "compensations")++ resumed_rows = self._collect(_FakeResumableManager(uninterrupted.saved[0]), monkeypatch, pages, "compensations")++ assert len({row["id"] for row in full_rows}) == 2+ assert [row["id"] for row in resumed_rows] == [full_rows[1]["id"]]++class TestCheckAccess:def _patch_session(self, monkeypatch: Any, response: Any) -> MagicMock:session = MagicMock()@@ -296,10 +423,6 @@ class TestSageHRSourceResponse:resumable_source_manager=MagicMock(),)assert response.name == endpoint- assert response.primary_keys == ["id"]+ assert response.primary_keys == SAGE_HR_ENDPOINTS[endpoint].primary_keys# No stable creation timestamp is guaranteed across every object, so we don't partition.assert response.partition_mode is None-- def test_every_endpoint_uses_id_primary_key(self) -> None:- assert all(config.primary_keys == ["id"] for config in SAGE_HR_ENDPOINTS.values())- assert set(SAGE_HR_ENDPOINTS) == set(ENDPOINTS)