Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(sage-hr): add 3 missing tables to the Sage HR source (#129121)

Tom Owers74dc0ca661bb
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index f9f994c60c76..35abe8c08985 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10544,10 +10544,10 @@ Today (12): `document_categories`, `documents`, `employees`, `individual_allowan
Diffed against: <https://jsapi.apiary.io/apis/sagehr.apib>
-- [ ] `timesheets/workdays (GET /timesheets/workdays/)` — actual hours worked per employee per day — the core time-tracking fact table (high)
-- [ ] `employees/{id}/compensations` — salary/compensation history per employee, needed for any pay or cost analysis (high)
-- [ ] `employees/{id}/leave-management/balances` — remaining leave per employee per policy; leave_requests alone can't answer balance questions (high)
-- [ ] `recruitment/positions/{id}/applicants` — hiring funnel — applicants per open role (high)
+- [ ] `timesheets/workdays (GET /timesheets/workdays/)` — actual hours worked per employee per day — the core time-tracking fact table (high) — skipped for now: the spec documents no success response for the GET, and spells its required employee filter `employe_id`
+- [x] `employees/{id}/compensations` — salary/compensation history per employee, needed for any pay or cost analysis (high)
+- [x] `employees/{id}/leave-management/balances` — remaining leave per employee per policy; leave_requests alone can't answer balance questions (high)
+- [x] `recruitment/positions/{id}/applicants` — hiring funnel — applicants per open role (high)
- [ ] `recruitment/positions` — open job requisitions; also the lookup that resolves position ids on applicants (medium)
- [ ] `timesheets/projects` — lookup resolving the project ids carried on workdays (medium)
- [ ] `timesheets/working_packages` — lookup resolving work package ids on timesheet entries (medium)
diff --git a/products/warehouse_sources/sources/sage_hr/canonical_descriptions.py b/products/warehouse_sources/sources/sage_hr/canonical_descriptions.py
index 8d692409de53..e41eec1f09e8 100644
--- a/products/warehouse_sources/sources/sage_hr/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/sage_hr/canonical_descriptions.py
@@ -152,4 +152,48 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"title": "The category title.",
},
},
+ "compensations": {
+ "description": "An entry in an active employee's compensation history, such as a salary or a bonus.",
+ "docs_url": "https://sagehr.docs.apiary.io",
+ "columns": {
+ "id": "Identifier PostHog derives from the row content, because Sage HR returns none.",
+ "employee_id": "The ID of the employee the compensation belongs to.",
+ "start_date": "The date the compensation starts.",
+ "end_date": "The date the compensation ends, or null when it is current.",
+ "currency": "Three-letter currency code of the amount.",
+ "amount": "The compensation amount in `currency`.",
+ "period": "How often the amount is paid (e.g. monthly).",
+ "comment": "A free-text note about the compensation.",
+ "category": "The compensation category (e.g. Salary).",
+ },
+ },
+ "leave_balances": {
+ "description": "An active employee's used and remaining time off under each leave policy.",
+ "docs_url": "https://sagehr.docs.apiary.io",
+ "columns": {
+ "employee_id": "The ID of the employee the balance belongs to.",
+ "policy_id": "The ID of the leave policy.",
+ "used": "The time off the employee has used under the policy.",
+ "available": "The time off the employee has left under the policy, or null when it is unlimited.",
+ },
+ },
+ "recruitment_applicants": {
+ "description": "An applicant for a recruitment position, with their current hiring stage and scores.",
+ "docs_url": "https://sagehr.docs.apiary.io",
+ "columns": {
+ "id": "The ID of the applicant.",
+ "position_id": "The ID of the recruitment position the applicant applied for.",
+ "full_name": "The applicant's full name.",
+ "first_name": "The applicant's first name.",
+ "last_name": "The applicant's last name.",
+ "email": "The applicant's email address.",
+ "source": "Where the applicant came from (e.g. recruiters, referral).",
+ "created_at": "When the applicant was added.",
+ "disqualified_date": "The date the applicant was disqualified, or null.",
+ "hired_date": "The date the applicant was hired, or null.",
+ "stage": "The applicant's current hiring stage (id, title, created_at).",
+ "tags": "Tags on the applicant.",
+ "scores": "Scores that reviewers gave the applicant.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/sage_hr/sage_hr.py b/products/warehouse_sources/sources/sage_hr/sage_hr.py
index eb9674b90198..64ae3e66084c 100644
--- a/products/warehouse_sources/sources/sage_hr/sage_hr.py
+++ b/products/warehouse_sources/sources/sage_hr/sage_hr.py
@@ -1,9 +1,12 @@
import re
+import json
+import hashlib
import dataclasses
+from collections import Counter
from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
-from urllib.parse import urlencode
+from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
@@ -43,12 +46,18 @@ class SageHRRetryableError(Exception):
pass
[email protected](frozen=True)
class SageHRResumeConfig:
# Next 1-indexed page to fetch (from the response `meta.next_page`). None means "start over".
next_page: int | None = None
# For the windowed leave_requests endpoint: ISO date of the window `next_page` belongs to.
window_from: str | None = None
+ # For fan-out endpoints: the parent page, and the index and id on it of the parent `next_page` belongs to.
+ parent_page: int | None = None
+ parent_index: int | None = None
+ parent_id: Any = None
+ # For endpoints with derived row ids: how often each row hash already appeared under `parent_id`.
+ row_occurrences: dict[str, int] | None = None
def normalize_subdomain(subdomain: str) -> str:
@@ -204,6 +213,116 @@ def _get_windowed_rows(
page = next_page
+def _derive_row_id(parent_id: Any, row: dict[str, Any], occurrences: Counter[str]) -> str:
+ payload = json.dumps({"parent": parent_id, "row": row}, sort_keys=True, default=str)
+ row_hash = hashlib.sha256(payload.encode()).hexdigest()[:32]
+ # Identical rows under one parent would share a hash, so each repeat adds its ordinal to stay unique.
+ occurrence = occurrences[row_hash]
+ occurrences[row_hash] += 1
+ if not occurrence:
+ return row_hash
+ return hashlib.sha256(f"{row_hash}#{occurrence}".encode()).hexdigest()[:32]
+
+
+def _shape_child_row(
+ config: SageHREndpointConfig, parent_key: str, parent_id: Any, row: dict[str, Any], occurrences: Counter[str]
+) -> dict[str, Any]:
+ shaped = {**row, parent_key: parent_id}
+ if config.derive_row_id:
+ shaped["id"] = _derive_row_id(parent_id, row, occurrences)
+ return shaped
+
+
+def _get_fan_out_rows(
+ session: requests.Session,
+ base_url: str,
+ config: SageHREndpointConfig,
+ resume: SageHRResumeConfig | None,
+ resumable_source_manager: ResumableSourceManager[SageHRResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ fan_out = config.fan_out
+ if fan_out is None:
+ raise ValueError(f"Sage HR endpoint {config.name} has no fan-out parent")
+
+ parent_page = resume.parent_page if resume is not None and resume.parent_page is not None else 1
+ resume_index = resume.parent_index if resume is not None and resume.parent_index is not None else 0
+ resume_child_page = resume.next_page if resume is not None and resume.next_page is not None else 1
+ is_resumed_page = resume is not None
+
+ while True:
+ parents, next_parent_page = _fetch_page(
+ session, _build_url(base_url, fan_out.parent_path, {"page": parent_page}), logger
+ )
+ has_next_parent_page = bool(next_parent_page and parents and next_parent_page > parent_page)
+ first_index, first_child_page = 0, 1
+ if is_resumed_page and resume is not None:
+ # The parent list can shift between attempts, so the saved child page applies only to the
+ # parent with the saved id. A parent that is gone restarts at its saved index from page 1.
+ first_index = resume_index
+ for candidate_index, parent in enumerate(parents):
+ if resume.parent_id is not None and parent.get("id") == resume.parent_id:
+ first_index, first_child_page = candidate_index, resume_child_page
+ break
+
+ for index in range(first_index, len(parents)):
+ parent_id = parents[index]["id"]
+ child_path = config.path.format(id=quote(str(parent_id), safe=""))
+ child_page = first_child_page if index == first_index else 1
+ occurrences: Counter[str] = Counter()
+ if is_resumed_page and resume is not None and child_page > 1 and index == first_index:
+ occurrences.update(resume.row_occurrences or {})
+
+ while True:
+ params = {"page": child_page} if config.paginated else {}
+ items, next_child_page = _fetch_page(session, _build_url(base_url, child_path, params), logger)
+ has_next_child_page = config.paginated and bool(
+ next_child_page and items and next_child_page > child_page
+ )
+
+ # Shape before staging the cursor, so the saved occurrences cover this page's rows.
+ rows = [_shape_child_row(config, fan_out.parent_key, parent_id, row, occurrences) for row in items]
+
+ if has_next_child_page:
+ resumable_source_manager.save_state(
+ SageHRResumeConfig(
+ next_page=next_child_page,
+ parent_page=parent_page,
+ parent_index=index,
+ parent_id=parent_id,
+ row_occurrences=dict(occurrences) if config.derive_row_id else None,
+ )
+ )
+ elif index + 1 < len(parents):
+ resumable_source_manager.save_state(
+ SageHRResumeConfig(
+ next_page=1,
+ parent_page=parent_page,
+ parent_index=index + 1,
+ parent_id=parents[index + 1]["id"],
+ )
+ )
+ elif has_next_parent_page:
+ resumable_source_manager.save_state(
+ SageHRResumeConfig(next_page=1, parent_page=next_parent_page, parent_index=0)
+ )
+
+ if rows:
+ yield rows
+ else:
+ # Many parents have no children, so the walk needs a stop point that does not depend on rows.
+ resumable_source_manager.safe_point()
+
+ if not has_next_child_page or next_child_page is None:
+ break
+ child_page = next_child_page
+
+ is_resumed_page = False
+ if not has_next_parent_page or next_parent_page is None:
+ break
+ parent_page = next_parent_page
+
+
def get_rows(
subdomain: str,
api_key: str,
@@ -219,6 +338,10 @@ def get_rows(
if resume is not None:
logger.debug(f"Sage HR: resuming {endpoint} from page={resume.next_page}, window={resume.window_from}")
+ if config.fan_out is not None:
+ yield from _get_fan_out_rows(session, base_url, config, resume, resumable_source_manager, logger)
+ return
+
if config.requires_date_window:
yield from _get_windowed_rows(session, base_url, config, resume, resumable_source_manager, logger)
return
diff --git a/products/warehouse_sources/sources/sage_hr/settings.py b/products/warehouse_sources/sources/sage_hr/settings.py
index cfb335ce8d82..3c532962ab1d 100644
--- a/products/warehouse_sources/sources/sage_hr/settings.py
+++ b/products/warehouse_sources/sources/sage_hr/settings.py
@@ -1,7 +1,15 @@
from dataclasses import dataclass, field
-@dataclass
+@dataclass(frozen=True)
+class SageHRFanOut:
+ # Paginated parent collection whose rows' `id` fills the `{id}` placeholder of the child path.
+ parent_path: str
+ # Column added to every child row so rows from different parents stay distinct.
+ parent_key: str
+
+
+@dataclass(frozen=True)
class SageHREndpointConfig:
name: str
path: str
@@ -13,6 +21,13 @@ class SageHREndpointConfig:
requires_date_window: bool = False
# Sage HR resource identifiers are account-unique integers, so `id` is a safe primary key.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ fan_out: SageHRFanOut | None = None
+ # Compensation rows carry no identifier, so the source derives `id` from the row content.
+ derive_row_id: bool = False
+
+
+_PER_EMPLOYEE = SageHRFanOut(parent_path="/employees", parent_key="employee_id")
+_PER_RECRUITMENT_POSITION = SageHRFanOut(parent_path="/recruitment/positions", parent_key="position_id")
# Sage HR API collection endpoints (https://sagehr.docs.apiary.io). All are full-refresh only:
@@ -38,6 +53,27 @@ SAGE_HR_ENDPOINTS: dict[str, SageHREndpointConfig] = {
),
"onboarding_categories": SageHREndpointConfig(name="onboarding_categories", path="/onboarding/categories"),
"offboarding_categories": SageHREndpointConfig(name="offboarding_categories", path="/offboarding/categories"),
+ "compensations": SageHREndpointConfig(
+ name="compensations",
+ path="/employees/{id}/compensations",
+ primary_keys=["employee_id", "id"],
+ fan_out=_PER_EMPLOYEE,
+ derive_row_id=True,
+ ),
+ "leave_balances": SageHREndpointConfig(
+ name="leave_balances",
+ path="/employees/{id}/leave-management/balances",
+ paginated=False,
+ primary_keys=["employee_id", "policy_id"],
+ fan_out=_PER_EMPLOYEE,
+ ),
+ # The docs don't say whether applicant ids are unique across positions, so key on both.
+ "recruitment_applicants": SageHREndpointConfig(
+ name="recruitment_applicants",
+ path="/recruitment/positions/{id}/applicants",
+ primary_keys=["position_id", "id"],
+ fan_out=_PER_RECRUITMENT_POSITION,
+ ),
}
ENDPOINTS = tuple(SAGE_HR_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py
index 17013c74c8b0..a362ebde5f91 100644
--- a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py
+++ b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py
@@ -216,6 +216,133 @@ class TestWindowedLeaveRequests:
assert rows == [{"id": 2}]
+class TestFanOutEndpoints:
+ @staticmethod
+ def _collect(
+ manager: _FakeResumableManager,
+ monkeypatch: Any,
+ pages: Mapping[str, tuple[list[dict], int | None]],
+ endpoint: str,
+ requested: list[str] | None = None,
+ ) -> list[dict]:
+ def fake_fetch(session: Any, url: str, logger: Any) -> tuple[list[dict], int | None]:
+ parsed = urlparse(url)
+ key = parsed.path.removeprefix("/api") + ("?" + parsed.query if parsed.query else "")
+ if requested is not None:
+ requested.append(key)
+ return pages[key]
+
+ monkeypatch.setattr(sage_hr, "_fetch_page", fake_fetch)
+ monkeypatch.setattr(sage_hr, "make_tracked_session", lambda **kwargs: MagicMock())
+
+ rows: list[dict] = []
+ for batch in get_rows(
+ subdomain="acme",
+ api_key="sage-key",
+ endpoint=endpoint,
+ logger=MagicMock(),
+ resumable_source_manager=manager, # type: ignore[arg-type]
+ ):
+ rows.extend(batch)
+ return rows
+
+ def test_walks_parent_and_child_pages_and_tags_rows_with_parent(self, monkeypatch: Any) -> None:
+ manager = _FakeResumableManager()
+ pages = {
+ "/recruitment/positions?page=1": ([{"id": 10}, {"id": 11}], 2),
+ "/recruitment/positions?page=2": ([{"id": 12}], None),
+ "/recruitment/positions/10/applicants?page=1": ([{"id": 1}], 2),
+ "/recruitment/positions/10/applicants?page=2": ([{"id": 2}], None),
+ "/recruitment/positions/11/applicants?page=1": ([], None),
+ "/recruitment/positions/12/applicants?page=1": ([{"id": 1}], None),
+ }
+ rows = self._collect(manager, monkeypatch, pages, "recruitment_applicants")
+ assert rows == [
+ {"id": 1, "position_id": 10},
+ {"id": 2, "position_id": 10},
+ {"id": 1, "position_id": 12},
+ ]
+ assert [(s.parent_page, s.parent_index, s.parent_id, s.next_page) for s in manager.saved] == [
+ (1, 0, 10, 2),
+ (1, 1, 11, 1),
+ (2, 0, None, 1),
+ ]
+
+ def test_unpaginated_child_is_fetched_once_per_parent_without_page_param(self, monkeypatch: Any) -> None:
+ manager = _FakeResumableManager()
+ requested: list[str] = []
+ pages: dict[str, tuple[list[dict], int | None]] = {
+ "/employees?page=1": ([{"id": 5}, {"id": 6}], None),
+ "/employees/5/leave-management/balances": ([{"policy_id": 1, "used": 2, "available": 3}], None),
+ "/employees/6/leave-management/balances": ([{"policy_id": 1, "used": 0, "available": None}], None),
+ }
+ rows = self._collect(manager, monkeypatch, pages, "leave_balances", requested)
+ assert rows == [
+ {"policy_id": 1, "used": 2, "available": 3, "employee_id": 5},
+ {"policy_id": 1, "used": 0, "available": None, "employee_id": 6},
+ ]
+ assert requested == list(pages)
+
+ @pytest.mark.parametrize(
+ "parent_index, parent_id, first_child_key",
+ [
+ pytest.param(1, 21, "/recruitment/positions/21/applicants?page=3", id="saved_parent_at_saved_index"),
+ pytest.param(2, 21, "/recruitment/positions/21/applicants?page=3", id="saved_parent_moved"),
+ pytest.param(1, 99, "/recruitment/positions/21/applicants?page=1", id="saved_parent_gone"),
+ ],
+ )
+ def test_resumes_from_saved_parent_and_child_page(
+ self, parent_index: int, parent_id: int, first_child_key: str, monkeypatch: Any
+ ) -> None:
+ manager = _FakeResumableManager(
+ SageHRResumeConfig(next_page=3, parent_page=2, parent_index=parent_index, parent_id=parent_id)
+ )
+ # Earlier parent pages, earlier parents on the page and earlier child pages must never be fetched.
+ pages: dict[str, tuple[list[dict], int | None]] = {
+ "/recruitment/positions?page=2": ([{"id": 20}, {"id": 21}, {"id": 22}], None),
+ first_child_key: ([{"id": 7}], None),
+ "/recruitment/positions/22/applicants?page=1": ([{"id": 8}], None),
+ }
+ rows = self._collect(manager, monkeypatch, pages, "recruitment_applicants")
+ assert rows == [{"id": 7, "position_id": 21}, {"id": 8, "position_id": 22}]
+
+ def test_derived_compensation_ids_are_stable_and_distinct(self, monkeypatch: Any) -> None:
+ salary: dict[str, Any] = {
+ "start_date": "2024-01-01",
+ "end_date": None,
+ "amount": 1000,
+ "currency": "EUR",
+ "category": "Salary",
+ }
+ bonus = {**salary, "amount": 200, "category": "Bonus"}
+ pages: dict[str, tuple[list[dict], int | None]] = {
+ "/employees?page=1": ([{"id": 5}, {"id": 6}], None),
+ "/employees/5/compensations?page=1": ([salary, bonus, dict(bonus)], None),
+ "/employees/6/compensations?page=1": ([salary], None),
+ }
+ first = self._collect(_FakeResumableManager(), monkeypatch, pages, "compensations")
+ second = self._collect(_FakeResumableManager(), monkeypatch, pages, "compensations")
+
+ assert [row["employee_id"] for row in first] == [5, 5, 5, 6]
+ assert len({row["id"] for row in first}) == 4
+ assert [row["id"] for row in first] == [row["id"] for row in second]
+
+ def test_resumed_compensation_ids_match_an_uninterrupted_walk(self, monkeypatch: Any) -> None:
+ salary: dict[str, Any] = {"start_date": "2024-01-01", "amount": 1000, "category": "Salary"}
+ pages: dict[str, tuple[list[dict], int | None]] = {
+ "/employees?page=1": ([{"id": 5}], None),
+ "/employees/5/compensations?page=1": ([salary], 2),
+ "/employees/5/compensations?page=2": ([dict(salary)], None),
+ }
+ uninterrupted = _FakeResumableManager()
+ full_rows = self._collect(uninterrupted, monkeypatch, pages, "compensations")
+
+ resumed_rows = self._collect(_FakeResumableManager(uninterrupted.saved[0]), monkeypatch, pages, "compensations")
+
+ assert len({row["id"] for row in full_rows}) == 2
+ assert [row["id"] for row in resumed_rows] == [full_rows[1]["id"]]
+
+
class TestCheckAccess:
def _patch_session(self, monkeypatch: Any, response: Any) -> MagicMock:
session = MagicMock()
@@ -296,10 +423,6 @@ class TestSageHRSourceResponse:
resumable_source_manager=MagicMock(),
)
assert response.name == endpoint
- assert response.primary_keys == ["id"]
+ assert response.primary_keys == SAGE_HR_ENDPOINTS[endpoint].primary_keys
# No stable creation timestamp is guaranteed across every object, so we don't partition.
assert response.partition_mode is None
-
- def test_every_endpoint_uses_id_primary_key(self) -> None:
- assert all(config.primary_keys == ["id"] for config in SAGE_HR_ENDPOINTS.values())
- assert set(SAGE_HR_ENDPOINTS) == set(ENDPOINTS)