Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add personio employments, compensations, org units and absence types (#125431)

Tom Owers23575f323509
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 3fc947f24e55..b51e60c064d1 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -847,7 +847,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/pinterest_ads.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 9050de6a6d57..657b05ed1d02 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9332,14 +9332,14 @@ Note: Persona's own repo note (products/warehouse_sources/backend/temporal/data_
## Personio — **thin**
-Today (7): `absence_periods`, `attendance_periods`, `cost_centers`, `persons`, `recruiting_categories`, `recruiting_jobs`, `salary_bands`
+Today (11): `absence_periods`, `absence_types`, `attendance_periods`, `compensations`, `cost_centers`, `employments`, `org_units`, `persons`, `recruiting_categories`, `recruiting_jobs`, `salary_bands`
Diffed against: <https://developer.personio.de/llms.txt>
-- [ ] `GET /v2/persons/{person_id}/employments` — employment records for the persons already synced - contract type, dates, job reference; the core HR fact table (high)
-- [ ] `GET /v2/absence-types` — lookup resolving the absence type on every absence period already synced (high)
-- [ ] `GET /v2/compensations` — salary, hourly, bonus and recurring compensation - the payroll dataset (high)
-- [ ] `GET /v2/org-units` — department/team hierarchy lookup for grouping persons, absences and attendance (high)
+- [x] `GET /v2/persons/{person_id}/employments` — employment records for the persons already synced - contract type, dates, job reference; the core HR fact table (high)
+- [x] `GET /v2/absence-types` — lookup resolving the absence type on every absence period already synced (high)
+- [x] `GET /v2/compensations` — salary, hourly, bonus and recurring compensation - the payroll dataset (high)
+- [x] `GET /v2/org-units` — department/team hierarchy lookup for grouping persons, absences and attendance (high)
- [x] `GET /v2/cost-centers` — cost center lookup for allocating attendance and compensation to finance dimensions (medium)
- [x] `GET /v2/salary-bands` — salary band ranges (min/max/mid, currency) and the workplaces they apply to (medium)
- [ ] `GET /v2/legal-entities` — legal entity lookup for multi-entity headcount and payroll splits (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.py
index 0cd0c0120506..5841b43c8e35 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.py
@@ -24,6 +24,47 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"attributes": "Whitelisted custom and standard employee attributes.",
},
},
+ "employments": {
+ "description": "An employment record of a Personio person, with its position, contract type and dates.",
+ "docs_url": "https://developer.personio.de/reference/get_v2-persons-person-id-employments",
+ "columns": {
+ "id": "Unique identifier for the employment.",
+ "person_id": "The person the employment belongs to.",
+ "position": "The position of the employment, carrying its title.",
+ "status": "Status of the employment (ACTIVE, INACTIVE, ONBOARDING, LEAVE or UNSPECIFIED).",
+ "type": "Employment type (INTERNAL, EXTERNAL or UNSPECIFIED).",
+ "weekly_working_hours": "Total weekly working hours of the employment.",
+ "full_time_weekly_working_hours": "Weekly working hours considered full time for the employment.",
+ "probation_end_date": "Date that marks the end of the probation period.",
+ "employment_start_date": "Date that marks the start of the employment.",
+ "employment_end_date": "Date that marks the end of the employment.",
+ "contract_end_date": "Date the contract ends for this employment.",
+ "supervisor": "The supervisor of the employment, carrying a person id.",
+ "office": "The office of the employment, carrying an office id.",
+ "org_units": "Org units (departments and teams) the employment belongs to.",
+ "cost_centers": "Cost centers the employment is allocated to.",
+ "legal_entity": "The legal entity of the employment, carrying a legal entity id.",
+ "job": "The job of the employment, carrying a job id.",
+ "created_at": "Time at which the employment was created.",
+ "updated_at": "Time at which the employment was last updated.",
+ },
+ },
+ "compensations": {
+ "description": "A payroll compensation (base salary, hourly, one-time, recurring or bonus) that applies to a person in a calendar month.",
+ "docs_url": "https://developer.personio.de/reference/get_v2-compensations",
+ "columns": {
+ "id": "Identifier of the compensation.",
+ "period_start": "First day of the calendar month the compensation was listed for.",
+ "person": "The person the compensation belongs to, carrying a person id.",
+ "legal_entity": "The legal entity of the compensation, carrying a legal entity id.",
+ "type": "The compensation type, with its name and category (e.g. FIXED_SALARY, ONE_TIME).",
+ "amount": "The compensation amount, with its value and currency.",
+ "interval": "Payout interval of the compensation (MONTHLY, ONE_TIME or YEARLY).",
+ "effective_from": "Effective start date of the compensation.",
+ "weekly_working_hours": "Hours the person is expected to work per week. Only set for fixed salaries.",
+ "full_time_weekly_working_hours": "Hours per week considered full time. Only set for fixed salaries.",
+ },
+ },
"absence_periods": {
"description": "A period of employee absence such as vacation or sick leave, including its dates and type.",
"docs_url": "https://developer.personio.de/reference/get_v2-absence-periods",
@@ -71,6 +112,30 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"name": "The name of the cost center.",
},
},
+ "absence_types": {
+ "description": "An absence type defined in Personio, such as vacation or sick leave, referenced by absence periods.",
+ "docs_url": "https://developer.personio.de/reference/get_v2-absence-types",
+ "columns": {
+ "id": "Unique identifier for the absence type.",
+ "name": "The name of the absence type.",
+ "category": "The category of the absence type (e.g. PAID_VACATION, SICK_LEAVE).",
+ "unit": "Unit absence periods of this type are measured in (DAY or HOUR).",
+ },
+ },
+ "org_units": {
+ "description": "An org unit (department or team) in Personio, with its place in the org unit hierarchy.",
+ "docs_url": "https://developer.personio.de/reference/listorgunits",
+ "columns": {
+ "id": "Unique identifier for the org unit.",
+ "type": "Type of the org unit (department or team).",
+ "name": "The name of the org unit.",
+ "abbreviation": "Short name of the org unit.",
+ "description": "Description of the org unit.",
+ "parent_id": "Identifier of the parent org unit. Null for a root org unit.",
+ "parent_chain": "Ordered ancestor org unit ids, from the immediate parent up to the root.",
+ "create_time": "Time at which the org unit was created.",
+ },
+ },
"recruiting_jobs": {
"description": "A recruiting job (open position) in Personio, with its category, department and hiring team.",
"docs_url": "https://developer.personio.de/reference/get_v2-recruiting-jobs",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py
index 6ccd56671969..9b30548d7e14 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py
@@ -1,10 +1,13 @@
import dataclasses
-from datetime import UTC, date, datetime
-from typing import Any, Optional
+from collections.abc import Callable, Iterable, Iterator, Mapping
+from datetime import UTC, date, datetime, timedelta
+from typing import Any, Optional, cast
import requests
from requests import Request, Response
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
@@ -13,27 +16,41 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
OAuth2Auth,
OAuth2AuthRequestError,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ FanoutEndpointLike,
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponsePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
Endpoint,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.personio.settings import PERSONIO_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.personio.settings import (
+ COMPENSATIONS_LOOKBACK_MONTHS,
+ PERSONIO_ENDPOINTS,
+ PersonioEndpointConfig,
+)
PERSONIO_BASE_URL = "https://api.personio.de"
TOKEN_URL = f"{PERSONIO_BASE_URL}/v2/auth/token"
[email protected](frozen=True)
class PersonioResumeConfig:
# Personio v2 paginates via _meta.links.next.href, a self-contained URL (opaque cursor), so
# the URL is all we persist. Optional/defaulted so previously saved state (which always carried
# next_url) still parses via dataclass(**saved).
next_url: Optional[str] = None
+ # Endpoints read as several listings (org unit types, compensation month windows) record which
+ # listing next_url belongs to.
+ segment: Optional[str] = None
+ # Fan-out endpoints checkpoint the shape `build_dependent_resource` emits.
+ fanout_state: Optional[dict[str, Any]] = None
def _make_auth(client_id: str, client_secret: str) -> OAuth2Auth:
@@ -60,6 +77,38 @@ def _format_updated_at(value: Any) -> str:
return str(value)
+def _month_start(value: Any) -> date:
+ if isinstance(value, datetime):
+ value = value.date()
+ elif not isinstance(value, date):
+ value = date.fromisoformat(str(value)[:10])
+ return value.replace(day=1)
+
+
+def _add_months(month: date, months: int) -> date:
+ index = month.year * 12 + month.month - 1 + months
+ return date(index // 12, index % 12 + 1, 1)
+
+
+def _month_starts(first: date, last: date) -> list[date]:
+ """The first day of each calendar month from the month of `first` through the month of `last`."""
+ months = []
+ month = _month_start(first)
+ while month <= last:
+ months.append(month)
+ month = _add_months(month, 1)
+ return months
+
+
+@frozen
+class _Segment:
+ """One listing of an endpoint that Personio serves as several listings."""
+
+ key: str
+ params: dict[str, Any]
+ row_map: Callable[[dict[str, Any]], dict[str, Any]] | None = None
+
+
class PersonioPaginator(JSONResponsePaginator):
"""Follow Personio's self-contained ``_meta.links.next.href`` link, stopping on an empty page.
@@ -91,6 +140,111 @@ def validate_credentials(client_id: str, client_secret: str) -> bool:
return True
+def _client_config(auth: OAuth2Auth) -> ClientConfig:
+ return {
+ "base_url": PERSONIO_BASE_URL,
+ "auth": auth,
+ # Pin every request — including _meta.links.next.href links and the seeded resume URL — to
+ # api.personio.de so a tampered next/resume link can't exfiltrate the bearer token. Disable
+ # redirects so a 3xx can't bounce the authenticated request (and token) off-host either.
+ "allowed_hosts": [],
+ "allow_redirects": False,
+ }
+
+
+def _segments(config: PersonioEndpointConfig) -> list[_Segment]:
+ """The listings a multi-listing endpoint reads, in order."""
+ if config.monthly_windows:
+ today = datetime.now(UTC).date()
+ first = _add_months(_month_start(today), -COMPENSATIONS_LOOKBACK_MONTHS)
+
+ def add_period(period_start: date) -> Callable[[dict[str, Any]], dict[str, Any]]:
+ return lambda row: {**row, "period_start": period_start}
+
+ return [
+ _Segment(
+ key=start.isoformat(),
+ params={
+ "start_date": start.isoformat(),
+ "end_date": (_add_months(start, 1) - timedelta(days=1)).isoformat(),
+ },
+ row_map=add_period(start),
+ )
+ for start in _month_starts(first, today)
+ ]
+ return [_Segment(key=variant["type"], params=dict(variant)) for variant in config.query_variants]
+
+
+def _segmented_resource(
+ endpoint: str,
+ config: PersonioEndpointConfig,
+ auth: OAuth2Auth,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[PersonioResumeConfig],
+ resume: Optional[PersonioResumeConfig],
+) -> Resource:
+ segments = _segments(config)
+ start_index = 0
+ if resume is not None and resume.segment is not None:
+ start_index = next((i for i, segment in enumerate(segments) if segment.key == resume.segment), 0)
+
+ def build(index: int) -> Resource:
+ segment = segments[index]
+ key = segment.key
+ next_key = segments[index + 1].key if index + 1 < len(segments) else None
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if index == start_index and resume is not None and resume.segment == key and resume.next_url:
+ initial_paginator_state = {"next_url": resume.next_url}
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state is not None and state.get("next_url") is not None:
+ resumable_source_manager.save_state(PersonioResumeConfig(segment=key, next_url=state["next_url"]))
+ elif next_key is not None:
+ # The last page of this listing is on its way, so a resume starts at the next one.
+ resumable_source_manager.save_state(PersonioResumeConfig(segment=next_key))
+
+ rest_config: RESTAPIConfig = {
+ "client": _client_config(auth),
+ "resources": [
+ {
+ "name": endpoint,
+ "endpoint": {
+ "path": config.path,
+ "params": {"limit": config.page_size, **segment.params},
+ "data_selector": "_data",
+ "paginator": PersonioPaginator(),
+ },
+ }
+ ],
+ }
+ resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+ return resource.add_map(segment.row_map) if segment.row_map is not None else resource
+
+ first = build(start_index)
+
+ def pages() -> Iterator[list[dict[str, Any]]]:
+ for index in range(start_index, len(segments)):
+ resource = first if index == start_index else build(index)
+ if index != start_index:
+ resumable_source_manager.save_state(PersonioResumeConfig(segment=segments[index].key))
+ # Every earlier listing is fully handed on, so a hand-off here loses nothing. Months with
+ # no compensations otherwise send requests with no yield in between.
+ resumable_source_manager.safe_point()
+ yield from resource
+
+ # As a `Resource`, the chain keeps the framework's checkpoints: each page reaches the pipeline
+ # with its cursor already staged.
+ return Resource(pages, name=endpoint, hints=first._hints)
+
+
def personio_source(
client_id: str,
client_secret: str,
@@ -106,62 +260,88 @@ def personio_source(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- params: dict[str, Any] = {}
- if config.page_size is not None:
- params["limit"] = config.page_size
+ incremental_params: dict[str, Any] = {}
if (
config.incremental_param is not None
and should_use_incremental_field
and db_incremental_field_last_value is not None
):
- params[config.incremental_param] = _format_updated_at(db_incremental_field_last_value)
-
- initial_paginator_state: Optional[dict[str, Any]] = None
- if resume is not None and resume.next_url is not None:
- initial_paginator_state = {"next_url": resume.next_url}
-
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # The framework calls the hook AFTER a page is yielded and only while a next page remains,
- # so a crash re-yields the last batch (merge dedupes on primary key) rather than skipping it.
- if state is not None and state.get("next_url") is not None:
- resumable_source_manager.save_state(PersonioResumeConfig(next_url=state["next_url"]))
-
- endpoint_config: Endpoint = {
- "path": config.path,
- "params": params,
- "data_selector": "_data",
- "paginator": PersonioPaginator(),
- }
- client_config: ClientConfig = {
- "base_url": PERSONIO_BASE_URL,
- "auth": auth,
- # Pin every request — including _meta.links.next.href links and the seeded resume URL — to
- # api.personio.de so a tampered next/resume link can't exfiltrate the bearer token. Disable
- # redirects so a 3xx can't bounce the authenticated request (and token) off-host either.
- "allowed_hosts": [],
- "allow_redirects": False,
- }
- rest_config: RESTAPIConfig = {
- "client": client_config,
- "resources": [{"name": endpoint, "endpoint": endpoint_config}],
- }
- resource = rest_api_resource(
- rest_config,
- team_id,
- job_id,
- None,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
- )
+ incremental_params[config.incremental_param] = _format_updated_at(db_incremental_field_last_value)
+
+ items: Callable[[], Iterable[Any]]
+ if config.fanout is not None:
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(PersonioResumeConfig(fanout_state=state))
+
+ resource = build_dependent_resource(
+ # Every endpoint in a fan-out is paginated, so page_size is set on both.
+ endpoint_configs=cast(Mapping[str, FanoutEndpointLike], PERSONIO_ENDPOINTS),
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ client_config=_client_config(auth),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ should_use_incremental_field=should_use_incremental_field,
+ # The cursor rides in child_params_extra; the factory only switches the child to merge.
+ incremental_config_factory=lambda _field: None,
+ parent_endpoint_extra={"data_selector": "_data", "paginator": PersonioPaginator()},
+ child_endpoint_extra={"data_selector": "_data", "paginator": PersonioPaginator()},
+ child_params_extra=incremental_params,
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume else None,
+ )
+ items = lambda: resource
+ elif config.monthly_windows or config.query_variants:
+ segmented = _segmented_resource(endpoint, config, auth, team_id, job_id, resumable_source_manager, resume)
+ items = lambda: segmented
+ else:
+ params: dict[str, Any] = {**incremental_params}
+ if config.page_size is not None:
+ params["limit"] = config.page_size
+
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if resume is not None and resume.next_url is not None:
+ initial_paginator_state = {"next_url": resume.next_url}
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ # The framework calls the hook AFTER a page is yielded and only while a next page remains,
+ # so a crash re-yields the last batch (merge dedupes on primary key) rather than skipping it.
+ if state is not None and state.get("next_url") is not None:
+ resumable_source_manager.save_state(PersonioResumeConfig(next_url=state["next_url"]))
+
+ endpoint_config: Endpoint = {
+ "path": config.path,
+ "params": params,
+ "data_selector": "_data",
+ "paginator": PersonioPaginator(),
+ }
+ rest_config: RESTAPIConfig = {
+ "client": _client_config(auth),
+ "resources": [{"name": endpoint, "endpoint": endpoint_config}],
+ }
+ single_resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+ items = lambda: single_resource
return SourceResponse(
name=endpoint,
- items=lambda: resource,
- primary_keys=[config.primary_key],
+ items=items,
+ primary_keys=list(config.primary_keys),
partition_count=1,
partition_size=1,
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
- sort_mode="asc",
+ # Fan-out rows arrive per person, newest-first, so they are not one ordered series.
+ sort_mode=None if config.fanout is not None else "asc",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.py
index 076a7eecba08..4e4a56f3a578 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.py
@@ -1,8 +1,15 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+# The compensations listing only answers for one calendar month at a time, so a sync walks month
+# windows. This bounds how far back the walk starts when there is no watermark yet.
+COMPENSATIONS_LOOKBACK_MONTHS = 24
+
_UPDATED_AT_INCREMENTAL_FIELDS: list[IncrementalField] = [
{
"label": "updated_at",
@@ -19,7 +26,7 @@ class PersonioEndpointConfig:
path: str
# OAuth scope the endpoint needs (surfaced in the source caption).
scope: str
- primary_key: str = "id"
+ primary_keys: tuple[str, ...] = ("id",)
# Server-side updated_at filter param. Personio v2 is inconsistent per
# endpoint: persons use strict `.gt`, the period endpoints use `.gte`.
incremental_param: Optional[str] = None
@@ -29,6 +36,13 @@ class PersonioEndpointConfig:
page_size: Optional[int] = 100
# Stable creation-time field used for datetime partitioning.
partition_key: Optional[str] = None
+ # Child endpoints listed once per parent row (e.g. employments per person).
+ fanout: Optional[DependentEndpointConfig] = None
+ default_incremental_field: Optional[str] = None
+ # Endpoints with a required filter are listed once per variant (e.g. org units per type).
+ query_variants: tuple[dict[str, str], ...] = ()
+ # Walk calendar-month start_date/end_date windows (compensations).
+ monthly_windows: bool = False
PERSONIO_ENDPOINTS: dict[str, PersonioEndpointConfig] = {
@@ -41,6 +55,35 @@ PERSONIO_ENDPOINTS: dict[str, PersonioEndpointConfig] = {
page_size=50,
partition_key="created_at",
),
+ # Employment ids are only documented per person, so the key carries the person id too. The
+ # listing is per person and newest-first, so incremental runs merge without an ordered cursor.
+ "employments": PersonioEndpointConfig(
+ name="employments",
+ path="/v2/persons/{person_id}/employments",
+ scope="personio:persons:read",
+ primary_keys=("person_id", "id"),
+ incremental_param="updated_at.gt",
+ incremental_fields=list(_UPDATED_AT_INCREMENTAL_FIELDS),
+ page_size=50,
+ partition_key="created_at",
+ fanout=DependentEndpointConfig(
+ parent_name="persons",
+ resolve_param="person_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "person_id"},
+ ),
+ ),
+ # One row per compensation per month window: recurring compensations repeat in every window,
+ # so the window start is part of the key. Full refresh only: a compensation can be added to a
+ # past month at any time, and the API has no change timestamp to find it with.
+ "compensations": PersonioEndpointConfig(
+ name="compensations",
+ path="/v2/compensations",
+ scope="personio:compensations:read",
+ primary_keys=("period_start", "id"),
+ monthly_windows=True,
+ ),
"absence_periods": PersonioEndpointConfig(
name="absence_periods",
path="/v2/absence-periods",
@@ -67,6 +110,23 @@ PERSONIO_ENDPOINTS: dict[str, PersonioEndpointConfig] = {
path="/v2/cost-centers",
scope="personio:cost-centers:read",
),
+ # Absence types and org units are lookups with no server-side timestamp filter — full refresh only.
+ "absence_types": PersonioEndpointConfig(
+ name="absence_types",
+ path="/v2/absence-types",
+ scope="personio:absences:read",
+ ),
+ # `type` is a required filter, so departments and teams are two listings in one table.
+ "org_units": PersonioEndpointConfig(
+ name="org_units",
+ path="/v2/org-units",
+ scope="personio:org-units:read",
+ primary_keys=("type", "id"),
+ query_variants=(
+ {"type": "department", "include_parent_chain": "true"},
+ {"type": "team", "include_parent_chain": "true"},
+ ),
+ ),
# Recruiting jobs support `updated_at.gt`, but their updated_at/created_at are
# {"date-time", "timezone"} objects that can't serve as a cursor or partition column, and the
# listing is newest-first — so full refresh only.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.py
index 31943dc6893f..f65b4d030002 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.py
@@ -67,7 +67,7 @@ class PersonioSource(ResumableSource[PersonioSourceConfig, PersonioResumeConfig]
label="Personio",
caption="""Enter your Personio API credentials to pull your Personio HR data into the PostHog Data warehouse.
-An admin can create API credentials in Personio under Settings > Integrations > API credentials. Grant the read scopes for the datasets you want to sync (`personio:persons:read`, `personio:absences:read`, `personio:attendances:read`, `personio:salary-bands:read`, `personio:cost-centers:read`, `personio:recruiting:read`) and whitelist the employee attributes you need — attributes that aren't whitelisted are silently omitted from responses.""",
+An admin can create API credentials in Personio under Settings > Integrations > API credentials. Grant the read scopes for the datasets you want to sync (`personio:persons:read`, `personio:absences:read`, `personio:attendances:read`, `personio:compensations:read`, `personio:org-units:read`, `personio:salary-bands:read`, `personio:cost-centers:read`, `personio:recruiting:read`) and whitelist the employee attributes you need — attributes that aren't whitelisted are silently omitted from responses.""",
iconPath="/static/services/personio.png",
docsUrl="https://posthog.com/docs/cdp/sources/personio",
releaseStatus=ReleaseStatus.BETA,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.py
index f891eaabcbe2..211eece6f8de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.py
@@ -5,6 +5,7 @@ from urllib.parse import parse_qs
import pytest
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
scripted_network,
@@ -18,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.personio.p
validate_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.personio.settings import (
+ COMPENSATIONS_LOOKBACK_MONTHS,
ENDPOINTS,
PERSONIO_ENDPOINTS,
)
@@ -40,6 +42,19 @@ def _token_response(status_code: int = 200, expires_in: int = 86400) -> Scripted
return ScriptedResponse(status=status_code, json=body)
+def _answer_empty(request: RecordedRequest) -> ScriptedResponse:
+ return _token_response() if request.path == "/v2/auth/token" else _page([])
+
+
+def _this_month() -> date:
+ return datetime.now(UTC).date().replace(day=1)
+
+
+def _months_before(month: date, months: int) -> date:
+ index = month.year * 12 + month.month - 1 - months
+ return date(index // 12, index % 12 + 1, 1)
+
+
def _driver() -> SourceDriver:
return SourceDriver(PersonioSource(), PersonioSourceConfig(client_id="cid", client_secret="sec"))
@@ -216,6 +231,100 @@ class TestGetRows:
assert result.paths == ["/v2/auth/token", "/v2/persons"]
assert result.saved_states == []
+ @pytest.mark.parametrize("incremental", [False, True])
+ def test_employments_fan_out_per_person(self, incremental: bool) -> None:
+ result = _driver().run(
+ "employments",
+ [
+ _token_response(),
+ _page([{"id": "p1"}, {"id": "p2"}]),
+ _page([{"id": "e1"}]),
+ _page([{"id": "e2"}]),
+ ],
+ should_use_incremental_field=incremental,
+ db_incremental_field_last_value=datetime(2024, 1, 2, tzinfo=UTC) if incremental else None,
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/v2/auth/token",
+ "/v2/persons",
+ "/v2/persons/p1/employments",
+ "/v2/persons/p2/employments",
+ ]
+ assert [(row["person_id"], row["id"]) for row in result.rows] == [("p1", "e1"), ("p2", "e2")]
+ # The cursor filters employments only: an employment update does not touch its person.
+ assert result.requests[1].param("updated_at.gt") is None
+ expected_filter = "2024-01-02T00:00:00Z" if incremental else None
+ assert [request.param("updated_at.gt") for request in result.requests[2:]] == [expected_filter] * 2
+ assert result.response is not None
+ assert result.response.sort_mode is None
+
+ def test_org_units_lists_each_type_and_stages_cursor_before_each_page(self) -> None:
+ next_url = "https://api.personio.de/v2/org-units?type=department&cursor=p2"
+ result = _driver().run(
+ "org_units",
+ [
+ _token_response(),
+ _page([{"id": "1", "type": "department"}], next_url=next_url),
+ _page([{"id": "2", "type": "department"}]),
+ _page([{"id": "1", "type": "team"}]),
+ ],
+ )
+
+ assert result.raised is None
+ assert [request.param("type") for request in result.requests[1:]] == ["department", "department", "team"]
+ assert result.requests[1].param("include_parent_chain") == "true"
+ assert [(row["type"], row["id"]) for row in result.rows] == [
+ ("department", "1"),
+ ("department", "2"),
+ ("team", "1"),
+ ]
+ # Each page reaches the pipeline with the cursor past it, so a hand-off after a write never
+ # re-reads that page. The last department page already points at the team listing.
+ assert result.committed_states == [
+ PersonioResumeConfig(segment="department", next_url=next_url),
+ PersonioResumeConfig(segment="team"),
+ ]
+
+ def test_compensations_walks_month_windows(self) -> None:
+ result = _driver().run(
+ "compensations",
+ lambda request: (
+ _token_response()
+ if request.path == "/v2/auth/token"
+ else _page([{"id": "c1"}] if request.param("start_date") == _this_month().isoformat() else [])
+ ),
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=date(2099, 1, 1),
+ )
+
+ assert result.raised is None
+ windows = [(request.param("start_date"), request.param("end_date")) for request in result.requests[1:]]
+ # Always the full lookback: a compensation can land in a past month after it was synced.
+ assert len(windows) == COMPENSATIONS_LOOKBACK_MONTHS + 1
+ assert windows[0][0] == _months_before(_this_month(), COMPENSATIONS_LOOKBACK_MONTHS).isoformat()
+ assert windows[-1][0] == _this_month().isoformat()
+ for start, end in windows:
+ assert start is not None and end is not None
+ # Personio rejects a window longer than one month.
+ assert start[:7] == end[:7] and start.endswith("-01")
+ assert [(row["period_start"], row["id"]) for row in result.rows] == [(_this_month(), "c1")]
+
+ def test_compensations_resume_skips_finished_windows(self) -> None:
+ last_month = _months_before(_this_month(), 1)
+ resume_url = "https://api.personio.de/v2/compensations?cursor=cur_resume"
+ result = _driver().run(
+ "compensations",
+ [_token_response(), _page([{"id": "c9"}]), _page([])],
+ resume_state=PersonioResumeConfig(segment=last_month.isoformat(), next_url=resume_url),
+ )
+
+ assert result.raised is None
+ assert result.requests[1].url == resume_url
+ assert result.requests[2].param("start_date") == _this_month().isoformat()
+ assert [row["period_start"] for row in result.rows] == [last_month]
+
def test_rejects_off_host_next_url(self) -> None:
# SSRF guard: a _meta.links.next.href pointing off api.personio.de is rejected before the
# request (and its bearer token) leaves the process.
@@ -243,14 +352,13 @@ class TestPersonioSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = PERSONIO_ENDPOINTS[endpoint]
- result = _driver().run(endpoint, [_token_response(), _page([])])
+ result = _driver().run(endpoint, _answer_empty)
response = result.response
assert result.raised is None
assert response is not None
assert response.name == endpoint
- assert response.primary_keys == [config.primary_key]
- assert response.sort_mode == "asc"
+ assert response.primary_keys == list(config.primary_keys)
if config.partition_key:
assert response.partition_mode == "datetime"
assert response.partition_keys == [config.partition_key]