:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add personio employments, compensations, org units and absence types (#125431)
Tom Owers23575f323509
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 3fc947f24e55..b51e60c064d1 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -847,7 +847,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/pinterest_ads.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 9050de6a6d57..657b05ed1d02 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9332,14 +9332,14 @@ Note: Persona's own repo note (products/warehouse_sources/backend/temporal/data_## Personio — **thin**-Today (7): `absence_periods`, `attendance_periods`, `cost_centers`, `persons`, `recruiting_categories`, `recruiting_jobs`, `salary_bands`+Today (11): `absence_periods`, `absence_types`, `attendance_periods`, `compensations`, `cost_centers`, `employments`, `org_units`, `persons`, `recruiting_categories`, `recruiting_jobs`, `salary_bands`Diffed against: <https://developer.personio.de/llms.txt>-- [ ] `GET /v2/persons/{person_id}/employments` — employment records for the persons already synced - contract type, dates, job reference; the core HR fact table (high)-- [ ] `GET /v2/absence-types` — lookup resolving the absence type on every absence period already synced (high)-- [ ] `GET /v2/compensations` — salary, hourly, bonus and recurring compensation - the payroll dataset (high)-- [ ] `GET /v2/org-units` — department/team hierarchy lookup for grouping persons, absences and attendance (high)+- [x] `GET /v2/persons/{person_id}/employments` — employment records for the persons already synced - contract type, dates, job reference; the core HR fact table (high)+- [x] `GET /v2/absence-types` — lookup resolving the absence type on every absence period already synced (high)+- [x] `GET /v2/compensations` — salary, hourly, bonus and recurring compensation - the payroll dataset (high)+- [x] `GET /v2/org-units` — department/team hierarchy lookup for grouping persons, absences and attendance (high)- [x] `GET /v2/cost-centers` — cost center lookup for allocating attendance and compensation to finance dimensions (medium)- [x] `GET /v2/salary-bands` — salary band ranges (min/max/mid, currency) and the workplaces they apply to (medium)- [ ] `GET /v2/legal-entities` — legal entity lookup for multi-entity headcount and payroll splits (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.pyindex 0cd0c0120506..5841b43c8e35 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/canonical_descriptions.py@@ -24,6 +24,47 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"attributes": "Whitelisted custom and standard employee attributes.",},},+ "employments": {+ "description": "An employment record of a Personio person, with its position, contract type and dates.",+ "docs_url": "https://developer.personio.de/reference/get_v2-persons-person-id-employments",+ "columns": {+ "id": "Unique identifier for the employment.",+ "person_id": "The person the employment belongs to.",+ "position": "The position of the employment, carrying its title.",+ "status": "Status of the employment (ACTIVE, INACTIVE, ONBOARDING, LEAVE or UNSPECIFIED).",+ "type": "Employment type (INTERNAL, EXTERNAL or UNSPECIFIED).",+ "weekly_working_hours": "Total weekly working hours of the employment.",+ "full_time_weekly_working_hours": "Weekly working hours considered full time for the employment.",+ "probation_end_date": "Date that marks the end of the probation period.",+ "employment_start_date": "Date that marks the start of the employment.",+ "employment_end_date": "Date that marks the end of the employment.",+ "contract_end_date": "Date the contract ends for this employment.",+ "supervisor": "The supervisor of the employment, carrying a person id.",+ "office": "The office of the employment, carrying an office id.",+ "org_units": "Org units (departments and teams) the employment belongs to.",+ "cost_centers": "Cost centers the employment is allocated to.",+ "legal_entity": "The legal entity of the employment, carrying a legal entity id.",+ "job": "The job of the employment, carrying a job id.",+ "created_at": "Time at which the employment was created.",+ "updated_at": "Time at which the employment was last updated.",+ },+ },+ "compensations": {+ "description": "A payroll compensation (base salary, hourly, one-time, recurring or bonus) that applies to a person in a calendar month.",+ "docs_url": "https://developer.personio.de/reference/get_v2-compensations",+ "columns": {+ "id": "Identifier of the compensation.",+ "period_start": "First day of the calendar month the compensation was listed for.",+ "person": "The person the compensation belongs to, carrying a person id.",+ "legal_entity": "The legal entity of the compensation, carrying a legal entity id.",+ "type": "The compensation type, with its name and category (e.g. FIXED_SALARY, ONE_TIME).",+ "amount": "The compensation amount, with its value and currency.",+ "interval": "Payout interval of the compensation (MONTHLY, ONE_TIME or YEARLY).",+ "effective_from": "Effective start date of the compensation.",+ "weekly_working_hours": "Hours the person is expected to work per week. Only set for fixed salaries.",+ "full_time_weekly_working_hours": "Hours per week considered full time. Only set for fixed salaries.",+ },+ },"absence_periods": {"description": "A period of employee absence such as vacation or sick leave, including its dates and type.","docs_url": "https://developer.personio.de/reference/get_v2-absence-periods",@@ -71,6 +112,30 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"name": "The name of the cost center.",},},+ "absence_types": {+ "description": "An absence type defined in Personio, such as vacation or sick leave, referenced by absence periods.",+ "docs_url": "https://developer.personio.de/reference/get_v2-absence-types",+ "columns": {+ "id": "Unique identifier for the absence type.",+ "name": "The name of the absence type.",+ "category": "The category of the absence type (e.g. PAID_VACATION, SICK_LEAVE).",+ "unit": "Unit absence periods of this type are measured in (DAY or HOUR).",+ },+ },+ "org_units": {+ "description": "An org unit (department or team) in Personio, with its place in the org unit hierarchy.",+ "docs_url": "https://developer.personio.de/reference/listorgunits",+ "columns": {+ "id": "Unique identifier for the org unit.",+ "type": "Type of the org unit (department or team).",+ "name": "The name of the org unit.",+ "abbreviation": "Short name of the org unit.",+ "description": "Description of the org unit.",+ "parent_id": "Identifier of the parent org unit. Null for a root org unit.",+ "parent_chain": "Ordered ancestor org unit ids, from the immediate parent up to the root.",+ "create_time": "Time at which the org unit was created.",+ },+ },"recruiting_jobs": {"description": "A recruiting job (open position) in Personio, with its category, department and hiring team.","docs_url": "https://developer.personio.de/reference/get_v2-recruiting-jobs",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.pyindex 6ccd56671969..9b30548d7e14 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py@@ -1,10 +1,13 @@import dataclasses-from datetime import UTC, date, datetime-from typing import Any, Optional+from collections.abc import Callable, Iterable, Iterator, Mapping+from datetime import UTC, date, datetime, timedelta+from typing import Any, Optional, castimport requestsfrom requests import Request, Response+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,@@ -13,27 +16,41 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resOAuth2Auth,OAuth2AuthRequestError,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ FanoutEndpointLike,+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponsePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (ClientConfig,Endpoint,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-from products.warehouse_sources.backend.temporal.data_imports.sources.personio.settings import PERSONIO_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.personio.settings import (+ COMPENSATIONS_LOOKBACK_MONTHS,+ PERSONIO_ENDPOINTS,+ PersonioEndpointConfig,+)PERSONIO_BASE_URL = "https://api.personio.de"TOKEN_URL = f"{PERSONIO_BASE_URL}/v2/auth/token"[email protected](frozen=True)class PersonioResumeConfig:# Personio v2 paginates via _meta.links.next.href, a self-contained URL (opaque cursor), so# the URL is all we persist. Optional/defaulted so previously saved state (which always carried# next_url) still parses via dataclass(**saved).next_url: Optional[str] = None+ # Endpoints read as several listings (org unit types, compensation month windows) record which+ # listing next_url belongs to.+ segment: Optional[str] = None+ # Fan-out endpoints checkpoint the shape `build_dependent_resource` emits.+ fanout_state: Optional[dict[str, Any]] = Nonedef _make_auth(client_id: str, client_secret: str) -> OAuth2Auth:@@ -60,6 +77,38 @@ def _format_updated_at(value: Any) -> str:return str(value)+def _month_start(value: Any) -> date:+ if isinstance(value, datetime):+ value = value.date()+ elif not isinstance(value, date):+ value = date.fromisoformat(str(value)[:10])+ return value.replace(day=1)+++def _add_months(month: date, months: int) -> date:+ index = month.year * 12 + month.month - 1 + months+ return date(index // 12, index % 12 + 1, 1)+++def _month_starts(first: date, last: date) -> list[date]:+ """The first day of each calendar month from the month of `first` through the month of `last`."""+ months = []+ month = _month_start(first)+ while month <= last:+ months.append(month)+ month = _add_months(month, 1)+ return months+++@frozen+class _Segment:+ """One listing of an endpoint that Personio serves as several listings."""++ key: str+ params: dict[str, Any]+ row_map: Callable[[dict[str, Any]], dict[str, Any]] | None = None++class PersonioPaginator(JSONResponsePaginator):"""Follow Personio's self-contained ``_meta.links.next.href`` link, stopping on an empty page.@@ -91,6 +140,111 @@ def validate_credentials(client_id: str, client_secret: str) -> bool:return True+def _client_config(auth: OAuth2Auth) -> ClientConfig:+ return {+ "base_url": PERSONIO_BASE_URL,+ "auth": auth,+ # Pin every request — including _meta.links.next.href links and the seeded resume URL — to+ # api.personio.de so a tampered next/resume link can't exfiltrate the bearer token. Disable+ # redirects so a 3xx can't bounce the authenticated request (and token) off-host either.+ "allowed_hosts": [],+ "allow_redirects": False,+ }+++def _segments(config: PersonioEndpointConfig) -> list[_Segment]:+ """The listings a multi-listing endpoint reads, in order."""+ if config.monthly_windows:+ today = datetime.now(UTC).date()+ first = _add_months(_month_start(today), -COMPENSATIONS_LOOKBACK_MONTHS)++ def add_period(period_start: date) -> Callable[[dict[str, Any]], dict[str, Any]]:+ return lambda row: {**row, "period_start": period_start}++ return [+ _Segment(+ key=start.isoformat(),+ params={+ "start_date": start.isoformat(),+ "end_date": (_add_months(start, 1) - timedelta(days=1)).isoformat(),+ },+ row_map=add_period(start),+ )+ for start in _month_starts(first, today)+ ]+ return [_Segment(key=variant["type"], params=dict(variant)) for variant in config.query_variants]+++def _segmented_resource(+ endpoint: str,+ config: PersonioEndpointConfig,+ auth: OAuth2Auth,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[PersonioResumeConfig],+ resume: Optional[PersonioResumeConfig],+) -> Resource:+ segments = _segments(config)+ start_index = 0+ if resume is not None and resume.segment is not None:+ start_index = next((i for i, segment in enumerate(segments) if segment.key == resume.segment), 0)++ def build(index: int) -> Resource:+ segment = segments[index]+ key = segment.key+ next_key = segments[index + 1].key if index + 1 < len(segments) else None+ initial_paginator_state: Optional[dict[str, Any]] = None+ if index == start_index and resume is not None and resume.segment == key and resume.next_url:+ initial_paginator_state = {"next_url": resume.next_url}++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state is not None and state.get("next_url") is not None:+ resumable_source_manager.save_state(PersonioResumeConfig(segment=key, next_url=state["next_url"]))+ elif next_key is not None:+ # The last page of this listing is on its way, so a resume starts at the next one.+ resumable_source_manager.save_state(PersonioResumeConfig(segment=next_key))++ rest_config: RESTAPIConfig = {+ "client": _client_config(auth),+ "resources": [+ {+ "name": endpoint,+ "endpoint": {+ "path": config.path,+ "params": {"limit": config.page_size, **segment.params},+ "data_selector": "_data",+ "paginator": PersonioPaginator(),+ },+ }+ ],+ }+ resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ )+ return resource.add_map(segment.row_map) if segment.row_map is not None else resource++ first = build(start_index)++ def pages() -> Iterator[list[dict[str, Any]]]:+ for index in range(start_index, len(segments)):+ resource = first if index == start_index else build(index)+ if index != start_index:+ resumable_source_manager.save_state(PersonioResumeConfig(segment=segments[index].key))+ # Every earlier listing is fully handed on, so a hand-off here loses nothing. Months with+ # no compensations otherwise send requests with no yield in between.+ resumable_source_manager.safe_point()+ yield from resource++ # As a `Resource`, the chain keeps the framework's checkpoints: each page reaches the pipeline+ # with its cursor already staged.+ return Resource(pages, name=endpoint, hints=first._hints)++def personio_source(client_id: str,client_secret: str,@@ -106,62 +260,88 @@ def personio_source(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- params: dict[str, Any] = {}- if config.page_size is not None:- params["limit"] = config.page_size+ incremental_params: dict[str, Any] = {}if (config.incremental_param is not Noneand should_use_incremental_fieldand db_incremental_field_last_value is not None):- params[config.incremental_param] = _format_updated_at(db_incremental_field_last_value)-- initial_paginator_state: Optional[dict[str, Any]] = None- if resume is not None and resume.next_url is not None:- initial_paginator_state = {"next_url": resume.next_url}-- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # The framework calls the hook AFTER a page is yielded and only while a next page remains,- # so a crash re-yields the last batch (merge dedupes on primary key) rather than skipping it.- if state is not None and state.get("next_url") is not None:- resumable_source_manager.save_state(PersonioResumeConfig(next_url=state["next_url"]))-- endpoint_config: Endpoint = {- "path": config.path,- "params": params,- "data_selector": "_data",- "paginator": PersonioPaginator(),- }- client_config: ClientConfig = {- "base_url": PERSONIO_BASE_URL,- "auth": auth,- # Pin every request — including _meta.links.next.href links and the seeded resume URL — to- # api.personio.de so a tampered next/resume link can't exfiltrate the bearer token. Disable- # redirects so a 3xx can't bounce the authenticated request (and token) off-host either.- "allowed_hosts": [],- "allow_redirects": False,- }- rest_config: RESTAPIConfig = {- "client": client_config,- "resources": [{"name": endpoint, "endpoint": endpoint_config}],- }- resource = rest_api_resource(- rest_config,- team_id,- job_id,- None,- resume_hook=save_checkpoint,- initial_paginator_state=initial_paginator_state,- )+ incremental_params[config.incremental_param] = _format_updated_at(db_incremental_field_last_value)++ items: Callable[[], Iterable[Any]]+ if config.fanout is not None:++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(PersonioResumeConfig(fanout_state=state))++ resource = build_dependent_resource(+ # Every endpoint in a fan-out is paginated, so page_size is set on both.+ endpoint_configs=cast(Mapping[str, FanoutEndpointLike], PERSONIO_ENDPOINTS),+ child_endpoint=endpoint,+ fanout=config.fanout,+ client_config=_client_config(auth),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ should_use_incremental_field=should_use_incremental_field,+ # The cursor rides in child_params_extra; the factory only switches the child to merge.+ incremental_config_factory=lambda _field: None,+ parent_endpoint_extra={"data_selector": "_data", "paginator": PersonioPaginator()},+ child_endpoint_extra={"data_selector": "_data", "paginator": PersonioPaginator()},+ child_params_extra=incremental_params,+ resume_hook=save_fanout_checkpoint,+ initial_paginator_state=resume.fanout_state if resume else None,+ )+ items = lambda: resource+ elif config.monthly_windows or config.query_variants:+ segmented = _segmented_resource(endpoint, config, auth, team_id, job_id, resumable_source_manager, resume)+ items = lambda: segmented+ else:+ params: dict[str, Any] = {**incremental_params}+ if config.page_size is not None:+ params["limit"] = config.page_size++ initial_paginator_state: Optional[dict[str, Any]] = None+ if resume is not None and resume.next_url is not None:+ initial_paginator_state = {"next_url": resume.next_url}++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ # The framework calls the hook AFTER a page is yielded and only while a next page remains,+ # so a crash re-yields the last batch (merge dedupes on primary key) rather than skipping it.+ if state is not None and state.get("next_url") is not None:+ resumable_source_manager.save_state(PersonioResumeConfig(next_url=state["next_url"]))++ endpoint_config: Endpoint = {+ "path": config.path,+ "params": params,+ "data_selector": "_data",+ "paginator": PersonioPaginator(),+ }+ rest_config: RESTAPIConfig = {+ "client": _client_config(auth),+ "resources": [{"name": endpoint, "endpoint": endpoint_config}],+ }+ single_resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ )+ items = lambda: single_resourcereturn SourceResponse(name=endpoint,- items=lambda: resource,- primary_keys=[config.primary_key],+ items=items,+ primary_keys=list(config.primary_keys),partition_count=1,partition_size=1,partition_mode="datetime" if config.partition_key else None,partition_format="month" if config.partition_key else None,partition_keys=[config.partition_key] if config.partition_key else None,- sort_mode="asc",+ # Fan-out rows arrive per person, newest-first, so they are not one ordered series.+ sort_mode=None if config.fanout is not None else "asc",)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.pyindex 076a7eecba08..4e4a56f3a578 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/settings.py@@ -1,8 +1,15 @@from dataclasses import dataclass, fieldfrom typing import Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType+# The compensations listing only answers for one calendar month at a time, so a sync walks month+# windows. This bounds how far back the walk starts when there is no watermark yet.+COMPENSATIONS_LOOKBACK_MONTHS = 24+_UPDATED_AT_INCREMENTAL_FIELDS: list[IncrementalField] = [{"label": "updated_at",@@ -19,7 +26,7 @@ class PersonioEndpointConfig:path: str# OAuth scope the endpoint needs (surfaced in the source caption).scope: str- primary_key: str = "id"+ primary_keys: tuple[str, ...] = ("id",)# Server-side updated_at filter param. Personio v2 is inconsistent per# endpoint: persons use strict `.gt`, the period endpoints use `.gte`.incremental_param: Optional[str] = None@@ -29,6 +36,13 @@ class PersonioEndpointConfig:page_size: Optional[int] = 100# Stable creation-time field used for datetime partitioning.partition_key: Optional[str] = None+ # Child endpoints listed once per parent row (e.g. employments per person).+ fanout: Optional[DependentEndpointConfig] = None+ default_incremental_field: Optional[str] = None+ # Endpoints with a required filter are listed once per variant (e.g. org units per type).+ query_variants: tuple[dict[str, str], ...] = ()+ # Walk calendar-month start_date/end_date windows (compensations).+ monthly_windows: bool = FalsePERSONIO_ENDPOINTS: dict[str, PersonioEndpointConfig] = {@@ -41,6 +55,35 @@ PERSONIO_ENDPOINTS: dict[str, PersonioEndpointConfig] = {page_size=50,partition_key="created_at",),+ # Employment ids are only documented per person, so the key carries the person id too. The+ # listing is per person and newest-first, so incremental runs merge without an ordered cursor.+ "employments": PersonioEndpointConfig(+ name="employments",+ path="/v2/persons/{person_id}/employments",+ scope="personio:persons:read",+ primary_keys=("person_id", "id"),+ incremental_param="updated_at.gt",+ incremental_fields=list(_UPDATED_AT_INCREMENTAL_FIELDS),+ page_size=50,+ partition_key="created_at",+ fanout=DependentEndpointConfig(+ parent_name="persons",+ resolve_param="person_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "person_id"},+ ),+ ),+ # One row per compensation per month window: recurring compensations repeat in every window,+ # so the window start is part of the key. Full refresh only: a compensation can be added to a+ # past month at any time, and the API has no change timestamp to find it with.+ "compensations": PersonioEndpointConfig(+ name="compensations",+ path="/v2/compensations",+ scope="personio:compensations:read",+ primary_keys=("period_start", "id"),+ monthly_windows=True,+ ),"absence_periods": PersonioEndpointConfig(name="absence_periods",path="/v2/absence-periods",@@ -67,6 +110,23 @@ PERSONIO_ENDPOINTS: dict[str, PersonioEndpointConfig] = {path="/v2/cost-centers",scope="personio:cost-centers:read",),+ # Absence types and org units are lookups with no server-side timestamp filter — full refresh only.+ "absence_types": PersonioEndpointConfig(+ name="absence_types",+ path="/v2/absence-types",+ scope="personio:absences:read",+ ),+ # `type` is a required filter, so departments and teams are two listings in one table.+ "org_units": PersonioEndpointConfig(+ name="org_units",+ path="/v2/org-units",+ scope="personio:org-units:read",+ primary_keys=("type", "id"),+ query_variants=(+ {"type": "department", "include_parent_chain": "true"},+ {"type": "team", "include_parent_chain": "true"},+ ),+ ),# Recruiting jobs support `updated_at.gt`, but their updated_at/created_at are# {"date-time", "timezone"} objects that can't serve as a cursor or partition column, and the# listing is newest-first — so full refresh only.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.pyindex 31943dc6893f..f65b4d030002 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/source.py@@ -67,7 +67,7 @@ class PersonioSource(ResumableSource[PersonioSourceConfig, PersonioResumeConfig]label="Personio",caption="""Enter your Personio API credentials to pull your Personio HR data into the PostHog Data warehouse.-An admin can create API credentials in Personio under Settings > Integrations > API credentials. Grant the read scopes for the datasets you want to sync (`personio:persons:read`, `personio:absences:read`, `personio:attendances:read`, `personio:salary-bands:read`, `personio:cost-centers:read`, `personio:recruiting:read`) and whitelist the employee attributes you need — attributes that aren't whitelisted are silently omitted from responses.""",+An admin can create API credentials in Personio under Settings > Integrations > API credentials. Grant the read scopes for the datasets you want to sync (`personio:persons:read`, `personio:absences:read`, `personio:attendances:read`, `personio:compensations:read`, `personio:org-units:read`, `personio:salary-bands:read`, `personio:cost-centers:read`, `personio:recruiting:read`) and whitelist the employee attributes you need — attributes that aren't whitelisted are silently omitted from responses.""",iconPath="/static/services/personio.png",docsUrl="https://posthog.com/docs/cdp/sources/personio",releaseStatus=ReleaseStatus.BETA,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.pyindex f891eaabcbe2..211eece6f8de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/personio/tests/test_personio.py@@ -5,6 +5,7 @@ from urllib.parse import parse_qsimport pytestfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,ScriptedResponse,SourceDriver,scripted_network,@@ -18,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.personio.pvalidate_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.personio.settings import (+ COMPENSATIONS_LOOKBACK_MONTHS,ENDPOINTS,PERSONIO_ENDPOINTS,)@@ -40,6 +42,19 @@ def _token_response(status_code: int = 200, expires_in: int = 86400) -> Scriptedreturn ScriptedResponse(status=status_code, json=body)+def _answer_empty(request: RecordedRequest) -> ScriptedResponse:+ return _token_response() if request.path == "/v2/auth/token" else _page([])+++def _this_month() -> date:+ return datetime.now(UTC).date().replace(day=1)+++def _months_before(month: date, months: int) -> date:+ index = month.year * 12 + month.month - 1 - months+ return date(index // 12, index % 12 + 1, 1)++def _driver() -> SourceDriver:return SourceDriver(PersonioSource(), PersonioSourceConfig(client_id="cid", client_secret="sec"))@@ -216,6 +231,100 @@ class TestGetRows:assert result.paths == ["/v2/auth/token", "/v2/persons"]assert result.saved_states == []+ @pytest.mark.parametrize("incremental", [False, True])+ def test_employments_fan_out_per_person(self, incremental: bool) -> None:+ result = _driver().run(+ "employments",+ [+ _token_response(),+ _page([{"id": "p1"}, {"id": "p2"}]),+ _page([{"id": "e1"}]),+ _page([{"id": "e2"}]),+ ],+ should_use_incremental_field=incremental,+ db_incremental_field_last_value=datetime(2024, 1, 2, tzinfo=UTC) if incremental else None,+ )++ assert result.raised is None+ assert result.paths == [+ "/v2/auth/token",+ "/v2/persons",+ "/v2/persons/p1/employments",+ "/v2/persons/p2/employments",+ ]+ assert [(row["person_id"], row["id"]) for row in result.rows] == [("p1", "e1"), ("p2", "e2")]+ # The cursor filters employments only: an employment update does not touch its person.+ assert result.requests[1].param("updated_at.gt") is None+ expected_filter = "2024-01-02T00:00:00Z" if incremental else None+ assert [request.param("updated_at.gt") for request in result.requests[2:]] == [expected_filter] * 2+ assert result.response is not None+ assert result.response.sort_mode is None++ def test_org_units_lists_each_type_and_stages_cursor_before_each_page(self) -> None:+ next_url = "https://api.personio.de/v2/org-units?type=department&cursor=p2"+ result = _driver().run(+ "org_units",+ [+ _token_response(),+ _page([{"id": "1", "type": "department"}], next_url=next_url),+ _page([{"id": "2", "type": "department"}]),+ _page([{"id": "1", "type": "team"}]),+ ],+ )++ assert result.raised is None+ assert [request.param("type") for request in result.requests[1:]] == ["department", "department", "team"]+ assert result.requests[1].param("include_parent_chain") == "true"+ assert [(row["type"], row["id"]) for row in result.rows] == [+ ("department", "1"),+ ("department", "2"),+ ("team", "1"),+ ]+ # Each page reaches the pipeline with the cursor past it, so a hand-off after a write never+ # re-reads that page. The last department page already points at the team listing.+ assert result.committed_states == [+ PersonioResumeConfig(segment="department", next_url=next_url),+ PersonioResumeConfig(segment="team"),+ ]++ def test_compensations_walks_month_windows(self) -> None:+ result = _driver().run(+ "compensations",+ lambda request: (+ _token_response()+ if request.path == "/v2/auth/token"+ else _page([{"id": "c1"}] if request.param("start_date") == _this_month().isoformat() else [])+ ),+ should_use_incremental_field=True,+ db_incremental_field_last_value=date(2099, 1, 1),+ )++ assert result.raised is None+ windows = [(request.param("start_date"), request.param("end_date")) for request in result.requests[1:]]+ # Always the full lookback: a compensation can land in a past month after it was synced.+ assert len(windows) == COMPENSATIONS_LOOKBACK_MONTHS + 1+ assert windows[0][0] == _months_before(_this_month(), COMPENSATIONS_LOOKBACK_MONTHS).isoformat()+ assert windows[-1][0] == _this_month().isoformat()+ for start, end in windows:+ assert start is not None and end is not None+ # Personio rejects a window longer than one month.+ assert start[:7] == end[:7] and start.endswith("-01")+ assert [(row["period_start"], row["id"]) for row in result.rows] == [(_this_month(), "c1")]++ def test_compensations_resume_skips_finished_windows(self) -> None:+ last_month = _months_before(_this_month(), 1)+ resume_url = "https://api.personio.de/v2/compensations?cursor=cur_resume"+ result = _driver().run(+ "compensations",+ [_token_response(), _page([{"id": "c9"}]), _page([])],+ resume_state=PersonioResumeConfig(segment=last_month.isoformat(), next_url=resume_url),+ )++ assert result.raised is None+ assert result.requests[1].url == resume_url+ assert result.requests[2].param("start_date") == _this_month().isoformat()+ assert [row["period_start"] for row in result.rows] == [last_month]+def test_rejects_off_host_next_url(self) -> None:# SSRF guard: a _meta.links.next.href pointing off api.personio.de is rejected before the# request (and its bearer token) leaves the process.@@ -243,14 +352,13 @@ class TestPersonioSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = PERSONIO_ENDPOINTS[endpoint]- result = _driver().run(endpoint, [_token_response(), _page([])])+ result = _driver().run(endpoint, _answer_empty)response = result.responseassert result.raised is Noneassert response is not Noneassert response.name == endpoint- assert response.primary_keys == [config.primary_key]- assert response.sort_mode == "asc"+ assert response.primary_keys == list(config.primary_keys)if config.partition_key:assert response.partition_mode == "datetime"assert response.partition_keys == [config.partition_key]