:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add four payfit tables from coverage gaps (#124790)
Tom Owers96c80dc4d819
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 210138414af6..0e3017f1e00e 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -842,8 +842,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/pardot/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/partnerstack/partnerstack.py1 products/warehouse_sources/backend/temporal/data_imports/sources/partnerstack/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/perigon/perigon.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 4f6e48b08acc..6a7f514f83f8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9203,10 +9203,10 @@ Today (4): `absences`, `collaborators`, `contracts`, `payslips`Diffed against: <https://developers.payfit.io/llms.txt>-- [ ] `/companies/{companyId}/contracts/time (worked time by contract)` — actual hours worked per pay period - the core time analytics table alongside absences (high)-- [ ] `/companies/{companyId}/accounting-v2` — payroll accounting entries broken down by analytical code and employee, the finance-facing dataset (high)-- [ ] `/companies/{companyId}` — company lookup resolving the company ID every other table is scoped by (medium)-- [ ] `/companies/{companyId}/payroll-status` — payroll state per pay period, needed to know which periods are final (medium)+- [x] `/companies/{companyId}/contracts/time (worked time by contract)` — actual hours worked per pay period - the core time analytics table alongside absences (high)+- [x] `/companies/{companyId}/accounting-v2` — payroll accounting entries broken down by analytical code and employee, the finance-facing dataset (high)+- [x] `/companies/{companyId}` — company lookup resolving the company ID every other table is scoped by (medium)+- [x] `/companies/{companyId}/payroll-status` — payroll state per pay period, needed to know which periods are final (medium)- [ ] `/companies/{companyId}/collaborators/meal-vouchers` — per-month meal voucher entitlements per collaborator (FR) (medium)- [ ] `/companies/{companyId}/health-insurance-contracts` — benefit contract lookup joined to employee contracts already synced (medium)- [ ] `/companies/{companyId}/provident-fund-contracts` — provident fund contract lookup, same join as health insurance (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.pyindex f720ea437827..bc65ebb8258e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.py@@ -69,4 +69,54 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"payslipUrl": "URL on the PayFit API to retrieve the payslip file.",},},+ "company": {+ "description": "The company the PayFit API key belongs to.",+ "docs_url": "https://developers.payfit.io/reference/get_company",+ "columns": {+ "id": "ID of the company.",+ "name": "Name of the company.",+ "country": "Country of the company.",+ "identificationNumber": "Identification number of the company (e.g. SIRET for France, CIF for Spain).",+ "address": "The company's address.",+ "city": "The city of the company.",+ "postalCode": "The postal code of the company.",+ "nbActiveContracts": "Number of current active contracts within the company.",+ },+ },+ "payroll_status": {+ "description": "The payroll status of the company for one pay period.",+ "docs_url": "https://developers.payfit.io/reference/get_companypayrollstatus",+ "columns": {+ "month": "First day of the pay period.",+ "status": "The payroll status: completed or not_completed.",+ "executionEndDate": "End execution date of the last payroll for the pay period.",+ },+ },+ "contracts_worked_time": {+ "description": "The time worked on each contract for one pay period. Only available for French companies.",+ "docs_url": "https://developers.payfit.io/reference/get_contracts_worked_time",+ "columns": {+ "month": "First day of the pay period.",+ "contractId": "The ID of the contract.",+ "effectiveWorkedTime": "Time effectively worked on the contract during the pay period.",+ "payedWorkedTime": "Worked time paid on the contract during the pay period.",+ "workTimeUnit": "Unit of the worked time values: days or hours.",+ },+ },+ "accounting_entries": {+ "description": "Payroll accounting entries of the company for one pay period, matching the accounting exports. Only available for French companies.",+ "docs_url": "https://developers.payfit.io/reference/get_companyaccountingv2",+ "columns": {+ "month": "First day of the pay period.",+ "lineNumber": "Position of the entry in the pay period's accounting data.",+ "operationDate": "Date of the accounting entry.",+ "analyticCodes": "Analytical accounting sections the collaborator belongs to, each with a type, code and value.",+ "employeeFullName": "The full name of the employee the entry concerns.",+ "contractId": "The contract ID of the employee the entry concerns.",+ "accountId": "The account's identifier in the accounting system.",+ "accountName": "Label of the accounting entry.",+ "debit": "Amount of the debit operation.",+ "credit": "Amount of the credit operation.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.pyindex 12787e9db490..2dc179fcbca8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py@@ -1,3 +1,4 @@+import datetimeimport dataclassesfrom collections.abc import Iteratorfrom typing import Any@@ -13,6 +14,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.payfit.settings import (+ INCREMENTAL_REREAD_MONTHS,+ MONTHLY_HISTORY_LOOKBACK,PAYFIT_ENDPOINTS,PayFitEndpointConfig,)@@ -38,12 +41,58 @@ class PayFitInvalidTokenError(Exception):pass[email protected](frozen=True)class PayFitResumeConfig:# `nextPageToken` cursor for the next page. For `payslips` this is the cursor of the# *collaborators* page being fanned out over. `None` means start from the first page. A crashed# sync resumes from the page after the last one yielded; merge dedupes on the primary key.next_page_token: str | None = None+ # Pay period (`YYYYMM`) to resume at, for month-scoped endpoints. `next_page_token` then pages+ # within that month.+ month: str | None = None+++def _today() -> datetime.date:+ return datetime.datetime.now(datetime.UTC).date()+++def _period(month: datetime.date) -> str:+ return f"{month.year:04d}{month.month:02d}"+++def _parse_period(period: str) -> datetime.date:+ return datetime.date(int(period[:4]), int(period[4:6]), 1)+++def _next_month(month: datetime.date) -> datetime.date:+ return datetime.date(month.year + month.month // 12, month.month % 12 + 1, 1)+++def _to_date(value: Any) -> datetime.date | None:+ if value is None:+ return None+ if isinstance(value, datetime.datetime):+ return value.date()+ if isinstance(value, datetime.date):+ return value+ try:+ return datetime.date.fromisoformat(str(value)[:10])+ except ValueError:+ return None+++def resolve_first_month(+ today: datetime.date,+ history_start: datetime.datetime | None,+ db_incremental_field_last_value: Any,+) -> datetime.date:+ watermark = _to_date(db_incremental_field_last_value)+ if watermark is not None:+ month = min(watermark, today).replace(day=1)+ months_back = month.year * 12 + month.month - 1 - INCREMENTAL_REREAD_MONTHS+ return datetime.date(months_back // 12, months_back % 12 + 1, 1)+ start = history_start.date() if history_start is not None else today - MONTHLY_HISTORY_LOOKBACK+ return start.replace(day=1)def _headers(api_key: str) -> dict[str, str]:@@ -117,6 +166,7 @@ def _fetch_page(next_page_token: str | None,extra_params: dict[str, str],logger: FilteringBoundLogger,+ missing_ok: bool = False,) -> tuple[list[dict[str, Any]], str | None]:params: dict[str, Any] = {"maxResults": PAGE_SIZE, **extra_params}if next_page_token is not None:@@ -131,6 +181,9 @@ def _fetch_page(if response.status_code == 429 or response.status_code >= 500:raise PayFitRetryableError(f"PayFit API error (retryable): status={response.status_code}, path={path}")+ if response.status_code == 404 and missing_ok:+ return [], None+if not response.ok:logger.error(f"PayFit API error: status={response.status_code}, body={response.text}, path={path}")response.raise_for_status()@@ -146,6 +199,39 @@ def _fetch_page(return rows, next_token or None+@retry_with_deadline(+ retry=retry_if_exception_type((PayFitRetryableError, requests.ReadTimeout, requests.ConnectionError)),+ stop=stop_after_attempt(5),+ wait=wait_exponential_jitter(initial=1, max=30),+ reraise=True,+)+def _fetch_unpaginated(+ session: requests.Session,+ path: str,+ params: dict[str, str],+ logger: FilteringBoundLogger,+ missing_ok: bool = False,+) -> list[dict[str, Any]]:+ response = session.get(f"{PAYFIT_BASE_URL}{path}", params=params, timeout=REQUEST_TIMEOUT_SECONDS)++ if response.status_code == 429 or response.status_code >= 500:+ raise PayFitRetryableError(f"PayFit API error (retryable): status={response.status_code}, path={path}")++ if response.status_code == 404 and missing_ok:+ return []++ if not response.ok:+ logger.error(f"PayFit API error: status={response.status_code}, body={response.text}, path={path}")+ response.raise_for_status()++ data = response.json()+ if isinstance(data, dict):+ return [data]+ if isinstance(data, list) and all(isinstance(row, dict) for row in data):+ return data+ raise PayFitRetryableError(f"PayFit returned an unexpected payload for {path}: {type(data).__name__}")++@retry_with_deadline(retry=retry_if_exception_type((PayFitRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),@@ -182,6 +268,8 @@ def get_rows(endpoint: str,logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[PayFitResumeConfig],+ history_start: datetime.datetime | None = None,+ db_incremental_field_last_value: Any = None,) -> Iterator[list[dict[str, Any]]]:config = PAYFIT_ENDPOINTS[endpoint]session = make_tracked_session(headers=_headers(api_key), redact_values=(api_key,))@@ -192,10 +280,35 @@ def get_rows(if resume and resume.next_page_token is not None:logger.debug(f"PayFit: resuming {endpoint} from page token {next_page_token}")+ if config.monthly:+ today = _today()+ first_month = (+ _parse_period(resume.month)+ if resume and resume.month+ else resolve_first_month(today, history_start, db_incremental_field_last_value)+ )+ yield from _get_monthly_rows(+ session,+ company_id,+ config,+ first_month,+ today.replace(day=1),+ next_page_token if resume and resume.month else None,+ logger,+ resumable_source_manager,+ )+ return+if config.fan_out_over_collaborators:yield from _get_payslip_rows(session, company_id, next_page_token, logger, resumable_source_manager)return+ if not config.paginated or config.data_key is None:+ rows = _fetch_unpaginated(session, f"/companies/{company_id}{config.path}", {}, logger)+ if rows:+ yield rows+ return+path = f"/companies/{company_id}{config.path}"while True:rows, next_token = _fetch_page(session, path, config.data_key, next_page_token, config.extra_params, logger)@@ -241,11 +354,52 @@ def _get_payslip_rows(resumable_source_manager.save_state(PayFitResumeConfig(next_page_token=next_page_token))+def _get_monthly_rows(+ session: requests.Session,+ company_id: str,+ config: PayFitEndpointConfig,+ month: datetime.date,+ last_month: datetime.date,+ next_page_token: str | None,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PayFitResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ path = f"/companies/{company_id}{config.path}"+ while month <= last_month:+ params = {**config.extra_params, "date": _period(month)}+ # A 404 for one pay period, such as a month before the company joined PayFit, skips that+ # month rather than failing the sync.+ if config.paginated and config.data_key is not None:+ rows, next_token = _fetch_page(+ session, path, config.data_key, next_page_token, params, logger, missing_ok=True+ )+ else:+ rows, next_token = _fetch_unpaginated(session, path, params, logger, missing_ok=True), None++ batch = [+ {**row, "month": month, **({"lineNumber": index} if config.number_rows else {})}+ for index, row in enumerate(rows)+ ]++ if next_token and rows:+ next_page_token = next_token+ else:+ month, next_page_token = _next_month(month), None+ resumable_source_manager.save_state(PayFitResumeConfig(next_page_token=next_page_token, month=_period(month)))++ if batch:+ yield batch+ else:+ resumable_source_manager.safe_point()++def payfit_source(api_key: str,endpoint: str,logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[PayFitResumeConfig],+ history_start: datetime.datetime | None = None,+ db_incremental_field_last_value: Any = None,) -> SourceResponse:config = PAYFIT_ENDPOINTS[endpoint]@@ -256,6 +410,8 @@ def payfit_source(endpoint=endpoint,logger=logger,resumable_source_manager=resumable_source_manager,+ history_start=history_start,+ db_incremental_field_last_value=db_incremental_field_last_value,),primary_keys=config.primary_keys,partition_count=1,@@ -295,7 +451,11 @@ def check_schema_access(api_key: str, schema_name: str) -> tuple[bool, str | Nonreturn False, f"Could not validate PayFit API key: {e}"probe = _probe_endpoint(endpoint_config)- probe_params: dict[str, Any] = {"maxResults": 1, **probe.extra_params}+ probe_params: dict[str, Any] = {**probe.extra_params}+ if probe.paginated:+ probe_params["maxResults"] = 1+ if probe.monthly:+ probe_params["date"] = _period(_today())try:response = session.get(f"{PAYFIT_BASE_URL}/companies/{company_id}{probe.path}",@@ -308,7 +468,12 @@ def check_schema_access(api_key: str, schema_name: str) -> tuple[bool, str | Nonif response.status_code == 401:return False, "Invalid PayFit API key"if response.status_code == 403:+ if probe.scope is None:+ return False, f"Your PayFit API key cannot read `{schema_name}`"return False, f"Your PayFit API key is missing the `{probe.scope}` scope required for `{schema_name}`"+ # A month-scoped endpoint answers 404 for a pay period with no data, which still proves access.+ if response.status_code == 404 and probe.monthly:+ return True, Noneif not response.ok:return False, f"PayFit returned HTTP {response.status_code}"return True, Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.pyindex c5ce4bd918a4..04e73955866b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py@@ -1,25 +1,53 @@+import datetimefrom dataclasses import dataclass, field+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+# Month-scoped endpoints take a required `date=YYYYMM` and return one pay period per request, so a+# first sync walks back this far and later full refreshes keep the same start.+MONTHLY_HISTORY_LOOKBACK = datetime.timedelta(days=3 * 365)++# A pay period's payroll can stay open after later months have rows, so an incremental run reads+# this many months before the watermark again.+INCREMENTAL_REREAD_MONTHS = 3++MONTH_INCREMENTAL_FIELDS: list[IncrementalField] = [+ {+ "label": "month",+ "type": IncrementalFieldType.Date,+ "field": "month",+ "field_type": IncrementalFieldType.Date,+ }+]+++@dataclass(frozen=True)class PayFitEndpointConfig:name: str# Appended to /companies/{companyId} on the Partner API host.path: str# Key wrapping the rows in the JSON response, e.g. {"collaborators": [...], "meta": {...}}.- data_key: str- # OAuth scope the API key must carry to read this endpoint.- scope: str+ # None means the body itself is the row (an object) or the rows (an array).+ data_key: str | None+ # OAuth scope the API key must carry to read this endpoint, or None when it needs none.+ scope: str | Noneprimary_keys: list[str] = field(default_factory=lambda: ["id"])# Payslips have no list-all endpoint; they are fetched per collaborator.fan_out_over_collaborators: bool = Falseextra_params: dict[str, str] = field(default_factory=dict)+ paginated: bool = True+ # Fetched once per pay period, with each row stamped with a `month` date column.+ monthly: bool = False+ incremental_fields: list[IncrementalField] = field(default_factory=list)+ # Stamp each row with its position in the response, for rows that carry no ID.+ number_rows: bool = False-# PayFit Partner API list endpoints. All are full-refresh only: PayFit exposes no server-side-# `updated_after`/`since` filter on any of them, so there is no incremental cursor to advance.-# Country-specific endpoints (meal vouchers, health insurance and provident funds, accounting CSV-# exports) are intentionally not synced.+# PayFit Partner API endpoints. List endpoints expose no server-side `updated_after`/`since`+# filter, so they are full refresh only. Month-scoped endpoints can sync incrementally by month,+# because the requested pay period is the server-side filter. Accounting entries and worked time+# are French-only endpoints. Meal vouchers, health insurance and provident funds, and the+# accounting CSV export are not synced.PAYFIT_ENDPOINTS: dict[str, PayFitEndpointConfig] = {"collaborators": PayFitEndpointConfig(name="collaborators",@@ -52,8 +80,49 @@ PAYFIT_ENDPOINTS: dict[str, PayFitEndpointConfig] = {primary_keys=["collaboratorId", "payslipId"],fan_out_over_collaborators=True,),+ "company": PayFitEndpointConfig(+ name="company",+ path="",+ data_key=None,+ scope=None,+ paginated=False,+ ),+ "payroll_status": PayFitEndpointConfig(+ name="payroll_status",+ path="/payroll-status",+ data_key=None,+ scope=None,+ primary_keys=["month"],+ paginated=False,+ monthly=True,+ incremental_fields=MONTH_INCREMENTAL_FIELDS,+ ),+ "contracts_worked_time": PayFitEndpointConfig(+ name="contracts_worked_time",+ path="/contracts/time",+ data_key="contracts",+ scope="time:read",+ primary_keys=["month", "contractId"],+ monthly=True,+ incremental_fields=MONTH_INCREMENTAL_FIELDS,+ ),+ "accounting_entries": PayFitEndpointConfig(+ name="accounting_entries",+ path="/accounting-v2",+ data_key=None,+ scope="accounting:read",+ # Entries carry no ID, and one contract and account can repeat across analytic codes, so+ # rows are keyed by their position in the month's export. That key is only stable within+ # one read, which is why this table is full refresh only.+ primary_keys=["month", "lineNumber"],+ paginated=False,+ monthly=True,+ number_rows=True,+ ),}ENDPOINTS = tuple(PAYFIT_ENDPOINTS.keys())-INCREMENTAL_FIELDS: dict[str, list[dict[str, str]]] = {}+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ name: config.incremental_fields for name, config in PAYFIT_ENDPOINTS.items() if config.incremental_fields+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.pyindex 0098cb17428f..541af4e2dd8d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.py@@ -1,3 +1,4 @@+import datetimefrom typing import Optional, castfrom products.warehouse_sources.backend.facade.codebase_detection import detect@@ -23,7 +24,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.paypayfit_source,validate_credentials,)-from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.settings import ENDPOINTS, PAYFIT_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.settings import (+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ MONTHLY_HISTORY_LOOKBACK,+ PAYFIT_ENDPOINTS,+)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -47,7 +53,7 @@ class PayFitSource(ResumableSource[PayFitSourceConfig, PayFitResumeConfig]):releaseStatus=ReleaseStatus.BETA,caption="""Enter your PayFit API key to pull your HR and payroll data into the PostHog Data warehouse.-You can create an API key from the **API access** tab on the [integrations page](https://app.payfit.com/integrations/hub/api) of your PayFit admin account. Grant it the `collaborators:read`, `contracts:read`, `time:read`, and `contracts:payslips:read` scopes so every table can sync.+You can create an API key from the **API access** tab on the [integrations page](https://app.payfit.com/integrations/hub/api) of your PayFit admin account. Grant it the `collaborators:read`, `contracts:read`, `time:read`, `contracts:payslips:read`, and `accounting:read` scopes so every table can sync. The `accounting_entries` and `contracts_worked_time` tables are only available for companies based in France.""",iconPath="/static/services/payfit.png",docsUrl="https://posthog.com/docs/cdp/sources/payfit",@@ -76,7 +82,7 @@ You can create an API key from the **API access** tab on the [integrations page]def get_non_retryable_errors(self) -> dict[str, str | None]:return {"401 Client Error: Unauthorized for url: https://partner-api.payfit.com": "Your PayFit API key is invalid or has been revoked. Create a new key under Integrations → API access in PayFit, then reconnect.",- "403 Client Error: Forbidden for url: https://partner-api.payfit.com": "Your PayFit API key is missing a scope this table needs. Create a key with the collaborators:read, contracts:read, time:read, and contracts:payslips:read scopes, then reconnect.",+ "403 Client Error: Forbidden for url: https://partner-api.payfit.com": "Your PayFit API key is missing a scope this table needs. Create a key with the collaborators:read, contracts:read, time:read, contracts:payslips:read, and accounting:read scopes, then reconnect.","PayFit API key is inactive or invalid": "Your PayFit API key is invalid or has been revoked. Create a new key under Integrations → API access in PayFit, then reconnect.",}@@ -89,14 +95,12 @@ You can create an API key from the **API access** tab on the [integrations page]force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Every endpoint is full refresh only — PayFit exposes no server-side updated_after/since- # filter on any list endpoint, so there is no incremental cursor to advance.schemas = [SourceSchema(name=endpoint,- supports_incremental=False,+ supports_incremental=endpoint in INCREMENTAL_FIELDS,supports_append=False,- incremental_fields=[],+ incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),)for endpoint in ENDPOINTS]@@ -118,6 +122,14 @@ You can create an API key from the **API access** tab on the [integrations page]return check_schema_access(config.api_key, schema_name)return validate_credentials(config.api_key)+ def history_lookback_for_schema(+ self, schema_name: str, config: PayFitSourceConfig | None = None+ ) -> datetime.timedelta | None:+ endpoint_config = PAYFIT_ENDPOINTS.get(schema_name)+ if endpoint_config is not None and endpoint_config.monthly:+ return MONTHLY_HISTORY_LOOKBACK+ return None+def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[PayFitResumeConfig]:return ResumableSourceManager[PayFitResumeConfig](inputs, PayFitResumeConfig)@@ -135,4 +147,8 @@ You can create an API key from the **API access** tab on the [integrations page]endpoint=inputs.schema_name,logger=inputs.logger,resumable_source_manager=resumable_source_manager,+ history_start=inputs.history_start,+ db_incremental_field_last_value=(+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None+ ),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.pyindex 52f5591c9240..484eea0fe288 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.py@@ -1,6 +1,9 @@+import datetime+from collections.abc import Iteratorfrom typing import Anyimport pytest+import time_machinefrom unittest.mock import MagicMockimport requests@@ -10,6 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.payfit import PayFitSourceConfig@@ -77,6 +81,126 @@ class TestGetRows:assert result.rows == [{"id": "5"}]assert result.params("nextPageToken") == [None, "tok-99"]+ def test_company_object_is_one_row(self) -> None:+ company = {"id": COMPANY_ID, "name": "Example company", "country": "FR", "nbActiveContracts": 3}+ result = _driver().run("company", [_introspection_response(), ScriptedResponse(json=company)])++ assert result.raised is None+ assert result.items == [[company]]+ assert result.paths == ["/introspect", f"/companies/{COMPANY_ID}"]+++class TestGetMonthlyRows:+ @pytest.fixture(autouse=True)+ def _frozen_today(self) -> Iterator[None]:+ with time_machine.travel(datetime.datetime(2026, 3, 15, 12, tzinfo=datetime.UTC), tick=False):+ yield++ def test_walks_pay_periods_paging_within_each_month(self) -> None:+ result = _driver().run(+ "contracts_worked_time",+ [+ _introspection_response(),+ ScriptedResponse(json={"contracts": [{"contractId": "c1"}], "meta": {"nextPageToken": "tok-2"}}),+ ScriptedResponse(json={"contracts": [{"contractId": "c2"}], "meta": {"nextPageToken": None}}),+ # A pay period PayFit holds nothing for answers 404 and is skipped.+ ScriptedResponse(status=404, json={"error": "Ressource not found"}),+ ScriptedResponse(json={"contracts": [{"contractId": "c1"}], "meta": {}}),+ ],+ history_start=datetime.datetime(2026, 1, 10, tzinfo=datetime.UTC),+ )++ assert result.raised is None+ assert result.items == [+ [{"contractId": "c1", "month": datetime.date(2026, 1, 1)}],+ [{"contractId": "c2", "month": datetime.date(2026, 1, 1)}],+ [{"contractId": "c1", "month": datetime.date(2026, 3, 1)}],+ ]+ assert result.params("date") == [None, "202601", "202601", "202602", "202603"]+ assert result.params("nextPageToken") == [None, None, "tok-2", None, None]+ assert result.saved_states == [+ PayFitResumeConfig(next_page_token="tok-2", month="202601"),+ PayFitResumeConfig(month="202602"),+ PayFitResumeConfig(month="202603"),+ PayFitResumeConfig(month="202604"),+ ]++ @parameterized.expand(+ [+ (+ "payroll_status_object",+ "payroll_status",+ {"status": "completed", "executionEndDate": "2026-03-10T11:56:24.221Z"},+ [{"status": "completed", "executionEndDate": "2026-03-10T11:56:24.221Z"}],+ ),+ (+ "accounting_array_numbers_rows",+ "accounting_entries",+ [{"accountId": "6411000", "debit": 10}, {"accountId": "6411000", "debit": 10}],+ [+ {"accountId": "6411000", "debit": 10, "lineNumber": 0},+ {"accountId": "6411000", "debit": 10, "lineNumber": 1},+ ],+ ),+ ]+ )+ def test_unpaginated_bodies_become_rows(+ self, _name: str, endpoint: str, body: Any, expected: list[dict[str, Any]]+ ) -> None:+ result = _driver().run(+ endpoint,+ [_introspection_response(), ScriptedResponse(json=body)],+ history_start=datetime.datetime(2026, 3, 1, tzinfo=datetime.UTC),+ )++ assert result.raised is None+ assert result.rows == [{**row, "month": datetime.date(2026, 3, 1)} for row in expected]+ assert result.params("maxResults") == [None, None]++ @parameterized.expand(+ [+ ("history_start", {"history_start": datetime.datetime(2026, 1, 20, tzinfo=datetime.UTC)}, "202601"),+ ("no_history_start_uses_lookback", {}, "202303"),+ (+ "incremental_rereads_months_before_watermark",+ {+ "history_start": datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC),+ "incremental_field": "month",+ "db_incremental_field_last_value": datetime.date(2026, 2, 1),+ },+ "202511",+ ),+ ]+ )+ def test_first_requested_pay_period(self, _name: str, inputs: dict[str, Any], expected_first: str) -> None:+ result = _driver().run(+ "payroll_status",+ route({"/introspect": [_introspection_response()], "/payroll-status": [ScriptedResponse(status=404)] * 40}),+ **inputs,+ )++ assert result.raised is None+ dates = result.params("date")[1:]+ assert dates[0] == expected_first+ assert dates[-1] == "202603"++ def test_resumes_from_saved_month_and_page_token(self) -> None:+ result = _driver().run(+ "contracts_worked_time",+ [+ _introspection_response(),+ ScriptedResponse(json={"contracts": [{"contractId": "c9"}], "meta": {}}),+ ScriptedResponse(json={"contracts": [], "meta": {}}),+ ],+ resume_state=PayFitResumeConfig(next_page_token="tok-9", month="202602"),+ history_start=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC),+ )++ assert result.raised is None+ assert result.rows == [{"contractId": "c9", "month": datetime.date(2026, 2, 1)}]+ assert result.params("date") == [None, "202602", "202603"]+ assert result.params("nextPageToken") == [None, "tok-9", None]+class TestGetPayslipRows:def test_fans_out_over_collaborator_pages_and_stamps_parent_id(self) -> None:@@ -292,6 +416,14 @@ class TestCheckSchemaAccess:# Payslip paths need a collaborator id, so the scope probe targets the collaborators list.assert network.requests_log[1].url == f"{PAYFIT_BASE_URL}/companies/{COMPANY_ID}/collaborators?maxResults=1"+ @time_machine.travel(datetime.datetime(2026, 3, 15, tzinfo=datetime.UTC), tick=False)+ def test_monthly_probe_accepts_empty_pay_period(self) -> None:+ with scripted_network([_introspection_response(), ScriptedResponse(status=404, json={})]) as network:+ valid, _message = check_schema_access("payfit-key", "accounting_entries")++ assert valid is True+ assert network.requests_log[1].url == f"{PAYFIT_BASE_URL}/companies/{COMPANY_ID}/accounting-v2?date=202603"+def test_inactive_token_fails_before_probing(self) -> None:with scripted_network([ScriptedResponse(json={"active": False})]) as network:valid, message = check_schema_access("payfit-key", "contracts")@@ -302,13 +434,11 @@ class TestCheckSchemaAccess:class TestPayFitSourceResponse:- @parameterized.expand([(e,) for e in ENDPOINTS])+ @parameterized.expand([(e,) for e in ENDPOINTS if not PAYFIT_ENDPOINTS[e].monthly])def test_source_response_shape(self, endpoint: str) -> None:data_key = "collaborators" if endpoint == "payslips" else PAYFIT_ENDPOINTS[endpoint].data_key- result = _driver().run(- endpoint,- [_introspection_response(), ScriptedResponse(json={data_key: [], "meta": {}})],- )+ body: dict[str, Any] = {data_key: [], "meta": {}} if data_key else {"id": COMPANY_ID}+ result = _driver().run(endpoint, [_introspection_response(), ScriptedResponse(json=body)])assert result.raised is Noneresponse = result.response