Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add four payfit tables from coverage gaps (#124790)

Tom Owers96c80dc4d819
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 210138414af6..0e3017f1e00e 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -842,8 +842,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/pardot/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/partnerstack/partnerstack.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/partnerstack/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/perigon/perigon.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 4f6e48b08acc..6a7f514f83f8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9203,10 +9203,10 @@ Today (4): `absences`, `collaborators`, `contracts`, `payslips`
Diffed against: <https://developers.payfit.io/llms.txt>
-- [ ] `/companies/{companyId}/contracts/time (worked time by contract)` — actual hours worked per pay period - the core time analytics table alongside absences (high)
-- [ ] `/companies/{companyId}/accounting-v2` — payroll accounting entries broken down by analytical code and employee, the finance-facing dataset (high)
-- [ ] `/companies/{companyId}` — company lookup resolving the company ID every other table is scoped by (medium)
-- [ ] `/companies/{companyId}/payroll-status` — payroll state per pay period, needed to know which periods are final (medium)
+- [x] `/companies/{companyId}/contracts/time (worked time by contract)` — actual hours worked per pay period - the core time analytics table alongside absences (high)
+- [x] `/companies/{companyId}/accounting-v2` — payroll accounting entries broken down by analytical code and employee, the finance-facing dataset (high)
+- [x] `/companies/{companyId}` — company lookup resolving the company ID every other table is scoped by (medium)
+- [x] `/companies/{companyId}/payroll-status` — payroll state per pay period, needed to know which periods are final (medium)
- [ ] `/companies/{companyId}/collaborators/meal-vouchers` — per-month meal voucher entitlements per collaborator (FR) (medium)
- [ ] `/companies/{companyId}/health-insurance-contracts` — benefit contract lookup joined to employee contracts already synced (medium)
- [ ] `/companies/{companyId}/provident-fund-contracts` — provident fund contract lookup, same join as health insurance (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.py
index f720ea437827..bc65ebb8258e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/canonical_descriptions.py
@@ -69,4 +69,54 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"payslipUrl": "URL on the PayFit API to retrieve the payslip file.",
},
},
+ "company": {
+ "description": "The company the PayFit API key belongs to.",
+ "docs_url": "https://developers.payfit.io/reference/get_company",
+ "columns": {
+ "id": "ID of the company.",
+ "name": "Name of the company.",
+ "country": "Country of the company.",
+ "identificationNumber": "Identification number of the company (e.g. SIRET for France, CIF for Spain).",
+ "address": "The company's address.",
+ "city": "The city of the company.",
+ "postalCode": "The postal code of the company.",
+ "nbActiveContracts": "Number of current active contracts within the company.",
+ },
+ },
+ "payroll_status": {
+ "description": "The payroll status of the company for one pay period.",
+ "docs_url": "https://developers.payfit.io/reference/get_companypayrollstatus",
+ "columns": {
+ "month": "First day of the pay period.",
+ "status": "The payroll status: completed or not_completed.",
+ "executionEndDate": "End execution date of the last payroll for the pay period.",
+ },
+ },
+ "contracts_worked_time": {
+ "description": "The time worked on each contract for one pay period. Only available for French companies.",
+ "docs_url": "https://developers.payfit.io/reference/get_contracts_worked_time",
+ "columns": {
+ "month": "First day of the pay period.",
+ "contractId": "The ID of the contract.",
+ "effectiveWorkedTime": "Time effectively worked on the contract during the pay period.",
+ "payedWorkedTime": "Worked time paid on the contract during the pay period.",
+ "workTimeUnit": "Unit of the worked time values: days or hours.",
+ },
+ },
+ "accounting_entries": {
+ "description": "Payroll accounting entries of the company for one pay period, matching the accounting exports. Only available for French companies.",
+ "docs_url": "https://developers.payfit.io/reference/get_companyaccountingv2",
+ "columns": {
+ "month": "First day of the pay period.",
+ "lineNumber": "Position of the entry in the pay period's accounting data.",
+ "operationDate": "Date of the accounting entry.",
+ "analyticCodes": "Analytical accounting sections the collaborator belongs to, each with a type, code and value.",
+ "employeeFullName": "The full name of the employee the entry concerns.",
+ "contractId": "The contract ID of the employee the entry concerns.",
+ "accountId": "The account's identifier in the accounting system.",
+ "accountName": "Label of the accounting entry.",
+ "debit": "Amount of the debit operation.",
+ "credit": "Amount of the credit operation.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py
index 12787e9db490..2dc179fcbca8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py
@@ -1,3 +1,4 @@
+import datetime
import dataclasses
from collections.abc import Iterator
from typing import Any
@@ -13,6 +14,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.settings import (
+ INCREMENTAL_REREAD_MONTHS,
+ MONTHLY_HISTORY_LOOKBACK,
PAYFIT_ENDPOINTS,
PayFitEndpointConfig,
)
@@ -38,12 +41,58 @@ class PayFitInvalidTokenError(Exception):
pass
[email protected](frozen=True)
class PayFitResumeConfig:
# `nextPageToken` cursor for the next page. For `payslips` this is the cursor of the
# *collaborators* page being fanned out over. `None` means start from the first page. A crashed
# sync resumes from the page after the last one yielded; merge dedupes on the primary key.
next_page_token: str | None = None
+ # Pay period (`YYYYMM`) to resume at, for month-scoped endpoints. `next_page_token` then pages
+ # within that month.
+ month: str | None = None
+
+
+def _today() -> datetime.date:
+ return datetime.datetime.now(datetime.UTC).date()
+
+
+def _period(month: datetime.date) -> str:
+ return f"{month.year:04d}{month.month:02d}"
+
+
+def _parse_period(period: str) -> datetime.date:
+ return datetime.date(int(period[:4]), int(period[4:6]), 1)
+
+
+def _next_month(month: datetime.date) -> datetime.date:
+ return datetime.date(month.year + month.month // 12, month.month % 12 + 1, 1)
+
+
+def _to_date(value: Any) -> datetime.date | None:
+ if value is None:
+ return None
+ if isinstance(value, datetime.datetime):
+ return value.date()
+ if isinstance(value, datetime.date):
+ return value
+ try:
+ return datetime.date.fromisoformat(str(value)[:10])
+ except ValueError:
+ return None
+
+
+def resolve_first_month(
+ today: datetime.date,
+ history_start: datetime.datetime | None,
+ db_incremental_field_last_value: Any,
+) -> datetime.date:
+ watermark = _to_date(db_incremental_field_last_value)
+ if watermark is not None:
+ month = min(watermark, today).replace(day=1)
+ months_back = month.year * 12 + month.month - 1 - INCREMENTAL_REREAD_MONTHS
+ return datetime.date(months_back // 12, months_back % 12 + 1, 1)
+ start = history_start.date() if history_start is not None else today - MONTHLY_HISTORY_LOOKBACK
+ return start.replace(day=1)
def _headers(api_key: str) -> dict[str, str]:
@@ -117,6 +166,7 @@ def _fetch_page(
next_page_token: str | None,
extra_params: dict[str, str],
logger: FilteringBoundLogger,
+ missing_ok: bool = False,
) -> tuple[list[dict[str, Any]], str | None]:
params: dict[str, Any] = {"maxResults": PAGE_SIZE, **extra_params}
if next_page_token is not None:
@@ -131,6 +181,9 @@ def _fetch_page(
if response.status_code == 429 or response.status_code >= 500:
raise PayFitRetryableError(f"PayFit API error (retryable): status={response.status_code}, path={path}")
+ if response.status_code == 404 and missing_ok:
+ return [], None
+
if not response.ok:
logger.error(f"PayFit API error: status={response.status_code}, body={response.text}, path={path}")
response.raise_for_status()
@@ -146,6 +199,39 @@ def _fetch_page(
return rows, next_token or None
+@retry_with_deadline(
+ retry=retry_if_exception_type((PayFitRetryableError, requests.ReadTimeout, requests.ConnectionError)),
+ stop=stop_after_attempt(5),
+ wait=wait_exponential_jitter(initial=1, max=30),
+ reraise=True,
+)
+def _fetch_unpaginated(
+ session: requests.Session,
+ path: str,
+ params: dict[str, str],
+ logger: FilteringBoundLogger,
+ missing_ok: bool = False,
+) -> list[dict[str, Any]]:
+ response = session.get(f"{PAYFIT_BASE_URL}{path}", params=params, timeout=REQUEST_TIMEOUT_SECONDS)
+
+ if response.status_code == 429 or response.status_code >= 500:
+ raise PayFitRetryableError(f"PayFit API error (retryable): status={response.status_code}, path={path}")
+
+ if response.status_code == 404 and missing_ok:
+ return []
+
+ if not response.ok:
+ logger.error(f"PayFit API error: status={response.status_code}, body={response.text}, path={path}")
+ response.raise_for_status()
+
+ data = response.json()
+ if isinstance(data, dict):
+ return [data]
+ if isinstance(data, list) and all(isinstance(row, dict) for row in data):
+ return data
+ raise PayFitRetryableError(f"PayFit returned an unexpected payload for {path}: {type(data).__name__}")
+
+
@retry_with_deadline(
retry=retry_if_exception_type((PayFitRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
@@ -182,6 +268,8 @@ def get_rows(
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[PayFitResumeConfig],
+ history_start: datetime.datetime | None = None,
+ db_incremental_field_last_value: Any = None,
) -> Iterator[list[dict[str, Any]]]:
config = PAYFIT_ENDPOINTS[endpoint]
session = make_tracked_session(headers=_headers(api_key), redact_values=(api_key,))
@@ -192,10 +280,35 @@ def get_rows(
if resume and resume.next_page_token is not None:
logger.debug(f"PayFit: resuming {endpoint} from page token {next_page_token}")
+ if config.monthly:
+ today = _today()
+ first_month = (
+ _parse_period(resume.month)
+ if resume and resume.month
+ else resolve_first_month(today, history_start, db_incremental_field_last_value)
+ )
+ yield from _get_monthly_rows(
+ session,
+ company_id,
+ config,
+ first_month,
+ today.replace(day=1),
+ next_page_token if resume and resume.month else None,
+ logger,
+ resumable_source_manager,
+ )
+ return
+
if config.fan_out_over_collaborators:
yield from _get_payslip_rows(session, company_id, next_page_token, logger, resumable_source_manager)
return
+ if not config.paginated or config.data_key is None:
+ rows = _fetch_unpaginated(session, f"/companies/{company_id}{config.path}", {}, logger)
+ if rows:
+ yield rows
+ return
+
path = f"/companies/{company_id}{config.path}"
while True:
rows, next_token = _fetch_page(session, path, config.data_key, next_page_token, config.extra_params, logger)
@@ -241,11 +354,52 @@ def _get_payslip_rows(
resumable_source_manager.save_state(PayFitResumeConfig(next_page_token=next_page_token))
+def _get_monthly_rows(
+ session: requests.Session,
+ company_id: str,
+ config: PayFitEndpointConfig,
+ month: datetime.date,
+ last_month: datetime.date,
+ next_page_token: str | None,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PayFitResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ path = f"/companies/{company_id}{config.path}"
+ while month <= last_month:
+ params = {**config.extra_params, "date": _period(month)}
+ # A 404 for one pay period, such as a month before the company joined PayFit, skips that
+ # month rather than failing the sync.
+ if config.paginated and config.data_key is not None:
+ rows, next_token = _fetch_page(
+ session, path, config.data_key, next_page_token, params, logger, missing_ok=True
+ )
+ else:
+ rows, next_token = _fetch_unpaginated(session, path, params, logger, missing_ok=True), None
+
+ batch = [
+ {**row, "month": month, **({"lineNumber": index} if config.number_rows else {})}
+ for index, row in enumerate(rows)
+ ]
+
+ if next_token and rows:
+ next_page_token = next_token
+ else:
+ month, next_page_token = _next_month(month), None
+ resumable_source_manager.save_state(PayFitResumeConfig(next_page_token=next_page_token, month=_period(month)))
+
+ if batch:
+ yield batch
+ else:
+ resumable_source_manager.safe_point()
+
+
def payfit_source(
api_key: str,
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[PayFitResumeConfig],
+ history_start: datetime.datetime | None = None,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
config = PAYFIT_ENDPOINTS[endpoint]
@@ -256,6 +410,8 @@ def payfit_source(
endpoint=endpoint,
logger=logger,
resumable_source_manager=resumable_source_manager,
+ history_start=history_start,
+ db_incremental_field_last_value=db_incremental_field_last_value,
),
primary_keys=config.primary_keys,
partition_count=1,
@@ -295,7 +451,11 @@ def check_schema_access(api_key: str, schema_name: str) -> tuple[bool, str | Non
return False, f"Could not validate PayFit API key: {e}"
probe = _probe_endpoint(endpoint_config)
- probe_params: dict[str, Any] = {"maxResults": 1, **probe.extra_params}
+ probe_params: dict[str, Any] = {**probe.extra_params}
+ if probe.paginated:
+ probe_params["maxResults"] = 1
+ if probe.monthly:
+ probe_params["date"] = _period(_today())
try:
response = session.get(
f"{PAYFIT_BASE_URL}/companies/{company_id}{probe.path}",
@@ -308,7 +468,12 @@ def check_schema_access(api_key: str, schema_name: str) -> tuple[bool, str | Non
if response.status_code == 401:
return False, "Invalid PayFit API key"
if response.status_code == 403:
+ if probe.scope is None:
+ return False, f"Your PayFit API key cannot read `{schema_name}`"
return False, f"Your PayFit API key is missing the `{probe.scope}` scope required for `{schema_name}`"
+ # A month-scoped endpoint answers 404 for a pay period with no data, which still proves access.
+ if response.status_code == 404 and probe.monthly:
+ return True, None
if not response.ok:
return False, f"PayFit returned HTTP {response.status_code}"
return True, None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py
index c5ce4bd918a4..04e73955866b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/settings.py
@@ -1,25 +1,53 @@
+import datetime
from dataclasses import dataclass, field
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+# Month-scoped endpoints take a required `date=YYYYMM` and return one pay period per request, so a
+# first sync walks back this far and later full refreshes keep the same start.
+MONTHLY_HISTORY_LOOKBACK = datetime.timedelta(days=3 * 365)
+
+# A pay period's payroll can stay open after later months have rows, so an incremental run reads
+# this many months before the watermark again.
+INCREMENTAL_REREAD_MONTHS = 3
+
+MONTH_INCREMENTAL_FIELDS: list[IncrementalField] = [
+ {
+ "label": "month",
+ "type": IncrementalFieldType.Date,
+ "field": "month",
+ "field_type": IncrementalFieldType.Date,
+ }
+]
+
+
+@dataclass(frozen=True)
class PayFitEndpointConfig:
name: str
# Appended to /companies/{companyId} on the Partner API host.
path: str
# Key wrapping the rows in the JSON response, e.g. {"collaborators": [...], "meta": {...}}.
- data_key: str
- # OAuth scope the API key must carry to read this endpoint.
- scope: str
+ # None means the body itself is the row (an object) or the rows (an array).
+ data_key: str | None
+ # OAuth scope the API key must carry to read this endpoint, or None when it needs none.
+ scope: str | None
primary_keys: list[str] = field(default_factory=lambda: ["id"])
# Payslips have no list-all endpoint; they are fetched per collaborator.
fan_out_over_collaborators: bool = False
extra_params: dict[str, str] = field(default_factory=dict)
+ paginated: bool = True
+ # Fetched once per pay period, with each row stamped with a `month` date column.
+ monthly: bool = False
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ # Stamp each row with its position in the response, for rows that carry no ID.
+ number_rows: bool = False
-# PayFit Partner API list endpoints. All are full-refresh only: PayFit exposes no server-side
-# `updated_after`/`since` filter on any of them, so there is no incremental cursor to advance.
-# Country-specific endpoints (meal vouchers, health insurance and provident funds, accounting CSV
-# exports) are intentionally not synced.
+# PayFit Partner API endpoints. List endpoints expose no server-side `updated_after`/`since`
+# filter, so they are full refresh only. Month-scoped endpoints can sync incrementally by month,
+# because the requested pay period is the server-side filter. Accounting entries and worked time
+# are French-only endpoints. Meal vouchers, health insurance and provident funds, and the
+# accounting CSV export are not synced.
PAYFIT_ENDPOINTS: dict[str, PayFitEndpointConfig] = {
"collaborators": PayFitEndpointConfig(
name="collaborators",
@@ -52,8 +80,49 @@ PAYFIT_ENDPOINTS: dict[str, PayFitEndpointConfig] = {
primary_keys=["collaboratorId", "payslipId"],
fan_out_over_collaborators=True,
),
+ "company": PayFitEndpointConfig(
+ name="company",
+ path="",
+ data_key=None,
+ scope=None,
+ paginated=False,
+ ),
+ "payroll_status": PayFitEndpointConfig(
+ name="payroll_status",
+ path="/payroll-status",
+ data_key=None,
+ scope=None,
+ primary_keys=["month"],
+ paginated=False,
+ monthly=True,
+ incremental_fields=MONTH_INCREMENTAL_FIELDS,
+ ),
+ "contracts_worked_time": PayFitEndpointConfig(
+ name="contracts_worked_time",
+ path="/contracts/time",
+ data_key="contracts",
+ scope="time:read",
+ primary_keys=["month", "contractId"],
+ monthly=True,
+ incremental_fields=MONTH_INCREMENTAL_FIELDS,
+ ),
+ "accounting_entries": PayFitEndpointConfig(
+ name="accounting_entries",
+ path="/accounting-v2",
+ data_key=None,
+ scope="accounting:read",
+ # Entries carry no ID, and one contract and account can repeat across analytic codes, so
+ # rows are keyed by their position in the month's export. That key is only stable within
+ # one read, which is why this table is full refresh only.
+ primary_keys=["month", "lineNumber"],
+ paginated=False,
+ monthly=True,
+ number_rows=True,
+ ),
}
ENDPOINTS = tuple(PAYFIT_ENDPOINTS.keys())
-INCREMENTAL_FIELDS: dict[str, list[dict[str, str]]] = {}
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ name: config.incremental_fields for name, config in PAYFIT_ENDPOINTS.items() if config.incremental_fields
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.py
index 0098cb17428f..541af4e2dd8d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/source.py
@@ -1,3 +1,4 @@
+import datetime
from typing import Optional, cast
from products.warehouse_sources.backend.facade.codebase_detection import detect
@@ -23,7 +24,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.pay
payfit_source,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.settings import ENDPOINTS, PAYFIT_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.settings import (
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ MONTHLY_HISTORY_LOOKBACK,
+ PAYFIT_ENDPOINTS,
+)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -47,7 +53,7 @@ class PayFitSource(ResumableSource[PayFitSourceConfig, PayFitResumeConfig]):
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your PayFit API key to pull your HR and payroll data into the PostHog Data warehouse.
-You can create an API key from the **API access** tab on the [integrations page](https://app.payfit.com/integrations/hub/api) of your PayFit admin account. Grant it the `collaborators:read`, `contracts:read`, `time:read`, and `contracts:payslips:read` scopes so every table can sync.
+You can create an API key from the **API access** tab on the [integrations page](https://app.payfit.com/integrations/hub/api) of your PayFit admin account. Grant it the `collaborators:read`, `contracts:read`, `time:read`, `contracts:payslips:read`, and `accounting:read` scopes so every table can sync. The `accounting_entries` and `contracts_worked_time` tables are only available for companies based in France.
""",
iconPath="/static/services/payfit.png",
docsUrl="https://posthog.com/docs/cdp/sources/payfit",
@@ -76,7 +82,7 @@ You can create an API key from the **API access** tab on the [integrations page]
def get_non_retryable_errors(self) -> dict[str, str | None]:
return {
"401 Client Error: Unauthorized for url: https://partner-api.payfit.com": "Your PayFit API key is invalid or has been revoked. Create a new key under Integrations → API access in PayFit, then reconnect.",
- "403 Client Error: Forbidden for url: https://partner-api.payfit.com": "Your PayFit API key is missing a scope this table needs. Create a key with the collaborators:read, contracts:read, time:read, and contracts:payslips:read scopes, then reconnect.",
+ "403 Client Error: Forbidden for url: https://partner-api.payfit.com": "Your PayFit API key is missing a scope this table needs. Create a key with the collaborators:read, contracts:read, time:read, contracts:payslips:read, and accounting:read scopes, then reconnect.",
"PayFit API key is inactive or invalid": "Your PayFit API key is invalid or has been revoked. Create a new key under Integrations → API access in PayFit, then reconnect.",
}
@@ -89,14 +95,12 @@ You can create an API key from the **API access** tab on the [integrations page]
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Every endpoint is full refresh only — PayFit exposes no server-side updated_after/since
- # filter on any list endpoint, so there is no incremental cursor to advance.
schemas = [
SourceSchema(
name=endpoint,
- supports_incremental=False,
+ supports_incremental=endpoint in INCREMENTAL_FIELDS,
supports_append=False,
- incremental_fields=[],
+ incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
)
for endpoint in ENDPOINTS
]
@@ -118,6 +122,14 @@ You can create an API key from the **API access** tab on the [integrations page]
return check_schema_access(config.api_key, schema_name)
return validate_credentials(config.api_key)
+ def history_lookback_for_schema(
+ self, schema_name: str, config: PayFitSourceConfig | None = None
+ ) -> datetime.timedelta | None:
+ endpoint_config = PAYFIT_ENDPOINTS.get(schema_name)
+ if endpoint_config is not None and endpoint_config.monthly:
+ return MONTHLY_HISTORY_LOOKBACK
+ return None
+
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[PayFitResumeConfig]:
return ResumableSourceManager[PayFitResumeConfig](inputs, PayFitResumeConfig)
@@ -135,4 +147,8 @@ You can create an API key from the **API access** tab on the [integrations page]
endpoint=inputs.schema_name,
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
+ history_start=inputs.history_start,
+ db_incremental_field_last_value=(
+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None
+ ),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.py
index 52f5591c9240..484eea0fe288 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/tests/test_payfit.py
@@ -1,6 +1,9 @@
+import datetime
+from collections.abc import Iterator
from typing import Any
import pytest
+import time_machine
from unittest.mock import MagicMock
import requests
@@ -10,6 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.payfit import PayFitSourceConfig
@@ -77,6 +81,126 @@ class TestGetRows:
assert result.rows == [{"id": "5"}]
assert result.params("nextPageToken") == [None, "tok-99"]
+ def test_company_object_is_one_row(self) -> None:
+ company = {"id": COMPANY_ID, "name": "Example company", "country": "FR", "nbActiveContracts": 3}
+ result = _driver().run("company", [_introspection_response(), ScriptedResponse(json=company)])
+
+ assert result.raised is None
+ assert result.items == [[company]]
+ assert result.paths == ["/introspect", f"/companies/{COMPANY_ID}"]
+
+
+class TestGetMonthlyRows:
+ @pytest.fixture(autouse=True)
+ def _frozen_today(self) -> Iterator[None]:
+ with time_machine.travel(datetime.datetime(2026, 3, 15, 12, tzinfo=datetime.UTC), tick=False):
+ yield
+
+ def test_walks_pay_periods_paging_within_each_month(self) -> None:
+ result = _driver().run(
+ "contracts_worked_time",
+ [
+ _introspection_response(),
+ ScriptedResponse(json={"contracts": [{"contractId": "c1"}], "meta": {"nextPageToken": "tok-2"}}),
+ ScriptedResponse(json={"contracts": [{"contractId": "c2"}], "meta": {"nextPageToken": None}}),
+ # A pay period PayFit holds nothing for answers 404 and is skipped.
+ ScriptedResponse(status=404, json={"error": "Ressource not found"}),
+ ScriptedResponse(json={"contracts": [{"contractId": "c1"}], "meta": {}}),
+ ],
+ history_start=datetime.datetime(2026, 1, 10, tzinfo=datetime.UTC),
+ )
+
+ assert result.raised is None
+ assert result.items == [
+ [{"contractId": "c1", "month": datetime.date(2026, 1, 1)}],
+ [{"contractId": "c2", "month": datetime.date(2026, 1, 1)}],
+ [{"contractId": "c1", "month": datetime.date(2026, 3, 1)}],
+ ]
+ assert result.params("date") == [None, "202601", "202601", "202602", "202603"]
+ assert result.params("nextPageToken") == [None, None, "tok-2", None, None]
+ assert result.saved_states == [
+ PayFitResumeConfig(next_page_token="tok-2", month="202601"),
+ PayFitResumeConfig(month="202602"),
+ PayFitResumeConfig(month="202603"),
+ PayFitResumeConfig(month="202604"),
+ ]
+
+ @parameterized.expand(
+ [
+ (
+ "payroll_status_object",
+ "payroll_status",
+ {"status": "completed", "executionEndDate": "2026-03-10T11:56:24.221Z"},
+ [{"status": "completed", "executionEndDate": "2026-03-10T11:56:24.221Z"}],
+ ),
+ (
+ "accounting_array_numbers_rows",
+ "accounting_entries",
+ [{"accountId": "6411000", "debit": 10}, {"accountId": "6411000", "debit": 10}],
+ [
+ {"accountId": "6411000", "debit": 10, "lineNumber": 0},
+ {"accountId": "6411000", "debit": 10, "lineNumber": 1},
+ ],
+ ),
+ ]
+ )
+ def test_unpaginated_bodies_become_rows(
+ self, _name: str, endpoint: str, body: Any, expected: list[dict[str, Any]]
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ [_introspection_response(), ScriptedResponse(json=body)],
+ history_start=datetime.datetime(2026, 3, 1, tzinfo=datetime.UTC),
+ )
+
+ assert result.raised is None
+ assert result.rows == [{**row, "month": datetime.date(2026, 3, 1)} for row in expected]
+ assert result.params("maxResults") == [None, None]
+
+ @parameterized.expand(
+ [
+ ("history_start", {"history_start": datetime.datetime(2026, 1, 20, tzinfo=datetime.UTC)}, "202601"),
+ ("no_history_start_uses_lookback", {}, "202303"),
+ (
+ "incremental_rereads_months_before_watermark",
+ {
+ "history_start": datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC),
+ "incremental_field": "month",
+ "db_incremental_field_last_value": datetime.date(2026, 2, 1),
+ },
+ "202511",
+ ),
+ ]
+ )
+ def test_first_requested_pay_period(self, _name: str, inputs: dict[str, Any], expected_first: str) -> None:
+ result = _driver().run(
+ "payroll_status",
+ route({"/introspect": [_introspection_response()], "/payroll-status": [ScriptedResponse(status=404)] * 40}),
+ **inputs,
+ )
+
+ assert result.raised is None
+ dates = result.params("date")[1:]
+ assert dates[0] == expected_first
+ assert dates[-1] == "202603"
+
+ def test_resumes_from_saved_month_and_page_token(self) -> None:
+ result = _driver().run(
+ "contracts_worked_time",
+ [
+ _introspection_response(),
+ ScriptedResponse(json={"contracts": [{"contractId": "c9"}], "meta": {}}),
+ ScriptedResponse(json={"contracts": [], "meta": {}}),
+ ],
+ resume_state=PayFitResumeConfig(next_page_token="tok-9", month="202602"),
+ history_start=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC),
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"contractId": "c9", "month": datetime.date(2026, 2, 1)}]
+ assert result.params("date") == [None, "202602", "202603"]
+ assert result.params("nextPageToken") == [None, "tok-9", None]
+
class TestGetPayslipRows:
def test_fans_out_over_collaborator_pages_and_stamps_parent_id(self) -> None:
@@ -292,6 +416,14 @@ class TestCheckSchemaAccess:
# Payslip paths need a collaborator id, so the scope probe targets the collaborators list.
assert network.requests_log[1].url == f"{PAYFIT_BASE_URL}/companies/{COMPANY_ID}/collaborators?maxResults=1"
+ @time_machine.travel(datetime.datetime(2026, 3, 15, tzinfo=datetime.UTC), tick=False)
+ def test_monthly_probe_accepts_empty_pay_period(self) -> None:
+ with scripted_network([_introspection_response(), ScriptedResponse(status=404, json={})]) as network:
+ valid, _message = check_schema_access("payfit-key", "accounting_entries")
+
+ assert valid is True
+ assert network.requests_log[1].url == f"{PAYFIT_BASE_URL}/companies/{COMPANY_ID}/accounting-v2?date=202603"
+
def test_inactive_token_fails_before_probing(self) -> None:
with scripted_network([ScriptedResponse(json={"active": False})]) as network:
valid, message = check_schema_access("payfit-key", "contracts")
@@ -302,13 +434,11 @@ class TestCheckSchemaAccess:
class TestPayFitSourceResponse:
- @parameterized.expand([(e,) for e in ENDPOINTS])
+ @parameterized.expand([(e,) for e in ENDPOINTS if not PAYFIT_ENDPOINTS[e].monthly])
def test_source_response_shape(self, endpoint: str) -> None:
data_key = "collaborators" if endpoint == "payslips" else PAYFIT_ENDPOINTS[endpoint].data_key
- result = _driver().run(
- endpoint,
- [_introspection_response(), ScriptedResponse(json={data_key: [], "meta": {}})],
- )
+ body: dict[str, Any] = {data_key: [], "meta": {}} if data_key else {"id": COMPANY_ID}
+ result = _driver().run(endpoint, [_introspection_response(), ScriptedResponse(json=body)])
assert result.raised is None
response = result.response