Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the Workable source (#120050)

Tom Owers2e2eb9bfe23f
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 626bc99c665c..5919573152b3 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1198,8 +1198,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/woocommerce.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/wordpress/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/wordpress/wordpress.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/workable/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/workable/workable.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/workday/workday.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/workos/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/workos/workos.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index cc5dbb9f8d26..4ac48c4dd843 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -13250,9 +13250,9 @@ Today (5): `candidates`, `jobs`, `members`, `recruiters`, `stages`
Diffed against: <https://workable.readme.io/reference/jobs>
-- [ ] `candidate activities (GET /candidates/{id}/activities)` — the stage-transition and action history behind every candidate; without it stages is a static snapshot (high)
-- [ ] `employees (GET /employees)` — the entire Workable HR side is absent, and it is what links hires back to headcount (high)
-- [ ] `departments (GET /departments)` — lookup resolving the department carried on jobs, requisitions and employees (high)
+- [x] `candidate activities (GET /candidates/{id}/activities)` — the stage-transition and action history behind every candidate; without it stages is a static snapshot (high)
+- [x] `employees (GET /employees)` — the entire Workable HR side is absent, and it is what links hires back to headcount (high)
+- [x] `departments (GET /departments)` — lookup resolving the department carried on jobs, requisitions and employees (high)
- [ ] `locations (GET /accounts/{subdomain}/locations)` — lookup resolving the location id on jobs and candidates (high)
- [ ] `time entries (GET /time-tracking/time-entries)` — time tracking fact table for employees (high)
- [ ] `job activities (GET /jobs/{shortcode}/activities)` — per-job event stream for pipeline throughput analysis (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/canonical_descriptions.py
index 12cfb93818ce..674e214acf5a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/canonical_descriptions.py
@@ -76,4 +76,52 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"position": "Zero-based position of the stage in the pipeline.",
},
},
+ "departments": {
+ "description": "Departments defined in your Workable account, arranged as a tree.",
+ "docs_url": "https://workable.readme.io/reference/departments",
+ "columns": {
+ "id": "Unique identifier for the department.",
+ "name": "Department name.",
+ "parent_id": "Identifier of the parent department, or null for a top-level department.",
+ },
+ },
+ "employees": {
+ "description": "Employees in the Workable HR module.",
+ "docs_url": "https://workable.readme.io/reference/employees",
+ "columns": {
+ "id": "Unique identifier for the employee.",
+ "firstname": "Employee's first name.",
+ "lastname": "Employee's last name.",
+ "email": "Email address assigned to the employee.",
+ "work_email": "Company email address of the employee.",
+ "start_date": "Start date of the employee.",
+ "state": "State of the employee profile (draft, published, inactive).",
+ "employee_number": "Employee number, often used for payroll or HR tracking.",
+ "job_title": "Role or position the employee holds.",
+ "manager_id": "Identifier of the employee's direct manager.",
+ "department_id": "Identifier of the employee's department, from the departments table.",
+ "division_id": "Identifier of the employee's division or business unit.",
+ "candidate_id": "Identifier of the candidate who was hired into this employee profile.",
+ "legal_entity_id": "Identifier of the legal entity the employee belongs to.",
+ "work_schedule_id": "Identifier of the employee's work schedule.",
+ "template_id": "Identifier of the employee profile template.",
+ },
+ },
+ "candidate_activities": {
+ "description": "Activity stream of each candidate: stage moves, comments, ratings, messages and other actions.",
+ "docs_url": "https://workable.readme.io/reference/candidate-activities",
+ "columns": {
+ "candidate_id": "Identifier of the candidate the activity belongs to.",
+ "action": "Action or type of the activity.",
+ "stage_name": "Stage the candidate was in when a stage move created the activity.",
+ "action_stage": "Stage the candidate was in when the action started (id and name).",
+ "target_stage": "Stage the candidate moved to, for move activities (id and name).",
+ "created_at": "Timestamp the activity was created.",
+ "updated_at": "Timestamp the activity was last updated.",
+ "member": "Member who created the activity, if any (id and name).",
+ "body": "Body of a comment, rating, message or disqualification activity.",
+ "rating": "Score of the activity, for rating activities.",
+ "visibility_roles": "Collaboration roles that can view the activity.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/settings.py
index 5d506482c946..e7f0515734a8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/settings.py
@@ -1,6 +1,9 @@
from dataclasses import dataclass, field
-from typing import Optional
+from typing import Literal, Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# Workable caps `limit` at 100 (default 50) across the list endpoints.
@@ -16,13 +19,14 @@ def _datetime_field(name: str) -> IncrementalField:
}
-@dataclass
+@dataclass(frozen=False)
class WorkableEndpointConfig:
name: str
# Path under the `/spi/v3` base (e.g. `/jobs`).
path: str
- # Key the result array is nested under in the JSON response (e.g. `{"jobs": [...]}`).
- data_key: str
+ # Key the result array is nested under in the JSON response (e.g. `{"jobs": [...]}`). `None`
+ # when the response body is the array itself.
+ data_key: Optional[str]
primary_keys: list[str] = field(default_factory=lambda: ["id"])
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Stable creation timestamp used for datetime partitioning. Never `updated_at` — partitions
@@ -30,11 +34,16 @@ class WorkableEndpointConfig:
partition_key: Optional[str] = None
# Only `True` when the endpoint exposes a genuine server-side `<field>_after` time filter.
supports_incremental: bool = False
+ # `next_url` follows `paging.next`, `offset` walks `offset`/`limit` against `totalCount`, and
+ # `none` is a single unpaginated request.
+ pagination: Literal["next_url", "offset", "none"] = "next_url"
+ page_size: int = PAGE_SIZE
+ default_incremental_field: Optional[str] = None
+ fanout: Optional[DependentEndpointConfig] = None
-# Account-level list endpoints. These mirror the canonical Workable connector stream set
-# (jobs, candidates, members, recruiters, stages). All return `{<data_key>: [...]}` and the
-# paginated ones carry a sibling `{"paging": {"next": "<full url>"}}`.
+# Account-level list endpoints plus the candidate activity fan-out. Most return
+# `{<data_key>: [...]}` and the paginated ones carry a sibling `{"paging": {"next": "<full url>"}}`.
WORKABLE_ENDPOINTS: dict[str, WorkableEndpointConfig] = {
"jobs": WorkableEndpointConfig(
name="jobs",
@@ -76,6 +85,46 @@ WORKABLE_ENDPOINTS: dict[str, WorkableEndpointConfig] = {
data_key="stages",
primary_keys=["slug"],
),
+ # Requires the `r_account` scope and Workable's employee management features. The response is
+ # a bare array with no pagination.
+ "departments": WorkableEndpointConfig(
+ name="departments",
+ path="/departments",
+ data_key=None,
+ primary_keys=["id"],
+ pagination="none",
+ ),
+ # Requires the `r_employees` scope. An account-level token only sees published employees.
+ # There is no server-side time filter, so this is full refresh only.
+ "employees": WorkableEndpointConfig(
+ name="employees",
+ path="/employees",
+ data_key="employees",
+ primary_keys=["id"],
+ pagination="offset",
+ ),
+ # Activities have no list endpoint of their own, so they fan out over every candidate. The docs
+ # list no activity id, so the key is the parent candidate plus the activity's creation time and
+ # action.
+ "candidate_activities": WorkableEndpointConfig(
+ name="candidate_activities",
+ path="/candidates/{id}/activities",
+ data_key="activities",
+ primary_keys=["candidate_id", "created_at", "action"],
+ partition_key="created_at",
+ supports_incremental=True,
+ incremental_fields=[_datetime_field("updated_at")],
+ default_incremental_field="updated_at",
+ fanout=DependentEndpointConfig(
+ parent_name="candidates",
+ resolve_param="id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "candidate_id"},
+ # A candidate deleted between the parent listing and its activity fetch answers 404.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(WORKABLE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/source.py
index 8c3ea2e90119..129678dbed2b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/source.py
@@ -59,6 +59,8 @@ class WorkableSource(ResumableSource[WorkableSourceConfig, WorkableResumeConfig]
You can create an access token in your Workable account under **Settings > Integrations > Access Tokens** (admins only). Grant the following read scopes:
- `r_jobs`
- `r_candidates`
+- `r_employees` (only for the `employees` table)
+- `r_account` (only for the `departments` table)
""",
iconPath="/static/services/workable.png",
docsUrl="https://posthog.com/docs/cdp/sources/workable",
@@ -125,8 +127,14 @@ You can create an access token in your Workable account under **Settings > Integ
schema_name: Optional[str] = None,
api_version: str | None = None,
) -> tuple[bool, str | None]:
- # Probe the specific schema's endpoint when given, otherwise a cheap `/jobs` probe.
- path = WORKABLE_ENDPOINTS[schema_name].path if schema_name in WORKABLE_ENDPOINTS else "/jobs"
+ # Probe the specific schema's endpoint when given, otherwise a cheap `/jobs` probe. A fan-out
+ # endpoint has no bare list path, so probe its parent, which needs the same scope.
+ path = "/jobs"
+ if schema_name in WORKABLE_ENDPOINTS:
+ endpoint_config = WORKABLE_ENDPOINTS[schema_name]
+ if endpoint_config.fanout is not None:
+ endpoint_config = WORKABLE_ENDPOINTS[endpoint_config.fanout.parent_name]
+ path = endpoint_config.path
try:
status_code, ok = validate_workable_credentials(config.subdomain, config.api_token, path)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable.py
index 4199ff25b63f..caafc4ac220a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable.py
@@ -10,6 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.workable import (
@@ -123,6 +124,16 @@ class TestPagination:
assert [r["id"] for r in result.rows] == ["99"]
assert result.urls == [resume_url]
+ def test_next_url_on_another_host_is_not_followed(self) -> None:
+ result = _driver().run(
+ "candidates",
+ [_response(_page([{"id": "1"}], next_url="https://attacker.example/collect"))],
+ )
+
+ assert isinstance(result.raised, ValueError)
+ assert "disallowed host" in str(result.raised)
+ assert len(result.urls) == 1
+
def test_a_run_with_no_saved_state_starts_from_the_first_page(self) -> None:
result = _driver().run("candidates", [_response(_page([{"id": "1"}]))])
@@ -234,3 +245,128 @@ class TestValidateCredentials:
def test_invalid_subdomain_raises(self) -> None:
with pytest.raises(ValueError):
validate_credentials("evil.com/", "tok")
+
+
+class TestEmployees:
+ @parameterized.expand(
+ [
+ # A server-side cap below PAGE_SIZE must move the offset by the rows returned, not the limit.
+ ("capped_page_size_stops_at_total", {"totalCount": 4}, ["0", "2"]),
+ # Without a total, the walk ends on the first empty page.
+ ("missing_total_stops_on_empty_page", {}, ["0", "2", "4"]),
+ ]
+ )
+ def test_offset_walk(self, _name: str, total: dict[str, Any], expected_offsets: list[str]) -> None:
+ pages = [
+ _response({**total, "employees": [{"id": "e1"}, {"id": "e2"}]}),
+ _response({**total, "employees": [{"id": "e3"}, {"id": "e4"}]}),
+ _response({**total, "employees": []}),
+ ]
+
+ result = _driver().run("employees", pages[: len(expected_offsets)])
+
+ assert result.raised is None
+ assert [r["id"] for r in result.rows] == ["e1", "e2", "e3", "e4"]
+ assert result.params("offset") == expected_offsets
+ assert set(result.params("limit")) == {"100"}
+
+
+class TestDepartments:
+ def test_bare_array_body_is_read_in_one_request(self) -> None:
+ result = _driver().run(
+ "departments",
+ [
+ ScriptedResponse(
+ json=[
+ {"id": "d1", "name": "Engineering", "parent_id": None},
+ {"id": "d2", "name": "Platform", "parent_id": "d1"},
+ ]
+ )
+ ],
+ )
+
+ assert result.raised is None
+ assert [(r["id"], r["parent_id"]) for r in result.rows] == [("d1", None), ("d2", "d1")]
+ assert result.paths == ["/spi/v3/departments"]
+ assert result.requests[0].query == {}
+
+
+class TestCandidateActivities:
+ @parameterized.expand(
+ [
+ ("full_refresh", False, None, None),
+ ("incremental", True, datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC), "2026-01-02T03:04:05Z"),
+ ]
+ )
+ def test_fans_out_over_candidates(
+ self, _name: str, use_incremental: bool, last_value: datetime | None, expected_filter: str | None
+ ) -> None:
+ candidates_next = "https://acme.workable.com/spi/v3/candidates?limit=100&since_id=c2"
+ activities_next = "https://acme.workable.com/spi/v3/candidates/c1/activities?limit=100&since_id=a2"
+ script = route(
+ {
+ "/c1/activities": [
+ _response(
+ _page(
+ [{"action": "applied", "created_at": "2026-01-03T00:00:00Z"}], "activities", activities_next
+ )
+ ),
+ _response(_page([{"action": "comment", "created_at": "2026-01-04T00:00:00Z"}], "activities")),
+ ],
+ # A candidate deleted between the listing and its activity fetch is skipped.
+ "/c2/activities": [_response({"error": "Not found"}, status=404)],
+ "/c3/activities": [
+ _response(_page([{"action": "moved", "created_at": "2026-01-05T00:00:00Z"}], "activities"))
+ ],
+ "/candidates": [
+ _response(_page([{"id": "c1"}, {"id": "c2"}], next_url=candidates_next)),
+ _response(_page([{"id": "c3"}])),
+ ],
+ }
+ )
+
+ result = _driver().run(
+ "candidate_activities",
+ script,
+ incremental_field="updated_at",
+ should_use_incremental_field=use_incremental,
+ db_incremental_field_last_value=last_value,
+ )
+
+ assert result.raised is None
+ assert [(r["candidate_id"], r["action"]) for r in result.rows] == [
+ ("c1", "applied"),
+ ("c1", "comment"),
+ ("c3", "moved"),
+ ]
+ child_requests = [r for r in result.requests if r.path.endswith("/activities")]
+ parent_requests = [r for r in result.requests if r.path.endswith("/candidates")]
+ assert [r.param("updated_after") for r in child_requests[:1] + child_requests[2:]] == [expected_filter] * 3
+ assert all(r.param("updated_after") is None for r in parent_requests)
+
+ def test_resume_skips_completed_candidates(self) -> None:
+ script = route(
+ {
+ "/c2/activities": [
+ _response(_page([{"action": "applied", "created_at": "2026-01-03T00:00:00Z"}], "activities"))
+ ],
+ "/candidates": [_response(_page([{"id": "c1"}, {"id": "c2"}]))],
+ }
+ )
+
+ result = _driver().run(
+ "candidate_activities",
+ script,
+ should_use_incremental_field=False,
+ resume_state=WorkableResumeConfig(
+ fanout_state={"completed": ["/candidates/c1/activities"], "current": None, "child_state": None}
+ ),
+ )
+
+ assert result.raised is None
+ assert [r["candidate_id"] for r in result.rows] == ["c2"]
+ assert not any(r.path.endswith("/c1/activities") for r in result.requests)
+ assert result.committed_states[-1].fanout_state["completed"] == [
+ "/candidates/c1/activities",
+ "/candidates/c2/activities",
+ ]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable_source.py
index eb9bf7fdbf09..251652f76d10 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/tests/test_workable_source.py
@@ -7,6 +7,8 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
+ always,
+ scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.workable import (
WorkableSourceConfig,
@@ -45,6 +47,21 @@ class TestValidateCredentials:
valid, _msg = WorkableSource().validate_credentials(_config(), team_id=1, schema_name=schema_name)
assert valid is expected_valid
+ @parameterized.expand(
+ [
+ ("default", None, "/spi/v3/jobs"),
+ ("top_level_endpoint", "departments", "/spi/v3/departments"),
+ # A fan-out endpoint has no bare list path; probing its literal template would 404.
+ ("fanout_probes_parent", "candidate_activities", "/spi/v3/candidates"),
+ ]
+ )
+ def test_probe_path(self, _name: str, schema_name: str | None, expected_path: str) -> None:
+ with scripted_network(always(ScriptedResponse(json={}))) as network:
+ valid, _msg = WorkableSource().validate_credentials(_config(), team_id=1, schema_name=schema_name)
+
+ assert valid is True
+ assert [r.path for r in network.requests_log] == [expected_path]
+
def test_invalid_subdomain_is_a_failure(self) -> None:
def _raise(*_a: Any, **_k: Any) -> Any:
raise ValueError("Invalid Workable subdomain")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/workable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/workable.py
index b30967b3884d..7fc7a6553774 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workable/workable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/workable/workable.py
@@ -1,18 +1,28 @@
import re
import dataclasses
+from collections.abc import Iterable
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import urlencode
+from requests import Response
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
JSONResponsePaginator,
+ OffsetPaginator,
+ SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
Endpoint,
IncrementalConfig,
)
@@ -22,6 +32,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.temporal.data_imports.sources.workable.settings import (
PAGE_SIZE,
WORKABLE_ENDPOINTS,
+ WorkableEndpointConfig,
)
# Workable account subdomains are DNS labels — letters, digits and hyphens. Validating this before
@@ -34,10 +45,34 @@ _SUBDOMAIN_RE = re.compile(r"^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$")
_TIME_FILTER_PARAM = {"updated_at": "updated_after", "created_at": "created_after"}
[email protected](frozen=False)
class WorkableResumeConfig:
# Full `paging.next` URL to fetch next. `None` means start the endpoint from its first page.
next_url: str | None = None
+ # Fan-out endpoints: the framework's dependent-resource resume state
+ # (`{"completed": [...], "current": ..., "child_state": ...}`).
+ fanout_state: dict[str, Any] | None = None
+
+
+class WorkableOffsetPaginator(OffsetPaginator):
+ """Advances `offset` by the rows a page returned rather than by the requested `limit`.
+
+ Workable does not document a maximum `limit` for `/employees`, so a server-side cap below
+ `PAGE_SIZE` would make the stock paginator skip rows or stop after the first short page.
+ """
+
+ def __init__(self, limit: int) -> None:
+ super().__init__(limit=limit, total_path="totalCount")
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ if not data:
+ self._has_next_page = False
+ return
+
+ self.offset += len(data)
+ body = response.json()
+ total = body.get("totalCount") if isinstance(body, dict) else None
+ self._has_next_page = not isinstance(total, int) or self.offset < total
def _validate_subdomain(subdomain: str) -> str:
@@ -86,6 +121,91 @@ def _sort_mode_for(endpoint: str, should_use_incremental_field: bool, incrementa
return "asc" if incremental_field == "created_at" else "desc"
+def _incremental_config(field_name: str) -> IncrementalConfig:
+ # `updated_after` / `created_after` are the documented filters; default to `updated_at` so edits
+ # to existing rows are picked up. The value is formatted ISO 8601 with a `Z` suffix.
+ filter_param = _TIME_FILTER_PARAM.get(field_name, "updated_after")
+ return {"start_param": filter_param, "cursor_path": field_name, "convert": _format_datetime}
+
+
+def _paginator(config: WorkableEndpointConfig) -> BasePaginator:
+ if config.pagination == "offset":
+ return WorkableOffsetPaginator(limit=config.page_size)
+ if config.pagination == "none":
+ return SinglePagePaginator()
+ # `paging.next` is a full URL carrying the cursor params; the paginator follows it verbatim
+ # and stops when it's absent.
+ return JSONResponsePaginator(next_url_path="paging.next")
+
+
+def _endpoint_extra(config: WorkableEndpointConfig) -> Endpoint:
+ extra: Endpoint = {"paginator": _paginator(config)}
+ # Rows are nested under the endpoint's data key (e.g. `{"jobs": [...]}`). A missing key
+ # yields an empty page (matching the original `.get(data_key, [])`), so no fail-loud here.
+ # Without a data key the body is the row array itself.
+ if config.data_key is not None:
+ extra["data_selector"] = config.data_key
+ return extra
+
+
+def _client_config(subdomain: str, api_token: str) -> ClientConfig:
+ return {
+ "base_url": _base_url(subdomain),
+ # Auth (Bearer) goes through the framework auth config so the token is redacted from logs
+ # and raised error messages; only the non-secret Accept header is set here.
+ "headers": {"Accept": "application/json"},
+ "auth": {"type": "bearer", "token": api_token},
+ # Pin every request, including `paging.next` follow-ups and resume URLs, so a crafted next
+ # link can't send the bearer token elsewhere. Workable's next links may point at
+ # `www.workable.com/spi/v3/accounts/<subdomain>/...` rather than the account subdomain host.
+ "allowed_hosts": ["www.workable.com"],
+ }
+
+
+def _fanout_items(
+ config: WorkableEndpointConfig,
+ client_config: ClientConfig,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[WorkableResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Optional[Any],
+ incremental_field: str | None,
+) -> Iterable[Any]:
+ assert config.fanout is not None
+ parent_config = WORKABLE_ENDPOINTS[config.fanout.parent_name]
+
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None and resume.fanout_state:
+ initial_paginator_state = resume.fanout_state
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state is not None:
+ resumable_source_manager.save_state(WorkableResumeConfig(fanout_state=state))
+
+ # The parent candidate walk is never windowed: a new activity is not documented to move the
+ # candidate's `updated_at`, so the time filter only applies to each candidate's activities.
+ return build_dependent_resource(
+ endpoint_configs=WORKABLE_ENDPOINTS,
+ child_endpoint=config.name,
+ fanout=config.fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=db_incremental_field_last_value,
+ should_use_incremental_field=should_use_incremental_field and db_incremental_field_last_value is not None,
+ incremental_field=incremental_field,
+ incremental_config_factory=_incremental_config,
+ parent_endpoint_extra=_endpoint_extra(parent_config),
+ child_endpoint_extra=_endpoint_extra(config),
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+
+
def workable_source(
subdomain: str,
api_token: str,
@@ -98,39 +218,36 @@ def workable_source(
incremental_field: str | None = None,
) -> SourceResponse:
config = WORKABLE_ENDPOINTS[endpoint]
+ client_config = _client_config(subdomain, api_token)
- params: dict[str, Any] = {"limit": PAGE_SIZE}
+ if config.fanout is not None:
+ fanout_items = _fanout_items(
+ config,
+ client_config,
+ team_id,
+ job_id,
+ resumable_source_manager,
+ should_use_incremental_field,
+ db_incremental_field_last_value if should_use_incremental_field else None,
+ incremental_field,
+ )
+ return _source_response(endpoint, config, fanout_items, should_use_incremental_field, incremental_field)
+
+ params: dict[str, Any] = {"limit": PAGE_SIZE} if config.pagination == "next_url" else {}
# Only inject the server-side time filter when the endpoint exposes one and we have a cursor to
- # filter on. `updated_after` / `created_after` are the documented filters; default to `updated_at`
- # so edits to existing rows are picked up. The value is formatted ISO 8601 with a `Z` suffix.
+ # filter on.
incremental: Optional[IncrementalConfig] = None
if config.supports_incremental and should_use_incremental_field and db_incremental_field_last_value is not None:
- field_name = incremental_field or "updated_at"
- filter_param = _TIME_FILTER_PARAM.get(field_name, "updated_after")
- incremental = {"start_param": filter_param, "cursor_path": field_name, "convert": _format_datetime}
-
- endpoint_config: Endpoint = {
- "path": config.path,
- "params": params,
- # Rows are nested under the endpoint's data key (e.g. `{"jobs": [...]}`). A missing key
- # yields an empty page (matching the original `.get(data_key, [])`), so no fail-loud here.
- "data_selector": config.data_key,
- # `paging.next` is a full URL carrying the cursor params; the paginator follows it verbatim
- # and stops when it's absent.
- "paginator": JSONResponsePaginator(next_url_path="paging.next"),
- }
+ incremental = _incremental_config(incremental_field or "updated_at")
+
+ endpoint_config: Endpoint = {"path": config.path, "params": params}
+ endpoint_config.update(_endpoint_extra(config))
if incremental is not None:
endpoint_config["incremental"] = incremental
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": _base_url(subdomain),
- # Auth (Bearer) goes through the framework auth config so the token is redacted from logs
- # and raised error messages; only the non-secret Accept header is set here.
- "headers": {"Accept": "application/json"},
- "auth": {"type": "bearer", "token": api_token},
- },
+ "client": client_config,
"resource_defaults": {},
"resources": [
{
@@ -161,6 +278,16 @@ def workable_source(
initial_paginator_state=initial_paginator_state,
)
+ return _source_response(endpoint, config, resource, should_use_incremental_field, incremental_field)
+
+
+def _source_response(
+ endpoint: str,
+ config: WorkableEndpointConfig,
+ resource: Iterable[Any],
+ should_use_incremental_field: bool,
+ incremental_field: str | None,
+) -> SourceResponse:
return SourceResponse(
name=endpoint,
items=lambda: resource,