:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import re
import secrets
import datetime
import dataclasses
from collections.abc import Iterator
from typing import Any, Optional
from urllib.parse import urlencode
import orjson
import pyarrow as pa
import structlog
from asgiref.sync import async_to_sync
from dateutil import parser as dateutil_parser
from requests import Session
from requests.exceptions import HTTPError, RequestException
from products.warehouse_sources.sources.pipedrive.settings import (
DEAL_IDS_PER_REQUEST,
DEAL_PARENT_PATHS,
MAIL_FOLDERS,
MAIL_THREADS_PATH,
PAGE_SIZE,
PROJECT_BOARDS_PATH,
PipedriveEndpointConfig,
endpoints_for_version,
)
from products.warehouse_sources.sources.sdk import (
APIKeyAuth,
BasePaginator,
ClientConfig,
ExternalWebhookInfo,
JSONResponseCursorPaginator,
OffsetPaginator,
RESTAPIConfig,
RESTClient,
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSourceManager,
create_response_hooks,
make_tracked_session,
rest_api_resource,
validate_via_probe,
)
from products.warehouse_sources.sources.sdk.internals import table_from_py_list
LOGGER = structlog.get_logger(__name__)
# Webhook management lives on v1; there is no v2 equivalent.
WEBHOOKS_PATH = "/api/v1/webhooks"
# The payload format we register. v2 deliveries carry the entity under `data` in the same shape
# the API v2 collections return, which is what lets pushed rows merge into the polled tables.
WEBHOOK_PAYLOAD_VERSION = "2.0"
WEBHOOK_NAME = "PostHog data warehouse"
# Pipedrive echoes the basic auth credentials set on the subscription back on every delivery.
# The username is fixed and the password is the generated secret.
WEBHOOK_AUTH_USER = "posthog"
WEBHOOK_REQUEST_TIMEOUT_SECONDS = 30
_SUBDOMAIN_RE = re.compile(r"^[a-z0-9-]+$")
# Never reflects the raw input back: it can be a full URL or website domain the user pasted by
# mistake, and echoing it gives no guidance on what a valid value looks like.
_INVALID_COMPANY_DOMAIN_ERROR = (
"Invalid Pipedrive company domain. Enter just your Pipedrive subdomain — the part before "
".pipedrive.com (for example, enter 'acme' for acme.pipedrive.com) — not a full URL or your "
"website's domain."
)
@dataclasses.dataclass
class PipedriveResumeConfig:
# Legacy field written by the hand-rolled paginator: the full next-page URL. Kept (optional,
# defaulted) so previously saved resume state still parses via ``dataclass(**saved)``. No longer
# written; a resume that carries only this restarts the endpoint from the beginning (merge
# dedupes any re-yielded rows).
next_url: Optional[str] = None
# Framework paginator resume snapshot: ``{"offset": N}`` for v1 offset endpoints or
# ``{"cursor": "…"}`` for v2 cursor endpoints. Seeds ``initial_paginator_state`` on resume.
paginator_state: Optional[dict[str, Any]] = None
def normalize_company_domain(raw: str) -> str:
"""Reduce whatever the user typed to the bare Pipedrive subdomain.
Accepts ``mycompany``, ``mycompany.pipedrive.com`` or ``https://mycompany.pipedrive.com``.
Raises ``ValueError`` if the result isn't a plain subdomain, which also pins outbound
traffic to ``*.pipedrive.com`` (no SSRF to arbitrary hosts).
"""
domain = raw.strip().lower()
domain = domain.removeprefix("https://").removeprefix("http://")
domain = domain.split("/")[0]
domain = domain.removesuffix(".pipedrive.com")
if not _SUBDOMAIN_RE.match(domain):
raise ValueError(_INVALID_COMPANY_DOMAIN_ERROR)
return domain
def base_url(company_domain: str) -> str:
return f"https://{normalize_company_domain(company_domain)}.pipedrive.com"
def _get_headers(api_token: str) -> dict[str, str]:
return {"x-api-token": api_token, "Accept": "application/json"}
def _build_url(company_domain: str, path: str, params: dict[str, Any]) -> str:
clean_params = {key: value for key, value in params.items() if value is not None}
url = f"{base_url(company_domain)}{path}"
if not clean_params:
return url
return f"{url}?{urlencode(clean_params)}"
def _cursor_paginator() -> JSONResponseCursorPaginator:
return JSONResponseCursorPaginator(cursor_path="additional_data.next_cursor", cursor_param="cursor")
def _offset_paginator(maximum_offset: Optional[int] = None) -> OffsetPaginator:
# v1 endpoints: start/limit offset. No top-level total; termination is a short/empty page.
return OffsetPaginator(
limit=PAGE_SIZE,
offset=0,
offset_param="start",
limit_param="limit",
total_path=None,
maximum_offset=maximum_offset,
stop_after_empty_page=True,
)
def _build_paginator(config: PipedriveEndpointConfig) -> BasePaginator:
if config.pagination == "cursor":
# v2 endpoints: opaque cursor in the response body; ``limit`` rides in the endpoint params.
return _cursor_paginator()
if config.pagination == "none":
# Documented as unpaginated: a full page must not send a second request at `start=500`.
return SinglePagePaginator()
return _offset_paginator()
def validate_credentials(company_domain: str, api_token: str) -> Optional[int]:
"""Return the status code of a cheap authenticated probe, or ``None`` on transport error.
``/api/v1/users/me`` resolves the token's own user and is reachable by any valid token.
"""
# Built outside the probe so an invalid-domain `ValueError` from `_build_url` propagates to the
# caller rather than being swallowed into `None` by the probe's broad transport-error handler.
url = _build_url(company_domain, "/api/v1/users/me", {})
_ok, status = validate_via_probe(
lambda: make_tracked_session(redact_values=(api_token,)),
url,
headers=_get_headers(api_token),
)
return status
def _coerce_payload(value: Any) -> Optional[dict[str, Any]]:
"""Webhook payload blocks arrive as dicts, or as JSON strings once buffered through parquet."""
if isinstance(value, str | bytes):
try:
value = orjson.loads(value)
except orjson.JSONDecodeError:
return None
return value if isinstance(value, dict) else None
def _parse_timestamp(value: Any) -> Optional[datetime.datetime]:
if not isinstance(value, str):
return None
try:
return dateutil_parser.parse(value)
except (ValueError, OverflowError):
return None
def _webhook_table_transformer(table: pa.Table) -> pa.Table:
"""Reshape raw webhook deliveries into rows matching the pull-API table shape.
A delivery lands as the whole v2 envelope (`{"meta": {...}, "data": {...}, "previous": {...}}`),
so the entity has to be lifted out of `data`. Only the newest delivery per id survives: delta
merge dedupes across syncs but not within one batch, so a `create` followed by a `change` for
the same deal must collapse here or the batch would merge the same key twice.
"""
if "data" not in table.column_names:
return table_from_py_list([])
meta_column = table.column("meta").to_pylist() if "meta" in table.column_names else [None] * table.num_rows
latest_by_id: dict[Any, tuple[Optional[datetime.datetime], dict[str, Any]]] = {}
for data, meta in zip(table.column("data").to_pylist(), meta_column):
row = _coerce_payload(data)
if row is None:
continue
row_id = row.get("id")
if row_id is None:
continue
meta_row = _coerce_payload(meta) or {}
# The Hog template already drops deletions; this keeps a hand-configured webhook that
# subscribed to them from blanking a polled row with a tombstone payload.
if meta_row.get("action") == "delete":
continue
timestamp = _parse_timestamp(meta_row.get("timestamp"))
existing = latest_by_id.get(row_id)
# Later rows win ties so batch arrival order breaks equal or unparseable timestamps.
if existing is None or existing[0] is None or (timestamp is not None and timestamp >= existing[0]):
latest_by_id[row_id] = (timestamp, row)
return table_from_py_list([row for _, row in latest_by_id.values()])
def _webhook_error_result(action: str, error: Exception, status_code: Optional[int]) -> str:
LOGGER.warning(f"Failed to {action} Pipedrive webhook", error=str(error), status_code=status_code)
if status_code in (401, 403):
return (
"Pipedrive rejected the API token while managing the webhook. The token's user needs "
"permission to manage webhooks."
)
if status_code is not None:
return f"Pipedrive API error ({status_code})."
return f"Could not reach Pipedrive: {error}"
def _webhooks_url(company_domain: str, webhook_id: Optional[int] = None) -> str:
path = WEBHOOKS_PATH if webhook_id is None else f"{WEBHOOKS_PATH}/{webhook_id}"
return _build_url(company_domain, path, {})
def _list_webhooks(company_domain: str, api_token: str) -> list[dict[str, Any]]:
session = make_tracked_session(redact_values=(api_token,))
response = session.get(
_webhooks_url(company_domain),
headers=_get_headers(api_token),
timeout=WEBHOOK_REQUEST_TIMEOUT_SECONDS,
)
response.raise_for_status()
data = response.json().get("data")
return [hook for hook in data if isinstance(hook, dict)] if isinstance(data, list) else []
def create_webhook(company_domain: str, api_token: str, webhook_url: str) -> WebhookCreationResult:
"""Register one wildcard webhook (`event_action="*"`, `event_object="*"`) on the account.
Pipedrive scopes a subscription to a single action/object pair, so a per-entity subscription
would burn one of the account's 40 webhook slots per table. The Hog template drops every
entity that isn't mapped to a selected schema before anything is persisted, and the wildcard
can't drift out of sync when the user enables another table later.
"""
password = secrets.token_urlsafe(32)
session = make_tracked_session(redact_values=(api_token, password))
try:
response = session.post(
_webhooks_url(company_domain),
json={
"name": WEBHOOK_NAME,
"subscription_url": webhook_url,
"event_action": "*",
"event_object": "*",
"version": WEBHOOK_PAYLOAD_VERSION,
"http_auth_user": WEBHOOK_AUTH_USER,
"http_auth_password": password,
},
headers=_get_headers(api_token),
timeout=WEBHOOK_REQUEST_TIMEOUT_SECONDS,
)
response.raise_for_status()
except ValueError as e:
return WebhookCreationResult(success=False, error=str(e))
except HTTPError as e:
status_code = e.response.status_code if e.response is not None else None
return WebhookCreationResult(success=False, error=_webhook_error_result("create", e, status_code))
except RequestException as e:
return WebhookCreationResult(success=False, error=_webhook_error_result("create", e, None))
created_id = (response.json().get("data") or {}).get("id")
_remove_superseded_webhooks(session, company_domain, api_token, webhook_url, created_id)
return WebhookCreationResult(
success=True,
extra_inputs={"http_auth_user": WEBHOOK_AUTH_USER, "http_auth_password": password},
)
def _remove_superseded_webhooks(
session: Session, company_domain: str, api_token: str, webhook_url: str, created_id: Any
) -> None:
"""Drop earlier subscriptions on the same URL, best effort, after the replacement is live.
Re-running setup mints a fresh password, so an older subscription would keep delivering with
credentials the Hog function no longer accepts. Cleaning up after the create (rather than
before) means the account is never left without a working webhook if the create fails.
"""
try:
for hook in _list_webhooks(company_domain, api_token):
if hook.get("subscription_url") != webhook_url or hook.get("id") == created_id:
continue
session.delete(
_webhooks_url(company_domain, hook.get("id")),
headers=_get_headers(api_token),
timeout=WEBHOOK_REQUEST_TIMEOUT_SECONDS,
).raise_for_status()
except RequestException as e:
LOGGER.warning("Could not remove superseded Pipedrive webhooks", error=str(e))
def get_external_webhook_info(company_domain: str, api_token: str, webhook_url: str) -> ExternalWebhookInfo:
try:
hooks = _list_webhooks(company_domain, api_token)
except ValueError as e:
return ExternalWebhookInfo(exists=False, error=str(e))
except HTTPError as e:
status_code = e.response.status_code if e.response is not None else None
return ExternalWebhookInfo(exists=False, error=_webhook_error_result("look up", e, status_code))
except RequestException as e:
return ExternalWebhookInfo(exists=False, error=_webhook_error_result("look up", e, None))
matches = [hook for hook in hooks if hook.get("subscription_url") == webhook_url]
if not matches:
return ExternalWebhookInfo(exists=False, url=webhook_url)
return ExternalWebhookInfo(
exists=True,
url=webhook_url,
enabled_events=[f"{hook.get('event_action')}.{hook.get('event_object')}" for hook in matches],
status="active" if any(hook.get("is_active") for hook in matches) else "inactive",
created_at=matches[0].get("add_time"),
description=matches[0].get("name"),
)
def delete_webhook(company_domain: str, api_token: str, webhook_url: str) -> WebhookDeletionResult:
try:
hooks = _list_webhooks(company_domain, api_token)
except ValueError as e:
return WebhookDeletionResult(success=False, error=str(e))
except HTTPError as e:
status_code = e.response.status_code if e.response is not None else None
return WebhookDeletionResult(success=False, error=_webhook_error_result("delete", e, status_code))
except RequestException as e:
return WebhookDeletionResult(success=False, error=_webhook_error_result("delete", e, None))
session = make_tracked_session(redact_values=(api_token,))
for hook in hooks:
if hook.get("subscription_url") != webhook_url:
continue
try:
response = session.delete(
_webhooks_url(company_domain, hook.get("id")),
headers=_get_headers(api_token),
timeout=WEBHOOK_REQUEST_TIMEOUT_SECONDS,
)
response.raise_for_status()
except HTTPError as e:
status_code = e.response.status_code if e.response is not None else None
return WebhookDeletionResult(success=False, error=_webhook_error_result("delete", e, status_code))
except RequestException as e:
return WebhookDeletionResult(success=False, error=_webhook_error_result("delete", e, None))
# Nothing matching left on the account, whether we deleted it now or it was already gone.
return WebhookDeletionResult(success=True)
def _deal_fan_out_rows(
client: RESTClient,
config: PipedriveEndpointConfig,
resumable_source_manager: ResumableSourceManager[PipedriveResumeConfig],
initial_state: Optional[dict[str, Any]],
) -> Iterator[list[dict[str, Any]]]:
"""Read a deal child endpoint for every active and archived deal.
Deals arrive in ascending id order, so the resume state stays small: the parent list and page
being read, the last deal id whose rows are all yielded, and the cursor inside the deal group
being read. The shared fan-out helper instead records every finished parent, which grows with
the account.
"""
state = initial_state or {}
group_size = DEAL_IDS_PER_REQUEST if config.deal_fan_out == "batched" else 1
# A deal deleted between the deals listing and this fetch has nothing left to read.
hooks = create_response_hooks([{"status_code": 404, "action": "ignore"}], resource_name=config.name)
start_parent = state.get("parent_index", 0)
for parent_index, parent_path in enumerate(DEAL_PARENT_PATHS):
if parent_index < start_parent:
continue
resuming = parent_index == start_parent
progress: dict[str, Any] = {
"parent_index": parent_index,
"parent_page": state.get("parent_page") if resuming else None,
"done_through": state.get("done_through") if resuming else None,
"group": None,
"child": None,
}
resume_group = state.get("group") if resuming else None
resume_child = state.get("child") if resuming else None
next_parent_page: dict[str, Any] = {}
def stage(
group: Optional[list[int]], child: Optional[dict[str, Any]], _progress: dict[str, Any] = progress
) -> None:
_progress["group"] = group
_progress["child"] = child
resumable_source_manager.save_state(PipedriveResumeConfig(paginator_state=dict(_progress)))
def on_parent_page(
paginator_state: Optional[dict[str, Any]], has_next: bool, _next: dict[str, Any] = next_parent_page
) -> None:
_next["state"] = paginator_state if has_next else None
def read_group(
group: list[int], child_initial: Optional[dict[str, Any]], _progress: dict[str, Any] = progress
) -> Iterator[list[dict[str, Any]]]:
if config.deal_fan_out == "batched":
path = config.path
params: dict[str, Any] = {"deal_ids": ",".join(str(deal_id) for deal_id in group)}
else:
path = config.path.format(id=group[0])
params = {}
params["limit"] = PAGE_SIZE
def on_child_page(
paginator_state: Optional[dict[str, Any]],
has_next: bool,
_group: list[int] = group,
) -> None:
# Staged before the page is yielded, so the cursor commits with the rows it covers.
if has_next:
stage(_group, paginator_state)
else:
_progress["done_through"] = _group[-1]
stage(None, None)
for rows in client.paginate(
path=path,
params=params,
paginator=_cursor_paginator(),
data_selector="data",
hooks=hooks,
initial_paginator_state=child_initial,
page_state_hook=on_child_page,
):
if config.deal_fan_out == "per_deal":
for row in rows:
row["deal_id"] = group[0]
yield rows
# A group with no rows stages nothing. Resuming from the last staged state reads such
# groups again, which costs requests but yields no duplicate rows.
_progress["done_through"] = group[-1]
resumable_source_manager.safe_point()
# Finish the saved group with its cursor before regrouping: deals added or deleted since the
# crash would change the rebuilt groups, and restarting a half-read batch duplicates rows.
if resume_group:
yield from read_group(resume_group, resume_child)
for deals in client.paginate(
path=parent_path,
params={"limit": PAGE_SIZE, "sort_by": "id", "sort_direction": "asc"},
paginator=_cursor_paginator(),
data_selector="data",
initial_paginator_state=progress["parent_page"],
page_state_hook=on_parent_page,
):
done_through = progress["done_through"]
deal_ids = [deal["id"] for deal in deals if done_through is None or deal["id"] > done_through]
for start in range(0, len(deal_ids), group_size):
yield from read_group(deal_ids[start : start + group_size], None)
progress["parent_page"] = next_parent_page.get("state")
def _list_thread_ids(client: RESTClient, folder: str, stop_offset: Optional[int] = None) -> set[Any]:
"""List the thread ids of a folder, or of its pages before `stop_offset` when it is set."""
if stop_offset is not None and stop_offset <= 0:
return set()
ids: set[Any] = set()
for threads in client.paginate(
path=MAIL_THREADS_PATH,
params={"folder": folder},
paginator=_offset_paginator(maximum_offset=stop_offset),
data_selector="data",
):
ids.update(thread.get("id") for thread in threads)
return ids
def _mail_rows(
client: RESTClient,
config: PipedriveEndpointConfig,
resumable_source_manager: ResumableSourceManager[PipedriveResumeConfig],
initial_state: Optional[dict[str, Any]],
) -> Iterator[list[dict[str, Any]]]:
"""Read the threads of every mail folder, or for `mail_messages` the messages of each thread.
A thread is listed in every folder that holds one of its messages, so a thread already read
from an earlier folder is skipped. Threads come newest message first rather than by id, so the
resume state is the folder, the page being read, and the ids of the threads of that page that
are done. A resume lists the folders and pages before it again to rebuild the skipped threads,
which keeps the state from growing with the mailbox.
"""
state = initial_state or {}
start_folder = state.get("folder_index", 0)
seen: set[Any] = set(state.get("done_ids", []))
if initial_state:
for folder in MAIL_FOLDERS[:start_folder]:
seen |= _list_thread_ids(client, folder)
resume_page = state.get("page")
if start_folder < len(MAIL_FOLDERS) and resume_page:
seen |= _list_thread_ids(client, MAIL_FOLDERS[start_folder], stop_offset=resume_page.get("offset"))
# A thread deleted between the folder listing and this fetch has nothing left to read.
hooks = create_response_hooks([{"status_code": 404, "action": "ignore"}], resource_name=config.name)
def stage(progress: dict[str, Any]) -> None:
resumable_source_manager.save_state(PipedriveResumeConfig(paginator_state=progress))
def stage_page_done(folder_index: int, next_page_state: Optional[dict[str, Any]]) -> None:
if next_page_state:
stage({"folder_index": folder_index, "page": next_page_state})
else:
stage({"folder_index": folder_index + 1})
for folder_index, folder in enumerate(MAIL_FOLDERS):
if folder_index < start_folder:
continue
resuming = folder_index == start_folder
page: Optional[dict[str, Any]] = state.get("page") if resuming else None
done_in_page: list[Any] = list(state.get("done_ids", [])) if resuming else []
next_page: dict[str, Any] = {}
def on_page(
paginator_state: Optional[dict[str, Any]], has_next: bool, _next: dict[str, Any] = next_page
) -> None:
_next["state"] = paginator_state if has_next else None
for threads in client.paginate(
path=MAIL_THREADS_PATH,
params={"folder": folder},
paginator=_offset_paginator(),
data_selector="data",
initial_paginator_state=page,
page_state_hook=on_page,
):
if config.custom_read == "mail_threads":
new_threads = [thread for thread in threads if thread.get("id") not in seen]
seen.update(thread.get("id") for thread in new_threads)
stage_page_done(folder_index, next_page.get("state"))
if new_threads:
yield new_threads
else:
resumable_source_manager.safe_point()
else:
for thread in threads:
thread_id = thread.get("id")
if thread_id in seen:
continue
seen.add(thread_id)
messages: list[dict[str, Any]] = []
for rows in client.paginate(
path=config.path.format(id=thread_id),
paginator=SinglePagePaginator(),
data_selector="data",
hooks=hooks,
):
messages.extend(rows)
if messages:
done_in_page.append(thread_id)
stage({"folder_index": folder_index, "page": page, "done_ids": list(done_in_page)})
yield messages
else:
resumable_source_manager.safe_point()
# Every thread of the page is done, so the next attempt can start at the next page.
stage_page_done(folder_index, next_page.get("state"))
resumable_source_manager.safe_point()
page = next_page.get("state")
done_in_page = []
def _project_phase_rows(
client: RESTClient,
config: PipedriveEndpointConfig,
resumable_source_manager: ResumableSourceManager[PipedriveResumeConfig],
initial_state: Optional[dict[str, Any]],
) -> Iterator[list[dict[str, Any]]]:
"""Read the phases of every project board. The resume state is the ids of the boards done."""
done_boards: list[Any] = list((initial_state or {}).get("done_board_ids", []))
# A board deleted between the board listing and this fetch has no phases left to read.
hooks = create_response_hooks([{"status_code": 404, "action": "ignore"}], resource_name=config.name)
for boards in client.paginate(path=PROJECT_BOARDS_PATH, paginator=SinglePagePaginator(), data_selector="data"):
for board in boards:
if board["id"] in done_boards:
continue
for phases in client.paginate(
path=config.path,
params={"board_id": board["id"]},
paginator=SinglePagePaginator(),
data_selector="data",
hooks=hooks,
):
if phases:
done_boards.append(board["id"])
resumable_source_manager.save_state(
PipedriveResumeConfig(paginator_state={"done_board_ids": list(done_boards)})
)
yield phases
else:
resumable_source_manager.safe_point()
def pipedrive_source(
company_domain: str,
api_token: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[PipedriveResumeConfig],
webhook_source_manager: WebhookSourceManager,
db_incremental_field_last_value: Optional[Any] = None,
api_version: str = "v2",
) -> SourceResponse:
endpoints = endpoints_for_version(api_version)
config = endpoints[endpoint]
client_config: ClientConfig = {
"base_url": base_url(company_domain),
# Only the non-secret Accept header here; the token travels via framework auth so it's
# redacted from logged URLs, headers, sampled bodies, and raised error messages.
"headers": {"Accept": "application/json"},
"auth": {"type": "api_key", "api_key": api_token, "name": "x-api-token", "location": "header"},
"paginator": _build_paginator(config),
# base_url host (`{subdomain}.pipedrive.com`) is implicitly allowed; `[]` pins every
# request — including resume URLs — to it, matching the source's SSRF posture.
"allowed_hosts": [],
}
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
# Only new-shape paginator state can seed a resume; legacy `next_url`-only state restarts
# the endpoint from the beginning (merge dedupes the re-yielded rows).
if resume is not None and resume.paginator_state:
initial_paginator_state = resume.paginator_state
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when a next page remains; saved AFTER a page is yielded so a crash re-yields
# the last page (merge dedupes) rather than skipping it.
if state:
resumable_source_manager.save_state(PipedriveResumeConfig(paginator_state=state))
resource: Any
column_hints = None
if config.deal_fan_out is not None or config.custom_read is not None:
client = RESTClient(
base_url=base_url(company_domain),
headers={"Accept": "application/json"},
auth=APIKeyAuth(api_key=api_token, name="x-api-token", location="header"),
allowed_hosts=[],
)
if config.custom_read == "project_phases":
resource = _project_phase_rows(client, config, resumable_source_manager, initial_paginator_state)
elif config.custom_read is not None:
resource = _mail_rows(client, config, resumable_source_manager, initial_paginator_state)
else:
resource = _deal_fan_out_rows(client, config, resumable_source_manager, initial_paginator_state)
else:
params: dict[str, Any] = {}
if config.pagination == "cursor":
# The cursor paginator only injects ``cursor``; ``limit`` must be a static param.
params["limit"] = PAGE_SIZE
rest_config: RESTAPIConfig = {
"client": client_config,
"resources": [
{
"name": endpoint,
"endpoint": {
"path": config.path,
"params": params,
# Mirrors the old `data.get("data") or []`: a 200 body without `data` yields an
# empty page rather than raising (not required).
"data_selector": "data",
},
}
],
}
resource = rest_api_resource(
rest_config,
team_id,
job_id,
db_incremental_field_last_value,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
column_hints = resource.column_hints
webhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)()
def items() -> Any:
# Once the backfill has landed and the schema is on webhook sync, pushed rows replace the
# poll for this run; the poll stays the only path until then.
if webhook_enabled:
return webhook_source_manager.get_items(table_transformer=_webhook_table_transformer)
return resource
return SourceResponse(
name=endpoint,
items=items,
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
partition_mode="datetime" if config.partition_key else None,
partition_format="week" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
column_hints=column_hints,
)