Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add pandadoc catalog, audit trail and document detail tables (#123998)

Tom Owersffe603ca16e1
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index e5efa6d3aa73..a9b80ad025aa 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -838,8 +838,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/packagist/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/pagerduty.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pagerduty/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/pandadoc.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/paperform/paperform.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/paperform/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/papersign/papersign.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 5c15a1f861ef..4f441aed949d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9103,10 +9103,10 @@ Today (7): `contacts`, `document_folders`, `documents`, `forms`, `members`, `tem
Diffed against: <https://developers.pandadoc.com/llms.txt>
-- [ ] `/public/v2/documents/{document_id}/audit-trail` — full state/transition history per document (sent, viewed, signed, edited) with timestamps - the core e-signature funnel dataset (high)
-- [ ] `/public/v2/product-catalog/items/search` — lookup table resolving the catalog item / SKU ids referenced by quotes and pricing tables (high)
-- [ ] `/public/v1/documents/{id} (detail expansion)` — recipients, tokens, and pricing table line items only appear in document details, not in the list response we sync (high)
-- [ ] `/public/v1/documents/{id}/fields` — per-document field values, the structured data captured on each document (high)
+- [x] `/public/v2/documents/{document_id}/audit-trail` — full state/transition history per document (sent, viewed, signed, edited) with timestamps - the core e-signature funnel dataset (high)
+- [x] `/public/v2/product-catalog/items/search` — lookup table resolving the catalog item / SKU ids referenced by quotes and pricing tables (high)
+- [x] `/public/v1/documents/{id} (detail expansion)` — recipients, tokens, and pricing table line items only appear in document details, not in the list response we sync (high)
+- [x] `/public/v1/documents/{id}/fields` — per-document field values, the structured data captured on each document (high)
- [ ] `/public/v1/documents/{id}/linked-objects` — CRM linkage (Salesforce/HubSpot object ids) - the join key back to deal data (medium)
- [ ] `/public/v1/content-library-items` — content library blocks reused across documents and templates (medium)
- [ ] `/public/v1/workspaces` — lookup resolving the workspace each document, template, and member belongs to (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/canonical_descriptions.py
index 1691fc5e2727..fd8a4d70ba55 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/canonical_descriptions.py
@@ -105,4 +105,89 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"parent_uuid": "Identifier of the parent folder, if nested.",
},
},
+ "product_catalog_items": {
+ "description": "An item in the PandaDoc product catalog, referenced by quotes and pricing tables.",
+ "docs_url": "https://developers.pandadoc.com/reference/search-catalog-items",
+ "columns": {
+ "uuid": "Unique identifier for the catalog item.",
+ "title": "The catalog item's title.",
+ "sku": "The catalog item's SKU.",
+ "description": "The catalog item's description.",
+ "type": "Catalog item type (e.g. regular, bundle).",
+ "billing_type": "Billing type of the item (one_time or recurring).",
+ "billing_cycle": "Billing cycle for recurring items.",
+ "price": "Unit price of the item.",
+ "cost": "Unit cost of the item.",
+ "currency": "Currency code of the item's price.",
+ "pricing_method": "Numeric code of the pricing method (flat or tiered).",
+ "tiers": "Tiered pricing steps, each with a minimum quantity and a value.",
+ "category_id": "Identifier of the catalog category the item belongs to.",
+ "category_name": "Name of the catalog category the item belongs to.",
+ "custom_fields": "Custom field names and values set on the item.",
+ "workspace_id": "Identifier of the workspace the item belongs to.",
+ "created_by": "Identifier of the user who created the item.",
+ "modified_by": "Identifier of the user who last modified the item.",
+ "date_created": "Time at which the item was created.",
+ "date_modified": "Time at which the item was last modified.",
+ },
+ },
+ "document_audit_trail": {
+ "description": "One event in a document's audit trail, such as the document being sent, viewed, signed, or edited.",
+ "docs_url": "https://developers.pandadoc.com/reference/list-document-audit-trail",
+ "columns": {
+ "id": "Unique identifier of the audit trail entry.",
+ "document_id": "Identifier of the document the entry belongs to.",
+ "user": "The user who triggered the action (id and email).",
+ "action": "Numeric code of the action (e.g. 1 new document, 6 sent, 8 viewed, 9 completed, 7 completed by all).",
+ "reason": "Optional reason or explanation for the action.",
+ "date_created": "Time at which the action happened.",
+ "ip_address": "IP address the action came from.",
+ },
+ },
+ "document_fields": {
+ "description": "A field placed on a document, with the value a recipient filled in.",
+ "docs_url": "https://developers.pandadoc.com/reference/list-document-fields",
+ "columns": {
+ "uuid": "Unique identifier of the field.",
+ "document_id": "Identifier of the document the field belongs to.",
+ "field_id": "The field's identifier within the document.",
+ "name": "The field's name.",
+ "title": "The field's title.",
+ "type": "Field type (e.g. text, signature, date, checkbox, dropdown).",
+ "value": "Value entered in the field.",
+ "assigned_to": "The recipient the field is assigned to.",
+ "settings": "Field settings such as required and placeholder.",
+ "layout": "Page and position of the field in the document.",
+ "merge_field": "Merge field the field is bound to, if any.",
+ },
+ },
+ "document_details": {
+ "description": "Full details of a document, including recipients, fields, tokens, and pricing.",
+ "docs_url": "https://developers.pandadoc.com/reference/document-details",
+ "columns": {
+ "id": "Unique identifier for the document.",
+ "name": "The document's name.",
+ "status": "Current status of the document (e.g. document.draft, document.sent, document.completed).",
+ "date_created": "Time at which the document was created.",
+ "date_modified": "Time of the last change to the document's recipients or status.",
+ "content_date_modified": "Time of the last change to the document's content.",
+ "date_sent": "Time at which the document was sent.",
+ "date_completed": "Time at which the document was completed, if applicable.",
+ "expiration_date": "Time at which the document expires, if set.",
+ "ref_number": "Reference number of the document.",
+ "folder_uuid": "Identifier of the folder that contains the document.",
+ "template": "The template the document was created from, if any.",
+ "created_by": "The user who created the document.",
+ "sent_by": "The user who sent the document.",
+ "recipients": "Recipients of the document, with completion status and signing order.",
+ "fields": "Fields on the document, with their values.",
+ "tokens": "Token (variable) names and values in the document.",
+ "pricing": "Pricing tables and quotes in the document, with their line items.",
+ "grand_total": "Grand total amount of the document's pricing.",
+ "metadata": "Custom key-value metadata attached to the document.",
+ "tags": "Tags attached to the document.",
+ "linked_objects": "CRM objects linked to the document.",
+ "version": "Version number of the document.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/pandadoc.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/pandadoc.py
index 440d873d5c85..6b387a167dee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/pandadoc.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/pandadoc.py
@@ -1,45 +1,56 @@
import dataclasses
from datetime import UTC, date, datetime
-from typing import Any, Optional
+from typing import Any, Optional, cast
from requests import Response
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
+ OffsetPaginator,
PageNumberPaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pandadoc.settings import (
+ PAGE_SIZE,
PANDADOC_ENDPOINTS,
PandaDocEndpointConfig,
)
# PandaDoc versions its API per endpoint via a URL segment (/public/v1, /public/v2), not
# globally: /public/v2 is a small, growing set of new endpoints (API logs, document audit
-# trail), NOT a mirror of v1. Every resource this source reads (documents, templates, forms,
-# contacts, members, folders) is served only under /public/v1 and has no v2 route, so both
-# pins resolve to the same wire here and the base URL stays on v1 regardless of the pin.
+# trail, product catalog search), NOT a mirror of v1. Each resource this source reads is
+# served under exactly one of the two, so every endpoint path carries its own version segment
+# and both pins resolve to the same wire.
API_VERSION_V1 = "v1"
API_VERSION_V2 = "v2"
-PANDADOC_BASE_URL = "https://api.pandadoc.com/public/v1"
-# PandaDoc list pages cap at 100 items.
-PAGE_SIZE = 100
+PANDADOC_BASE_URL = "https://api.pandadoc.com/public"
+@frozen
class PandaDocResumeConfig:
# PandaDoc paginates with a 1-based page number; the static query params
# (count, incremental filters, sort) are deterministically rebuilt from the
# endpoint config and job inputs on resume.
- page: int
+ page: Optional[int] = None
+ # Per-document child endpoints checkpoint the `build_dependent_resource` state instead
+ # (which documents finished, where the current one stopped).
+ fanout_state: Optional[dict[str, Any]] = None
class PandaDocPagePaginator(PageNumberPaginator):
@@ -59,6 +70,41 @@ class PandaDocPagePaginator(PageNumberPaginator):
self._has_next_page = False
+class PandaDocHasMorePaginator(PageNumberPaginator):
+ """1-based page paginator for v2 list endpoints that flag the last page in the body."""
+
+ def __init__(self, has_more_path: str) -> None:
+ super().__init__(base_page=1, page_param="page")
+ self.has_more_path = has_more_path
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ super().update_state(response, data)
+ body = response.json()
+ if isinstance(body, dict) and body.get(self.has_more_path) is False:
+ self._has_next_page = False
+
+
+def _paginator_for(config: PandaDocEndpointConfig) -> BasePaginator:
+ if config.pagination == "none":
+ return SinglePagePaginator()
+ if config.pagination == "offset":
+ return OffsetPaginator(
+ limit=config.page_size, limit_param=config.page_size_param, offset_param="offset", total_path="count"
+ )
+ if config.has_more_path:
+ return PandaDocHasMorePaginator(config.has_more_path)
+ return PandaDocPagePaginator()
+
+
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": PANDADOC_BASE_URL,
+ # Auth is supplied via the framework so the key is redacted from logs and errors;
+ # PandaDoc uses an "API-Key <key>" Authorization scheme rather than a bare Bearer.
+ "auth": {"type": "api_key", "api_key": f"API-Key {api_key}", "name": "Authorization"},
+ }
+
+
def _format_date_filter(value: Any) -> str:
"""Format an incremental cursor for PandaDoc's date filters (ISO 8601 UTC, e.g. 2024-01-02T03:04:05.000000Z)."""
if isinstance(value, datetime):
@@ -83,8 +129,8 @@ def _build_query_params(
"""
params: dict[str, Any] = {}
- if config.paginated:
- params["count"] = PAGE_SIZE
+ if config.pagination == "page":
+ params[config.page_size_param] = config.page_size
if not config.incremental_params:
return params
@@ -105,6 +151,55 @@ def _build_query_params(
return params
+def _document_child_resource(
+ api_key: str,
+ config: PandaDocEndpointConfig,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[PandaDocResumeConfig],
+) -> Resource:
+ assert config.fanout is not None
+ parent = PANDADOC_ENDPOINTS[config.fanout.parent_name]
+ # The parent listing is a full walk sorted on the stable creation date, like a full refresh of
+ # the documents table, so documents modified mid-sync don't move across page boundaries.
+ fanout = dataclasses.replace(
+ config.fanout,
+ parent_params=_build_query_params(
+ parent, should_use_incremental_field=False, db_incremental_field_last_value=None, incremental_field=None
+ ),
+ )
+
+ initial_fanout_state: Optional[dict[str, Any]] = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None:
+ initial_fanout_state = resume.fanout_state
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(PandaDocResumeConfig(fanout_state=state))
+
+ return cast(
+ Resource,
+ build_dependent_resource(
+ endpoint_configs={config.name: config, parent.name: parent},
+ child_endpoint=config.name,
+ fanout=fanout,
+ client_config=_client_config(api_key),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ # Parent and child name their page-size params differently, so each sets its own.
+ page_size_param=None,
+ parent_endpoint_extra={"paginator": _paginator_for(parent), "data_selector": parent.data_key},
+ child_endpoint_extra={"paginator": _paginator_for(config), "data_selector": config.data_key},
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=initial_fanout_state,
+ ),
+ )
+
+
def pandadoc_source(
api_key: str,
endpoint: str,
@@ -116,29 +211,40 @@ def pandadoc_source(
incremental_field: str | None = None,
) -> SourceResponse:
config = PANDADOC_ENDPOINTS[endpoint]
+ partition_kwargs: dict[str, Any] = {
+ "partition_count": 1,
+ "partition_size": 1,
+ "partition_mode": "datetime" if config.partition_key else None,
+ "partition_format": "month" if config.partition_key else None,
+ "partition_keys": [config.partition_key] if config.partition_key else None,
+ }
+
+ if config.fanout is not None:
+ child = _document_child_resource(api_key, config, team_id, job_id, resumable_source_manager)
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: child,
+ primary_keys=config.primary_keys,
+ column_hints=child.column_hints,
+ **partition_kwargs,
+ )
params = _build_query_params(
config, should_use_incremental_field, db_incremental_field_last_value, incremental_field
)
- paginator = PandaDocPagePaginator() if config.paginated else SinglePagePaginator()
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": PANDADOC_BASE_URL,
- # Auth is supplied via the framework so the key is redacted from logs and errors;
- # PandaDoc uses an "API-Key <key>" Authorization scheme rather than a bare Bearer.
- "auth": {"type": "api_key", "api_key": f"API-Key {api_key}", "name": "Authorization"},
- },
+ "client": _client_config(api_key),
"resources": [
{
"name": endpoint,
"endpoint": {
"path": config.path,
"params": params,
- # Every list endpoint wraps its rows in "results"; a missing key is treated as
- # an empty page (matching the hand-rolled source), so this is not required.
+ # List endpoints wrap their rows in "results" (or "items" on v2); a missing key is
+ # treated as an empty page (matching the hand-rolled source), so this is not required.
"data_selector": config.data_key,
- "paginator": paginator,
+ "paginator": _paginator_for(config),
},
}
],
@@ -147,7 +253,7 @@ def pandadoc_source(
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
- if resume is not None:
+ if resume is not None and resume.page is not None:
initial_paginator_state = {"page": resume.page}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
@@ -168,14 +274,10 @@ def pandadoc_source(
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=[config.primary_key],
- partition_count=1,
- partition_size=1,
- partition_mode="datetime" if config.partition_key else None,
- partition_format="month" if config.partition_key else None,
- partition_keys=[config.partition_key] if config.partition_key else None,
+ primary_keys=config.primary_keys,
sort_mode="asc",
column_hints=resource.column_hints,
+ **partition_kwargs,
)
@@ -183,7 +285,7 @@ def validate_credentials(api_key: str) -> bool:
"""Confirm the API key is valid with a cheap one-document listing probe."""
ok, _status = validate_via_probe(
lambda: make_tracked_session(redact_values=(api_key,)),
- f"{PANDADOC_BASE_URL}/documents?count=1&page=1",
+ f"{PANDADOC_BASE_URL}/v1/documents?count=1&page=1",
headers={"Authorization": f"API-Key {api_key}"},
)
return ok
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/settings.py
index bf9ebb72d9fe..638fae8b11f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/settings.py
@@ -1,15 +1,25 @@
-from dataclasses import dataclass, field
-from typing import Optional
+from dataclasses import field
+from typing import Literal, Optional
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ResponseAction
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+# PandaDoc list pages cap at 100 items.
+PAGE_SIZE = 100
+
-@dataclass
+@frozen
class PandaDocEndpointConfig:
name: str
+ # Relative to https://api.pandadoc.com/public, so it carries the endpoint's own /v1 or /v2 segment.
path: str
- primary_key: str = "id"
- # Body key the list of rows lives under (every list endpoint wraps in "results").
+ primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ # Body key the list of rows lives under ("$" for a single-object response).
data_key: str = "results"
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Maps an incremental cursor field to the server-side query param that filters on it
@@ -18,14 +28,43 @@ class PandaDocEndpointConfig:
# Stable creation-time field used for datetime partitioning. Never an
# updated_at-style field, which would rewrite partitions on every sync.
partition_key: Optional[str] = None
- # Some endpoints (contacts, members) return the full list without pagination.
- paginated: bool = True
+ # "page": 1-based page number. "offset": limit/offset with a `count` total.
+ # "none": the endpoint returns the full list in one response (contacts, members, per-document children).
+ pagination: Literal["page", "offset", "none"] = "page"
+ page_size_param: str = "count"
+ page_size: int = PAGE_SIZE
+ # Body flag some v2 list endpoints return to mark whether another page exists.
+ has_more_path: Optional[str] = None
+ default_incremental_field: Optional[str] = None
+ fanout: Optional[DependentEndpointConfig] = None
+
+
+# A document deleted between the parent listing and the child fetch 404s, and a document still
+# being processed (document.uploaded) 409s. Both are skipped so the next sync picks them up.
+_DOCUMENT_CHILD_IGNORE_404: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]
+_DOCUMENT_CHILD_IGNORE_404_409: list[ResponseAction] = [
+ {"status_code": 404, "action": "ignore"},
+ {"status_code": 409, "action": "ignore"},
+]
+
+
+def _document_fanout(
+ include_document_id: bool, child_response_actions: list[ResponseAction]
+) -> DependentEndpointConfig:
+ return DependentEndpointConfig(
+ parent_name="documents",
+ resolve_param="document_id",
+ resolve_field="id",
+ include_from_parent=["id"] if include_document_id else [],
+ parent_field_renames={"id": "document_id"} if include_document_id else {},
+ child_response_actions=child_response_actions,
+ )
PANDADOC_ENDPOINTS: dict[str, PandaDocEndpointConfig] = {
"documents": PandaDocEndpointConfig(
name="documents",
- path="/documents",
+ path="/v1/documents",
partition_key="date_created",
incremental_params={
"date_modified": "modified_from",
@@ -48,32 +87,71 @@ PANDADOC_ENDPOINTS: dict[str, PandaDocEndpointConfig] = {
),
"templates": PandaDocEndpointConfig(
name="templates",
- path="/templates",
+ path="/v1/templates",
),
"forms": PandaDocEndpointConfig(
name="forms",
- path="/forms",
+ path="/v1/forms",
),
"contacts": PandaDocEndpointConfig(
name="contacts",
- path="/contacts",
- paginated=False,
+ path="/v1/contacts",
+ pagination="none",
),
"members": PandaDocEndpointConfig(
name="members",
- path="/members",
- primary_key="user_id",
- paginated=False,
+ path="/v1/members",
+ primary_keys=["user_id"],
+ pagination="none",
),
"document_folders": PandaDocEndpointConfig(
name="document_folders",
- path="/documents/folders",
- primary_key="uuid",
+ path="/v1/documents/folders",
+ primary_keys=["uuid"],
),
"template_folders": PandaDocEndpointConfig(
name="template_folders",
- path="/templates/folders",
- primary_key="uuid",
+ path="/v1/templates/folders",
+ primary_keys=["uuid"],
+ ),
+ # The search endpoint has no date filter, so the catalog is full refresh only.
+ "product_catalog_items": PandaDocEndpointConfig(
+ name="product_catalog_items",
+ path="/v2/product-catalog/items/search",
+ primary_keys=["uuid"],
+ data_key="items",
+ partition_key="date_created",
+ page_size_param="per_page",
+ has_more_path="has_more_items",
+ ),
+ # The document children are full refresh only. The audit trail and fields endpoints take no
+ # date filter, and PandaDoc's `date_modified` does not move on content edits, so bounding the
+ # parent listing by it would leave tokens, pricing and field values stale.
+ "document_audit_trail": PandaDocEndpointConfig(
+ name="document_audit_trail",
+ path="/v2/documents/{document_id}/audit-trail",
+ primary_keys=["document_id", "id"],
+ partition_key="date_created",
+ pagination="offset",
+ page_size_param="limit",
+ fanout=_document_fanout(include_document_id=True, child_response_actions=_DOCUMENT_CHILD_IGNORE_404),
+ ),
+ "document_fields": PandaDocEndpointConfig(
+ name="document_fields",
+ path="/v1/documents/{document_id}/fields",
+ primary_keys=["document_id", "uuid"],
+ data_key="fields",
+ pagination="none",
+ fanout=_document_fanout(include_document_id=True, child_response_actions=_DOCUMENT_CHILD_IGNORE_404_409),
+ ),
+ "document_details": PandaDocEndpointConfig(
+ name="document_details",
+ path="/v1/documents/{document_id}/details",
+ data_key="$",
+ partition_key="date_created",
+ pagination="none",
+ # The detail row already carries the document `id`.
+ fanout=_document_fanout(include_document_id=False, child_response_actions=_DOCUMENT_CHILD_IGNORE_404_409),
),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/source.py
index e7dcf127cb7a..405055e0e369 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/source.py
@@ -40,8 +40,8 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
class PandaDocSource(ResumableSource[PandaDocSourceConfig, PandaDocResumeConfig]):
codebase_detection = detect(env=("^PANDADOC_",), npm=("pandadoc-node-client",), pypi=("pandadoc-python-client",))
- # v2 is declared so new sources are stamped with the vendor's current API generation, but the
- # endpoints this source reads are v1-only (see pandadoc.py), so both pins resolve to the same wire.
+ # v2 is declared so new sources are stamped with the vendor's current API generation. PandaDoc
+ # versions per endpoint (see pandadoc.py), so both pins resolve to the same wire.
supported_versions = (API_VERSION_V1, API_VERSION_V2)
default_version = API_VERSION_V2
api_docs_url = "https://developers.pandadoc.com/reference/version"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/tests/test_pandadoc.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/tests/test_pandadoc.py
index b24cd8d027fa..6353065939b2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/tests/test_pandadoc.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pandadoc/tests/test_pandadoc.py
@@ -5,6 +5,7 @@ import pytest
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pandadoc import (
@@ -96,3 +97,106 @@ class TestPagination:
assert result.raised is None
assert result.requests[0].param("modified_from") == "2024-01-01T00:00:00.000000Z"
assert result.requests[0].param("order_by") == "date_modified"
+
+
+def _documents_listing(*ids: str) -> list[ScriptedResponse]:
+ return [ScriptedResponse(json={"results": [{"id": doc_id} for doc_id in ids]})]
+
+
+class TestProductCatalogItems:
+ def test_walks_pages_until_has_more_items_is_false(self) -> None:
+ result = TestPagination._driver().run(
+ "product_catalog_items",
+ [
+ ScriptedResponse(json={"items": [{"uuid": "a"}], "has_more_items": True, "total": 2}),
+ ScriptedResponse(json={"items": [{"uuid": "b"}], "has_more_items": False, "total": 2}),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["uuid"] for row in result.rows] == ["a", "b"]
+ assert result.paths == ["/public/v2/product-catalog/items/search"] * 2
+ assert result.params("page") == ["1", "2"]
+ assert result.params("per_page") == [str(PAGE_SIZE)] * 2
+
+
+class TestDocumentChildren:
+ def test_audit_trail_pages_each_document_and_skips_deleted_ones(self) -> None:
+ result = TestPagination._driver().run(
+ "document_audit_trail",
+ route(
+ {
+ "/public/v1/documents": _documents_listing("docA", "docB"),
+ "/public/v2/documents/docA/audit-trail": [
+ ScriptedResponse(json={"count": 101, "results": [{"id": f"e{i}"} for i in range(PAGE_SIZE)]}),
+ ScriptedResponse(json={"count": 101, "results": [{"id": "e100"}]}),
+ ],
+ "/public/v2/documents/docB/audit-trail": [
+ ScriptedResponse(status=404, json={"detail": "Not found"})
+ ],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert len(result.rows) == PAGE_SIZE + 1
+ assert {row["document_id"] for row in result.rows} == {"docA"}
+ assert [r.param("offset") for r in result.requests if "audit-trail" in r.path] == ["0", "100", "0"]
+ assert result.response is not None
+ assert result.response.primary_keys == ["document_id", "id"]
+
+ @pytest.mark.parametrize(
+ "endpoint, path_suffix, body, expected_rows",
+ [
+ (
+ "document_fields",
+ "fields",
+ {"fields": [{"uuid": "f1", "value": "x"}]},
+ [{"uuid": "f1", "value": "x", "document_id": "docA"}],
+ ),
+ (
+ "document_details",
+ "details",
+ {"id": "docA", "status": "document.completed", "recipients": [{"email": "[email protected]"}]},
+ [{"id": "docA", "status": "document.completed", "recipients": [{"email": "[email protected]"}]}],
+ ),
+ ],
+ )
+ def test_per_document_endpoints_skip_documents_still_processing(
+ self, endpoint, path_suffix, body, expected_rows
+ ) -> None:
+ result = TestPagination._driver().run(
+ endpoint,
+ route(
+ {
+ "/public/v1/documents": _documents_listing("docA", "docB"),
+ f"/public/v1/documents/docA/{path_suffix}": [ScriptedResponse(json=body)],
+ f"/public/v1/documents/docB/{path_suffix}": [
+ ScriptedResponse(status=409, json={"id": "docB", "status": "document.uploaded"})
+ ],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == expected_rows
+
+ def test_resume_skips_already_completed_document(self) -> None:
+ result = TestPagination._driver().run(
+ "document_fields",
+ route(
+ {
+ "/public/v1/documents": _documents_listing("docA", "docB"),
+ # No entry for docA: a request there would fail, proving a resumed run
+ # does not re-fetch a document it already finished.
+ "/public/v1/documents/docB/fields": [ScriptedResponse(json={"fields": [{"uuid": "fB"}]})],
+ }
+ ),
+ resume_state=PandaDocResumeConfig(
+ fanout_state={"completed": ["/v1/documents/docA/fields"], "current": None, "child_state": None}
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/public/v1/documents", "/public/v1/documents/docB/fields"]
+ assert [row["uuid"] for row in result.rows] == ["fB"]