Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Wufoo source (#120956)

Tom Owers175bbcf72c30
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 51367c42e2f2..984f45fe0231 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1192,8 +1192,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/wrike/wrike.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/writesonic/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/writesonic/writesonic.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/wufoo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/xendit/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/xendit/xendit.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/yousign/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index e49a26475ce0..4888c976d54c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -13377,10 +13377,10 @@ Today (3): `forms`, `reports`, `users`
Diffed against: <https://wufoo.github.io/docs/>
-- [ ] `form entries (GET /api/v3/forms/{hash}/entries.json)` — the actual form submission data; without it the source syncs metadata only (high)
-- [ ] `form fields (GET /api/v3/forms/{hash}/fields.json)` — lookup mapping Field1/Field2 entry column ids to human field titles and choice values (high)
-- [ ] `report entries (GET /api/v3/reports/{hash}/entries.json)` — the filtered entry set behind each report we already sync (medium)
-- [ ] `report fields (GET /api/v3/reports/{hash}/fields.json)` — lookup resolving column ids on report entries (medium)
+- [x] `form entries (GET /api/v3/forms/{hash}/entries.json)` — the actual form submission data; without it the source syncs metadata only (high)
+- [x] `form fields (GET /api/v3/forms/{hash}/fields.json)` — lookup mapping Field1/Field2 entry column ids to human field titles and choice values (high)
+- [x] `report entries (GET /api/v3/reports/{hash}/entries.json)` — the filtered entry set behind each report we already sync (medium)
+- [x] `report fields (GET /api/v3/reports/{hash}/fields.json)` — lookup resolving column ids on report entries (medium)
- [ ] `form entries count (GET /api/v3/forms/{hash}/entries/count.json)` — cheap submission-volume metric per form (low)
- [ ] `report widgets (GET /api/v3/reports/{hash}/widgets.json)` — the aggregate/chart definitions attached to a report (low)
- [ ] `form comments (GET /api/v3/forms/{hash}/comments.json)` — entry-level comments for review workflows (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/canonical_descriptions.py
index b2f4b5a9c958..29e65d22f1f7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/canonical_descriptions.py
@@ -54,4 +54,66 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"TimeZone": "The user's configured time zone.",
},
},
+ "form_entries": {
+ "description": "An entry submitted to a Wufoo form. Each FieldN column holds the answer to one form "
+ "field; join to form_fields on form_hash and ID to read the field titles.",
+ "docs_url": "https://wufoo.github.io/docs/#form-entries",
+ "columns": {
+ "form_hash": "The hash of the form that the entry belongs to.",
+ "EntryId": "The identifier of the entry, unique within its form.",
+ "DateCreated": "The date and time the entry was submitted, in the timezone of the API key's user.",
+ "CreatedBy": "Who created the entry: 'public' for a form submission, or the user name for an entry "
+ "added in the Entry Manager.",
+ "DateUpdated": "The date and time the entry was last edited in the Entry Manager; blank if never edited.",
+ "UpdatedBy": "The user name of the person who last edited the entry in the Entry Manager.",
+ "IP": "The IP address of the person who submitted the form.",
+ "LastPage": "The last page of the form that the person submitted.",
+ "CompleteSubmission": "Whether the entry is complete ('1') or partial ('0').",
+ "Status": "The payment status of the entry, for example 'Paid'.",
+ "PurchaseTotal": "The total amount sent to the payment processor.",
+ "Currency": "The currency sent to the payment processor.",
+ "TransactionId": "The confirmation number returned by the payment processor.",
+ "MerchantType": "The name of the payment processor used.",
+ },
+ },
+ "form_fields": {
+ "description": "A field on a Wufoo form. Maps the FieldN column IDs on form_entries to field titles, types, "
+ "and choices.",
+ "docs_url": "https://wufoo.github.io/docs/#form-fields",
+ "columns": {
+ "form_hash": "The hash of the form that the field belongs to.",
+ "ID": "The API ID of the field, for example 'Field105'. Matches a column on form_entries.",
+ "Title": "The field label.",
+ "Type": "The field type, for example 'text', 'checkbox', or 'radio'.",
+ "Instructions": "The instructions shown to the user for this field.",
+ "IsRequired": "Whether the field is required ('1') or optional ('0').",
+ "ClassNames": "The CSS keywords set on the field in the Form Builder.",
+ "DefaultVal": "The predefined value of the field.",
+ "Page": "The page of the form that holds the field.",
+ "SubFields": "The sub-fields of a multi-part field, such as checkbox options or address parts, each "
+ "with its own ID and label.",
+ "Choices": "The choices of a dropdown, multiple choice, or likert field.",
+ "IsSystem": "Whether the field is a system field, such as IP address or payment status.",
+ },
+ },
+ "report_entries": {
+ "description": "An entry that a Wufoo report includes, after the report's filters are applied.",
+ "docs_url": "https://wufoo.github.io/docs/#report-entries",
+ "columns": {
+ "report_hash": "The hash of the report that includes the entry.",
+ "EntryId": "The identifier of the entry, unique within the report's form.",
+ "DateCreated": "The date and time the entry was submitted.",
+ },
+ },
+ "report_fields": {
+ "description": "A field on the form that a Wufoo report is built on. Maps the FieldN column IDs on "
+ "report_entries to field titles.",
+ "docs_url": "https://wufoo.github.io/docs/#report-fields",
+ "columns": {
+ "report_hash": "The hash of the report.",
+ "ID": "The API ID of the field, for example 'Field105'. Matches a column on report_entries.",
+ "Title": "The field label.",
+ "Type": "The field type, for example 'text', 'checkbox', or 'radio'.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/settings.py
index 859f3093b071..eac5e818600b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/settings.py
@@ -1,9 +1,35 @@
from dataclasses import dataclass, field
+from typing import Any
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField
+# Wufoo caps `pageSize` at 100 rows per request.
+PAGE_SIZE = 100
+# `forms.json` documents `page`/`limit` paging with a maximum `limit` of 1000.
+FORMS_PAGE_LIMIT = 1000
-@dataclass
+FORM_FANOUT = DependentEndpointConfig(
+ parent_name="forms",
+ resolve_param="hash",
+ resolve_field="Hash",
+ include_from_parent=["Hash"],
+ parent_field_renames={"Hash": "form_hash"},
+ parent_params={"limit": FORMS_PAGE_LIMIT},
+)
+
+REPORT_FANOUT = DependentEndpointConfig(
+ parent_name="reports",
+ resolve_param="hash",
+ resolve_field="Hash",
+ include_from_parent=["Hash"],
+ parent_field_renames={"Hash": "report_hash"},
+)
+
+
+@dataclass(frozen=True)
class WufooEndpointConfig:
name: str
# Path relative to https://<subdomain>.wufoo.com/api/v3 (includes the .json suffix).
@@ -13,19 +39,62 @@ class WufooEndpointConfig:
# Wufoo identifies every top-level object with a stable `Hash`, so it is a safe table-wide
# primary key (forms, reports, and users all expose one).
primary_keys: list[str] = field(default_factory=lambda: ["Hash"])
+ fanout: DependentEndpointConfig | None = None
+ # Whether the endpoint takes `pageStart`/`pageSize`. Fields endpoints return the whole list.
+ paginated: bool = True
+ params: dict[str, Any] = field(default_factory=dict)
+ page_size: int = PAGE_SIZE
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ default_incremental_field: str | None = None
-# Top-level Wufoo REST API v3 list endpoints. Per-form fan-out resources (a form's fields,
-# entries, comments, widgets) require a parent form hash and are intentionally left out of this
-# first cut — they can't be listed without first enumerating forms.
WUFOO_ENDPOINTS: dict[str, WufooEndpointConfig] = {
"forms": WufooEndpointConfig(name="forms", path="forms.json", data_key="Forms"),
"reports": WufooEndpointConfig(name="reports", path="reports.json", data_key="Reports"),
"users": WufooEndpointConfig(name="users", path="users.json", data_key="Users"),
+ # EntryId and field IDs are unique only within one form or report, so the parent hash is part
+ # of every child key.
+ "form_entries": WufooEndpointConfig(
+ name="form_entries",
+ path="forms/{hash}/entries.json",
+ data_key="Entries",
+ primary_keys=["form_hash", "EntryId"],
+ fanout=FORM_FANOUT,
+ # An explicit sort keeps offset pages stable while new entries arrive during the sync.
+ params={"system": "true", "sort": "EntryId", "sortDirection": "ASC"},
+ ),
+ "form_fields": WufooEndpointConfig(
+ name="form_fields",
+ path="forms/{hash}/fields.json",
+ data_key="Fields",
+ primary_keys=["form_hash", "ID"],
+ fanout=FORM_FANOUT,
+ paginated=False,
+ params={"system": "true"},
+ ),
+ "report_entries": WufooEndpointConfig(
+ name="report_entries",
+ path="reports/{hash}/entries.json",
+ data_key="Entries",
+ primary_keys=["report_hash", "EntryId"],
+ fanout=REPORT_FANOUT,
+ params={"system": "true"},
+ ),
+ "report_fields": WufooEndpointConfig(
+ name="report_fields",
+ path="reports/{hash}/fields.json",
+ data_key="Fields",
+ primary_keys=["report_hash", "ID"],
+ fanout=REPORT_FANOUT,
+ paginated=False,
+ params={"system": "true"},
+ ),
}
ENDPOINTS = tuple(WUFOO_ENDPOINTS.keys())
-# Every endpoint is full refresh — Wufoo's account-level list endpoints expose no server-side
-# `updated_after`-style cursor, so there is no genuine incremental field to advance.
+# Every endpoint is full refresh. The account-level list endpoints expose no server-side
+# `updated_after`-style cursor. Entries can be edited after submission, and the Entries API filter
+# reads dates in a fixed PST/PDT zone while it returns them in the account's zone, so a
+# `DateCreated` watermark could skip rows.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/source.py
index eb0fe9d04b43..d6df4ac31089 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/source.py
@@ -82,9 +82,8 @@ class WufooSource(ResumableSource[WufooSourceConfig, WufooResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Every endpoint is full refresh only — Wufoo's account-level list endpoints expose no
- # server-side timestamp filter, so there is no incremental cursor to advance (INCREMENTAL_FIELDS
- # is empty, so build_endpoint_schemas marks every endpoint non-incremental / non-append).
+ # Every endpoint is full refresh only (see INCREMENTAL_FIELDS in settings.py), so
+ # build_endpoint_schemas marks every endpoint non-incremental / non-append.
return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
def validate_credentials(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/tests/test_wufoo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/tests/test_wufoo.py
index d358289b64c9..7443f8b30838 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/tests/test_wufoo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/tests/test_wufoo.py
@@ -6,10 +6,11 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.wufoo import WufooSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.wufoo.settings import ENDPOINTS, WUFOO_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.wufoo.settings import FORMS_PAGE_LIMIT
from products.warehouse_sources.backend.temporal.data_imports.sources.wufoo.source import WufooSource
from products.warehouse_sources.backend.temporal.data_imports.sources.wufoo.wufoo import (
PAGE_SIZE,
@@ -88,6 +89,189 @@ class TestWufooSourceResponse:
assert response.name == endpoint
assert response.primary_keys == ["Hash"]
- def test_every_endpoint_uses_hash_primary_key(self) -> None:
- assert all(config.primary_keys == ["Hash"] for config in WUFOO_ENDPOINTS.values())
- assert set(WUFOO_ENDPOINTS) == set(ENDPOINTS)
+
+def _rows(data_key: str, rows: list[dict[str, Any]]) -> ScriptedResponse:
+ return ScriptedResponse(json={data_key: rows})
+
+
+def _entries(start: int, count: int) -> ScriptedResponse:
+ return _rows("Entries", [{"EntryId": str(i), "Field1": f"answer {i}"} for i in range(start, start + count)])
+
+
+class TestFanout:
+ def test_form_entries_walk_every_form_and_tag_rows_with_form_hash(self) -> None:
+ result = _driver().run(
+ "form_entries",
+ route(
+ {
+ "/forms.json": [_rows("Forms", [{"Hash": "forma"}, {"Hash": "formb"}])],
+ "/forms/forma/entries.json": [_entries(1, PAGE_SIZE), _entries(PAGE_SIZE + 1, 2)],
+ "/forms/formb/entries.json": [_entries(1, 0)],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert len(result.rows) == PAGE_SIZE + 2
+ assert {row["form_hash"] for row in result.rows} == {"forma"}
+ assert all("Hash" not in row and "_forms_Hash" not in row for row in result.rows)
+
+ parent, *children = result.requests
+ assert (parent.param("page"), parent.param("limit"), parent.param("pageStart")) == (
+ "1",
+ str(FORMS_PAGE_LIMIT),
+ None,
+ )
+ assert [(r.path.rsplit("/", 2)[-2], r.param("pageStart")) for r in children] == [
+ ("forma", "0"),
+ ("forma", str(PAGE_SIZE)),
+ ("formb", "0"),
+ ]
+ assert all(
+ (r.param("pageSize"), r.param("sort"), r.param("sortDirection"), r.param("system"))
+ == (str(PAGE_SIZE), "EntryId", "ASC", "true")
+ for r in children
+ )
+
+ @parameterized.expand(
+ [
+ # Fields endpoints are single responses, so a full-looking page is not followed.
+ (
+ "form_fields_single_page",
+ "form_fields",
+ "/forms.json",
+ "Forms",
+ "/forms/p1/fields.json",
+ "Fields",
+ PAGE_SIZE,
+ 1,
+ ),
+ (
+ "report_fields_single_page",
+ "report_fields",
+ "/reports.json",
+ "Reports",
+ "/reports/p1/fields.json",
+ "Fields",
+ PAGE_SIZE,
+ 1,
+ ),
+ # A report that ignores pageSize and sends every entry must not be asked for the same rows again.
+ (
+ "report_entries_unpaged_response",
+ "report_entries",
+ "/reports.json",
+ "Reports",
+ "/reports/p1/entries.json",
+ "Entries",
+ PAGE_SIZE + 50,
+ 1,
+ ),
+ (
+ "report_entries_short_page",
+ "report_entries",
+ "/reports.json",
+ "Reports",
+ "/reports/p1/entries.json",
+ "Entries",
+ 3,
+ 1,
+ ),
+ ]
+ )
+ def test_child_stops_after_its_last_page(
+ self,
+ _name: str,
+ endpoint: str,
+ parent_path: str,
+ parent_key: str,
+ child_path: str,
+ child_key: str,
+ child_rows: int,
+ expected_child_requests: int,
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ parent_path: [_rows(parent_key, [{"Hash": "p1"}])],
+ child_path: [_rows(child_key, [{"EntryId": str(i), "ID": f"Field{i}"} for i in range(child_rows)])],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert len(result.rows) == child_rows
+ assert result.paths.count(f"/api/v3{child_path}") == expected_child_requests
+ # Neither parent list takes the offset params the top-level tables send.
+ assert result.requests[0].param("pageStart") is None
+
+ def test_report_entries_stop_when_an_unpaged_response_repeats_a_full_page(self) -> None:
+ def respond(request: Any) -> ScriptedResponse:
+ if request.path.endswith("/reports.json"):
+ return _rows("Reports", [{"Hash": "p1"}])
+ return _entries(1, PAGE_SIZE)
+
+ result = _driver().run("report_entries", respond)
+
+ assert result.raised is None
+ assert result.paths.count("/api/v3/reports/p1/entries.json") == 2
+ assert len(result.rows) == PAGE_SIZE
+
+ def test_forms_parent_requests_next_page_only_after_a_full_page(self) -> None:
+ def respond(request: Any) -> ScriptedResponse:
+ if request.path.endswith("/forms.json"):
+ count = FORMS_PAGE_LIMIT if request.param("page") == "1" else 1
+ offset = 0 if request.param("page") == "1" else FORMS_PAGE_LIMIT
+ return _rows("Forms", [{"Hash": f"f{offset + i}"} for i in range(count)])
+ return _rows("Fields", [{"ID": "Field1"}])
+
+ result = _driver().run("form_fields", respond)
+
+ assert result.raised is None
+ assert result.params("page")[0] == "1"
+ assert [p for p in result.params("page") if p is not None] == ["1", "2"]
+ assert len(result.rows) == FORMS_PAGE_LIMIT + 1
+ assert len({row["form_hash"] for row in result.rows}) == FORMS_PAGE_LIMIT + 1
+
+ def test_resumes_fanout_from_saved_parent_and_offset(self) -> None:
+ resume = WufooResumeConfig(
+ fanout_state={
+ "completed": ["forms/forma/entries.json"],
+ "current": "forms/formb/entries.json",
+ "child_state": {"offset": PAGE_SIZE},
+ }
+ )
+ result = _driver().run(
+ "form_entries",
+ route(
+ {
+ "/forms.json": [_rows("Forms", [{"Hash": "forma"}, {"Hash": "formb"}])],
+ "/forms/formb/entries.json": [_entries(PAGE_SIZE + 1, 2)],
+ }
+ ),
+ resume_state=resume,
+ )
+
+ assert result.raised is None
+ assert "/api/v3/forms/forma/entries.json" not in result.paths
+ assert result.requests[-1].param("pageStart") == str(PAGE_SIZE)
+ assert [row["EntryId"] for row in result.rows] == [str(PAGE_SIZE + 1), str(PAGE_SIZE + 2)]
+ assert result.final_state is not None
+ assert result.final_state.fanout_state["completed"] == [
+ "forms/forma/entries.json",
+ "forms/formb/entries.json",
+ ]
+
+ def test_child_error_fails_the_sync(self) -> None:
+ result = _driver().run(
+ "report_entries",
+ route(
+ {
+ "/reports.json": [_rows("Reports", [{"Hash": "r1"}])],
+ "/reports/r1/entries.json": [ScriptedResponse(status=401, json={})],
+ }
+ ),
+ )
+
+ assert isinstance(result.raised, requests.HTTPError)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/wufoo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/wufoo.py
index db1fc8939df3..5e5b737bbcb3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/wufoo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/wufoo/wufoo.py
@@ -1,8 +1,10 @@
import re
+import copy
import base64
import dataclasses
from typing import Any, Optional
+from requests import Response
from urllib3.util.retry import Retry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -10,29 +12,75 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
OffsetPaginator,
+ PageNumberPaginator,
+ SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.wufoo.settings import WUFOO_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.wufoo.settings import (
+ FORMS_PAGE_LIMIT,
+ PAGE_SIZE,
+ WUFOO_ENDPOINTS,
+ WufooEndpointConfig,
+)
# Wufoo enforces a per-account hostname, so only the subdomain label is user-supplied. Restricting
# it to host-safe characters keeps the request pinned to *.wufoo.com.
SUBDOMAIN_REGEX = re.compile(r"^[a-zA-Z0-9-]+$")
-# Wufoo caps `pageSize` at 100 rows per request.
-PAGE_SIZE = 100
VALIDATE_TIMEOUT_SECONDS = 10
[email protected](frozen=True)
class WufooResumeConfig:
# Row offset (Wufoo `pageStart`) of the next page to fetch. Limit/offset pagination is
# deterministic, so a crashed full-refresh sync resumes from the offset after the last page
# yielded; merge dedupes any re-pulled rows on the primary key.
page_start: int = 0
+ # Fan-out endpoints checkpoint the shape `build_dependent_resource` uses: which parents
+ # finished and where the current one stopped.
+ fanout_state: Optional[dict[str, Any]] = None
+
+
+class WufooEntriesPaginator(OffsetPaginator):
+ # The Report Entries docs list no paging params. If the endpoint ignores them, every offset
+ # returns the same rows: a page longer than the requested size, or a page that starts with the
+ # previous page's first row, means the walk is done.
+ def __init__(self) -> None:
+ super().__init__(limit=PAGE_SIZE, offset_param="pageStart", limit_param="pageSize", total_path=None)
+ self._previous_first_row: Any = None
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ if data and self._previous_first_row is not None and data[0] == self._previous_first_row:
+ # Drop the repeated page in place so its rows are not yielded twice.
+ data.clear()
+ self._has_next_page = False
+ return
+ super().update_state(response, data)
+ if data:
+ # Copied because the fan-out adds the parent hash to rows after they are yielded.
+ self._previous_first_row = copy.deepcopy(data[0])
+ if data is not None and len(data) > self.limit:
+ self._has_next_page = False
+
+
+class WufooFormsPaginator(PageNumberPaginator):
+ # `forms.json` returns no page count, so a short page is the last one.
+ def __init__(self) -> None:
+ super().__init__(base_page=1, page_param="page")
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ super().update_state(response, data)
+ if data is not None and len(data) < FORMS_PAGE_LIMIT:
+ self._has_next_page = False
def base_url(subdomain: str) -> str:
@@ -54,6 +102,47 @@ def _redact_values(api_key: str) -> tuple[str, ...]:
return (api_key, _basic_token(api_key))
+def _client_config(api_key: str, subdomain: str) -> ClientConfig:
+ return {
+ "base_url": base_url(subdomain),
+ # Only the non-secret Accept header goes here; the credential rides on framework auth
+ # so its value is redacted from logs.
+ "headers": {"Accept": "application/json"},
+ # Wufoo authenticates with HTTP Basic: the API key is the username and any non-empty
+ # string is the (ignored) password.
+ "auth": {"type": "http_basic", "username": api_key, "password": "footastic"},
+ # Wufoo exposes no top-level total; termination is a short/empty page (OffsetPaginator
+ # default). pageStart/pageSize are Wufoo's offset/limit params.
+ "paginator": OffsetPaginator(
+ limit=PAGE_SIZE,
+ offset_param="pageStart",
+ limit_param="pageSize",
+ total_path=None,
+ ),
+ # Pin every request to the validated subdomain host and reject redirects so the
+ # credential can't be replayed to a cross-host target (SSRF / exfiltration defense).
+ "allowed_hosts": [],
+ "allow_redirects": False,
+ }
+
+
+def _parent_paginator(parent_name: str) -> BasePaginator:
+ # `reports.json` documents no paging params and returns every report in one response.
+ return WufooFormsPaginator() if parent_name == "forms" else SinglePagePaginator()
+
+
+def _child_paginator(config: WufooEndpointConfig) -> BasePaginator:
+ return WufooEntriesPaginator() if config.paginated else SinglePagePaginator()
+
+
+def _load_resume_state(
+ resumable_source_manager: ResumableSourceManager[WufooResumeConfig],
+) -> Optional[WufooResumeConfig]:
+ if not resumable_source_manager.can_resume():
+ return None
+ return resumable_source_manager.load_state()
+
+
def wufoo_source(
api_key: str,
subdomain: str,
@@ -64,29 +153,50 @@ def wufoo_source(
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = WUFOO_ENDPOINTS[endpoint]
+ client_config = _client_config(api_key, subdomain)
+ resume = _load_resume_state(resumable_source_manager)
+
+ if config.fanout is not None:
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(WufooResumeConfig(fanout_state=state))
+
+ child = build_dependent_resource(
+ endpoint_configs=WUFOO_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=db_incremental_field_last_value,
+ parent_endpoint_extra={
+ "paginator": _parent_paginator(config.fanout.parent_name),
+ "data_selector": WUFOO_ENDPOINTS[config.fanout.parent_name].data_key,
+ "data_selector_required": True,
+ },
+ child_endpoint_extra={
+ "paginator": _child_paginator(config),
+ "data_selector": config.data_key,
+ "data_selector_required": True,
+ },
+ child_params_extra=dict(config.params),
+ # The child paginator sets its own page size, and the parent's goes in parent_params.
+ page_size_param=None,
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume else None,
+ )
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: child,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ )
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": base_url(subdomain),
- # Only the non-secret Accept header goes here; the credential rides on framework auth
- # so its value is redacted from logs.
- "headers": {"Accept": "application/json"},
- # Wufoo authenticates with HTTP Basic: the API key is the username and any non-empty
- # string is the (ignored) password.
- "auth": {"type": "http_basic", "username": api_key, "password": "footastic"},
- # Wufoo exposes no top-level total; termination is a short/empty page (OffsetPaginator
- # default). pageStart/pageSize are Wufoo's offset/limit params.
- "paginator": OffsetPaginator(
- limit=PAGE_SIZE,
- offset_param="pageStart",
- limit_param="pageSize",
- total_path=None,
- ),
- # Pin every request to the validated subdomain host and reject redirects so the
- # credential can't be replayed to a cross-host target (SSRF / exfiltration defense).
- "allowed_hosts": [],
- "allow_redirects": False,
- },
+ "client": client_config,
"resource_defaults": {},
"resources": [
{
@@ -103,10 +213,8 @@ def wufoo_source(
}
initial_paginator_state: Optional[dict[str, Any]] = None
- if resumable_source_manager.can_resume():
- resume = resumable_source_manager.load_state()
- if resume is not None:
- initial_paginator_state = {"offset": resume.page_start}
+ if resume is not None:
+ initial_paginator_state = {"offset": resume.page_start}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields