Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the ZapSign source (#124819)

Tom Owers05b1ad800ceb
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 9214ba454813..70475f4bdba1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -13533,9 +13533,9 @@ Today (3): `documents`, `signers`, `templates`
Diffed against: <https://docs.zapsign.com.br/english/sitemap-pages.xml>
-- [ ] `docs/signer-log/{doc_token} (document audit trail)` — per-document signature event history (viewed, signed, refused, timestamps) - the state-transition table behind document status (high)
-- [ ] `users (GET /api/v1/users/, account users)` — lookup table resolving the created_by / account user IDs already carried on documents (medium)
-- [ ] `signer-verification-details/{signer_token}` — authentication and identity-verification outcome per signer, needed to analyze signing friction and failures (medium)
+- [x] `docs/signer-log/{doc_token} (document audit trail)` — per-document signature event history (viewed, signed, refused, timestamps) - the state-transition table behind document status (high)
+- [x] `users (GET /api/v1/users/, account users)` — lookup table resolving the created_by / account user IDs already carried on documents (medium)
+- [x] `signer-verification-details/{signer_token}` — authentication and identity-verification outcome per signer, needed to analyze signing friction and failures (medium)
- [ ] `checks/{check_id} and checks/{check_id}/details (background checks)` — background-check status and results tied to signers; no list endpoint, so it must fan out from signers (low)
Note: Docs are a GitBook site with no OpenAPI spec; the English sitemap enumerates every endpoint page and each page embeds the literal api.zapsign.com.br path, which is what I diffed against. Webhook create/delete/logs endpoints exist but were excluded as plumbing.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/canonical_descriptions.py
index b48f0204a01d..5ded652d25ff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/canonical_descriptions.py
@@ -62,4 +62,36 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"geo_longitude": "Longitude captured at signature time, when geolocation is required.",
},
},
+ "audit_trail": {
+ "description": "One activity in a document's audit trail, such as a view, a reading confirmation or a signature.",
+ "docs_url": "https://docs.zapsign.com.br/zapsign-api/english/documentos/document-audit-trail",
+ "columns": {
+ "_documents_token": "Token of the document the activity belongs to.",
+ "time": "When the activity happened.",
+ "user": "Email of the person who did the activity.",
+ "event": "Name of the activity, in the language of the account.",
+ "description": "Full text of the activity as it appears in the audit trail.",
+ },
+ },
+ "users": {
+ "description": "A user of the ZapSign organization, with their permission level.",
+ "docs_url": "https://docs.zapsign.com.br/zapsign-api/english/account/list-account-users",
+ "columns": {
+ "first_name": "User's first name.",
+ "last_name": "User's last name.",
+ "email": "User's email address. Documents reference their creator by this email.",
+ "email_verified": "Whether the user has verified their email address.",
+ "user_group": "Permission level of the user in the organization (e.g. owner).",
+ },
+ },
+ "signer_verifications": {
+ "description": "Identity verification attempts of one signer, with the status and failure reason of each step.",
+ "docs_url": "https://docs.zapsign.com.br/zapsign-api/english/signer-verification-details",
+ "columns": {
+ "_signers_token": "Token of the signer the verification attempts belong to.",
+ "token": "Token of the signer.",
+ "selfie_validation_type": "Verification method configured for the signer (e.g. identity-verification-global).",
+ "validations": "Each verification step with its type, status, failure reason and creation time.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/settings.py
index addfe8b6e7f5..914eb076e616 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/settings.py
@@ -7,29 +7,41 @@ ZAPSIGN_SANDBOX_BASE_URL = "https://sandbox.api.zapsign.com.br"
DOCUMENTS_RESOURCE = "documents"
TEMPLATES_RESOURCE = "templates"
SIGNERS_RESOURCE = "signers"
+AUDIT_TRAIL_RESOURCE = "audit_trail"
+USERS_RESOURCE = "users"
+SIGNER_VERIFICATIONS_RESOURCE = "signer_verifications"
ENDPOINTS = (
DOCUMENTS_RESOURCE,
TEMPLATES_RESOURCE,
SIGNERS_RESOURCE,
+ AUDIT_TRAIL_RESOURCE,
+ USERS_RESOURCE,
+ SIGNER_VERIFICATIONS_RESOURCE,
)
# Only the documents list exposes a server-side time filter (`created_from`/`created_to`, on the
# creation date). ZapSign has no `updated_at` filter anywhere, so `created_at` is the sole
-# incremental cursor; templates and signers are full refresh only.
+# incremental cursor; every other endpoint is full refresh only.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
DOCUMENTS_RESOURCE: [incremental_field("created_at", IncrementalFieldType.DateTime)],
TEMPLATES_RESOURCE: [],
SIGNERS_RESOURCE: [],
+ AUDIT_TRAIL_RESOURCE: [],
+ USERS_RESOURCE: [],
+ SIGNER_VERIFICATIONS_RESOURCE: [],
}
ENDPOINT_PATHS: dict[str, str] = {
DOCUMENTS_RESOURCE: "/api/v1/docs/",
TEMPLATES_RESOURCE: "/api/v1/templates/",
+ USERS_RESOURCE: "/api/v1/users/",
}
# Signers have no list endpoint of their own — they're read from each document's detail response.
DOCUMENT_DETAIL_PATH = "/api/v1/docs/{token}/"
+AUDIT_TRAIL_PATH = "/api/v1/docs/signer-log/{token}"
+SIGNER_VERIFICATION_PATH = "/api/v1/signer-verification-details/{token}/"
PRIMARY_KEYS: dict[str, list[str]] = {
DOCUMENTS_RESOURCE: ["token"],
@@ -37,12 +49,19 @@ PRIMARY_KEYS: dict[str, list[str]] = {
# Signer tokens look globally unique, but ZapSign doesn't document that guarantee, so the
# parent document token stays in the key (`_documents_token` is injected by the fan-out).
SIGNERS_RESOURCE: ["_documents_token", "token"],
+ # Audit trail events carry no id of their own.
+ AUDIT_TRAIL_RESOURCE: ["_documents_token", "time", "event"],
+ # Account users carry no id; the email is what documents reference in `created_by`.
+ USERS_RESOURCE: ["email"],
+ # The verification endpoint is addressed by the signer token alone, so it identifies the row.
+ SIGNER_VERIFICATIONS_RESOURCE: ["_signers_token"],
}
# `created_at` never changes after creation, unlike `last_update_at`, so partitions stay stable.
PARTITION_KEYS: dict[str, list[str]] = {
DOCUMENTS_RESOURCE: ["created_at"],
TEMPLATES_RESOURCE: ["created_at"],
+ AUDIT_TRAIL_RESOURCE: ["time"],
}
# String timestamp columns coerced to datetimes by the framework's `convert_types`.
@@ -50,4 +69,5 @@ TIMESTAMP_COLUMNS: dict[str, tuple[str, ...]] = {
DOCUMENTS_RESOURCE: ("created_at", "last_update_at"),
TEMPLATES_RESOURCE: ("created_at", "last_update_at"),
SIGNERS_RESOURCE: ("last_view_at", "signed_at"),
+ AUDIT_TRAIL_RESOURCE: ("time",),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py
index dd1a5a0133d0..3df161fedb44 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py
@@ -18,9 +18,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
ZapSignSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.settings import (
+ AUDIT_TRAIL_RESOURCE,
DOCUMENTS_RESOURCE,
+ SIGNER_VERIFICATIONS_RESOURCE,
SIGNERS_RESOURCE,
TEMPLATES_RESOURCE,
+ USERS_RESOURCE,
ZAPSIGN_BASE_URL,
ZAPSIGN_SANDBOX_BASE_URL,
)
@@ -200,6 +203,91 @@ class TestSignersSource:
assert result.response.partition_keys is None
+class TestUsersSource:
+ def test_walks_every_page_until_next_is_null(self) -> None:
+ page2_url = f"{ZAPSIGN_BASE_URL}/api/v1/users/?page=2"
+ result = _driver().run(
+ USERS_RESOURCE,
+ [
+ _scripted_page([{"email": "[email protected]", "user_group": {"name": "owner"}}], next_url=page2_url),
+ _scripted_page([{"email": "[email protected]", "user_group": {"name": "self_docs_limited"}}]),
+ ],
+ )
+
+ assert [row["email"] for row in result.rows] == ["[email protected]", "[email protected]"]
+ assert result.urls[0].startswith(f"{ZAPSIGN_BASE_URL}/api/v1/users/")
+ assert result.urls[1] == page2_url
+
+
+class TestAuditTrailSource:
+ def test_fans_out_per_document_and_skips_documents_that_answer_404(self) -> None:
+ result = _driver().run(
+ AUDIT_TRAIL_RESOURCE,
+ [
+ _scripted_page([{"token": "d1"}, {"token": "d2"}, {"token": "d3"}]),
+ ScriptedResponse(
+ json=[
+ {
+ "time": "2026-01-02 10:00:00.500000+00:00",
+ "user": "[email protected]",
+ "event": "Document viewed",
+ "description": "Document viewed by [email protected]",
+ },
+ {
+ "time": "2026-01-01 09:00:00+00:00",
+ "user": "[email protected]",
+ "event": "Document created",
+ "description": "Document created by [email protected] via web",
+ },
+ ]
+ ),
+ ScriptedResponse(status=404, json={"detail": "Not found."}),
+ ScriptedResponse(json=[]),
+ ],
+ )
+
+ assert [(row["_documents_token"], row["event"], row["time"]) for row in result.rows] == [
+ ("d1", "Document viewed", datetime.datetime(2026, 1, 2, 10, 0, 0, 500000, tzinfo=datetime.UTC)),
+ ("d1", "Document created", datetime.datetime(2026, 1, 1, 9, 0, tzinfo=datetime.UTC)),
+ ]
+ assert [url.split("?")[0] for url in result.urls[1:]] == [
+ f"{ZAPSIGN_BASE_URL}/api/v1/docs/signer-log/{token}" for token in ("d1", "d2", "d3")
+ ]
+
+
+class TestSignerVerificationsSource:
+ def test_fans_out_from_documents_through_signers_and_skips_missing_documents_and_attempts(self) -> None:
+ verification = {
+ "token": "s1",
+ "selfie_validation_type": "identity-verification-global",
+ "validations": [{"type": "Name validation", "status": "success", "reason": ""}],
+ }
+ result = _driver().run(
+ SIGNER_VERIFICATIONS_RESOURCE,
+ [
+ _scripted_page([{"token": "d1"}, {"token": "d2"}, {"token": "d3"}]),
+ ScriptedResponse(json={"token": "d1", "signers": [{"token": "s1"}, {"token": "s2"}]}),
+ ScriptedResponse(json=verification),
+ ScriptedResponse(status=404, json={"error": "Signer has no attempts"}),
+ ScriptedResponse(status=404, json={"detail": "Not found."}),
+ ScriptedResponse(json={"token": "d3", "signers": []}),
+ ],
+ )
+
+ assert [(row["_signers_token"], row["selfie_validation_type"]) for row in result.rows] == [
+ ("s1", "identity-verification-global")
+ ]
+ assert result.rows[0]["validations"] == verification["validations"]
+ assert [url.split("?")[0] for url in result.urls] == [
+ f"{ZAPSIGN_BASE_URL}/api/v1/docs/",
+ f"{ZAPSIGN_BASE_URL}/api/v1/docs/d1/",
+ f"{ZAPSIGN_BASE_URL}/api/v1/signer-verification-details/s1/",
+ f"{ZAPSIGN_BASE_URL}/api/v1/signer-verification-details/s2/",
+ f"{ZAPSIGN_BASE_URL}/api/v1/docs/d2/",
+ f"{ZAPSIGN_BASE_URL}/api/v1/docs/d3/",
+ ]
+
+
class TestWebhookTableTransformer:
def test_parses_timestamps_and_skips_rows_without_token(self) -> None:
table = table_from_py_list(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
index 61cb43c7590d..acaae9b1d21d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
@@ -37,14 +37,19 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.settings import (
+ AUDIT_TRAIL_PATH,
+ AUDIT_TRAIL_RESOURCE,
DOCUMENT_DETAIL_PATH,
DOCUMENTS_RESOURCE,
ENDPOINT_PATHS,
PARTITION_KEYS,
PRIMARY_KEYS,
+ SIGNER_VERIFICATION_PATH,
+ SIGNER_VERIFICATIONS_RESOURCE,
SIGNERS_RESOURCE,
TEMPLATES_RESOURCE,
TIMESTAMP_COLUMNS,
+ USERS_RESOURCE,
ZAPSIGN_BASE_URL,
ZAPSIGN_SANDBOX_BASE_URL,
)
@@ -56,13 +61,13 @@ REQUEST_TIMEOUT_SECONDS = 30
WEBHOOK_CREATE_PATH = "/api/v1/user/company/webhook/"
[email protected](frozen=True)
class ZapSignResumeConfig:
"""Resume state for one endpoint's pagination.
``paginator_state`` holds whatever the framework paginator checkpoints: ``{"next_url": ...}``
for the top-level list endpoints, or the fan-out state dict
- (``{"completed": [...], "current": ..., "child_state": ...}``) for signers.
+ (``{"completed": [...], "current": ..., "child_state": ...}``) for single-level fan-outs.
"""
endpoint: str
@@ -141,21 +146,39 @@ def _templates_resource() -> EndpointResource:
}
-def _signers_resources() -> list[EndpointResource]:
- """Parent documents list + per-document detail fan-out that extracts the signer arrays."""
- parent: EndpointResource = {
+def _users_resource() -> EndpointResource:
+ # The users list exposes no filter or sort params — full refresh only.
+ return {
+ "name": USERS_RESOURCE,
+ "table_name": USERS_RESOURCE,
+ "write_disposition": "replace",
+ "endpoint": {
+ "path": ENDPOINT_PATHS[USERS_RESOURCE],
+ "data_selector": "results",
+ "data_selector_required": True,
+ },
+ "table_format": "delta",
+ }
+
+
+def _documents_parent_resource() -> EndpointResource:
+ return {
"name": DOCUMENTS_RESOURCE,
"table_name": DOCUMENTS_RESOURCE,
"write_disposition": "replace",
"endpoint": {
"path": ENDPOINT_PATHS[DOCUMENTS_RESOURCE],
- "params": {"sort_order": "asc"},
+ # `include_signers` guarantees the `results` envelope the selector requires.
+ "params": {"sort_order": "asc", "include_signers": "true"},
"data_selector": "results",
"data_selector_required": True,
},
"table_format": "delta",
}
- child: EndpointResource = {
+
+
+def _signers_child_resource() -> EndpointResource:
+ return {
"name": SIGNERS_RESOURCE,
"table_name": SIGNERS_RESOURCE,
"write_disposition": "replace",
@@ -170,11 +193,69 @@ def _signers_resources() -> list[EndpointResource]:
},
},
"data_selector": "signers",
+ # A document deleted between the list and the detail request answers 404.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
},
"columns": _timestamp_columns(SIGNERS_RESOURCE),
"table_format": "delta",
}
- return [parent, child]
+
+
+def _signers_resources() -> list[EndpointResource]:
+ """Parent documents list + per-document detail fan-out that extracts the signer arrays."""
+ return [_documents_parent_resource(), _signers_child_resource()]
+
+
+def _audit_trail_resources() -> list[EndpointResource]:
+ """Parent documents list + per-document audit trail, which answers with a bare JSON list."""
+ child: EndpointResource = {
+ "name": AUDIT_TRAIL_RESOURCE,
+ "table_name": AUDIT_TRAIL_RESOURCE,
+ "write_disposition": "replace",
+ "include_from_parent": ["token"],
+ "endpoint": {
+ "path": AUDIT_TRAIL_PATH,
+ "params": {
+ "token": {
+ "type": "resolve",
+ "resource": DOCUMENTS_RESOURCE,
+ "field": "token",
+ },
+ },
+ "paginator": "single_page",
+ "data_selector_required": True,
+ # A document deleted between the list and the audit trail request answers 404.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ "columns": _timestamp_columns(AUDIT_TRAIL_RESOURCE),
+ "table_format": "delta",
+ }
+ return [_documents_parent_resource(), child]
+
+
+def _signer_verifications_resources() -> list[EndpointResource]:
+ """Documents list -> document detail (signers) -> one verification request per signer."""
+ child: EndpointResource = {
+ "name": SIGNER_VERIFICATIONS_RESOURCE,
+ "table_name": SIGNER_VERIFICATIONS_RESOURCE,
+ "write_disposition": "replace",
+ "include_from_parent": ["token"],
+ "endpoint": {
+ "path": SIGNER_VERIFICATION_PATH,
+ "params": {
+ "token": {
+ "type": "resolve",
+ "resource": SIGNERS_RESOURCE,
+ "field": "token",
+ },
+ },
+ "paginator": "single_page",
+ # ZapSign answers 404 for a signer with no verification attempts, which is most signers.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ "table_format": "delta",
+ }
+ return [_documents_parent_resource(), _signers_child_resource(), child]
def _webhook_table_transformer(table: pa.Table) -> pa.Table:
@@ -247,6 +328,12 @@ def zapsign_source(
resources = [_documents_resource(should_use_incremental_field)]
elif endpoint == TEMPLATES_RESOURCE:
resources = [_templates_resource()]
+ elif endpoint == USERS_RESOURCE:
+ resources = [_users_resource()]
+ elif endpoint == AUDIT_TRAIL_RESOURCE:
+ resources = [*_audit_trail_resources()]
+ elif endpoint == SIGNER_VERIFICATIONS_RESOURCE:
+ resources = [*_signer_verifications_resources()]
else:
raise ValueError(f"Unknown ZapSign endpoint: {endpoint}")