Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add docusign audit events, form data, templates and bulk send tables (#119241)

Tom Owers9a5ccf1a78fa
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 0282f37f4431..0739df97bac4 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -489,7 +489,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/docuseal/docuseal.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/docuseal/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/docusign/docusign.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/docusign/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/dodopayments.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/doppler/doppler.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/doppler/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index d1f9da8a328a..447501d2863a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -3862,11 +3862,11 @@ Today (6): `envelope_documents`, `envelope_recipients`, `envelopes`, `folders`,
Diffed against: <https://raw.githubusercontent.com/docusign/OpenAPI-Specifications/master/esignature.rest.swagger-v2.1.json>
-- [ ] `Envelope audit events (GET /v2.1/accounts/{accountId}/envelopes/{envelopeId}/audit_events)` — reconstruct envelope activity and signing history beyond the latest status (high)
-- [ ] `Envelope form data (GET /v2.1/accounts/{accountId}/envelopes/{envelopeId}/form_data)` — analyze submitted contract fields and recipient responses across envelopes (high)
+- [x] `Envelope audit events (GET /v2.1/accounts/{accountId}/envelopes/{envelopeId}/audit_events)` — reconstruct envelope activity and signing history beyond the latest status (high)
+- [x] `Envelope form data (GET /v2.1/accounts/{accountId}/envelopes/{envelopeId}/form_data)` — analyze submitted contract fields and recipient responses across envelopes (high)
- [ ] `Envelope custom fields (GET /v2.1/accounts/{accountId}/envelopes/{envelopeId}/custom_fields)` — join envelopes to business records using sender-defined field values (medium)
-- [ ] `Envelope templates (GET /v2.1/accounts/{accountId}/envelopes/{envelopeId}/templates)` — associate imported envelopes with templates to compare completion rates and turnaround times (medium)
-- [ ] `Bulk send batch summaries (GET /v2.1/accounts/{accountId}/bulk_send_batch)` — measure bulk signing campaign volume, delivery progress, and errors (medium)
+- [x] `Envelope templates (GET /v2.1/accounts/{accountId}/envelopes/{envelopeId}/templates)` — associate imported envelopes with templates to compare completion rates and turnaround times (medium)
+- [x] `Bulk send batch summaries (GET /v2.1/accounts/{accountId}/bulk_send_batch)` — measure bulk signing campaign volume, delivery progress, and errors (medium)
- [ ] `Bulk send batch envelopes (GET /v2.1/accounts/{accountId}/bulk_send_batch/{bulkSendBatchId}/envelopes)` — associate imported envelopes with bulk campaigns for campaign completion analysis (medium)
- [ ] `PowerForms and form data (GET /v2.1/accounts/{accountId}/powerforms; GET /v2.1/accounts/{accountId}/powerforms/{powerFormId}/form_data)` — analyze self-service form submissions and their envelope outcomes (medium)
- [ ] `Billing invoices (GET /v2.1/accounts/{accountId}/billing_invoices)` — compare account charges and invoice balances with envelope activity (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/canonical_descriptions.py
index 27cc71d687a5..0afd587d6904 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/canonical_descriptions.py
@@ -64,6 +64,54 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"envelopeStatusChangedDateTime": "Last status change of the parent envelope, carried over as the incremental cursor.",
},
},
+ "envelope_form_data": {
+ "description": "Field values (tab data) captured on each envelope the user sent, overall and per recipient. Needs the account's 'Allow sender to download form data' sending setting.",
+ "docs_url": "https://developers.docusign.com/docs/esign-rest-api/reference/envelopes/envelopeformdata/get/",
+ "columns": {
+ "envelopeId": "Identifier of the envelope the form data belongs to.",
+ "status": "Status of the envelope when the form data was read.",
+ "emailSubject": "Subject line of the email sent to all recipients of the envelope.",
+ "sentDateTime": "Date and time the envelope was sent.",
+ "formData": "Name and value of each form field on the envelope, with the original value where it changed.",
+ "recipientFormData": "Form field values grouped by recipient, with the times each recipient was sent, viewed, signed, or declined.",
+ "prefillFormData": "Values of prefill fields the sender filled in before sending.",
+ "envelopeCreatedDateTime": "Creation time of the parent envelope, carried over for partitioning.",
+ "envelopeStatusChangedDateTime": "Last status change of the parent envelope, carried over as the incremental cursor.",
+ },
+ },
+ "envelope_audit_events": {
+ "description": "Audit trail of each envelope: one row per event, such as sent, viewed, signed, or voided.",
+ "docs_url": "https://developers.docusign.com/docs/esign-rest-api/reference/envelopes/envelopes/listauditevents/",
+ "columns": {
+ "envelopeId": "Identifier of the envelope the event belongs to.",
+ "eventIndex": "Position of the event in the envelope's audit log, starting at 0 for the oldest event.",
+ "logTime": "Date and time the event happened.",
+ "Action": "What happened, for example Registered, Sent Invitations, Viewed, or Signed.",
+ "Message": "Human-readable description of the event.",
+ "EnvelopeStatus": "Status of the envelope after the event.",
+ "UserName": "Name of the user or recipient who caused the event.",
+ "UserId": "Identifier of the user who caused the event.",
+ "ClientIPAddress": "IP address the event came from.",
+ "Source": "Application or channel the event came from, such as web or API.",
+ "envelopeCreatedDateTime": "Creation time of the parent envelope, carried over for partitioning.",
+ "envelopeStatusChangedDateTime": "Last status change of the parent envelope, carried over as the incremental cursor.",
+ },
+ },
+ "envelope_templates": {
+ "description": "Server templates applied to each envelope, to link envelopes back to the templates they came from.",
+ "docs_url": "https://developers.docusign.com/docs/esign-rest-api/reference/envelopes/envelopetemplates/list/",
+ "columns": {
+ "envelopeId": "Identifier of the envelope.",
+ "templateId": "Identifier of the template applied to the envelope.",
+ "name": "Name of the template.",
+ "uri": "Relative URI of the template.",
+ "documentId": "Envelope document that the template supplied.",
+ "documentName": "Name of the envelope document that the template supplied.",
+ "applied": "Whether the template was applied to the envelope.",
+ "envelopeCreatedDateTime": "Creation time of the parent envelope, carried over for partitioning.",
+ "envelopeStatusChangedDateTime": "Last status change of the parent envelope, carried over as the incremental cursor.",
+ },
+ },
"templates": {
"description": "Reusable envelope templates defined on the account.",
"docs_url": "https://developers.docusign.com/docs/esign-rest-api/reference/templates/templates/list/",
@@ -112,4 +160,20 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"itemCount": "Number of items stored in the folder.",
},
},
+ "bulk_send_batches": {
+ "description": "Bulk send batches submitted on the account, with how many envelopes were sent, queued, or failed.",
+ "docs_url": "https://developers.docusign.com/docs/esign-rest-api/reference/bulkenvelopes/bulksend/getbulksendbatches/",
+ "columns": {
+ "batchId": "Unique identifier of the batch.",
+ "batchName": "Name of the batch.",
+ "batchSize": "Number of envelopes in the batch.",
+ "sent": "Number of envelopes in the batch that were sent.",
+ "queued": "Number of envelopes in the batch still waiting to be processed.",
+ "failed": "Number of envelopes in the batch that failed to send.",
+ "submittedDate": "Date and time the batch was submitted.",
+ "action": "Action last taken on the batch, such as resend or void.",
+ "actionStatus": "Status of the action last taken on the batch.",
+ "batchUri": "Relative URI of the batch details.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/docusign.py b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/docusign.py
index 6cad016d9124..500dfe3094ce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/docusign.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/docusign.py
@@ -1,9 +1,9 @@
import time
import datetime as dt
import dataclasses
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
from typing import Any, Optional
-from urllib.parse import urlencode, urlparse
+from urllib.parse import quote, urlencode, urlparse
import jwt
import requests
@@ -11,6 +11,9 @@ from requests.auth import HTTPBasicAuth
from structlog.types import FilteringBoundLogger
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
+ interruptible_wait,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.docusign.settings import (
@@ -38,6 +41,11 @@ REQUEST_TIMEOUT_SECONDS = 120
# Hard stop so a misbehaving endpoint that keeps advertising `nextUri` can't page forever.
MAX_PAGES = 20_000
+# Per-envelope tables cost one request per envelope against the account's hourly API quota, which
+# the customer's other DocuSign integrations share. Pause once only this fraction of it is left.
+QUOTA_RESERVE_FRACTION = 0.1
+MAX_QUOTA_WAIT_SECONDS = 3600
+
class DocusignAuthError(Exception):
"""Raised when DocuSign refuses to issue an access token."""
@@ -258,11 +266,7 @@ def _flatten_envelope_children(
continue
# Carry the parent's timestamps so the child table can be synced incrementally against
# the same `from_date` window that produced its parent, and partitioned on a stable key.
- parent = {
- "envelopeId": str(envelope_id),
- "envelopeCreatedDateTime": envelope.get("createdDateTime"),
- "envelopeStatusChangedDateTime": envelope.get("statusChangedDateTime"),
- }
+ parent = _envelope_parent(envelope)
if endpoint.derived_from_envelopes == "recipients":
rows.extend(_flatten_recipients(envelope, parent))
@@ -276,6 +280,55 @@ def _flatten_envelope_children(
return rows
+def _quota_wait_seconds(headers: Mapping[str, str], now: float) -> float:
+ """Seconds until DocuSign resets the hourly quota, when the last response left too little of it."""
+ try:
+ remaining = int(headers["X-RateLimit-Remaining"])
+ limit = int(headers["X-RateLimit-Limit"])
+ reset_at = float(headers["X-RateLimit-Reset"])
+ except (KeyError, ValueError):
+ return 0.0
+ if remaining > limit * QUOTA_RESERVE_FRACTION:
+ return 0.0
+ return min(max(reset_at - now, 0.0), MAX_QUOTA_WAIT_SECONDS)
+
+
+def _envelope_parent(envelope: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "envelopeId": str(envelope["envelopeId"]),
+ "envelopeCreatedDateTime": envelope.get("createdDateTime"),
+ "envelopeStatusChangedDateTime": envelope.get("statusChangedDateTime"),
+ }
+
+
+def _shape_envelope_sub_resource(
+ endpoint: DocusignEndpointConfig, body: dict[str, Any], parent: dict[str, Any]
+) -> list[dict[str, Any]]:
+ if endpoint.envelope_sub_resource == "form_data":
+ return [{**body, **parent}]
+
+ if endpoint.envelope_sub_resource == "audit_events":
+ rows: list[dict[str, Any]] = []
+ for index, event in enumerate(body.get("auditEvents") or []):
+ if not isinstance(event, dict):
+ continue
+ # Each event arrives as a list of name/value pairs (logTime, Action, UserName, ...).
+ fields = {
+ str(item["name"]): item.get("value")
+ for item in event.get("eventFields") or []
+ if isinstance(item, dict) and item.get("name")
+ }
+ rows.append({**fields, **parent, "eventIndex": index})
+ return rows
+
+ # DocuSign can list a template once per envelope document it supplied; keep one row per template.
+ templates: dict[str, dict[str, Any]] = {}
+ for template in body.get("templates") or []:
+ if isinstance(template, dict) and template.get("templateId"):
+ templates.setdefault(str(template["templateId"]), {**template, **parent})
+ return list(templates.values())
+
+
def get_rows(
credentials: DocusignCredentials,
endpoint_name: str,
@@ -304,9 +357,13 @@ def get_rows(
start_position = resume.start_position
logger.debug(f"Resuming DocuSign {endpoint_name} from start_position={start_position}")
- def request_page(position: int) -> dict[str, Any]:
- nonlocal token
- url = f"{account.base_url}{endpoint.path}?{urlencode(_build_params(endpoint, position, from_date))}"
+ last_headers: Mapping[str, str] = {}
+
+ def request(path: str, params: dict[str, str]) -> dict[str, Any]:
+ nonlocal token, last_headers
+ url = f"{account.base_url}{path}"
+ if params:
+ url = f"{url}?{urlencode(params)}"
def _do() -> requests.Response:
return api_session.get(url, headers={"Authorization": f"Bearer {token}"}, timeout=REQUEST_TIMEOUT_SECONDS)
@@ -317,6 +374,7 @@ def get_rows(
token = mint_access_token(auth_session, credentials)
response = _do()
+ last_headers = response.headers
if not response.ok:
logger.error(f"DocuSign API error: url={url}, {_error_detail(response)}")
response.raise_for_status()
@@ -325,12 +383,34 @@ def get_rows(
return body if isinstance(body, dict) else {}
for _ in range(MAX_PAGES):
- body = request_page(start_position)
+ body = request(endpoint.path, _build_params(endpoint, start_position, from_date))
page = [item for item in (body.get(endpoint.data_key) or []) if isinstance(item, dict)]
- rows = _flatten_envelope_children(endpoint, page) if endpoint.derived_from_envelopes else page
- if rows:
- yield rows
+ if endpoint.envelope_sub_resource:
+ for envelope in page:
+ if not envelope.get("envelopeId"):
+ continue
+ wait_seconds = _quota_wait_seconds(last_headers, time.time())
+ if wait_seconds > 0:
+ logger.info(f"DocuSign hourly API quota is low, pausing {wait_seconds:.0f}s until it resets")
+ interruptible_wait(wait_seconds, safe_point=resumable_source_manager.safe_point)
+
+ parent = _envelope_parent(envelope)
+ sub_resource_path = (
+ f"{endpoint.path}/{quote(parent['envelopeId'], safe='')}/{endpoint.envelope_sub_resource}"
+ )
+ sub_rows = _shape_envelope_sub_resource(
+ endpoint, request(sub_resource_path, endpoint.envelope_sub_resource_params), parent
+ )
+ if sub_rows:
+ yield sub_rows
+ else:
+ # The staged cursor is the start of this page, so resuming re-reads it whole.
+ resumable_source_manager.safe_point()
+ else:
+ rows = _flatten_envelope_children(endpoint, page) if endpoint.derived_from_envelopes else page
+ if rows:
+ yield rows
if not endpoint.supports_pagination or not page or not _has_next_page(body, page):
break
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/settings.py
index b6042da7e932..97da1242e2fd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/settings.py
@@ -16,7 +16,7 @@ PAGE_SIZE = 100
DEFAULT_LOOKBACK_DAYS = 730
-@dataclass
+@dataclass(frozen=True)
class DocusignEndpointConfig:
name: str
# Path under `/restapi/v2.1/accounts/{accountId}`.
@@ -40,6 +40,11 @@ class DocusignEndpointConfig:
# Candidate keys the inline child collection arrives under on each envelope. DocuSign is not
# consistent between the list and per-envelope shapes, so we accept either spelling.
envelope_child_keys: tuple[str, ...] = ()
+ # Per-envelope sub-resource (`/envelopes/{envelopeId}/<this>`) that the envelope listing has
+ # no `include` for. Each listed envelope costs one extra request against the account's hourly
+ # API quota, so these endpoints should be synced incrementally.
+ envelope_sub_resource: Optional[str] = None
+ envelope_sub_resource_params: dict[str, str] = field(default_factory=dict)
# False for endpoints that return everything in one response and ignore `start_position` —
# re-requesting with an advanced offset would replay the same rows forever.
supports_pagination: bool = True
@@ -103,6 +108,45 @@ DOCUSIGN_ENDPOINTS: dict[str, DocusignEndpointConfig] = {
derived_from_envelopes="documents",
envelope_child_keys=("envelopeDocuments", "documents"),
),
+ "envelope_form_data": DocusignEndpointConfig(
+ name="envelope_form_data",
+ path="/envelopes",
+ data_key="envelopes",
+ primary_key=["envelopeId"],
+ # DocuSign only releases form data to the sender (or a full admin), so received envelopes
+ # would fail the per-envelope request.
+ params={"order_by": "status_changed", "order": "asc", "user_filter": "sender"},
+ date_filter_param="from_date",
+ incremental_fields=_DERIVED_INCREMENTAL_FIELDS,
+ partition_key="envelopeCreatedDateTime",
+ envelope_sub_resource="form_data",
+ ),
+ "envelope_audit_events": DocusignEndpointConfig(
+ name="envelope_audit_events",
+ path="/envelopes",
+ data_key="envelopes",
+ # Audit events carry no id of their own. The log is append-only and returned oldest
+ # first, so an event's position in its envelope's log is stable.
+ primary_key=["envelopeId", "eventIndex"],
+ params={"order_by": "status_changed", "order": "asc"},
+ date_filter_param="from_date",
+ incremental_fields=_DERIVED_INCREMENTAL_FIELDS,
+ partition_key="envelopeCreatedDateTime",
+ envelope_sub_resource="audit_events",
+ ),
+ "envelope_templates": DocusignEndpointConfig(
+ name="envelope_templates",
+ path="/envelopes",
+ data_key="envelopes",
+ primary_key=["envelopeId", "templateId"],
+ params={"order_by": "status_changed", "order": "asc"},
+ date_filter_param="from_date",
+ incremental_fields=_DERIVED_INCREMENTAL_FIELDS,
+ partition_key="envelopeCreatedDateTime",
+ envelope_sub_resource="templates",
+ # Without a filter DocuSign can also return templates that merely match the documents.
+ envelope_sub_resource_params={"include": "applied"},
+ ),
"templates": DocusignEndpointConfig(
name="templates",
path="/templates",
@@ -135,6 +179,17 @@ DOCUSIGN_ENDPOINTS: dict[str, DocusignEndpointConfig] = {
primary_key=["folderId"],
supports_pagination=False,
),
+ "bulk_send_batches": DocusignEndpointConfig(
+ name="bulk_send_batches",
+ path="/bulk_send_batch",
+ data_key="bulkBatchSummaries",
+ primary_key=["batchId"],
+ # Without `from_date` DocuSign returns one account-wide status report instead of the
+ # batch list. Full refresh only: the sent/failed/queued counts keep changing after a
+ # batch is submitted, and `from_date` filters on submission time.
+ date_filter_param="from_date",
+ partition_key="submittedDate",
+ ),
}
ENDPOINTS = tuple(DOCUSIGN_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/source.py
index f21e6d07c7ab..c7c7d3db091b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/source.py
@@ -93,9 +93,11 @@ class DocusignSource(ResumableSource[DocusignSourceConfig, DocusignResumeConfig]
name=ExternalDataSourceType.DOCUSIGN,
category=DataWarehouseSourceCategory.SALES,
label="DocuSign",
- caption="""Connect your DocuSign account to pull envelopes, recipients, documents, templates, users, and folders into the PostHog Data warehouse.
+ caption="""Connect your DocuSign account to pull envelopes, recipients, documents, form data, audit events, templates, bulk send batches, users, and folders into the PostHog Data warehouse.
-Create an integration key in [DocuSign Apps and Keys](https://apps.docusign.com/admin/apps-and-keys). JWT grant is the option to pick: add an RSA keypair to the key, grant consent once for the user you want to impersonate, and paste that user's API user ID. Production keys have to pass DocuSign's go-live review before they work outside the demo environment.""",
+Create an integration key in [DocuSign Apps and Keys](https://apps.docusign.com/admin/apps-and-keys). JWT grant is the option to pick: add an RSA keypair to the key, grant consent once for the user you want to impersonate, and paste that user's API user ID. Production keys have to pass DocuSign's go-live review before they work outside the demo environment.
+
+The form data, audit events, and envelope templates tables make one API request per envelope, which counts toward your DocuSign hourly API limit. Sync them incrementally. Form data also needs the "Allow sender to download form data" sending setting turned on.""",
iconPath="/static/services/docusign.png",
docsUrl="https://posthog.com/docs/cdp/sources/docusign",
releaseStatus=ReleaseStatus.ALPHA,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/conftest.py b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/conftest.py
index c4f4bdc8d7e1..4ac0454b7d05 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/conftest.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/conftest.py
@@ -35,6 +35,7 @@ class FakeResponse:
self.status_code = status_code
self._payload = payload
self.url = url
+ self.headers: dict[str, str] = {}
@property
def ok(self) -> bool:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/test_docusign.py b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/test_docusign.py
index 80adf5d25931..bf851a4adb47 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/test_docusign.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/docusign/tests/test_docusign.py
@@ -1,3 +1,4 @@
+import time
import datetime as dt
from collections.abc import Iterable
from typing import Any, Optional, cast
@@ -37,9 +38,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
DocusignSourceConfig,
)
-_SESSION_FACTORY = (
- "products.warehouse_sources.backend.temporal.data_imports.sources.docusign.docusign.make_tracked_session"
-)
+_DOCUSIGN_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.docusign.docusign"
+_SESSION_FACTORY = f"{_DOCUSIGN_MODULE}.make_tracked_session"
USERINFO_PAYLOAD: dict[str, Any] = {
"sub": "user-guid",
@@ -263,6 +263,131 @@ class TestDocusignTransport:
assert [state.start_position for state in result.saved_states] == [PAGE_SIZE]
assert result.params("start_position")[-2:] == ["0", "100"]
+ @pytest.mark.parametrize(
+ "endpoint_name,sub_resource,sub_body,empty_body,expected_rows",
+ [
+ (
+ "envelope_audit_events",
+ "audit_events",
+ {
+ "auditEvents": [
+ {"eventFields": [{"name": "logTime", "value": "t0"}, {"name": "Action", "value": "Sent"}]},
+ {"eventFields": [{"name": "logTime", "value": "t1"}, {"name": "Action", "value": "Signed"}]},
+ ]
+ },
+ {"auditEvents": []},
+ [
+ {"envelopeId": "env-0", "logTime": "t0", "Action": "Sent", "eventIndex": 0},
+ {"envelopeId": "env-0", "logTime": "t1", "Action": "Signed", "eventIndex": 1},
+ ],
+ ),
+ (
+ "envelope_templates",
+ "templates",
+ {
+ "templates": [
+ {"templateId": "tpl-1", "name": "NDA", "documentId": "1"},
+ {"templateId": "tpl-1", "name": "NDA", "documentId": "2"},
+ {"templateId": "tpl-2", "name": "Order form", "documentId": "3"},
+ ]
+ },
+ {},
+ [
+ {"envelopeId": "env-0", "templateId": "tpl-1", "name": "NDA", "documentId": "1"},
+ {"envelopeId": "env-0", "templateId": "tpl-2", "name": "Order form", "documentId": "3"},
+ ],
+ ),
+ (
+ "envelope_form_data",
+ "form_data",
+ {"envelopeId": "env-0", "status": "completed", "formData": [{"name": "Company", "value": "Acme"}]},
+ {"envelopeId": "env-1", "status": "sent", "formData": []},
+ [
+ {"envelopeId": "env-0", "status": "completed", "formData": [{"name": "Company", "value": "Acme"}]},
+ {"envelopeId": "env-1", "status": "sent", "formData": []},
+ ],
+ ),
+ ],
+ )
+ def test_envelope_sub_resources_are_fetched_per_envelope(
+ self,
+ endpoint_name: str,
+ sub_resource: str,
+ sub_body: dict[str, Any],
+ empty_body: dict[str, Any],
+ expected_rows: list[dict[str, Any]],
+ logger: FilteringBoundLogger,
+ ) -> None:
+ result = driver().run(
+ endpoint_name,
+ [
+ *auth_responses(),
+ ScriptedResponse(json=envelope_page(2)),
+ ScriptedResponse(json=sub_body),
+ ScriptedResponse(json=empty_body),
+ ],
+ logger=logger,
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {
+ **row,
+ "envelopeCreatedDateTime": "2024-01-01T00:00:00.0000000Z",
+ "envelopeStatusChangedDateTime": "2024-02-01T00:00:00.0000000Z",
+ }
+ for row in expected_rows
+ ]
+ assert [path.rsplit("/", 2)[-2:] for path in result.paths[-2:]] == [
+ ["env-0", sub_resource],
+ ["env-1", sub_resource],
+ ]
+ # Form data is only released to the sender, so received envelopes must not be listed.
+ assert result.requests[-3].param("user_filter") == ("sender" if sub_resource == "form_data" else None)
+ assert result.final_state is None
+
+ @pytest.mark.parametrize("remaining,expect_pause", [("500", False), ("50", True)])
+ def test_per_envelope_requests_pause_when_the_hourly_quota_runs_low(
+ self, remaining: str, expect_pause: bool, logger: FilteringBoundLogger
+ ) -> None:
+ quota_headers = {
+ "X-RateLimit-Limit": "1000",
+ "X-RateLimit-Remaining": remaining,
+ "X-RateLimit-Reset": str(int(time.time()) + 600),
+ }
+ with mock.patch(f"{_DOCUSIGN_MODULE}.interruptible_wait") as wait:
+ result = driver().run(
+ "envelope_audit_events",
+ [
+ *auth_responses(),
+ ScriptedResponse(json=envelope_page(1), headers=quota_headers),
+ ScriptedResponse(json={"auditEvents": []}),
+ ],
+ logger=logger,
+ )
+
+ assert result.raised is None
+ waits = [call.args[0] for call in wait.call_args_list]
+ if expect_pause:
+ assert len(waits) == 1 and 500 < waits[0] <= 600
+ else:
+ assert waits == []
+
+ def test_bulk_send_batches_always_send_from_date(self, logger: FilteringBoundLogger) -> None:
+ result = driver().run(
+ "bulk_send_batches",
+ [
+ *auth_responses(),
+ ScriptedResponse(json={"bulkBatchSummaries": [{"batchId": "b-1", "submittedDate": "2024-01-01"}]}),
+ ],
+ logger=logger,
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"batchId": "b-1", "submittedDate": "2024-01-01"}]
+ # Without from_date DocuSign answers with an account-wide summary that has no batch list.
+ assert result.requests[-1].param("from_date") is not None
+
@pytest.mark.parametrize(
"value,expected",
[