Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add addresses, emails, notes and tasks tables to Bloomerang (#119529)

Tom Owers89ab868902fd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index d433001e28bd..8c31b94b3a56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1734,13 +1734,13 @@ Diffed against: <https://bloomerang-api-documentation.s3.us-west-2.amazonaws.com
- [x] `Households (GET /v2/households)` — resolve HouseholdId in constituents and aggregate giving across household members (high)
- [x] `Soft credits (GET /v2/softcredits)` — resolve SoftCreditIds in transaction designations and attribute giving credit to additional donors (high)
- [x] `Refunds (GET /v2/refunds)` — resolve RefundIds and report refund amounts and dates for accurate net giving (high)
-- [ ] `Addresses (GET /v2/addresses)` — resolve AddressIds beyond the primary address for geographic and household analysis (high)
-- [ ] `Email addresses (GET /v2/emails)` — resolve EmailIds beyond the primary email for contact coverage and donor matching (high)
+- [x] `Addresses (GET /v2/addresses)` — resolve AddressIds beyond the primary address for geographic and household analysis (high)
+- [x] `Email addresses (GET /v2/emails)` — resolve EmailIds beyond the primary email for contact coverage and donor matching (high)
- [x] `Phones (GET /v2/phones)` — resolve PhoneIds beyond the primary phone for contact coverage and outreach analysis (high)
- [ ] `Relationships (GET /v2/constituent/{id}/relationships)` — analyze donor relationships, employers, and connections that affect fundraising (medium)
- [ ] `Pledge payment failures (GET /v2/pledge/{id}/paymentFailures)` — measure failed pledge payments and identify donations at risk (medium)
-- [ ] `Notes (GET /v2/notes)` — analyze donor notes alongside transactions and interactions (medium)
-- [ ] `Tasks (GET /v2/tasks)` — measure fundraising follow-up activity, completion, and outstanding work (medium)
+- [x] `Notes (GET /v2/notes)` — analyze donor notes alongside transactions and interactions (medium)
+- [x] `Tasks (GET /v2/tasks)` — measure fundraising follow-up activity, completion, and outstanding work (medium)
Note: The OpenAPI file linked from the official REST reference lists all collection operations. HouseholdId, contact IDs, SoftCreditIds, and RefundIds lack complete lookup tables. Transactions already embed designations, custom field labels, and pledge installments, so those endpoints are excluded. Relationship and payment failure tables need calls for each constituent or pledge.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py
index cae24e7136c9..7bdef3db2188 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py
@@ -33,9 +33,9 @@ class BloomerangResumeConfig:
def _flatten_audit_trail(item: dict[str, Any]) -> dict[str, Any]:
"""Promote AuditTrail.CreatedDate/LastModifiedDate onto the row root.
- Constituents, Transactions, Interactions, Refunds, SoftCredits, and Households nest their
- timestamps under an `AuditTrail` object; flattening them makes `CreatedDate` usable as a
- partition key and `LastModifiedDate` usable as the incremental cursor column.
+ Every endpoint except Appeals, Campaigns, Funds, and Phones nests its timestamps under an
+ `AuditTrail` object; flattening them makes `CreatedDate` usable as a partition key and
+ `LastModifiedDate` usable as the incremental cursor column.
"""
audit_trail = item.pop("AuditTrail", None)
if isinstance(audit_trail, dict):
@@ -59,18 +59,21 @@ def _build_params(
should_use_incremental_field: bool,
db_incremental_field_last_value: Optional[Any],
) -> dict[str, Any]:
+ params: dict[str, Any] = dict(config.extra_params)
if config.supports_incremental and should_use_incremental_field and db_incremental_field_last_value:
- return {
- "lastModified": _format_last_modified(db_incremental_field_last_value),
- "orderBy": "LastModifiedDate",
- "orderDirection": "Asc",
- }
- if config.supports_sort:
+ params.update(
+ {
+ "lastModified": _format_last_modified(db_incremental_field_last_value),
+ "orderBy": "LastModifiedDate",
+ "orderDirection": "Asc",
+ }
+ )
+ elif config.supports_sort:
# Explicit stable sort even for full refresh: Bloomerang's own default order direction
# varies by endpoint (Transactions defaults to Desc), so an unstated default can't be
# relied on to keep page boundaries stable as rows are inserted mid-sync.
- return {"orderBy": config.sort_field, "orderDirection": "Asc"}
- return {}
+ params.update({"orderBy": config.sort_field, "orderDirection": "Asc"})
+ return params
def _client_config(api_key: str) -> ClientConfig:
@@ -78,8 +81,8 @@ def _client_config(api_key: str) -> ClientConfig:
"base_url": BASE_URL,
"headers": {"Accept": "application/json"},
"auth": {"type": "api_key", "name": "X-Api-Key", "api_key": api_key, "location": "header"},
- # `capture=False`: Constituents, Transactions, and Interactions carry donor PII and
- # free-form interaction notes that the name-based sample scrubbers can't recognise, so
+ # `capture=False`: Constituents, Transactions, Interactions, Notes, and Tasks carry donor PII
+ # and free-form notes that the name-based sample scrubbers can't recognise, so
# keep every response body out of HTTP sample storage. Requests stay metered and logged.
"session": make_tracked_session(redact_values=(api_key,), capture=False),
# `X-Api-Key` isn't the standard Authorization header, which `requests` strips on a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py
index 53825b4be52f..11646dd9e506 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py
@@ -155,4 +155,72 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"IsPrimary": "Whether this is the constituent's primary phone number.",
},
},
+ "Addresses": {
+ "description": "A mailing address on a constituent record, including non-primary addresses.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the address record used by the API.",
+ "AccountId": "ID of the constituent this address belongs to.",
+ "Type": "Kind of address, such as Home or Work.",
+ "Street": "Street portion of the address.",
+ "City": "City of the address.",
+ "State": "State or province name, matching a state in the Bloomerang CRM.",
+ "StateAbbreviation": "Two- or three-letter abbreviation for the state or province.",
+ "PostalCode": "Postal or ZIP code.",
+ "Country": "Country name, matching a country in the Bloomerang CRM.",
+ "CountryCode": "Two-letter ISO code for the country.",
+ "IsPrimary": "Whether this is the constituent's primary address.",
+ "IsBad": "Whether the address is marked as bad (undeliverable).",
+ "CreatedDate": "When the address record was created.",
+ "LastModifiedDate": "When the address record was last modified.",
+ },
+ },
+ "Emails": {
+ "description": "An email address on a constituent record, including non-primary addresses.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the email record used by the API.",
+ "AccountId": "ID of the constituent this email address belongs to.",
+ "Type": "Kind of email address, such as Home or Work.",
+ "Value": "The email address.",
+ "IsPrimary": "Whether this is the constituent's primary email address.",
+ "IsBad": "Whether the email address is marked as bad (undeliverable).",
+ "CreatedDate": "When the email record was created.",
+ "LastModifiedDate": "When the email record was last modified.",
+ },
+ },
+ "Notes": {
+ "description": "A free-form note recorded against a constituent.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the note used by the API.",
+ "AccountId": "ID of the constituent the note is about.",
+ "Date": "Date the note applies to.",
+ "Note": "Text of the note.",
+ "CustomValues": "Values of custom fields set on the note.",
+ "AttachmentIds": "IDs of files attached to the note.",
+ "CreatedDate": "When the note was created.",
+ "LastModifiedDate": "When the note was last modified.",
+ },
+ },
+ "Tasks": {
+ "description": "A follow-up task assigned to a Bloomerang user for a constituent.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the task used by the API.",
+ "AccountId": "ID of the constituent the task is for.",
+ "Status": "Whether the task is Active, Complete, or Archived.",
+ "DueDate": "Date the task is due.",
+ "CompletedDate": "Date the task was completed, if it was.",
+ "Subject": "Short subject line of the task.",
+ "Note": "Free-form note on the task.",
+ "Channel": "Channel the task uses, such as Email, Phone, or InPerson.",
+ "Purpose": "Purpose of the task, such as Acknowledgement, Solicitation, or PledgeReminder.",
+ "UserId": "ID of the Bloomerang user the task is assigned to.",
+ "InteractionId": "ID of the interaction created when the task was completed.",
+ "AttachmentIds": "IDs of files attached to the task.",
+ "CreatedDate": "When the task was created.",
+ "LastModifiedDate": "When the task was last modified.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py
index 65d04433c3cd..877b783727d3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py
@@ -28,8 +28,8 @@ class BloomerangEndpointConfig:
name: str
path: str # path under https://api.bloomerang.co/v2
# Whether the list action accepts a server-side `lastModified` filter *and* `orderBy=LastModifiedDate`,
- # so incremental pages arrive in ascending cursor order. Households accepts the filter but not
- # `orderBy`, so it stays full refresh.
+ # so incremental pages arrive in ascending cursor order. Households, Addresses, and Emails accept
+ # the filter but not `orderBy`, so they stay full refresh.
supports_incremental: bool
# Whether the list action accepts `orderBy`/`orderDirection` at all (Appeals/Campaigns/Funds
# accept neither, per the API's own parameter docs).
@@ -37,9 +37,10 @@ class BloomerangEndpointConfig:
# Whether rows carry a nested `AuditTrail` object ({CreatedDate, LastModifiedDate, ...}) that
# gets flattened onto the row root. Appeals/Campaigns/Funds have no AuditTrail field.
has_audit_trail: bool
- # Column for the stable full-refresh sort. The `orderBy` enum varies per endpoint: Refunds and
- # SoftCredits don't accept `Id`.
+ # Column for the stable full-refresh sort. The `orderBy` enum varies per endpoint: Refunds,
+ # SoftCredits, Notes, and Tasks don't accept `Id`.
sort_field: str = "Id"
+ extra_params: dict[str, str] = field(default_factory=dict)
incremental_fields: list[IncrementalField] = field(default_factory=list)
primary_keys: list[str] = field(default_factory=lambda: ["Id"])
# Stable creation-time column to partition by. None when the endpoint has no AuditTrail.
@@ -126,6 +127,38 @@ BLOOMERANG_ENDPOINTS: dict[str, BloomerangEndpointConfig] = {
has_audit_trail=False,
partition_key=None,
),
+ "Addresses": BloomerangEndpointConfig(
+ name="Addresses",
+ path="addresses",
+ supports_incremental=False,
+ supports_sort=False,
+ has_audit_trail=True,
+ ),
+ "Emails": BloomerangEndpointConfig(
+ name="Emails",
+ path="emails",
+ supports_incremental=False,
+ supports_sort=False,
+ has_audit_trail=True,
+ ),
+ "Notes": BloomerangEndpointConfig(
+ name="Notes",
+ path="notes",
+ supports_incremental=False,
+ supports_sort=True,
+ has_audit_trail=True,
+ sort_field="CreatedDate",
+ ),
+ "Tasks": BloomerangEndpointConfig(
+ name="Tasks",
+ path="tasks",
+ supports_incremental=False,
+ supports_sort=True,
+ has_audit_trail=True,
+ sort_field="CreatedDate",
+ # The API doesn't document which statuses an unfiltered request returns, so ask for all of them.
+ extra_params={"status": "Active|Complete|Archived"},
+ ),
}
ENDPOINTS = tuple(BLOOMERANG_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py
index e02ea8304aef..8d9d0716efb6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py
@@ -127,7 +127,7 @@ class BloomerangSource(ResumableSource[BloomerangSourceConfig, BloomerangResumeC
category=DataWarehouseSourceCategory.CRM,
keywords=["donor", "nonprofit", "fundraising"],
releaseStatus=ReleaseStatus.ALPHA,
- caption="""Enter your Bloomerang private API key to pull your donor database into the PostHog Data warehouse, including constituents, households, phones, transactions, refunds, soft credits, interactions, appeals, campaigns, and funds.
+ caption="""Enter your Bloomerang private API key to pull your donor database into the PostHog Data warehouse, including constituents, households, addresses, emails, phones, notes, tasks, transactions, refunds, soft credits, interactions, appeals, campaigns, and funds.
Generate a private API key under **Settings → User Settings → API Keys** in your Bloomerang account. This key grants full read/write access to your Bloomerang data, so keep it secret.""",
iconPath="/static/services/bloomerang.png",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py
index ea49f77d000c..1f2580f2a5b8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py
@@ -55,8 +55,9 @@ class TestBuildParams:
@parameterized.expand(
[
("Constituents", "Id"),
- # Refunds and SoftCredits reject `orderBy=Id`.
+ # Refunds, SoftCredits, and Notes reject `orderBy=Id`.
("SoftCredits", "CreatedDate"),
+ ("Notes", "CreatedDate"),
]
)
def test_incremental_endpoint_without_cursor_falls_back_to_stable_sort(
@@ -180,6 +181,13 @@ class TestBloomerangSourceTransport:
assert result.requests[0].param("lastModified") == "2026-03-04T02:58:14Z"
assert result.requests[0].param("orderBy") == "LastModifiedDate"
+ def test_tasks_requests_every_status(self) -> None:
+ result = self._driver().run("Tasks", [_response([{"Id": 1}], total_filtered=1)])
+
+ assert result.raised is None
+ assert result.requests[0].param("status") == "Active|Complete|Archived"
+ assert result.requests[0].param("orderBy") == "CreatedDate"
+
def test_audit_trail_endpoint_flattens_rows(self) -> None:
result = self._driver().run(
"Constituents",