Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Bloomerang source (#118991)

Tom Owersf4badf9f065d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index e7e9121da8cd..246d70bdf631 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1731,12 +1731,12 @@ Today (6): `Appeals`, `Campaigns`, `Constituents`, `Funds`, `Interactions`, `Tra
Diffed against: <https://bloomerang-api-documentation.s3.us-west-2.amazonaws.com/public_crm_generated.json>
-- [ ] `Households (GET /v2/households)` — resolve HouseholdId in constituents and aggregate giving across household members (high)
-- [ ] `Soft credits (GET /v2/softcredits)` — resolve SoftCreditIds in transaction designations and attribute giving credit to additional donors (high)
-- [ ] `Refunds (GET /v2/refunds)` — resolve RefundIds and report refund amounts and dates for accurate net giving (high)
+- [x] `Households (GET /v2/households)` — resolve HouseholdId in constituents and aggregate giving across household members (high)
+- [x] `Soft credits (GET /v2/softcredits)` — resolve SoftCreditIds in transaction designations and attribute giving credit to additional donors (high)
+- [x] `Refunds (GET /v2/refunds)` — resolve RefundIds and report refund amounts and dates for accurate net giving (high)
- [ ] `Addresses (GET /v2/addresses)` — resolve AddressIds beyond the primary address for geographic and household analysis (high)
- [ ] `Email addresses (GET /v2/emails)` — resolve EmailIds beyond the primary email for contact coverage and donor matching (high)
-- [ ] `Phones (GET /v2/phones)` — resolve PhoneIds beyond the primary phone for contact coverage and outreach analysis (high)
+- [x] `Phones (GET /v2/phones)` — resolve PhoneIds beyond the primary phone for contact coverage and outreach analysis (high)
- [ ] `Relationships (GET /v2/constituent/{id}/relationships)` — analyze donor relationships, employers, and connections that affect fundraising (medium)
- [ ] `Pledge payment failures (GET /v2/pledge/{id}/paymentFailures)` — measure failed pledge payments and identify donations at risk (medium)
- [ ] `Notes (GET /v2/notes)` — analyze donor notes alongside transactions and interactions (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py
index bed48d7bcc5d..cae24e7136c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/bloomerang.py
@@ -33,9 +33,9 @@ class BloomerangResumeConfig:
def _flatten_audit_trail(item: dict[str, Any]) -> dict[str, Any]:
"""Promote AuditTrail.CreatedDate/LastModifiedDate onto the row root.
- Constituents, Transactions, and Interactions nest their timestamps under an `AuditTrail`
- object; flattening them makes `CreatedDate` usable as a partition key and `LastModifiedDate`
- usable as the incremental cursor column.
+ Constituents, Transactions, Interactions, Refunds, SoftCredits, and Households nest their
+ timestamps under an `AuditTrail` object; flattening them makes `CreatedDate` usable as a
+ partition key and `LastModifiedDate` usable as the incremental cursor column.
"""
audit_trail = item.pop("AuditTrail", None)
if isinstance(audit_trail, dict):
@@ -69,7 +69,7 @@ def _build_params(
# Explicit stable sort even for full refresh: Bloomerang's own default order direction
# varies by endpoint (Transactions defaults to Desc), so an unstated default can't be
# relied on to keep page boundaries stable as rows are inserted mid-sync.
- return {"orderBy": "Id", "orderDirection": "Asc"}
+ return {"orderBy": config.sort_field, "orderDirection": "Asc"}
return {}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py
index bad7d560da12..53825b4be52f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/canonical_descriptions.py
@@ -100,4 +100,59 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"IsActive": "Whether the fund is currently active.",
},
},
+ "Refunds": {
+ "description": "A refund issued against a transaction, used to compute net giving.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the refund used by the API.",
+ "TransactionId": "ID of the refunded transaction used by the API (not the transaction number).",
+ "Note": "Reason the refund was made.",
+ "Date": "Date of the refund.",
+ "Amount": "Amount refunded.",
+ "CreatedDate": "When the refund record was created, flattened from AuditTrail.",
+ "LastModifiedDate": "When the refund record was last modified, flattened from AuditTrail.",
+ },
+ },
+ "SoftCredits": {
+ "description": "Giving credit attributed to a constituent other than the donor for a transaction designation.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the soft credit used by the API.",
+ "DesignationId": "ID of the transaction designation this soft credit applies to.",
+ "TransactionId": "ID of the transaction this soft credit applies to.",
+ "PledgeId": "ID of the pledge this soft credit pays down, if any.",
+ "IsAppliedToPledge": "Whether this soft credit pays down a pledge.",
+ "Amount": "Soft credit amount.",
+ "AccountId": "ID of the constituent who receives the soft credit.",
+ "CreatedDate": "When the soft credit record was created, flattened from AuditTrail.",
+ "LastModifiedDate": "When the soft credit record was last modified, flattened from AuditTrail.",
+ },
+ },
+ "Households": {
+ "description": "A household that groups individual constituents so giving can be aggregated across members.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the household used by the API.",
+ "AccountNumber": "User-friendly account number shown in the Bloomerang CRM UI.",
+ "FullName": "Household's full display name.",
+ "Status": "Whether the household is Active, Inactive, or Deceased.",
+ "HeadId": "Constituent ID of the head of household.",
+ "MemberIds": "Constituent IDs of the household members, including the head of household.",
+ "EngagementScore": "Bloomerang's calculated engagement level for the household.",
+ "CreatedDate": "When the household record was created, flattened from AuditTrail.",
+ "LastModifiedDate": "When the household record was last modified, flattened from AuditTrail.",
+ },
+ },
+ "Phones": {
+ "description": "A phone number on a constituent record, including non-primary numbers.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "Id": "Unique identifier for the phone record used by the API.",
+ "AccountId": "ID of the constituent this phone number belongs to.",
+ "Type": "Kind of phone number: Home, Work, Mobile, or Fax.",
+ "Extension": "Phone extension, if any.",
+ "Number": "Phone number including formatting, without the extension.",
+ "IsPrimary": "Whether this is the constituent's primary phone number.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py
index 3f31d7c84654..65d04433c3cd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/settings.py
@@ -27,8 +27,9 @@ def _last_modified_incremental_fields() -> list[IncrementalField]:
class BloomerangEndpointConfig:
name: str
path: str # path under https://api.bloomerang.co/v2
- # Whether the list action accepts a server-side `lastModified` filter (Constituents only —
- # Transactions/Interactions accept `orderBy=LastModifiedDate` for sorting but no filter param).
+ # Whether the list action accepts a server-side `lastModified` filter *and* `orderBy=LastModifiedDate`,
+ # so incremental pages arrive in ascending cursor order. Households accepts the filter but not
+ # `orderBy`, so it stays full refresh.
supports_incremental: bool
# Whether the list action accepts `orderBy`/`orderDirection` at all (Appeals/Campaigns/Funds
# accept neither, per the API's own parameter docs).
@@ -36,6 +37,9 @@ class BloomerangEndpointConfig:
# Whether rows carry a nested `AuditTrail` object ({CreatedDate, LastModifiedDate, ...}) that
# gets flattened onto the row root. Appeals/Campaigns/Funds have no AuditTrail field.
has_audit_trail: bool
+ # Column for the stable full-refresh sort. The `orderBy` enum varies per endpoint: Refunds and
+ # SoftCredits don't accept `Id`.
+ sort_field: str = "Id"
incremental_fields: list[IncrementalField] = field(default_factory=list)
primary_keys: list[str] = field(default_factory=lambda: ["Id"])
# Stable creation-time column to partition by. None when the endpoint has no AuditTrail.
@@ -89,6 +93,39 @@ BLOOMERANG_ENDPOINTS: dict[str, BloomerangEndpointConfig] = {
has_audit_trail=False,
partition_key=None,
),
+ "Refunds": BloomerangEndpointConfig(
+ name="Refunds",
+ path="refunds",
+ supports_incremental=True,
+ supports_sort=True,
+ has_audit_trail=True,
+ sort_field="CreatedDate",
+ incremental_fields=_last_modified_incremental_fields(),
+ ),
+ "SoftCredits": BloomerangEndpointConfig(
+ name="SoftCredits",
+ path="softcredits",
+ supports_incremental=True,
+ supports_sort=True,
+ has_audit_trail=True,
+ sort_field="CreatedDate",
+ incremental_fields=_last_modified_incremental_fields(),
+ ),
+ "Households": BloomerangEndpointConfig(
+ name="Households",
+ path="households",
+ supports_incremental=False,
+ supports_sort=False,
+ has_audit_trail=True,
+ ),
+ "Phones": BloomerangEndpointConfig(
+ name="Phones",
+ path="phones",
+ supports_incremental=False,
+ supports_sort=False,
+ has_audit_trail=False,
+ partition_key=None,
+ ),
}
ENDPOINTS = tuple(BLOOMERANG_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py
index 7e5e36e26f67..e02ea8304aef 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/source.py
@@ -127,7 +127,7 @@ class BloomerangSource(ResumableSource[BloomerangSourceConfig, BloomerangResumeC
category=DataWarehouseSourceCategory.CRM,
keywords=["donor", "nonprofit", "fundraising"],
releaseStatus=ReleaseStatus.ALPHA,
- caption="""Enter your Bloomerang private API key to pull your donor database into the PostHog Data warehouse, including constituents, transactions, interactions, appeals, campaigns, and funds.
+ caption="""Enter your Bloomerang private API key to pull your donor database into the PostHog Data warehouse, including constituents, households, phones, transactions, refunds, soft credits, interactions, appeals, campaigns, and funds.
Generate a private API key under **Settings → User Settings → API Keys** in your Bloomerang account. This key grants full read/write access to your Bloomerang data, so keep it secret.""",
iconPath="/static/services/bloomerang.png",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py
index 0862c8657fd3..ea49f77d000c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bloomerang/tests/test_bloomerang.py
@@ -52,15 +52,24 @@ class TestBuildParams:
"orderDirection": "Asc",
}
- def test_incremental_endpoint_without_cursor_falls_back_to_stable_sort(self) -> None:
+ @parameterized.expand(
+ [
+ ("Constituents", "Id"),
+ # Refunds and SoftCredits reject `orderBy=Id`.
+ ("SoftCredits", "CreatedDate"),
+ ]
+ )
+ def test_incremental_endpoint_without_cursor_falls_back_to_stable_sort(
+ self, endpoint: str, expected_sort: str
+ ) -> None:
# First-ever sync: no watermark yet, but the endpoint still supports sorting, so page
# boundaries must stay stable rather than sending no params at all.
params = _build_params(
- BLOOMERANG_ENDPOINTS["Constituents"],
+ BLOOMERANG_ENDPOINTS[endpoint],
should_use_incremental_field=True,
db_incremental_field_last_value=None,
)
- assert params == {"orderBy": "Id", "orderDirection": "Asc"}
+ assert params == {"orderBy": expected_sort, "orderDirection": "Asc"}
def test_full_refresh_endpoint_with_sort_support_never_filters(self) -> None:
# Transactions has no server-side lastModified filter; a cursor must not leak into the request.