Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 1 missing table to the gleif source (#124780)

Tom Owersce0995905f6b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 475efb6ac397..9d3612fdd590 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5371,7 +5371,7 @@ Today (6): `Countries`, `EntityLegalForms`, `Jurisdictions`, `LeiIssuers`, `LeiR
Diffed against: <https://documenter.gw.postman.com/api/collections/7679680/SVYrrxuU?segregateAuth=true&versionTag=latest>
-- [ ] `Region Codes (GET /api/v1/regions)` — resolve address region codes to named subdivisions for geographic entity analysis (high)
+- [x] `Region Codes (GET /api/v1/regions)` — resolve address region codes to named subdivisions for geographic entity analysis (high)
- [ ] `Direct Child Relationship(s) (GET /api/v1/lei-records/{id}/direct-child-relationships)` — build direct consolidation hierarchies and identify related entities for counterparty exposure analysis (high)
- [ ] `Ultimate Child Relationship(s) (GET /api/v1/lei-records/{id}/ultimate-child-relationships)` — connect entities to their ultimate consolidation parents for group exposure and corporate structure analysis (high)
- [ ] `ISINs (GET /api/v1/lei-records/{id}/isins)` — join legal entities to issued securities and connect issuer identity with investment data (high)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.py
index def50c98b57f..24b6d8b32568 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.py
@@ -86,4 +86,14 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"names": "Localized names for the jurisdiction (nested list).",
},
},
+ "Regions": {
+ "description": "An ISO 3166-2 country subdivision (region) code, used for the region of LEI record addresses.",
+ "docs_url": "https://www.gleif.org/en/lei-data/gleif-api",
+ "columns": {
+ "id": "ISO 3166-2 subdivision code, same as `code`.",
+ "code": "ISO 3166-2 subdivision code (e.g. `US-CA`, `DE-BY`).",
+ "name": "A single display name for the region. Often null; the localized names are in `names`.",
+ "names": "Localized names for the region, each with its language code (nested list).",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.py
index ccea82872d88..e6f382a42256 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.py
@@ -12,6 +12,7 @@ ENTITY_LEGAL_FORMS = "EntityLegalForms"
REGISTRATION_AUTHORITIES = "RegistrationAuthorities"
COUNTRIES = "Countries"
JURISDICTIONS = "Jurisdictions"
+REGIONS = "Regions"
ENDPOINTS = (
LEI_RECORDS,
@@ -20,6 +21,7 @@ ENDPOINTS = (
REGISTRATION_AUTHORITIES,
COUNTRIES,
JURISDICTIONS,
+ REGIONS,
)
ENDPOINT_PATHS: dict[str, str] = {
@@ -29,11 +31,12 @@ ENDPOINT_PATHS: dict[str, str] = {
REGISTRATION_AUTHORITIES: "/registration-authorities",
COUNTRIES: "/countries",
JURISDICTIONS: "/jurisdictions",
+ REGIONS: "/regions",
}
# Only `lei-records` exposes a server-side timestamp filter (verified live:
# `filter[registration.lastUpdateDate]=>=DATE`). The reference tables (issuers, legal forms,
-# registration authorities, countries, jurisdictions) change rarely and expose no filter, so
+# registration authorities, countries, jurisdictions, regions) change rarely and expose no filter, so
# they sync full refresh only.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
LEI_RECORDS: [incremental_field("last_update_date")],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.py
index 9e480979e423..a246656aaaad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.py
@@ -14,7 +14,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.glei
_flatten_json_api_item,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.settings import LEI_RECORDS
+from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.settings import (
+ COUNTRIES,
+ LEI_RECORDS,
+ REGIONS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.source import GleifSource
_LEI_ITEM: dict[str, Any] = {
@@ -75,6 +79,43 @@ class TestGleifSourceResumeBehavior:
assert result.rows[0]["last_update_date"] == "2026-08-17T22:46:53Z"
assert result.saved_states == [GleifResumeConfig(next_url=next_url)]
+ @pytest.mark.parametrize(
+ ("endpoint", "expected_path", "item"),
+ [
+ (
+ REGIONS,
+ "/api/v1/regions",
+ {
+ "type": "regions",
+ "id": "XX-01",
+ "attributes": {
+ "code": "XX-01",
+ "name": None,
+ "names": [{"language": "en", "name": "North Testland"}],
+ },
+ },
+ ),
+ (
+ COUNTRIES,
+ "/api/v1/countries",
+ {"type": "countries", "id": "XX", "attributes": {"code": "XX", "name": "Testland"}},
+ ),
+ ],
+ )
+ def test_reference_endpoint_walks_page_number_links_to_last_page(
+ self, endpoint: str, expected_path: str, item: dict[str, Any]
+ ) -> None:
+ next_url = f"https://api.gleif.org{expected_path}?page%5Bnumber%5D=2&page%5Bsize%5D=200"
+ result = self._driver().run(endpoint, [self._page([item], next_url), self._page([item])])
+
+ assert result.raised is None
+ assert result.requests[0].path == expected_path
+ # Reference tables use page-number links, so the lei-records cursor and sort must not leak in.
+ assert result.queries[0] == {"page[size]": ("200",)}
+ assert result.urls[1] == next_url
+ assert len(result.requests) == 2
+ assert result.rows == [{"id": item["id"], **item["attributes"]}] * 2
+
def test_lei_records_incremental_run_adds_gte_filter(self) -> None:
result = self._driver().run(
LEI_RECORDS,