:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 1 missing table to the gleif source (#124780)
Tom Owersce0995905f6b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 475efb6ac397..9d3612fdd590 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -5371,7 +5371,7 @@ Today (6): `Countries`, `EntityLegalForms`, `Jurisdictions`, `LeiIssuers`, `LeiRDiffed against: <https://documenter.gw.postman.com/api/collections/7679680/SVYrrxuU?segregateAuth=true&versionTag=latest>-- [ ] `Region Codes (GET /api/v1/regions)` — resolve address region codes to named subdivisions for geographic entity analysis (high)+- [x] `Region Codes (GET /api/v1/regions)` — resolve address region codes to named subdivisions for geographic entity analysis (high)- [ ] `Direct Child Relationship(s) (GET /api/v1/lei-records/{id}/direct-child-relationships)` — build direct consolidation hierarchies and identify related entities for counterparty exposure analysis (high)- [ ] `Ultimate Child Relationship(s) (GET /api/v1/lei-records/{id}/ultimate-child-relationships)` — connect entities to their ultimate consolidation parents for group exposure and corporate structure analysis (high)- [ ] `ISINs (GET /api/v1/lei-records/{id}/isins)` — join legal entities to issued securities and connect issuer identity with investment data (high)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.pyindex def50c98b57f..24b6d8b32568 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/canonical_descriptions.py@@ -86,4 +86,14 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"names": "Localized names for the jurisdiction (nested list).",},},+ "Regions": {+ "description": "An ISO 3166-2 country subdivision (region) code, used for the region of LEI record addresses.",+ "docs_url": "https://www.gleif.org/en/lei-data/gleif-api",+ "columns": {+ "id": "ISO 3166-2 subdivision code, same as `code`.",+ "code": "ISO 3166-2 subdivision code (e.g. `US-CA`, `DE-BY`).",+ "name": "A single display name for the region. Often null; the localized names are in `names`.",+ "names": "Localized names for the region, each with its language code (nested list).",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.pyindex ccea82872d88..e6f382a42256 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/settings.py@@ -12,6 +12,7 @@ ENTITY_LEGAL_FORMS = "EntityLegalForms"REGISTRATION_AUTHORITIES = "RegistrationAuthorities"COUNTRIES = "Countries"JURISDICTIONS = "Jurisdictions"+REGIONS = "Regions"ENDPOINTS = (LEI_RECORDS,@@ -20,6 +21,7 @@ ENDPOINTS = (REGISTRATION_AUTHORITIES,COUNTRIES,JURISDICTIONS,+ REGIONS,)ENDPOINT_PATHS: dict[str, str] = {@@ -29,11 +31,12 @@ ENDPOINT_PATHS: dict[str, str] = {REGISTRATION_AUTHORITIES: "/registration-authorities",COUNTRIES: "/countries",JURISDICTIONS: "/jurisdictions",+ REGIONS: "/regions",}# Only `lei-records` exposes a server-side timestamp filter (verified live:# `filter[registration.lastUpdateDate]=>=DATE`). The reference tables (issuers, legal forms,-# registration authorities, countries, jurisdictions) change rarely and expose no filter, so+# registration authorities, countries, jurisdictions, regions) change rarely and expose no filter, so# they sync full refresh only.INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {LEI_RECORDS: [incremental_field("last_update_date")],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.pyindex 9e480979e423..a246656aaaad 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gleif/tests/test_gleif.py@@ -14,7 +14,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.glei_flatten_json_api_item,validate_credentials,)-from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.settings import LEI_RECORDS+from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.settings import (+ COUNTRIES,+ LEI_RECORDS,+ REGIONS,+)from products.warehouse_sources.backend.temporal.data_imports.sources.gleif.source import GleifSource_LEI_ITEM: dict[str, Any] = {@@ -75,6 +79,43 @@ class TestGleifSourceResumeBehavior:assert result.rows[0]["last_update_date"] == "2026-08-17T22:46:53Z"assert result.saved_states == [GleifResumeConfig(next_url=next_url)]+ @pytest.mark.parametrize(+ ("endpoint", "expected_path", "item"),+ [+ (+ REGIONS,+ "/api/v1/regions",+ {+ "type": "regions",+ "id": "XX-01",+ "attributes": {+ "code": "XX-01",+ "name": None,+ "names": [{"language": "en", "name": "North Testland"}],+ },+ },+ ),+ (+ COUNTRIES,+ "/api/v1/countries",+ {"type": "countries", "id": "XX", "attributes": {"code": "XX", "name": "Testland"}},+ ),+ ],+ )+ def test_reference_endpoint_walks_page_number_links_to_last_page(+ self, endpoint: str, expected_path: str, item: dict[str, Any]+ ) -> None:+ next_url = f"https://api.gleif.org{expected_path}?page%5Bnumber%5D=2&page%5Bsize%5D=200"+ result = self._driver().run(endpoint, [self._page([item], next_url), self._page([item])])++ assert result.raised is None+ assert result.requests[0].path == expected_path+ # Reference tables use page-number links, so the lei-records cursor and sort must not leak in.+ assert result.queries[0] == {"page[size]": ("200",)}+ assert result.urls[1] == next_url+ assert len(result.requests) == 2+ assert result.rows == [{"id": item["id"], **item["attributes"]}] * 2+def test_lei_records_incremental_run_adds_gte_filter(self) -> None:result = self._driver().run(LEI_RECORDS,