Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add opencorporates filings, statements and jurisdiction tables (#123363)

Tom Owers5e999590bfd3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c483a4703c55..46d1560d5abd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8781,11 +8781,11 @@ Today (2): `Companies`, `Officers`
Diffed against: <https://api.opencorporates.com/documentation/API-Reference>
-- [ ] `Jurisdictions (GET /jurisdictions)` — resolve jurisdiction_code values in company records for geographic comparisons (high)
-- [ ] `Company Filings (GET /companies/:jurisdiction_code/:company_number/filings)` — analyze statutory filing history and changes in company status (high)
-- [ ] `Company Statements (GET /companies/:jurisdiction_code/:company_number/statements)` — analyze ownership relationships and other sourced facts omitted from company search results (high)
+- [x] `Jurisdictions (GET /jurisdictions)` — resolve jurisdiction_code values in company records for geographic comparisons (high)
+- [x] `Company Filings (GET /companies/:jurisdiction_code/:company_number/filings)` — analyze statutory filing history and changes in company status (high)
+- [x] `Company Statements (GET /companies/:jurisdiction_code/:company_number/statements)` — analyze ownership relationships and other sourced facts omitted from company search results (high)
- [ ] `Gazette Notices (GET /statements/gazette_notices/search)` — track official notices about insolvency, dissolution, and other company events (medium)
-- [ ] `Registration Statements (GET /statements/alternate_registrations/search; GET /statements/subsequent_registrations/search)` — link alternate and successor registrations to avoid duplicate company counts (medium)
+- [x] `Registration Statements (GET /statements/alternate_registrations/search; GET /statements/subsequent_registrations/search)` — link alternate and successor registrations to avoid duplicate company counts (medium)
- [ ] `Trademark Registrations (GET /statements/trademark_registrations/search)` — analyze trademark ownership, registration dates, and expirations (medium)
- [ ] `Industry Codes (GET /industry_codes; GET /industry_codes/:code_scheme_id)` — build industry classifications for company segmentation and comparisons across code schemes (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/canonical_descriptions.py
index f57739e05475..9003dffb42b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/canonical_descriptions.py
@@ -42,4 +42,65 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"opencorporates_url": "The canonical OpenCorporates URL for the officer.",
},
},
+ "Jurisdictions": {
+ "description": "Every jurisdiction OpenCorporates knows about, including jurisdictions it has no companies for.",
+ "docs_url": "https://api.opencorporates.com/documentation/API-Reference#jurisdictions",
+ "columns": {
+ "code": "The jurisdiction code used in company records, e.g. `gb` or `us_de`.",
+ "name": "The name of the jurisdiction.",
+ "country": "The country the jurisdiction belongs to.",
+ },
+ },
+ "CompanyFilings": {
+ "description": "Statutory filings for each company matching the configured search term (and optional jurisdiction).",
+ "docs_url": "https://api.opencorporates.com/documentation/API-Reference#get-companies-jurisdiction_code-company_number-filings",
+ "columns": {
+ "id": "The OpenCorporates identifier of the filing.",
+ "company_jurisdiction_code": "The jurisdiction code of the company the filing belongs to.",
+ "company_number": "The registry-issued number of the company the filing belongs to.",
+ "date": "The date of the filing.",
+ "uid": "The identifier given to the filing by the company registry.",
+ "title": "The title of the filing, its filing type, or a readable version of its date.",
+ "description": "The description of the filing given by the registry.",
+ "url": "A link to the filing, either an HTML page or the document itself.",
+ "filing_type_name": "A human-readable name of the type of filing.",
+ "filing_code": "The code the company registry gives to this type of filing.",
+ "opencorporates_url": "The canonical OpenCorporates URL for the filing.",
+ },
+ },
+ "CompanyStatements": {
+ "description": "Statements of fact (e.g. subsidiary, shareholding, or branch relationships) about each company matching the configured search term (and optional jurisdiction).",
+ "docs_url": "https://api.opencorporates.com/documentation/API-Reference#get-companies-jurisdiction_code-company_number-statements",
+ "columns": {
+ "id": "The OpenCorporates identifier of the statement.",
+ "company_jurisdiction_code": "The jurisdiction code of the company the statement was listed for.",
+ "company_number": "The registry-issued number of the company the statement was listed for.",
+ "data_type": "The type of statement, e.g. subsidiary, shareholding, or branch_relationship.",
+ "properties": "The core of what the statement says. The attributes vary by `data_type`.",
+ "provenances": "Where the statement came from: a datetime, source URL, creating actor, and confidence for each provenance.",
+ "opencorporates_url": "The canonical OpenCorporates URL for the statement.",
+ },
+ },
+ "AlternateRegistrations": {
+ "description": "Alternate registrations matching the configured search term: the same company represented in another official registration, e.g. when more than one headquarters is registered.",
+ "docs_url": "https://api.opencorporates.com/documentation/API-Reference#get-statements-alternate_registrations-search",
+ "columns": {
+ "id": "The OpenCorporates identifier of the statement.",
+ "data_type": "The type of statement.",
+ "properties": "The registrations the statement links. The attributes vary by `data_type`.",
+ "provenances": "Where the statement came from: a datetime, source URL, creating actor, and confidence for each provenance.",
+ "opencorporates_url": "The canonical OpenCorporates URL for the statement.",
+ },
+ },
+ "SubsequentRegistrations": {
+ "description": "Subsequent registrations matching the configured search term: a later registration of the same company that supersedes an earlier one.",
+ "docs_url": "https://api.opencorporates.com/documentation/API-Reference#get-statements-subsequent_registrations-search",
+ "columns": {
+ "id": "The OpenCorporates identifier of the statement.",
+ "data_type": "The type of statement.",
+ "properties": "The registrations the statement links. The attributes vary by `data_type`.",
+ "provenances": "Where the statement came from: a datetime, source URL, creating actor, and confidence for each provenance.",
+ "opencorporates_url": "The canonical OpenCorporates URL for the statement.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/opencorporates.py b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/opencorporates.py
index 6e402ebb0428..9c13aa3c3463 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/opencorporates.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/opencorporates.py
@@ -8,7 +8,10 @@ from requests import Response
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
- rest_api_resource,
+ rest_api_resources,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ rename_parent_fields,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
PageNumberPaginator,
@@ -17,6 +20,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.opencorporates.settings import (
+ COMPANY_FANOUT_PARENT,
MAX_PAGE,
OPENCORPORATES_ENDPOINTS,
PER_PAGE,
@@ -30,7 +34,9 @@ OPENCORPORATES_BASE_URL = "https://api.opencorporates.com/v0.4"
@dataclasses.dataclass(frozen=True)
class OpencorporatesResumeConfig:
# Next 1-indexed page to fetch — OpenCorporates uses page-number pagination.
- next_page: int
+ next_page: int | None = None
+ # Per-company progress of the company filings/statements fan-out.
+ fanout_state: dict[str, Any] | None = None
def _format_incremental_filter(value: Any) -> str:
@@ -47,6 +53,14 @@ def _format_incremental_filter(value: Any) -> str:
return f"{str(value)[:10]}:"
+def _unwrap(item_key: str) -> Any:
+ def _mapper(row: dict[str, Any]) -> dict[str, Any]:
+ inner = row.get(item_key)
+ return inner if isinstance(inner, dict) else row
+
+ return _mapper
+
+
class OpencorporatesPaginator(PageNumberPaginator):
"""Page-number pagination capped at OpenCorporates' documented page<=100 limit.
@@ -80,27 +94,68 @@ def opencorporates_source(
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = OPENCORPORATES_ENDPOINTS[endpoint]
-
- params: dict[str, Any] = {"q": query, "per_page": PER_PAGE}
- if jurisdiction_code:
- params["jurisdiction_code"] = jurisdiction_code
+ search_config = OPENCORPORATES_ENDPOINTS[COMPANY_FANOUT_PARENT] if config.company_fanout else config
+
+ search_params: dict[str, Any] = {}
+ if search_config.paginated:
+ search_params["per_page"] = PER_PAGE
+ if search_config.sends_query:
+ search_params["q"] = query
+ if search_config.sends_jurisdiction_code and jurisdiction_code:
+ search_params["jurisdiction_code"] = jurisdiction_code
if config.supports_incremental and should_use_incremental_field and db_incremental_field_last_value is not None:
- params["updated_at"] = _format_incremental_filter(db_incremental_field_last_value)
+ search_params["updated_at"] = _format_incremental_filter(db_incremental_field_last_value)
# Date facets are only ever returned newest-first — there is no ascending option.
- params["order"] = "updated_at"
+ search_params["order"] = "updated_at"
- resource: EndpointResource = {
- "name": endpoint,
+ search_resource: EndpointResource = {
+ "name": search_config.name,
"endpoint": {
- "path": config.path,
- "params": params,
- "data_selector": config.data_selector,
+ "path": search_config.path,
+ "params": search_params,
+ "data_selector": search_config.data_selector,
+ "paginator": OpencorporatesPaginator() if search_config.paginated else "single_page",
},
"write_disposition": {"disposition": "merge", "strategy": "upsert"}
if should_use_incremental_field
else "replace",
"table_format": "delta",
}
+ if search_config.item_key:
+ search_resource["data_map"] = _unwrap(search_config.item_key)
+
+ resources: list[str | EndpointResource] = [search_resource]
+ if config.company_fanout:
+ resources.append(
+ {
+ "name": endpoint,
+ "endpoint": {
+ "path": config.path,
+ "params": {
+ "jurisdiction_code": {
+ "type": "resolve",
+ "resource": COMPANY_FANOUT_PARENT,
+ "field": "jurisdiction_code",
+ },
+ "company_number": {
+ "type": "resolve",
+ "resource": COMPANY_FANOUT_PARENT,
+ "field": "company_number",
+ },
+ "per_page": PER_PAGE,
+ },
+ "data_selector": config.data_selector,
+ "paginator": OpencorporatesPaginator(),
+ },
+ "include_from_parent": ["jurisdiction_code", "company_number"],
+ "data_map": rename_parent_fields(
+ COMPANY_FANOUT_PARENT,
+ {"jurisdiction_code": "company_jurisdiction_code", "company_number": "company_number"},
+ ),
+ "write_disposition": "replace",
+ "table_format": "delta",
+ }
+ )
rest_config: RESTAPIConfig = {
"client": {
@@ -108,31 +163,36 @@ def opencorporates_source(
# api_token rides in the query string (the API's only auth method); the framework auth
# redacts its value from every logged URL, captured sample, and raised error message.
"auth": {"type": "api_key", "api_key": api_token, "name": "api_token", "location": "query"},
- "paginator": OpencorporatesPaginator(),
},
- "resources": [resource],
+ "resources": resources,
}
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None:
- initial_paginator_state = {"page": resume.next_page}
+ if config.company_fanout:
+ initial_paginator_state = resume.fanout_state
+ elif resume.next_page is not None:
+ initial_paginator_state = {"page": resume.next_page}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if config.company_fanout:
+ if state is not None:
+ resumable_source_manager.save_state(OpencorporatesResumeConfig(fanout_state=state))
# Persist only while a next page remains; save AFTER a page is yielded so a crash re-yields
# the last page (merge dedupes on primary key) rather than skipping it.
- if state and state.get("page") is not None:
+ elif state and state.get("page") is not None:
resumable_source_manager.save_state(OpencorporatesResumeConfig(next_page=int(state["page"])))
- rest_resource = rest_api_resource(
+ rest_resource = rest_api_resources(
rest_config,
team_id,
job_id,
db_incremental_field_last_value,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
- )
+ )[-1]
partition_kwargs: dict[str, Any] = {}
if config.partition_key is not None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/settings.py
index 7fe38f90c644..ed68c83eb332 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/settings.py
@@ -29,6 +29,13 @@ class OpencorporatesEndpointConfig:
# Stable creation-time field to partition by. None when the resource has no reliable created_at.
partition_key: str | None = None
incremental_fields: list[IncrementalField] = field(default_factory=list)
+ sends_query: bool = True
+ sends_jurisdiction_code: bool = True
+ paginated: bool = True
+ # Rows wrapped in a single key (e.g. `{"jurisdiction": {...}}`) are unwrapped when present.
+ item_key: str | None = None
+ # Fan-out child of the company search: `path` holds `{jurisdiction_code}` and `{company_number}`.
+ company_fanout: bool = False
OPENCORPORATES_ENDPOINTS: dict[str, OpencorporatesEndpointConfig] = {
@@ -53,8 +60,56 @@ OPENCORPORATES_ENDPOINTS: dict[str, OpencorporatesEndpointConfig] = {
# officers/search documents no created_at/updated_at date filter, so this stays full refresh.
supports_incremental=False,
),
+ "Jurisdictions": OpencorporatesEndpointConfig(
+ name="Jurisdictions",
+ path="/jurisdictions",
+ data_selector="results.jurisdictions",
+ primary_keys=["code"],
+ supports_incremental=False,
+ sends_query=False,
+ sends_jurisdiction_code=False,
+ paginated=False,
+ item_key="jurisdiction",
+ ),
+ "CompanyFilings": OpencorporatesEndpointConfig(
+ name="CompanyFilings",
+ path="/companies/{jurisdiction_code}/{company_number}/filings",
+ data_selector="results.filings[*].filing",
+ primary_keys=["company_jurisdiction_code", "company_number", "id"],
+ # The filings endpoint documents no date filter.
+ supports_incremental=False,
+ company_fanout=True,
+ ),
+ "CompanyStatements": OpencorporatesEndpointConfig(
+ name="CompanyStatements",
+ path="/companies/{jurisdiction_code}/{company_number}/statements",
+ data_selector="results.statements[*].statement",
+ # One statement (e.g. a subsidiary relationship) can relate to several companies.
+ primary_keys=["company_jurisdiction_code", "company_number", "id"],
+ supports_incremental=False,
+ company_fanout=True,
+ ),
+ "AlternateRegistrations": OpencorporatesEndpointConfig(
+ name="AlternateRegistrations",
+ path="/statements/alternate_registrations/search",
+ data_selector="results.statements[*].statement",
+ primary_keys=["id"],
+ # Ordered by created_at, but the endpoint documents no date filter.
+ supports_incremental=False,
+ sends_jurisdiction_code=False,
+ ),
+ "SubsequentRegistrations": OpencorporatesEndpointConfig(
+ name="SubsequentRegistrations",
+ path="/statements/subsequent_registrations/search",
+ data_selector="results.statements[*].statement",
+ primary_keys=["id"],
+ supports_incremental=False,
+ sends_jurisdiction_code=False,
+ ),
}
+COMPANY_FANOUT_PARENT = "Companies"
+
ENDPOINTS = tuple(OPENCORPORATES_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/source.py
index 1d45ddb77f6a..d7438d5c595f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/source.py
@@ -57,7 +57,7 @@ class OpencorporatesSource(ResumableSource[OpencorporatesSourceConfig, Opencorpo
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your OpenCorporates API token and a search term to pull matching company and officer records into the PostHog Data warehouse.
-Get an API token from [OpenCorporates](https://opencorporates.com/api_accounts/new). The free open-data tier is limited to around 50 requests a day and 200 a month, so scope your search with a jurisdiction to stay within it.""",
+Get an API token from [OpenCorporates](https://opencorporates.com/api_accounts/new). The free open-data tier is limited to around 50 requests a day and 200 a month, so scope your search with a jurisdiction to stay within it. The company filings and company statements tables make at least one request for every matching company.""",
iconPath="/static/services/opencorporates.png",
docsUrl="https://posthog.com/docs/cdp/sources/opencorporates",
keywords=["company registry", "kyb", "firmographics"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/tests/test_opencorporates.py b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/tests/test_opencorporates.py
index 0d4bc3a13de1..374f10dec8b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/tests/test_opencorporates.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/opencorporates/tests/test_opencorporates.py
@@ -12,6 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.opencorporates import (
@@ -93,6 +94,122 @@ class TestCompaniesPagination:
assert result.params("page") == ["3"]
+def _statements_page(statements: list[dict[str, Any]], *, page: int = 1, total_pages: int = 1) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={
+ "results": {"statements": [{"statement": s} for s in statements], "page": page, "total_pages": total_pages}
+ }
+ )
+
+
+class TestCompanyFanout:
+ @parameterized.expand(
+ [
+ ("filings", "CompanyFilings", "filings", "filing"),
+ ("statements", "CompanyStatements", "statements", "statement"),
+ ]
+ )
+ def test_walks_every_company_and_tags_rows_with_it(
+ self, _name: str, endpoint: str, collection: str, item_key: str
+ ) -> None:
+ def child_page(ids: list[int], *, page: int, total_pages: int) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={
+ "results": {
+ collection: [{item_key: {"id": i}} for i in ids],
+ "page": page,
+ "total_pages": total_pages,
+ }
+ }
+ )
+
+ result = _driver(jurisdiction_code="gb").run(
+ endpoint,
+ route(
+ {
+ "/companies/search": [
+ _companies_page([{"company_number": "001", "jurisdiction_code": "gb"}], page=1, total_pages=2),
+ _companies_page([{"company_number": "002", "jurisdiction_code": "gb"}], page=2, total_pages=2),
+ ],
+ f"/companies/gb/001/{collection}": [
+ child_page([1, 2], page=1, total_pages=2),
+ child_page([3], page=2, total_pages=2),
+ ],
+ f"/companies/gb/002/{collection}": [child_page([1], page=1, total_pages=1)],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert [(r["company_number"], r["id"]) for r in result.rows] == [("001", 1), ("001", 2), ("001", 3), ("002", 1)]
+ assert {r["company_jurisdiction_code"] for r in result.rows} == {"gb"}
+ assert result.requests[0].param("q") == "acme"
+ assert result.requests[0].param("jurisdiction_code") == "gb"
+ assert "supersecret" not in str(result.rows)
+
+ def test_resume_skips_completed_companies(self) -> None:
+ result = _driver().run(
+ "CompanyFilings",
+ route(
+ {
+ "/companies/search": [
+ _companies_page(
+ [
+ {"company_number": "001", "jurisdiction_code": "gb"},
+ {"company_number": "002", "jurisdiction_code": "gb"},
+ ],
+ page=1,
+ total_pages=1,
+ )
+ ],
+ "/companies/gb/002/filings": [
+ ScriptedResponse(json={"results": {"filings": [{"filing": {"id": 9}}], "total_pages": 1}})
+ ],
+ }
+ ),
+ resume_state=OpencorporatesResumeConfig(
+ fanout_state={"completed": ["/companies/gb/001/filings"], "current": None, "child_state": None}
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/v0.4/companies/search", "/v0.4/companies/gb/002/filings"]
+ assert [r["id"] for r in result.rows] == [9]
+
+
+class TestUnscopedEndpoints:
+ @parameterized.expand(
+ [
+ ("wrapped", {"jurisdiction": {"code": "gb", "name": "United Kingdom"}}),
+ ("unwrapped", {"code": "gb", "name": "United Kingdom"}),
+ ]
+ )
+ def test_jurisdictions_is_one_unfiltered_request(self, _name: str, row: dict[str, Any]) -> None:
+ result = _driver(jurisdiction_code="us_de").run(
+ "Jurisdictions", [ScriptedResponse(json={"results": {"jurisdictions": [row]}})]
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"code": "gb", "name": "United Kingdom"}]
+ assert len(result.requests) == 1
+ assert result.queries[0].keys() == {"api_token"}
+
+ @parameterized.expand([("AlternateRegistrations",), ("SubsequentRegistrations",)])
+ def test_registration_search_sends_query_but_not_jurisdiction(self, endpoint: str) -> None:
+ result = _driver(jurisdiction_code="gb").run(
+ endpoint,
+ [
+ _statements_page([{"id": 1}], page=1, total_pages=2),
+ _statements_page([{"id": 2}], page=2, total_pages=2),
+ ],
+ )
+
+ assert result.raised is None
+ assert [r["id"] for r in result.rows] == [1, 2]
+ assert result.params("q") == ["acme", "acme"]
+ assert result.params("jurisdiction_code") == [None, None]
+
+
class TestRequestParams:
def test_jurisdiction_code_is_passed_when_set(self) -> None:
result = _driver(jurisdiction_code="gb").run("Companies", [_companies_page([], page=1, total_pages=0)])