:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add openalex study designs, indexes, source lists and locations tables (#123367)
Tom Owers53665740f3aa
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 46d1560d5abd..a059307bc00b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8751,11 +8751,11 @@ Today (20): `authors`, `awards`, `continents`, `countries`, `domains`, `fields`,Diffed against: <https://help.openalex.org/api/endpoints.md>-- [ ] `Study Designs (GET /study-designs)` — resolve study design IDs on works and compare evidence types (high)-- [ ] `Indexes (GET /indexes)` — resolve indexed_in values on works and compare coverage across bibliographic indexes (high)-- [ ] `Source Lists (GET /source-lists)` — resolve listed_in IDs on sources for journal quality and coverage analysis (high)+- [x] `Study Designs (GET /study-designs)` — resolve study design IDs on works and compare evidence types (high)+- [x] `Indexes (GET /indexes)` — resolve indexed_in values on works and compare coverage across bibliographic indexes (high)+- [x] `Source Lists (GET /source-lists)` — resolve listed_in IDs on sources for journal quality and coverage analysis (high)- [ ] `Collections and Members (GET /collections; GET /collections/{id}/members)` — track research portfolios and collection membership across works, authors, and journals (medium)-- [ ] `Locations (GET /locations)` — analyze each harvested copy and its metadata beyond the locations embedded in works (medium)+- [x] `Locations (GET /locations)` — analyze each harvested copy and its metadata beyond the locations embedded in works (medium)Note: Parsed the current endpoint index and entity references, including the collections member endpoint. Existing works already contain locations and affiliation strings; dedicated locations expose additional metadata. Collection members need requests for each collection. Excluded deprecated concepts, autocomplete, text tagging, saved searches, and correction workflows.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.pyindex 7f68394891db..9fe589819531 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.py@@ -10,3 +10,4 @@ class OpenalexSourceConfig(config.Config):works_filter: str | None = Noneauthors_filter: str | None = Noneawards_filter: str | None = None+ locations_filter: str | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.pyindex c48a7c54000a..b628d910f8b9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.py@@ -448,4 +448,82 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {url="Link to the full text of the license.",),},+ "study_designs": {+ "description": "The study designs a work's research can have, such as randomized controlled trial or systematic review.",+ "docs_url": f"{_DOCS_BASE}/study-designs/list-study-designs",+ "columns": _columns(+ "id",+ "display_name",+ "created_date",+ "updated_date",+ "works_count",+ "cited_by_count",+ "works_api_url",+ "description",+ pubmed_publication_types="PubMed publication types that map to this study design.",+ ),+ },+ "indexes": {+ "description": "External bibliographic indexes and registries that list works, matching the indexed_in values on works.",+ "docs_url": f"{_DOCS_BASE}/indexes/list-indexes",+ "columns": _columns(+ "id",+ "display_name",+ "created_date",+ "updated_date",+ "works_count",+ "cited_by_count",+ "works_api_url",+ "description",+ ),+ },+ "source_lists": {+ "description": "Externally maintained lists of journals that a source can appear on, matching the listed_in values on sources.",+ "docs_url": f"{_DOCS_BASE}/source-lists/list-source-lists",+ "columns": _columns(+ "id",+ "display_name",+ "created_date",+ "updated_date",+ "works_count",+ "cited_by_count",+ "description",+ works_api_url="URL of the OpenAlex works list filtered to works whose primary location is on this list.",+ maintainer="Organization that maintains the list.",+ url="Where the maintainer publishes the list.",+ list_version="Edition of the list currently loaded, as YYYY-MM-DD. Null for lists derived continuously from another feed.",+ sources_count="Number of OpenAlex sources on this list.",+ sources_api_url="URL of the OpenAlex sources list filtered to sources on this list.",+ ),+ },+ "locations": {+ "description": "Every harvested copy of a work, one row per copy, such as the publisher version, a repository manuscript or a preprint.",+ "docs_url": f"{_DOCS_BASE}/locations/list-locations",+ "columns": {+ "id": "Namespaced identifier the copy was harvested under, for example doi:10.7717/peerj.4375. Primary key.",+ "work_id": "OpenAlex ID of the work this copy was matched into.",+ "native_id": "Identifier the copy was harvested under, without its namespace prefix.",+ "native_id_namespace": "Identifier system of native_id: doi, pmh, pmid, mag or openalex_curation.",+ "provenance": "Pipeline that delivered the record, such as crossref, datacite, pubmed, mag or repo.",+ "doi": "DOI of this copy, as a lowercase URL.",+ "endpoint_id": "For repository copies, ID of the OAI-PMH endpoint the record was harvested from.",+ "title": "Title as it appears on this copy's record.",+ "type": "Normalized work type declared on this copy's record.",+ "raw_type": "Work type as this copy's source labeled it, before normalization.",+ "source_id": "OpenAlex ID of the source hosting this copy.",+ "source_name": "Name of the hosting source as it arrived on the record.",+ "publisher": "Publisher named on the record.",+ "is_oa": "True if this copy is free to read without paying or logging in.",+ "version": "Version this copy holds: publishedVersion, acceptedVersion or submittedVersion.",+ "license": "License this copy is available under, as a short code such as cc-by.",+ "language": "Language code of the record.",+ "is_retracted": "True if this copy's record is flagged as retracted.",+ "landing_page_url": "URL of the page that describes and links to this copy.",+ "pdf_url": "Direct link to a full-text PDF of this copy, if known.",+ "ids": "Every identifier on the record, each with its namespace.",+ "urls": "Every URL on the record, each with a content type.",+ "merge_key": "Keys used to match this copy into a work: doi, arxiv, pmid and title_author.",+ "ingested_at": "When OpenAlex fetched this copy. Null on many older copies.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.pyindex 0260f5f647cc..f36466f15307 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py@@ -34,6 +34,11 @@ REQUEST_TIMEOUT_SECONDS = 60LARGE_ROW_CHUNK_SIZE = 1000LARGE_ROW_CHUNK_SIZE_BYTES = 100 * 1024 * 1024+# OpenAlex does not document how long a copy takes to reach the API after its `ingested_at`, so+# an incremental `locations` run re-reads the watermark day and the day before it. The merge+# dedupes the overlap. Two days, because a date-only `ingested_at:>` bound excludes the day it names.+INGESTED_AT_LOOKBACK = datetime.timedelta(days=2)+@dataclasses.dataclassclass OpenAlexResumeConfig:@@ -97,6 +102,21 @@ def build_filter(return ",".join(clauses) or None+def build_ingested_at_filter(base_filter: Optional[str], from_ingested_at: Any = None) -> Optional[str]:+ """Combine the user's scoping filter with the incremental `ingested_at` bound for locations.++ The first incremental run has no watermark, so it sends no bound and also reads the older+ copies that have no `ingested_at`.+ """+ clauses = []+ if base_filter and base_filter.strip():+ clauses.append(base_filter.strip())+ from_date = _as_date(from_ingested_at)+ if from_date is not None:+ clauses.append(f"ingested_at:>{(from_date - INGESTED_AT_LOOKBACK).isoformat()}")+ return ",".join(clauses) or None++def get_resource(endpoint_config: OpenAlexEndpointConfig,entity_filter: Optional[str],@@ -110,7 +130,14 @@ def get_resource("sort": endpoint_config.sort,}- if should_use_incremental_field:+ if should_use_incremental_field and endpoint_config.name == "locations":+ params["filter"] = {+ "type": "incremental",+ "cursor_path": "ingested_at",+ "initial_value": None,+ "convert": lambda last_value: build_ingested_at_filter(entity_filter, last_value),+ }+ elif should_use_incremental_field:# Pinned once per run rather than read per page: the filter has to stay identical for# the life of a cursor walk, and a long sync can cross midnight.cap = utc_today()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.pyindex 2e00207d989f..3694fd3b478e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.py@@ -55,6 +55,7 @@ OPENALEX_ENDPOINTS: dict[str, OpenAlexEndpointConfig] = {"fields": OpenAlexEndpointConfig(name="fields", path="/fields"),"subfields": OpenAlexEndpointConfig(name="subfields", path="/subfields"),"sdgs": OpenAlexEndpointConfig(name="sdgs", path="/sdgs"),+ "study_designs": OpenAlexEndpointConfig(name="study_designs", path="/study-designs"),"countries": OpenAlexEndpointConfig(name="countries", path="/countries"),"continents": OpenAlexEndpointConfig(name="continents", path="/continents"),"languages": OpenAlexEndpointConfig(name="languages", path="/languages"),@@ -62,6 +63,18 @@ OPENALEX_ENDPOINTS: dict[str, OpenAlexEndpointConfig] = {"source_types": OpenAlexEndpointConfig(name="source_types", path="/source-types"),"institution_types": OpenAlexEndpointConfig(name="institution_types", path="/institution-types"),"licenses": OpenAlexEndpointConfig(name="licenses", path="/licenses"),+ "indexes": OpenAlexEndpointConfig(name="indexes", path="/indexes"),+ "source_lists": OpenAlexEndpointConfig(name="source_lists", path="/source-lists"),+ # One row per harvested copy of a work. The `id` is a namespaced harvest identifier such as+ # `doi:10.7717/peerj.4375` rather than an OpenAlex URL, but it is still unique per copy.+ # Rows carry no `created_date`, and `ingested_at` is null on many older copies, so there is+ # no stable date to partition on.+ "locations": OpenAlexEndpointConfig(+ name="locations",+ path="/locations",+ partition_key=None,+ filter_config_field="locations_filter",+ ),}ENDPOINTS = tuple(OPENALEX_ENDPOINTS.keys())@@ -74,6 +87,11 @@ ENDPOINTS = tuple(OPENALEX_ENDPOINTS.keys())# late edits to older records; full refresh those when you need the corrections. Incremental# runs also cap the window at today, because OpenAlex carries future publication dates and one# of those rows would otherwise checkpoint a watermark nothing can lower again.+#+# `locations` filters on `ingested_at`, the time OpenAlex fetched the copy, which is also free.+# New copies always carry it, so an incremental run picks up newly harvested copies. Older copies+# without it only arrive through the first run or a full refresh.INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {"works": [incremental_field("publication_date", IncrementalFieldType.Date)],+ "locations": [incremental_field("ingested_at", IncrementalFieldType.DateTime)],}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.pyindex 69fc9d8bf291..b7e0ea70a2ef 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.py@@ -129,7 +129,7 @@ class OpenalexSource(ResumableSource[OpenalexSourceConfig, OpenAlexResumeConfig]Keys are free and take about a minute to create at [openalex.org/settings/api](https://openalex.org/settings/api).-OpenAlex indexes over 300 million works and 120 million authors, which is more than a free key's daily allowance covers. Use the filter fields to narrow the works, authors and awards tables to the records you care about, for example `authorships.institutions.lineage:i27837315` for one institution's papers. The [filtering guide](https://developers.openalex.org/guides/filtering) lists the available filter fields. Leave a filter blank to sync the whole entity.""",+OpenAlex indexes over 300 million works and 120 million authors, which is more than a free key's daily allowance covers. Use the filter fields to narrow the works, authors, awards and locations tables to the records you care about, for example `authorships.institutions.lineage:i27837315` for one institution's papers. The [filtering guide](https://developers.openalex.org/guides/filtering) lists the available filter fields. Leave a filter blank to sync the whole entity.""",iconPath="/static/services/openalex.png",docsUrl="https://posthog.com/docs/cdp/sources/openalex",releaseStatus=ReleaseStatus.BETA,@@ -169,6 +169,14 @@ OpenAlex indexes over 300 million works and 120 million authors, which is more tplaceholder="funder.id:f4320306076",secret=False,),+ SourceFieldInputConfig(+ name="locations_filter",+ label="Locations filter (optional)",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="source_id:S4306400194",+ secret=False,+ ),],),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.pyindex c05b9fab8718..10967c02c9fe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.py@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openalex.sfrom products.warehouse_sources.backend.temporal.data_imports.sources.openalex.source import OpenalexSourceINSTITUTION_FILTER = "authorships.institutions.lineage:i27837315"+SOURCE_FILTER = "source_id:S4306400194"TODAY = utc_today()@@ -49,6 +50,7 @@ def _drive(works_filter=entity_filter if endpoint == "works" else None,authors_filter=entity_filter if endpoint == "authors" else None,awards_filter=entity_filter if endpoint == "awards" else None,+ locations_filter=entity_filter if endpoint == "locations" else None,)return SourceDriver(OpenalexSource(), config).run(endpoint,@@ -226,6 +228,33 @@ class TestRequestParams:assert result.raised is Noneassert result.requests[0].param("filter") == INSTITUTION_FILTER+ @pytest.mark.parametrize(+ "should_use_incremental_field,watermark,expected_filter",+ [+ # A date-only `ingested_at:>` bound excludes the day it names, so this re-reads the+ # watermark day and the day before it for copies that reached the API late.+ (True, datetime.datetime(2026, 7, 10, 13, 45), f"{SOURCE_FILTER},ingested_at:>2026-07-08"),+ (True, "2026-07-10 05:06:08.047", f"{SOURCE_FILTER},ingested_at:>2026-07-08"),+ # The first run has no bound, so it also reads older copies with no `ingested_at`.+ (True, None, SOURCE_FILTER),+ (False, datetime.datetime(2026, 7, 10, 13, 45), SOURCE_FILTER),+ ],+ )+ def test_locations_filters_on_ingested_at(+ self, should_use_incremental_field: bool, watermark: Any, expected_filter: str+ ) -> None:+ result = _drive(+ "locations",+ [_page("c1"), _page(None)],+ entity_filter=SOURCE_FILTER,+ should_use_incremental_field=should_use_incremental_field,+ db_incremental_field_last_value=watermark,+ )++ assert result.raised is None+ # `locations` rejects the works-only publication date filters with a 400.+ assert result.params("filter") == [expected_filter, expected_filter]+@pytest.mark.parametrize("endpoint", [name for name in ENDPOINTS if name != "works"])def test_only_works_sends_a_sort(self, endpoint: str) -> None:# `sort=publication_date` is a works-only field; sending it elsewhere is a 400.@@ -234,7 +263,9 @@ class TestRequestParams:assert result.raised is Noneassert result.requests[0].param("sort") is None- @pytest.mark.parametrize("endpoint", [name for name in ENDPOINTS if name not in ("works", "authors", "awards")])+ @pytest.mark.parametrize(+ "endpoint", [name for name in ENDPOINTS if name not in ("works", "authors", "awards", "locations")]+ )def test_unfilterable_endpoints_send_no_filter(self, endpoint: str) -> None:result = _drive(endpoint, [_page(None)])@@ -254,8 +285,12 @@ class TestSourceResponseMetadata:assert response.name == endpoint# Every OpenAlex entity is keyed by its canonical OpenAlex URL, unique per entity type.assert response.primary_keys == ["id"]- assert response.partition_mode == "datetime"- assert response.partition_keys == [endpoint_config.partition_key]+ if endpoint_config.partition_key is None:+ assert response.partition_mode is None+ assert response.partition_keys is None+ else:+ assert response.partition_mode == "datetime"+ assert response.partition_keys == [endpoint_config.partition_key]# Declaring "asc" on an endpoint we cannot order would checkpoint a bogus watermark.assert response.sort_mode == ("asc" if endpoint == "works" else None)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.pyindex b26006781266..a304891b120e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.py@@ -25,14 +25,15 @@ class TestOpenalexSource:self.config = OpenalexSourceConfig(api_key="key")@pytest.mark.parametrize("endpoint", list(ENDPOINTS))- def test_only_works_advertises_incremental_sync(self, endpoint: str) -> None:- # OpenAlex gates its updated/created date filters behind a paid plan, so `works` is the- # only table with a usable server-side timestamp filter.+ def test_only_works_and_locations_advertise_incremental_sync(self, endpoint: str) -> None:+ # OpenAlex gates its updated/created date filters behind a paid plan, so only the free+ # `publication_date` and `ingested_at` filters give a usable server-side timestamp filter.schema = next(s for s in self.source.get_schemas(self.config, team_id=1) if s.name == endpoint)+ expected_fields = {"works": ["publication_date"], "locations": ["ingested_at"]}- if endpoint == "works":+ if endpoint in expected_fields:assert schema.supports_incremental is True- assert [f["field"] for f in schema.incremental_fields] == ["publication_date"]+ assert [f["field"] for f in schema.incremental_fields] == expected_fields[endpoint]else:assert schema.supports_incremental is Falseassert schema.incremental_fields == []@@ -55,6 +56,7 @@ class TestOpenalexSource:("works", "works-scope"),("authors", "authors-scope"),("awards", "awards-scope"),+ ("locations", "locations-scope"),("topics", None),("institutions", None),],@@ -66,6 +68,7 @@ class TestOpenalexSource:works_filter="works-scope",authors_filter="authors-scope",awards_filter="awards-scope",+ locations_filter="locations-scope",)result = SourceDriver(self.source, config).run(@@ -89,4 +92,5 @@ class TestOpenalexSource:"works": "works-scope","authors": None,"awards": "awards-scope",+ "locations": None,}