Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add openalex study designs, indexes, source lists and locations tables (#123367)

Tom Owers53665740f3aa
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 46d1560d5abd..a059307bc00b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8751,11 +8751,11 @@ Today (20): `authors`, `awards`, `continents`, `countries`, `domains`, `fields`,
Diffed against: <https://help.openalex.org/api/endpoints.md>
-- [ ] `Study Designs (GET /study-designs)` — resolve study design IDs on works and compare evidence types (high)
-- [ ] `Indexes (GET /indexes)` — resolve indexed_in values on works and compare coverage across bibliographic indexes (high)
-- [ ] `Source Lists (GET /source-lists)` — resolve listed_in IDs on sources for journal quality and coverage analysis (high)
+- [x] `Study Designs (GET /study-designs)` — resolve study design IDs on works and compare evidence types (high)
+- [x] `Indexes (GET /indexes)` — resolve indexed_in values on works and compare coverage across bibliographic indexes (high)
+- [x] `Source Lists (GET /source-lists)` — resolve listed_in IDs on sources for journal quality and coverage analysis (high)
- [ ] `Collections and Members (GET /collections; GET /collections/{id}/members)` — track research portfolios and collection membership across works, authors, and journals (medium)
-- [ ] `Locations (GET /locations)` — analyze each harvested copy and its metadata beyond the locations embedded in works (medium)
+- [x] `Locations (GET /locations)` — analyze each harvested copy and its metadata beyond the locations embedded in works (medium)
Note: Parsed the current endpoint index and entity references, including the collections member endpoint. Existing works already contain locations and affiliation strings; dedicated locations expose additional metadata. Collection members need requests for each collection. Excluded deprecated concepts, autocomplete, text tagging, saved searches, and correction workflows.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.py
index 7f68394891db..9fe589819531 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openalex.py
@@ -10,3 +10,4 @@ class OpenalexSourceConfig(config.Config):
works_filter: str | None = None
authors_filter: str | None = None
awards_filter: str | None = None
+ locations_filter: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.py
index c48a7c54000a..b628d910f8b9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/canonical_descriptions.py
@@ -448,4 +448,82 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
url="Link to the full text of the license.",
),
},
+ "study_designs": {
+ "description": "The study designs a work's research can have, such as randomized controlled trial or systematic review.",
+ "docs_url": f"{_DOCS_BASE}/study-designs/list-study-designs",
+ "columns": _columns(
+ "id",
+ "display_name",
+ "created_date",
+ "updated_date",
+ "works_count",
+ "cited_by_count",
+ "works_api_url",
+ "description",
+ pubmed_publication_types="PubMed publication types that map to this study design.",
+ ),
+ },
+ "indexes": {
+ "description": "External bibliographic indexes and registries that list works, matching the indexed_in values on works.",
+ "docs_url": f"{_DOCS_BASE}/indexes/list-indexes",
+ "columns": _columns(
+ "id",
+ "display_name",
+ "created_date",
+ "updated_date",
+ "works_count",
+ "cited_by_count",
+ "works_api_url",
+ "description",
+ ),
+ },
+ "source_lists": {
+ "description": "Externally maintained lists of journals that a source can appear on, matching the listed_in values on sources.",
+ "docs_url": f"{_DOCS_BASE}/source-lists/list-source-lists",
+ "columns": _columns(
+ "id",
+ "display_name",
+ "created_date",
+ "updated_date",
+ "works_count",
+ "cited_by_count",
+ "description",
+ works_api_url="URL of the OpenAlex works list filtered to works whose primary location is on this list.",
+ maintainer="Organization that maintains the list.",
+ url="Where the maintainer publishes the list.",
+ list_version="Edition of the list currently loaded, as YYYY-MM-DD. Null for lists derived continuously from another feed.",
+ sources_count="Number of OpenAlex sources on this list.",
+ sources_api_url="URL of the OpenAlex sources list filtered to sources on this list.",
+ ),
+ },
+ "locations": {
+ "description": "Every harvested copy of a work, one row per copy, such as the publisher version, a repository manuscript or a preprint.",
+ "docs_url": f"{_DOCS_BASE}/locations/list-locations",
+ "columns": {
+ "id": "Namespaced identifier the copy was harvested under, for example doi:10.7717/peerj.4375. Primary key.",
+ "work_id": "OpenAlex ID of the work this copy was matched into.",
+ "native_id": "Identifier the copy was harvested under, without its namespace prefix.",
+ "native_id_namespace": "Identifier system of native_id: doi, pmh, pmid, mag or openalex_curation.",
+ "provenance": "Pipeline that delivered the record, such as crossref, datacite, pubmed, mag or repo.",
+ "doi": "DOI of this copy, as a lowercase URL.",
+ "endpoint_id": "For repository copies, ID of the OAI-PMH endpoint the record was harvested from.",
+ "title": "Title as it appears on this copy's record.",
+ "type": "Normalized work type declared on this copy's record.",
+ "raw_type": "Work type as this copy's source labeled it, before normalization.",
+ "source_id": "OpenAlex ID of the source hosting this copy.",
+ "source_name": "Name of the hosting source as it arrived on the record.",
+ "publisher": "Publisher named on the record.",
+ "is_oa": "True if this copy is free to read without paying or logging in.",
+ "version": "Version this copy holds: publishedVersion, acceptedVersion or submittedVersion.",
+ "license": "License this copy is available under, as a short code such as cc-by.",
+ "language": "Language code of the record.",
+ "is_retracted": "True if this copy's record is flagged as retracted.",
+ "landing_page_url": "URL of the page that describes and links to this copy.",
+ "pdf_url": "Direct link to a full-text PDF of this copy, if known.",
+ "ids": "Every identifier on the record, each with its namespace.",
+ "urls": "Every URL on the record, each with a content type.",
+ "merge_key": "Keys used to match this copy into a work: doi, arxiv, pmid and title_author.",
+ "ingested_at": "When OpenAlex fetched this copy. Null on many older copies.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py
index 0260f5f647cc..f36466f15307 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py
@@ -34,6 +34,11 @@ REQUEST_TIMEOUT_SECONDS = 60
LARGE_ROW_CHUNK_SIZE = 1000
LARGE_ROW_CHUNK_SIZE_BYTES = 100 * 1024 * 1024
+# OpenAlex does not document how long a copy takes to reach the API after its `ingested_at`, so
+# an incremental `locations` run re-reads the watermark day and the day before it. The merge
+# dedupes the overlap. Two days, because a date-only `ingested_at:>` bound excludes the day it names.
+INGESTED_AT_LOOKBACK = datetime.timedelta(days=2)
+
@dataclasses.dataclass
class OpenAlexResumeConfig:
@@ -97,6 +102,21 @@ def build_filter(
return ",".join(clauses) or None
+def build_ingested_at_filter(base_filter: Optional[str], from_ingested_at: Any = None) -> Optional[str]:
+ """Combine the user's scoping filter with the incremental `ingested_at` bound for locations.
+
+ The first incremental run has no watermark, so it sends no bound and also reads the older
+ copies that have no `ingested_at`.
+ """
+ clauses = []
+ if base_filter and base_filter.strip():
+ clauses.append(base_filter.strip())
+ from_date = _as_date(from_ingested_at)
+ if from_date is not None:
+ clauses.append(f"ingested_at:>{(from_date - INGESTED_AT_LOOKBACK).isoformat()}")
+ return ",".join(clauses) or None
+
+
def get_resource(
endpoint_config: OpenAlexEndpointConfig,
entity_filter: Optional[str],
@@ -110,7 +130,14 @@ def get_resource(
"sort": endpoint_config.sort,
}
- if should_use_incremental_field:
+ if should_use_incremental_field and endpoint_config.name == "locations":
+ params["filter"] = {
+ "type": "incremental",
+ "cursor_path": "ingested_at",
+ "initial_value": None,
+ "convert": lambda last_value: build_ingested_at_filter(entity_filter, last_value),
+ }
+ elif should_use_incremental_field:
# Pinned once per run rather than read per page: the filter has to stay identical for
# the life of a cursor walk, and a long sync can cross midnight.
cap = utc_today()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.py
index 2e00207d989f..3694fd3b478e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/settings.py
@@ -55,6 +55,7 @@ OPENALEX_ENDPOINTS: dict[str, OpenAlexEndpointConfig] = {
"fields": OpenAlexEndpointConfig(name="fields", path="/fields"),
"subfields": OpenAlexEndpointConfig(name="subfields", path="/subfields"),
"sdgs": OpenAlexEndpointConfig(name="sdgs", path="/sdgs"),
+ "study_designs": OpenAlexEndpointConfig(name="study_designs", path="/study-designs"),
"countries": OpenAlexEndpointConfig(name="countries", path="/countries"),
"continents": OpenAlexEndpointConfig(name="continents", path="/continents"),
"languages": OpenAlexEndpointConfig(name="languages", path="/languages"),
@@ -62,6 +63,18 @@ OPENALEX_ENDPOINTS: dict[str, OpenAlexEndpointConfig] = {
"source_types": OpenAlexEndpointConfig(name="source_types", path="/source-types"),
"institution_types": OpenAlexEndpointConfig(name="institution_types", path="/institution-types"),
"licenses": OpenAlexEndpointConfig(name="licenses", path="/licenses"),
+ "indexes": OpenAlexEndpointConfig(name="indexes", path="/indexes"),
+ "source_lists": OpenAlexEndpointConfig(name="source_lists", path="/source-lists"),
+ # One row per harvested copy of a work. The `id` is a namespaced harvest identifier such as
+ # `doi:10.7717/peerj.4375` rather than an OpenAlex URL, but it is still unique per copy.
+ # Rows carry no `created_date`, and `ingested_at` is null on many older copies, so there is
+ # no stable date to partition on.
+ "locations": OpenAlexEndpointConfig(
+ name="locations",
+ path="/locations",
+ partition_key=None,
+ filter_config_field="locations_filter",
+ ),
}
ENDPOINTS = tuple(OPENALEX_ENDPOINTS.keys())
@@ -74,6 +87,11 @@ ENDPOINTS = tuple(OPENALEX_ENDPOINTS.keys())
# late edits to older records; full refresh those when you need the corrections. Incremental
# runs also cap the window at today, because OpenAlex carries future publication dates and one
# of those rows would otherwise checkpoint a watermark nothing can lower again.
+#
+# `locations` filters on `ingested_at`, the time OpenAlex fetched the copy, which is also free.
+# New copies always carry it, so an incremental run picks up newly harvested copies. Older copies
+# without it only arrive through the first run or a full refresh.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
"works": [incremental_field("publication_date", IncrementalFieldType.Date)],
+ "locations": [incremental_field("ingested_at", IncrementalFieldType.DateTime)],
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.py
index 69fc9d8bf291..b7e0ea70a2ef 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/source.py
@@ -129,7 +129,7 @@ class OpenalexSource(ResumableSource[OpenalexSourceConfig, OpenAlexResumeConfig]
Keys are free and take about a minute to create at [openalex.org/settings/api](https://openalex.org/settings/api).
-OpenAlex indexes over 300 million works and 120 million authors, which is more than a free key's daily allowance covers. Use the filter fields to narrow the works, authors and awards tables to the records you care about, for example `authorships.institutions.lineage:i27837315` for one institution's papers. The [filtering guide](https://developers.openalex.org/guides/filtering) lists the available filter fields. Leave a filter blank to sync the whole entity.""",
+OpenAlex indexes over 300 million works and 120 million authors, which is more than a free key's daily allowance covers. Use the filter fields to narrow the works, authors, awards and locations tables to the records you care about, for example `authorships.institutions.lineage:i27837315` for one institution's papers. The [filtering guide](https://developers.openalex.org/guides/filtering) lists the available filter fields. Leave a filter blank to sync the whole entity.""",
iconPath="/static/services/openalex.png",
docsUrl="https://posthog.com/docs/cdp/sources/openalex",
releaseStatus=ReleaseStatus.BETA,
@@ -169,6 +169,14 @@ OpenAlex indexes over 300 million works and 120 million authors, which is more t
placeholder="funder.id:f4320306076",
secret=False,
),
+ SourceFieldInputConfig(
+ name="locations_filter",
+ label="Locations filter (optional)",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="source_id:S4306400194",
+ secret=False,
+ ),
],
),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.py
index c05b9fab8718..10967c02c9fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex.py
@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openalex.s
from products.warehouse_sources.backend.temporal.data_imports.sources.openalex.source import OpenalexSource
INSTITUTION_FILTER = "authorships.institutions.lineage:i27837315"
+SOURCE_FILTER = "source_id:S4306400194"
TODAY = utc_today()
@@ -49,6 +50,7 @@ def _drive(
works_filter=entity_filter if endpoint == "works" else None,
authors_filter=entity_filter if endpoint == "authors" else None,
awards_filter=entity_filter if endpoint == "awards" else None,
+ locations_filter=entity_filter if endpoint == "locations" else None,
)
return SourceDriver(OpenalexSource(), config).run(
endpoint,
@@ -226,6 +228,33 @@ class TestRequestParams:
assert result.raised is None
assert result.requests[0].param("filter") == INSTITUTION_FILTER
+ @pytest.mark.parametrize(
+ "should_use_incremental_field,watermark,expected_filter",
+ [
+ # A date-only `ingested_at:>` bound excludes the day it names, so this re-reads the
+ # watermark day and the day before it for copies that reached the API late.
+ (True, datetime.datetime(2026, 7, 10, 13, 45), f"{SOURCE_FILTER},ingested_at:>2026-07-08"),
+ (True, "2026-07-10 05:06:08.047", f"{SOURCE_FILTER},ingested_at:>2026-07-08"),
+ # The first run has no bound, so it also reads older copies with no `ingested_at`.
+ (True, None, SOURCE_FILTER),
+ (False, datetime.datetime(2026, 7, 10, 13, 45), SOURCE_FILTER),
+ ],
+ )
+ def test_locations_filters_on_ingested_at(
+ self, should_use_incremental_field: bool, watermark: Any, expected_filter: str
+ ) -> None:
+ result = _drive(
+ "locations",
+ [_page("c1"), _page(None)],
+ entity_filter=SOURCE_FILTER,
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=watermark,
+ )
+
+ assert result.raised is None
+ # `locations` rejects the works-only publication date filters with a 400.
+ assert result.params("filter") == [expected_filter, expected_filter]
+
@pytest.mark.parametrize("endpoint", [name for name in ENDPOINTS if name != "works"])
def test_only_works_sends_a_sort(self, endpoint: str) -> None:
# `sort=publication_date` is a works-only field; sending it elsewhere is a 400.
@@ -234,7 +263,9 @@ class TestRequestParams:
assert result.raised is None
assert result.requests[0].param("sort") is None
- @pytest.mark.parametrize("endpoint", [name for name in ENDPOINTS if name not in ("works", "authors", "awards")])
+ @pytest.mark.parametrize(
+ "endpoint", [name for name in ENDPOINTS if name not in ("works", "authors", "awards", "locations")]
+ )
def test_unfilterable_endpoints_send_no_filter(self, endpoint: str) -> None:
result = _drive(endpoint, [_page(None)])
@@ -254,8 +285,12 @@ class TestSourceResponseMetadata:
assert response.name == endpoint
# Every OpenAlex entity is keyed by its canonical OpenAlex URL, unique per entity type.
assert response.primary_keys == ["id"]
- assert response.partition_mode == "datetime"
- assert response.partition_keys == [endpoint_config.partition_key]
+ if endpoint_config.partition_key is None:
+ assert response.partition_mode is None
+ assert response.partition_keys is None
+ else:
+ assert response.partition_mode == "datetime"
+ assert response.partition_keys == [endpoint_config.partition_key]
# Declaring "asc" on an endpoint we cannot order would checkpoint a bogus watermark.
assert response.sort_mode == ("asc" if endpoint == "works" else None)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.py
index b26006781266..a304891b120e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openalex/tests/test_openalex_source.py
@@ -25,14 +25,15 @@ class TestOpenalexSource:
self.config = OpenalexSourceConfig(api_key="key")
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
- def test_only_works_advertises_incremental_sync(self, endpoint: str) -> None:
- # OpenAlex gates its updated/created date filters behind a paid plan, so `works` is the
- # only table with a usable server-side timestamp filter.
+ def test_only_works_and_locations_advertise_incremental_sync(self, endpoint: str) -> None:
+ # OpenAlex gates its updated/created date filters behind a paid plan, so only the free
+ # `publication_date` and `ingested_at` filters give a usable server-side timestamp filter.
schema = next(s for s in self.source.get_schemas(self.config, team_id=1) if s.name == endpoint)
+ expected_fields = {"works": ["publication_date"], "locations": ["ingested_at"]}
- if endpoint == "works":
+ if endpoint in expected_fields:
assert schema.supports_incremental is True
- assert [f["field"] for f in schema.incremental_fields] == ["publication_date"]
+ assert [f["field"] for f in schema.incremental_fields] == expected_fields[endpoint]
else:
assert schema.supports_incremental is False
assert schema.incremental_fields == []
@@ -55,6 +56,7 @@ class TestOpenalexSource:
("works", "works-scope"),
("authors", "authors-scope"),
("awards", "awards-scope"),
+ ("locations", "locations-scope"),
("topics", None),
("institutions", None),
],
@@ -66,6 +68,7 @@ class TestOpenalexSource:
works_filter="works-scope",
authors_filter="authors-scope",
awards_filter="awards-scope",
+ locations_filter="locations-scope",
)
result = SourceDriver(self.source, config).run(
@@ -89,4 +92,5 @@ class TestOpenalexSource:
"works": "works-scope",
"authors": None,
"awards": "awards-scope",
+ "locations": None,
}