:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add crossref journals table (#119236)
Tom Owers2e3ee267d211
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex e954181e87d0..e01773f8e3df 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -3382,7 +3382,7 @@ Today (5): `Funders`, `Licenses`, `Members`, `Types`, `Works`Diffed against: <https://api.crossref.org/swagger-docs>-- [ ] `Journals (GET /journals)` — resolve work ISSNs to journal metadata, subject categories, and coverage statistics (high)+- [x] `Journals (GET /journals)` — resolve work ISSNs to journal metadata, subject categories, and coverage statistics (high)Note: Parsed the current vendor Swagger document and compared every GET operation with the five static tables. Journal listings are absent, while scoped work collections return objects already covered by Works. Prefix metadata has only a single-record endpoint, so it was excluded from collection gaps.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/canonical_descriptions.pyindex 31a8f0a283de..f7afed62b523 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/canonical_descriptions.py@@ -74,4 +74,22 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"work-count": "Number of works referencing this license.",},},+ "Journals": {+ "description": "Journals in Crossref with their ISSNs, publisher, subjects, DOI counts, and metadata coverage statistics.",+ "docs_url": "https://github.com/CrossRef/rest-api-doc#journals",+ "columns": {+ "title": "The journal's title.",+ "publisher": "Name of the journal's publisher.",+ "ISSN": "The journal's ISSNs.",+ "issn-type": "The journal's ISSNs, each labelled as print or electronic.",+ "issn_key": "The journal's ISSNs, sorted and joined with commas. Empty when the journal has no ISSN.",+ "subjects": "Subject categories assigned to the journal.",+ "counts": "Number of current, backfile, and total DOIs registered for the journal.",+ "breakdowns": "DOI counts broken down by issued year.",+ "coverage": "Share of the journal's current and backfile DOIs that carry each metadata element (references, ORCIDs, licenses, etc).",+ "coverage-type": "Metadata coverage ratios grouped by all, current, and backfile DOIs.",+ "flags": "Whether the journal deposits each metadata element for its current and backfile DOIs.",+ "last-status-check-time": "When Crossref last recalculated the journal's statistics, as epoch milliseconds.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/crossref.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/crossref.pyindex 48da45c4715b..a8799978599e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/crossref.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/crossref.py@@ -70,6 +70,14 @@ def _normalize_work(item: dict[str, Any]) -> dict[str, Any]:return item+def _normalize_journal(item: dict[str, Any]) -> dict[str, Any]:+ item["issn_key"] = ",".join(sorted(item.get("ISSN") or []))+ return item+++_NORMALIZERS = {"Works": _normalize_work, "Journals": _normalize_journal}++def validate_credentials(mailto: Optional[str]) -> bool:params: dict[str, Any] = {"rows": 0}if mailto:@@ -178,8 +186,9 @@ def get_rows(next_cursor = message.get("next-cursor")+ normalize = _NORMALIZERS.get(endpoint)for item in items:- batcher.batch(_normalize_work(item) if endpoint == "Works" else item)+ batcher.batch(normalize(item) if normalize else item)if batcher.should_yield():yield batcher.get_table()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/settings.pyindex a9fe52379d94..26ccc65ce6ba 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/settings.py@@ -91,6 +91,14 @@ ENDPOINTS: dict[str, CrossrefEndpointConfig] = {primary_keys=["URL"],description="Distinct license URLs referenced by works, with a count of works that use each.",),+ "Journals": CrossrefEndpointConfig(+ name="Journals",+ path="/journals",+ # Journals carry no id of their own, and a handful have no ISSN at all, so the derived+ # ISSN key needs the title alongside it to stay unique.+ primary_keys=["issn_key", "title"],+ description="Journals in Crossref with their ISSNs, publisher, subjects, DOI counts, and metadata coverage statistics.",+ ),}INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/source.pyindex 957a3117f0be..bd8bc64180b5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/source.py@@ -60,7 +60,7 @@ class CrossrefSource(ResumableSource[CrossrefSourceConfig, CrossrefResumeConfig]releaseStatus=ReleaseStatus.ALPHA,caption="""Crossref's API is free and public — no API key needed. Add a contact email to get routed to Crossref's faster "polite pool".-The Works table covers Crossref's full DOI registry (160 million+ records), so set a member ID, funder ID, or journal ISSN below to scope which works sync. The Members, Funders, Types, and Licenses tables always sync in full.+The Works table covers Crossref's full DOI registry (160 million+ records), so set a member ID, funder ID, or journal ISSN below to scope which works sync. The Members, Funders, Types, Licenses, and Journals tables always sync in full.""",iconPath="/static/services/crossref.png",docsUrl="https://posthog.com/docs/cdp/sources/crossref",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref.pyindex 9a6c4d92684e..a8cb3cc26e87 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref.py@@ -129,11 +129,30 @@ class TestGetRowsCursorPagination:assert result.committed_states == [CrossrefResumeConfig(cursor="c2")]assert result.final_state == CrossrefResumeConfig(cursor="c2")- def test_works_rows_are_normalized(self) -> None:- result = self._driver().run("Works", [_page([{"DOI": "1", "indexed": {"date-time": "2026-01-01T00:00:00Z"}}])])+ @parameterized.expand(+ [+ (+ "works_date",+ "Works",+ {"DOI": "1", "indexed": {"date-time": "2026-01-01T00:00:00Z"}},+ "indexed_date",+ "2026-01-01T00:00:00Z",+ ),+ (+ "journal_issns_sorted",+ "Journals",+ {"title": "J", "ISSN": ["1097-4547", "0360-4012"]},+ "issn_key",+ "0360-4012,1097-4547",+ ),+ ("journal_without_issn", "Journals", {"title": "J", "ISSN": []}, "issn_key", ""),+ ]+ )+ def test_rows_are_normalized(self, _name, endpoint, item, column, expected) -> None:+ result = self._driver().run(endpoint, [_page([item])])assert result.raised is None- assert result.rows[0].to_pylist()[0]["indexed_date"] == "2026-01-01T00:00:00Z"+ assert result.rows[0].to_pylist()[0][column] == expecteddef test_mailto_passed_as_redact_value(self) -> None:"""mailto is a user-typed contact email, not a secret, but it should still be kept outdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref_source.pyindex a3e02d384f3c..05010a8097bc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crossref/tests/test_crossref_source.py@@ -42,7 +42,7 @@ class TestCrossrefSource:if not expected_valid:assert "member ID, funder ID, or journal ISSN" in (message or "")- @pytest.mark.parametrize("schema_name", [None, "Members", "Funders", "Types", "Licenses"])+ @pytest.mark.parametrize("schema_name", [None, "Members", "Funders", "Types", "Licenses", "Journals"])@mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.crossref.source.validate_crossref_credentials")