:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the RKI Covid source (#127357)
Tom Owers243e73360412
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 82f325437c4d..e106358f8750 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -906,7 +906,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rippling/rippling.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rippling/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/roark/roark.py1 products/warehouse_sources/backend/temporal/data_imports/sources/roark/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rocketlane/rocketlane.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 0c648475e300..cae094c4dafa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10336,10 +10336,10 @@ Today (11): `districts`, `germany`, `germany_age_groups`, `germany_history_casesDiffed against: <https://api.corona-zahlen.org/docs/>-- [ ] `GET /vaccinations` — national vaccination totals and quotas - an entire headline dataset with no table today (high)-- [ ] `GET /vaccinations/history/:days` — daily vaccination time series, the analytical form of the vaccination data (high)-- [ ] `GET /vaccinations/states/:state` — per-state vaccination breakdown, the standard comparison dimension (high)-- [ ] `GET /states/history/cases/:days` — per-state case time series; only Germany-level history is synced, so no state comparison over time is possible (high)+- [x] `GET /vaccinations` — national vaccination totals and quotas - an entire headline dataset with no table today (high) → `vaccinations` (per-state figures go to `vaccinations_states`)+- [x] `GET /vaccinations/history/:days` — daily vaccination time series, the analytical form of the vaccination data (high) → `vaccinations_history`+- [x] `GET /vaccinations/states/:state` — per-state vaccination breakdown, the standard comparison dimension (high) → `vaccinations_states` (reads `GET /vaccinations/states`, which returns every state in one response)+- [x] `GET /states/history/cases/:days` — per-state case time series; only Germany-level history is synced, so no state comparison over time is possible (high) → `states_history_cases`- [ ] `GET /states/history/deaths/:days` — per-state death time series to match the national germany_history_deaths table (high)- [ ] `GET /states/history/incidence/:days` — per-state 7-day incidence, the metric the whole dataset is organized around (high)- [ ] `GET /districts/history/cases/:days` — per-district case time series - the finest granularity the API offers and the reason to use it over aggregate sources (high)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.pyindex 10077531177b..885156e0fa8b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.py@@ -117,6 +117,17 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"hospitalization": "Current 7-day hospitalization cases and incidence for the state.",},},+ "states_history_cases": {+ "description": "Daily new reported COVID-19 cases per German federal state.",+ "docs_url": _DOCS_URL,+ "columns": {+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",+ "id": "Numeric state identifier used by the RKI.",+ "name": "State name.",+ "date": "Reporting day the row covers.",+ "cases": "New cases reported in the state on this day.",+ },+ },"districts": {"description": "Current COVID-19 snapshot per German district (Landkreis / kreisfreie Stadt).","docs_url": _DOCS_URL,@@ -148,4 +159,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"laboratoryCount": "Number of laboratories that reported figures for this week.",},},+ "vaccinations": {+ "description": "Current nationwide COVID-19 vaccination totals for Germany, replaced on every sync.",+ "docs_url": _DOCS_URL,+ "columns": {+ "administeredVaccinations": "Total vaccine doses administered.",+ "vaccinated": "People with at least one vaccine dose.",+ "vaccination": "First doses per vaccine product, with the day-over-day change per product.",+ "delta": "Day-over-day change in people with at least one dose.",+ "quote": "Share of the population with at least one dose (0-1).",+ "quotes": "Share of the population with at least one dose, in total and per age group.",+ "secondVaccination": "People with a completed primary series, with per-vaccine counts and quotas.",+ "boosterVaccination": "People with a first booster dose, with per-vaccine counts and quotas.",+ "2ndBoosterVaccination": "People with a second booster dose, with per-vaccine counts and quotas.",+ "3rdBoosterVaccination": "People with a third booster dose, with per-vaccine counts and quotas.",+ "4thBoosterVaccination": "People with a fourth booster dose, with per-vaccine counts and quotas.",+ "meta": "API metadata, including the last update timestamp of the underlying RKI data.",+ },+ },+ "vaccinations_history": {+ "description": "Daily COVID-19 vaccinations administered in Germany.",+ "docs_url": _DOCS_URL,+ "columns": {+ "date": "Day the vaccinations were administered.",+ "vaccinated": "People who received a first dose on this day (same as firstVaccination).",+ "firstVaccination": "First doses administered on this day.",+ "secondVaccination": "Doses that completed a primary series on this day.",+ "firstBoosterVaccination": "First booster doses administered on this day.",+ "secondBoosterVaccination": "Second booster doses administered on this day.",+ "thirdBoosterVaccination": "Third booster doses administered on this day.",+ "fourthBoosterVaccination": "Fourth booster doses administered on this day.",+ "totalVacciantionOfTheDay": "All vaccine doses administered on this day (the column name keeps the API's spelling).",+ },+ },+ "vaccinations_states": {+ "description": "Current COVID-19 vaccination totals and quotas per German federal state.",+ "docs_url": _DOCS_URL,+ "columns": {+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",+ "name": "State name.",+ "administeredVaccinations": "Total vaccine doses administered.",+ "vaccinated": "People with at least one vaccine dose.",+ "vaccination": "First doses per vaccine product, with the day-over-day change per product.",+ "delta": "Day-over-day change in people with at least one dose.",+ "quote": "Share of the population with at least one dose (0-1).",+ "quotes": "Share of the population with at least one dose, in total and per age group.",+ "secondVaccination": "People with a completed primary series, with per-vaccine counts and quotas.",+ "boosterVaccination": "People with a first booster dose, with per-vaccine counts and quotas.",+ "2ndBoosterVaccination": "People with a second booster dose, with per-vaccine counts and quotas.",+ "3rdBoosterVaccination": "People with a third booster dose, with per-vaccine counts and quotas.",+ "4thBoosterVaccination": "People with a fourth booster dose, with per-vaccine counts and quotas.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.pyindex 4981937de7d8..2016a8162fa1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.py@@ -49,6 +49,14 @@ def _parse_snapshot(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iteyield body+def _parse_data_snapshot(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:+ data = body.get("data")+ if not isinstance(data, dict):+ return+ # The per-state figures are synced separately by vaccinations_states.+ yield {**{k: v for k, v in data.items() if k != "states"}, "meta": body.get("meta")}++def _parse_dict_rows(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:data = body.get("data")if not isinstance(data, dict):@@ -69,6 +77,23 @@ def _parse_keyed_rows(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iyield {config.key_field: key, **value}+def _parse_keyed_history(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:+ data = body.get("data")+ if not isinstance(data, dict):+ return+ assert config.key_field is not None+ for key, value in data.items():+ if not isinstance(value, dict):+ continue+ history = value.get("history")+ if not isinstance(history, list):+ continue+ parent = {k: v for k, v in value.items() if k != "history"}+ for row in history:+ if isinstance(row, dict):+ yield {config.key_field: key, **parent, **row}++def _parse_data_list(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:data = body.get("data")if not isinstance(data, list):@@ -92,8 +117,10 @@ def _parse_data_history(body: dict[str, Any], config: RKICovidEndpointConfig) ->_PARSERS = {"snapshot": _parse_snapshot,+ "data_snapshot": _parse_data_snapshot,"dict_rows": _parse_dict_rows,"keyed_rows": _parse_keyed_rows,+ "keyed_history": _parse_keyed_history,"data_list": _parse_data_list,"data_history": _parse_data_history,}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.pyindex 26871c2d7fb3..193ac242f6ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py@@ -1,6 +1,7 @@-import dataclassesfrom typing import Literal, Optional+from posthog.dataclasses import frozen+# The RKI COVID-19 API (api.corona-zahlen.org) is a public, unauthenticated JSON wrapper over# published Robert Koch-Institut figures, maintained by Marlon Lückert. Every endpoint returns its# full dataset in a single response — no pagination and no server-side timestamp cursor (history@@ -8,23 +9,29 @@ from typing import Literal, Optional## Each endpoint returns a bespoke JSON shape, so endpoints are grouped by a `kind` that tells the# transport how to reshape the response into flat rows:-# snapshot -> the response body is a single row (e.g. /germany)-# dict_rows -> body["data"] is a dict whose values are self-describing rows (e.g. /states)-# keyed_rows -> body["data"] is a dict whose keys carry the row identity and must be injected-# into each row under `key_field` (e.g. /germany/age-groups)-# data_list -> body["data"] is already a list of rows (e.g. /germany/history/cases)-# data_history -> the rows live at body["data"]["history"] (e.g. /germany/history/frozen-incidence)-ParseKind = Literal["snapshot", "dict_rows", "keyed_rows", "data_list", "data_history"]+# snapshot -> the response body is a single row (e.g. /germany)+# data_snapshot -> body["data"] is a single row (e.g. /vaccinations)+# dict_rows -> body["data"] is a dict whose values are self-describing rows (e.g. /states)+# keyed_rows -> body["data"] is a dict whose keys carry the row identity and must be injected+# into each row under `key_field` (e.g. /germany/age-groups)+# keyed_history -> body["data"] is a dict like keyed_rows, but each value nests its rows under+# "history"; each history row gets the key and the parent fields+# (e.g. /states/history/cases)+# data_list -> body["data"] is already a list of rows (e.g. /germany/history/cases)+# data_history -> the rows live at body["data"]["history"] (e.g. /germany/history/frozen-incidence)+ParseKind = Literal[+ "snapshot", "data_snapshot", "dict_rows", "keyed_rows", "keyed_history", "data_list", "data_history"+]+@frozenclass RKICovidEndpointConfig:name: strpath: strkind: ParseKind# Unique across the whole table. None for single-row snapshot tables that are replaced wholesale.primary_keys: Optional[list[str]]- # Set for keyed_rows endpoints: the column the dict key is injected under.+ # Set for keyed_rows and keyed_history endpoints: the column the dict key is injected under.key_field: Optional[str] = None# A stable date column used for datetime partitioning. History rows are keyed by reporting day# and never move to another day, so `date` is safe. None for snapshot tables.@@ -112,6 +119,16 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {primary_keys=["abbreviation"],description="Current COVID-19 snapshot per German federal state (Bundesland): cases, deaths, recoveries, week incidence, and hospitalization. One row per state. Full refresh.",),+ "states_history_cases": RKICovidEndpointConfig(+ name="states_history_cases",+ path="/states/history/cases",+ kind="keyed_history",+ primary_keys=["abbreviation", "date"],+ key_field="abbreviation",+ partition_key="date",+ supports_days=True,+ description="Daily new reported COVID-19 cases per German federal state. One row per state and day. Full refresh.",+ ),"districts": RKICovidEndpointConfig(name="districts",path="/districts",@@ -126,6 +143,30 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {primary_keys=["calendarWeek"],description="Weekly PCR testing figures for Germany: performed tests, positive tests, positivity rate, and reporting laboratories. One row per calendar week. Full refresh.",),+ "vaccinations": RKICovidEndpointConfig(+ name="vaccinations",+ path="/vaccinations",+ kind="data_snapshot",+ primary_keys=None,+ description="Current nationwide COVID-19 vaccination totals for Germany: administered doses, vaccinated people per dose, doses per vaccine, and vaccination quotas. One row, replaced on every sync.",+ ),+ "vaccinations_history": RKICovidEndpointConfig(+ name="vaccinations_history",+ path="/vaccinations/history",+ kind="data_history",+ primary_keys=["date"],+ partition_key="date",+ supports_days=True,+ description="Daily COVID-19 vaccinations administered in Germany, split by first, second, and booster doses. One row per day. Full refresh.",+ ),+ "vaccinations_states": RKICovidEndpointConfig(+ name="vaccinations_states",+ path="/vaccinations/states",+ kind="keyed_rows",+ primary_keys=["abbreviation"],+ key_field="abbreviation",+ description="Current COVID-19 vaccination totals and quotas per German federal state. One row per state. Full refresh.",+ ),}ENDPOINTS = tuple(RKI_COVID_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.pyindex d59bc6b945ba..283a657d3aae 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.py@@ -88,6 +88,37 @@ class TestRKICovid:{"age_group": "A80+", "casesMale": 3, "casesFemale": 4},]+ def test_vaccinations_snapshot_drops_per_state_figures_and_keeps_meta(self) -> None:+ # The per-state figures belong to vaccinations_states; keeping them would nest 17 states into one cell.+ body = {+ "data": {"vaccinated": 10, "quote": 0.5, "states": {"BY": {"vaccinated": 3}}},+ "meta": {"lastUpdate": "2024-07-09T07:00:22.000Z"},+ }+ assert _run("vaccinations", [_ok(body)]).rows == [+ {"vaccinated": 10, "quote": 0.5, "meta": {"lastUpdate": "2024-07-09T07:00:22.000Z"}}+ ]++ def test_states_history_flattens_each_state_history_with_state_identity(self) -> None:+ # History rows only carry date and cases; without the state key they collide on the primary key.+ body = {+ "data": {+ "SH": {"id": 1, "name": "Schleswig-Holstein", "history": [{"cases": 4, "date": "2026-07-19"}]},+ "BY": {+ "id": 9,+ "name": "Bayern",+ "history": [{"cases": 1, "date": "2026-07-19"}, {"cases": 2, "date": "2026-07-20"}],+ },+ },+ "meta": {},+ }+ result = _run("states_history_cases", [_ok(body)], history_days=30)+ assert result.paths == ["/states/history/cases/30"]+ assert result.rows == [+ {"abbreviation": "SH", "id": 1, "name": "Schleswig-Holstein", "cases": 4, "date": "2026-07-19"},+ {"abbreviation": "BY", "id": 9, "name": "Bayern", "cases": 1, "date": "2026-07-19"},+ {"abbreviation": "BY", "id": 9, "name": "Bayern", "cases": 2, "date": "2026-07-20"},+ ]+def test_states_yield_one_row_per_dict_value(self) -> None:body = {"data": {@@ -148,6 +179,8 @@ class TestRKICovid:("missing_data", "states", {"meta": {}}),("wrong_data_type", "germany_history_cases", {"data": {"unexpected": "shape"}, "meta": {}}),("missing_history", "testing_history", {"data": {}, "meta": {}}),+ ("missing_state_history", "states_history_cases", {"data": {"BY": {"id": 9}}, "meta": {}}),+ ("missing_vaccinations_data", "vaccinations", {"meta": {}}),])def test_malformed_data_yields_no_rows_instead_of_crashing(self, _name: str, endpoint: str, body: dict) -> None:@@ -160,6 +193,7 @@ class TestRKICovid:("germany_history_cases", ["date"], "date"),("germany_history_hospitalization", ["date"], "date"),("states", ["abbreviation"], None),+ ("states_history_cases", ["abbreviation", "date"], "date"),("districts", ["ags"], None),("testing_history", ["calendarWeek"], None),]