Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the RKI Covid source (#127357)

Tom Owers243e73360412
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 82f325437c4d..e106358f8750 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -906,7 +906,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/rippling/rippling.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/rippling/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/roark/roark.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/roark/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/rocketlane/rocketlane.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 0c648475e300..cae094c4dafa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10336,10 +10336,10 @@ Today (11): `districts`, `germany`, `germany_age_groups`, `germany_history_cases
Diffed against: <https://api.corona-zahlen.org/docs/>
-- [ ] `GET /vaccinations` — national vaccination totals and quotas - an entire headline dataset with no table today (high)
-- [ ] `GET /vaccinations/history/:days` — daily vaccination time series, the analytical form of the vaccination data (high)
-- [ ] `GET /vaccinations/states/:state` — per-state vaccination breakdown, the standard comparison dimension (high)
-- [ ] `GET /states/history/cases/:days` — per-state case time series; only Germany-level history is synced, so no state comparison over time is possible (high)
+- [x] `GET /vaccinations` — national vaccination totals and quotas - an entire headline dataset with no table today (high) → `vaccinations` (per-state figures go to `vaccinations_states`)
+- [x] `GET /vaccinations/history/:days` — daily vaccination time series, the analytical form of the vaccination data (high) → `vaccinations_history`
+- [x] `GET /vaccinations/states/:state` — per-state vaccination breakdown, the standard comparison dimension (high) → `vaccinations_states` (reads `GET /vaccinations/states`, which returns every state in one response)
+- [x] `GET /states/history/cases/:days` — per-state case time series; only Germany-level history is synced, so no state comparison over time is possible (high) → `states_history_cases`
- [ ] `GET /states/history/deaths/:days` — per-state death time series to match the national germany_history_deaths table (high)
- [ ] `GET /states/history/incidence/:days` — per-state 7-day incidence, the metric the whole dataset is organized around (high)
- [ ] `GET /districts/history/cases/:days` — per-district case time series - the finest granularity the API offers and the reason to use it over aggregate sources (high)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.py
index 10077531177b..885156e0fa8b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/canonical_descriptions.py
@@ -117,6 +117,17 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"hospitalization": "Current 7-day hospitalization cases and incidence for the state.",
},
},
+ "states_history_cases": {
+ "description": "Daily new reported COVID-19 cases per German federal state.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",
+ "id": "Numeric state identifier used by the RKI.",
+ "name": "State name.",
+ "date": "Reporting day the row covers.",
+ "cases": "New cases reported in the state on this day.",
+ },
+ },
"districts": {
"description": "Current COVID-19 snapshot per German district (Landkreis / kreisfreie Stadt).",
"docs_url": _DOCS_URL,
@@ -148,4 +159,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"laboratoryCount": "Number of laboratories that reported figures for this week.",
},
},
+ "vaccinations": {
+ "description": "Current nationwide COVID-19 vaccination totals for Germany, replaced on every sync.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "administeredVaccinations": "Total vaccine doses administered.",
+ "vaccinated": "People with at least one vaccine dose.",
+ "vaccination": "First doses per vaccine product, with the day-over-day change per product.",
+ "delta": "Day-over-day change in people with at least one dose.",
+ "quote": "Share of the population with at least one dose (0-1).",
+ "quotes": "Share of the population with at least one dose, in total and per age group.",
+ "secondVaccination": "People with a completed primary series, with per-vaccine counts and quotas.",
+ "boosterVaccination": "People with a first booster dose, with per-vaccine counts and quotas.",
+ "2ndBoosterVaccination": "People with a second booster dose, with per-vaccine counts and quotas.",
+ "3rdBoosterVaccination": "People with a third booster dose, with per-vaccine counts and quotas.",
+ "4thBoosterVaccination": "People with a fourth booster dose, with per-vaccine counts and quotas.",
+ "meta": "API metadata, including the last update timestamp of the underlying RKI data.",
+ },
+ },
+ "vaccinations_history": {
+ "description": "Daily COVID-19 vaccinations administered in Germany.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "date": "Day the vaccinations were administered.",
+ "vaccinated": "People who received a first dose on this day (same as firstVaccination).",
+ "firstVaccination": "First doses administered on this day.",
+ "secondVaccination": "Doses that completed a primary series on this day.",
+ "firstBoosterVaccination": "First booster doses administered on this day.",
+ "secondBoosterVaccination": "Second booster doses administered on this day.",
+ "thirdBoosterVaccination": "Third booster doses administered on this day.",
+ "fourthBoosterVaccination": "Fourth booster doses administered on this day.",
+ "totalVacciantionOfTheDay": "All vaccine doses administered on this day (the column name keeps the API's spelling).",
+ },
+ },
+ "vaccinations_states": {
+ "description": "Current COVID-19 vaccination totals and quotas per German federal state.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",
+ "name": "State name.",
+ "administeredVaccinations": "Total vaccine doses administered.",
+ "vaccinated": "People with at least one vaccine dose.",
+ "vaccination": "First doses per vaccine product, with the day-over-day change per product.",
+ "delta": "Day-over-day change in people with at least one dose.",
+ "quote": "Share of the population with at least one dose (0-1).",
+ "quotes": "Share of the population with at least one dose, in total and per age group.",
+ "secondVaccination": "People with a completed primary series, with per-vaccine counts and quotas.",
+ "boosterVaccination": "People with a first booster dose, with per-vaccine counts and quotas.",
+ "2ndBoosterVaccination": "People with a second booster dose, with per-vaccine counts and quotas.",
+ "3rdBoosterVaccination": "People with a third booster dose, with per-vaccine counts and quotas.",
+ "4thBoosterVaccination": "People with a fourth booster dose, with per-vaccine counts and quotas.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.py
index 4981937de7d8..2016a8162fa1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/rki_covid.py
@@ -49,6 +49,14 @@ def _parse_snapshot(body: dict[str, Any], config: RKICovidEndpointConfig) -> Ite
yield body
+def _parse_data_snapshot(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:
+ data = body.get("data")
+ if not isinstance(data, dict):
+ return
+ # The per-state figures are synced separately by vaccinations_states.
+ yield {**{k: v for k, v in data.items() if k != "states"}, "meta": body.get("meta")}
+
+
def _parse_dict_rows(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:
data = body.get("data")
if not isinstance(data, dict):
@@ -69,6 +77,23 @@ def _parse_keyed_rows(body: dict[str, Any], config: RKICovidEndpointConfig) -> I
yield {config.key_field: key, **value}
+def _parse_keyed_history(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:
+ data = body.get("data")
+ if not isinstance(data, dict):
+ return
+ assert config.key_field is not None
+ for key, value in data.items():
+ if not isinstance(value, dict):
+ continue
+ history = value.get("history")
+ if not isinstance(history, list):
+ continue
+ parent = {k: v for k, v in value.items() if k != "history"}
+ for row in history:
+ if isinstance(row, dict):
+ yield {config.key_field: key, **parent, **row}
+
+
def _parse_data_list(body: dict[str, Any], config: RKICovidEndpointConfig) -> Iterator[dict[str, Any]]:
data = body.get("data")
if not isinstance(data, list):
@@ -92,8 +117,10 @@ def _parse_data_history(body: dict[str, Any], config: RKICovidEndpointConfig) ->
_PARSERS = {
"snapshot": _parse_snapshot,
+ "data_snapshot": _parse_data_snapshot,
"dict_rows": _parse_dict_rows,
"keyed_rows": _parse_keyed_rows,
+ "keyed_history": _parse_keyed_history,
"data_list": _parse_data_list,
"data_history": _parse_data_history,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py
index 26871c2d7fb3..193ac242f6ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/settings.py
@@ -1,6 +1,7 @@
-import dataclasses
from typing import Literal, Optional
+from posthog.dataclasses import frozen
+
# The RKI COVID-19 API (api.corona-zahlen.org) is a public, unauthenticated JSON wrapper over
# published Robert Koch-Institut figures, maintained by Marlon Lückert. Every endpoint returns its
# full dataset in a single response — no pagination and no server-side timestamp cursor (history
@@ -8,23 +9,29 @@ from typing import Literal, Optional
#
# Each endpoint returns a bespoke JSON shape, so endpoints are grouped by a `kind` that tells the
# transport how to reshape the response into flat rows:
-# snapshot -> the response body is a single row (e.g. /germany)
-# dict_rows -> body["data"] is a dict whose values are self-describing rows (e.g. /states)
-# keyed_rows -> body["data"] is a dict whose keys carry the row identity and must be injected
-# into each row under `key_field` (e.g. /germany/age-groups)
-# data_list -> body["data"] is already a list of rows (e.g. /germany/history/cases)
-# data_history -> the rows live at body["data"]["history"] (e.g. /germany/history/frozen-incidence)
-ParseKind = Literal["snapshot", "dict_rows", "keyed_rows", "data_list", "data_history"]
+# snapshot -> the response body is a single row (e.g. /germany)
+# data_snapshot -> body["data"] is a single row (e.g. /vaccinations)
+# dict_rows -> body["data"] is a dict whose values are self-describing rows (e.g. /states)
+# keyed_rows -> body["data"] is a dict whose keys carry the row identity and must be injected
+# into each row under `key_field` (e.g. /germany/age-groups)
+# keyed_history -> body["data"] is a dict like keyed_rows, but each value nests its rows under
+# "history"; each history row gets the key and the parent fields
+# (e.g. /states/history/cases)
+# data_list -> body["data"] is already a list of rows (e.g. /germany/history/cases)
+# data_history -> the rows live at body["data"]["history"] (e.g. /germany/history/frozen-incidence)
+ParseKind = Literal[
+ "snapshot", "data_snapshot", "dict_rows", "keyed_rows", "keyed_history", "data_list", "data_history"
+]
+@frozen
class RKICovidEndpointConfig:
name: str
path: str
kind: ParseKind
# Unique across the whole table. None for single-row snapshot tables that are replaced wholesale.
primary_keys: Optional[list[str]]
- # Set for keyed_rows endpoints: the column the dict key is injected under.
+ # Set for keyed_rows and keyed_history endpoints: the column the dict key is injected under.
key_field: Optional[str] = None
# A stable date column used for datetime partitioning. History rows are keyed by reporting day
# and never move to another day, so `date` is safe. None for snapshot tables.
@@ -112,6 +119,16 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {
primary_keys=["abbreviation"],
description="Current COVID-19 snapshot per German federal state (Bundesland): cases, deaths, recoveries, week incidence, and hospitalization. One row per state. Full refresh.",
),
+ "states_history_cases": RKICovidEndpointConfig(
+ name="states_history_cases",
+ path="/states/history/cases",
+ kind="keyed_history",
+ primary_keys=["abbreviation", "date"],
+ key_field="abbreviation",
+ partition_key="date",
+ supports_days=True,
+ description="Daily new reported COVID-19 cases per German federal state. One row per state and day. Full refresh.",
+ ),
"districts": RKICovidEndpointConfig(
name="districts",
path="/districts",
@@ -126,6 +143,30 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {
primary_keys=["calendarWeek"],
description="Weekly PCR testing figures for Germany: performed tests, positive tests, positivity rate, and reporting laboratories. One row per calendar week. Full refresh.",
),
+ "vaccinations": RKICovidEndpointConfig(
+ name="vaccinations",
+ path="/vaccinations",
+ kind="data_snapshot",
+ primary_keys=None,
+ description="Current nationwide COVID-19 vaccination totals for Germany: administered doses, vaccinated people per dose, doses per vaccine, and vaccination quotas. One row, replaced on every sync.",
+ ),
+ "vaccinations_history": RKICovidEndpointConfig(
+ name="vaccinations_history",
+ path="/vaccinations/history",
+ kind="data_history",
+ primary_keys=["date"],
+ partition_key="date",
+ supports_days=True,
+ description="Daily COVID-19 vaccinations administered in Germany, split by first, second, and booster doses. One row per day. Full refresh.",
+ ),
+ "vaccinations_states": RKICovidEndpointConfig(
+ name="vaccinations_states",
+ path="/vaccinations/states",
+ kind="keyed_rows",
+ primary_keys=["abbreviation"],
+ key_field="abbreviation",
+ description="Current COVID-19 vaccination totals and quotas per German federal state. One row per state. Full refresh.",
+ ),
}
ENDPOINTS = tuple(RKI_COVID_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.py
index d59bc6b945ba..283a657d3aae 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rki_covid/tests/test_rki_covid.py
@@ -88,6 +88,37 @@ class TestRKICovid:
{"age_group": "A80+", "casesMale": 3, "casesFemale": 4},
]
+ def test_vaccinations_snapshot_drops_per_state_figures_and_keeps_meta(self) -> None:
+ # The per-state figures belong to vaccinations_states; keeping them would nest 17 states into one cell.
+ body = {
+ "data": {"vaccinated": 10, "quote": 0.5, "states": {"BY": {"vaccinated": 3}}},
+ "meta": {"lastUpdate": "2024-07-09T07:00:22.000Z"},
+ }
+ assert _run("vaccinations", [_ok(body)]).rows == [
+ {"vaccinated": 10, "quote": 0.5, "meta": {"lastUpdate": "2024-07-09T07:00:22.000Z"}}
+ ]
+
+ def test_states_history_flattens_each_state_history_with_state_identity(self) -> None:
+ # History rows only carry date and cases; without the state key they collide on the primary key.
+ body = {
+ "data": {
+ "SH": {"id": 1, "name": "Schleswig-Holstein", "history": [{"cases": 4, "date": "2026-07-19"}]},
+ "BY": {
+ "id": 9,
+ "name": "Bayern",
+ "history": [{"cases": 1, "date": "2026-07-19"}, {"cases": 2, "date": "2026-07-20"}],
+ },
+ },
+ "meta": {},
+ }
+ result = _run("states_history_cases", [_ok(body)], history_days=30)
+ assert result.paths == ["/states/history/cases/30"]
+ assert result.rows == [
+ {"abbreviation": "SH", "id": 1, "name": "Schleswig-Holstein", "cases": 4, "date": "2026-07-19"},
+ {"abbreviation": "BY", "id": 9, "name": "Bayern", "cases": 1, "date": "2026-07-19"},
+ {"abbreviation": "BY", "id": 9, "name": "Bayern", "cases": 2, "date": "2026-07-20"},
+ ]
+
def test_states_yield_one_row_per_dict_value(self) -> None:
body = {
"data": {
@@ -148,6 +179,8 @@ class TestRKICovid:
("missing_data", "states", {"meta": {}}),
("wrong_data_type", "germany_history_cases", {"data": {"unexpected": "shape"}, "meta": {}}),
("missing_history", "testing_history", {"data": {}, "meta": {}}),
+ ("missing_state_history", "states_history_cases", {"data": {"BY": {"id": 9}}, "meta": {}}),
+ ("missing_vaccinations_data", "vaccinations", {"meta": {}}),
]
)
def test_malformed_data_yields_no_rows_instead_of_crashing(self, _name: str, endpoint: str, body: dict) -> None:
@@ -160,6 +193,7 @@ class TestRKICovid:
("germany_history_cases", ["date"], "date"),
("germany_history_hospitalization", ["date"], "date"),
("states", ["abbreviation"], None),
+ ("states_history_cases", ["abbreviation", "date"], "date"),
("districts", ["ags"], None),
("testing_history", ["calendarWeek"], None),
]