Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(rki-covid): add state and district history tables (#128430)

Tom Owers0c8fba0ec1ad
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index bef70508ad25..baf01f2ff5a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10353,10 +10353,10 @@ Diffed against: <https://api.corona-zahlen.org/docs/>
- [x] `GET /vaccinations/history/:days` — daily vaccination time series, the analytical form of the vaccination data (high) → `vaccinations_history`
- [x] `GET /vaccinations/states/:state` — per-state vaccination breakdown, the standard comparison dimension (high) → `vaccinations_states` (reads `GET /vaccinations/states`, which returns every state in one response)
- [x] `GET /states/history/cases/:days` — per-state case time series; only Germany-level history is synced, so no state comparison over time is possible (high) → `states_history_cases`
-- [ ] `GET /states/history/deaths/:days` — per-state death time series to match the national germany_history_deaths table (high)
-- [ ] `GET /states/history/incidence/:days` — per-state 7-day incidence, the metric the whole dataset is organized around (high)
-- [ ] `GET /districts/history/cases/:days` — per-district case time series - the finest granularity the API offers and the reason to use it over aggregate sources (high)
-- [ ] `GET /districts/history/incidence/:days` — per-district incidence over time, the main district-level reporting metric (high)
+- [x] `GET /states/history/deaths/:days` — per-state death time series to match the national germany_history_deaths table (high) → `states_history_deaths`
+- [x] `GET /states/history/incidence/:days` — per-state 7-day incidence, the metric the whole dataset is organized around (high) → `states_history_incidence`
+- [x] `GET /districts/history/cases/:days` — per-district case time series - the finest granularity the API offers and the reason to use it over aggregate sources (high) → `districts_history_cases`
+- [x] `GET /districts/history/incidence/:days` — per-district incidence over time, the main district-level reporting metric (high) → `districts_history_incidence`
- [ ] `GET /germany/history/rValue/:days` — the reproduction number time series, a headline national metric with no equivalent column elsewhere (medium)
- [ ] `GET /states/age-groups` — age-group breakdown per state; the source has only the Germany-level age groups (medium)
- [ ] `GET /districts/age-groups` — age-group breakdown per district (medium)
diff --git a/products/warehouse_sources/sources/rki_covid/canonical_descriptions.py b/products/warehouse_sources/sources/rki_covid/canonical_descriptions.py
index d8c962d930c8..eddf68762dcf 100644
--- a/products/warehouse_sources/sources/rki_covid/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/rki_covid/canonical_descriptions.py
@@ -126,6 +126,28 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"cases": "New cases reported in the state on this day.",
},
},
+ "states_history_deaths": {
+ "description": "Daily new reported COVID-19 deaths per German federal state.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",
+ "id": "Numeric state identifier used by the RKI.",
+ "name": "State name.",
+ "date": "Reporting day the row covers.",
+ "deaths": "New deaths reported in the state on this day.",
+ },
+ },
+ "states_history_incidence": {
+ "description": "Daily 7-day incidence per 100,000 inhabitants per German federal state.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",
+ "id": "Numeric state identifier used by the RKI.",
+ "name": "State name.",
+ "date": "Reporting day the row covers.",
+ "weekIncidence": "7-day incidence per 100,000 inhabitants in the state on this day.",
+ },
+ },
"districts": {
"description": "Current COVID-19 snapshot per German district (Landkreis / kreisfreie Stadt).",
"docs_url": _DOCS_URL,
@@ -146,6 +168,26 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"delta": "Day-over-day change in cases, deaths, recovered, and week incidence.",
},
},
+ "districts_history_cases": {
+ "description": "Daily new reported COVID-19 cases per German district (Landkreis / kreisfreie Stadt).",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "ags": "Official municipality key (Allgemeiner Gemeindeschlüssel) identifying the district.",
+ "name": "District name.",
+ "date": "Reporting day the row covers.",
+ "cases": "New cases reported in the district on this day.",
+ },
+ },
+ "districts_history_incidence": {
+ "description": "Daily 7-day incidence per 100,000 inhabitants per German district (Landkreis / kreisfreie Stadt).",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "ags": "Official municipality key (Allgemeiner Gemeindeschlüssel) identifying the district.",
+ "name": "District name.",
+ "date": "Reporting day the row covers.",
+ "weekIncidence": "7-day incidence per 100,000 inhabitants in the district on this day.",
+ },
+ },
"testing_history": {
"description": "Weekly PCR testing figures for Germany.",
"docs_url": _DOCS_URL,
diff --git a/products/warehouse_sources/sources/rki_covid/rki_covid.py b/products/warehouse_sources/sources/rki_covid/rki_covid.py
index 211bdce67120..ffaebc37ce60 100644
--- a/products/warehouse_sources/sources/rki_covid/rki_covid.py
+++ b/products/warehouse_sources/sources/rki_covid/rki_covid.py
@@ -1,4 +1,5 @@
from collections.abc import Iterator
+from itertools import batched
from typing import Any, Optional
import requests
@@ -9,8 +10,9 @@ from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_
RKI_COVID_BASE_URL = "https://api.corona-zahlen.org"
-# Every dataset is bounded (the pandemic time series is a few thousand rows, districts ~400 rows),
-# so no history window cap is needed beyond the optional user-configured `history_days`.
+# The full per-district history is ~1M rows in one response, so rows are yielded in batches instead
+# of one list to avoid holding a second copy of every row in memory.
+ROW_BATCH_SIZE = 5000
class RKICovidAPIError(Exception):
@@ -132,10 +134,11 @@ def get_rows(
session = make_tracked_session()
body = _fetch(session, build_url(config, history_days))
- rows = list(parser(body, config))
- logger.debug(f"RKI COVID-19: fetched {len(rows)} rows for {endpoint}")
- if rows:
- yield rows
+ row_count = 0
+ for batch in batched(parser(body, config), ROW_BATCH_SIZE, strict=False):
+ row_count += len(batch)
+ yield list(batch)
+ logger.debug(f"RKI COVID-19: fetched {row_count} rows for {endpoint}")
def rki_covid_source(
diff --git a/products/warehouse_sources/sources/rki_covid/settings.py b/products/warehouse_sources/sources/rki_covid/settings.py
index e3f2aba103ef..56f24f5b716e 100644
--- a/products/warehouse_sources/sources/rki_covid/settings.py
+++ b/products/warehouse_sources/sources/rki_covid/settings.py
@@ -129,6 +129,26 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {
supports_days=True,
description="Daily new reported COVID-19 cases per German federal state. One row per state and day. Full refresh.",
),
+ "states_history_deaths": RKICovidEndpointConfig(
+ name="states_history_deaths",
+ path="/states/history/deaths",
+ kind="keyed_history",
+ primary_keys=["abbreviation", "date"],
+ key_field="abbreviation",
+ partition_key="date",
+ supports_days=True,
+ description="Daily new reported COVID-19 deaths per German federal state. One row per state and day. Full refresh.",
+ ),
+ "states_history_incidence": RKICovidEndpointConfig(
+ name="states_history_incidence",
+ path="/states/history/incidence",
+ kind="keyed_history",
+ primary_keys=["abbreviation", "date"],
+ key_field="abbreviation",
+ partition_key="date",
+ supports_days=True,
+ description="Daily 7-day incidence per 100k inhabitants per German federal state. One row per state and day. Full refresh.",
+ ),
"districts": RKICovidEndpointConfig(
name="districts",
path="/districts",
@@ -136,6 +156,26 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {
primary_keys=["ags"],
description="Current COVID-19 snapshot per German district (Landkreis), keyed by the official municipality key (AGS). One row per district. Full refresh.",
),
+ "districts_history_cases": RKICovidEndpointConfig(
+ name="districts_history_cases",
+ path="/districts/history/cases",
+ kind="keyed_history",
+ primary_keys=["ags", "date"],
+ key_field="ags",
+ partition_key="date",
+ supports_days=True,
+ description="Daily new reported COVID-19 cases per German district (Landkreis), keyed by AGS. One row per district and day. Full refresh.",
+ ),
+ "districts_history_incidence": RKICovidEndpointConfig(
+ name="districts_history_incidence",
+ path="/districts/history/incidence",
+ kind="keyed_history",
+ primary_keys=["ags", "date"],
+ key_field="ags",
+ partition_key="date",
+ supports_days=True,
+ description="Daily 7-day incidence per 100k inhabitants per German district (Landkreis), keyed by AGS. One row per district and day. Full refresh.",
+ ),
"testing_history": RKICovidEndpointConfig(
name="testing_history",
path="/testing/history",
diff --git a/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.py b/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.py
index 1557edf30ec2..18e9d39a867f 100644
--- a/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.py
+++ b/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.py
@@ -4,7 +4,12 @@ import requests
from parameterized import parameterized
from products.warehouse_sources.sources.rki_covid._config import RKICovidSourceConfig
-from products.warehouse_sources.sources.rki_covid.rki_covid import RKICovidAPIError, build_url, validate_connection
+from products.warehouse_sources.sources.rki_covid.rki_covid import (
+ ROW_BATCH_SIZE,
+ RKICovidAPIError,
+ build_url,
+ validate_connection,
+)
from products.warehouse_sources.sources.rki_covid.settings import RKI_COVID_ENDPOINTS
from products.warehouse_sources.sources.rki_covid.source import RKICovidSource
from products.warehouse_sources.sources.sdk.testing import (
@@ -113,6 +118,46 @@ class TestRKICovid:
{"abbreviation": "BY", "id": 9, "name": "Bayern", "cases": 2, "date": "2026-07-20"},
]
+ @parameterized.expand(
+ [
+ ("states_deaths", "states_history_deaths", "abbreviation", "SH", "BY", "deaths"),
+ ("states_incidence", "states_history_incidence", "abbreviation", "SH", "BY", "weekIncidence"),
+ ("districts_cases", "districts_history_cases", "ags", "01001", "09162", "cases"),
+ ("districts_incidence", "districts_history_incidence", "ags", "01001", "09162", "weekIncidence"),
+ ]
+ )
+ def test_keyed_history_endpoints_request_their_path_and_key_rows_by_entity_and_day(
+ self, _name: str, endpoint: str, key_field: str, first_key: str, second_key: str, value_field: str
+ ) -> None:
+ # Both entities report the same day, so rows only stay unique if the entity key is injected.
+ body = {
+ "data": {
+ first_key: {"name": "first", "history": [{value_field: 1, "date": "2026-07-19T00:00:00.000Z"}]},
+ second_key: {"name": "second", "history": [{value_field: 2, "date": "2026-07-19T00:00:00.000Z"}]},
+ },
+ "meta": {},
+ }
+ result = _run(endpoint, [_ok(body)], history_days=7)
+ assert result.paths == [f"{RKI_COVID_ENDPOINTS[endpoint].path}/7"]
+ assert result.response is not None
+ assert result.response.primary_keys == [key_field, "date"]
+ assert result.rows == [
+ {key_field: first_key, "name": "first", value_field: 1, "date": "2026-07-19T00:00:00.000Z"},
+ {key_field: second_key, "name": "second", value_field: 2, "date": "2026-07-19T00:00:00.000Z"},
+ ]
+
+ def test_large_responses_are_yielded_in_bounded_batches_without_losing_rows(self) -> None:
+ districts = {
+ f"{n:05d}": {
+ "ags": f"{n:05d}",
+ "history": [{"cases": n, "date": f"2026-07-{day:02d}"} for day in (1, 2, 3)],
+ }
+ for n in range(ROW_BATCH_SIZE)
+ }
+ result = _run("districts_history_cases", [_ok({"data": districts, "meta": {}})])
+ assert [len(batch) for batch in result.items] == [ROW_BATCH_SIZE, ROW_BATCH_SIZE, ROW_BATCH_SIZE]
+ assert len({(row["ags"], row["date"]) for row in result.rows}) == 3 * ROW_BATCH_SIZE
+
def test_states_yield_one_row_per_dict_value(self) -> None:
body = {
"data": {