:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(rki-covid): add state and district history tables (#128430)
Tom Owers0c8fba0ec1ad
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex bef70508ad25..baf01f2ff5a6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10353,10 +10353,10 @@ Diffed against: <https://api.corona-zahlen.org/docs/>- [x] `GET /vaccinations/history/:days` — daily vaccination time series, the analytical form of the vaccination data (high) → `vaccinations_history`- [x] `GET /vaccinations/states/:state` — per-state vaccination breakdown, the standard comparison dimension (high) → `vaccinations_states` (reads `GET /vaccinations/states`, which returns every state in one response)- [x] `GET /states/history/cases/:days` — per-state case time series; only Germany-level history is synced, so no state comparison over time is possible (high) → `states_history_cases`-- [ ] `GET /states/history/deaths/:days` — per-state death time series to match the national germany_history_deaths table (high)-- [ ] `GET /states/history/incidence/:days` — per-state 7-day incidence, the metric the whole dataset is organized around (high)-- [ ] `GET /districts/history/cases/:days` — per-district case time series - the finest granularity the API offers and the reason to use it over aggregate sources (high)-- [ ] `GET /districts/history/incidence/:days` — per-district incidence over time, the main district-level reporting metric (high)+- [x] `GET /states/history/deaths/:days` — per-state death time series to match the national germany_history_deaths table (high) → `states_history_deaths`+- [x] `GET /states/history/incidence/:days` — per-state 7-day incidence, the metric the whole dataset is organized around (high) → `states_history_incidence`+- [x] `GET /districts/history/cases/:days` — per-district case time series - the finest granularity the API offers and the reason to use it over aggregate sources (high) → `districts_history_cases`+- [x] `GET /districts/history/incidence/:days` — per-district incidence over time, the main district-level reporting metric (high) → `districts_history_incidence`- [ ] `GET /germany/history/rValue/:days` — the reproduction number time series, a headline national metric with no equivalent column elsewhere (medium)- [ ] `GET /states/age-groups` — age-group breakdown per state; the source has only the Germany-level age groups (medium)- [ ] `GET /districts/age-groups` — age-group breakdown per district (medium)diff --git a/products/warehouse_sources/sources/rki_covid/canonical_descriptions.py b/products/warehouse_sources/sources/rki_covid/canonical_descriptions.pyindex d8c962d930c8..eddf68762dcf 100644--- a/products/warehouse_sources/sources/rki_covid/canonical_descriptions.py+++ b/products/warehouse_sources/sources/rki_covid/canonical_descriptions.py@@ -126,6 +126,28 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"cases": "New cases reported in the state on this day.",},},+ "states_history_deaths": {+ "description": "Daily new reported COVID-19 deaths per German federal state.",+ "docs_url": _DOCS_URL,+ "columns": {+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",+ "id": "Numeric state identifier used by the RKI.",+ "name": "State name.",+ "date": "Reporting day the row covers.",+ "deaths": "New deaths reported in the state on this day.",+ },+ },+ "states_history_incidence": {+ "description": "Daily 7-day incidence per 100,000 inhabitants per German federal state.",+ "docs_url": _DOCS_URL,+ "columns": {+ "abbreviation": "Two-letter state abbreviation (e.g. BY for Bayern).",+ "id": "Numeric state identifier used by the RKI.",+ "name": "State name.",+ "date": "Reporting day the row covers.",+ "weekIncidence": "7-day incidence per 100,000 inhabitants in the state on this day.",+ },+ },"districts": {"description": "Current COVID-19 snapshot per German district (Landkreis / kreisfreie Stadt).","docs_url": _DOCS_URL,@@ -146,6 +168,26 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"delta": "Day-over-day change in cases, deaths, recovered, and week incidence.",},},+ "districts_history_cases": {+ "description": "Daily new reported COVID-19 cases per German district (Landkreis / kreisfreie Stadt).",+ "docs_url": _DOCS_URL,+ "columns": {+ "ags": "Official municipality key (Allgemeiner Gemeindeschlüssel) identifying the district.",+ "name": "District name.",+ "date": "Reporting day the row covers.",+ "cases": "New cases reported in the district on this day.",+ },+ },+ "districts_history_incidence": {+ "description": "Daily 7-day incidence per 100,000 inhabitants per German district (Landkreis / kreisfreie Stadt).",+ "docs_url": _DOCS_URL,+ "columns": {+ "ags": "Official municipality key (Allgemeiner Gemeindeschlüssel) identifying the district.",+ "name": "District name.",+ "date": "Reporting day the row covers.",+ "weekIncidence": "7-day incidence per 100,000 inhabitants in the district on this day.",+ },+ },"testing_history": {"description": "Weekly PCR testing figures for Germany.","docs_url": _DOCS_URL,diff --git a/products/warehouse_sources/sources/rki_covid/rki_covid.py b/products/warehouse_sources/sources/rki_covid/rki_covid.pyindex 211bdce67120..ffaebc37ce60 100644--- a/products/warehouse_sources/sources/rki_covid/rki_covid.py+++ b/products/warehouse_sources/sources/rki_covid/rki_covid.py@@ -1,4 +1,5 @@from collections.abc import Iterator+from itertools import batchedfrom typing import Any, Optionalimport requests@@ -9,8 +10,9 @@ from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_RKI_COVID_BASE_URL = "https://api.corona-zahlen.org"-# Every dataset is bounded (the pandemic time series is a few thousand rows, districts ~400 rows),-# so no history window cap is needed beyond the optional user-configured `history_days`.+# The full per-district history is ~1M rows in one response, so rows are yielded in batches instead+# of one list to avoid holding a second copy of every row in memory.+ROW_BATCH_SIZE = 5000class RKICovidAPIError(Exception):@@ -132,10 +134,11 @@ def get_rows(session = make_tracked_session()body = _fetch(session, build_url(config, history_days))- rows = list(parser(body, config))- logger.debug(f"RKI COVID-19: fetched {len(rows)} rows for {endpoint}")- if rows:- yield rows+ row_count = 0+ for batch in batched(parser(body, config), ROW_BATCH_SIZE, strict=False):+ row_count += len(batch)+ yield list(batch)+ logger.debug(f"RKI COVID-19: fetched {row_count} rows for {endpoint}")def rki_covid_source(diff --git a/products/warehouse_sources/sources/rki_covid/settings.py b/products/warehouse_sources/sources/rki_covid/settings.pyindex e3f2aba103ef..56f24f5b716e 100644--- a/products/warehouse_sources/sources/rki_covid/settings.py+++ b/products/warehouse_sources/sources/rki_covid/settings.py@@ -129,6 +129,26 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {supports_days=True,description="Daily new reported COVID-19 cases per German federal state. One row per state and day. Full refresh.",),+ "states_history_deaths": RKICovidEndpointConfig(+ name="states_history_deaths",+ path="/states/history/deaths",+ kind="keyed_history",+ primary_keys=["abbreviation", "date"],+ key_field="abbreviation",+ partition_key="date",+ supports_days=True,+ description="Daily new reported COVID-19 deaths per German federal state. One row per state and day. Full refresh.",+ ),+ "states_history_incidence": RKICovidEndpointConfig(+ name="states_history_incidence",+ path="/states/history/incidence",+ kind="keyed_history",+ primary_keys=["abbreviation", "date"],+ key_field="abbreviation",+ partition_key="date",+ supports_days=True,+ description="Daily 7-day incidence per 100k inhabitants per German federal state. One row per state and day. Full refresh.",+ ),"districts": RKICovidEndpointConfig(name="districts",path="/districts",@@ -136,6 +156,26 @@ RKI_COVID_ENDPOINTS: dict[str, RKICovidEndpointConfig] = {primary_keys=["ags"],description="Current COVID-19 snapshot per German district (Landkreis), keyed by the official municipality key (AGS). One row per district. Full refresh.",),+ "districts_history_cases": RKICovidEndpointConfig(+ name="districts_history_cases",+ path="/districts/history/cases",+ kind="keyed_history",+ primary_keys=["ags", "date"],+ key_field="ags",+ partition_key="date",+ supports_days=True,+ description="Daily new reported COVID-19 cases per German district (Landkreis), keyed by AGS. One row per district and day. Full refresh.",+ ),+ "districts_history_incidence": RKICovidEndpointConfig(+ name="districts_history_incidence",+ path="/districts/history/incidence",+ kind="keyed_history",+ primary_keys=["ags", "date"],+ key_field="ags",+ partition_key="date",+ supports_days=True,+ description="Daily 7-day incidence per 100k inhabitants per German district (Landkreis), keyed by AGS. One row per district and day. Full refresh.",+ ),"testing_history": RKICovidEndpointConfig(name="testing_history",path="/testing/history",diff --git a/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.py b/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.pyindex 1557edf30ec2..18e9d39a867f 100644--- a/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.py+++ b/products/warehouse_sources/sources/rki_covid/tests/test_rki_covid.py@@ -4,7 +4,12 @@ import requestsfrom parameterized import parameterizedfrom products.warehouse_sources.sources.rki_covid._config import RKICovidSourceConfig-from products.warehouse_sources.sources.rki_covid.rki_covid import RKICovidAPIError, build_url, validate_connection+from products.warehouse_sources.sources.rki_covid.rki_covid import (+ ROW_BATCH_SIZE,+ RKICovidAPIError,+ build_url,+ validate_connection,+)from products.warehouse_sources.sources.rki_covid.settings import RKI_COVID_ENDPOINTSfrom products.warehouse_sources.sources.rki_covid.source import RKICovidSourcefrom products.warehouse_sources.sources.sdk.testing import (@@ -113,6 +118,46 @@ class TestRKICovid:{"abbreviation": "BY", "id": 9, "name": "Bayern", "cases": 2, "date": "2026-07-20"},]+ @parameterized.expand(+ [+ ("states_deaths", "states_history_deaths", "abbreviation", "SH", "BY", "deaths"),+ ("states_incidence", "states_history_incidence", "abbreviation", "SH", "BY", "weekIncidence"),+ ("districts_cases", "districts_history_cases", "ags", "01001", "09162", "cases"),+ ("districts_incidence", "districts_history_incidence", "ags", "01001", "09162", "weekIncidence"),+ ]+ )+ def test_keyed_history_endpoints_request_their_path_and_key_rows_by_entity_and_day(+ self, _name: str, endpoint: str, key_field: str, first_key: str, second_key: str, value_field: str+ ) -> None:+ # Both entities report the same day, so rows only stay unique if the entity key is injected.+ body = {+ "data": {+ first_key: {"name": "first", "history": [{value_field: 1, "date": "2026-07-19T00:00:00.000Z"}]},+ second_key: {"name": "second", "history": [{value_field: 2, "date": "2026-07-19T00:00:00.000Z"}]},+ },+ "meta": {},+ }+ result = _run(endpoint, [_ok(body)], history_days=7)+ assert result.paths == [f"{RKI_COVID_ENDPOINTS[endpoint].path}/7"]+ assert result.response is not None+ assert result.response.primary_keys == [key_field, "date"]+ assert result.rows == [+ {key_field: first_key, "name": "first", value_field: 1, "date": "2026-07-19T00:00:00.000Z"},+ {key_field: second_key, "name": "second", value_field: 2, "date": "2026-07-19T00:00:00.000Z"},+ ]++ def test_large_responses_are_yielded_in_bounded_batches_without_losing_rows(self) -> None:+ districts = {+ f"{n:05d}": {+ "ags": f"{n:05d}",+ "history": [{"cases": n, "date": f"2026-07-{day:02d}"} for day in (1, 2, 3)],+ }+ for n in range(ROW_BATCH_SIZE)+ }+ result = _run("districts_history_cases", [_ok({"data": districts, "meta": {}})])+ assert [len(batch) for batch in result.items] == [ROW_BATCH_SIZE, ROW_BATCH_SIZE, ROW_BATCH_SIZE]+ assert len({(row["ags"], row["date"]) for row in result.rows}) == 3 * ROW_BATCH_SIZE+def test_states_yield_one_row_per_dict_value(self) -> None:body = {"data": {