:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(managed-warehouse): reconnect to trino after each backfill export (#129801)
Eric Duong3dd257a1716b
diff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.pyindex 2eae79f1bec5..b3e1c1e1eafa 100644--- a/posthog/dags/events_backfill_to_duckling.py+++ b/posthog/dags/events_backfill_to_duckling.py@@ -40,7 +40,7 @@ import calendarimport functoolsimport dataclassesfrom collections.abc import Callable, Mapping, Sequence-from contextlib import closing, nullcontext+from contextlib import closingfrom dataclasses import replacefrom datetime import UTC, date, datetime, timedeltafrom typing import Any, Literal, Protocol@@ -2644,12 +2644,12 @@ def backfill_through_trino(and all(isinstance(item.replacement, DuckLakeTableReplacement) for item in exports))rebuild = False- with (- connect_backfill_catalog(target.organization_id, canary=config.use_trino_canary, team_id=target.team_id)- if register- else nullcontext()- ) as catalog:- if catalog is not None:+ connect = functools.partial(+ connect_backfill_catalog, target.organization_id, canary=config.use_trino_canary, team_id=target.team_id+ )++ if register:+ with connect() as catalog:try:catalog.ensure_table(table,@@ -2666,24 +2666,26 @@ def backfill_through_trino(f"{table} does not have the column types of the export, so this run rebuilds it before it ""registers files")- for item in exports:- # A new prefix for every export, because an export under an earlier prefix would overwrite files that- # DuckLake already registered with their row counts and statistics.- prefix = f"{run_id}_{secrets.token_hex(4)}"-- def export_and_list(client: Client, item: TrinoExport = item, prefix: str = prefix) -> list[str] | None:- with tags_context(kind="dagster", dagster=tags):- s3_glob = item.export(client=client, run_id=prefix)- return None if s3_glob is None else list_exported_files(client, target, s3_glob, settings)-- files = cluster.any_host_by_role(fn=export_and_list, workload=workload, node_role=NodeRole.DATA).result()- if files is None:- continue- partitions_exported += 1- if catalog is None:- context.log.info(f"Exported {len(files)} file(s) for {item.label} without registering them")- continue- replacement = item.replacement if config.cleanup_existing_partition_data else None+ for item in exports:+ # A new prefix for every export, because an export under an earlier prefix would overwrite files that+ # DuckLake already registered with their row counts and statistics.+ prefix = f"{run_id}_{secrets.token_hex(4)}"++ def export_and_list(client: Client, item: TrinoExport = item, prefix: str = prefix) -> list[str] | None:+ with tags_context(kind="dagster", dagster=tags):+ s3_glob = item.export(client=client, run_id=prefix)+ return None if s3_glob is None else list_exported_files(client, target, s3_glob, settings)++ files = cluster.any_host_by_role(fn=export_and_list, workload=workload, node_role=NodeRole.DATA).result()+ if files is None:+ continue+ partitions_exported += 1+ if not register:+ context.log.info(f"Exported {len(files)} file(s) for {item.label} without registering them")+ continue+ replacement = item.replacement if config.cleanup_existing_partition_data else None+ # An export can outlive the Trino service credential, which cannot be renewed after it expires.+ with connect() as catalog:if rebuild:# The rebuild waits until the export finished, to keep the window with an empty table short. A file# count that add_files refuses fails here, before the rebuild empties the table.@@ -2695,14 +2697,14 @@ def backfill_through_trino(files_registered += len(files)elif replacement is not None:catalog.delete_rows(table, replacement)- context.log.info(f"Registered {len(files)} file(s) for {item.label} in {table} through Trino")- logger.info(- "duckling_trino_files_registered",- team_id=target.team_id,- table=table,- partition=item.label,- file_count=len(files),- )+ context.log.info(f"Registered {len(files)} file(s) for {item.label} in {table} through Trino")+ logger.info(+ "duckling_trino_files_registered",+ team_id=target.team_id,+ table=table,+ partition=item.label,+ file_count=len(files),+ )return TrinoBackfillResult(partitions_exported=partitions_exported, files_registered=files_registered)diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.pyindex 09d19c2ba7d1..e1d9df00bb59 100644--- a/posthog/dags/test_events_backfill_to_duckling.py+++ b/posthog/dags/test_events_backfill_to_duckling.py@@ -1,5 +1,7 @@import osimport re+from collections.abc import Callable, Iterator+from contextlib import contextmanagerfrom datetime import UTC, date, datetime, timedeltafrom fnmatch import fnmatchfrom typing import Any, cast@@ -2635,15 +2637,35 @@ class _FakeTrinoBackfillCatalog:self.tables: list[tuple[str, dict[str, str]]] = []self.registered: list[tuple[str, list[str], str, DuckLakeDayReplacement | DuckLakeTableReplacement | None]] = []self.deleted: list[tuple[str, DuckLakeDayReplacement | DuckLakeTableReplacement]] = []+ self.connection_open = False+ self.exports_at_connect = 0+ self.count_exports: Callable[[], int] = lambda: 0++ @contextmanager+ def connect(self, *_args: Any, **_kwargs: Any) -> Iterator["_FakeTrinoBackfillCatalog"]:+ self.connection_open = True+ self.exports_at_connect = self.count_exports()+ try:+ yield self+ finally:+ self.connection_open = False++ def _require_live_connection(self) -> None:+ assert self.connection_open, "a statement ran without an open Trino connection"+ assert self.count_exports() == self.exports_at_connect, (+ "a Trino connection stayed open across an export, so its credential lapsed"+ )def ensure_table(self, table: str, columns: dict[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool) -> None:+ self._require_live_connection()if self.layout_mismatch and validate and not self.rebuilt:raise DuckLakeTableLayoutError("the table has unsigned columns")self.tables.append((table, dict(columns)))def replace_table(self, table: str, columns: dict[str, str]) -> None:+ self._require_live_connection()self.calls.append(("replace", table))self.rebuilt = True@@ -2659,11 +2681,13 @@ class _FakeTrinoBackfillCatalog:import_id: str,replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,) -> int:+ self._require_live_connection()self.calls.append(("register", table, replacement))self.registered.append((table, list(files), import_id, replacement))return len(files)def delete_rows(self, table: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement) -> None:+ self._require_live_connection()self.deleted.append((table, replacement))@@ -2768,7 +2792,8 @@ class TestBackfillThroughTrino:patch("posthog.dags.events_backfill_to_duckling.record_duckling_backfill_workload"),patch("tenacity.nap.time.sleep"),):- connect_catalog.return_value.__enter__.return_value = catalog+ catalog.count_exports = lambda: len(clickhouse.inserts)+ connect_catalog.side_effect = catalog.connectrun(context, config)run(context, config)duckgres.assert_not_called()diff --git a/products/managed_warehouse/backend/tests/test_groups_backfill.py b/products/managed_warehouse/backend/tests/test_groups_backfill.pyindex 211af4a91dea..7d19d0d13463 100644--- a/products/managed_warehouse/backend/tests/test_groups_backfill.py+++ b/products/managed_warehouse/backend/tests/test_groups_backfill.py@@ -340,7 +340,7 @@ def test_groups_backfill_registers_through_trino_without_duckgres() -> None:inserts, catalog, connect_catalog, context = _run_groups_through_trino(partition_key, DucklingBackfillConfig())- connect_catalog.assert_called_once_with("org", canary=False, team_id=7)+ connect_catalog.assert_called_with("org", canary=False, team_id=7)assert catalog.tables == [("groups_team_7", GROUPS_TRINO_COLUMN_TYPES)][(table, files, registered_replacement)] = catalog.registeredassert (table, registered_replacement) == ("groups_team_7", replacement)@@ -361,7 +361,7 @@ def test_groups_backfill_opts_in_to_trino_when_the_org_is_not_on_the_flag() -> N"7", DucklingBackfillConfig(register_through_trino=True), org_on_flag=False)- connect_catalog.assert_called_once()+ connect_catalog.assert_called()assert catalog.registered