Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(managed-warehouse): reconnect to trino after each backfill export (#129801)

Eric Duong3dd257a1716b
diff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.py
index 2eae79f1bec5..b3e1c1e1eafa 100644
--- a/posthog/dags/events_backfill_to_duckling.py
+++ b/posthog/dags/events_backfill_to_duckling.py
@@ -40,7 +40,7 @@ import calendar
import functools
import dataclasses
from collections.abc import Callable, Mapping, Sequence
-from contextlib import closing, nullcontext
+from contextlib import closing
from dataclasses import replace
from datetime import UTC, date, datetime, timedelta
from typing import Any, Literal, Protocol
@@ -2644,12 +2644,12 @@ def backfill_through_trino(
and all(isinstance(item.replacement, DuckLakeTableReplacement) for item in exports)
)
rebuild = False
- with (
- connect_backfill_catalog(target.organization_id, canary=config.use_trino_canary, team_id=target.team_id)
- if register
- else nullcontext()
- ) as catalog:
- if catalog is not None:
+ connect = functools.partial(
+ connect_backfill_catalog, target.organization_id, canary=config.use_trino_canary, team_id=target.team_id
+ )
+
+ if register:
+ with connect() as catalog:
try:
catalog.ensure_table(
table,
@@ -2666,24 +2666,26 @@ def backfill_through_trino(
f"{table} does not have the column types of the export, so this run rebuilds it before it "
"registers files"
)
- for item in exports:
- # A new prefix for every export, because an export under an earlier prefix would overwrite files that
- # DuckLake already registered with their row counts and statistics.
- prefix = f"{run_id}_{secrets.token_hex(4)}"
-
- def export_and_list(client: Client, item: TrinoExport = item, prefix: str = prefix) -> list[str] | None:
- with tags_context(kind="dagster", dagster=tags):
- s3_glob = item.export(client=client, run_id=prefix)
- return None if s3_glob is None else list_exported_files(client, target, s3_glob, settings)
-
- files = cluster.any_host_by_role(fn=export_and_list, workload=workload, node_role=NodeRole.DATA).result()
- if files is None:
- continue
- partitions_exported += 1
- if catalog is None:
- context.log.info(f"Exported {len(files)} file(s) for {item.label} without registering them")
- continue
- replacement = item.replacement if config.cleanup_existing_partition_data else None
+ for item in exports:
+ # A new prefix for every export, because an export under an earlier prefix would overwrite files that
+ # DuckLake already registered with their row counts and statistics.
+ prefix = f"{run_id}_{secrets.token_hex(4)}"
+
+ def export_and_list(client: Client, item: TrinoExport = item, prefix: str = prefix) -> list[str] | None:
+ with tags_context(kind="dagster", dagster=tags):
+ s3_glob = item.export(client=client, run_id=prefix)
+ return None if s3_glob is None else list_exported_files(client, target, s3_glob, settings)
+
+ files = cluster.any_host_by_role(fn=export_and_list, workload=workload, node_role=NodeRole.DATA).result()
+ if files is None:
+ continue
+ partitions_exported += 1
+ if not register:
+ context.log.info(f"Exported {len(files)} file(s) for {item.label} without registering them")
+ continue
+ replacement = item.replacement if config.cleanup_existing_partition_data else None
+ # An export can outlive the Trino service credential, which cannot be renewed after it expires.
+ with connect() as catalog:
if rebuild:
# The rebuild waits until the export finished, to keep the window with an empty table short. A file
# count that add_files refuses fails here, before the rebuild empties the table.
@@ -2695,14 +2697,14 @@ def backfill_through_trino(
files_registered += len(files)
elif replacement is not None:
catalog.delete_rows(table, replacement)
- context.log.info(f"Registered {len(files)} file(s) for {item.label} in {table} through Trino")
- logger.info(
- "duckling_trino_files_registered",
- team_id=target.team_id,
- table=table,
- partition=item.label,
- file_count=len(files),
- )
+ context.log.info(f"Registered {len(files)} file(s) for {item.label} in {table} through Trino")
+ logger.info(
+ "duckling_trino_files_registered",
+ team_id=target.team_id,
+ table=table,
+ partition=item.label,
+ file_count=len(files),
+ )
return TrinoBackfillResult(partitions_exported=partitions_exported, files_registered=files_registered)
diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.py
index 09d19c2ba7d1..e1d9df00bb59 100644
--- a/posthog/dags/test_events_backfill_to_duckling.py
+++ b/posthog/dags/test_events_backfill_to_duckling.py
@@ -1,5 +1,7 @@
import os
import re
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
from datetime import UTC, date, datetime, timedelta
from fnmatch import fnmatch
from typing import Any, cast
@@ -2635,15 +2637,35 @@ class _FakeTrinoBackfillCatalog:
self.tables: list[tuple[str, dict[str, str]]] = []
self.registered: list[tuple[str, list[str], str, DuckLakeDayReplacement | DuckLakeTableReplacement | None]] = []
self.deleted: list[tuple[str, DuckLakeDayReplacement | DuckLakeTableReplacement]] = []
+ self.connection_open = False
+ self.exports_at_connect = 0
+ self.count_exports: Callable[[], int] = lambda: 0
+
+ @contextmanager
+ def connect(self, *_args: Any, **_kwargs: Any) -> Iterator["_FakeTrinoBackfillCatalog"]:
+ self.connection_open = True
+ self.exports_at_connect = self.count_exports()
+ try:
+ yield self
+ finally:
+ self.connection_open = False
+
+ def _require_live_connection(self) -> None:
+ assert self.connection_open, "a statement ran without an open Trino connection"
+ assert self.count_exports() == self.exports_at_connect, (
+ "a Trino connection stayed open across an export, so its credential lapsed"
+ )
def ensure_table(
self, table: str, columns: dict[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool
) -> None:
+ self._require_live_connection()
if self.layout_mismatch and validate and not self.rebuilt:
raise DuckLakeTableLayoutError("the table has unsigned columns")
self.tables.append((table, dict(columns)))
def replace_table(self, table: str, columns: dict[str, str]) -> None:
+ self._require_live_connection()
self.calls.append(("replace", table))
self.rebuilt = True
@@ -2659,11 +2681,13 @@ class _FakeTrinoBackfillCatalog:
import_id: str,
replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,
) -> int:
+ self._require_live_connection()
self.calls.append(("register", table, replacement))
self.registered.append((table, list(files), import_id, replacement))
return len(files)
def delete_rows(self, table: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement) -> None:
+ self._require_live_connection()
self.deleted.append((table, replacement))
@@ -2768,7 +2792,8 @@ class TestBackfillThroughTrino:
patch("posthog.dags.events_backfill_to_duckling.record_duckling_backfill_workload"),
patch("tenacity.nap.time.sleep"),
):
- connect_catalog.return_value.__enter__.return_value = catalog
+ catalog.count_exports = lambda: len(clickhouse.inserts)
+ connect_catalog.side_effect = catalog.connect
run(context, config)
run(context, config)
duckgres.assert_not_called()
diff --git a/products/managed_warehouse/backend/tests/test_groups_backfill.py b/products/managed_warehouse/backend/tests/test_groups_backfill.py
index 211af4a91dea..7d19d0d13463 100644
--- a/products/managed_warehouse/backend/tests/test_groups_backfill.py
+++ b/products/managed_warehouse/backend/tests/test_groups_backfill.py
@@ -340,7 +340,7 @@ def test_groups_backfill_registers_through_trino_without_duckgres(
) -> None:
inserts, catalog, connect_catalog, context = _run_groups_through_trino(partition_key, DucklingBackfillConfig())
- connect_catalog.assert_called_once_with("org", canary=False, team_id=7)
+ connect_catalog.assert_called_with("org", canary=False, team_id=7)
assert catalog.tables == [("groups_team_7", GROUPS_TRINO_COLUMN_TYPES)]
[(table, files, registered_replacement)] = catalog.registered
assert (table, registered_replacement) == ("groups_team_7", replacement)
@@ -361,7 +361,7 @@ def test_groups_backfill_opts_in_to_trino_when_the_org_is_not_on_the_flag() -> N
"7", DucklingBackfillConfig(register_through_trino=True), org_on_flag=False
)
- connect_catalog.assert_called_once()
+ connect_catalog.assert_called()
assert catalog.registered