Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(managed-warehouse): register duckling backfills through trino (#125930)

Eric Duong6251c29ee663
diff --git a/posthog/dags/README_DUCKLINGS.md b/posthog/dags/README_DUCKLINGS.md
index b4a130c9c768..5e06fbd7959a 100644
--- a/posthog/dags/README_DUCKLINGS.md
+++ b/posthog/dags/README_DUCKLINGS.md
@@ -22,6 +22,27 @@ Duckling RDS Catalog (PostgreSQL)
- **ClickHouse EC2 role**: Allowed in duckling bucket policy for direct S3 writes
- **Dagster IRSA role**: Can assume duckling cross-account roles for DuckDB registration
+### Warehouses without duckgres
+
+Organizations with the `managed-warehouse-trino-import-registration` flag have no duckgres, so the events, persons, and groups backfills register through the organization's Trino catalog instead.
+The ClickHouse export is the same, except that it casts each column to the Parquet type that DuckLake writes for its Trino column, because `add_files` refuses any type conversion.
+An export-only run (`skip_ducklake_registration: true`) writes the same types, so `add_files` can register its files later.
+ClickHouse lists the files it exported, because nothing else on the path can list the duckling bucket.
+Each export writes under a new file prefix, so a retry never overwrites a file that DuckLake has already registered.
+
+One `add_files` call registers the files of an export and removes the rows they replace in a single DuckLake snapshot:
+
+- A daily export replaces the rows of its UTC day, by `timestamp` for events and by `_timestamp` for persons and groups.
+- A full persons or groups export replaces the whole table.
+- An export with no files deletes the rows it would replace.
+- With `cleanup_existing_partition_data: false`, the files are added and no rows are removed.
+
+The call has an import ID, so a retry after a lost response adds nothing.
+A statement that fails for a transient reason, such as a DuckLake commit conflict, a lost coordinator, or a cluster fault, runs again up to four times.
+Readers never see a day without rows or with two copies of it, so this path needs no delete-then-register replay.
+Day and table replacements remove rows without a team filter, so the Trino path refuses to replace rows in the shared `events`, `persons`, and `groups` tables, or to drop them with `delete_tables: true`.
+Trino-created tables have no year/month/day partitioning, because Trino cannot partition a timestamp with time zone by those transforms. Each file holds one day, so file statistics still prune reads.
+
## Jobs and Sensors
### Events Backfill
diff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.py
index 524764e165b8..77066e04e0cb 100644
--- a/posthog/dags/events_backfill_to_duckling.py
+++ b/posthog/dags/events_backfill_to_duckling.py
@@ -35,13 +35,15 @@ import re
import math
import time
import random
+import secrets
import calendar
+import functools
import dataclasses
-from collections.abc import Callable
-from contextlib import closing
+from collections.abc import Callable, Mapping, Sequence
+from contextlib import closing, nullcontext
from dataclasses import replace
from datetime import UTC, date, datetime, timedelta
-from typing import Any, Literal
+from typing import Any, Literal, Protocol
from django.db import DatabaseError
from django.utils import timezone
@@ -69,11 +71,14 @@ from dagster import (
from psycopg import sql as psql
from tenacity import retry, retry_if_exception_type, stop_after_attempt, stop_after_delay, wait_exponential, wait_fixed
+from posthog.hogql.escape_sql import escape_clickhouse_string
+
from posthog.clickhouse.client.connection import NodeRole, Workload
from posthog.clickhouse.cluster import ClickhouseCluster, get_cluster
from posthog.clickhouse.query_tagging import tags_context
from posthog.cloud_utils import is_cloud
from posthog.dags.common import JobOwners, dagster_tags, settings_with_log_comment
+from posthog.dataclasses import frozen
from products.data_warehouse.backend.facade.backfill_status import (
BackfillOutcome,
@@ -91,6 +96,7 @@ from products.managed_warehouse.backend.facade.api import (
get_stored_bucket_config,
resolve_team_earliest_event_date,
)
+from products.managed_warehouse.backend.facade.backfill_trino import backfills_through_trino, connect_backfill_catalog
from products.managed_warehouse.backend.facade.client import (
ServiceCredential,
ServiceCredentialUnavailable,
@@ -98,7 +104,11 @@ from products.managed_warehouse.backend.facade.client import (
mint_service_credential,
refresh_service_credential,
)
-from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTeamMembership
+from products.managed_warehouse.backend.facade.contracts import (
+ DuckLakeDayReplacement,
+ DuckLakeTableReplacement,
+ ManagedWarehouseTeamMembership,
+)
from products.managed_warehouse.backend.facade.metrics import record_duckling_backfill_workload, track_duckling_backfill
from products.managed_warehouse.backend.facade.team_state import (
list_enabled_backfill_team_memberships,
@@ -821,6 +831,95 @@ EXPECTED_DUCKLAKE_PERSONS_COLUMNS = {
"_inserted_at",
}
+# Export columns for warehouses that register through Trino. add_files refuses any type conversion, so each
+# column has the exact Parquet type that DuckLake writes for its Trino column in EVENTS_TRINO_COLUMN_TYPES.
+# ClickHouse writes the DateTime64 columns without a precision in milliseconds, and the Enum8 as a Parquet enum.
+EVENTS_TRINO_COLUMNS = """
+ toString(uuid) as uuid,
+ event,
+ properties,
+ timestamp,
+ team_id,
+ toInt64(team_id) as project_id,
+ distinct_id,
+ elements_chain,
+ created_at,
+ toString(person_id) as person_id,
+ toDateTime64(person_created_at, 6, 'UTC') as person_created_at,
+ person_properties,
+ group0_properties,
+ group1_properties,
+ group2_properties,
+ group3_properties,
+ group4_properties,
+ toDateTime64(group0_created_at, 6, 'UTC') as group0_created_at,
+ toDateTime64(group1_created_at, 6, 'UTC') as group1_created_at,
+ toDateTime64(group2_created_at, 6, 'UTC') as group2_created_at,
+ toDateTime64(group3_created_at, 6, 'UTC') as group3_created_at,
+ toDateTime64(group4_created_at, 6, 'UTC') as group4_created_at,
+ toString(person_mode) as person_mode,
+ historical_migration,
+ now64(6) as _inserted_at
+"""
+
+EVENTS_TRINO_COLUMN_TYPES: dict[str, str] = {
+ "uuid": "varchar",
+ "event": "varchar",
+ "properties": "varchar",
+ "timestamp": "timestamp(6) with time zone",
+ "team_id": "bigint",
+ "project_id": "bigint",
+ "distinct_id": "varchar",
+ "elements_chain": "varchar",
+ "created_at": "timestamp(6) with time zone",
+ "person_id": "varchar",
+ "person_created_at": "timestamp(6) with time zone",
+ "person_properties": "varchar",
+ "group0_properties": "varchar",
+ "group1_properties": "varchar",
+ "group2_properties": "varchar",
+ "group3_properties": "varchar",
+ "group4_properties": "varchar",
+ "group0_created_at": "timestamp(6) with time zone",
+ "group1_created_at": "timestamp(6) with time zone",
+ "group2_created_at": "timestamp(6) with time zone",
+ "group3_created_at": "timestamp(6) with time zone",
+ "group4_created_at": "timestamp(6) with time zone",
+ "person_mode": "varchar",
+ "historical_migration": "boolean",
+ "_inserted_at": "timestamp(6) with time zone",
+}
+
+# Trino has no unsigned BIGINT, so the UInt64 person version is exported as a signed Int64.
+PERSONS_TRINO_COLUMNS = """
+ pd.team_id AS team_id,
+ pd.distinct_id AS distinct_id,
+ toString(p.id) AS id,
+ p.properties AS properties,
+ toDateTime64(p.created_at, 6, 'UTC') AS created_at,
+ toBool(p.is_identified) AS is_identified,
+ pd.version AS person_distinct_id_version,
+ toInt64(p.version) AS person_version,
+ toDateTime64(p._timestamp, 6) AS _timestamp,
+ now64(6) AS _inserted_at
+"""
+
+PERSONS_TRINO_COLUMN_TYPES: dict[str, str] = {
+ "team_id": "bigint",
+ "distinct_id": "varchar",
+ "id": "varchar",
+ "properties": "varchar",
+ "created_at": "timestamp(6) with time zone",
+ "is_identified": "boolean",
+ "person_distinct_id_version": "bigint",
+ "person_version": "bigint",
+ "_timestamp": "timestamp(6) with time zone",
+ "_inserted_at": "timestamp(6) with time zone",
+}
+
+# A Trino VARCHAR column expects the Parquet UTF-8 string type, not plain binary.
+TRINO_EXPORT_SETTINGS: dict[str, Any] = {"output_format_parquet_string_as_string": 1}
+
duckling_events_partitions_def = DynamicPartitionsDefinition(name="duckling_events_backfill")
duckling_persons_partitions_def = DynamicPartitionsDefinition(name="duckling_persons_backfill")
@@ -1544,6 +1643,7 @@ def export_events_to_duckling_s3(
date: datetime,
run_id: str,
settings: dict[str, Any],
+ columns: str = EVENTS_COLUMNS,
) -> str | None:
"""Export events for a team/date to the duckling's S3 bucket.
@@ -1623,7 +1723,7 @@ def export_events_to_duckling_s3(
)
PARTITION BY toString(cityHash64(distinct_id) % {fanout})
SELECT
- {EVENTS_COLUMNS}
+ {columns}
FROM events
WHERE {where_clause}
SETTINGS s3_truncate_on_insert=1, use_hive_partitioning=0
@@ -2166,6 +2266,7 @@ def export_persons_to_duckling_s3(
date: datetime,
run_id: str,
settings: dict[str, Any],
+ columns: str = PERSONS_COLUMNS,
) -> str | None:
"""Export persons for a team/date to the duckling's S3 bucket.
@@ -2225,7 +2326,7 @@ def export_persons_to_duckling_s3(
)
PARTITION BY toString(cityHash64(distinct_id) % {fanout})
SELECT
- {PERSONS_COLUMNS}
+ {columns}
FROM person AS p FINAL
INNER JOIN person_distinct_id2 AS pd FINAL ON p.id = pd.person_id AND p.team_id = pd.team_id
WHERE p.team_id = {team_id}
@@ -2270,6 +2371,7 @@ def export_persons_full_to_duckling_s3(
team_id: int,
run_id: str,
settings: dict[str, Any],
+ columns: str = PERSONS_COLUMNS,
) -> str | None:
"""Export ALL persons for a team to the duckling's S3 bucket.
@@ -2328,7 +2430,7 @@ def export_persons_full_to_duckling_s3(
)
PARTITION BY toString(cityHash64(distinct_id) % {fanout})
SELECT
- {PERSONS_COLUMNS}
+ {columns}
FROM person AS p FINAL
INNER JOIN person_distinct_id2 AS pd FINAL ON p.id = pd.person_id AND p.team_id = pd.team_id
WHERE p.team_id = {team_id}
@@ -2429,6 +2531,129 @@ def register_persons_files_with_duckling(
return len(files)
+def _merged_clickhouse_settings(context: AssetExecutionContext, config: DucklingBackfillConfig) -> dict[str, Any]:
+ merged_settings = DEFAULT_CLICKHOUSE_SETTINGS.copy()
+ merged_settings.update(settings_with_log_comment(context))
+ if config.clickhouse_settings:
+ merged_settings.update(config.clickhouse_settings)
+ context.log.info(f"Using custom ClickHouse settings: {config.clickhouse_settings}")
+ return merged_settings
+
+
+@retry(
+ stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
+ wait=wait_exponential(multiplier=1, min=4, max=60),
+ retry=retry_if_exception_type((ClickHouseError, OSError, TimeoutError)),
+ reraise=True,
+)
+def list_exported_files(client: Client, target: DucklingTarget, s3_glob: str, settings: dict[str, Any]) -> list[str]:
+ """List the files an export wrote, for a warehouse without duckgres to glob them.
+
+ ClickHouse wrote the files, and the duckling bucket policy lets it list them. The export globs have their
+ wildcard in the file name only, so every listed file name joins the directory of the glob.
+ """
+ bucket_prefix = f"s3://{target.bucket}/"
+ if not s3_glob.startswith(bucket_prefix):
+ raise ValueError(f"Export glob {s3_glob} is outside the bucket {target.bucket}")
+ directory, _, pattern = s3_glob.removeprefix(bucket_prefix).rpartition("/")
+ url = get_s3_url_for_clickhouse(target.bucket, target.bucket_region, f"{directory}/{pattern}")
+ # The One format reads no file contents, so the query only lists the objects. An export of no rows writes no
+ # files, so its glob must list none, whatever a profile or a Launchpad override sets.
+ rows = client.execute(
+ f"SELECT _file FROM s3({escape_clickhouse_string(url)}, 'One') ORDER BY _file",
+ settings={**settings, "s3_throw_on_zero_files_match": 0},
+ )
+ return [f"{bucket_prefix}{directory}/{row[0]}" for row in rows]
+
+
+class _ExportUnderPrefix(Protocol):
+ def __call__(self, *, client: Client, run_id: str) -> str | None: ...
+
+
+@frozen
+class TrinoExport:
+ """One export that the Trino path registers in one DuckLake snapshot.
+
+ `export` writes the files under the given file prefix and returns their S3 glob.
+ """
+
+ label: str
+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement
+ export: _ExportUnderPrefix
+
+
+@frozen
+class TrinoBackfillResult:
+ partitions_exported: int
+ files_registered: int
+
+
+def backfill_through_trino(
+ context: AssetExecutionContext,
+ config: DucklingBackfillConfig,
+ target: DucklingTarget,
+ *,
+ table: str,
+ column_types: Mapping[str, str],
+ exports: Sequence[TrinoExport],
+ settings: dict[str, Any],
+) -> TrinoBackfillResult:
+ """Export each item from ClickHouse and register its files through Trino, for a warehouse without duckgres.
+
+ One add_files call adds the files of an export and removes the rows they replace in the same snapshot. A retried
+ call with the same import id is a no-op, so this path needs no delete-then-register replay. A run that skips
+ registration still writes the Trino types, because add_files can register only those files later.
+ """
+ run_id = context.run.run_id[:16]
+ cluster = _get_cluster()
+ tags = dagster_tags(context)
+ workload = Workload.OFFLINE if is_cloud() else Workload.DEFAULT
+ partitions_exported = 0
+ files_registered = 0
+ register = not (config.dry_run or config.skip_ducklake_registration)
+ with connect_backfill_catalog(target.organization_id) if register else nullcontext() as catalog:
+ if catalog is not None:
+ catalog.ensure_table(
+ table,
+ column_types,
+ drop_first=config.delete_tables,
+ create_if_missing=config.create_tables_if_missing,
+ validate=not config.skip_schema_validation,
+ )
+ for item in exports:
+ # A new prefix for every export, because an export under an earlier prefix would overwrite files that
+ # DuckLake already registered with their row counts and statistics.
+ prefix = f"{run_id}_{secrets.token_hex(4)}"
+
+ def export_and_list(client: Client, item: TrinoExport = item, prefix: str = prefix) -> list[str] | None:
+ with tags_context(kind="dagster", dagster=tags):
+ s3_glob = item.export(client=client, run_id=prefix)
+ return None if s3_glob is None else list_exported_files(client, target, s3_glob, settings)
+
+ files = cluster.any_host_by_role(fn=export_and_list, workload=workload, node_role=NodeRole.DATA).result()
+ if files is None:
+ continue
+ partitions_exported += 1
+ if catalog is None:
+ context.log.info(f"Exported {len(files)} file(s) for {item.label} without registering them")
+ continue
+ replacement = item.replacement if config.cleanup_existing_partition_data else None
+ if files:
+ catalog.register_files(table, files, import_id=prefix, replacement=replacement)
+ files_registered += len(files)
+ elif replacement is not None:
+ catalog.delete_rows(table, replacement)
+ context.log.info(f"Registered {len(files)} file(s) for {item.label} in {table} through Trino")
+ logger.info(
+ "duckling_trino_files_registered",
+ team_id=target.team_id,
+ table=table,
+ partition=item.label,
+ file_count=len(files),
+ )
+ return TrinoBackfillResult(partitions_exported=partitions_exported, files_registered=files_registered)
+
+
@asset(
partitions_def=duckling_events_partitions_def,
name="duckling_events_backfill",
@@ -2473,6 +2698,87 @@ def duckling_events_backfill(context: AssetExecutionContext, config: DucklingBac
)
+def _report_events_backfill(
+ context: AssetExecutionContext,
+ config: DucklingBackfillConfig,
+ target: DucklingTarget,
+ dates: list[datetime],
+ *,
+ days_exported: int,
+ files_registered: int,
+) -> None:
+ team_id = target.team_id
+ context.add_output_metadata(
+ {
+ "team_id": team_id,
+ "partition_key": context.partition_key,
+ "dates_processed": len(dates),
+ "days_exported": days_exported,
+ "files_registered": files_registered,
+ "bucket": target.bucket,
+ }
+ )
+ if not config.dry_run:
+ record_duckling_backfill_workload(
+ team_id=team_id,
+ dataset=ManagedWarehouseBackfillPartition.Dataset.EVENTS,
+ mode="monthly" if len(dates) > 1 else "daily",
+ files_registered=files_registered,
+ partitions_exported=days_exported,
+ )
+
+ context.log.info(
+ f"Completed duckling backfill for team_id={team_id}: "
+ f"{days_exported}/{len(dates)} days exported, {files_registered} files registered"
+ )
+ logger.info(
+ "duckling_backfill_complete",
+ team_id=team_id,
+ dates_processed=len(dates),
+ days_exported=days_exported,
+ files_registered=files_registered,
+ )
+
+
+def _backfill_events_through_trino(
+ context: AssetExecutionContext, config: DucklingBackfillConfig, target: DucklingTarget, dates: list[datetime]
+) -> None:
+ trino_settings = {**_merged_clickhouse_settings(context, config), **TRINO_EXPORT_SETTINGS}
+ result = backfill_through_trino(
+ context,
+ config,
+ target,
+ table=target.events_table,
+ column_types=EVENTS_TRINO_COLUMN_TYPES,
+ settings=trino_settings,
+ exports=[
+ TrinoExport(
+ label=partition_date.strftime("%Y-%m-%d"),
+ replacement=DuckLakeDayReplacement(column="timestamp", day=partition_date.date()),
+ export=functools.partial(
+ export_events_to_duckling_s3,
+ context=context,
+ config=config,
+ target=target,
+ team_id=target.team_id,
+ date=partition_date,
+ settings=trino_settings,
+ columns=EVENTS_TRINO_COLUMNS,
+ ),
+ )
+ for partition_date in dates
+ ],
+ )
+ _report_events_backfill(
+ context,
+ config,
+ target,
+ dates,
+ days_exported=result.partitions_exported,
+ files_registered=result.files_registered,
+ )
+
+
def _run_duckling_events_backfill(context: AssetExecutionContext, config: DucklingBackfillConfig) -> None:
"""Backfill events from ClickHouse to a customer's duckling.
@@ -2511,10 +2817,15 @@ def _run_duckling_events_backfill(context: AssetExecutionContext, config: Duckli
context.log.info(f"Backfill ready for team_id={team_id}: org={target.organization_id}, bucket={target.bucket}")
+ if backfills_through_trino(target.organization_id):
+ _backfill_events_through_trino(context, config, target, dates)
+ return
+
+ should_register = not (config.dry_run or config.skip_ducklake_registration)
+
# Open one duckgres connection for all metadata operations, but skip it
# entirely when no duckgres-backed work will run (dry_run / skip_ducklake_registration).
- should_use_duckgres = not (config.dry_run or config.skip_ducklake_registration)
- session = _DuckgresSession(context, target) if should_use_duckgres else None
+ session = _DuckgresSession(context, target) if should_register else None
try:
if session is not None:
# Delete events table if requested (dangerous - loses all data)
@@ -2545,11 +2856,7 @@ def _run_duckling_events_backfill(context: AssetExecutionContext, config: Duckli
session.run("validate events schema", lambda c: validate_duckling_schema(context, target, c))
# Prepare ClickHouse settings
- merged_settings = DEFAULT_CLICKHOUSE_SETTINGS.copy()
- merged_settings.update(settings_with_log_comment(context))
- if config.clickhouse_settings:
- merged_settings.update(config.clickhouse_settings)
- context.log.info(f"Using custom ClickHouse settings: {config.clickhouse_settings}")
+ merged_settings = _merged_clickhouse_settings(context, config)
cluster = _get_cluster()
tags = dagster_tags(context)
@@ -2612,35 +2919,8 @@ def _run_duckling_events_backfill(context: AssetExecutionContext, config: Duckli
total_registered += session.run(f"register events files {date_str}", register_events_files)
- context.add_output_metadata(
- {
- "team_id": team_id,
- "partition_key": context.partition_key,
- "dates_processed": len(dates),
- "days_exported": days_exported,
- "files_registered": total_registered,
- "bucket": target.bucket,
- }
- )
- if not config.dry_run:
- record_duckling_backfill_workload(
- team_id=team_id,
- dataset=ManagedWarehouseBackfillPartition.Dataset.EVENTS,
- mode="monthly" if len(dates) > 1 else "daily",
- files_registered=total_registered,
- partitions_exported=days_exported,
- )
-
- context.log.info(
- f"Completed duckling backfill for team_id={team_id}: "
- f"{days_exported}/{len(dates)} days exported, {total_registered} files registered"
- )
- logger.info(
- "duckling_backfill_complete",
- team_id=team_id,
- dates_processed=len(dates),
- days_exported=days_exported,
- files_registered=total_registered,
+ _report_events_backfill(
+ context, config, target, dates, days_exported=days_exported, files_registered=total_registered
)
finally:
@@ -2695,6 +2975,123 @@ def duckling_persons_backfill(context: AssetExecutionContext, config: DucklingBa
)
+def _report_persons_backfill(
+ context: AssetExecutionContext,
+ config: DucklingBackfillConfig,
+ target: DucklingTarget,
+ dates: list[datetime] | None,
+ *,
+ partitions_exported: int,
+ files_registered: int,
+) -> None:
+ """Record the outcome of a persons run. `dates` is None for a full export."""
+ team_id = target.team_id
+ export_mode = "full" if dates is None else "daily"
+ metadata: dict[str, Any] = {
+ "team_id": team_id,
+ "partition_key": context.partition_key,
+ "export_mode": export_mode,
+ }
+ if dates is not None:
+ metadata.update(dates_processed=len(dates), days_exported=partitions_exported)
+ metadata.update(files_registered=files_registered, bucket=target.bucket)
+ context.add_output_metadata(metadata)
+ if not config.dry_run:
+ record_duckling_backfill_workload(
+ team_id=team_id,
+ dataset=ManagedWarehouseBackfillPartition.Dataset.PERSONS,
+ mode=export_mode,
+ files_registered=files_registered,
+ partitions_exported=partitions_exported,
+ )
+
+ if dates is None:
+ context.log.info(
+ f"Completed duckling persons full backfill for team_id={team_id}: {files_registered} files registered"
+ )
+ logger.info(
+ "duckling_persons_backfill_complete",
+ team_id=team_id,
+ export_mode="full",
+ files_registered=files_registered,
+ )
+ else:
+ context.log.info(
+ f"Completed duckling persons daily backfill for team_id={team_id}: "
+ f"{partitions_exported}/{len(dates)} days exported, {files_registered} files registered"
+ )
+ logger.info(
+ "duckling_persons_backfill_complete",
+ team_id=team_id,
+ export_mode="daily",
+ dates_processed=len(dates),
+ days_exported=partitions_exported,
+ files_registered=files_registered,
+ )
+
+
+def _backfill_persons_through_trino(
+ context: AssetExecutionContext,
+ config: DucklingBackfillConfig,
+ target: DucklingTarget,
+ dates: list[datetime] | None,
+) -> None:
+ """Export persons and register them through Trino. `dates` is None for a full export."""
+ trino_settings = {**_merged_clickhouse_settings(context, config), **TRINO_EXPORT_SETTINGS}
+ exports = (
+ [
+ TrinoExport(
+ label="full",
+ replacement=DuckLakeTableReplacement(),
+ export=functools.partial(
+ export_persons_full_to_duckling_s3,
+ context=context,
+ config=config,
+ target=target,
+ team_id=target.team_id,
+ settings=trino_settings,
+ columns=PERSONS_TRINO_COLUMNS,
+ ),
+ )
+ ]
+ if dates is None
+ else [
+ TrinoExport(
+ label=partition_date.strftime("%Y-%m-%d"),
+ replacement=DuckLakeDayReplacement(column="_timestamp", day=partition_date.date()),
+ export=functools.partial(
+ export_persons_to_duckling_s3,
+ context=context,
+ config=config,
+ target=target,
+ team_id=target.team_id,
+ date=partition_date,
+ settings=trino_settings,
+ columns=PERSONS_TRINO_COLUMNS,
+ ),
+ )
+ for partition_date in dates
+ ]
+ )
+ result = backfill_through_trino(
+ context,
+ config,
+ target,
+ table=target.persons_table,
+ column_types=PERSONS_TRINO_COLUMN_TYPES,
+ settings=trino_settings,
+ exports=exports,
+ )
+ _report_persons_backfill(
+ context,
+ config,
+ target,
+ dates,
+ partitions_exported=result.partitions_exported,
+ files_registered=result.files_registered,
+ )
+
+
def _run_duckling_persons_backfill(context: AssetExecutionContext, config: DucklingBackfillConfig) -> None:
"""Backfill persons from ClickHouse to a customer's duckling.
@@ -2742,10 +3139,15 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Duckl
context.log.info(f"Backfill ready for team_id={team_id}: org={target.organization_id}, bucket={target.bucket}")
+ if backfills_through_trino(target.organization_id):
+ _backfill_persons_through_trino(context, config, target, None if is_full else dates)
+ return
+
+ should_register = not (config.dry_run or config.skip_ducklake_registration)
+
# Open one duckgres connection for all metadata operations, but skip it
# entirely when no duckgres-backed work will run (dry_run / skip_ducklake_registration).
- should_use_duckgres = not (config.dry_run or config.skip_ducklake_registration)
- session = _DuckgresSession(context, target) if should_use_duckgres else None
+ session = _DuckgresSession(context, target) if should_register else None
try:
if session is not None:
# Delete persons table if requested (dangerous - loses all data)
@@ -2774,11 +3176,7 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Duckl
context.log.info("Validating duckling persons schema compatibility...")
session.run("validate persons schema", lambda c: validate_duckling_persons_schema(context, target, c))
- merged_settings = DEFAULT_CLICKHOUSE_SETTINGS.copy()
- merged_settings.update(settings_with_log_comment(context))
- if config.clickhouse_settings:
- merged_settings.update(config.clickhouse_settings)
- context.log.info(f"Using custom ClickHouse settings: {config.clickhouse_settings}")
+ merged_settings = _merged_clickhouse_settings(context, config)
cluster = _get_cluster()
tags = dagster_tags(context)
@@ -2826,32 +3224,8 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Duckl
files_registered = session.run("register persons files (full)", register_full_persons_files)
- context.add_output_metadata(
- {
- "team_id": team_id,
- "partition_key": partition_key,
- "export_mode": "full",
- "files_registered": files_registered,
- "bucket": target.bucket,
- }
- )
- if not config.dry_run:
- record_duckling_backfill_workload(
- team_id=team_id,
- dataset=ManagedWarehouseBackfillPartition.Dataset.PERSONS,
- mode="full",
- files_registered=files_registered,
- partitions_exported=int(bool(s3_glob)),
- )
-
- context.log.info(
- f"Completed duckling persons full backfill for team_id={team_id}: {files_registered} files registered"
- )
- logger.info(
- "duckling_persons_backfill_complete",
- team_id=team_id,
- export_mode="full",
- files_registered=files_registered,
+ _report_persons_backfill(
+ context, config, target, None, partitions_exported=int(bool(s3_glob)), files_registered=files_registered
)
else:
# DAILY EXPORT MODE - process each date in the partition
@@ -2910,37 +3284,8 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Duckl
total_registered += session.run(f"register persons files {date_str}", register_persons_files)
- context.add_output_metadata(
- {
- "team_id": team_id,
- "partition_key": partition_key,
- "export_mode": "daily",
- "dates_processed": len(dates),
- "days_exported": days_exported,
- "files_registered": total_registered,
- "bucket": target.bucket,
- }
- )
- if not config.dry_run:
- record_duckling_backfill_workload(
- team_id=team_id,
- dataset=ManagedWarehouseBackfillPartition.Dataset.PERSONS,
- mode="daily",
- files_registered=total_registered,
- partitions_exported=days_exported,
- )
-
- context.log.info(
- f"Completed duckling persons daily backfill for team_id={team_id}: "
- f"{days_exported}/{len(dates)} days exported, {total_registered} files registered"
- )
- logger.info(
- "duckling_persons_backfill_complete",
- team_id=team_id,
- export_mode="daily",
- dates_processed=len(dates),
- days_exported=days_exported,
- files_registered=total_registered,
+ _report_persons_backfill(
+ context, config, target, dates, partitions_exported=days_exported, files_registered=total_registered
)
finally:
diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.py
index a180ccce4698..baae112a0c68 100644
--- a/posthog/dags/test_events_backfill_to_duckling.py
+++ b/posthog/dags/test_events_backfill_to_duckling.py
@@ -1,14 +1,19 @@
import os
+import re
from datetime import UTC, date, datetime, timedelta
+from fnmatch import fnmatch
from typing import Any, cast
import pytest
+import time_machine
from unittest.mock import MagicMock, call, patch
import duckdb
import psycopg
+from clickhouse_driver.errors import Error as ClickHouseError
from parameterized import parameterized
+from posthog.clickhouse.client.connection import get_client_from_pool
from posthog.dags.common import JobOwners
from posthog.dags.events_backfill_to_duckling import (
_DUCKLAKE_FILE_PARTITION_VALUE_FIXUP_ENV_VAR,
@@ -17,12 +22,16 @@ from posthog.dags.events_backfill_to_duckling import (
EVENTS_COLUMNS,
EVENTS_CONCURRENCY_TAG,
EVENTS_TABLE_DDL,
+ EVENTS_TRINO_COLUMN_TYPES,
+ EVENTS_TRINO_COLUMNS,
EXPECTED_DUCKLAKE_EVENTS_COLUMNS,
EXPECTED_DUCKLAKE_PERSONS_COLUMNS,
MAX_S3_FILE_FANOUT,
PERSONS_COLUMNS,
PERSONS_CONCURRENCY_TAG,
PERSONS_TABLE_DDL,
+ PERSONS_TRINO_COLUMN_TYPES,
+ PERSONS_TRINO_COLUMNS,
TARGET_ROWS_PER_FILE,
DucklingBackfillConfig,
DucklingTarget,
@@ -41,6 +50,7 @@ from posthog.dags.events_backfill_to_duckling import (
_resolve_duckling_target,
_resolve_table_names,
_run_duckling_events_backfill,
+ _run_duckling_persons_backfill,
_set_table_partitioning,
_stale_run_outcome,
_validate_identifier,
@@ -65,6 +75,8 @@ from products.data_warehouse.backend.facade.backfill_status import BackfillOutco
from products.managed_warehouse.backend.facade.api import EARLIEST_BACKFILL_DATE
from products.managed_warehouse.backend.facade.client import ServiceCredential, ServiceCredentialUnavailable
from products.managed_warehouse.backend.facade.contracts import (
+ DuckLakeDayReplacement,
+ DuckLakeTableReplacement,
DucklingTables,
ManagedWarehouseTableNames,
ManagedWarehouseTeamMembership,
@@ -2611,3 +2623,234 @@ class TestFannedOutLayoutRoundTrip:
# The missing column is NULL-filled for every backfilled row.
cap_vals = [r[0] for r in conn.execute(f"SELECT captured_at FROM {DUCKLAKE_ALIAS}.posthog.events").fetchall()]
assert cap_vals == [None] * 5
+
+
+class _FakeTrinoBackfillCatalog:
+ def __init__(self) -> None:
+ self.tables: list[tuple[str, dict[str, str]]] = []
+ self.registered: list[tuple[str, list[str], str, DuckLakeDayReplacement | DuckLakeTableReplacement | None]] = []
+ self.deleted: list[tuple[str, DuckLakeDayReplacement | DuckLakeTableReplacement]] = []
+
+ def ensure_table(
+ self, table: str, columns: dict[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool
+ ) -> None:
+ self.tables.append((table, dict(columns)))
+
+ def register_files(
+ self,
+ table: str,
+ files: list[str],
+ *,
+ import_id: str,
+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,
+ ) -> int:
+ self.registered.append((table, list(files), import_id, replacement))
+ return len(files)
+
+ def delete_rows(self, table: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement) -> None:
+ self.deleted.append((table, replacement))
+
+
+class _FakeExportClickHouse:
+ def __init__(self, files_per_export: int) -> None:
+ self.files_per_export = files_per_export
+ self.inserts: list[tuple[str, dict[str, Any]]] = []
+ self.objects: list[str] = []
+
+ def execute(
+ self, query: str, params: object = None, settings: dict[str, Any] | None = None
+ ) -> list[tuple[Any, ...]]:
+ if "INSERT INTO FUNCTION" in query:
+ self.inserts.append((query, settings or {}))
+ url = re.search(r"s3\(\s*'([^']+)'", query)
+ assert url is not None
+ self.objects += [url[1].replace("{_partition_id}", str(index)) for index in range(self.files_per_export)]
+ return []
+ if "'One'" in query:
+ pattern = re.search(r"s3\('([^']+)'", query)
+ assert pattern is not None
+ matched = [(url.rsplit("/", 1)[1],) for url in sorted(self.objects) if fnmatch(url, pattern[1])]
+ if not matched and (settings or {}).get("s3_throw_on_zero_files_match"):
+ raise ClickHouseError(f"Can not match any files with path {pattern[1]}")
+ return matched
+ return [(10,)]
+
+
+class TestBackfillThroughTrino:
+ TARGET = DucklingTarget(
+ team_id=2,
+ organization_id="org-1",
+ bucket="bkt",
+ bucket_region="us-east-1",
+ events_table="events_team_2",
+ persons_table="persons_team_2",
+ )
+
+ def _run(self, run, partition_key: str, clickhouse: _FakeExportClickHouse, config: DucklingBackfillConfig):
+ catalog = _FakeTrinoBackfillCatalog()
+ cluster = MagicMock()
+ cluster.any_host_by_role.side_effect = lambda fn, **_: MagicMock(result=MagicMock(return_value=fn(clickhouse)))
+ context = MagicMock(partition_key=partition_key)
+ context.run.run_id = "0123456789abcdef-run"
+ with (
+ patch("posthog.dags.events_backfill_to_duckling._resolve_duckling_target", return_value=self.TARGET),
+ patch("posthog.dags.events_backfill_to_duckling.backfills_through_trino", return_value=True),
+ patch("posthog.dags.events_backfill_to_duckling.connect_backfill_catalog") as connect_catalog,
+ patch("posthog.dags.events_backfill_to_duckling._get_cluster", return_value=cluster),
+ patch("posthog.dags.events_backfill_to_duckling._DuckgresSession") as duckgres,
+ patch("posthog.dags.events_backfill_to_duckling.record_duckling_backfill_workload"),
+ patch("tenacity.nap.time.sleep"),
+ ):
+ connect_catalog.return_value.__enter__.return_value = catalog
+ run(context, config)
+ run(context, config)
+ duckgres.assert_not_called()
+ return catalog, connect_catalog
+
+ @parameterized.expand(
+ [
+ (
+ "events_day",
+ _run_duckling_events_backfill,
+ "2_2026-06-17",
+ "events_team_2",
+ EVENTS_TRINO_COLUMN_TYPES,
+ DuckLakeDayReplacement(column="timestamp", day=date(2026, 6, 17)),
+ "toString(person_mode) as person_mode",
+ ),
+ (
+ "persons_day",
+ _run_duckling_persons_backfill,
+ "2_2026-06-17",
+ "persons_team_2",
+ PERSONS_TRINO_COLUMN_TYPES,
+ DuckLakeDayReplacement(column="_timestamp", day=date(2026, 6, 17)),
+ "toInt64(p.version) AS person_version",
+ ),
+ (
+ "persons_full",
+ _run_duckling_persons_backfill,
+ "2",
+ "persons_team_2",
+ PERSONS_TRINO_COLUMN_TYPES,
+ DuckLakeTableReplacement(),
+ "toInt64(p.version) AS person_version",
+ ),
+ ]
+ )
+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)
+ def test_registers_each_export_in_one_snapshot_under_a_new_prefix(
+ self, _name, run, partition_key, table, column_types, replacement, cast_column
+ ):
+ clickhouse = _FakeExportClickHouse(files_per_export=2)
+
+ catalog, connect_catalog = self._run(run, partition_key, clickhouse, DucklingBackfillConfig())
+
+ connect_catalog.assert_called_with("org-1")
+ assert catalog.tables == [(table, column_types)] * 2
+ assert [(t, r) for t, _, _, r in catalog.registered] == [(table, replacement)] * 2
+ first, second = catalog.registered
+ assert len(first[1]) == 2 and all(path.startswith("s3://bkt/backfill/") for path in first[1])
+ assert all(f"/{first[2]}_" in path for path in first[1])
+ assert first[2] != second[2] and not set(first[1]) & set(second[1])
+ for insert, settings in clickhouse.inserts:
+ assert cast_column in insert
+ assert settings["output_format_parquet_string_as_string"] == 1
+
+ @parameterized.expand(
+ [
+ ("empty_day_deletes_its_rows", 0, True),
+ ("append_keeps_existing_rows", 2, False),
+ ("empty_append_changes_nothing", 0, False),
+ ]
+ )
+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)
+ def test_replaces_the_day_only_when_cleanup_is_on(self, _name, files_per_export, cleanup):
+ clickhouse = _FakeExportClickHouse(files_per_export=files_per_export)
+ day = DuckLakeDayReplacement(column="timestamp", day=date(2026, 6, 17))
+
+ catalog, _ = self._run(
+ _run_duckling_events_backfill,
+ "2_2026-06-17",
+ clickhouse,
+ # A server profile or a Launchpad override can make ClickHouse throw for a glob that matches no files.
+ DucklingBackfillConfig(
+ cleanup_existing_partition_data=cleanup, clickhouse_settings={"s3_throw_on_zero_files_match": 1}
+ ),
+ )
+
+ assert catalog.deleted == ([("events_team_2", day)] * 2 if cleanup and not files_per_export else [])
+ assert [r for _, _, _, r in catalog.registered] == ([None] * 2 if files_per_export else [])
+
+ @parameterized.expand(
+ [
+ ("events_day", _run_duckling_events_backfill, "2_2026-06-17", "toString(person_mode) as person_mode"),
+ ("persons_day", _run_duckling_persons_backfill, "2_2026-06-17", "toInt64(p.version) AS person_version"),
+ ("persons_full", _run_duckling_persons_backfill, "2", "toInt64(p.version) AS person_version"),
+ ]
+ )
+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)
+ def test_export_only_runs_write_the_trino_types_without_registering(self, _name, run, partition_key, cast_column):
+ clickhouse = _FakeExportClickHouse(files_per_export=2)
+
+ catalog, connect_catalog = self._run(
+ run, partition_key, clickhouse, DucklingBackfillConfig(skip_ducklake_registration=True)
+ )
+
+ connect_catalog.assert_not_called()
+ assert catalog.registered == []
+ assert len(clickhouse.inserts) == 2
+ for insert, settings in clickhouse.inserts:
+ assert cast_column in insert
+ assert settings["output_format_parquet_string_as_string"] == 1
+
+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)
+ def test_dry_runs_neither_export_nor_open_the_catalog(self):
+ clickhouse = _FakeExportClickHouse(files_per_export=2)
+
+ _, connect_catalog = self._run(
+ _run_duckling_events_backfill, "2_2026-06-17", clickhouse, DucklingBackfillConfig(dry_run=True)
+ )
+
+ connect_catalog.assert_not_called()
+ assert clickhouse.inserts == []
+
+
+# The Trino column type that DuckLake reads back from the Parquet type ClickHouse writes for a ClickHouse type.
+TRINO_TYPE_OF_CLICKHOUSE_TYPE = {
+ "String": "varchar",
+ "Int64": "bigint",
+ "Int16": "smallint",
+ "Bool": "boolean",
+ "DateTime64(6)": "timestamp(6) with time zone",
+ "DateTime64(6, 'UTC')": "timestamp(6) with time zone",
+}
+EVENTS_SOURCE = (
+ "generateRandom('uuid UUID, event String, properties String, timestamp DateTime64(6, ''UTC''), team_id Int64, "
+ "distinct_id String, elements_chain String, created_at DateTime64(6, ''UTC''), person_id UUID, "
+ "person_created_at DateTime64(3), person_properties String, group0_properties String, group1_properties String, "
+ "group2_properties String, group3_properties String, group4_properties String, group0_created_at DateTime64(3), "
+ "group1_created_at DateTime64(3), group2_created_at DateTime64(3), group3_created_at DateTime64(3), "
+ "group4_created_at DateTime64(3), person_mode Enum8(''full'' = 0, ''propertyless'' = 1, ''force_upgrade'' = 2), "
+ "historical_migration Bool')"
+)
+PERSONS_SOURCE = (
+ "generateRandom('id UUID, created_at DateTime64(3), team_id Int64, properties String, is_identified Int8, "
+ "version UInt64, _timestamp DateTime') AS p "
+ "CROSS JOIN generateRandom('team_id Int64, distinct_id String, person_id UUID, version Int64') AS pd"
+)
+
+
+class TestTrinoExportColumnTypes:
+ @parameterized.expand(
+ [
+ ("events", f"SELECT {EVENTS_TRINO_COLUMNS} FROM {EVENTS_SOURCE}", EVENTS_TRINO_COLUMN_TYPES),
+ ("persons", f"SELECT {PERSONS_TRINO_COLUMNS} FROM {PERSONS_SOURCE}", PERSONS_TRINO_COLUMN_TYPES),
+ ]
+ )
+ def test_export_writes_the_types_of_the_trino_table(self, _name, select, column_types):
+ with get_client_from_pool() as client:
+ described = client.execute(f"DESCRIBE ({select})")
+
+ exported = {str(row[0]): TRINO_TYPE_OF_CLICKHOUSE_TYPE.get(str(row[1]), str(row[1])) for row in described}
+ assert exported == column_types
diff --git a/products/managed_warehouse/backend/backfill_trino.py b/products/managed_warehouse/backend/backfill_trino.py
new file mode 100644
index 000000000000..7488bc59149d
--- /dev/null
+++ b/products/managed_warehouse/backend/backfill_trino.py
@@ -0,0 +1,207 @@
+"""Register duckling backfill exports in DuckLake through the organization's Trino catalog.
+
+The Dagster backfills use this for warehouses that have no duckgres. One `add_files` call registers the files of an
+export and removes the rows they replace in a single DuckLake snapshot, so readers never see a day without rows or
+with two copies of it. The call refuses any type conversion, so the export must write the exact Parquet types that
+DuckLake writes for each Trino column.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Iterator, Mapping, Sequence
+from datetime import date, timedelta
+from typing import TYPE_CHECKING, Any
+
+import structlog
+from tenacity import RetryCallState, retry, retry_if_exception, stop_after_attempt, wait_exponential
+from trino.exceptions import Http502Error, Http503Error, Http504Error, TrinoConnectionError
+
+from posthog.hogql.escape_sql import escape_trino_identifier, escape_trino_string
+
+from posthog.exceptions_capture import capture_exception
+from posthog.ph_client import feature_enabled_or_false
+
+from products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacement
+from products.managed_warehouse.backend.facade.feature_flags import DUCKLAKE_TRINO_REGISTRATION_FLAG
+from products.managed_warehouse.backend.trino_connection import connect_managed_warehouse_trino
+from products.managed_warehouse.backend.trino_execution import is_transient_trino_error
+
+if TYPE_CHECKING:
+ from trino.dbapi import Connection
+
+LOGGER = structlog.get_logger(__name__)
+
+BACKFILL_SCHEMA = "posthog"
+BACKFILL_TRINO_PRINCIPAL = "dagster:duckling-backfill"
+# One call is one snapshot. The connector accepts more files, but Trino rejects a statement longer than
+# 1,000,000 characters, and this many export paths stay well below that.
+ADD_FILES_MAX_FILES = 2_000
+# A replacement removes rows by time or by table, never by team. In a table that several teams share, it would also
+# remove the rows of the other teams.
+SHARED_TABLES = frozenset({"events", "persons", "groups"})
+# DuckLake asks for the statement to be run again when another writer changed the table while it was planned or
+# committed.
+TRANSIENT_DUCKLAKE_ERROR_NAMES = frozenset(
+ {"DUCKLAKE_COMMIT_CONFLICT", "DUCKLAKE_CONCURRENT_MODIFICATION", "DUCKLAKE_COMMIT_FAILED"}
+)
+STATEMENT_ATTEMPTS = 4
+
+
+Replacement = DuckLakeDayReplacement | DuckLakeTableReplacement
+
+
+def backfills_through_trino(organization_id: str) -> bool:
+ try:
+ return feature_enabled_or_false(
+ DUCKLAKE_TRINO_REGISTRATION_FLAG,
+ organization_id,
+ groups={"organization": organization_id},
+ group_properties={"organization": {"id": organization_id}},
+ only_evaluate_locally=True,
+ send_feature_flag_events=False,
+ )
+ except Exception as error:
+ LOGGER.warning("Failed to evaluate the DuckLake Trino registration feature flag", error=str(error))
+ capture_exception(error)
+ return False
+
+
+def _timestamp_bound(day: date) -> str:
+ # The connector reads a TIMESTAMP WITH TIME ZONE bound as an instant with an explicit offset.
+ return f"{day.isoformat()} 00:00:00+00"
+
+
+def _refuse_shared_table(table: str, change: str) -> None:
+ if table in SHARED_TABLES:
+ raise ValueError(
+ f"Refusing to {change} the shared table '{table}' through Trino, because that would also remove the rows "
+ "of other teams"
+ )
+
+
+def _is_transient(error: BaseException) -> bool:
+ # The client already retried each HTTP request, so these mean that the coordinator or the gateway went away.
+ if isinstance(error, TrinoConnectionError | Http502Error | Http503Error | Http504Error):
+ return True
+ return is_transient_trino_error(error) or getattr(error, "error_name", None) in TRANSIENT_DUCKLAKE_ERROR_NAMES
+
+
+def _log_retry(state: RetryCallState) -> None:
+ error = state.outcome.exception() if state.outcome is not None else None
+ LOGGER.warning(
+ "A DuckLake backfill statement failed for a transient reason, retrying it",
+ attempt=state.attempt_number,
+ error_name=getattr(error, "error_name", None),
+ error=str(error),
+ )
+
+
+class TrinoBackfillCatalog:
+ """The DuckLake statements of one backfill, run in the organization's Trino catalog."""
+
+ def __init__(self, connection: Connection) -> None:
+ self._connection = connection
+ self._catalog = escape_trino_identifier(connection.catalog)
+
+ @classmethod
+ @contextlib.contextmanager
+ def connect(cls, organization_id: str) -> Iterator[TrinoBackfillCatalog]:
+ with connect_managed_warehouse_trino(organization_id, principal=BACKFILL_TRINO_PRINCIPAL) as connection:
+ yield cls(connection)
+
+ def _table(self, table: str) -> str:
+ return f"{self._catalog}.{escape_trino_identifier(BACKFILL_SCHEMA)}.{escape_trino_identifier(table)}"
+
+ @retry(
+ retry=retry_if_exception(_is_transient),
+ stop=stop_after_attempt(STATEMENT_ATTEMPTS),
+ wait=wait_exponential(multiplier=1, min=4, max=60),
+ before_sleep=_log_retry,
+ reraise=True,
+ )
+ def _run(self, statement: str) -> list[list[Any]]:
+ # Every statement here can run again. DDL has IF [NOT] EXISTS, deleting a range twice deletes nothing more,
+ # and add_files records its import_id, so a repeat of an applied import adds nothing.
+ cursor = self._connection.cursor()
+ try:
+ cursor.execute(statement)
+ return cursor.fetchall()
+ finally:
+ cursor.close()
+
+ def ensure_table(
+ self, table: str, columns: Mapping[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool
+ ) -> None:
+ if drop_first:
+ _refuse_shared_table(table, "drop")
+ self._run(f"DROP TABLE IF EXISTS {self._table(table)}")
+ if create_if_missing:
+ self._run(f"CREATE SCHEMA IF NOT EXISTS {self._catalog}.{escape_trino_identifier(BACKFILL_SCHEMA)}")
+ definitions = ", ".join(
+ f"{escape_trino_identifier(name)} {data_type}" for name, data_type in columns.items()
+ )
+ self._run(f"CREATE TABLE IF NOT EXISTS {self._table(table)} ({definitions})")
+ if validate:
+ self._validate(table, columns)
+
+ def _validate(self, table: str, columns: Mapping[str, str]) -> None:
+ # A table can have more columns than the export, because add_files reads a missing column as NULL.
+ rows = self._run(
+ f"SELECT column_name, data_type FROM {self._catalog}.information_schema.columns "
+ f"WHERE table_schema = {escape_trino_string(BACKFILL_SCHEMA)} AND table_name = {escape_trino_string(table)}"
+ )
+ actual = {str(name): str(data_type) for name, data_type in rows}
+ mismatched = {
+ name: actual.get(name, "missing") for name, data_type in columns.items() if actual.get(name) != data_type
+ }
+ if mismatched:
+ raise ValueError(
+ f"Table {BACKFILL_SCHEMA}.{table} does not match the backfill export, so add_files would refuse its "
+ f"files. Expected {dict(columns)}, but these columns differ: {mismatched}"
+ )
+
+ def register_files(
+ self, table: str, files: Sequence[str], *, import_id: str, replacement: Replacement | None
+ ) -> int:
+ """Add the files and remove the rows they replace in one snapshot.
+
+ Repeating a call with the same import_id and files is a no-op, so a call whose response was lost can be
+ repeated.
+ """
+ if replacement is not None:
+ _refuse_shared_table(table, "replace rows in")
+ if len(files) > ADD_FILES_MAX_FILES:
+ raise ValueError(
+ f"The export wrote {len(files)} files, but one add_files call registers at most "
+ f"{ADD_FILES_MAX_FILES}. Lower max_s3_file_fanout."
+ )
+ options = [
+ f"files => ARRAY[{', '.join(escape_trino_string(path) for path in files)}]",
+ f"import_id => {escape_trino_string(import_id)}",
+ "allow_missing => true",
+ ]
+ match replacement:
+ case DuckLakeDayReplacement(column=column, day=day):
+ options += [
+ f"replace_range_column => {escape_trino_string(column)}",
+ f"replace_range_start => {escape_trino_string(_timestamp_bound(day))}",
+ f"replace_range_end => {escape_trino_string(_timestamp_bound(day + timedelta(days=1)))}",
+ ]
+ case DuckLakeTableReplacement():
+ options.append("replace => true")
+ rows = self._run(f"ALTER TABLE {self._table(table)} EXECUTE add_files({', '.join(options)})")
+ return sum(int(row[1]) for row in rows if len(row) > 1 and row[0] == "added_data_files")
+
+ def delete_rows(self, table: str, replacement: Replacement) -> None:
+ """Remove the rows an export replaces when the export wrote no files."""
+ _refuse_shared_table(table, "replace rows in")
+ statement = f"DELETE FROM {self._table(table)}"
+ if isinstance(replacement, DuckLakeDayReplacement):
+ column = escape_trino_identifier(replacement.column)
+ start = replacement.day.isoformat()
+ end = (replacement.day + timedelta(days=1)).isoformat()
+ statement += (
+ f" WHERE {column} >= TIMESTAMP '{start} 00:00:00 UTC' AND {column} < TIMESTAMP '{end} 00:00:00 UTC'"
+ )
+ self._run(statement)
diff --git a/products/managed_warehouse/backend/facade/backfill_trino.py b/products/managed_warehouse/backend/facade/backfill_trino.py
new file mode 100644
index 000000000000..b74d5c9fa230
--- /dev/null
+++ b/products/managed_warehouse/backend/facade/backfill_trino.py
@@ -0,0 +1,18 @@
+"""Register duckling backfill exports in DuckLake through Trino, for warehouses without duckgres."""
+
+from __future__ import annotations
+
+from contextlib import AbstractContextManager
+
+from products.managed_warehouse.backend import backfill_trino
+from products.managed_warehouse.backend.facade.contracts import DuckLakeBackfillCatalog
+
+__all__ = ["backfills_through_trino", "connect_backfill_catalog"]
+
+
+def backfills_through_trino(organization_id: str) -> bool:
+ return backfill_trino.backfills_through_trino(organization_id)
+
+
+def connect_backfill_catalog(organization_id: str) -> AbstractContextManager[DuckLakeBackfillCatalog]:
+ return backfill_trino.TrinoBackfillCatalog.connect(organization_id)
diff --git a/products/managed_warehouse/backend/facade/contracts.py b/products/managed_warehouse/backend/facade/contracts.py
index 466ff42fa65d..6db19bd36917 100644
--- a/products/managed_warehouse/backend/facade/contracts.py
+++ b/products/managed_warehouse/backend/facade/contracts.py
@@ -12,16 +12,20 @@ behavior change, not a contract improvement.
from __future__ import annotations
+from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from datetime import date, datetime
from enum import StrEnum
-from typing import Any, Literal
+from typing import Any, Literal, Protocol
from uuid import UUID
from posthog.dataclasses import frozen
__all__ = [
"CPUnavailableError",
+ "DuckLakeBackfillCatalog",
+ "DuckLakeDayReplacement",
+ "DuckLakeTableReplacement",
"DuckgresQueryServerConfig",
"DuckgresStoredBucketConfig",
"DuckgresStoredServerConfig",
@@ -428,3 +432,35 @@ class DuckLakeTableResult:
# Highest incremental key written, for builds that track one. None when no rows were written.
watermark: Any = None
merged: bool = False
+
+
+@frozen
+class DuckLakeDayReplacement:
+ """The rows of one UTC day of `column`, which a backfill export of that day replaces."""
+
+ column: str
+ day: date
+
+
+@frozen
+class DuckLakeTableReplacement:
+ """All rows of the table, which a full backfill export replaces."""
+
+
+class DuckLakeBackfillCatalog(Protocol):
+ """The DuckLake statements a backfill runs in the organization's Trino catalog."""
+
+ def ensure_table(
+ self, table: str, columns: Mapping[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool
+ ) -> None: ...
+
+ def register_files(
+ self,
+ table: str,
+ files: Sequence[str],
+ *,
+ import_id: str,
+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,
+ ) -> int: ...
+
+ def delete_rows(self, table: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement) -> None: ...
diff --git a/products/managed_warehouse/backend/facade/feature_flags.py b/products/managed_warehouse/backend/facade/feature_flags.py
index cf1004202adb..e8980d00d8ed 100644
--- a/products/managed_warehouse/backend/facade/feature_flags.py
+++ b/products/managed_warehouse/backend/facade/feature_flags.py
@@ -3,3 +3,5 @@ MANAGED_TRINO_QUERY_FLAG = "managed-warehouse-trino-query"
# Either flag gives an organization the managed warehouse.
MANAGED_WAREHOUSE_ACCESS_FLAGS = (MANAGED_TRINO_QUERY_FLAG, DATA_WAREHOUSE_SCENE_FLAG)
DATA_MODELING_SHADOW_FLAG = "managed-warehouse-data-modeling-shadow"
+# Marks an organization whose warehouse has no duckgres, so its data imports and backfills register through Trino.
+DUCKLAKE_TRINO_REGISTRATION_FLAG = "managed-warehouse-trino-import-registration"
diff --git a/products/managed_warehouse/backend/tests/test_backfill_trino.py b/products/managed_warehouse/backend/tests/test_backfill_trino.py
new file mode 100644
index 000000000000..4768ee0e6066
--- /dev/null
+++ b/products/managed_warehouse/backend/tests/test_backfill_trino.py
@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+from datetime import date
+from typing import TYPE_CHECKING, cast
+
+import pytest
+from unittest.mock import patch
+
+from parameterized import parameterized
+from trino.exceptions import Http503Error, TrinoConnectionError, TrinoExternalError, TrinoQueryError, TrinoUserError
+
+from products.managed_warehouse.backend.backfill_trino import ADD_FILES_MAX_FILES, TrinoBackfillCatalog
+from products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacement
+
+if TYPE_CHECKING:
+ from trino.dbapi import Connection
+
+COLUMNS = {"team_id": "bigint", "timestamp": "timestamp(6) with time zone", "person_version": "bigint"}
+
+
+class _Cursor:
+ def __init__(self, connection: _FakeTrino) -> None:
+ self._connection = connection
+ self._rows: list[list[object]] = []
+
+ def execute(self, statement: str) -> None:
+ self._connection.statements.append(statement)
+ if self._connection.failures:
+ raise self._connection.failures.pop(0)
+ self._rows = self._connection.rows_for(statement)
+
+ def fetchall(self) -> list[list[object]]:
+ return self._rows
+
+ def close(self) -> None:
+ pass
+
+
+class _FakeTrino:
+ catalog = "warehouse"
+
+ def __init__(self, columns: dict[str, str] | None = None, failures: list[Exception] | None = None) -> None:
+ self.statements: list[str] = []
+ self.failures = failures or []
+ self._columns = columns or {}
+
+ def cursor(self) -> _Cursor:
+ return _Cursor(self)
+
+ def rows_for(self, statement: str) -> list[list[object]]:
+ if "information_schema.columns" in statement:
+ return [[name, data_type] for name, data_type in self._columns.items()]
+ if "EXECUTE add_files" in statement:
+ return [["added_data_files", 2]]
+ return []
+
+
+def _catalog(
+ columns: dict[str, str] | None = None, failures: list[Exception] | None = None
+) -> tuple[TrinoBackfillCatalog, _FakeTrino]:
+ connection = _FakeTrino(columns, failures)
+ return TrinoBackfillCatalog(cast("Connection", connection)), connection
+
+
+class TestRegisterFiles:
+ @parameterized.expand(
+ [
+ (
+ "day",
+ DuckLakeDayReplacement(column="timestamp", day=date(2024, 1, 31)),
+ "replace_range_column => 'timestamp', replace_range_start => '2024-01-31 00:00:00+00', "
+ "replace_range_end => '2024-02-01 00:00:00+00'",
+ ),
+ ("table", DuckLakeTableReplacement(), "replace => true"),
+ ("append", None, ""),
+ ]
+ )
+ def test_registers_all_files_in_one_snapshot_with_the_replacement(
+ self, _name: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None, options: str
+ ) -> None:
+ catalog, connection = _catalog()
+
+ added = catalog.register_files(
+ "events_team_7",
+ ["s3://bucket/backfill/a.parquet", "s3://bucket/backfill/it's.parquet"],
+ import_id="run1_ab12",
+ replacement=replacement,
+ )
+
+ assert added == 2
+ [statement] = connection.statements
+ assert statement.startswith(
+ 'ALTER TABLE "warehouse"."posthog"."events_team_7" EXECUTE add_files('
+ "files => ARRAY['s3://bucket/backfill/a.parquet', 's3://bucket/backfill/it''s.parquet'], "
+ "import_id => 'run1_ab12', allow_missing => true"
+ )
+ assert statement.endswith(f", {options})" if options else "allow_missing => true)")
+
+ @parameterized.expand(
+ [
+ (
+ "register_day",
+ lambda c: c.register_files(
+ "events",
+ ["s3://b/f.parquet"],
+ import_id="i",
+ replacement=DuckLakeDayReplacement(column="timestamp", day=date(2024, 1, 1)),
+ ),
+ ),
+ (
+ "register_table",
+ lambda c: c.register_files(
+ "persons", ["s3://b/f.parquet"], import_id="i", replacement=DuckLakeTableReplacement()
+ ),
+ ),
+ (
+ "delete_day",
+ lambda c: c.delete_rows("groups", DuckLakeDayReplacement(column="_timestamp", day=date(2024, 1, 1))),
+ ),
+ ("delete_table", lambda c: c.delete_rows("events", DuckLakeTableReplacement())),
+ ]
+ )
+ def test_refuses_to_replace_rows_in_a_table_other_teams_share(self, _name: str, action) -> None:
+ catalog, connection = _catalog()
+
+ with pytest.raises(ValueError, match="shared"):
+ action(catalog)
+
+ assert connection.statements == []
+
+ def test_refuses_more_files_than_one_call_registers(self) -> None:
+ catalog, connection = _catalog()
+ files = [f"s3://bucket/f{index}.parquet" for index in range(ADD_FILES_MAX_FILES + 1)]
+
+ with pytest.raises(ValueError, match="max_s3_file_fanout"):
+ catalog.register_files("events_team_7", files, import_id="i", replacement=None)
+
+ assert connection.statements == []
+
+
+class TestDeleteRows:
+ @parameterized.expand(
+ [
+ (
+ "day",
+ DuckLakeDayReplacement(column="_timestamp", day=date(2024, 12, 31)),
+ 'DELETE FROM "warehouse"."posthog"."persons_team_7" '
+ "WHERE \"_timestamp\" >= TIMESTAMP '2024-12-31 00:00:00 UTC' "
+ "AND \"_timestamp\" < TIMESTAMP '2025-01-01 00:00:00 UTC'",
+ ),
+ ("table", DuckLakeTableReplacement(), 'DELETE FROM "warehouse"."posthog"."persons_team_7"'),
+ ]
+ )
+ def test_deletes_the_rows_an_empty_export_replaces(
+ self, _name: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement, expected: str
+ ) -> None:
+ catalog, connection = _catalog()
+
+ catalog.delete_rows("persons_team_7", replacement)
+
+ assert connection.statements == [expected]
+
+
+class TestEnsureTable:
+ def test_creates_the_schema_and_table_with_the_column_types(self) -> None:
+ catalog, connection = _catalog(COLUMNS)
+
+ catalog.ensure_table("events_team_7", COLUMNS, drop_first=True, create_if_missing=True, validate=True)
+
+ assert connection.statements[:3] == [
+ 'DROP TABLE IF EXISTS "warehouse"."posthog"."events_team_7"',
+ 'CREATE SCHEMA IF NOT EXISTS "warehouse"."posthog"',
+ 'CREATE TABLE IF NOT EXISTS "warehouse"."posthog"."events_team_7" '
+ '("team_id" bigint, "timestamp" timestamp(6) with time zone, "person_version" bigint)',
+ ]
+
+ @parameterized.expand([("events",), ("persons",), ("groups",)])
+ def test_refuses_to_drop_a_table_other_teams_share(self, table: str) -> None:
+ catalog, connection = _catalog(COLUMNS)
+
+ with pytest.raises(ValueError, match="shared"):
+ catalog.ensure_table(table, COLUMNS, drop_first=True, create_if_missing=True, validate=True)
+
+ assert connection.statements == []
+
+ @parameterized.expand(
+ [
+ ("extra_column", {**COLUMNS, "added_later": "varchar"}, None),
+ ("missing_column", {"team_id": "bigint", "timestamp": "timestamp(6) with time zone"}, "person_version"),
+ ("unsigned_column", {**COLUMNS, "person_version": "decimal(20,0)"}, "person_version"),
+ ]
+ )
+ def test_validates_that_exported_files_match_the_table(
+ self, _name: str, actual: dict[str, str], mismatch: str | None
+ ) -> None:
+ catalog, _ = _catalog(actual)
+
+ def ensure() -> None:
+ catalog.ensure_table("events_team_7", COLUMNS, drop_first=False, create_if_missing=False, validate=True)
+
+ if mismatch is None:
+ ensure()
+ else:
+ with pytest.raises(ValueError, match=mismatch):
+ ensure()
+
+
+def _query_error(error_class: type[TrinoQueryError], name: str, error_type: str) -> TrinoQueryError:
+ return error_class({"errorName": name, "errorType": error_type, "message": name}, "query-1")
+
+
+def _register(catalog: TrinoBackfillCatalog) -> int:
+ return catalog.register_files(
+ "events_team_7",
+ ["s3://bucket/backfill/a.parquet"],
+ import_id="run1_ab12",
+ replacement=DuckLakeDayReplacement(column="timestamp", day=date(2024, 1, 31)),
+ )
+
+
+@patch("tenacity.nap.time.sleep")
+class TestTransientFailures:
+ @parameterized.expand(
+ [
+ ("commit_conflict", _query_error(TrinoExternalError, "DUCKLAKE_COMMIT_CONFLICT", "EXTERNAL")),
+ (
+ "concurrent_modification",
+ _query_error(TrinoExternalError, "DUCKLAKE_CONCURRENT_MODIFICATION", "EXTERNAL"),
+ ),
+ ("commit_retries_exhausted", _query_error(TrinoExternalError, "DUCKLAKE_COMMIT_FAILED", "EXTERNAL")),
+ ("cluster_out_of_memory", _query_error(TrinoQueryError, "CLUSTER_OUT_OF_MEMORY", "INSUFFICIENT_RESOURCES")),
+ ("worker_lost", _query_error(TrinoQueryError, "TOO_MANY_REQUESTS_FAILED", "INTERNAL_ERROR")),
+ ("connection_lost", TrinoConnectionError("failed to fetch: connection reset")),
+ ("gateway_unavailable", Http503Error("error 503: service unavailable")),
+ ]
+ )
+ def test_repeats_a_statement_that_failed_for_a_transient_reason(
+ self, _sleep, _name: str, failure: Exception
+ ) -> None:
+ catalog, connection = _catalog(failures=[failure])
+
+ assert _register(catalog) == 2
+
+ assert len(connection.statements) == 2
+ assert connection.statements[0] == connection.statements[1]
+
+ @parameterized.expand(
+ [
+ ("import_id_reused", _query_error(TrinoUserError, "INVALID_PROCEDURE_ARGUMENT", "USER_ERROR")),
+ ("file_types_differ", _query_error(TrinoExternalError, "DUCKLAKE_BAD_DATA", "EXTERNAL")),
+ ]
+ )
+ def test_raises_a_statement_error_without_repeating_the_statement(
+ self, _sleep, _name: str, failure: Exception
+ ) -> None:
+ catalog, connection = _catalog(failures=[failure])
+
+ with pytest.raises(type(failure)):
+ _register(catalog)
+
+ assert len(connection.statements) == 1
+
+ def test_raises_the_last_transient_failure_after_bounded_attempts(self, _sleep) -> None:
+ failures: list[Exception] = [
+ _query_error(TrinoExternalError, "DUCKLAKE_COMMIT_CONFLICT", "EXTERNAL") for _ in range(10)
+ ]
+ catalog, connection = _catalog(failures=failures)
+
+ with pytest.raises(TrinoExternalError):
+ _register(catalog)
+
+ assert 1 < len(connection.statements) < 10
diff --git a/products/managed_warehouse/backend/tests/test_groups_backfill.py b/products/managed_warehouse/backend/tests/test_groups_backfill.py
index d8480d874a25..8eab45c3085f 100644
--- a/products/managed_warehouse/backend/tests/test_groups_backfill.py
+++ b/products/managed_warehouse/backend/tests/test_groups_backfill.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import re
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -21,6 +22,8 @@ from posthog.clickhouse.client.connection import get_client_from_pool
from posthog.dags.events_backfill_to_duckling import DUCKLAKE_ALIAS, DucklingBackfillConfig, DucklingTarget
from products.managed_warehouse.backend.facade.contracts import (
+ DuckLakeDayReplacement,
+ DuckLakeTableReplacement,
ManagedWarehouseTableNames,
ManagedWarehouseTeamMembership,
)
@@ -28,6 +31,7 @@ from products.managed_warehouse.dags.groups_backfill import (
GROUPS_JOB_NAME,
GROUPS_REFRESH_DATE_TAG,
GROUPS_SNAPSHOT_TAG,
+ GROUPS_TRINO_COLUMN_TYPES,
MAX_TEAMS_PER_TICK,
RETRY_BASE_SECONDS,
duckling_groups_backfill,
@@ -45,9 +49,10 @@ if TYPE_CHECKING:
from dagster import RunRequest
[email protected]("for_trino", [False, True])
@pytest.mark.parametrize("partition_date", [None, datetime(2025, 1, 1), datetime(2025, 1, 2), datetime(2025, 1, 3)])
def test_groups_snapshot_deduplicates_keys_within_each_team_and_group_type(
- monkeypatch: pytest.MonkeyPatch, partition_date: datetime | None
+ monkeypatch: pytest.MonkeyPatch, partition_date: datetime | None, for_trino: bool
) -> None:
monkeypatch.setattr(settings, "CLICKHOUSE_USE_HTTP", False)
table_name = f"test_groups_snapshot_{uuid4().hex}"
@@ -79,10 +84,11 @@ def test_groups_snapshot_deduplicates_keys_within_each_team_and_group_type(
(7, 0, "tied", '{"name":"last insert"}', created, latest, 1),
],
)
- query = groups_snapshot_select_sql(7, partition_date).replace(
+ query = groups_snapshot_select_sql(7, partition_date, for_trino=for_trino).replace(
"FROM groups FINAL", f"FROM {table_name} FINAL"
)
rows = client.execute(query)
+ described = {str(row[0]): str(row[1]) for row in client.execute(f"DESCRIBE ({query})")}
finally:
client.execute(f"DROP TABLE {table_name}")
expected = [
@@ -90,9 +96,20 @@ def test_groups_snapshot_deduplicates_keys_within_each_team_and_group_type(
(7, 0, "tied", '{"name":"last insert"}', created, latest),
(7, 1, "company", '{"name":"another type"}', created, latest),
]
- assert sorted(row[:6] for row in rows) == [
+ assert sorted((*row[:4], row[4].replace(tzinfo=None), row[5]) for row in rows) == [
row for row in expected if partition_date is None or row[5] == partition_date
]
+ if for_trino:
+ # add_files refuses any type conversion, so these must be the types DuckLake writes for the Trino columns.
+ assert described == {
+ "team_id": "Int64",
+ "group_type_index": "Int16",
+ "group_key": "String",
+ "group_properties": "String",
+ "created_at": "DateTime64(6, 'UTC')",
+ "_timestamp": "DateTime64(6)",
+ "_inserted_at": "DateTime64(6)",
+ }
@pytest.mark.parametrize("memory_override", [None, 20 * 1024**3])
@@ -288,6 +305,108 @@ def test_groups_backfill_cleans_only_its_partition_and_replays_registration(
assert metadata["export_mode"] == ("daily" if daily else "full")
+class _FakeGroupsCatalog:
+ def __init__(self) -> None:
+ self.tables: list[tuple[str, dict[str, str]]] = []
+ self.registered: list[tuple[str, list[str], DuckLakeDayReplacement | DuckLakeTableReplacement | None]] = []
+
+ def ensure_table(
+ self, table: str, columns: dict[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool
+ ) -> None:
+ self.tables.append((table, dict(columns)))
+
+ def register_files(
+ self,
+ table: str,
+ files: list[str],
+ *,
+ import_id: str,
+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,
+ ) -> int:
+ self.registered.append((table, list(files), replacement))
+ return len(files)
+
+
+ "partition_key,replacement",
+ [
+ ("7", DuckLakeTableReplacement()),
+ ("7_2025-12-31", DuckLakeDayReplacement(column="_timestamp", day=datetime(2025, 12, 31).date())),
+ ],
+)
+@time_machine.travel(datetime(2026, 1, 1, tzinfo=UTC), tick=False)
+def test_groups_backfill_registers_through_trino_without_duckgres(
+ partition_key: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement
+) -> None:
+ inserts, catalog, connect_catalog, context = _run_groups_through_trino(partition_key, DucklingBackfillConfig())
+
+ connect_catalog.assert_called_once_with("org")
+ assert catalog.tables == [("groups_team_7", GROUPS_TRINO_COLUMN_TYPES)]
+ [(table, files, registered_replacement)] = catalog.registered
+ assert (table, registered_replacement) == ("groups_team_7", replacement)
+ [insert] = inserts
+ export_path = re.search(r"s3\('([^']+)'", insert)
+ assert export_path is not None
+ assert files == [
+ export_path[1].replace("https://bkt.s3.us-east-1.amazonaws.com/", "s3://bkt/").replace("{_partition_id}", "0")
+ ]
+ assert "toInt16(group_type_index) AS group_type_index" in insert
+ assert "toDateTime64(created_at, 6, 'UTC') AS created_at" in insert
+ assert context.add_output_metadata.call_args.args[0]["files_registered"] == 1
+
+
[email protected]("partition_key", ["7", "7_2025-12-31"])
+@time_machine.travel(datetime(2026, 1, 1, tzinfo=UTC), tick=False)
+def test_groups_export_only_writes_the_trino_types_without_registering(partition_key: str) -> None:
+ inserts, catalog, connect_catalog, _ = _run_groups_through_trino(
+ partition_key, DucklingBackfillConfig(skip_ducklake_registration=True)
+ )
+
+ connect_catalog.assert_not_called()
+ assert catalog.registered == []
+ [insert] = inserts
+ assert "toInt16(group_type_index) AS group_type_index" in insert
+ assert "toDateTime64(created_at, 6, 'UTC') AS created_at" in insert
+
+
+def _run_groups_through_trino(
+ partition_key: str, config: DucklingBackfillConfig
+) -> tuple[list[str], _FakeGroupsCatalog, MagicMock, MagicMock]:
+ target = DucklingTarget(team_id=7, organization_id="org", bucket="bkt", bucket_region="us-east-1")
+ inserts: list[str] = []
+
+ def execute(query: str, params: object = None, settings: object = None) -> list[tuple[object, ...]]:
+ if "INSERT INTO FUNCTION" in query:
+ inserts.append(query)
+ return []
+ if "'One'" in query:
+ return [("0.parquet",)]
+ return [(10,)]
+
+ client = MagicMock()
+ client.execute.side_effect = execute
+ cluster = MagicMock()
+ cluster.any_host_by_role.side_effect = lambda fn, **_: MagicMock(result=MagicMock(return_value=fn(client)))
+ catalog = _FakeGroupsCatalog()
+ context = MagicMock()
+ context.partition_key = partition_key
+ context.run.run_id = "synthetic-run"
+ with (
+ patch("products.managed_warehouse.dags.groups_backfill._resolve_duckling_target", return_value=target),
+ patch("products.managed_warehouse.dags.groups_backfill.resolve_groups_table", return_value="groups_team_7"),
+ patch("products.managed_warehouse.dags.groups_backfill.backfills_through_trino", return_value=True),
+ patch("products.managed_warehouse.dags.groups_backfill._DuckgresSession") as duckgres,
+ patch("products.managed_warehouse.dags.groups_backfill.record_duckling_backfill_workload"),
+ patch("posthog.dags.events_backfill_to_duckling._get_cluster", return_value=cluster),
+ patch("posthog.dags.events_backfill_to_duckling.connect_backfill_catalog") as connect_catalog,
+ ):
+ connect_catalog.return_value.__enter__.return_value = catalog
+ run_groups_backfill(context, config)
+
+ duckgres.assert_not_called()
+ return inserts, catalog, connect_catalog, context
+
+
_NOW = datetime(2026, 1, 1, tzinfo=UTC)
diff --git a/products/managed_warehouse/dags/groups_backfill.py b/products/managed_warehouse/dags/groups_backfill.py
index fc4e6a798846..1b2ddefa994d 100644
--- a/products/managed_warehouse/dags/groups_backfill.py
+++ b/products/managed_warehouse/dags/groups_backfill.py
@@ -1,3 +1,4 @@
+import functools
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING
@@ -31,15 +32,19 @@ from posthog.dags.events_backfill_to_duckling import (
DUCKLAKE_ALIAS,
DUCKLING_BACKFILL_CONCURRENCY_TAG,
PARQUET_WRITER_SETTINGS,
+ TRINO_EXPORT_SETTINGS,
DucklingBackfillConfig,
DucklingTarget,
+ TrinoExport,
_compute_fanout,
_DuckgresSession,
_execute_export_with_retry,
_get_cluster,
_glob_run_files,
+ _merged_clickhouse_settings,
_repair_stale_running_statuses,
_resolve_duckling_target,
+ backfill_through_trino,
get_s3_url_for_clickhouse,
is_full_export_partition,
parse_partition_key_dates,
@@ -47,6 +52,8 @@ from posthog.dags.events_backfill_to_duckling import (
from products.data_warehouse.backend.facade.backfill_status import record_backfill_finished, record_backfill_started
from products.data_warehouse.backend.facade.models import ManagedWarehouseBackfillPartition
+from products.managed_warehouse.backend.facade.backfill_trino import backfills_through_trino
+from products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacement
from products.managed_warehouse.backend.facade.metrics import record_duckling_backfill_workload, track_duckling_backfill
from products.managed_warehouse.backend.facade.team_state import (
list_enabled_backfill_team_memberships,
@@ -75,9 +82,20 @@ GROUPS_COLUMN_TYPES = {
"_timestamp": "TIMESTAMP WITH TIME ZONE",
"_inserted_at": "TIMESTAMP WITH TIME ZONE",
}
+# Trino has no unsigned types, and add_files refuses any type conversion, so the Trino export casts
+# group_type_index to Int16 and the millisecond created_at to microseconds.
+GROUPS_TRINO_COLUMN_TYPES = {
+ "team_id": "bigint",
+ "group_type_index": "smallint",
+ "group_key": "varchar",
+ "group_properties": "varchar",
+ "created_at": "timestamp(6) with time zone",
+ "_timestamp": "timestamp(6) with time zone",
+ "_inserted_at": "timestamp(6) with time zone",
+}
-def groups_snapshot_select_sql(team_id: int, date: datetime | None = None) -> str:
+def groups_snapshot_select_sql(team_id: int, date: datetime | None = None, *, for_trino: bool = False) -> str:
if team_id <= 0:
raise ValueError("Groups backfill requires a positive team ID")
date_filter = ""
@@ -85,9 +103,11 @@ def groups_snapshot_select_sql(team_id: int, date: datetime | None = None) -> st
start = escape_clickhouse_string(date.strftime("%Y-%m-%d"))
end = escape_clickhouse_string((date + timedelta(days=1)).strftime("%Y-%m-%d"))
date_filter = f"AND _timestamp >= toDateTime({start}, 'UTC') AND _timestamp < toDateTime({end}, 'UTC')"
+ group_type_index = "toInt16(group_type_index) AS group_type_index" if for_trino else "group_type_index"
+ created_at = "toDateTime64(created_at, 6, 'UTC') AS created_at" if for_trino else "created_at"
return f"""
- SELECT team_id, group_type_index, group_key,
- group_properties, created_at,
+ SELECT team_id, {group_type_index}, group_key,
+ group_properties, {created_at},
toDateTime64(_timestamp, 6) AS _timestamp,
now64(6) AS _inserted_at
FROM groups FINAL
@@ -101,9 +121,12 @@ def export_groups_to_duckling_s3(
config: DucklingBackfillConfig,
target: DucklingTarget,
date: datetime | None = None,
+ *,
+ run_id: str | None = None,
+ for_trino: bool = False,
) -> str | None:
period = f"/{date.strftime('%Y-%m-%d')}" if date is not None else ""
- path = f"backfill/groups/{target.team_id}{period}/{context.run.run_id}/{{_partition_id}}.parquet"
+ path = f"backfill/groups/{target.team_id}{period}/{run_id or context.run.run_id}/{{_partition_id}}.parquet"
s3_url = get_s3_url_for_clickhouse(target.bucket, target.bucket_region, path)
s3_glob = f"s3://{target.bucket}/{path.replace('{_partition_id}', '*')}"
if config.dry_run:
@@ -117,6 +140,8 @@ def export_groups_to_duckling_s3(
**(config.clickhouse_settings or {}),
}
settings.update(PARQUET_WRITER_SETTINGS)
+ if for_trino:
+ settings.update(TRINO_EXPORT_SETTINGS)
count_sql = "SELECT count() FROM groups WHERE team_id = %(team_id)s"
count_params: dict[str, int | str] = {"team_id": target.team_id}
if date is not None:
@@ -127,7 +152,7 @@ def export_groups_to_duckling_s3(
export_sql = f"""
INSERT INTO FUNCTION s3({escape_clickhouse_string(s3_url)}, 'Parquet')
PARTITION BY toString(cityHash64(group_type_index, group_key) % {fanout})
- {groups_snapshot_select_sql(target.team_id, date)}
+ {groups_snapshot_select_sql(target.team_id, date, for_trino=for_trino)}
SETTINGS s3_truncate_on_insert=1, use_hive_partitioning=0
"""
# A failed export can leave partial S3 objects; only register files after the whole query succeeds.
@@ -191,6 +216,36 @@ def register_groups_files_with_duckling(
return len(files)
+def _report_groups_backfill(
+ context: AssetExecutionContext,
+ config: DucklingBackfillConfig,
+ *,
+ team_id: int,
+ mode: str,
+ table_name: str,
+ files_registered: int,
+ partitions_exported: int,
+) -> None:
+ context.add_output_metadata(
+ {
+ "team_id": team_id,
+ "partition_key": context.partition_key,
+ "export_mode": mode,
+ "groups_table": table_name,
+ "files_registered": files_registered,
+ "partitions_exported": partitions_exported,
+ }
+ )
+ if not config.dry_run and not config.skip_ducklake_registration:
+ record_duckling_backfill_workload(
+ team_id=team_id,
+ dataset=GROUPS_DATASET,
+ mode=mode,
+ files_registered=files_registered,
+ partitions_exported=partitions_exported,
+ )
+
+
def run_groups_backfill(context: AssetExecutionContext, config: DucklingBackfillConfig) -> None:
partition_key = context.partition_key
is_full = is_full_export_partition(partition_key)
@@ -206,8 +261,46 @@ def run_groups_backfill(context: AssetExecutionContext, config: DucklingBackfill
target = _resolve_duckling_target(team_id)
table_name = resolve_groups_table(team_id)
tags = dagster_tags(context)
- should_use_duckgres = not (config.dry_run or config.skip_ducklake_registration)
- session = _DuckgresSession(context, target) if should_use_duckgres else None
+ if backfills_through_trino(target.organization_id):
+ result = backfill_through_trino(
+ context,
+ config,
+ target,
+ table=table_name,
+ column_types=GROUPS_TRINO_COLUMN_TYPES,
+ settings=_merged_clickhouse_settings(context, config),
+ exports=[
+ TrinoExport(
+ label=partition_date.strftime("%Y-%m-%d") if partition_date is not None else "full",
+ replacement=(
+ DuckLakeTableReplacement()
+ if partition_date is None
+ else DuckLakeDayReplacement(column="_timestamp", day=partition_date.date())
+ ),
+ export=functools.partial(
+ export_groups_to_duckling_s3,
+ context=context,
+ config=config,
+ target=target,
+ date=partition_date,
+ for_trino=True,
+ ),
+ )
+ for partition_date in dates
+ ],
+ )
+ _report_groups_backfill(
+ context,
+ config,
+ team_id=team_id,
+ mode=mode,
+ table_name=table_name,
+ files_registered=result.files_registered,
+ partitions_exported=result.partitions_exported,
+ )
+ return
+ should_register = not (config.dry_run or config.skip_ducklake_registration)
+ session = _DuckgresSession(context, target) if should_register else None
files_registered = 0
partitions_exported = 0
try:
@@ -253,24 +346,15 @@ def run_groups_backfill(context: AssetExecutionContext, config: DucklingBackfill
finally:
if session is not None:
session.close()
- context.add_output_metadata(
- {
- "team_id": team_id,
- "partition_key": partition_key,
- "export_mode": mode,
- "groups_table": table_name,
- "files_registered": files_registered,
- "partitions_exported": partitions_exported,
- }
+ _report_groups_backfill(
+ context,
+ config,
+ team_id=team_id,
+ mode=mode,
+ table_name=table_name,
+ files_registered=files_registered,
+ partitions_exported=partitions_exported,
)
- if not config.dry_run and not config.skip_ducklake_registration:
- record_duckling_backfill_workload(
- team_id=team_id,
- dataset=GROUPS_DATASET,
- mode=mode,
- files_registered=files_registered,
- partitions_exported=partitions_exported,
- )
@asset(partitions_def=duckling_groups_partitions_def, tags=GROUPS_TAGS)