:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(managed-warehouse): register duckling backfills through trino (#125930)
Eric Duong6251c29ee663
diff --git a/posthog/dags/README_DUCKLINGS.md b/posthog/dags/README_DUCKLINGS.mdindex b4a130c9c768..5e06fbd7959a 100644--- a/posthog/dags/README_DUCKLINGS.md+++ b/posthog/dags/README_DUCKLINGS.md@@ -22,6 +22,27 @@ Duckling RDS Catalog (PostgreSQL)- **ClickHouse EC2 role**: Allowed in duckling bucket policy for direct S3 writes- **Dagster IRSA role**: Can assume duckling cross-account roles for DuckDB registration+### Warehouses without duckgres++Organizations with the `managed-warehouse-trino-import-registration` flag have no duckgres, so the events, persons, and groups backfills register through the organization's Trino catalog instead.+The ClickHouse export is the same, except that it casts each column to the Parquet type that DuckLake writes for its Trino column, because `add_files` refuses any type conversion.+An export-only run (`skip_ducklake_registration: true`) writes the same types, so `add_files` can register its files later.+ClickHouse lists the files it exported, because nothing else on the path can list the duckling bucket.+Each export writes under a new file prefix, so a retry never overwrites a file that DuckLake has already registered.++One `add_files` call registers the files of an export and removes the rows they replace in a single DuckLake snapshot:++- A daily export replaces the rows of its UTC day, by `timestamp` for events and by `_timestamp` for persons and groups.+- A full persons or groups export replaces the whole table.+- An export with no files deletes the rows it would replace.+- With `cleanup_existing_partition_data: false`, the files are added and no rows are removed.++The call has an import ID, so a retry after a lost response adds nothing.+A statement that fails for a transient reason, such as a DuckLake commit conflict, a lost coordinator, or a cluster fault, runs again up to four times.+Readers never see a day without rows or with two copies of it, so this path needs no delete-then-register replay.+Day and table replacements remove rows without a team filter, so the Trino path refuses to replace rows in the shared `events`, `persons`, and `groups` tables, or to drop them with `delete_tables: true`.+Trino-created tables have no year/month/day partitioning, because Trino cannot partition a timestamp with time zone by those transforms. Each file holds one day, so file statistics still prune reads.+## Jobs and Sensors### Events Backfilldiff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.pyindex 524764e165b8..77066e04e0cb 100644--- a/posthog/dags/events_backfill_to_duckling.py+++ b/posthog/dags/events_backfill_to_duckling.py@@ -35,13 +35,15 @@ import reimport mathimport timeimport random+import secretsimport calendar+import functoolsimport dataclasses-from collections.abc import Callable-from contextlib import closing+from collections.abc import Callable, Mapping, Sequence+from contextlib import closing, nullcontextfrom dataclasses import replacefrom datetime import UTC, date, datetime, timedelta-from typing import Any, Literal+from typing import Any, Literal, Protocolfrom django.db import DatabaseErrorfrom django.utils import timezone@@ -69,11 +71,14 @@ from dagster import (from psycopg import sql as psqlfrom tenacity import retry, retry_if_exception_type, stop_after_attempt, stop_after_delay, wait_exponential, wait_fixed+from posthog.hogql.escape_sql import escape_clickhouse_string+from posthog.clickhouse.client.connection import NodeRole, Workloadfrom posthog.clickhouse.cluster import ClickhouseCluster, get_clusterfrom posthog.clickhouse.query_tagging import tags_contextfrom posthog.cloud_utils import is_cloudfrom posthog.dags.common import JobOwners, dagster_tags, settings_with_log_comment+from posthog.dataclasses import frozenfrom products.data_warehouse.backend.facade.backfill_status import (BackfillOutcome,@@ -91,6 +96,7 @@ from products.managed_warehouse.backend.facade.api import (get_stored_bucket_config,resolve_team_earliest_event_date,)+from products.managed_warehouse.backend.facade.backfill_trino import backfills_through_trino, connect_backfill_catalogfrom products.managed_warehouse.backend.facade.client import (ServiceCredential,ServiceCredentialUnavailable,@@ -98,7 +104,11 @@ from products.managed_warehouse.backend.facade.client import (mint_service_credential,refresh_service_credential,)-from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTeamMembership+from products.managed_warehouse.backend.facade.contracts import (+ DuckLakeDayReplacement,+ DuckLakeTableReplacement,+ ManagedWarehouseTeamMembership,+)from products.managed_warehouse.backend.facade.metrics import record_duckling_backfill_workload, track_duckling_backfillfrom products.managed_warehouse.backend.facade.team_state import (list_enabled_backfill_team_memberships,@@ -821,6 +831,95 @@ EXPECTED_DUCKLAKE_PERSONS_COLUMNS = {"_inserted_at",}+# Export columns for warehouses that register through Trino. add_files refuses any type conversion, so each+# column has the exact Parquet type that DuckLake writes for its Trino column in EVENTS_TRINO_COLUMN_TYPES.+# ClickHouse writes the DateTime64 columns without a precision in milliseconds, and the Enum8 as a Parquet enum.+EVENTS_TRINO_COLUMNS = """+ toString(uuid) as uuid,+ event,+ properties,+ timestamp,+ team_id,+ toInt64(team_id) as project_id,+ distinct_id,+ elements_chain,+ created_at,+ toString(person_id) as person_id,+ toDateTime64(person_created_at, 6, 'UTC') as person_created_at,+ person_properties,+ group0_properties,+ group1_properties,+ group2_properties,+ group3_properties,+ group4_properties,+ toDateTime64(group0_created_at, 6, 'UTC') as group0_created_at,+ toDateTime64(group1_created_at, 6, 'UTC') as group1_created_at,+ toDateTime64(group2_created_at, 6, 'UTC') as group2_created_at,+ toDateTime64(group3_created_at, 6, 'UTC') as group3_created_at,+ toDateTime64(group4_created_at, 6, 'UTC') as group4_created_at,+ toString(person_mode) as person_mode,+ historical_migration,+ now64(6) as _inserted_at+"""++EVENTS_TRINO_COLUMN_TYPES: dict[str, str] = {+ "uuid": "varchar",+ "event": "varchar",+ "properties": "varchar",+ "timestamp": "timestamp(6) with time zone",+ "team_id": "bigint",+ "project_id": "bigint",+ "distinct_id": "varchar",+ "elements_chain": "varchar",+ "created_at": "timestamp(6) with time zone",+ "person_id": "varchar",+ "person_created_at": "timestamp(6) with time zone",+ "person_properties": "varchar",+ "group0_properties": "varchar",+ "group1_properties": "varchar",+ "group2_properties": "varchar",+ "group3_properties": "varchar",+ "group4_properties": "varchar",+ "group0_created_at": "timestamp(6) with time zone",+ "group1_created_at": "timestamp(6) with time zone",+ "group2_created_at": "timestamp(6) with time zone",+ "group3_created_at": "timestamp(6) with time zone",+ "group4_created_at": "timestamp(6) with time zone",+ "person_mode": "varchar",+ "historical_migration": "boolean",+ "_inserted_at": "timestamp(6) with time zone",+}++# Trino has no unsigned BIGINT, so the UInt64 person version is exported as a signed Int64.+PERSONS_TRINO_COLUMNS = """+ pd.team_id AS team_id,+ pd.distinct_id AS distinct_id,+ toString(p.id) AS id,+ p.properties AS properties,+ toDateTime64(p.created_at, 6, 'UTC') AS created_at,+ toBool(p.is_identified) AS is_identified,+ pd.version AS person_distinct_id_version,+ toInt64(p.version) AS person_version,+ toDateTime64(p._timestamp, 6) AS _timestamp,+ now64(6) AS _inserted_at+"""++PERSONS_TRINO_COLUMN_TYPES: dict[str, str] = {+ "team_id": "bigint",+ "distinct_id": "varchar",+ "id": "varchar",+ "properties": "varchar",+ "created_at": "timestamp(6) with time zone",+ "is_identified": "boolean",+ "person_distinct_id_version": "bigint",+ "person_version": "bigint",+ "_timestamp": "timestamp(6) with time zone",+ "_inserted_at": "timestamp(6) with time zone",+}++# A Trino VARCHAR column expects the Parquet UTF-8 string type, not plain binary.+TRINO_EXPORT_SETTINGS: dict[str, Any] = {"output_format_parquet_string_as_string": 1}+duckling_events_partitions_def = DynamicPartitionsDefinition(name="duckling_events_backfill")duckling_persons_partitions_def = DynamicPartitionsDefinition(name="duckling_persons_backfill")@@ -1544,6 +1643,7 @@ def export_events_to_duckling_s3(date: datetime,run_id: str,settings: dict[str, Any],+ columns: str = EVENTS_COLUMNS,) -> str | None:"""Export events for a team/date to the duckling's S3 bucket.@@ -1623,7 +1723,7 @@ def export_events_to_duckling_s3()PARTITION BY toString(cityHash64(distinct_id) % {fanout})SELECT- {EVENTS_COLUMNS}+ {columns}FROM eventsWHERE {where_clause}SETTINGS s3_truncate_on_insert=1, use_hive_partitioning=0@@ -2166,6 +2266,7 @@ def export_persons_to_duckling_s3(date: datetime,run_id: str,settings: dict[str, Any],+ columns: str = PERSONS_COLUMNS,) -> str | None:"""Export persons for a team/date to the duckling's S3 bucket.@@ -2225,7 +2326,7 @@ def export_persons_to_duckling_s3()PARTITION BY toString(cityHash64(distinct_id) % {fanout})SELECT- {PERSONS_COLUMNS}+ {columns}FROM person AS p FINALINNER JOIN person_distinct_id2 AS pd FINAL ON p.id = pd.person_id AND p.team_id = pd.team_idWHERE p.team_id = {team_id}@@ -2270,6 +2371,7 @@ def export_persons_full_to_duckling_s3(team_id: int,run_id: str,settings: dict[str, Any],+ columns: str = PERSONS_COLUMNS,) -> str | None:"""Export ALL persons for a team to the duckling's S3 bucket.@@ -2328,7 +2430,7 @@ def export_persons_full_to_duckling_s3()PARTITION BY toString(cityHash64(distinct_id) % {fanout})SELECT- {PERSONS_COLUMNS}+ {columns}FROM person AS p FINALINNER JOIN person_distinct_id2 AS pd FINAL ON p.id = pd.person_id AND p.team_id = pd.team_idWHERE p.team_id = {team_id}@@ -2429,6 +2531,129 @@ def register_persons_files_with_duckling(return len(files)+def _merged_clickhouse_settings(context: AssetExecutionContext, config: DucklingBackfillConfig) -> dict[str, Any]:+ merged_settings = DEFAULT_CLICKHOUSE_SETTINGS.copy()+ merged_settings.update(settings_with_log_comment(context))+ if config.clickhouse_settings:+ merged_settings.update(config.clickhouse_settings)+ context.log.info(f"Using custom ClickHouse settings: {config.clickhouse_settings}")+ return merged_settings+++@retry(+ stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),+ wait=wait_exponential(multiplier=1, min=4, max=60),+ retry=retry_if_exception_type((ClickHouseError, OSError, TimeoutError)),+ reraise=True,+)+def list_exported_files(client: Client, target: DucklingTarget, s3_glob: str, settings: dict[str, Any]) -> list[str]:+ """List the files an export wrote, for a warehouse without duckgres to glob them.++ ClickHouse wrote the files, and the duckling bucket policy lets it list them. The export globs have their+ wildcard in the file name only, so every listed file name joins the directory of the glob.+ """+ bucket_prefix = f"s3://{target.bucket}/"+ if not s3_glob.startswith(bucket_prefix):+ raise ValueError(f"Export glob {s3_glob} is outside the bucket {target.bucket}")+ directory, _, pattern = s3_glob.removeprefix(bucket_prefix).rpartition("/")+ url = get_s3_url_for_clickhouse(target.bucket, target.bucket_region, f"{directory}/{pattern}")+ # The One format reads no file contents, so the query only lists the objects. An export of no rows writes no+ # files, so its glob must list none, whatever a profile or a Launchpad override sets.+ rows = client.execute(+ f"SELECT _file FROM s3({escape_clickhouse_string(url)}, 'One') ORDER BY _file",+ settings={**settings, "s3_throw_on_zero_files_match": 0},+ )+ return [f"{bucket_prefix}{directory}/{row[0]}" for row in rows]+++class _ExportUnderPrefix(Protocol):+ def __call__(self, *, client: Client, run_id: str) -> str | None: ...+++@frozen+class TrinoExport:+ """One export that the Trino path registers in one DuckLake snapshot.++ `export` writes the files under the given file prefix and returns their S3 glob.+ """++ label: str+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement+ export: _ExportUnderPrefix+++@frozen+class TrinoBackfillResult:+ partitions_exported: int+ files_registered: int+++def backfill_through_trino(+ context: AssetExecutionContext,+ config: DucklingBackfillConfig,+ target: DucklingTarget,+ *,+ table: str,+ column_types: Mapping[str, str],+ exports: Sequence[TrinoExport],+ settings: dict[str, Any],+) -> TrinoBackfillResult:+ """Export each item from ClickHouse and register its files through Trino, for a warehouse without duckgres.++ One add_files call adds the files of an export and removes the rows they replace in the same snapshot. A retried+ call with the same import id is a no-op, so this path needs no delete-then-register replay. A run that skips+ registration still writes the Trino types, because add_files can register only those files later.+ """+ run_id = context.run.run_id[:16]+ cluster = _get_cluster()+ tags = dagster_tags(context)+ workload = Workload.OFFLINE if is_cloud() else Workload.DEFAULT+ partitions_exported = 0+ files_registered = 0+ register = not (config.dry_run or config.skip_ducklake_registration)+ with connect_backfill_catalog(target.organization_id) if register else nullcontext() as catalog:+ if catalog is not None:+ catalog.ensure_table(+ table,+ column_types,+ drop_first=config.delete_tables,+ create_if_missing=config.create_tables_if_missing,+ validate=not config.skip_schema_validation,+ )+ for item in exports:+ # A new prefix for every export, because an export under an earlier prefix would overwrite files that+ # DuckLake already registered with their row counts and statistics.+ prefix = f"{run_id}_{secrets.token_hex(4)}"++ def export_and_list(client: Client, item: TrinoExport = item, prefix: str = prefix) -> list[str] | None:+ with tags_context(kind="dagster", dagster=tags):+ s3_glob = item.export(client=client, run_id=prefix)+ return None if s3_glob is None else list_exported_files(client, target, s3_glob, settings)++ files = cluster.any_host_by_role(fn=export_and_list, workload=workload, node_role=NodeRole.DATA).result()+ if files is None:+ continue+ partitions_exported += 1+ if catalog is None:+ context.log.info(f"Exported {len(files)} file(s) for {item.label} without registering them")+ continue+ replacement = item.replacement if config.cleanup_existing_partition_data else None+ if files:+ catalog.register_files(table, files, import_id=prefix, replacement=replacement)+ files_registered += len(files)+ elif replacement is not None:+ catalog.delete_rows(table, replacement)+ context.log.info(f"Registered {len(files)} file(s) for {item.label} in {table} through Trino")+ logger.info(+ "duckling_trino_files_registered",+ team_id=target.team_id,+ table=table,+ partition=item.label,+ file_count=len(files),+ )+ return TrinoBackfillResult(partitions_exported=partitions_exported, files_registered=files_registered)++@asset(partitions_def=duckling_events_partitions_def,name="duckling_events_backfill",@@ -2473,6 +2698,87 @@ def duckling_events_backfill(context: AssetExecutionContext, config: DucklingBac)+def _report_events_backfill(+ context: AssetExecutionContext,+ config: DucklingBackfillConfig,+ target: DucklingTarget,+ dates: list[datetime],+ *,+ days_exported: int,+ files_registered: int,+) -> None:+ team_id = target.team_id+ context.add_output_metadata(+ {+ "team_id": team_id,+ "partition_key": context.partition_key,+ "dates_processed": len(dates),+ "days_exported": days_exported,+ "files_registered": files_registered,+ "bucket": target.bucket,+ }+ )+ if not config.dry_run:+ record_duckling_backfill_workload(+ team_id=team_id,+ dataset=ManagedWarehouseBackfillPartition.Dataset.EVENTS,+ mode="monthly" if len(dates) > 1 else "daily",+ files_registered=files_registered,+ partitions_exported=days_exported,+ )++ context.log.info(+ f"Completed duckling backfill for team_id={team_id}: "+ f"{days_exported}/{len(dates)} days exported, {files_registered} files registered"+ )+ logger.info(+ "duckling_backfill_complete",+ team_id=team_id,+ dates_processed=len(dates),+ days_exported=days_exported,+ files_registered=files_registered,+ )+++def _backfill_events_through_trino(+ context: AssetExecutionContext, config: DucklingBackfillConfig, target: DucklingTarget, dates: list[datetime]+) -> None:+ trino_settings = {**_merged_clickhouse_settings(context, config), **TRINO_EXPORT_SETTINGS}+ result = backfill_through_trino(+ context,+ config,+ target,+ table=target.events_table,+ column_types=EVENTS_TRINO_COLUMN_TYPES,+ settings=trino_settings,+ exports=[+ TrinoExport(+ label=partition_date.strftime("%Y-%m-%d"),+ replacement=DuckLakeDayReplacement(column="timestamp", day=partition_date.date()),+ export=functools.partial(+ export_events_to_duckling_s3,+ context=context,+ config=config,+ target=target,+ team_id=target.team_id,+ date=partition_date,+ settings=trino_settings,+ columns=EVENTS_TRINO_COLUMNS,+ ),+ )+ for partition_date in dates+ ],+ )+ _report_events_backfill(+ context,+ config,+ target,+ dates,+ days_exported=result.partitions_exported,+ files_registered=result.files_registered,+ )++def _run_duckling_events_backfill(context: AssetExecutionContext, config: DucklingBackfillConfig) -> None:"""Backfill events from ClickHouse to a customer's duckling.@@ -2511,10 +2817,15 @@ def _run_duckling_events_backfill(context: AssetExecutionContext, config: Ducklicontext.log.info(f"Backfill ready for team_id={team_id}: org={target.organization_id}, bucket={target.bucket}")+ if backfills_through_trino(target.organization_id):+ _backfill_events_through_trino(context, config, target, dates)+ return++ should_register = not (config.dry_run or config.skip_ducklake_registration)+# Open one duckgres connection for all metadata operations, but skip it# entirely when no duckgres-backed work will run (dry_run / skip_ducklake_registration).- should_use_duckgres = not (config.dry_run or config.skip_ducklake_registration)- session = _DuckgresSession(context, target) if should_use_duckgres else None+ session = _DuckgresSession(context, target) if should_register else Nonetry:if session is not None:# Delete events table if requested (dangerous - loses all data)@@ -2545,11 +2856,7 @@ def _run_duckling_events_backfill(context: AssetExecutionContext, config: Ducklisession.run("validate events schema", lambda c: validate_duckling_schema(context, target, c))# Prepare ClickHouse settings- merged_settings = DEFAULT_CLICKHOUSE_SETTINGS.copy()- merged_settings.update(settings_with_log_comment(context))- if config.clickhouse_settings:- merged_settings.update(config.clickhouse_settings)- context.log.info(f"Using custom ClickHouse settings: {config.clickhouse_settings}")+ merged_settings = _merged_clickhouse_settings(context, config)cluster = _get_cluster()tags = dagster_tags(context)@@ -2612,35 +2919,8 @@ def _run_duckling_events_backfill(context: AssetExecutionContext, config: Ducklitotal_registered += session.run(f"register events files {date_str}", register_events_files)- context.add_output_metadata(- {- "team_id": team_id,- "partition_key": context.partition_key,- "dates_processed": len(dates),- "days_exported": days_exported,- "files_registered": total_registered,- "bucket": target.bucket,- }- )- if not config.dry_run:- record_duckling_backfill_workload(- team_id=team_id,- dataset=ManagedWarehouseBackfillPartition.Dataset.EVENTS,- mode="monthly" if len(dates) > 1 else "daily",- files_registered=total_registered,- partitions_exported=days_exported,- )-- context.log.info(- f"Completed duckling backfill for team_id={team_id}: "- f"{days_exported}/{len(dates)} days exported, {total_registered} files registered"- )- logger.info(- "duckling_backfill_complete",- team_id=team_id,- dates_processed=len(dates),- days_exported=days_exported,- files_registered=total_registered,+ _report_events_backfill(+ context, config, target, dates, days_exported=days_exported, files_registered=total_registered)finally:@@ -2695,6 +2975,123 @@ def duckling_persons_backfill(context: AssetExecutionContext, config: DucklingBa)+def _report_persons_backfill(+ context: AssetExecutionContext,+ config: DucklingBackfillConfig,+ target: DucklingTarget,+ dates: list[datetime] | None,+ *,+ partitions_exported: int,+ files_registered: int,+) -> None:+ """Record the outcome of a persons run. `dates` is None for a full export."""+ team_id = target.team_id+ export_mode = "full" if dates is None else "daily"+ metadata: dict[str, Any] = {+ "team_id": team_id,+ "partition_key": context.partition_key,+ "export_mode": export_mode,+ }+ if dates is not None:+ metadata.update(dates_processed=len(dates), days_exported=partitions_exported)+ metadata.update(files_registered=files_registered, bucket=target.bucket)+ context.add_output_metadata(metadata)+ if not config.dry_run:+ record_duckling_backfill_workload(+ team_id=team_id,+ dataset=ManagedWarehouseBackfillPartition.Dataset.PERSONS,+ mode=export_mode,+ files_registered=files_registered,+ partitions_exported=partitions_exported,+ )++ if dates is None:+ context.log.info(+ f"Completed duckling persons full backfill for team_id={team_id}: {files_registered} files registered"+ )+ logger.info(+ "duckling_persons_backfill_complete",+ team_id=team_id,+ export_mode="full",+ files_registered=files_registered,+ )+ else:+ context.log.info(+ f"Completed duckling persons daily backfill for team_id={team_id}: "+ f"{partitions_exported}/{len(dates)} days exported, {files_registered} files registered"+ )+ logger.info(+ "duckling_persons_backfill_complete",+ team_id=team_id,+ export_mode="daily",+ dates_processed=len(dates),+ days_exported=partitions_exported,+ files_registered=files_registered,+ )+++def _backfill_persons_through_trino(+ context: AssetExecutionContext,+ config: DucklingBackfillConfig,+ target: DucklingTarget,+ dates: list[datetime] | None,+) -> None:+ """Export persons and register them through Trino. `dates` is None for a full export."""+ trino_settings = {**_merged_clickhouse_settings(context, config), **TRINO_EXPORT_SETTINGS}+ exports = (+ [+ TrinoExport(+ label="full",+ replacement=DuckLakeTableReplacement(),+ export=functools.partial(+ export_persons_full_to_duckling_s3,+ context=context,+ config=config,+ target=target,+ team_id=target.team_id,+ settings=trino_settings,+ columns=PERSONS_TRINO_COLUMNS,+ ),+ )+ ]+ if dates is None+ else [+ TrinoExport(+ label=partition_date.strftime("%Y-%m-%d"),+ replacement=DuckLakeDayReplacement(column="_timestamp", day=partition_date.date()),+ export=functools.partial(+ export_persons_to_duckling_s3,+ context=context,+ config=config,+ target=target,+ team_id=target.team_id,+ date=partition_date,+ settings=trino_settings,+ columns=PERSONS_TRINO_COLUMNS,+ ),+ )+ for partition_date in dates+ ]+ )+ result = backfill_through_trino(+ context,+ config,+ target,+ table=target.persons_table,+ column_types=PERSONS_TRINO_COLUMN_TYPES,+ settings=trino_settings,+ exports=exports,+ )+ _report_persons_backfill(+ context,+ config,+ target,+ dates,+ partitions_exported=result.partitions_exported,+ files_registered=result.files_registered,+ )++def _run_duckling_persons_backfill(context: AssetExecutionContext, config: DucklingBackfillConfig) -> None:"""Backfill persons from ClickHouse to a customer's duckling.@@ -2742,10 +3139,15 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Ducklcontext.log.info(f"Backfill ready for team_id={team_id}: org={target.organization_id}, bucket={target.bucket}")+ if backfills_through_trino(target.organization_id):+ _backfill_persons_through_trino(context, config, target, None if is_full else dates)+ return++ should_register = not (config.dry_run or config.skip_ducklake_registration)+# Open one duckgres connection for all metadata operations, but skip it# entirely when no duckgres-backed work will run (dry_run / skip_ducklake_registration).- should_use_duckgres = not (config.dry_run or config.skip_ducklake_registration)- session = _DuckgresSession(context, target) if should_use_duckgres else None+ session = _DuckgresSession(context, target) if should_register else Nonetry:if session is not None:# Delete persons table if requested (dangerous - loses all data)@@ -2774,11 +3176,7 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Ducklcontext.log.info("Validating duckling persons schema compatibility...")session.run("validate persons schema", lambda c: validate_duckling_persons_schema(context, target, c))- merged_settings = DEFAULT_CLICKHOUSE_SETTINGS.copy()- merged_settings.update(settings_with_log_comment(context))- if config.clickhouse_settings:- merged_settings.update(config.clickhouse_settings)- context.log.info(f"Using custom ClickHouse settings: {config.clickhouse_settings}")+ merged_settings = _merged_clickhouse_settings(context, config)cluster = _get_cluster()tags = dagster_tags(context)@@ -2826,32 +3224,8 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Ducklfiles_registered = session.run("register persons files (full)", register_full_persons_files)- context.add_output_metadata(- {- "team_id": team_id,- "partition_key": partition_key,- "export_mode": "full",- "files_registered": files_registered,- "bucket": target.bucket,- }- )- if not config.dry_run:- record_duckling_backfill_workload(- team_id=team_id,- dataset=ManagedWarehouseBackfillPartition.Dataset.PERSONS,- mode="full",- files_registered=files_registered,- partitions_exported=int(bool(s3_glob)),- )-- context.log.info(- f"Completed duckling persons full backfill for team_id={team_id}: {files_registered} files registered"- )- logger.info(- "duckling_persons_backfill_complete",- team_id=team_id,- export_mode="full",- files_registered=files_registered,+ _report_persons_backfill(+ context, config, target, None, partitions_exported=int(bool(s3_glob)), files_registered=files_registered)else:# DAILY EXPORT MODE - process each date in the partition@@ -2910,37 +3284,8 @@ def _run_duckling_persons_backfill(context: AssetExecutionContext, config: Duckltotal_registered += session.run(f"register persons files {date_str}", register_persons_files)- context.add_output_metadata(- {- "team_id": team_id,- "partition_key": partition_key,- "export_mode": "daily",- "dates_processed": len(dates),- "days_exported": days_exported,- "files_registered": total_registered,- "bucket": target.bucket,- }- )- if not config.dry_run:- record_duckling_backfill_workload(- team_id=team_id,- dataset=ManagedWarehouseBackfillPartition.Dataset.PERSONS,- mode="daily",- files_registered=total_registered,- partitions_exported=days_exported,- )-- context.log.info(- f"Completed duckling persons daily backfill for team_id={team_id}: "- f"{days_exported}/{len(dates)} days exported, {total_registered} files registered"- )- logger.info(- "duckling_persons_backfill_complete",- team_id=team_id,- export_mode="daily",- dates_processed=len(dates),- days_exported=days_exported,- files_registered=total_registered,+ _report_persons_backfill(+ context, config, target, dates, partitions_exported=days_exported, files_registered=total_registered)finally:diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.pyindex a180ccce4698..baae112a0c68 100644--- a/posthog/dags/test_events_backfill_to_duckling.py+++ b/posthog/dags/test_events_backfill_to_duckling.py@@ -1,14 +1,19 @@import os+import refrom datetime import UTC, date, datetime, timedelta+from fnmatch import fnmatchfrom typing import Any, castimport pytest+import time_machinefrom unittest.mock import MagicMock, call, patchimport duckdbimport psycopg+from clickhouse_driver.errors import Error as ClickHouseErrorfrom parameterized import parameterized+from posthog.clickhouse.client.connection import get_client_from_poolfrom posthog.dags.common import JobOwnersfrom posthog.dags.events_backfill_to_duckling import (_DUCKLAKE_FILE_PARTITION_VALUE_FIXUP_ENV_VAR,@@ -17,12 +22,16 @@ from posthog.dags.events_backfill_to_duckling import (EVENTS_COLUMNS,EVENTS_CONCURRENCY_TAG,EVENTS_TABLE_DDL,+ EVENTS_TRINO_COLUMN_TYPES,+ EVENTS_TRINO_COLUMNS,EXPECTED_DUCKLAKE_EVENTS_COLUMNS,EXPECTED_DUCKLAKE_PERSONS_COLUMNS,MAX_S3_FILE_FANOUT,PERSONS_COLUMNS,PERSONS_CONCURRENCY_TAG,PERSONS_TABLE_DDL,+ PERSONS_TRINO_COLUMN_TYPES,+ PERSONS_TRINO_COLUMNS,TARGET_ROWS_PER_FILE,DucklingBackfillConfig,DucklingTarget,@@ -41,6 +50,7 @@ from posthog.dags.events_backfill_to_duckling import (_resolve_duckling_target,_resolve_table_names,_run_duckling_events_backfill,+ _run_duckling_persons_backfill,_set_table_partitioning,_stale_run_outcome,_validate_identifier,@@ -65,6 +75,8 @@ from products.data_warehouse.backend.facade.backfill_status import BackfillOutcofrom products.managed_warehouse.backend.facade.api import EARLIEST_BACKFILL_DATEfrom products.managed_warehouse.backend.facade.client import ServiceCredential, ServiceCredentialUnavailablefrom products.managed_warehouse.backend.facade.contracts import (+ DuckLakeDayReplacement,+ DuckLakeTableReplacement,DucklingTables,ManagedWarehouseTableNames,ManagedWarehouseTeamMembership,@@ -2611,3 +2623,234 @@ class TestFannedOutLayoutRoundTrip:# The missing column is NULL-filled for every backfilled row.cap_vals = [r[0] for r in conn.execute(f"SELECT captured_at FROM {DUCKLAKE_ALIAS}.posthog.events").fetchall()]assert cap_vals == [None] * 5+++class _FakeTrinoBackfillCatalog:+ def __init__(self) -> None:+ self.tables: list[tuple[str, dict[str, str]]] = []+ self.registered: list[tuple[str, list[str], str, DuckLakeDayReplacement | DuckLakeTableReplacement | None]] = []+ self.deleted: list[tuple[str, DuckLakeDayReplacement | DuckLakeTableReplacement]] = []++ def ensure_table(+ self, table: str, columns: dict[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool+ ) -> None:+ self.tables.append((table, dict(columns)))++ def register_files(+ self,+ table: str,+ files: list[str],+ *,+ import_id: str,+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,+ ) -> int:+ self.registered.append((table, list(files), import_id, replacement))+ return len(files)++ def delete_rows(self, table: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement) -> None:+ self.deleted.append((table, replacement))+++class _FakeExportClickHouse:+ def __init__(self, files_per_export: int) -> None:+ self.files_per_export = files_per_export+ self.inserts: list[tuple[str, dict[str, Any]]] = []+ self.objects: list[str] = []++ def execute(+ self, query: str, params: object = None, settings: dict[str, Any] | None = None+ ) -> list[tuple[Any, ...]]:+ if "INSERT INTO FUNCTION" in query:+ self.inserts.append((query, settings or {}))+ url = re.search(r"s3\(\s*'([^']+)'", query)+ assert url is not None+ self.objects += [url[1].replace("{_partition_id}", str(index)) for index in range(self.files_per_export)]+ return []+ if "'One'" in query:+ pattern = re.search(r"s3\('([^']+)'", query)+ assert pattern is not None+ matched = [(url.rsplit("/", 1)[1],) for url in sorted(self.objects) if fnmatch(url, pattern[1])]+ if not matched and (settings or {}).get("s3_throw_on_zero_files_match"):+ raise ClickHouseError(f"Can not match any files with path {pattern[1]}")+ return matched+ return [(10,)]+++class TestBackfillThroughTrino:+ TARGET = DucklingTarget(+ team_id=2,+ organization_id="org-1",+ bucket="bkt",+ bucket_region="us-east-1",+ events_table="events_team_2",+ persons_table="persons_team_2",+ )++ def _run(self, run, partition_key: str, clickhouse: _FakeExportClickHouse, config: DucklingBackfillConfig):+ catalog = _FakeTrinoBackfillCatalog()+ cluster = MagicMock()+ cluster.any_host_by_role.side_effect = lambda fn, **_: MagicMock(result=MagicMock(return_value=fn(clickhouse)))+ context = MagicMock(partition_key=partition_key)+ context.run.run_id = "0123456789abcdef-run"+ with (+ patch("posthog.dags.events_backfill_to_duckling._resolve_duckling_target", return_value=self.TARGET),+ patch("posthog.dags.events_backfill_to_duckling.backfills_through_trino", return_value=True),+ patch("posthog.dags.events_backfill_to_duckling.connect_backfill_catalog") as connect_catalog,+ patch("posthog.dags.events_backfill_to_duckling._get_cluster", return_value=cluster),+ patch("posthog.dags.events_backfill_to_duckling._DuckgresSession") as duckgres,+ patch("posthog.dags.events_backfill_to_duckling.record_duckling_backfill_workload"),+ patch("tenacity.nap.time.sleep"),+ ):+ connect_catalog.return_value.__enter__.return_value = catalog+ run(context, config)+ run(context, config)+ duckgres.assert_not_called()+ return catalog, connect_catalog++ @parameterized.expand(+ [+ (+ "events_day",+ _run_duckling_events_backfill,+ "2_2026-06-17",+ "events_team_2",+ EVENTS_TRINO_COLUMN_TYPES,+ DuckLakeDayReplacement(column="timestamp", day=date(2026, 6, 17)),+ "toString(person_mode) as person_mode",+ ),+ (+ "persons_day",+ _run_duckling_persons_backfill,+ "2_2026-06-17",+ "persons_team_2",+ PERSONS_TRINO_COLUMN_TYPES,+ DuckLakeDayReplacement(column="_timestamp", day=date(2026, 6, 17)),+ "toInt64(p.version) AS person_version",+ ),+ (+ "persons_full",+ _run_duckling_persons_backfill,+ "2",+ "persons_team_2",+ PERSONS_TRINO_COLUMN_TYPES,+ DuckLakeTableReplacement(),+ "toInt64(p.version) AS person_version",+ ),+ ]+ )+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)+ def test_registers_each_export_in_one_snapshot_under_a_new_prefix(+ self, _name, run, partition_key, table, column_types, replacement, cast_column+ ):+ clickhouse = _FakeExportClickHouse(files_per_export=2)++ catalog, connect_catalog = self._run(run, partition_key, clickhouse, DucklingBackfillConfig())++ connect_catalog.assert_called_with("org-1")+ assert catalog.tables == [(table, column_types)] * 2+ assert [(t, r) for t, _, _, r in catalog.registered] == [(table, replacement)] * 2+ first, second = catalog.registered+ assert len(first[1]) == 2 and all(path.startswith("s3://bkt/backfill/") for path in first[1])+ assert all(f"/{first[2]}_" in path for path in first[1])+ assert first[2] != second[2] and not set(first[1]) & set(second[1])+ for insert, settings in clickhouse.inserts:+ assert cast_column in insert+ assert settings["output_format_parquet_string_as_string"] == 1++ @parameterized.expand(+ [+ ("empty_day_deletes_its_rows", 0, True),+ ("append_keeps_existing_rows", 2, False),+ ("empty_append_changes_nothing", 0, False),+ ]+ )+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)+ def test_replaces_the_day_only_when_cleanup_is_on(self, _name, files_per_export, cleanup):+ clickhouse = _FakeExportClickHouse(files_per_export=files_per_export)+ day = DuckLakeDayReplacement(column="timestamp", day=date(2026, 6, 17))++ catalog, _ = self._run(+ _run_duckling_events_backfill,+ "2_2026-06-17",+ clickhouse,+ # A server profile or a Launchpad override can make ClickHouse throw for a glob that matches no files.+ DucklingBackfillConfig(+ cleanup_existing_partition_data=cleanup, clickhouse_settings={"s3_throw_on_zero_files_match": 1}+ ),+ )++ assert catalog.deleted == ([("events_team_2", day)] * 2 if cleanup and not files_per_export else [])+ assert [r for _, _, _, r in catalog.registered] == ([None] * 2 if files_per_export else [])++ @parameterized.expand(+ [+ ("events_day", _run_duckling_events_backfill, "2_2026-06-17", "toString(person_mode) as person_mode"),+ ("persons_day", _run_duckling_persons_backfill, "2_2026-06-17", "toInt64(p.version) AS person_version"),+ ("persons_full", _run_duckling_persons_backfill, "2", "toInt64(p.version) AS person_version"),+ ]+ )+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)+ def test_export_only_runs_write_the_trino_types_without_registering(self, _name, run, partition_key, cast_column):+ clickhouse = _FakeExportClickHouse(files_per_export=2)++ catalog, connect_catalog = self._run(+ run, partition_key, clickhouse, DucklingBackfillConfig(skip_ducklake_registration=True)+ )++ connect_catalog.assert_not_called()+ assert catalog.registered == []+ assert len(clickhouse.inserts) == 2+ for insert, settings in clickhouse.inserts:+ assert cast_column in insert+ assert settings["output_format_parquet_string_as_string"] == 1++ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)+ def test_dry_runs_neither_export_nor_open_the_catalog(self):+ clickhouse = _FakeExportClickHouse(files_per_export=2)++ _, connect_catalog = self._run(+ _run_duckling_events_backfill, "2_2026-06-17", clickhouse, DucklingBackfillConfig(dry_run=True)+ )++ connect_catalog.assert_not_called()+ assert clickhouse.inserts == []+++# The Trino column type that DuckLake reads back from the Parquet type ClickHouse writes for a ClickHouse type.+TRINO_TYPE_OF_CLICKHOUSE_TYPE = {+ "String": "varchar",+ "Int64": "bigint",+ "Int16": "smallint",+ "Bool": "boolean",+ "DateTime64(6)": "timestamp(6) with time zone",+ "DateTime64(6, 'UTC')": "timestamp(6) with time zone",+}+EVENTS_SOURCE = (+ "generateRandom('uuid UUID, event String, properties String, timestamp DateTime64(6, ''UTC''), team_id Int64, "+ "distinct_id String, elements_chain String, created_at DateTime64(6, ''UTC''), person_id UUID, "+ "person_created_at DateTime64(3), person_properties String, group0_properties String, group1_properties String, "+ "group2_properties String, group3_properties String, group4_properties String, group0_created_at DateTime64(3), "+ "group1_created_at DateTime64(3), group2_created_at DateTime64(3), group3_created_at DateTime64(3), "+ "group4_created_at DateTime64(3), person_mode Enum8(''full'' = 0, ''propertyless'' = 1, ''force_upgrade'' = 2), "+ "historical_migration Bool')"+)+PERSONS_SOURCE = (+ "generateRandom('id UUID, created_at DateTime64(3), team_id Int64, properties String, is_identified Int8, "+ "version UInt64, _timestamp DateTime') AS p "+ "CROSS JOIN generateRandom('team_id Int64, distinct_id String, person_id UUID, version Int64') AS pd"+)+++class TestTrinoExportColumnTypes:+ @parameterized.expand(+ [+ ("events", f"SELECT {EVENTS_TRINO_COLUMNS} FROM {EVENTS_SOURCE}", EVENTS_TRINO_COLUMN_TYPES),+ ("persons", f"SELECT {PERSONS_TRINO_COLUMNS} FROM {PERSONS_SOURCE}", PERSONS_TRINO_COLUMN_TYPES),+ ]+ )+ def test_export_writes_the_types_of_the_trino_table(self, _name, select, column_types):+ with get_client_from_pool() as client:+ described = client.execute(f"DESCRIBE ({select})")++ exported = {str(row[0]): TRINO_TYPE_OF_CLICKHOUSE_TYPE.get(str(row[1]), str(row[1])) for row in described}+ assert exported == column_typesdiff --git a/products/managed_warehouse/backend/backfill_trino.py b/products/managed_warehouse/backend/backfill_trino.pynew file mode 100644index 000000000000..7488bc59149d--- /dev/null+++ b/products/managed_warehouse/backend/backfill_trino.py@@ -0,0 +1,207 @@+"""Register duckling backfill exports in DuckLake through the organization's Trino catalog.++The Dagster backfills use this for warehouses that have no duckgres. One `add_files` call registers the files of an+export and removes the rows they replace in a single DuckLake snapshot, so readers never see a day without rows or+with two copies of it. The call refuses any type conversion, so the export must write the exact Parquet types that+DuckLake writes for each Trino column.+"""++from __future__ import annotations++import contextlib+from collections.abc import Iterator, Mapping, Sequence+from datetime import date, timedelta+from typing import TYPE_CHECKING, Any++import structlog+from tenacity import RetryCallState, retry, retry_if_exception, stop_after_attempt, wait_exponential+from trino.exceptions import Http502Error, Http503Error, Http504Error, TrinoConnectionError++from posthog.hogql.escape_sql import escape_trino_identifier, escape_trino_string++from posthog.exceptions_capture import capture_exception+from posthog.ph_client import feature_enabled_or_false++from products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacement+from products.managed_warehouse.backend.facade.feature_flags import DUCKLAKE_TRINO_REGISTRATION_FLAG+from products.managed_warehouse.backend.trino_connection import connect_managed_warehouse_trino+from products.managed_warehouse.backend.trino_execution import is_transient_trino_error++if TYPE_CHECKING:+ from trino.dbapi import Connection++LOGGER = structlog.get_logger(__name__)++BACKFILL_SCHEMA = "posthog"+BACKFILL_TRINO_PRINCIPAL = "dagster:duckling-backfill"+# One call is one snapshot. The connector accepts more files, but Trino rejects a statement longer than+# 1,000,000 characters, and this many export paths stay well below that.+ADD_FILES_MAX_FILES = 2_000+# A replacement removes rows by time or by table, never by team. In a table that several teams share, it would also+# remove the rows of the other teams.+SHARED_TABLES = frozenset({"events", "persons", "groups"})+# DuckLake asks for the statement to be run again when another writer changed the table while it was planned or+# committed.+TRANSIENT_DUCKLAKE_ERROR_NAMES = frozenset(+ {"DUCKLAKE_COMMIT_CONFLICT", "DUCKLAKE_CONCURRENT_MODIFICATION", "DUCKLAKE_COMMIT_FAILED"}+)+STATEMENT_ATTEMPTS = 4+++Replacement = DuckLakeDayReplacement | DuckLakeTableReplacement+++def backfills_through_trino(organization_id: str) -> bool:+ try:+ return feature_enabled_or_false(+ DUCKLAKE_TRINO_REGISTRATION_FLAG,+ organization_id,+ groups={"organization": organization_id},+ group_properties={"organization": {"id": organization_id}},+ only_evaluate_locally=True,+ send_feature_flag_events=False,+ )+ except Exception as error:+ LOGGER.warning("Failed to evaluate the DuckLake Trino registration feature flag", error=str(error))+ capture_exception(error)+ return False+++def _timestamp_bound(day: date) -> str:+ # The connector reads a TIMESTAMP WITH TIME ZONE bound as an instant with an explicit offset.+ return f"{day.isoformat()} 00:00:00+00"+++def _refuse_shared_table(table: str, change: str) -> None:+ if table in SHARED_TABLES:+ raise ValueError(+ f"Refusing to {change} the shared table '{table}' through Trino, because that would also remove the rows "+ "of other teams"+ )+++def _is_transient(error: BaseException) -> bool:+ # The client already retried each HTTP request, so these mean that the coordinator or the gateway went away.+ if isinstance(error, TrinoConnectionError | Http502Error | Http503Error | Http504Error):+ return True+ return is_transient_trino_error(error) or getattr(error, "error_name", None) in TRANSIENT_DUCKLAKE_ERROR_NAMES+++def _log_retry(state: RetryCallState) -> None:+ error = state.outcome.exception() if state.outcome is not None else None+ LOGGER.warning(+ "A DuckLake backfill statement failed for a transient reason, retrying it",+ attempt=state.attempt_number,+ error_name=getattr(error, "error_name", None),+ error=str(error),+ )+++class TrinoBackfillCatalog:+ """The DuckLake statements of one backfill, run in the organization's Trino catalog."""++ def __init__(self, connection: Connection) -> None:+ self._connection = connection+ self._catalog = escape_trino_identifier(connection.catalog)++ @classmethod+ @contextlib.contextmanager+ def connect(cls, organization_id: str) -> Iterator[TrinoBackfillCatalog]:+ with connect_managed_warehouse_trino(organization_id, principal=BACKFILL_TRINO_PRINCIPAL) as connection:+ yield cls(connection)++ def _table(self, table: str) -> str:+ return f"{self._catalog}.{escape_trino_identifier(BACKFILL_SCHEMA)}.{escape_trino_identifier(table)}"++ @retry(+ retry=retry_if_exception(_is_transient),+ stop=stop_after_attempt(STATEMENT_ATTEMPTS),+ wait=wait_exponential(multiplier=1, min=4, max=60),+ before_sleep=_log_retry,+ reraise=True,+ )+ def _run(self, statement: str) -> list[list[Any]]:+ # Every statement here can run again. DDL has IF [NOT] EXISTS, deleting a range twice deletes nothing more,+ # and add_files records its import_id, so a repeat of an applied import adds nothing.+ cursor = self._connection.cursor()+ try:+ cursor.execute(statement)+ return cursor.fetchall()+ finally:+ cursor.close()++ def ensure_table(+ self, table: str, columns: Mapping[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool+ ) -> None:+ if drop_first:+ _refuse_shared_table(table, "drop")+ self._run(f"DROP TABLE IF EXISTS {self._table(table)}")+ if create_if_missing:+ self._run(f"CREATE SCHEMA IF NOT EXISTS {self._catalog}.{escape_trino_identifier(BACKFILL_SCHEMA)}")+ definitions = ", ".join(+ f"{escape_trino_identifier(name)} {data_type}" for name, data_type in columns.items()+ )+ self._run(f"CREATE TABLE IF NOT EXISTS {self._table(table)} ({definitions})")+ if validate:+ self._validate(table, columns)++ def _validate(self, table: str, columns: Mapping[str, str]) -> None:+ # A table can have more columns than the export, because add_files reads a missing column as NULL.+ rows = self._run(+ f"SELECT column_name, data_type FROM {self._catalog}.information_schema.columns "+ f"WHERE table_schema = {escape_trino_string(BACKFILL_SCHEMA)} AND table_name = {escape_trino_string(table)}"+ )+ actual = {str(name): str(data_type) for name, data_type in rows}+ mismatched = {+ name: actual.get(name, "missing") for name, data_type in columns.items() if actual.get(name) != data_type+ }+ if mismatched:+ raise ValueError(+ f"Table {BACKFILL_SCHEMA}.{table} does not match the backfill export, so add_files would refuse its "+ f"files. Expected {dict(columns)}, but these columns differ: {mismatched}"+ )++ def register_files(+ self, table: str, files: Sequence[str], *, import_id: str, replacement: Replacement | None+ ) -> int:+ """Add the files and remove the rows they replace in one snapshot.++ Repeating a call with the same import_id and files is a no-op, so a call whose response was lost can be+ repeated.+ """+ if replacement is not None:+ _refuse_shared_table(table, "replace rows in")+ if len(files) > ADD_FILES_MAX_FILES:+ raise ValueError(+ f"The export wrote {len(files)} files, but one add_files call registers at most "+ f"{ADD_FILES_MAX_FILES}. Lower max_s3_file_fanout."+ )+ options = [+ f"files => ARRAY[{', '.join(escape_trino_string(path) for path in files)}]",+ f"import_id => {escape_trino_string(import_id)}",+ "allow_missing => true",+ ]+ match replacement:+ case DuckLakeDayReplacement(column=column, day=day):+ options += [+ f"replace_range_column => {escape_trino_string(column)}",+ f"replace_range_start => {escape_trino_string(_timestamp_bound(day))}",+ f"replace_range_end => {escape_trino_string(_timestamp_bound(day + timedelta(days=1)))}",+ ]+ case DuckLakeTableReplacement():+ options.append("replace => true")+ rows = self._run(f"ALTER TABLE {self._table(table)} EXECUTE add_files({', '.join(options)})")+ return sum(int(row[1]) for row in rows if len(row) > 1 and row[0] == "added_data_files")++ def delete_rows(self, table: str, replacement: Replacement) -> None:+ """Remove the rows an export replaces when the export wrote no files."""+ _refuse_shared_table(table, "replace rows in")+ statement = f"DELETE FROM {self._table(table)}"+ if isinstance(replacement, DuckLakeDayReplacement):+ column = escape_trino_identifier(replacement.column)+ start = replacement.day.isoformat()+ end = (replacement.day + timedelta(days=1)).isoformat()+ statement += (+ f" WHERE {column} >= TIMESTAMP '{start} 00:00:00 UTC' AND {column} < TIMESTAMP '{end} 00:00:00 UTC'"+ )+ self._run(statement)diff --git a/products/managed_warehouse/backend/facade/backfill_trino.py b/products/managed_warehouse/backend/facade/backfill_trino.pynew file mode 100644index 000000000000..b74d5c9fa230--- /dev/null+++ b/products/managed_warehouse/backend/facade/backfill_trino.py@@ -0,0 +1,18 @@+"""Register duckling backfill exports in DuckLake through Trino, for warehouses without duckgres."""++from __future__ import annotations++from contextlib import AbstractContextManager++from products.managed_warehouse.backend import backfill_trino+from products.managed_warehouse.backend.facade.contracts import DuckLakeBackfillCatalog++__all__ = ["backfills_through_trino", "connect_backfill_catalog"]+++def backfills_through_trino(organization_id: str) -> bool:+ return backfill_trino.backfills_through_trino(organization_id)+++def connect_backfill_catalog(organization_id: str) -> AbstractContextManager[DuckLakeBackfillCatalog]:+ return backfill_trino.TrinoBackfillCatalog.connect(organization_id)diff --git a/products/managed_warehouse/backend/facade/contracts.py b/products/managed_warehouse/backend/facade/contracts.pyindex 466ff42fa65d..6db19bd36917 100644--- a/products/managed_warehouse/backend/facade/contracts.py+++ b/products/managed_warehouse/backend/facade/contracts.py@@ -12,16 +12,20 @@ behavior change, not a contract improvement.from __future__ import annotations+from collections.abc import Mapping, Sequencefrom dataclasses import dataclass, fieldfrom datetime import date, datetimefrom enum import StrEnum-from typing import Any, Literal+from typing import Any, Literal, Protocolfrom uuid import UUIDfrom posthog.dataclasses import frozen__all__ = ["CPUnavailableError",+ "DuckLakeBackfillCatalog",+ "DuckLakeDayReplacement",+ "DuckLakeTableReplacement","DuckgresQueryServerConfig","DuckgresStoredBucketConfig","DuckgresStoredServerConfig",@@ -428,3 +432,35 @@ class DuckLakeTableResult:# Highest incremental key written, for builds that track one. None when no rows were written.watermark: Any = Nonemerged: bool = False+++@frozen+class DuckLakeDayReplacement:+ """The rows of one UTC day of `column`, which a backfill export of that day replaces."""++ column: str+ day: date+++@frozen+class DuckLakeTableReplacement:+ """All rows of the table, which a full backfill export replaces."""+++class DuckLakeBackfillCatalog(Protocol):+ """The DuckLake statements a backfill runs in the organization's Trino catalog."""++ def ensure_table(+ self, table: str, columns: Mapping[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool+ ) -> None: ...++ def register_files(+ self,+ table: str,+ files: Sequence[str],+ *,+ import_id: str,+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,+ ) -> int: ...++ def delete_rows(self, table: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement) -> None: ...diff --git a/products/managed_warehouse/backend/facade/feature_flags.py b/products/managed_warehouse/backend/facade/feature_flags.pyindex cf1004202adb..e8980d00d8ed 100644--- a/products/managed_warehouse/backend/facade/feature_flags.py+++ b/products/managed_warehouse/backend/facade/feature_flags.py@@ -3,3 +3,5 @@ MANAGED_TRINO_QUERY_FLAG = "managed-warehouse-trino-query"# Either flag gives an organization the managed warehouse.MANAGED_WAREHOUSE_ACCESS_FLAGS = (MANAGED_TRINO_QUERY_FLAG, DATA_WAREHOUSE_SCENE_FLAG)DATA_MODELING_SHADOW_FLAG = "managed-warehouse-data-modeling-shadow"+# Marks an organization whose warehouse has no duckgres, so its data imports and backfills register through Trino.+DUCKLAKE_TRINO_REGISTRATION_FLAG = "managed-warehouse-trino-import-registration"diff --git a/products/managed_warehouse/backend/tests/test_backfill_trino.py b/products/managed_warehouse/backend/tests/test_backfill_trino.pynew file mode 100644index 000000000000..4768ee0e6066--- /dev/null+++ b/products/managed_warehouse/backend/tests/test_backfill_trino.py@@ -0,0 +1,272 @@+from __future__ import annotations++from datetime import date+from typing import TYPE_CHECKING, cast++import pytest+from unittest.mock import patch++from parameterized import parameterized+from trino.exceptions import Http503Error, TrinoConnectionError, TrinoExternalError, TrinoQueryError, TrinoUserError++from products.managed_warehouse.backend.backfill_trino import ADD_FILES_MAX_FILES, TrinoBackfillCatalog+from products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacement++if TYPE_CHECKING:+ from trino.dbapi import Connection++COLUMNS = {"team_id": "bigint", "timestamp": "timestamp(6) with time zone", "person_version": "bigint"}+++class _Cursor:+ def __init__(self, connection: _FakeTrino) -> None:+ self._connection = connection+ self._rows: list[list[object]] = []++ def execute(self, statement: str) -> None:+ self._connection.statements.append(statement)+ if self._connection.failures:+ raise self._connection.failures.pop(0)+ self._rows = self._connection.rows_for(statement)++ def fetchall(self) -> list[list[object]]:+ return self._rows++ def close(self) -> None:+ pass+++class _FakeTrino:+ catalog = "warehouse"++ def __init__(self, columns: dict[str, str] | None = None, failures: list[Exception] | None = None) -> None:+ self.statements: list[str] = []+ self.failures = failures or []+ self._columns = columns or {}++ def cursor(self) -> _Cursor:+ return _Cursor(self)++ def rows_for(self, statement: str) -> list[list[object]]:+ if "information_schema.columns" in statement:+ return [[name, data_type] for name, data_type in self._columns.items()]+ if "EXECUTE add_files" in statement:+ return [["added_data_files", 2]]+ return []+++def _catalog(+ columns: dict[str, str] | None = None, failures: list[Exception] | None = None+) -> tuple[TrinoBackfillCatalog, _FakeTrino]:+ connection = _FakeTrino(columns, failures)+ return TrinoBackfillCatalog(cast("Connection", connection)), connection+++class TestRegisterFiles:+ @parameterized.expand(+ [+ (+ "day",+ DuckLakeDayReplacement(column="timestamp", day=date(2024, 1, 31)),+ "replace_range_column => 'timestamp', replace_range_start => '2024-01-31 00:00:00+00', "+ "replace_range_end => '2024-02-01 00:00:00+00'",+ ),+ ("table", DuckLakeTableReplacement(), "replace => true"),+ ("append", None, ""),+ ]+ )+ def test_registers_all_files_in_one_snapshot_with_the_replacement(+ self, _name: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None, options: str+ ) -> None:+ catalog, connection = _catalog()++ added = catalog.register_files(+ "events_team_7",+ ["s3://bucket/backfill/a.parquet", "s3://bucket/backfill/it's.parquet"],+ import_id="run1_ab12",+ replacement=replacement,+ )++ assert added == 2+ [statement] = connection.statements+ assert statement.startswith(+ 'ALTER TABLE "warehouse"."posthog"."events_team_7" EXECUTE add_files('+ "files => ARRAY['s3://bucket/backfill/a.parquet', 's3://bucket/backfill/it''s.parquet'], "+ "import_id => 'run1_ab12', allow_missing => true"+ )+ assert statement.endswith(f", {options})" if options else "allow_missing => true)")++ @parameterized.expand(+ [+ (+ "register_day",+ lambda c: c.register_files(+ "events",+ ["s3://b/f.parquet"],+ import_id="i",+ replacement=DuckLakeDayReplacement(column="timestamp", day=date(2024, 1, 1)),+ ),+ ),+ (+ "register_table",+ lambda c: c.register_files(+ "persons", ["s3://b/f.parquet"], import_id="i", replacement=DuckLakeTableReplacement()+ ),+ ),+ (+ "delete_day",+ lambda c: c.delete_rows("groups", DuckLakeDayReplacement(column="_timestamp", day=date(2024, 1, 1))),+ ),+ ("delete_table", lambda c: c.delete_rows("events", DuckLakeTableReplacement())),+ ]+ )+ def test_refuses_to_replace_rows_in_a_table_other_teams_share(self, _name: str, action) -> None:+ catalog, connection = _catalog()++ with pytest.raises(ValueError, match="shared"):+ action(catalog)++ assert connection.statements == []++ def test_refuses_more_files_than_one_call_registers(self) -> None:+ catalog, connection = _catalog()+ files = [f"s3://bucket/f{index}.parquet" for index in range(ADD_FILES_MAX_FILES + 1)]++ with pytest.raises(ValueError, match="max_s3_file_fanout"):+ catalog.register_files("events_team_7", files, import_id="i", replacement=None)++ assert connection.statements == []+++class TestDeleteRows:+ @parameterized.expand(+ [+ (+ "day",+ DuckLakeDayReplacement(column="_timestamp", day=date(2024, 12, 31)),+ 'DELETE FROM "warehouse"."posthog"."persons_team_7" '+ "WHERE \"_timestamp\" >= TIMESTAMP '2024-12-31 00:00:00 UTC' "+ "AND \"_timestamp\" < TIMESTAMP '2025-01-01 00:00:00 UTC'",+ ),+ ("table", DuckLakeTableReplacement(), 'DELETE FROM "warehouse"."posthog"."persons_team_7"'),+ ]+ )+ def test_deletes_the_rows_an_empty_export_replaces(+ self, _name: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement, expected: str+ ) -> None:+ catalog, connection = _catalog()++ catalog.delete_rows("persons_team_7", replacement)++ assert connection.statements == [expected]+++class TestEnsureTable:+ def test_creates_the_schema_and_table_with_the_column_types(self) -> None:+ catalog, connection = _catalog(COLUMNS)++ catalog.ensure_table("events_team_7", COLUMNS, drop_first=True, create_if_missing=True, validate=True)++ assert connection.statements[:3] == [+ 'DROP TABLE IF EXISTS "warehouse"."posthog"."events_team_7"',+ 'CREATE SCHEMA IF NOT EXISTS "warehouse"."posthog"',+ 'CREATE TABLE IF NOT EXISTS "warehouse"."posthog"."events_team_7" '+ '("team_id" bigint, "timestamp" timestamp(6) with time zone, "person_version" bigint)',+ ]++ @parameterized.expand([("events",), ("persons",), ("groups",)])+ def test_refuses_to_drop_a_table_other_teams_share(self, table: str) -> None:+ catalog, connection = _catalog(COLUMNS)++ with pytest.raises(ValueError, match="shared"):+ catalog.ensure_table(table, COLUMNS, drop_first=True, create_if_missing=True, validate=True)++ assert connection.statements == []++ @parameterized.expand(+ [+ ("extra_column", {**COLUMNS, "added_later": "varchar"}, None),+ ("missing_column", {"team_id": "bigint", "timestamp": "timestamp(6) with time zone"}, "person_version"),+ ("unsigned_column", {**COLUMNS, "person_version": "decimal(20,0)"}, "person_version"),+ ]+ )+ def test_validates_that_exported_files_match_the_table(+ self, _name: str, actual: dict[str, str], mismatch: str | None+ ) -> None:+ catalog, _ = _catalog(actual)++ def ensure() -> None:+ catalog.ensure_table("events_team_7", COLUMNS, drop_first=False, create_if_missing=False, validate=True)++ if mismatch is None:+ ensure()+ else:+ with pytest.raises(ValueError, match=mismatch):+ ensure()+++def _query_error(error_class: type[TrinoQueryError], name: str, error_type: str) -> TrinoQueryError:+ return error_class({"errorName": name, "errorType": error_type, "message": name}, "query-1")+++def _register(catalog: TrinoBackfillCatalog) -> int:+ return catalog.register_files(+ "events_team_7",+ ["s3://bucket/backfill/a.parquet"],+ import_id="run1_ab12",+ replacement=DuckLakeDayReplacement(column="timestamp", day=date(2024, 1, 31)),+ )+++@patch("tenacity.nap.time.sleep")+class TestTransientFailures:+ @parameterized.expand(+ [+ ("commit_conflict", _query_error(TrinoExternalError, "DUCKLAKE_COMMIT_CONFLICT", "EXTERNAL")),+ (+ "concurrent_modification",+ _query_error(TrinoExternalError, "DUCKLAKE_CONCURRENT_MODIFICATION", "EXTERNAL"),+ ),+ ("commit_retries_exhausted", _query_error(TrinoExternalError, "DUCKLAKE_COMMIT_FAILED", "EXTERNAL")),+ ("cluster_out_of_memory", _query_error(TrinoQueryError, "CLUSTER_OUT_OF_MEMORY", "INSUFFICIENT_RESOURCES")),+ ("worker_lost", _query_error(TrinoQueryError, "TOO_MANY_REQUESTS_FAILED", "INTERNAL_ERROR")),+ ("connection_lost", TrinoConnectionError("failed to fetch: connection reset")),+ ("gateway_unavailable", Http503Error("error 503: service unavailable")),+ ]+ )+ def test_repeats_a_statement_that_failed_for_a_transient_reason(+ self, _sleep, _name: str, failure: Exception+ ) -> None:+ catalog, connection = _catalog(failures=[failure])++ assert _register(catalog) == 2++ assert len(connection.statements) == 2+ assert connection.statements[0] == connection.statements[1]++ @parameterized.expand(+ [+ ("import_id_reused", _query_error(TrinoUserError, "INVALID_PROCEDURE_ARGUMENT", "USER_ERROR")),+ ("file_types_differ", _query_error(TrinoExternalError, "DUCKLAKE_BAD_DATA", "EXTERNAL")),+ ]+ )+ def test_raises_a_statement_error_without_repeating_the_statement(+ self, _sleep, _name: str, failure: Exception+ ) -> None:+ catalog, connection = _catalog(failures=[failure])++ with pytest.raises(type(failure)):+ _register(catalog)++ assert len(connection.statements) == 1++ def test_raises_the_last_transient_failure_after_bounded_attempts(self, _sleep) -> None:+ failures: list[Exception] = [+ _query_error(TrinoExternalError, "DUCKLAKE_COMMIT_CONFLICT", "EXTERNAL") for _ in range(10)+ ]+ catalog, connection = _catalog(failures=failures)++ with pytest.raises(TrinoExternalError):+ _register(catalog)++ assert 1 < len(connection.statements) < 10diff --git a/products/managed_warehouse/backend/tests/test_groups_backfill.py b/products/managed_warehouse/backend/tests/test_groups_backfill.pyindex d8480d874a25..8eab45c3085f 100644--- a/products/managed_warehouse/backend/tests/test_groups_backfill.py+++ b/products/managed_warehouse/backend/tests/test_groups_backfill.py@@ -1,5 +1,6 @@from __future__ import annotations+import refrom collections.abc import Iteratorfrom datetime import UTC, datetime, timedeltafrom pathlib import Path@@ -21,6 +22,8 @@ from posthog.clickhouse.client.connection import get_client_from_poolfrom posthog.dags.events_backfill_to_duckling import DUCKLAKE_ALIAS, DucklingBackfillConfig, DucklingTargetfrom products.managed_warehouse.backend.facade.contracts import (+ DuckLakeDayReplacement,+ DuckLakeTableReplacement,ManagedWarehouseTableNames,ManagedWarehouseTeamMembership,)@@ -28,6 +31,7 @@ from products.managed_warehouse.dags.groups_backfill import (GROUPS_JOB_NAME,GROUPS_REFRESH_DATE_TAG,GROUPS_SNAPSHOT_TAG,+ GROUPS_TRINO_COLUMN_TYPES,MAX_TEAMS_PER_TICK,RETRY_BASE_SECONDS,duckling_groups_backfill,@@ -45,9 +49,10 @@ if TYPE_CHECKING:from dagster import RunRequest[email protected]("for_trino", [False, True])@pytest.mark.parametrize("partition_date", [None, datetime(2025, 1, 1), datetime(2025, 1, 2), datetime(2025, 1, 3)])def test_groups_snapshot_deduplicates_keys_within_each_team_and_group_type(- monkeypatch: pytest.MonkeyPatch, partition_date: datetime | None+ monkeypatch: pytest.MonkeyPatch, partition_date: datetime | None, for_trino: bool) -> None:monkeypatch.setattr(settings, "CLICKHOUSE_USE_HTTP", False)table_name = f"test_groups_snapshot_{uuid4().hex}"@@ -79,10 +84,11 @@ def test_groups_snapshot_deduplicates_keys_within_each_team_and_group_type((7, 0, "tied", '{"name":"last insert"}', created, latest, 1),],)- query = groups_snapshot_select_sql(7, partition_date).replace(+ query = groups_snapshot_select_sql(7, partition_date, for_trino=for_trino).replace("FROM groups FINAL", f"FROM {table_name} FINAL")rows = client.execute(query)+ described = {str(row[0]): str(row[1]) for row in client.execute(f"DESCRIBE ({query})")}finally:client.execute(f"DROP TABLE {table_name}")expected = [@@ -90,9 +96,20 @@ def test_groups_snapshot_deduplicates_keys_within_each_team_and_group_type((7, 0, "tied", '{"name":"last insert"}', created, latest),(7, 1, "company", '{"name":"another type"}', created, latest),]- assert sorted(row[:6] for row in rows) == [+ assert sorted((*row[:4], row[4].replace(tzinfo=None), row[5]) for row in rows) == [row for row in expected if partition_date is None or row[5] == partition_date]+ if for_trino:+ # add_files refuses any type conversion, so these must be the types DuckLake writes for the Trino columns.+ assert described == {+ "team_id": "Int64",+ "group_type_index": "Int16",+ "group_key": "String",+ "group_properties": "String",+ "created_at": "DateTime64(6, 'UTC')",+ "_timestamp": "DateTime64(6)",+ "_inserted_at": "DateTime64(6)",+ }@pytest.mark.parametrize("memory_override", [None, 20 * 1024**3])@@ -288,6 +305,108 @@ def test_groups_backfill_cleans_only_its_partition_and_replays_registration(assert metadata["export_mode"] == ("daily" if daily else "full")+class _FakeGroupsCatalog:+ def __init__(self) -> None:+ self.tables: list[tuple[str, dict[str, str]]] = []+ self.registered: list[tuple[str, list[str], DuckLakeDayReplacement | DuckLakeTableReplacement | None]] = []++ def ensure_table(+ self, table: str, columns: dict[str, str], *, drop_first: bool, create_if_missing: bool, validate: bool+ ) -> None:+ self.tables.append((table, dict(columns)))++ def register_files(+ self,+ table: str,+ files: list[str],+ *,+ import_id: str,+ replacement: DuckLakeDayReplacement | DuckLakeTableReplacement | None,+ ) -> int:+ self.registered.append((table, list(files), replacement))+ return len(files)+++ "partition_key,replacement",+ [+ ("7", DuckLakeTableReplacement()),+ ("7_2025-12-31", DuckLakeDayReplacement(column="_timestamp", day=datetime(2025, 12, 31).date())),+ ],+)+@time_machine.travel(datetime(2026, 1, 1, tzinfo=UTC), tick=False)+def test_groups_backfill_registers_through_trino_without_duckgres(+ partition_key: str, replacement: DuckLakeDayReplacement | DuckLakeTableReplacement+) -> None:+ inserts, catalog, connect_catalog, context = _run_groups_through_trino(partition_key, DucklingBackfillConfig())++ connect_catalog.assert_called_once_with("org")+ assert catalog.tables == [("groups_team_7", GROUPS_TRINO_COLUMN_TYPES)]+ [(table, files, registered_replacement)] = catalog.registered+ assert (table, registered_replacement) == ("groups_team_7", replacement)+ [insert] = inserts+ export_path = re.search(r"s3\('([^']+)'", insert)+ assert export_path is not None+ assert files == [+ export_path[1].replace("https://bkt.s3.us-east-1.amazonaws.com/", "s3://bkt/").replace("{_partition_id}", "0")+ ]+ assert "toInt16(group_type_index) AS group_type_index" in insert+ assert "toDateTime64(created_at, 6, 'UTC') AS created_at" in insert+ assert context.add_output_metadata.call_args.args[0]["files_registered"] == 1++[email protected]("partition_key", ["7", "7_2025-12-31"])+@time_machine.travel(datetime(2026, 1, 1, tzinfo=UTC), tick=False)+def test_groups_export_only_writes_the_trino_types_without_registering(partition_key: str) -> None:+ inserts, catalog, connect_catalog, _ = _run_groups_through_trino(+ partition_key, DucklingBackfillConfig(skip_ducklake_registration=True)+ )++ connect_catalog.assert_not_called()+ assert catalog.registered == []+ [insert] = inserts+ assert "toInt16(group_type_index) AS group_type_index" in insert+ assert "toDateTime64(created_at, 6, 'UTC') AS created_at" in insert+++def _run_groups_through_trino(+ partition_key: str, config: DucklingBackfillConfig+) -> tuple[list[str], _FakeGroupsCatalog, MagicMock, MagicMock]:+ target = DucklingTarget(team_id=7, organization_id="org", bucket="bkt", bucket_region="us-east-1")+ inserts: list[str] = []++ def execute(query: str, params: object = None, settings: object = None) -> list[tuple[object, ...]]:+ if "INSERT INTO FUNCTION" in query:+ inserts.append(query)+ return []+ if "'One'" in query:+ return [("0.parquet",)]+ return [(10,)]++ client = MagicMock()+ client.execute.side_effect = execute+ cluster = MagicMock()+ cluster.any_host_by_role.side_effect = lambda fn, **_: MagicMock(result=MagicMock(return_value=fn(client)))+ catalog = _FakeGroupsCatalog()+ context = MagicMock()+ context.partition_key = partition_key+ context.run.run_id = "synthetic-run"+ with (+ patch("products.managed_warehouse.dags.groups_backfill._resolve_duckling_target", return_value=target),+ patch("products.managed_warehouse.dags.groups_backfill.resolve_groups_table", return_value="groups_team_7"),+ patch("products.managed_warehouse.dags.groups_backfill.backfills_through_trino", return_value=True),+ patch("products.managed_warehouse.dags.groups_backfill._DuckgresSession") as duckgres,+ patch("products.managed_warehouse.dags.groups_backfill.record_duckling_backfill_workload"),+ patch("posthog.dags.events_backfill_to_duckling._get_cluster", return_value=cluster),+ patch("posthog.dags.events_backfill_to_duckling.connect_backfill_catalog") as connect_catalog,+ ):+ connect_catalog.return_value.__enter__.return_value = catalog+ run_groups_backfill(context, config)++ duckgres.assert_not_called()+ return inserts, catalog, connect_catalog, context++_NOW = datetime(2026, 1, 1, tzinfo=UTC)diff --git a/products/managed_warehouse/dags/groups_backfill.py b/products/managed_warehouse/dags/groups_backfill.pyindex fc4e6a798846..1b2ddefa994d 100644--- a/products/managed_warehouse/dags/groups_backfill.py+++ b/products/managed_warehouse/dags/groups_backfill.py@@ -1,3 +1,4 @@+import functoolsfrom datetime import UTC, datetime, timedeltafrom typing import TYPE_CHECKING@@ -31,15 +32,19 @@ from posthog.dags.events_backfill_to_duckling import (DUCKLAKE_ALIAS,DUCKLING_BACKFILL_CONCURRENCY_TAG,PARQUET_WRITER_SETTINGS,+ TRINO_EXPORT_SETTINGS,DucklingBackfillConfig,DucklingTarget,+ TrinoExport,_compute_fanout,_DuckgresSession,_execute_export_with_retry,_get_cluster,_glob_run_files,+ _merged_clickhouse_settings,_repair_stale_running_statuses,_resolve_duckling_target,+ backfill_through_trino,get_s3_url_for_clickhouse,is_full_export_partition,parse_partition_key_dates,@@ -47,6 +52,8 @@ from posthog.dags.events_backfill_to_duckling import (from products.data_warehouse.backend.facade.backfill_status import record_backfill_finished, record_backfill_startedfrom products.data_warehouse.backend.facade.models import ManagedWarehouseBackfillPartition+from products.managed_warehouse.backend.facade.backfill_trino import backfills_through_trino+from products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacementfrom products.managed_warehouse.backend.facade.metrics import record_duckling_backfill_workload, track_duckling_backfillfrom products.managed_warehouse.backend.facade.team_state import (list_enabled_backfill_team_memberships,@@ -75,9 +82,20 @@ GROUPS_COLUMN_TYPES = {"_timestamp": "TIMESTAMP WITH TIME ZONE","_inserted_at": "TIMESTAMP WITH TIME ZONE",}+# Trino has no unsigned types, and add_files refuses any type conversion, so the Trino export casts+# group_type_index to Int16 and the millisecond created_at to microseconds.+GROUPS_TRINO_COLUMN_TYPES = {+ "team_id": "bigint",+ "group_type_index": "smallint",+ "group_key": "varchar",+ "group_properties": "varchar",+ "created_at": "timestamp(6) with time zone",+ "_timestamp": "timestamp(6) with time zone",+ "_inserted_at": "timestamp(6) with time zone",+}-def groups_snapshot_select_sql(team_id: int, date: datetime | None = None) -> str:+def groups_snapshot_select_sql(team_id: int, date: datetime | None = None, *, for_trino: bool = False) -> str:if team_id <= 0:raise ValueError("Groups backfill requires a positive team ID")date_filter = ""@@ -85,9 +103,11 @@ def groups_snapshot_select_sql(team_id: int, date: datetime | None = None) -> ststart = escape_clickhouse_string(date.strftime("%Y-%m-%d"))end = escape_clickhouse_string((date + timedelta(days=1)).strftime("%Y-%m-%d"))date_filter = f"AND _timestamp >= toDateTime({start}, 'UTC') AND _timestamp < toDateTime({end}, 'UTC')"+ group_type_index = "toInt16(group_type_index) AS group_type_index" if for_trino else "group_type_index"+ created_at = "toDateTime64(created_at, 6, 'UTC') AS created_at" if for_trino else "created_at"return f"""- SELECT team_id, group_type_index, group_key,- group_properties, created_at,+ SELECT team_id, {group_type_index}, group_key,+ group_properties, {created_at},toDateTime64(_timestamp, 6) AS _timestamp,now64(6) AS _inserted_atFROM groups FINAL@@ -101,9 +121,12 @@ def export_groups_to_duckling_s3(config: DucklingBackfillConfig,target: DucklingTarget,date: datetime | None = None,+ *,+ run_id: str | None = None,+ for_trino: bool = False,) -> str | None:period = f"/{date.strftime('%Y-%m-%d')}" if date is not None else ""- path = f"backfill/groups/{target.team_id}{period}/{context.run.run_id}/{{_partition_id}}.parquet"+ path = f"backfill/groups/{target.team_id}{period}/{run_id or context.run.run_id}/{{_partition_id}}.parquet"s3_url = get_s3_url_for_clickhouse(target.bucket, target.bucket_region, path)s3_glob = f"s3://{target.bucket}/{path.replace('{_partition_id}', '*')}"if config.dry_run:@@ -117,6 +140,8 @@ def export_groups_to_duckling_s3(**(config.clickhouse_settings or {}),}settings.update(PARQUET_WRITER_SETTINGS)+ if for_trino:+ settings.update(TRINO_EXPORT_SETTINGS)count_sql = "SELECT count() FROM groups WHERE team_id = %(team_id)s"count_params: dict[str, int | str] = {"team_id": target.team_id}if date is not None:@@ -127,7 +152,7 @@ def export_groups_to_duckling_s3(export_sql = f"""INSERT INTO FUNCTION s3({escape_clickhouse_string(s3_url)}, 'Parquet')PARTITION BY toString(cityHash64(group_type_index, group_key) % {fanout})- {groups_snapshot_select_sql(target.team_id, date)}+ {groups_snapshot_select_sql(target.team_id, date, for_trino=for_trino)}SETTINGS s3_truncate_on_insert=1, use_hive_partitioning=0"""# A failed export can leave partial S3 objects; only register files after the whole query succeeds.@@ -191,6 +216,36 @@ def register_groups_files_with_duckling(return len(files)+def _report_groups_backfill(+ context: AssetExecutionContext,+ config: DucklingBackfillConfig,+ *,+ team_id: int,+ mode: str,+ table_name: str,+ files_registered: int,+ partitions_exported: int,+) -> None:+ context.add_output_metadata(+ {+ "team_id": team_id,+ "partition_key": context.partition_key,+ "export_mode": mode,+ "groups_table": table_name,+ "files_registered": files_registered,+ "partitions_exported": partitions_exported,+ }+ )+ if not config.dry_run and not config.skip_ducklake_registration:+ record_duckling_backfill_workload(+ team_id=team_id,+ dataset=GROUPS_DATASET,+ mode=mode,+ files_registered=files_registered,+ partitions_exported=partitions_exported,+ )++def run_groups_backfill(context: AssetExecutionContext, config: DucklingBackfillConfig) -> None:partition_key = context.partition_keyis_full = is_full_export_partition(partition_key)@@ -206,8 +261,46 @@ def run_groups_backfill(context: AssetExecutionContext, config: DucklingBackfilltarget = _resolve_duckling_target(team_id)table_name = resolve_groups_table(team_id)tags = dagster_tags(context)- should_use_duckgres = not (config.dry_run or config.skip_ducklake_registration)- session = _DuckgresSession(context, target) if should_use_duckgres else None+ if backfills_through_trino(target.organization_id):+ result = backfill_through_trino(+ context,+ config,+ target,+ table=table_name,+ column_types=GROUPS_TRINO_COLUMN_TYPES,+ settings=_merged_clickhouse_settings(context, config),+ exports=[+ TrinoExport(+ label=partition_date.strftime("%Y-%m-%d") if partition_date is not None else "full",+ replacement=(+ DuckLakeTableReplacement()+ if partition_date is None+ else DuckLakeDayReplacement(column="_timestamp", day=partition_date.date())+ ),+ export=functools.partial(+ export_groups_to_duckling_s3,+ context=context,+ config=config,+ target=target,+ date=partition_date,+ for_trino=True,+ ),+ )+ for partition_date in dates+ ],+ )+ _report_groups_backfill(+ context,+ config,+ team_id=team_id,+ mode=mode,+ table_name=table_name,+ files_registered=result.files_registered,+ partitions_exported=result.partitions_exported,+ )+ return+ should_register = not (config.dry_run or config.skip_ducklake_registration)+ session = _DuckgresSession(context, target) if should_register else Nonefiles_registered = 0partitions_exported = 0try:@@ -253,24 +346,15 @@ def run_groups_backfill(context: AssetExecutionContext, config: DucklingBackfillfinally:if session is not None:session.close()- context.add_output_metadata(- {- "team_id": team_id,- "partition_key": partition_key,- "export_mode": mode,- "groups_table": table_name,- "files_registered": files_registered,- "partitions_exported": partitions_exported,- }+ _report_groups_backfill(+ context,+ config,+ team_id=team_id,+ mode=mode,+ table_name=table_name,+ files_registered=files_registered,+ partitions_exported=partitions_exported,)- if not config.dry_run and not config.skip_ducklake_registration:- record_duckling_backfill_workload(- team_id=team_id,- dataset=GROUPS_DATASET,- mode=mode,- files_registered=files_registered,- partitions_exported=partitions_exported,- )@asset(partitions_def=duckling_groups_partitions_def, tags=GROUPS_TAGS)