:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): record bytes read by trino materializations (#121173)
Jovan Sakovic491205a54ea0
diff --git a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.pyindex 9976e02a5fce..4cee3b3e69b1 100644--- a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py+++ b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py@@ -99,6 +99,7 @@ class ManagedWarehouseShadowResult:file_size_bytes: int = 0file_size_delta_bytes: int = 0run_mode: str = UNKNOWN_RUN_MODE+ bytes_read: int | None = Nonedef _failed_run_mode(plan: WritePlan | None, use_trino: bool) -> str:@@ -235,6 +236,7 @@ def _resolve_managed_warehouse_job(job_id: str, result: "ManagedWarehouseShadowRjob.status = DataModelingJobStatus.FAILEDjob.rows_materialized = 0job.error = result.error+ job.bytes_read = result.bytes_readjob.last_run_at = dt.datetime.now(dt.UTC)job.save()return job.engine@@ -265,6 +267,7 @@ async def _materialize_view_managed_warehouse(values: dict[str, object] = {}s3_secrets: tuple[DuckLakeS3Secret, ...] = ()plan: WritePlan | None = None+ processed_bytes: list[int] = []try:from products.managed_warehouse.backend.facade.client import execute_ducklake_create_table, execute_trino_model@@ -287,6 +290,7 @@ async def _materialize_view_managed_warehouse(saved_query_id=saved_query.id,source_query=saved_query.query,incremental=_trino_incremental_write(plan),+ on_processed_bytes=processed_bytes.append,)if plan.config is not None:# A quiet window writes nothing, so it records the run without moving the watermark.@@ -339,6 +343,7 @@ async def _materialize_view_managed_warehouse(file_size_bytes=result.file_size_bytes,file_size_delta_bytes=result.file_size_delta_bytes,run_mode=INCREMENTAL_RUN_MODE if inputs.use_trino and result.merged else FULL_REFRESH_RUN_MODE,+ bytes_read=_total_bytes_read(processed_bytes),)job_engine = await _resolve_managed_warehouse_job(inputs.job_id, shadow_result)await clear_node_suspension_for_engine(@@ -364,6 +369,7 @@ async def _materialize_view_managed_warehouse(table_name=table_name,error=str(e),run_mode=_failed_run_mode(plan, inputs.use_trino),+ bytes_read=_total_bytes_read(processed_bytes),)if plan is not None and plan.incremental:# The merge may have stopped partway or failed on a schema change. Rebuilding next run@@ -390,6 +396,11 @@ async def _materialize_view_managed_warehouse(return shadow_result+def _total_bytes_read(processed_bytes: list[int]) -> int | None:+ # No statement reported its bytes, so the run was not measured. That is not the same as reading zero bytes.+ return sum(processed_bytes) if processed_bytes else None++def _trino_incremental_write(plan: WritePlan) -> TrinoIncrementalWrite | None:if plan.config is None:return Nonediff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex 4dbfa7fd4e37..ff1b6089456c 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -179,6 +179,14 @@ class TestMaterializeViewManagedWarehouseActivity:asaved_query.origin = DataWarehouseSavedQuery.Origin.ENDPOINTawait database_sync_to_async(asaved_query.save)(update_fields=["origin"])executable_query = {"kind": "HogQLQuery", "query": "SELECT 2"}++ def execute_trino_model(**kwargs: Any) -> DuckLakeTableResult:+ kwargs["on_processed_bytes"](300)+ kwargs["on_processed_bytes"](45)+ if compile_fails:+ raise ValueError("Trino compilation failed")+ return DuckLakeTableResult(schema_name="shadow_models", table_name="test_model", row_count=12)+with (unittest.mock.patch("products.managed_warehouse.backend.facade.client.request_model_alias_reconciliation",@@ -186,8 +194,7 @@ class TestMaterializeViewManagedWarehouseActivity:) as reconcile_aliases,unittest.mock.patch("products.managed_warehouse.backend.facade.client.execute_trino_model",- return_value=DuckLakeTableResult(schema_name="shadow_models", table_name="test_model", row_count=12),- side_effect=ValueError("Trino compilation failed") if compile_fails else None,+ side_effect=execute_trino_model,) as execute,unittest.mock.patch("products.managed_warehouse.backend.facade.client.execute_ducklake_create_table"@@ -205,9 +212,11 @@ class TestMaterializeViewManagedWarehouseActivity:saved_query_id=asaved_query.id,source_query=executable_query,incremental=None,+ on_processed_bytes=unittest.mock.ANY,)legacy_execute.assert_not_called()await database_sync_to_async(ajob.refresh_from_db)()+ assert ajob.bytes_read == 345if compile_fails:reconcile_aliases.assert_not_awaited()assert result.error == "Trino compilation failed"diff --git a/products/data_modeling/backend/migrations/0049_datamodelingjob_bytes_read.py b/products/data_modeling/backend/migrations/0049_datamodelingjob_bytes_read.pynew file mode 100644index 000000000000..7385fb615a9a--- /dev/null+++ b/products/data_modeling/backend/migrations/0049_datamodelingjob_bytes_read.py@@ -0,0 +1,21 @@+# Generated by Django 5.2.18 on 2026-10-21 12:38++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("data_modeling", "0048_drop_datawarehousemodelpath_table"),+ ]++ operations = [+ migrations.AddField(+ model_name="datamodelingjob",+ name="bytes_read",+ field=models.BigIntegerField(+ blank=True,+ help_text="Bytes the engine read to build the model. Null when the run was not measured.",+ null=True,+ ),+ ),+ ]diff --git a/products/data_modeling/backend/migrations/max_migration.txt b/products/data_modeling/backend/migrations/max_migration.txtindex f70645e6b2bf..fd735afd57e9 100644--- a/products/data_modeling/backend/migrations/max_migration.txt+++ b/products/data_modeling/backend/migrations/max_migration.txt@@ -1 +1 @@-0048_drop_datawarehousemodelpath_table+0049_datamodelingjob_bytes_readdiff --git a/products/data_modeling/backend/models/data_modeling_job.py b/products/data_modeling/backend/models/data_modeling_job.pyindex e1c97f402df7..fc97c7776ed1 100644--- a/products/data_modeling/backend/models/data_modeling_job.py+++ b/products/data_modeling/backend/models/data_modeling_job.py@@ -55,6 +55,11 @@ class DataModelingJob(CreatedMetaFields, UpdatedMetaFields, UUIDTModel):last_run_at = models.DateTimeField(default=timezone.now)rows_expected = models.IntegerField(null=True, blank=True, help_text="Total rows expected to be materialized")storage_delta_mib = models.FloatField(null=True, blank=True, default=0)+ bytes_read = models.BigIntegerField(+ null=True,+ blank=True,+ help_text="Bytes the engine read to build the model. Null when the run was not measured.",+ )class Meta:db_table = "posthog_datamodelingjob"diff --git a/products/managed_warehouse/backend/facade/client.py b/products/managed_warehouse/backend/facade/client.pyindex 5fc3ab254ebb..f25342261292 100644--- a/products/managed_warehouse/backend/facade/client.py+++ b/products/managed_warehouse/backend/facade/client.py@@ -12,7 +12,7 @@ result types live in ``facade.contracts``.from __future__ import annotations-from collections.abc import Sequence+from collections.abc import Callable, Sequencefrom contextlib import AbstractContextManagerfrom typing import TYPE_CHECKINGfrom uuid import UUID@@ -239,7 +239,10 @@ async def execute_trino_model(saved_query_id: str | UUID,source_query: object,incremental: TrinoIncrementalWrite | None = None,+ on_processed_bytes: Callable[[int], None] | None = None,) -> DuckLakeTableResult:+ """Build the model on Trino. ``on_processed_bytes`` receives the bytes each statement processed,+ including statements of attempts that fail."""from products.managed_warehouse.backend.trino_execution import ( # noqa: PLC0415 -- keeps executor creation off startup pathsrun_trino_model_with_retries,)@@ -256,4 +259,5 @@ async def execute_trino_model(incremental=incremental,control=control,),+ on_processed_bytes=on_processed_bytes,)diff --git a/products/managed_warehouse/backend/tests/test_trino_execution.py b/products/managed_warehouse/backend/tests/test_trino_execution.pyindex 3b8d10434ef7..1243ab4dee08 100644--- a/products/managed_warehouse/backend/tests/test_trino_execution.py+++ b/products/managed_warehouse/backend/tests/test_trino_execution.py@@ -70,17 +70,27 @@ async def test_success_preserves_result() -> None:)async def test_retries_only_cluster_out_of_memory(error_name: str, expected_attempts: int) -> None:attempts = 0+ processed: list[int] = []def execute(control: execution.TrinoQueryControl) -> int:nonlocal attemptsattempts += 1+ control.record_processed_bytes(10)if attempts == 1:raise TrinoQueryError({"errorName": error_name, "message": "Query killed"})return 42if expected_attempts == 1:with pytest.raises(TrinoQueryError):- await execution.run_trino_model_with_retries(execute, query_seconds=60, retry_delays=(0,))+ await execution.run_trino_model_with_retries(+ execute, query_seconds=60, retry_delays=(0,), on_processed_bytes=processed.append+ )else:- assert await execution.run_trino_model_with_retries(execute, query_seconds=60, retry_delays=(0,)) == 42+ assert (+ await execution.run_trino_model_with_retries(+ execute, query_seconds=60, retry_delays=(0,), on_processed_bytes=processed.append+ )+ == 42+ )assert attempts == expected_attempts+ assert processed == [10] * expected_attemptsdiff --git a/products/managed_warehouse/backend/tests/test_trino_materialization.py b/products/managed_warehouse/backend/tests/test_trino_materialization.pyindex 8d53fee7538e..88540c49d3b7 100644--- a/products/managed_warehouse/backend/tests/test_trino_materialization.py+++ b/products/managed_warehouse/backend/tests/test_trino_materialization.py@@ -29,7 +29,11 @@ from products.managed_warehouse.backend.models import (ManagedWarehouseViewTranslationResult,)from products.managed_warehouse.backend.table_binding import build_trino_table_locators-from products.managed_warehouse.backend.trino_materialization import DuplicateUniqueKeyError+from products.managed_warehouse.backend.trino_execution import TrinoQueryControl+from products.managed_warehouse.backend.trino_materialization import (+ DuplicateUniqueKeyError,+ execute_trino_shadow_materialization as execute_trino_model_statements,+)from products.managed_warehouse.backend.view_translation_status import source_query_hashfrom products.warehouse_sources.backend.facade.models import DataWarehouseTable@@ -377,7 +381,16 @@ class TestTrinoShadowMaterialization(BaseTest):class _ScriptedCursor:"""Answers each statement by its leading words, and records every statement it ran."""- def __init__(self, *, table_exists: bool, staged: int, duplicate: bool = False, max_key: Any = None) -> None:+ def __init__(+ self,+ *,+ table_exists: bool,+ staged: int,+ duplicate: bool = False,+ max_key: Any = None,+ processed_bytes: int = 0,+ unreachable_from: str | None = None,+ ) -> None:self.table_exists = table_existsself.staged = stagedself.duplicate = duplicate@@ -385,11 +398,17 @@ class _ScriptedCursor:self.statements: list[tuple[str, Any]] = []self.rowcount = -1self.description: list[tuple[str]] | None = None+ self.processed_bytes = processed_bytes+ self.unreachable_from = unreachable_from+ self.stats: dict[str, Any] | None = Noneself._rows: list[tuple[Any, ...]] = []def execute(self, statement: str, parameters: Any = None) -> None:+ if self.unreachable_from and statement.startswith(self.unreachable_from):+ raise ConnectionError("coordinator unreachable")self.statements.append((statement, parameters))self.rowcount, self._rows = -1, []+ self.stats = {"processedBytes": self.processed_bytes}if "information_schema.tables" in statement:self._rows = [(1,)] if self.table_exists else []elif statement.startswith("CREATE OR REPLACE TABLE"):@@ -416,7 +435,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):since = dt.datetime(2026, 10, 1, tzinfo=dt.UTC)def run_build(- self, cursor: _ScriptedCursor, since: Any+ self, cursor: _ScriptedCursor, since: Any, control: TrinoQueryControl | None = None) -> tuple[DuckLakeTableResult | Exception, list[str], list[Any]]:compiled = TrinoCompiledQuery(sql="SELECT id, ts FROM orders", values={})connection = MagicMock(catalog="cat")@@ -432,12 +451,13 @@ class TestTrinoIncrementalMaterialization(BaseTest):):connect.return_value.__enter__.return_value = connectiontry:- result: DuckLakeTableResult | Exception = execute_trino_shadow_materialization(+ result: DuckLakeTableResult | Exception = execute_trino_model_statements(organization_id=str(self.organization.pk),team_id=self.team.pk,saved_query_id=self.saved_query_id,source_query=self.query,incremental=TrinoIncrementalWrite(incremental_key="ts", unique_key=("id",), since=since),+ control=control,)except Exception as error:result = error@@ -497,3 +517,28 @@ class TestTrinoIncrementalMaterialization(BaseTest):assert f"CREATE OR REPLACE TABLE {table} AS SELECT id, ts FROM orders" in statementsassert not any("__ph_incremental_stage" in statement for statement in statements)assert statements[-1] == f'SELECT max("ts") FROM {table}'++ @parameterized.expand(+ [+ # Schema, table check, stage build, duplicate check, column read, merge and watermark read.+ ("merge_succeeds", False, 7),+ # The run stops at the duplicate check, and its statements still count.+ ("duplicate_keys_fail_the_merge", True, 4),+ ]+ )+ def test_reports_bytes_processed_by_every_statement(self, _name: str, duplicate: bool, statements: int) -> None:+ processed: list[int] = []+ cursor = _ScriptedCursor(table_exists=True, staged=3, duplicate=duplicate, processed_bytes=100)++ self.run_build(cursor, self.since, TrinoQueryControl(on_processed_bytes=processed.append))++ assert processed == [100] * statements++ def test_a_statement_that_never_starts_does_not_recount_the_previous_one(self) -> None:+ processed: list[int] = []+ cursor = _ScriptedCursor(table_exists=True, staged=3, processed_bytes=100, unreachable_from="MERGE")++ result, _, _ = self.run_build(cursor, self.since, TrinoQueryControl(on_processed_bytes=processed.append))++ assert isinstance(result, ConnectionError)+ assert processed == [100] * 5diff --git a/products/managed_warehouse/backend/trino_execution.py b/products/managed_warehouse/backend/trino_execution.pyindex 6b1833dba319..29748ed21be9 100644--- a/products/managed_warehouse/backend/trino_execution.py+++ b/products/managed_warehouse/backend/trino_execution.py@@ -28,11 +28,16 @@ logger = structlog.get_logger(__name__)class TrinoQueryControl:- def __init__(self, query_seconds: float = TRINO_QUERY_SECONDS) -> None:+ def __init__(+ self,+ query_seconds: float = TRINO_QUERY_SECONDS,+ on_processed_bytes: Callable[[int], None] | None = None,+ ) -> None:self.cancelled = threading.Event()self.finished = threading.Event()self.cursor: Cursor | None = Noneself.deadline = monotonic() + query_seconds+ self.on_processed_bytes = on_processed_bytesdef checkpoint(self) -> None:if self.cancelled.is_set() or monotonic() >= self.deadline:@@ -41,6 +46,10 @@ class TrinoQueryControl:def attach(self, cursor: Cursor) -> None:self.cursor = cursor+ def record_processed_bytes(self, processed_bytes: object) -> None:+ if self.on_processed_bytes is not None and isinstance(processed_bytes, int):+ self.on_processed_bytes(processed_bytes)+def _cancel_until_finished(self) -> None:# execute() can still be waiting for its first nextUri when cancellation arrives.while not self.finished.is_set():@@ -58,9 +67,12 @@ class TrinoQueryControl:async def run_trino_model(- execute: Callable[[TrinoQueryControl], _Result], *, query_seconds: float = TRINO_QUERY_SECONDS+ execute: Callable[[TrinoQueryControl], _Result],+ *,+ query_seconds: float = TRINO_QUERY_SECONDS,+ on_processed_bytes: Callable[[int], None] | None = None,) -> _Result:- control = TrinoQueryControl(query_seconds)+ control = TrinoQueryControl(query_seconds, on_processed_bytes)task = asyncio.create_task(database_sync_to_async_pool(execute, executor=_executor)(control))timeout = asyncio.timeout(query_seconds)try:@@ -91,15 +103,21 @@ async def run_trino_model_with_retries(*,query_seconds: float = TRINO_QUERY_SECONDS,retry_delays: tuple[float, ...] = CLUSTER_OUT_OF_MEMORY_RETRY_DELAYS_SECONDS,+ on_processed_bytes: Callable[[int], None] | None = None,) -> _Result:- """Run ``execute`` again after a cluster out-of-memory kill. All attempts share one deadline."""+ """Run ``execute`` again after a cluster out-of-memory kill. All attempts share one deadline.++ ``on_processed_bytes`` receives the bytes each statement processed, in every attempt.+ """deadline = monotonic() + query_secondsfor delay in retry_delays:try:- return await run_trino_model(execute, query_seconds=deadline - monotonic())+ return await run_trino_model(+ execute, query_seconds=deadline - monotonic(), on_processed_bytes=on_processed_bytes+ )except Exception as error:if not is_cluster_out_of_memory(error) or monotonic() + delay >= deadline:raiselogger.warning("Trino cluster is out of memory, retrying the model", retry_in_seconds=delay)await asyncio.sleep(delay)- return await run_trino_model(execute, query_seconds=deadline - monotonic())+ return await run_trino_model(execute, query_seconds=deadline - monotonic(), on_processed_bytes=on_processed_bytes)diff --git a/products/managed_warehouse/backend/trino_materialization.py b/products/managed_warehouse/backend/trino_materialization.pyindex 1fda738e7246..d82af1125f9d 100644--- a/products/managed_warehouse/backend/trino_materialization.py+++ b/products/managed_warehouse/backend/trino_materialization.py@@ -140,13 +140,7 @@ def _merge(return 0, Nonekeys = [escape_trino_identifier(key) for key in incremental.unique_key]- _run(- cursor,- control,- f"SELECT 1 FROM {stage} GROUP BY {', '.join(keys)} HAVING count(*) > 1 LIMIT 1",- fetch=False,- )- if cursor.fetchall():+ if _fetch(cursor, control, f"SELECT 1 FROM {stage} GROUP BY {', '.join(keys)} HAVING count(*) > 1 LIMIT 1"):raise DuplicateUniqueKeyError(f"The incremental rows hold more than one row per unique key ({', '.join(incremental.unique_key)})")@@ -178,20 +172,18 @@ def _drop_stage(cursor: Cursor, stage: str) -> None:def _table_exists(cursor: Cursor, control: TrinoQueryControl | None, catalog: str, schema_name: str, table_name: str) -> bool:- _run(+ rows = _fetch(cursor,control,f"SELECT 1 FROM {escape_trino_identifier(catalog)}.information_schema.tables ""WHERE table_schema = ? AND table_name = ? AND table_type = 'BASE TABLE'",[schema_name, table_name],- fetch=False,)- return bool(cursor.fetchall())+ return bool(rows)def _max_key(cursor: Cursor, control: TrinoQueryControl | None, table: str, incremental_key: str) -> Any:- _run(cursor, control, f"SELECT max({escape_trino_identifier(incremental_key)}) FROM {table}", fetch=False)- rows = cursor.fetchall()+ rows = _fetch(cursor, control, f"SELECT max({escape_trino_identifier(incremental_key)}) FROM {table}")return rows[0][0] if rows else None@@ -200,23 +192,37 @@ def _run(control: TrinoQueryControl | None,statement: str,parameters: list[object] | None = None,- *,- fetch: bool = True,) -> int:- """Run one statement and return the row count Trino reports for a write.-- ``fetch=False`` leaves the result for the caller to read.- """- if control:- control.checkpoint()- if parameters:- cursor.execute(statement, parameters)- else:- cursor.execute(statement)- if not fetch:- return -1- cursor.fetchall()+ """Run one statement and return the row count Trino reports for a write."""+ _fetch(cursor, control, statement, parameters)row_count = cursor.rowcountif statement.startswith(("CREATE OR REPLACE TABLE", "MERGE")) and row_count < 0:raise RuntimeError("Trino did not report the materialized row count")return row_count+++def _fetch(+ cursor: Cursor,+ control: TrinoQueryControl | None,+ statement: str,+ parameters: list[object] | None = None,+) -> list[Any]:+ """Run one statement, read all of its rows, and report the bytes it processed to ``control``."""+ if control:+ control.checkpoint()+ stats_before = cursor.stats+ try:+ if parameters:+ cursor.execute(statement, parameters)+ else:+ cursor.execute(statement)+ return cursor.fetchall()+ finally:+ # The client updates the statistics with every response, so they are final once the rows are read.+ # A failed statement reports what it processed as of the last response before the error: the client+ # raises on the error response before it reads that response's statistics, so the figure is a lower bound.+ # When ``execute`` fails before it starts a query, the cursor still holds the previous statement's+ # statistics, which are already counted.+ stats = cursor.stats+ if control is not None and stats is not None and stats is not stats_before:+ control.record_processed_bytes(stats.get("processedBytes"))