Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): record bytes read by trino materializations (#121173)

Jovan Sakovic491205a54ea0
diff --git a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
index 9976e02a5fce..4cee3b3e69b1 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
@@ -99,6 +99,7 @@ class ManagedWarehouseShadowResult:
file_size_bytes: int = 0
file_size_delta_bytes: int = 0
run_mode: str = UNKNOWN_RUN_MODE
+ bytes_read: int | None = None
def _failed_run_mode(plan: WritePlan | None, use_trino: bool) -> str:
@@ -235,6 +236,7 @@ def _resolve_managed_warehouse_job(job_id: str, result: "ManagedWarehouseShadowR
job.status = DataModelingJobStatus.FAILED
job.rows_materialized = 0
job.error = result.error
+ job.bytes_read = result.bytes_read
job.last_run_at = dt.datetime.now(dt.UTC)
job.save()
return job.engine
@@ -265,6 +267,7 @@ async def _materialize_view_managed_warehouse(
values: dict[str, object] = {}
s3_secrets: tuple[DuckLakeS3Secret, ...] = ()
plan: WritePlan | None = None
+ processed_bytes: list[int] = []
try:
from products.managed_warehouse.backend.facade.client import execute_ducklake_create_table, execute_trino_model
@@ -287,6 +290,7 @@ async def _materialize_view_managed_warehouse(
saved_query_id=saved_query.id,
source_query=saved_query.query,
incremental=_trino_incremental_write(plan),
+ on_processed_bytes=processed_bytes.append,
)
if plan.config is not None:
# A quiet window writes nothing, so it records the run without moving the watermark.
@@ -339,6 +343,7 @@ async def _materialize_view_managed_warehouse(
file_size_bytes=result.file_size_bytes,
file_size_delta_bytes=result.file_size_delta_bytes,
run_mode=INCREMENTAL_RUN_MODE if inputs.use_trino and result.merged else FULL_REFRESH_RUN_MODE,
+ bytes_read=_total_bytes_read(processed_bytes),
)
job_engine = await _resolve_managed_warehouse_job(inputs.job_id, shadow_result)
await clear_node_suspension_for_engine(
@@ -364,6 +369,7 @@ async def _materialize_view_managed_warehouse(
table_name=table_name,
error=str(e),
run_mode=_failed_run_mode(plan, inputs.use_trino),
+ bytes_read=_total_bytes_read(processed_bytes),
)
if plan is not None and plan.incremental:
# The merge may have stopped partway or failed on a schema change. Rebuilding next run
@@ -390,6 +396,11 @@ async def _materialize_view_managed_warehouse(
return shadow_result
+def _total_bytes_read(processed_bytes: list[int]) -> int | None:
+ # No statement reported its bytes, so the run was not measured. That is not the same as reading zero bytes.
+ return sum(processed_bytes) if processed_bytes else None
+
+
def _trino_incremental_write(plan: WritePlan) -> TrinoIncrementalWrite | None:
if plan.config is None:
return None
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index 4dbfa7fd4e37..ff1b6089456c 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -179,6 +179,14 @@ class TestMaterializeViewManagedWarehouseActivity:
asaved_query.origin = DataWarehouseSavedQuery.Origin.ENDPOINT
await database_sync_to_async(asaved_query.save)(update_fields=["origin"])
executable_query = {"kind": "HogQLQuery", "query": "SELECT 2"}
+
+ def execute_trino_model(**kwargs: Any) -> DuckLakeTableResult:
+ kwargs["on_processed_bytes"](300)
+ kwargs["on_processed_bytes"](45)
+ if compile_fails:
+ raise ValueError("Trino compilation failed")
+ return DuckLakeTableResult(schema_name="shadow_models", table_name="test_model", row_count=12)
+
with (
unittest.mock.patch(
"products.managed_warehouse.backend.facade.client.request_model_alias_reconciliation",
@@ -186,8 +194,7 @@ class TestMaterializeViewManagedWarehouseActivity:
) as reconcile_aliases,
unittest.mock.patch(
"products.managed_warehouse.backend.facade.client.execute_trino_model",
- return_value=DuckLakeTableResult(schema_name="shadow_models", table_name="test_model", row_count=12),
- side_effect=ValueError("Trino compilation failed") if compile_fails else None,
+ side_effect=execute_trino_model,
) as execute,
unittest.mock.patch(
"products.managed_warehouse.backend.facade.client.execute_ducklake_create_table"
@@ -205,9 +212,11 @@ class TestMaterializeViewManagedWarehouseActivity:
saved_query_id=asaved_query.id,
source_query=executable_query,
incremental=None,
+ on_processed_bytes=unittest.mock.ANY,
)
legacy_execute.assert_not_called()
await database_sync_to_async(ajob.refresh_from_db)()
+ assert ajob.bytes_read == 345
if compile_fails:
reconcile_aliases.assert_not_awaited()
assert result.error == "Trino compilation failed"
diff --git a/products/data_modeling/backend/migrations/0049_datamodelingjob_bytes_read.py b/products/data_modeling/backend/migrations/0049_datamodelingjob_bytes_read.py
new file mode 100644
index 000000000000..7385fb615a9a
--- /dev/null
+++ b/products/data_modeling/backend/migrations/0049_datamodelingjob_bytes_read.py
@@ -0,0 +1,21 @@
+# Generated by Django 5.2.18 on 2026-10-21 12:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("data_modeling", "0048_drop_datawarehousemodelpath_table"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="datamodelingjob",
+ name="bytes_read",
+ field=models.BigIntegerField(
+ blank=True,
+ help_text="Bytes the engine read to build the model. Null when the run was not measured.",
+ null=True,
+ ),
+ ),
+ ]
diff --git a/products/data_modeling/backend/migrations/max_migration.txt b/products/data_modeling/backend/migrations/max_migration.txt
index f70645e6b2bf..fd735afd57e9 100644
--- a/products/data_modeling/backend/migrations/max_migration.txt
+++ b/products/data_modeling/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0048_drop_datawarehousemodelpath_table
+0049_datamodelingjob_bytes_read
diff --git a/products/data_modeling/backend/models/data_modeling_job.py b/products/data_modeling/backend/models/data_modeling_job.py
index e1c97f402df7..fc97c7776ed1 100644
--- a/products/data_modeling/backend/models/data_modeling_job.py
+++ b/products/data_modeling/backend/models/data_modeling_job.py
@@ -55,6 +55,11 @@ class DataModelingJob(CreatedMetaFields, UpdatedMetaFields, UUIDTModel):
last_run_at = models.DateTimeField(default=timezone.now)
rows_expected = models.IntegerField(null=True, blank=True, help_text="Total rows expected to be materialized")
storage_delta_mib = models.FloatField(null=True, blank=True, default=0)
+ bytes_read = models.BigIntegerField(
+ null=True,
+ blank=True,
+ help_text="Bytes the engine read to build the model. Null when the run was not measured.",
+ )
class Meta:
db_table = "posthog_datamodelingjob"
diff --git a/products/managed_warehouse/backend/facade/client.py b/products/managed_warehouse/backend/facade/client.py
index 5fc3ab254ebb..f25342261292 100644
--- a/products/managed_warehouse/backend/facade/client.py
+++ b/products/managed_warehouse/backend/facade/client.py
@@ -12,7 +12,7 @@ result types live in ``facade.contracts``.
from __future__ import annotations
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
from contextlib import AbstractContextManager
from typing import TYPE_CHECKING
from uuid import UUID
@@ -239,7 +239,10 @@ async def execute_trino_model(
saved_query_id: str | UUID,
source_query: object,
incremental: TrinoIncrementalWrite | None = None,
+ on_processed_bytes: Callable[[int], None] | None = None,
) -> DuckLakeTableResult:
+ """Build the model on Trino. ``on_processed_bytes`` receives the bytes each statement processed,
+ including statements of attempts that fail."""
from products.managed_warehouse.backend.trino_execution import ( # noqa: PLC0415 -- keeps executor creation off startup paths
run_trino_model_with_retries,
)
@@ -256,4 +259,5 @@ async def execute_trino_model(
incremental=incremental,
control=control,
),
+ on_processed_bytes=on_processed_bytes,
)
diff --git a/products/managed_warehouse/backend/tests/test_trino_execution.py b/products/managed_warehouse/backend/tests/test_trino_execution.py
index 3b8d10434ef7..1243ab4dee08 100644
--- a/products/managed_warehouse/backend/tests/test_trino_execution.py
+++ b/products/managed_warehouse/backend/tests/test_trino_execution.py
@@ -70,17 +70,27 @@ async def test_success_preserves_result() -> None:
)
async def test_retries_only_cluster_out_of_memory(error_name: str, expected_attempts: int) -> None:
attempts = 0
+ processed: list[int] = []
def execute(control: execution.TrinoQueryControl) -> int:
nonlocal attempts
attempts += 1
+ control.record_processed_bytes(10)
if attempts == 1:
raise TrinoQueryError({"errorName": error_name, "message": "Query killed"})
return 42
if expected_attempts == 1:
with pytest.raises(TrinoQueryError):
- await execution.run_trino_model_with_retries(execute, query_seconds=60, retry_delays=(0,))
+ await execution.run_trino_model_with_retries(
+ execute, query_seconds=60, retry_delays=(0,), on_processed_bytes=processed.append
+ )
else:
- assert await execution.run_trino_model_with_retries(execute, query_seconds=60, retry_delays=(0,)) == 42
+ assert (
+ await execution.run_trino_model_with_retries(
+ execute, query_seconds=60, retry_delays=(0,), on_processed_bytes=processed.append
+ )
+ == 42
+ )
assert attempts == expected_attempts
+ assert processed == [10] * expected_attempts
diff --git a/products/managed_warehouse/backend/tests/test_trino_materialization.py b/products/managed_warehouse/backend/tests/test_trino_materialization.py
index 8d53fee7538e..88540c49d3b7 100644
--- a/products/managed_warehouse/backend/tests/test_trino_materialization.py
+++ b/products/managed_warehouse/backend/tests/test_trino_materialization.py
@@ -29,7 +29,11 @@ from products.managed_warehouse.backend.models import (
ManagedWarehouseViewTranslationResult,
)
from products.managed_warehouse.backend.table_binding import build_trino_table_locators
-from products.managed_warehouse.backend.trino_materialization import DuplicateUniqueKeyError
+from products.managed_warehouse.backend.trino_execution import TrinoQueryControl
+from products.managed_warehouse.backend.trino_materialization import (
+ DuplicateUniqueKeyError,
+ execute_trino_shadow_materialization as execute_trino_model_statements,
+)
from products.managed_warehouse.backend.view_translation_status import source_query_hash
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
@@ -377,7 +381,16 @@ class TestTrinoShadowMaterialization(BaseTest):
class _ScriptedCursor:
"""Answers each statement by its leading words, and records every statement it ran."""
- def __init__(self, *, table_exists: bool, staged: int, duplicate: bool = False, max_key: Any = None) -> None:
+ def __init__(
+ self,
+ *,
+ table_exists: bool,
+ staged: int,
+ duplicate: bool = False,
+ max_key: Any = None,
+ processed_bytes: int = 0,
+ unreachable_from: str | None = None,
+ ) -> None:
self.table_exists = table_exists
self.staged = staged
self.duplicate = duplicate
@@ -385,11 +398,17 @@ class _ScriptedCursor:
self.statements: list[tuple[str, Any]] = []
self.rowcount = -1
self.description: list[tuple[str]] | None = None
+ self.processed_bytes = processed_bytes
+ self.unreachable_from = unreachable_from
+ self.stats: dict[str, Any] | None = None
self._rows: list[tuple[Any, ...]] = []
def execute(self, statement: str, parameters: Any = None) -> None:
+ if self.unreachable_from and statement.startswith(self.unreachable_from):
+ raise ConnectionError("coordinator unreachable")
self.statements.append((statement, parameters))
self.rowcount, self._rows = -1, []
+ self.stats = {"processedBytes": self.processed_bytes}
if "information_schema.tables" in statement:
self._rows = [(1,)] if self.table_exists else []
elif statement.startswith("CREATE OR REPLACE TABLE"):
@@ -416,7 +435,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):
since = dt.datetime(2026, 10, 1, tzinfo=dt.UTC)
def run_build(
- self, cursor: _ScriptedCursor, since: Any
+ self, cursor: _ScriptedCursor, since: Any, control: TrinoQueryControl | None = None
) -> tuple[DuckLakeTableResult | Exception, list[str], list[Any]]:
compiled = TrinoCompiledQuery(sql="SELECT id, ts FROM orders", values={})
connection = MagicMock(catalog="cat")
@@ -432,12 +451,13 @@ class TestTrinoIncrementalMaterialization(BaseTest):
):
connect.return_value.__enter__.return_value = connection
try:
- result: DuckLakeTableResult | Exception = execute_trino_shadow_materialization(
+ result: DuckLakeTableResult | Exception = execute_trino_model_statements(
organization_id=str(self.organization.pk),
team_id=self.team.pk,
saved_query_id=self.saved_query_id,
source_query=self.query,
incremental=TrinoIncrementalWrite(incremental_key="ts", unique_key=("id",), since=since),
+ control=control,
)
except Exception as error:
result = error
@@ -497,3 +517,28 @@ class TestTrinoIncrementalMaterialization(BaseTest):
assert f"CREATE OR REPLACE TABLE {table} AS SELECT id, ts FROM orders" in statements
assert not any("__ph_incremental_stage" in statement for statement in statements)
assert statements[-1] == f'SELECT max("ts") FROM {table}'
+
+ @parameterized.expand(
+ [
+ # Schema, table check, stage build, duplicate check, column read, merge and watermark read.
+ ("merge_succeeds", False, 7),
+ # The run stops at the duplicate check, and its statements still count.
+ ("duplicate_keys_fail_the_merge", True, 4),
+ ]
+ )
+ def test_reports_bytes_processed_by_every_statement(self, _name: str, duplicate: bool, statements: int) -> None:
+ processed: list[int] = []
+ cursor = _ScriptedCursor(table_exists=True, staged=3, duplicate=duplicate, processed_bytes=100)
+
+ self.run_build(cursor, self.since, TrinoQueryControl(on_processed_bytes=processed.append))
+
+ assert processed == [100] * statements
+
+ def test_a_statement_that_never_starts_does_not_recount_the_previous_one(self) -> None:
+ processed: list[int] = []
+ cursor = _ScriptedCursor(table_exists=True, staged=3, processed_bytes=100, unreachable_from="MERGE")
+
+ result, _, _ = self.run_build(cursor, self.since, TrinoQueryControl(on_processed_bytes=processed.append))
+
+ assert isinstance(result, ConnectionError)
+ assert processed == [100] * 5
diff --git a/products/managed_warehouse/backend/trino_execution.py b/products/managed_warehouse/backend/trino_execution.py
index 6b1833dba319..29748ed21be9 100644
--- a/products/managed_warehouse/backend/trino_execution.py
+++ b/products/managed_warehouse/backend/trino_execution.py
@@ -28,11 +28,16 @@ logger = structlog.get_logger(__name__)
class TrinoQueryControl:
- def __init__(self, query_seconds: float = TRINO_QUERY_SECONDS) -> None:
+ def __init__(
+ self,
+ query_seconds: float = TRINO_QUERY_SECONDS,
+ on_processed_bytes: Callable[[int], None] | None = None,
+ ) -> None:
self.cancelled = threading.Event()
self.finished = threading.Event()
self.cursor: Cursor | None = None
self.deadline = monotonic() + query_seconds
+ self.on_processed_bytes = on_processed_bytes
def checkpoint(self) -> None:
if self.cancelled.is_set() or monotonic() >= self.deadline:
@@ -41,6 +46,10 @@ class TrinoQueryControl:
def attach(self, cursor: Cursor) -> None:
self.cursor = cursor
+ def record_processed_bytes(self, processed_bytes: object) -> None:
+ if self.on_processed_bytes is not None and isinstance(processed_bytes, int):
+ self.on_processed_bytes(processed_bytes)
+
def _cancel_until_finished(self) -> None:
# execute() can still be waiting for its first nextUri when cancellation arrives.
while not self.finished.is_set():
@@ -58,9 +67,12 @@ class TrinoQueryControl:
async def run_trino_model(
- execute: Callable[[TrinoQueryControl], _Result], *, query_seconds: float = TRINO_QUERY_SECONDS
+ execute: Callable[[TrinoQueryControl], _Result],
+ *,
+ query_seconds: float = TRINO_QUERY_SECONDS,
+ on_processed_bytes: Callable[[int], None] | None = None,
) -> _Result:
- control = TrinoQueryControl(query_seconds)
+ control = TrinoQueryControl(query_seconds, on_processed_bytes)
task = asyncio.create_task(database_sync_to_async_pool(execute, executor=_executor)(control))
timeout = asyncio.timeout(query_seconds)
try:
@@ -91,15 +103,21 @@ async def run_trino_model_with_retries(
*,
query_seconds: float = TRINO_QUERY_SECONDS,
retry_delays: tuple[float, ...] = CLUSTER_OUT_OF_MEMORY_RETRY_DELAYS_SECONDS,
+ on_processed_bytes: Callable[[int], None] | None = None,
) -> _Result:
- """Run ``execute`` again after a cluster out-of-memory kill. All attempts share one deadline."""
+ """Run ``execute`` again after a cluster out-of-memory kill. All attempts share one deadline.
+
+ ``on_processed_bytes`` receives the bytes each statement processed, in every attempt.
+ """
deadline = monotonic() + query_seconds
for delay in retry_delays:
try:
- return await run_trino_model(execute, query_seconds=deadline - monotonic())
+ return await run_trino_model(
+ execute, query_seconds=deadline - monotonic(), on_processed_bytes=on_processed_bytes
+ )
except Exception as error:
if not is_cluster_out_of_memory(error) or monotonic() + delay >= deadline:
raise
logger.warning("Trino cluster is out of memory, retrying the model", retry_in_seconds=delay)
await asyncio.sleep(delay)
- return await run_trino_model(execute, query_seconds=deadline - monotonic())
+ return await run_trino_model(execute, query_seconds=deadline - monotonic(), on_processed_bytes=on_processed_bytes)
diff --git a/products/managed_warehouse/backend/trino_materialization.py b/products/managed_warehouse/backend/trino_materialization.py
index 1fda738e7246..d82af1125f9d 100644
--- a/products/managed_warehouse/backend/trino_materialization.py
+++ b/products/managed_warehouse/backend/trino_materialization.py
@@ -140,13 +140,7 @@ def _merge(
return 0, None
keys = [escape_trino_identifier(key) for key in incremental.unique_key]
- _run(
- cursor,
- control,
- f"SELECT 1 FROM {stage} GROUP BY {', '.join(keys)} HAVING count(*) > 1 LIMIT 1",
- fetch=False,
- )
- if cursor.fetchall():
+ if _fetch(cursor, control, f"SELECT 1 FROM {stage} GROUP BY {', '.join(keys)} HAVING count(*) > 1 LIMIT 1"):
raise DuplicateUniqueKeyError(
f"The incremental rows hold more than one row per unique key ({', '.join(incremental.unique_key)})"
)
@@ -178,20 +172,18 @@ def _drop_stage(cursor: Cursor, stage: str) -> None:
def _table_exists(
cursor: Cursor, control: TrinoQueryControl | None, catalog: str, schema_name: str, table_name: str
) -> bool:
- _run(
+ rows = _fetch(
cursor,
control,
f"SELECT 1 FROM {escape_trino_identifier(catalog)}.information_schema.tables "
"WHERE table_schema = ? AND table_name = ? AND table_type = 'BASE TABLE'",
[schema_name, table_name],
- fetch=False,
)
- return bool(cursor.fetchall())
+ return bool(rows)
def _max_key(cursor: Cursor, control: TrinoQueryControl | None, table: str, incremental_key: str) -> Any:
- _run(cursor, control, f"SELECT max({escape_trino_identifier(incremental_key)}) FROM {table}", fetch=False)
- rows = cursor.fetchall()
+ rows = _fetch(cursor, control, f"SELECT max({escape_trino_identifier(incremental_key)}) FROM {table}")
return rows[0][0] if rows else None
@@ -200,23 +192,37 @@ def _run(
control: TrinoQueryControl | None,
statement: str,
parameters: list[object] | None = None,
- *,
- fetch: bool = True,
) -> int:
- """Run one statement and return the row count Trino reports for a write.
-
- ``fetch=False`` leaves the result for the caller to read.
- """
- if control:
- control.checkpoint()
- if parameters:
- cursor.execute(statement, parameters)
- else:
- cursor.execute(statement)
- if not fetch:
- return -1
- cursor.fetchall()
+ """Run one statement and return the row count Trino reports for a write."""
+ _fetch(cursor, control, statement, parameters)
row_count = cursor.rowcount
if statement.startswith(("CREATE OR REPLACE TABLE", "MERGE")) and row_count < 0:
raise RuntimeError("Trino did not report the materialized row count")
return row_count
+
+
+def _fetch(
+ cursor: Cursor,
+ control: TrinoQueryControl | None,
+ statement: str,
+ parameters: list[object] | None = None,
+) -> list[Any]:
+ """Run one statement, read all of its rows, and report the bytes it processed to ``control``."""
+ if control:
+ control.checkpoint()
+ stats_before = cursor.stats
+ try:
+ if parameters:
+ cursor.execute(statement, parameters)
+ else:
+ cursor.execute(statement)
+ return cursor.fetchall()
+ finally:
+ # The client updates the statistics with every response, so they are final once the rows are read.
+ # A failed statement reports what it processed as of the last response before the error: the client
+ # raises on the error response before it reads that response's statistics, so the figure is a lower bound.
+ # When ``execute`` fails before it starts a query, the cursor still holds the previous statement's
+ # statistics, which are already counted.
+ stats = cursor.stats
+ if control is not None and stats is not None and stats is not stats_before:
+ control.record_processed_bytes(stats.get("processedBytes"))