Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(marketing-analytics): stop the precompute run at a wall-clock budget (#122988)

Lucas Ricoy047c78665683
diff --git a/products/marketing_analytics/dags/marketing_precompute.py b/products/marketing_analytics/dags/marketing_precompute.py
index 187ef6b36897..2baf98c662b9 100644
--- a/products/marketing_analytics/dags/marketing_precompute.py
+++ b/products/marketing_analytics/dags/marketing_precompute.py
@@ -30,12 +30,18 @@ The audience is `MARKETING_PRECOMPUTE_TEAM_IDS`: comma-separated team IDs, empty
(query_log_archive). Unset, it warms the teams whose reads are precompute-only, on PostHog Cloud only: teams
with a conversion goal and the conversion read flag on, and recently active teams with a warehouse table
and the cost read flag on. `MARKETING_PRECOMPUTE_ACTIVE_DAYS` tunes the `auto` activity window.
+`MARKETING_PRECOMPUTE_WALL_CLOCK_BUDGET_SECONDS` bounds each run: a shard stops starting new teams and chunks
+once the budget is spent and reports the teams it never started as deferred, so a run ends on its own under
+the job's max runtime. Every team a shard starts still materializes one chunk past the budget, so a run whose
+planning spent the budget still makes progress. That chunk is the newest one reads cannot serve, so the
+expired newest chunks do not take it on every run and the backfill keeps advancing.
"""
import os
import time
from collections.abc import Callable
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from functools import partial
from itertools import islice
@@ -79,6 +85,9 @@ from products.marketing_analytics.backend.hogql_queries.marketing_analytics_base
costs_precompute_ttl_schedule,
)
from products.marketing_analytics.backend.hogql_queries.marketing_analytics_config import MarketingAnalyticsConfig
+from products.marketing_analytics.backend.hogql_queries.marketing_lazy_precompute import (
+ PRECOMPUTE_ONLY_MAX_STALE_SECONDS,
+)
from products.marketing_analytics.backend.hogql_queries.utils import convert_team_conversion_goals_to_objects
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
@@ -127,6 +136,11 @@ ACTIVE_TEAMS_QUERY_TIMEOUT_SECONDS = 60
TEAM_CONCURRENCY_ENV_VAR = "MARKETING_PRECOMPUTE_TEAM_CONCURRENCY"
DEFAULT_TEAM_CONCURRENCY = 4
+# Per-run budget after which a shard starts no new team. Kept well under the job's 2h `dagster/max_runtime`:
+# a run Dagster kills alerts, a run that defers teams does not, and both leave the same READY chunks behind.
+WALL_CLOCK_BUDGET_ENV_VAR = "MARKETING_PRECOMPUTE_WALL_CLOCK_BUDGET_SECONDS"
+DEFAULT_WALL_CLOCK_BUDGET_SECONDS = 100 * 60
+
# Processes the job fans teams out into. Each shard prints HogQL on its own core; the run pod requests
# 6 CPUs, so more shards than that only contend for cores.
SHARDS_ENV_VAR = "MARKETING_PRECOMPUTE_SHARDS"
@@ -258,6 +272,37 @@ def get_selected_team_ids() -> list[int]:
return sorted(goal_team_ids & active_team_ids)
+# Mutable: `materialized` flips once the team builds its first chunk, across all of its tables, and
+# `stale_refresh` holds the first stale chunk the team skipped after the deadline.
+@dataclass(frozen=False, slots=True)
+class _TeamBudget:
+ """Planning can spend the whole budget before any worker starts, so the deadline only stops a team after it has
+ built (or failed to build) one chunk. READY chunks don't count: they cost a Postgres check and add no coverage.
+
+ After the deadline, a chunk that reads can still serve stale is not built. The newest chunks have TTLs shorter
+ than the hourly schedule, so they are always expired when a run starts. If they took the one guaranteed chunk,
+ every run that starts past the deadline would rebuild the same newest chunk and the backfill would never
+ advance. The guaranteed chunk goes to the newest window that reads cannot serve. Only when the team has no
+ such window does it refresh the first stale chunk it skipped.
+ """
+
+ deadline: float
+ materialized: bool = False
+ stale_refresh: Callable[[], int] | None = None
+
+ def spent(self) -> bool:
+ return time.monotonic() >= self.deadline
+
+ def exhausted(self) -> bool:
+ return self.materialized and self.spent()
+
+ def refresh_if_nothing_built(self) -> int:
+ """Build the skipped stale chunk when no unservable chunk took the guarantee. Returns the failure count."""
+ if self.materialized or self.stale_refresh is None:
+ return 0
+ return self.stale_refresh()
+
+
def _ensure_chunks(
context: dagster.OpExecutionContext,
team: Team,
@@ -268,11 +313,13 @@ def _ensure_chunks(
end: datetime,
chunk_days: int,
database: Database | None = None,
+ budget: _TeamBudget | None = None,
) -> int:
"""Drive ensure_precomputed for one (team, table, query) across the window, one bounded chunk at a
time. `build_insert_query` is called fresh per chunk (the executor resolves the time-window
placeholders in place). Failures per chunk are isolated so one bad chunk doesn't poison the rest;
- already-fresh chunks are cheap PG checks with no INSERT. Returns the failure count.
+ already-fresh chunks are cheap PG checks with no INSERT. Once `budget` is exhausted, the remaining chunks
+ are left for the next run. Returns the failure count.
"""
table_label = table.value
failures = 0
@@ -281,12 +328,20 @@ def _ensure_chunks(
ensure_seconds = 0.0
started = time.monotonic()
for chunk_start, chunk_end in chunk_ranges(start, end, chunk_days):
+ if budget is not None and budget.exhausted():
+ context.log.info(
+ f"marketing_precompute_chunks_deferred team={team.pk} table={table_label} from={chunk_start}"
+ )
+ break
chunks += 1
build_started = time.monotonic()
insert_query = build_insert_query()
build_seconds += time.monotonic() - build_started
if insert_query is None:
continue # source can't materialize this chunk (deterministic) — nothing to warm
+ # Past the deadline, the reads' stale grace makes ensure_precomputed return a servable chunk as a stale hit
+ # instead of building it (see `_TeamBudget`).
+ stale_grace = PRECOMPUTE_ONLY_MAX_STALE_SECONDS if budget is not None and budget.spent() else None
ensure_started = time.monotonic()
try:
result = ensure_precomputed(
@@ -297,9 +352,12 @@ def _ensure_chunks(
ttl_seconds=ttl_seconds,
table=table,
database=database,
+ stale_while_revalidate_seconds=stale_grace,
)
except Exception:
ensure_seconds += time.monotonic() - ensure_started
+ if budget is not None:
+ budget.materialized = True
MARKETING_PRECOMPUTE_CHUNK_FAILED.labels(table=table_label, error_type="exception").inc()
context.log.exception(
f"marketing_precompute_failed team={team.pk} table={table_label} chunk=[{chunk_start}, {chunk_end})"
@@ -307,6 +365,22 @@ def _ensure_chunks(
failures += 1
continue
ensure_seconds += time.monotonic() - ensure_started
+ if budget is not None and (result.freshly_built or not result.ready):
+ budget.materialized = True
+ elif budget is not None and result.stale and budget.stale_refresh is None:
+ # Without a budget, the single chunk [chunk_start, chunk_end) is rebuilt with no stale grace.
+ budget.stale_refresh = partial(
+ _ensure_chunks,
+ context,
+ team,
+ table,
+ build_insert_query,
+ ttl_seconds,
+ chunk_start,
+ chunk_end,
+ chunk_days,
+ database,
+ )
if result.ready:
MARKETING_PRECOMPUTE_CHUNK_DONE.labels(table=table_label).inc()
@@ -333,6 +407,7 @@ def _ensure_touchpoints_for_team(
end: datetime,
chunk_days: int,
database: Database | None = None,
+ budget: _TeamBudget | None = None,
) -> int:
"""Warm the goal-agnostic touchpoints table over [start, end] (start already reaches back past the
attribution window). One warmed window serves every conversion goal / attribution mode.
@@ -353,6 +428,7 @@ def _ensure_touchpoints_for_team(
end,
chunk_days,
database,
+ budget,
)
@@ -366,6 +442,7 @@ def _ensure_conversions_for_team(
end: datetime,
chunk_days: int,
database: Database | None = None,
+ budget: _TeamBudget | None = None,
) -> tuple[int, int]:
"""Warm the per-goal conversions table over [start, end] (no attribution backfill — the conversion
event itself must fall in-range). One lazy job per precomputable goal; ineligible goals are skipped
@@ -396,6 +473,7 @@ def _ensure_conversions_for_team(
end,
chunk_days,
database,
+ budget,
)
return goals_warmed, failures
@@ -429,6 +507,7 @@ def _ensure_costs_for_team(
end: datetime,
chunk_days: int,
database: Database | None = None,
+ budget: _TeamBudget | None = None,
) -> tuple[int, int]:
"""Warm the per-source cost table at every supported grain over [start, end] (no attribution
backfill). The database is built userless with warehouse access control bypassed — the materialization
@@ -473,6 +552,7 @@ def _ensure_costs_for_team(
end,
chunk_days,
database,
+ budget,
)
return warmed, failures
@@ -543,7 +623,9 @@ def _plan_team(team: Team) -> _TeamWarmPlan | None:
return None
-def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: datetime) -> _WarmCounts:
+def _warm_team(
+ context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: datetime, deadline: float | None = None
+) -> _WarmCounts:
"""Warm one planned team's precomputes over the rolling window.
Returns (conversion_teams, costs_teams, failures) increments. Runs in a worker thread of the op's
@@ -555,6 +637,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da
# and schema introspection would be un-attributable in query_log.
tag_queries(product=Product.MARKETING_ANALYTICS, feature=Feature.CACHE_WARMUP)
team = plan.team
+ budget = _TeamBudget(deadline) if deadline is not None else None
# Reads span whole team-local days and step back in local wall-clock days, so warm the same bounds. A
# UTC-based range misses the UTC day that a local day crosses into, and precompute-only reads then report
# not-ready or fall back to live costs.
@@ -585,6 +668,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da
local_day_end,
PRECOMPUTE_CHUNK_DAYS,
plan.database,
+ budget,
)
# Conversions need no attribution backfill — the conversion event must fall in the query range.
# Goals that aren't precomputable (non-Events/Actions, schema remaps, person/cohort filters) are
@@ -600,6 +684,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da
local_day_end,
PRECOMPUTE_CHUNK_DAYS,
plan.database,
+ budget,
)
failures += conv_failures
conversion_teams += 1
@@ -612,7 +697,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da
try:
costs_start = days_before_today(PRECOMPUTE_WINDOW_DAYS)
_sources_warmed, costs_failures = _ensure_costs_for_team(
- context, team, costs_start, local_day_end, PRECOMPUTE_CHUNK_DAYS, plan.database
+ context, team, costs_start, local_day_end, PRECOMPUTE_CHUNK_DAYS, plan.database, budget
)
failures += costs_failures
costs_teams += 1 # after the block, mirroring conversion_teams: not counted if it raised
@@ -620,6 +705,14 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da
MARKETING_PRECOMPUTE_TEAM_FAILED.labels(stage="costs").inc()
logger.exception("marketing_precompute_costs_failed", team_id=team.pk)
failures += 1
+
+ if budget is not None:
+ try:
+ failures += budget.refresh_if_nothing_built()
+ except Exception:
+ MARKETING_PRECOMPUTE_TEAM_FAILED.labels(stage="stale_refresh").inc()
+ logger.exception("marketing_precompute_stale_refresh_failed", team_id=team.pk)
+ failures += 1
finally:
connections.close_all()
context.log.info(
@@ -630,7 +723,18 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da
def _empty_result() -> dict[str, int]:
- return {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0}
+ return {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}
+
+
+def _wall_clock_budget_seconds() -> float:
+ return max(0.0, float(os.getenv(WALL_CLOCK_BUDGET_ENV_VAR, str(DEFAULT_WALL_CLOCK_BUDGET_SECONDS))))
+
+
+def _rotated_for_hour(plans: list[_TeamWarmPlan], end: datetime) -> list[_TeamWarmPlan]:
+ if not plans:
+ return plans
+ offset = int(end.timestamp() // 3600) % len(plans)
+ return plans[offset:] + plans[:offset]
def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: datetime) -> dict[str, int]:
@@ -643,6 +747,11 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: d
conversion goals; costs stay gated on the costs precompute flag plus the team having warehouse tables.
Every team, and each warming block within it, is isolated — one failure never aborts the rest.
+ Once the wall-clock budget (`MARKETING_PRECOMPUTE_WALL_CLOCK_BUDGET_SECONDS`, counted from the start of
+ planning) is spent, no further team starts and the teams never started are returned as `deferred`. A
+ started team stops between chunks once it has materialized at least one (see `_TeamBudget`). The start team rotates by the hour of `end`, so a budget that runs out never defers the
+ same teams on every run.
+
`conversion_teams` / `costs_teams` count teams whose block ran to completion without an unexpected
error (its raw material present — goals / warehouse tables), symmetric to each other. They
are not success counts: per-chunk outcomes live in `failures` and the MARKETING_PRECOMPUTE_CHUNK_*
@@ -653,6 +762,8 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: d
if not team_ids:
return _empty_result()
+ started = time.monotonic()
+ deadline = started + _wall_clock_budget_seconds()
teams_by_id = {t.pk: t for t in Team.objects.filter(pk__in=team_ids)}
teams = [teams_by_id[team_id] for team_id in team_ids if team_id in teams_by_id]
if len(teams) != len(team_ids):
@@ -697,10 +808,13 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: d
# A sliding window of `concurrency` teams: when one finishes, the next team's database is built
# and submitted, so a slow team never idles the other workers and at most `concurrency`
# databases are held in memory.
- pending = iter(plans)
+ pending = iter(_rotated_for_hour(plans, end))
in_flight = {
- pool.submit(_warm_team, context, with_database(plan), end) for plan in islice(pending, concurrency)
+ pool.submit(_warm_team, context, with_database(plan), end, deadline)
+ for plan in islice(pending, concurrency)
}
+ submitted = len(in_flight)
+ budget_reached = False
while in_flight:
finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED)
for future in finished:
@@ -713,19 +827,33 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: d
f"marketing_precompute_progress done={done}/{len(plans)} "
f"elapsed_s={round(time.monotonic() - warm_started)}"
)
+ if budget_reached:
+ continue
+ if time.monotonic() >= deadline:
+ # Deferred teams need nothing more: the next hourly run reselects the audience and
+ # ensure_precomputed skips the chunks that are already READY.
+ budget_reached = True
+ context.log.info(
+ f"marketing_precompute_budget_reached deferred={len(plans) - submitted} "
+ f"elapsed_s={round(time.monotonic() - started)}"
+ )
+ continue
next_plan = next(pending, None)
if next_plan is not None:
- in_flight.add(pool.submit(_warm_team, context, with_database(next_plan), end))
+ in_flight.add(pool.submit(_warm_team, context, with_database(next_plan), end, deadline))
+ submitted += 1
+ deferred = len(plans) - submitted
context.log.info(
f"marketing_precompute_complete teams={len(teams)} conversion_teams={conversion_teams} "
- f"costs_teams={costs_teams} failures={failures}"
+ f"costs_teams={costs_teams} failures={failures} deferred={deferred}"
)
return {
"teams": len(teams),
"conversion_teams": conversion_teams,
"costs_teams": costs_teams,
"failures": failures,
+ "deferred": deferred,
}
@@ -779,7 +907,7 @@ def summarize_marketing_precompute_op(context: dagster.OpExecutionContext, resul
context.log.info(
f"marketing_precompute_summary shards={len(results)} teams={total['teams']} "
f"conversion_teams={total['conversion_teams']} costs_teams={total['costs_teams']} "
- f"failures={total['failures']}"
+ f"failures={total['failures']} deferred={total['deferred']}"
)
context.add_output_metadata(total)
return total
diff --git a/products/marketing_analytics/dags/tests/test_marketing_precompute.py b/products/marketing_analytics/dags/tests/test_marketing_precompute.py
index 80ffbee37331..8e2c21e1ade2 100644
--- a/products/marketing_analytics/dags/tests/test_marketing_precompute.py
+++ b/products/marketing_analytics/dags/tests/test_marketing_precompute.py
@@ -37,10 +37,13 @@ from products.marketing_analytics.dags.marketing_precompute import (
SELECTED_TEAM_IDS_ENV_VAR,
SHARDS_ENV_VAR,
TEAM_CONCURRENCY_ENV_VAR,
+ WALL_CLOCK_BUDGET_ENV_VAR,
_recently_active_team_ids,
+ _WarmCounts,
get_selected_team_ids,
marketing_precompute_job,
split_marketing_precompute_teams_op,
+ summarize_marketing_precompute_op,
warm_selected_teams,
)
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
@@ -53,10 +56,14 @@ _FF = "products.marketing_analytics.backend.hogql_queries.marketing_analytics_co
_DB = "products.marketing_analytics.dags.marketing_precompute.Database"
_FACTORY = "products.marketing_analytics.dags.marketing_precompute.MarketingSourceFactory"
_DWT = "products.marketing_analytics.dags.marketing_precompute.DataWarehouseTable"
+_WARM_TEAM = "products.marketing_analytics.dags.marketing_precompute._warm_team"
+_TIME = "products.marketing_analytics.dags.marketing_precompute.time"
# Patch chunking to a single chunk so call counts are deterministic in the op tests. Must exceed
# PRECOMPUTE_WINDOW_DAYS + the team's attribution window (default 90) to collapse to one chunk.
_SINGLE_CHUNK = "products.marketing_analytics.dags.marketing_precompute.PRECOMPUTE_CHUNK_DAYS"
_BIG_CHUNK = 100000
+_TP = LazyComputationTable.MARKETING_TOUCHPOINTS_PREAGGREGATED
+_CONV = LazyComputationTable.MARKETING_CONVERSIONS_PREAGGREGATED
# Converts cleanly but is_goal_precomputable() rejects it (schema_map remaps a tracked UTM field, which
# the config-agnostic touchpoints table can't serve), so a team with only this goal warms touchpoints but
@@ -193,6 +200,14 @@ class TestShardSplit:
with patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: ""}):
assert list(split_marketing_precompute_teams_op(dagster.build_op_context())) == []
+ def test_summary_sums_every_shard_key(self):
+ shards = [
+ {"teams": 3, "conversion_teams": 2, "costs_teams": 1, "failures": 1, "deferred": 0},
+ {"teams": 4, "conversion_teams": 1, "costs_teams": 0, "failures": 0, "deferred": 3},
+ ]
+ total = summarize_marketing_precompute_op(dagster.build_op_context(), shards)
+ assert total == {"teams": 7, "conversion_teams": 3, "costs_teams": 1, "failures": 1, "deferred": 3}
+
class TestConversionWarming(APIBaseTest):
"""Touchpoints + conversions orchestration; ensure_precomputed is patched so no ClickHouse traffic."""
@@ -370,7 +385,7 @@ class TestConversionWarming(APIBaseTest):
team = self._make_team("A", goals=None)
with patch(_FF, _flag_fn(conversion=True)), patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}"}):
result = warm_selected_teams(dagster.build_op_context())
- assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0}
+ assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}
ensure_mock.assert_not_called()
@parameterized.expand([("database_built", False), ("database_build_fails", True)])
@@ -392,10 +407,124 @@ class TestConversionWarming(APIBaseTest):
database.create_for.side_effect = Exception("warehouse source broken")
result = warm_selected_teams(dagster.build_op_context())
- assert result == {"teams": 3, "conversion_teams": 3, "costs_teams": 0, "failures": 0}
+ assert result == {"teams": 3, "conversion_teams": 3, "costs_teams": 0, "failures": 0, "deferred": 0}
assert ensure_mock.call_count > len(teams)
assert database.create_for.call_count == len(teams)
+ @parameterized.expand([("budget_spent", "0", 2, 3), ("budget_generous", "3600", 5, 0)])
+ @patch(_WARM_TEAM, return_value=_WarmCounts(1, 0, 0))
+ def test_budget_defers_teams_not_yet_started(self, _name, budget, warmed, deferred, warm_team_mock):
+ # A shard that kept starting teams past the budget would run until Dagster kills it at max runtime.
+ # The first `concurrency` teams always start, and a started team always finishes.
+ teams = [self._make_team(name, goals=[_PRECOMPUTABLE_GOAL]) for name in ("A", "B", "C", "D", "E")]
+ ids = ",".join(str(team.pk) for team in teams)
+ with (
+ patch(_FF, _flag_fn(conversion=True)),
+ patch.dict(
+ os.environ,
+ {SELECTED_TEAM_IDS_ENV_VAR: ids, TEAM_CONCURRENCY_ENV_VAR: "2", WALL_CLOCK_BUDGET_ENV_VAR: budget},
+ ),
+ patch(_DB),
+ ):
+ result = warm_selected_teams(dagster.build_op_context())
+
+ assert result == {
+ "teams": 5,
+ "conversion_teams": warmed,
+ "costs_teams": 0,
+ "failures": 0,
+ "deferred": deferred,
+ }
+ assert warm_team_mock.call_count == warmed
+
+ @parameterized.expand(
+ [
+ ("runs_out_mid_backfill", "15", {}, {}, [[(_TP, 0), (_TP, 1)]]),
+ ("spent_before_warming", "0", {}, {}, [[(_TP, 0)]]),
+ ("spent_before_warming_newest_chunk_ready", "0", {(_TP, 0): "fresh"}, {}, [[(_TP, 1)]]),
+ ("spent_before_warming_newest_chunk_stale", "0", {(_TP, 0): "stale"}, {}, [[(_TP, 1)]]),
+ (
+ "spent_runs_advance_the_backfill",
+ "0",
+ {(_TP, 0): "stale"},
+ {},
+ [[(_TP, 1)], [(_TP, 2)], [(_TP, 3)]],
+ ),
+ ("spent_with_touchpoints_served_stale", "0", {}, {_TP: "stale"}, [[(_CONV, 0)]]),
+ ("spent_with_every_chunk_stale", "0", {}, {_TP: "stale", _CONV: "stale"}, [[(_TP, 0)]]),
+ ]
+ )
+ @patch(_ENSURE, new_callable=_ready_mock)
+ def test_budget_stops_a_started_team_between_chunks(
+ self, _name, budget, chunk_states, table_states, expected_built_per_run, ensure_mock
+ ):
+ # A cold team that starts just before the deadline must not run its whole backfill past it, and a
+ # team started after planning spent the budget must still materialize one chunk, or no run progresses.
+ # That chunk must be one reads cannot serve: the newest touchpoints chunk expires before every hourly
+ # run, and rebuilding it each time would stall the backfill. Chunks are keyed by table and index,
+ # newest first.
+ team = self._make_team("A", goals=[_PRECOMPUTABLE_GOAL])
+ clock = MagicMock()
+ chunk_days = timedelta(days=7)
+ newest_end = datetime(2024, 4, 2, 23, 59, 59, 999999, tzinfo=UTC)
+ states = dict(chunk_states)
+ built: list[tuple[LazyComputationTable, int]] = []
+
+ def fake_ensure(**kwargs):
+ clock.monotonic.return_value += 10.0
+ key = (kwargs["table"], (newest_end - kwargs["time_range_end"]) // chunk_days)
+ state = states.get(key, table_states.get(key[0]))
+ if state == "fresh":
+ return MagicMock(ready=True, freshly_built=False, stale=False)
+ if state == "stale" and kwargs["stale_while_revalidate_seconds"] is not None:
+ return MagicMock(ready=True, freshly_built=False, stale=True)
+ states[key] = "fresh"
+ built.append(key)
+ return MagicMock(ready=True, freshly_built=True, stale=False)
+
+ ensure_mock.side_effect = fake_ensure
+ built_per_run = []
+ for _run in expected_built_per_run:
+ clock.monotonic.return_value = 0.0
+ built.clear()
+ with (
+ time_machine.travel(datetime(2024, 4, 2, 12, 5, tzinfo=UTC), tick=False),
+ patch(_TIME, clock),
+ patch(_SINGLE_CHUNK, 7),
+ patch(_FF, _flag_fn(conversion=True)),
+ patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}", WALL_CLOCK_BUDGET_ENV_VAR: budget}),
+ patch(_DB),
+ ):
+ result = warm_selected_teams(dagster.build_op_context())
+ assert result["deferred"] == 0
+ built_per_run.append(list(built))
+ # The newest touchpoints chunk has a 15-minute TTL, so it has expired again by the next hourly run.
+ states[(_TP, 0)] = "stale"
+
+ assert built_per_run == expected_built_per_run
+
+ @patch(_WARM_TEAM, return_value=_WarmCounts(1, 0, 0))
+ def test_budget_spent_runs_start_with_a_different_team_each_hour(self, warm_team_mock):
+ teams = [self._make_team(name, goals=[_PRECOMPUTABLE_GOAL]) for name in ("A", "B", "C")]
+ ids = ",".join(str(team.pk) for team in teams)
+ started: set[int] = set()
+ for hour in range(len(teams)):
+ warm_team_mock.reset_mock()
+ with (
+ time_machine.travel(datetime(2024, 4, 2, hour, 5, tzinfo=UTC), tick=False),
+ patch(_FF, _flag_fn(conversion=True)),
+ patch.dict(
+ os.environ,
+ {SELECTED_TEAM_IDS_ENV_VAR: ids, TEAM_CONCURRENCY_ENV_VAR: "1", WALL_CLOCK_BUDGET_ENV_VAR: "0"},
+ ),
+ patch(_DB),
+ ):
+ warm_selected_teams(dagster.build_op_context())
+ assert warm_team_mock.call_count == 1
+ started.add(warm_team_mock.call_args.args[1].team.pk)
+
+ assert started == {team.pk for team in teams}
+
@parameterized.expand([("utc", "UTC"), ("behind_utc", "America/Los_Angeles"), ("ahead_of_utc", "Asia/Jerusalem")])
@patch(_ENSURE, new_callable=_ready_mock)
@patch(_SINGLE_CHUNK, _BIG_CHUNK)
@@ -485,14 +614,14 @@ class TestConversionWarming(APIBaseTest):
def test_empty_allowlist_is_a_noop(self, ensure_mock):
with patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: ""}):
result = warm_selected_teams(dagster.build_op_context())
- assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0}
+ assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}
ensure_mock.assert_not_called()
@patch(_ENSURE, new_callable=_ready_mock)
def test_missing_team_is_skipped(self, ensure_mock):
with patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: "999999999"}):
result = warm_selected_teams(dagster.build_op_context())
- assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0}
+ assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}
ensure_mock.assert_not_called()
def test_job_has_owner_and_runtime_tags(self):
@@ -528,7 +657,7 @@ class TestCostsWarming(APIBaseTest):
team = self._make_team("A")
with patch(_FF, _flag_fn(costs=False)), patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}"}):
result = warm_selected_teams(dagster.build_op_context())
- assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0}
+ assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}
ensure_mock.assert_not_called()
@patch(_ENSURE, new_callable=_ready_mock)