:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(marketing-analytics): stop the precompute run at a wall-clock budget (#122988)
Lucas Ricoy047c78665683
diff --git a/products/marketing_analytics/dags/marketing_precompute.py b/products/marketing_analytics/dags/marketing_precompute.pyindex 187ef6b36897..2baf98c662b9 100644--- a/products/marketing_analytics/dags/marketing_precompute.py+++ b/products/marketing_analytics/dags/marketing_precompute.py@@ -30,12 +30,18 @@ The audience is `MARKETING_PRECOMPUTE_TEAM_IDS`: comma-separated team IDs, empty(query_log_archive). Unset, it warms the teams whose reads are precompute-only, on PostHog Cloud only: teamswith a conversion goal and the conversion read flag on, and recently active teams with a warehouse tableand the cost read flag on. `MARKETING_PRECOMPUTE_ACTIVE_DAYS` tunes the `auto` activity window.+`MARKETING_PRECOMPUTE_WALL_CLOCK_BUDGET_SECONDS` bounds each run: a shard stops starting new teams and chunks+once the budget is spent and reports the teams it never started as deferred, so a run ends on its own under+the job's max runtime. Every team a shard starts still materializes one chunk past the budget, so a run whose+planning spent the budget still makes progress. That chunk is the newest one reads cannot serve, so the+expired newest chunks do not take it on every run and the backfill keeps advancing."""import osimport timefrom collections.abc import Callablefrom concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait+from dataclasses import dataclassfrom datetime import UTC, datetime, timedeltafrom functools import partialfrom itertools import islice@@ -79,6 +85,9 @@ from products.marketing_analytics.backend.hogql_queries.marketing_analytics_basecosts_precompute_ttl_schedule,)from products.marketing_analytics.backend.hogql_queries.marketing_analytics_config import MarketingAnalyticsConfig+from products.marketing_analytics.backend.hogql_queries.marketing_lazy_precompute import (+ PRECOMPUTE_ONLY_MAX_STALE_SECONDS,+)from products.marketing_analytics.backend.hogql_queries.utils import convert_team_conversion_goals_to_objectsfrom products.warehouse_sources.backend.facade.models import DataWarehouseTable@@ -127,6 +136,11 @@ ACTIVE_TEAMS_QUERY_TIMEOUT_SECONDS = 60TEAM_CONCURRENCY_ENV_VAR = "MARKETING_PRECOMPUTE_TEAM_CONCURRENCY"DEFAULT_TEAM_CONCURRENCY = 4+# Per-run budget after which a shard starts no new team. Kept well under the job's 2h `dagster/max_runtime`:+# a run Dagster kills alerts, a run that defers teams does not, and both leave the same READY chunks behind.+WALL_CLOCK_BUDGET_ENV_VAR = "MARKETING_PRECOMPUTE_WALL_CLOCK_BUDGET_SECONDS"+DEFAULT_WALL_CLOCK_BUDGET_SECONDS = 100 * 60+# Processes the job fans teams out into. Each shard prints HogQL on its own core; the run pod requests# 6 CPUs, so more shards than that only contend for cores.SHARDS_ENV_VAR = "MARKETING_PRECOMPUTE_SHARDS"@@ -258,6 +272,37 @@ def get_selected_team_ids() -> list[int]:return sorted(goal_team_ids & active_team_ids)+# Mutable: `materialized` flips once the team builds its first chunk, across all of its tables, and+# `stale_refresh` holds the first stale chunk the team skipped after the deadline.+@dataclass(frozen=False, slots=True)+class _TeamBudget:+ """Planning can spend the whole budget before any worker starts, so the deadline only stops a team after it has+ built (or failed to build) one chunk. READY chunks don't count: they cost a Postgres check and add no coverage.++ After the deadline, a chunk that reads can still serve stale is not built. The newest chunks have TTLs shorter+ than the hourly schedule, so they are always expired when a run starts. If they took the one guaranteed chunk,+ every run that starts past the deadline would rebuild the same newest chunk and the backfill would never+ advance. The guaranteed chunk goes to the newest window that reads cannot serve. Only when the team has no+ such window does it refresh the first stale chunk it skipped.+ """++ deadline: float+ materialized: bool = False+ stale_refresh: Callable[[], int] | None = None++ def spent(self) -> bool:+ return time.monotonic() >= self.deadline++ def exhausted(self) -> bool:+ return self.materialized and self.spent()++ def refresh_if_nothing_built(self) -> int:+ """Build the skipped stale chunk when no unservable chunk took the guarantee. Returns the failure count."""+ if self.materialized or self.stale_refresh is None:+ return 0+ return self.stale_refresh()++def _ensure_chunks(context: dagster.OpExecutionContext,team: Team,@@ -268,11 +313,13 @@ def _ensure_chunks(end: datetime,chunk_days: int,database: Database | None = None,+ budget: _TeamBudget | None = None,) -> int:"""Drive ensure_precomputed for one (team, table, query) across the window, one bounded chunk at atime. `build_insert_query` is called fresh per chunk (the executor resolves the time-windowplaceholders in place). Failures per chunk are isolated so one bad chunk doesn't poison the rest;- already-fresh chunks are cheap PG checks with no INSERT. Returns the failure count.+ already-fresh chunks are cheap PG checks with no INSERT. Once `budget` is exhausted, the remaining chunks+ are left for the next run. Returns the failure count."""table_label = table.valuefailures = 0@@ -281,12 +328,20 @@ def _ensure_chunks(ensure_seconds = 0.0started = time.monotonic()for chunk_start, chunk_end in chunk_ranges(start, end, chunk_days):+ if budget is not None and budget.exhausted():+ context.log.info(+ f"marketing_precompute_chunks_deferred team={team.pk} table={table_label} from={chunk_start}"+ )+ breakchunks += 1build_started = time.monotonic()insert_query = build_insert_query()build_seconds += time.monotonic() - build_startedif insert_query is None:continue # source can't materialize this chunk (deterministic) — nothing to warm+ # Past the deadline, the reads' stale grace makes ensure_precomputed return a servable chunk as a stale hit+ # instead of building it (see `_TeamBudget`).+ stale_grace = PRECOMPUTE_ONLY_MAX_STALE_SECONDS if budget is not None and budget.spent() else Noneensure_started = time.monotonic()try:result = ensure_precomputed(@@ -297,9 +352,12 @@ def _ensure_chunks(ttl_seconds=ttl_seconds,table=table,database=database,+ stale_while_revalidate_seconds=stale_grace,)except Exception:ensure_seconds += time.monotonic() - ensure_started+ if budget is not None:+ budget.materialized = TrueMARKETING_PRECOMPUTE_CHUNK_FAILED.labels(table=table_label, error_type="exception").inc()context.log.exception(f"marketing_precompute_failed team={team.pk} table={table_label} chunk=[{chunk_start}, {chunk_end})"@@ -307,6 +365,22 @@ def _ensure_chunks(failures += 1continueensure_seconds += time.monotonic() - ensure_started+ if budget is not None and (result.freshly_built or not result.ready):+ budget.materialized = True+ elif budget is not None and result.stale and budget.stale_refresh is None:+ # Without a budget, the single chunk [chunk_start, chunk_end) is rebuilt with no stale grace.+ budget.stale_refresh = partial(+ _ensure_chunks,+ context,+ team,+ table,+ build_insert_query,+ ttl_seconds,+ chunk_start,+ chunk_end,+ chunk_days,+ database,+ )if result.ready:MARKETING_PRECOMPUTE_CHUNK_DONE.labels(table=table_label).inc()@@ -333,6 +407,7 @@ def _ensure_touchpoints_for_team(end: datetime,chunk_days: int,database: Database | None = None,+ budget: _TeamBudget | None = None,) -> int:"""Warm the goal-agnostic touchpoints table over [start, end] (start already reaches back past theattribution window). One warmed window serves every conversion goal / attribution mode.@@ -353,6 +428,7 @@ def _ensure_touchpoints_for_team(end,chunk_days,database,+ budget,)@@ -366,6 +442,7 @@ def _ensure_conversions_for_team(end: datetime,chunk_days: int,database: Database | None = None,+ budget: _TeamBudget | None = None,) -> tuple[int, int]:"""Warm the per-goal conversions table over [start, end] (no attribution backfill — the conversionevent itself must fall in-range). One lazy job per precomputable goal; ineligible goals are skipped@@ -396,6 +473,7 @@ def _ensure_conversions_for_team(end,chunk_days,database,+ budget,)return goals_warmed, failures@@ -429,6 +507,7 @@ def _ensure_costs_for_team(end: datetime,chunk_days: int,database: Database | None = None,+ budget: _TeamBudget | None = None,) -> tuple[int, int]:"""Warm the per-source cost table at every supported grain over [start, end] (no attributionbackfill). The database is built userless with warehouse access control bypassed — the materialization@@ -473,6 +552,7 @@ def _ensure_costs_for_team(end,chunk_days,database,+ budget,)return warmed, failures@@ -543,7 +623,9 @@ def _plan_team(team: Team) -> _TeamWarmPlan | None:return None-def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: datetime) -> _WarmCounts:+def _warm_team(+ context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: datetime, deadline: float | None = None+) -> _WarmCounts:"""Warm one planned team's precomputes over the rolling window.Returns (conversion_teams, costs_teams, failures) increments. Runs in a worker thread of the op's@@ -555,6 +637,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da# and schema introspection would be un-attributable in query_log.tag_queries(product=Product.MARKETING_ANALYTICS, feature=Feature.CACHE_WARMUP)team = plan.team+ budget = _TeamBudget(deadline) if deadline is not None else None# Reads span whole team-local days and step back in local wall-clock days, so warm the same bounds. A# UTC-based range misses the UTC day that a local day crosses into, and precompute-only reads then report# not-ready or fall back to live costs.@@ -585,6 +668,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: dalocal_day_end,PRECOMPUTE_CHUNK_DAYS,plan.database,+ budget,)# Conversions need no attribution backfill — the conversion event must fall in the query range.# Goals that aren't precomputable (non-Events/Actions, schema remaps, person/cohort filters) are@@ -600,6 +684,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: dalocal_day_end,PRECOMPUTE_CHUNK_DAYS,plan.database,+ budget,)failures += conv_failuresconversion_teams += 1@@ -612,7 +697,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: datry:costs_start = days_before_today(PRECOMPUTE_WINDOW_DAYS)_sources_warmed, costs_failures = _ensure_costs_for_team(- context, team, costs_start, local_day_end, PRECOMPUTE_CHUNK_DAYS, plan.database+ context, team, costs_start, local_day_end, PRECOMPUTE_CHUNK_DAYS, plan.database, budget)failures += costs_failurescosts_teams += 1 # after the block, mirroring conversion_teams: not counted if it raised@@ -620,6 +705,14 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: daMARKETING_PRECOMPUTE_TEAM_FAILED.labels(stage="costs").inc()logger.exception("marketing_precompute_costs_failed", team_id=team.pk)failures += 1++ if budget is not None:+ try:+ failures += budget.refresh_if_nothing_built()+ except Exception:+ MARKETING_PRECOMPUTE_TEAM_FAILED.labels(stage="stale_refresh").inc()+ logger.exception("marketing_precompute_stale_refresh_failed", team_id=team.pk)+ failures += 1finally:connections.close_all()context.log.info(@@ -630,7 +723,18 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: dadef _empty_result() -> dict[str, int]:- return {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0}+ return {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}+++def _wall_clock_budget_seconds() -> float:+ return max(0.0, float(os.getenv(WALL_CLOCK_BUDGET_ENV_VAR, str(DEFAULT_WALL_CLOCK_BUDGET_SECONDS))))+++def _rotated_for_hour(plans: list[_TeamWarmPlan], end: datetime) -> list[_TeamWarmPlan]:+ if not plans:+ return plans+ offset = int(end.timestamp() // 3600) % len(plans)+ return plans[offset:] + plans[:offset]def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: datetime) -> dict[str, int]:@@ -643,6 +747,11 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: dconversion goals; costs stay gated on the costs precompute flag plus the team having warehouse tables.Every team, and each warming block within it, is isolated — one failure never aborts the rest.+ Once the wall-clock budget (`MARKETING_PRECOMPUTE_WALL_CLOCK_BUDGET_SECONDS`, counted from the start of+ planning) is spent, no further team starts and the teams never started are returned as `deferred`. A+ started team stops between chunks once it has materialized at least one (see `_TeamBudget`). The start team rotates by the hour of `end`, so a budget that runs out never defers the+ same teams on every run.+`conversion_teams` / `costs_teams` count teams whose block ran to completion without an unexpectederror (its raw material present — goals / warehouse tables), symmetric to each other. Theyare not success counts: per-chunk outcomes live in `failures` and the MARKETING_PRECOMPUTE_CHUNK_*@@ -653,6 +762,8 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: dif not team_ids:return _empty_result()+ started = time.monotonic()+ deadline = started + _wall_clock_budget_seconds()teams_by_id = {t.pk: t for t in Team.objects.filter(pk__in=team_ids)}teams = [teams_by_id[team_id] for team_id in team_ids if team_id in teams_by_id]if len(teams) != len(team_ids):@@ -697,10 +808,13 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: d# A sliding window of `concurrency` teams: when one finishes, the next team's database is built# and submitted, so a slow team never idles the other workers and at most `concurrency`# databases are held in memory.- pending = iter(plans)+ pending = iter(_rotated_for_hour(plans, end))in_flight = {- pool.submit(_warm_team, context, with_database(plan), end) for plan in islice(pending, concurrency)+ pool.submit(_warm_team, context, with_database(plan), end, deadline)+ for plan in islice(pending, concurrency)}+ submitted = len(in_flight)+ budget_reached = Falsewhile in_flight:finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED)for future in finished:@@ -713,19 +827,33 @@ def _warm_teams(context: dagster.OpExecutionContext, team_ids: list[int], end: df"marketing_precompute_progress done={done}/{len(plans)} "f"elapsed_s={round(time.monotonic() - warm_started)}")+ if budget_reached:+ continue+ if time.monotonic() >= deadline:+ # Deferred teams need nothing more: the next hourly run reselects the audience and+ # ensure_precomputed skips the chunks that are already READY.+ budget_reached = True+ context.log.info(+ f"marketing_precompute_budget_reached deferred={len(plans) - submitted} "+ f"elapsed_s={round(time.monotonic() - started)}"+ )+ continuenext_plan = next(pending, None)if next_plan is not None:- in_flight.add(pool.submit(_warm_team, context, with_database(next_plan), end))+ in_flight.add(pool.submit(_warm_team, context, with_database(next_plan), end, deadline))+ submitted += 1+ deferred = len(plans) - submittedcontext.log.info(f"marketing_precompute_complete teams={len(teams)} conversion_teams={conversion_teams} "- f"costs_teams={costs_teams} failures={failures}"+ f"costs_teams={costs_teams} failures={failures} deferred={deferred}")return {"teams": len(teams),"conversion_teams": conversion_teams,"costs_teams": costs_teams,"failures": failures,+ "deferred": deferred,}@@ -779,7 +907,7 @@ def summarize_marketing_precompute_op(context: dagster.OpExecutionContext, resulcontext.log.info(f"marketing_precompute_summary shards={len(results)} teams={total['teams']} "f"conversion_teams={total['conversion_teams']} costs_teams={total['costs_teams']} "- f"failures={total['failures']}"+ f"failures={total['failures']} deferred={total['deferred']}")context.add_output_metadata(total)return totaldiff --git a/products/marketing_analytics/dags/tests/test_marketing_precompute.py b/products/marketing_analytics/dags/tests/test_marketing_precompute.pyindex 80ffbee37331..8e2c21e1ade2 100644--- a/products/marketing_analytics/dags/tests/test_marketing_precompute.py+++ b/products/marketing_analytics/dags/tests/test_marketing_precompute.py@@ -37,10 +37,13 @@ from products.marketing_analytics.dags.marketing_precompute import (SELECTED_TEAM_IDS_ENV_VAR,SHARDS_ENV_VAR,TEAM_CONCURRENCY_ENV_VAR,+ WALL_CLOCK_BUDGET_ENV_VAR,_recently_active_team_ids,+ _WarmCounts,get_selected_team_ids,marketing_precompute_job,split_marketing_precompute_teams_op,+ summarize_marketing_precompute_op,warm_selected_teams,)from products.warehouse_sources.backend.facade.models import DataWarehouseTable@@ -53,10 +56,14 @@ _FF = "products.marketing_analytics.backend.hogql_queries.marketing_analytics_co_DB = "products.marketing_analytics.dags.marketing_precompute.Database"_FACTORY = "products.marketing_analytics.dags.marketing_precompute.MarketingSourceFactory"_DWT = "products.marketing_analytics.dags.marketing_precompute.DataWarehouseTable"+_WARM_TEAM = "products.marketing_analytics.dags.marketing_precompute._warm_team"+_TIME = "products.marketing_analytics.dags.marketing_precompute.time"# Patch chunking to a single chunk so call counts are deterministic in the op tests. Must exceed# PRECOMPUTE_WINDOW_DAYS + the team's attribution window (default 90) to collapse to one chunk._SINGLE_CHUNK = "products.marketing_analytics.dags.marketing_precompute.PRECOMPUTE_CHUNK_DAYS"_BIG_CHUNK = 100000+_TP = LazyComputationTable.MARKETING_TOUCHPOINTS_PREAGGREGATED+_CONV = LazyComputationTable.MARKETING_CONVERSIONS_PREAGGREGATED# Converts cleanly but is_goal_precomputable() rejects it (schema_map remaps a tracked UTM field, which# the config-agnostic touchpoints table can't serve), so a team with only this goal warms touchpoints but@@ -193,6 +200,14 @@ class TestShardSplit:with patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: ""}):assert list(split_marketing_precompute_teams_op(dagster.build_op_context())) == []+ def test_summary_sums_every_shard_key(self):+ shards = [+ {"teams": 3, "conversion_teams": 2, "costs_teams": 1, "failures": 1, "deferred": 0},+ {"teams": 4, "conversion_teams": 1, "costs_teams": 0, "failures": 0, "deferred": 3},+ ]+ total = summarize_marketing_precompute_op(dagster.build_op_context(), shards)+ assert total == {"teams": 7, "conversion_teams": 3, "costs_teams": 1, "failures": 1, "deferred": 3}+class TestConversionWarming(APIBaseTest):"""Touchpoints + conversions orchestration; ensure_precomputed is patched so no ClickHouse traffic."""@@ -370,7 +385,7 @@ class TestConversionWarming(APIBaseTest):team = self._make_team("A", goals=None)with patch(_FF, _flag_fn(conversion=True)), patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}"}):result = warm_selected_teams(dagster.build_op_context())- assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0}+ assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}ensure_mock.assert_not_called()@parameterized.expand([("database_built", False), ("database_build_fails", True)])@@ -392,10 +407,124 @@ class TestConversionWarming(APIBaseTest):database.create_for.side_effect = Exception("warehouse source broken")result = warm_selected_teams(dagster.build_op_context())- assert result == {"teams": 3, "conversion_teams": 3, "costs_teams": 0, "failures": 0}+ assert result == {"teams": 3, "conversion_teams": 3, "costs_teams": 0, "failures": 0, "deferred": 0}assert ensure_mock.call_count > len(teams)assert database.create_for.call_count == len(teams)+ @parameterized.expand([("budget_spent", "0", 2, 3), ("budget_generous", "3600", 5, 0)])+ @patch(_WARM_TEAM, return_value=_WarmCounts(1, 0, 0))+ def test_budget_defers_teams_not_yet_started(self, _name, budget, warmed, deferred, warm_team_mock):+ # A shard that kept starting teams past the budget would run until Dagster kills it at max runtime.+ # The first `concurrency` teams always start, and a started team always finishes.+ teams = [self._make_team(name, goals=[_PRECOMPUTABLE_GOAL]) for name in ("A", "B", "C", "D", "E")]+ ids = ",".join(str(team.pk) for team in teams)+ with (+ patch(_FF, _flag_fn(conversion=True)),+ patch.dict(+ os.environ,+ {SELECTED_TEAM_IDS_ENV_VAR: ids, TEAM_CONCURRENCY_ENV_VAR: "2", WALL_CLOCK_BUDGET_ENV_VAR: budget},+ ),+ patch(_DB),+ ):+ result = warm_selected_teams(dagster.build_op_context())++ assert result == {+ "teams": 5,+ "conversion_teams": warmed,+ "costs_teams": 0,+ "failures": 0,+ "deferred": deferred,+ }+ assert warm_team_mock.call_count == warmed++ @parameterized.expand(+ [+ ("runs_out_mid_backfill", "15", {}, {}, [[(_TP, 0), (_TP, 1)]]),+ ("spent_before_warming", "0", {}, {}, [[(_TP, 0)]]),+ ("spent_before_warming_newest_chunk_ready", "0", {(_TP, 0): "fresh"}, {}, [[(_TP, 1)]]),+ ("spent_before_warming_newest_chunk_stale", "0", {(_TP, 0): "stale"}, {}, [[(_TP, 1)]]),+ (+ "spent_runs_advance_the_backfill",+ "0",+ {(_TP, 0): "stale"},+ {},+ [[(_TP, 1)], [(_TP, 2)], [(_TP, 3)]],+ ),+ ("spent_with_touchpoints_served_stale", "0", {}, {_TP: "stale"}, [[(_CONV, 0)]]),+ ("spent_with_every_chunk_stale", "0", {}, {_TP: "stale", _CONV: "stale"}, [[(_TP, 0)]]),+ ]+ )+ @patch(_ENSURE, new_callable=_ready_mock)+ def test_budget_stops_a_started_team_between_chunks(+ self, _name, budget, chunk_states, table_states, expected_built_per_run, ensure_mock+ ):+ # A cold team that starts just before the deadline must not run its whole backfill past it, and a+ # team started after planning spent the budget must still materialize one chunk, or no run progresses.+ # That chunk must be one reads cannot serve: the newest touchpoints chunk expires before every hourly+ # run, and rebuilding it each time would stall the backfill. Chunks are keyed by table and index,+ # newest first.+ team = self._make_team("A", goals=[_PRECOMPUTABLE_GOAL])+ clock = MagicMock()+ chunk_days = timedelta(days=7)+ newest_end = datetime(2024, 4, 2, 23, 59, 59, 999999, tzinfo=UTC)+ states = dict(chunk_states)+ built: list[tuple[LazyComputationTable, int]] = []++ def fake_ensure(**kwargs):+ clock.monotonic.return_value += 10.0+ key = (kwargs["table"], (newest_end - kwargs["time_range_end"]) // chunk_days)+ state = states.get(key, table_states.get(key[0]))+ if state == "fresh":+ return MagicMock(ready=True, freshly_built=False, stale=False)+ if state == "stale" and kwargs["stale_while_revalidate_seconds"] is not None:+ return MagicMock(ready=True, freshly_built=False, stale=True)+ states[key] = "fresh"+ built.append(key)+ return MagicMock(ready=True, freshly_built=True, stale=False)++ ensure_mock.side_effect = fake_ensure+ built_per_run = []+ for _run in expected_built_per_run:+ clock.monotonic.return_value = 0.0+ built.clear()+ with (+ time_machine.travel(datetime(2024, 4, 2, 12, 5, tzinfo=UTC), tick=False),+ patch(_TIME, clock),+ patch(_SINGLE_CHUNK, 7),+ patch(_FF, _flag_fn(conversion=True)),+ patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}", WALL_CLOCK_BUDGET_ENV_VAR: budget}),+ patch(_DB),+ ):+ result = warm_selected_teams(dagster.build_op_context())+ assert result["deferred"] == 0+ built_per_run.append(list(built))+ # The newest touchpoints chunk has a 15-minute TTL, so it has expired again by the next hourly run.+ states[(_TP, 0)] = "stale"++ assert built_per_run == expected_built_per_run++ @patch(_WARM_TEAM, return_value=_WarmCounts(1, 0, 0))+ def test_budget_spent_runs_start_with_a_different_team_each_hour(self, warm_team_mock):+ teams = [self._make_team(name, goals=[_PRECOMPUTABLE_GOAL]) for name in ("A", "B", "C")]+ ids = ",".join(str(team.pk) for team in teams)+ started: set[int] = set()+ for hour in range(len(teams)):+ warm_team_mock.reset_mock()+ with (+ time_machine.travel(datetime(2024, 4, 2, hour, 5, tzinfo=UTC), tick=False),+ patch(_FF, _flag_fn(conversion=True)),+ patch.dict(+ os.environ,+ {SELECTED_TEAM_IDS_ENV_VAR: ids, TEAM_CONCURRENCY_ENV_VAR: "1", WALL_CLOCK_BUDGET_ENV_VAR: "0"},+ ),+ patch(_DB),+ ):+ warm_selected_teams(dagster.build_op_context())+ assert warm_team_mock.call_count == 1+ started.add(warm_team_mock.call_args.args[1].team.pk)++ assert started == {team.pk for team in teams}+@parameterized.expand([("utc", "UTC"), ("behind_utc", "America/Los_Angeles"), ("ahead_of_utc", "Asia/Jerusalem")])@patch(_ENSURE, new_callable=_ready_mock)@patch(_SINGLE_CHUNK, _BIG_CHUNK)@@ -485,14 +614,14 @@ class TestConversionWarming(APIBaseTest):def test_empty_allowlist_is_a_noop(self, ensure_mock):with patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: ""}):result = warm_selected_teams(dagster.build_op_context())- assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0}+ assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}ensure_mock.assert_not_called()@patch(_ENSURE, new_callable=_ready_mock)def test_missing_team_is_skipped(self, ensure_mock):with patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: "999999999"}):result = warm_selected_teams(dagster.build_op_context())- assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0}+ assert result == {"teams": 0, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}ensure_mock.assert_not_called()def test_job_has_owner_and_runtime_tags(self):@@ -528,7 +657,7 @@ class TestCostsWarming(APIBaseTest):team = self._make_team("A")with patch(_FF, _flag_fn(costs=False)), patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}"}):result = warm_selected_teams(dagster.build_op_context())- assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0}+ assert result == {"teams": 1, "conversion_teams": 0, "costs_teams": 0, "failures": 0, "deferred": 0}ensure_mock.assert_not_called()@patch(_ENSURE, new_callable=_ready_mock)