:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): skip trino shadow runs whose upstream table is missing (#123153)
Jovan Sakovicabeb14141161
diff --git a/posthog/temporal/data_modeling/activities/fail_materialization.py b/posthog/temporal/data_modeling/activities/fail_materialization.pyindex d036f7514baa..1543a2db937d 100644--- a/posthog/temporal/data_modeling/activities/fail_materialization.py+++ b/posthog/temporal/data_modeling/activities/fail_materialization.py@@ -11,6 +11,7 @@ from posthog.exceptions_capture import capture_exceptionfrom posthog.sync import database_sync_to_async_poolfrom products.data_modeling.backend.facade.models import (+ TERMINAL_JOB_STATUSES,DataModelingJob,DataModelingJobEngine,DataModelingJobStatus,@@ -73,7 +74,7 @@ def _fail_node_and_data_modeling_job(inputs: FailMaterializationInputs):job = DataModelingJob.objects.get(id=inputs.job_id)# if the job is already in a terminal state, don't overwrite it — preserves the first error- if job.status in (DataModelingJobStatus.FAILED, DataModelingJobStatus.CANCELLED, DataModelingJobStatus.COMPLETED):+ if job.status in TERMINAL_JOB_STATUSES:return node, jobjob.status = DataModelingJobStatus.CANCELLED if inputs.cancelled else DataModelingJobStatus.FAILEDdiff --git a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.pyindex 224422ac6e65..ccb502c6cab7 100644--- a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py+++ b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py@@ -3,6 +3,7 @@ import timeimport typingimport datetime as dtimport dataclasses+from uuid import UUIDfrom structlog.contextvars import bind_contextvarsfrom temporalio import activity@@ -23,9 +24,12 @@ from products.data_modeling.backend.facade.api import (clear_incremental_state,resolve_write_plan,set_incremental_state,+ unbuilt_upstream_saved_query_ids,)from products.data_modeling.backend.facade.models import (+ TERMINAL_JOB_STATUSES,DataModelingJob,+ DataModelingJobEngine,DataModelingJobRunMode,DataModelingJobStatus,DataWarehouseSavedQuery,@@ -108,6 +112,24 @@ class ManagedWarehouseShadowResult:run_mode: str = UNKNOWN_RUN_MODEread_bytes: int | None = Nonecpu_microseconds: int | None = None+ # The run read an upstream model that has never been built on Trino. That says nothing about the+ # view itself, so the job is skipped rather than failed and does not count toward suspension.+ skipped: bool = False+++_TABLE_MISSING_ERROR_NAMES = frozenset({"CATALOG_NOT_FOUND", "SCHEMA_NOT_FOUND", "TABLE_NOT_FOUND"})+++def _upstream_model_unbuilt(missing_object_error_name: str | None, team_id: int, saved_query_id: UUID) -> bool:+ """Whether a missing Trino table is explained by an upstream model that has never been built there.++ A missing imported table or events table is a sync or provisioning fault, so it still fails the job.+ """+ if missing_object_error_name not in _TABLE_MISSING_ERROR_NAMES:+ return False+ return bool(+ unbuilt_upstream_saved_query_ids(team_id, saved_query_id, engine=DataModelingJobEngine.MANAGED_WAREHOUSE)+ )def _failed_run_mode(plan: WritePlan | None, use_trino: bool) -> str:@@ -235,12 +257,16 @@ async def check_managed_warehouse_shadow_eligibility_activity(def _resolve_managed_warehouse_job(job_id: str, result: "ManagedWarehouseShadowResult") -> str:"""Update the managed warehouse job to its terminal state based on the result."""job = DataModelingJob.objects.get(id=job_id)- if job.status in (DataModelingJobStatus.FAILED, DataModelingJobStatus.CANCELLED, DataModelingJobStatus.COMPLETED):+ if job.status in TERMINAL_JOB_STATUSES:return job.engineif result.error is None:job.status = DataModelingJobStatus.COMPLETEDjob.rows_materialized = result.row_countjob.error = None+ elif result.skipped:+ job.status = DataModelingJobStatus.SKIPPED+ job.rows_materialized = 0+ job.error = result.errorelse:job.status = DataModelingJobStatus.FAILEDjob.rows_materialized = 0@@ -411,6 +437,9 @@ async def _materialize_view_managed_warehouse(run_mode=_failed_run_mode(plan, inputs.use_trino),read_bytes=_total_read_bytes(processed_bytes),cpu_microseconds=_total_cpu_microseconds(cpu_milliseconds),+ skipped=await database_sync_to_async_pool(_upstream_model_unbuilt)(+ missing_object_error_name, team.pk, saved_query.id+ ),)if plan is not None and plan.incremental:# The merge may have stopped partway or failed on a schema change. Rebuilding next run@@ -420,6 +449,8 @@ async def _materialize_view_managed_warehouse(except Exception as clear_error:capture_exception(clear_error)job_engine = await _resolve_managed_warehouse_job(inputs.job_id, shadow_result)+ if shadow_result.skipped:+ return shadow_resultsuspended = await maybe_suspend_node_for_engine(node_id=inputs.node_id,team_id=inputs.team_id,diff --git a/posthog/temporal/data_modeling/activities/quality_block_materialization.py b/posthog/temporal/data_modeling/activities/quality_block_materialization.pyindex 5d5325b064d9..be7b7581fbaa 100644--- a/posthog/temporal/data_modeling/activities/quality_block_materialization.py+++ b/posthog/temporal/data_modeling/activities/quality_block_materialization.py@@ -9,7 +9,12 @@ from temporalio import activityfrom posthog.sync import database_sync_to_async_pool-from products.data_modeling.backend.facade.models import DataModelingJob, DataModelingJobStatus, Node+from products.data_modeling.backend.facade.models import (+ TERMINAL_JOB_STATUSES,+ DataModelingJob,+ DataModelingJobStatus,+ Node,+)from products.data_quality.backend.facade import api as data_quality_facadefrom .utils import QUALITY_BLOCKED_ERROR_PREFIX, update_node_system_properties@@ -80,7 +85,7 @@ def _block_node_and_job(inputs: QualityBlockMaterializationInputs) -> None:node.save()job = DataModelingJob.objects.get(id=inputs.job_id)- if job.status in (DataModelingJobStatus.FAILED, DataModelingJobStatus.CANCELLED, DataModelingJobStatus.COMPLETED):+ if job.status in TERMINAL_JOB_STATUSES:returnjob.status = DataModelingJobStatus.FAILEDjob.error = errordiff --git a/posthog/temporal/data_modeling/activities/succeed_materialization.py b/posthog/temporal/data_modeling/activities/succeed_materialization.pyindex 4bdcc763f102..6fb65f6c5cf2 100644--- a/posthog/temporal/data_modeling/activities/succeed_materialization.py+++ b/posthog/temporal/data_modeling/activities/succeed_materialization.py@@ -11,6 +11,7 @@ from posthog.exceptions_capture import capture_exceptionfrom posthog.sync import database_sync_to_async_poolfrom products.data_modeling.backend.facade.models import (+ TERMINAL_JOB_STATUSES,DataModelingJob,DataModelingJobEngine,DataModelingJobStatus,@@ -109,7 +110,7 @@ def _succeed_node_and_data_modeling_job(job = DataModelingJob.objects.get(id=inputs.job_id)# if the job is already in a terminal state, don't overwrite it- if job.status in (DataModelingJobStatus.FAILED, DataModelingJobStatus.CANCELLED, DataModelingJobStatus.COMPLETED):+ if job.status in TERMINAL_JOB_STATUSES:return SucceedNodeAndJobOutcome(node=node,job=job,diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.pyindex 52e8b24afe2b..92eba4c9a5eb 100644--- a/posthog/temporal/data_modeling/workflows/materialize_view.py+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py@@ -177,6 +177,9 @@ def _run_mode(materialize_result: MaterializeViewResult | None) -> str:def _record_managed_warehouse_node_metrics(result: ManagedWarehouseShadowResult, tier: str) -> None:+ if result.skipped:+ get_node_finished_metric("skipped", MANAGED_WAREHOUSE_ENGINE, tier=tier, run_mode=result.run_mode).add(1)+ returnif result.error is not None:get_node_finished_metric("failed",@@ -893,7 +896,12 @@ class MaterializeViewWorkflow(PostHogWorkflow):shadow_result: ManagedWarehouseShadowResult = await shadow_handlerow_count_matched = clickhouse_row_count == shadow_result.row_count- status = "completed" if shadow_result.error is None else "failed"+ if shadow_result.skipped:+ status = "skipped"+ elif shadow_result.error is None:+ status = "completed"+ else:+ status = "failed"get_managed_warehouse_shadow_finished_metrics(status).add(1)get_clickhouse_materialization_duration_metric().record(clickhouse_duration_seconds)diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex 95f389851464..58536897699e 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -163,14 +163,46 @@ class TestMaterializeViewManagedWarehouseActivity:is_ready.assert_not_called()@pytest.mark.parametrize(- "compile_error,error_tracked,alias_dispatch_fails",+ "compile_error,error_tracked,alias_dispatch_fails,upstream_unbuilt,failed_status",[- (None, False, False),- (ValueError("Trino compilation failed"), True, False),- (TrinoLoweringError("TRINO_TABLE_LOCATOR_MISSING", "table"), False, False),- (TrinoUserError({"errorName": "COLUMN_NOT_FOUND", "errorType": "USER_ERROR"}), False, False),- (TrinoUserError({"errorName": "SYNTAX_ERROR", "errorType": "USER_ERROR"}), True, False),- (None, True, True),+ (None, False, False, False, None),+ (ValueError("Trino compilation failed"), True, False, False, DataModelingJobStatus.FAILED),+ (+ TrinoLoweringError("TRINO_TABLE_LOCATOR_MISSING", "table"),+ False,+ False,+ False,+ DataModelingJobStatus.FAILED,+ ),+ (+ TrinoUserError({"errorName": "COLUMN_NOT_FOUND", "errorType": "USER_ERROR"}),+ False,+ False,+ False,+ DataModelingJobStatus.FAILED,+ ),+ (+ TrinoUserError({"errorName": "TABLE_NOT_FOUND", "errorType": "USER_ERROR"}),+ False,+ False,+ True,+ DataModelingJobStatus.SKIPPED,+ ),+ (+ TrinoUserError({"errorName": "TABLE_NOT_FOUND", "errorType": "USER_ERROR"}),+ False,+ False,+ False,+ DataModelingJobStatus.FAILED,+ ),+ (+ TrinoUserError({"errorName": "SYNTAX_ERROR", "errorType": "USER_ERROR"}),+ True,+ False,+ False,+ DataModelingJobStatus.FAILED,+ ),+ (None, True, True, False, None),],)async def test_trino_shadow_records_execution_or_compilation_result(@@ -184,6 +216,8 @@ class TestMaterializeViewManagedWarehouseActivity:compile_error: Exception | None,error_tracked: bool,alias_dispatch_fails: bool,+ upstream_unbuilt: bool,+ failed_status: DataModelingJobStatus | None,) -> None:inputs = ManagedWarehouseShadowInputs(team_id=ateam.pk,@@ -222,6 +256,10 @@ class TestMaterializeViewManagedWarehouseActivity:"posthog.temporal.data_modeling.activities.materialize_view_managed_warehouse.prepare_executable_query",side_effect=lambda saved_query: setattr(saved_query, "query", executable_query),),+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.materialize_view_managed_warehouse.unbuilt_upstream_saved_query_ids",+ return_value=frozenset({uuid4()}) if upstream_unbuilt else frozenset(),+ ),unittest.mock.patch("posthog.temporal.data_modeling.activities.materialize_view_managed_warehouse.capture_exception") as capture,@@ -246,7 +284,7 @@ class TestMaterializeViewManagedWarehouseActivity:if compile_error is not None:reconcile_aliases.assert_not_awaited()assert result.error == str(compile_error)- assert ajob.status == DataModelingJobStatus.FAILED+ assert ajob.status == failed_statuselse:reconcile_aliases.assert_awaited_once_with(ateam.pk, str(asaved_query.id))assert result.row_count == 12@@ -394,11 +432,23 @@ class TestFailMaterializationActivity:# terminal transition must stamp it (the model default is the job's start time).assert ajob.last_run_at > ajob.created_at- async def test_does_not_overwrite_already_terminal_job(self, activity_environment, ateam, anode, ajob, adag):- ajob.status = DataModelingJob.Status.COMPLETED+ @pytest.mark.parametrize(+ "status,error",+ [+ (DataModelingJob.Status.COMPLETED, None),+ # A Trino shadow saved as skipped, then backstopped as orphaned after the worker died,+ # would otherwise become a failure that counts toward suspension.+ (DataModelingJob.Status.SKIPPED, "TABLE_NOT_FOUND: upstream missing"),+ ],+ )+ async def test_does_not_overwrite_already_terminal_job(+ self, activity_environment, ateam, anode, ajob, adag, status, error+ ):+ ajob.status = status+ ajob.error = errorawait database_sync_to_async(ajob.save)()await database_sync_to_async(ajob.refresh_from_db)()- completed_last_run_at = ajob.last_run_at+ terminal_last_run_at = ajob.last_run_atinputs = FailMaterializationInputs(team_id=ateam.pk,@@ -406,12 +456,13 @@ class TestFailMaterializationActivity:dag_id=str(adag.id),job_id=str(ajob.id),error="Late failure",+ update_node=False,)await activity_environment.run(fail_materialization_activity, inputs)await database_sync_to_async(ajob.refresh_from_db)()- assert ajob.status == DataModelingJob.Status.COMPLETED- assert ajob.error is None- assert ajob.last_run_at == completed_last_run_at+ assert ajob.status == status+ assert ajob.error == error+ assert ajob.last_run_at == terminal_last_run_atasync def test_updates_node_system_properties(self, activity_environment, ateam, anode, ajob, adag):inputs = FailMaterializationInputs(diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.pyindex bedeeb31e7f3..7d6237d1d7da 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py@@ -107,9 +107,17 @@ class TestQualityGateBranching:result = await MaterializeViewWorkflow().run(inputs or _inputs())return result, execute_activity- @pytest.mark.parametrize("shadow_error,blocked", [(None, False), ("Upstream write failed", False), (None, True)])+ @pytest.mark.parametrize(+ "shadow_error,skipped,blocked,expected_status",+ [+ (None, False, False, "completed"),+ ("Upstream write failed", False, False, "failed"),+ ("Upstream table missing", True, False, "skipped"),+ (None, False, True, None),+ ],+ )async def test_a_history_without_the_patch_replays_the_in_workflow_shadow(- self, shadow_error: str | None, blocked: bool+ self, shadow_error: str | None, skipped: bool, blocked: bool, expected_status: str | None) -> None:shadow = asyncio.get_running_loop().create_future()shadow.set_result(@@ -119,6 +127,7 @@ class TestQualityGateBranching:schema_name="models",table_name="orders",error=shadow_error,+ skipped=skipped,))activity_results = [@@ -140,9 +149,23 @@ class TestQualityGateBranching:assert result.job_id == "job-1"assert result.rows_materialized == 10assert result.trino_materialized is (not blocked and shadow_error is None)+ shadow_finished = self.metrics["get_managed_warehouse_shadow_finished_metrics"]+ if expected_status is None:+ shadow_finished.assert_not_called()+ else:+ shadow_finished.assert_called_once_with(expected_status)- @pytest.mark.parametrize("shadow_error", [None, "Trino write failed"])- async def test_trino_only_reports_materialization_failure(self, shadow_error: str | None) -> None:+ @pytest.mark.parametrize(+ "shadow_error,skipped,expected_status,expected_error_class",+ [+ (None, False, "completed", "none"),+ ("Trino write failed", False, "failed", "other"),+ ("Trino write failed: upstream table missing", True, "skipped", "none"),+ ],+ )+ async def test_trino_only_reports_materialization_failure(+ self, shadow_error: str | None, skipped: bool, expected_status: str, expected_error_class: str+ ) -> None:shadow = asyncio.get_running_loop().create_future()shadow.set_result(ManagedWarehouseShadowResult(@@ -152,6 +175,7 @@ class TestQualityGateBranching:table_name="orders",error=shadow_error,run_mode="incremental",+ skipped=skipped,))with patch.object(temporalio.workflow, "start_activity", return_value=shadow):@@ -168,8 +192,8 @@ class TestQualityGateBranching:assert result.job_id == "shadow-job"finished = self.metrics["get_node_finished_metric"]- assert finished.call_args.args == ("failed" if shadow_error else "completed", "managed_warehouse")- assert finished.call_args.kwargs.get("error_class", "none") == ("other" if shadow_error else "none")+ assert finished.call_args.args == (expected_status, "managed_warehouse")+ assert finished.call_args.kwargs.get("error_class", "none") == expected_error_classassert finished.call_args.kwargs["run_mode"] == "incremental"assert self.metrics["get_node_duration_metric"].called is (shadow_error is None)diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex 984a1b6dba99..7438acad3411 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -56,6 +56,7 @@ _LAZY = {"saved_query_node_ids": "logic.saved_query_reads","upstream_table_refs": "logic.saved_query_reads","upstream_saved_query_ids": "logic.saved_query_reads",+ "unbuilt_upstream_saved_query_ids": "logic.saved_query_reads","last_query_change_at": "logic.saved_query_reads","completed_full_refresh_workflow_ids": "logic.saved_query_reads","MaterializationFailedError": "facade.contracts",diff --git a/products/data_modeling/backend/facade/models.py b/products/data_modeling/backend/facade/models.pyindex e59d0b217d84..e600aba1a997 100644--- a/products/data_modeling/backend/facade/models.py+++ b/products/data_modeling/backend/facade/models.py@@ -11,6 +11,7 @@ methods, so importing this adds nothing beyond the models Django loads at ``setufrom products.data_modeling.backend.graph import Graphfrom products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAMEfrom products.data_modeling.backend.models.data_modeling_job import (+ TERMINAL_JOB_STATUSES,DataModelingJob,DataModelingJobEngine,DataModelingJobRunMode,@@ -36,6 +37,7 @@ __all__ = ["DEFAULT_DAG_NAME","NON_SCHEDULING_EDGES","SAVED_QUERY_NODE_TYPES",+ "TERMINAL_JOB_STATUSES","DataModelingJob","DataModelingJobEngine","DataModelingJobRunMode",diff --git a/products/data_modeling/backend/logic/saved_query_reads.py b/products/data_modeling/backend/logic/saved_query_reads.pyindex 5982273372cc..59a758240362 100644--- a/products/data_modeling/backend/logic/saved_query_reads.py+++ b/products/data_modeling/backend/logic/saved_query_reads.py@@ -421,6 +421,20 @@ def upstream_saved_query_ids(team_id: int, saved_query_id: UUID | str) -> frozen)+def unbuilt_upstream_saved_query_ids(+ team_id: int, saved_query_id: UUID | str, *, engine: DataModelingJobEngine+) -> frozenset[UUID]:+ """The upstream saved queries of this one that have never completed a build on ``engine``."""+ upstream_ids = upstream_saved_query_ids(team_id, saved_query_id)+ built_ids = DataModelingJob.objects.filter(+ team_id=team_id,+ saved_query_id__in=upstream_ids,+ engine=engine,+ status=DataModelingJobStatus.COMPLETED,+ ).values_list("saved_query_id", flat=True)+ return upstream_ids - frozenset(built_ids)++def last_query_change_at(team_id: int, saved_query_ids: Collection[UUID]) -> datetime | None:"""When someone last changed the query of any of these saved queries, per the activity log."""return ActivityLog.objects.filter(diff --git a/products/data_modeling/backend/models/data_modeling_job.py b/products/data_modeling/backend/models/data_modeling_job.pyindex 22100bfeb108..8b0f13e9c882 100644--- a/products/data_modeling/backend/models/data_modeling_job.py+++ b/products/data_modeling/backend/models/data_modeling_job.py@@ -12,6 +12,17 @@ class DataModelingJobStatus(models.TextChoices):SKIPPED = "Skipped", "Skipped"+# Once a job reaches one of these, later activities must not overwrite its status or error.+TERMINAL_JOB_STATUSES = frozenset(+ {+ DataModelingJobStatus.CANCELLED,+ DataModelingJobStatus.COMPLETED,+ DataModelingJobStatus.FAILED,+ DataModelingJobStatus.SKIPPED,+ }+)++class DataModelingJobEngine(models.TextChoices):CLICKHOUSE = "clickhouse", "ClickHouse"LEGACY_DUCKGRES = "duckgres", "Duckgres"diff --git a/products/data_modeling/backend/test/test_saved_query_reads.py b/products/data_modeling/backend/test/test_saved_query_reads.pyindex dc029bc48887..e455cd804eb3 100644--- a/products/data_modeling/backend/test/test_saved_query_reads.py+++ b/products/data_modeling/backend/test/test_saved_query_reads.py@@ -14,6 +14,11 @@ from products.data_modeling.backend.facade import apifrom products.data_modeling.backend.facade.contracts import SavedQueryDefinition, UpstreamTableReffrom products.data_modeling.backend.logic.saved_query_reads import POSTHOG_TABLE_ORIGINfrom products.data_modeling.backend.models.dag import DAG+from products.data_modeling.backend.models.data_modeling_job import (+ DataModelingJob,+ DataModelingJobEngine,+ DataModelingJobStatus,+)from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQueryfrom products.data_modeling.backend.models.edge import Edgefrom products.data_modeling.backend.models.node import Node, NodeType@@ -181,3 +186,24 @@ class TestSavedQueryReads(BaseTest):UpstreamTableRef(name="legacy_orders"),})++ def test_unbuilt_upstream_saved_query_ids_needs_a_completed_build_on_that_engine(self) -> None:+ dag = DAG.objects.create(team=self.team, name="Default")+ events = table_node(self.team, dag, "events", {"origin": POSTHOG_TABLE_ORIGIN})+ built = saved_query_node(self.team, dag, "built", NodeType.TABLE)+ built_elsewhere = saved_query_node(self.team, dag, "built_elsewhere", NodeType.TABLE)+ failed = saved_query_node(self.team, dag, "failed", NodeType.TABLE)+ target = saved_query_node(self.team, dag, "target", NodeType.VIEW)+ for source in [events, built, built_elsewhere, failed]:+ Edge.objects.create(team=self.team, dag=dag, source=source, target=target)+ for node, engine, status in [+ (built, DataModelingJobEngine.MANAGED_WAREHOUSE, DataModelingJobStatus.FAILED),+ (built, DataModelingJobEngine.MANAGED_WAREHOUSE, DataModelingJobStatus.COMPLETED),+ (built_elsewhere, DataModelingJobEngine.CLICKHOUSE, DataModelingJobStatus.COMPLETED),+ (failed, DataModelingJobEngine.MANAGED_WAREHOUSE, DataModelingJobStatus.FAILED),+ ]:+ DataModelingJob.objects.create(team=self.team, saved_query=node.saved_query, engine=engine, status=status)++ assert api.unbuilt_upstream_saved_query_ids(+ self.team.id, target.saved_query_id, engine=DataModelingJobEngine.MANAGED_WAREHOUSE+ ) == frozenset({built_elsewhere.saved_query_id, failed.saved_query_id})