:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(tasks): add task source columns (#127897)
Charles Vien1f3726bbb67f
diff --git a/products/posthog_ai/backend/tests/test_conversation_mirror.py b/products/posthog_ai/backend/tests/test_conversation_mirror.pyindex f92e565c1fda..d9ac330cdf69 100644--- a/products/posthog_ai/backend/tests/test_conversation_mirror.py+++ b/products/posthog_ai/backend/tests/test_conversation_mirror.py@@ -254,6 +254,7 @@ class TestMirrorConversation(APIBaseTest):assert result.skipped_reason is Nonetask = Task.objects.get(id=result.task_id)assert task.origin_key == origin_key_for_conversation(self.conversation.id)+ assert task.idempotency_key == task.origin_keyassert task.origin_product == Task.OriginProduct.POSTHOG_AIassert task.title == "Pageviews chat"assert task.title_manually_set is Falsediff --git a/products/review_hog/backend/receivers.py b/products/review_hog/backend/receivers.pyindex 0e8a1cdc0c9b..46df955c6d55 100644--- a/products/review_hog/backend/receivers.py+++ b/products/review_hog/backend/receivers.py@@ -142,7 +142,8 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:if pr_url is None:returntask = instance.task # first DB hit- if task.signal_report_id is None:+ signal_report_id = task.signal_report_source_id+ if signal_report_id is None:returnif task.deleted:# A soft-deleted task's run can still save output. Its PR is disowned work, and the@@ -154,7 +155,7 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:if (instance.state or {}).get("ai_stage") != "implementation":returnrepository = (task.repository or "").strip() or None- resolved = resolve_assigned_reviewers(instance.team_id, task.signal_report_id)+ resolved = resolve_assigned_reviewers(instance.team_id, signal_report_id)if not resolved:returnsettings_by_user = ReviewUserSettings.load_preferences_many(instance.team_id, resolved)@@ -166,14 +167,14 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:# The priority as it stood when the implementation task was created. The agent can append# judgments through the artefact API, so a later one may be its own vote on its review.signal_priority = persisted_report_priority(- team_id=instance.team_id, report_id=str(task.signal_report_id), before=task.created_at+ team_id=instance.team_id, report_id=str(signal_report_id), before=task.created_at)transaction.on_commit(lambda: _start_review(pr_url=pr_url,team_id=instance.team_id,user_id=acting_user_id,- signal_report_id=str(task.signal_report_id),+ signal_report_id=str(signal_report_id),signal_priority=signal_priority,repository=repository,),@@ -188,7 +189,7 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:repository=stamphog_repository,team_id=instance.team_id,acting_user_id=stamphog_user_id,- signal_report_id=str(task.signal_report_id),+ signal_report_id=str(signal_report_id),task_run_id=str(instance.id),),robust=True,diff --git a/products/signals/backend/inbox_push_notifications.py b/products/signals/backend/inbox_push_notifications.pyindex 9857e35437e7..78466e1edf48 100644--- a/products/signals/backend/inbox_push_notifications.py+++ b/products/signals/backend/inbox_push_notifications.py@@ -191,7 +191,10 @@ def notify_report_task_update(task = task_run.taskreports = list(SignalReport.objects.filter(team_id=task.team_id)- .filter(Q(id=task.signal_report_id) | SignalReport.reports_for_task_ids_filter([task.id], team_id=task.team_id))+ .filter(+ Q(id=task.signal_report_source_id)+ | SignalReport.reports_for_task_ids_filter([task.id], team_id=task.team_id)+ ).select_related("team").order_by("created_at", "id"))diff --git a/products/signals/backend/scout_harness/runner.py b/products/signals/backend/scout_harness/runner.pyindex 7e38bb845231..e8f8a18f8f8b 100644--- a/products/signals/backend/scout_harness/runner.py+++ b/products/signals/backend/scout_harness/runner.py@@ -181,12 +181,10 @@ def _get_trial_config(trial: TrialLaunch) -> SignalScoutConfig:def _existing_trial_result(team_id: int, trial: TrialLaunch) -> RunResult | None:- run = (- SignalScoutRun.objects.for_team(team_id)- .select_related("task_run")- .filter(task_run__task__origin_key=f"scout-trial:{trial.id}")- .first()- )+ task_id = tasks_facade.get_task_id_by_idempotency_key(team_id, f"scout-trial:{trial.id}")+ if task_id is None:+ return None+ run = SignalScoutRun.objects.for_team(team_id).select_related("task_run").filter(task_run__task_id=task_id).first()if run is None:return Nonereturn RunResult(diff --git a/products/signals/backend/scout_harness/trial_evaluation.py b/products/signals/backend/scout_harness/trial_evaluation.pyindex eed236a20e22..df68322a88c5 100644--- a/products/signals/backend/scout_harness/trial_evaluation.py+++ b/products/signals/backend/scout_harness/trial_evaluation.py@@ -351,7 +351,7 @@ def _bound_run(launch: TrialLaunch) -> SignalScoutRun | None:or task.created_by_id != launch.user_idor task.deletedor task.origin_product != "signals_scout"- or task.origin_key != f"scout-trial:{launch.id}"+ or task.stored_idempotency_key != f"scout-trial:{launch.id}"or not isinstance(marker, dict)or marker.get("version") != 1or marker.get("context_id") != str(launch.context_id)diff --git a/products/signals/backend/scout_harness/trial_inspection.py b/products/signals/backend/scout_harness/trial_inspection.pyindex 9c84ef5e009b..c58e2b0c2f17 100644--- a/products/signals/backend/scout_harness/trial_inspection.py+++ b/products/signals/backend/scout_harness/trial_inspection.py@@ -176,7 +176,7 @@ class ScoutTrialInspection:if (not isinstance(marker.get("launch_id"), str)or run.task_run.task.origin_product != "signals_scout"- or run.task_run.task.origin_key != f"scout-trial:{marker['launch_id']}"+ or run.task_run.task.stored_idempotency_key != f"scout-trial:{marker['launch_id']}"):continuetry:diff --git a/products/signals/backend/scout_harness/trial_judge.py b/products/signals/backend/scout_harness/trial_judge.pyindex 3cb222142f36..4ad804fc1460 100644--- a/products/signals/backend/scout_harness/trial_judge.py+++ b/products/signals/backend/scout_harness/trial_judge.py@@ -78,7 +78,6 @@ def _assert_scout_available(snapshot: TrialEvaluationSnapshot, evidence: TrialRutask_run__task__created_by_id=snapshot.user_id,task_run__task__deleted=False,task_run__task__origin_product="signals_scout",- task_run__task__origin_key=f"scout-trial:{evidence.launch_id}",).first())@@ -90,6 +89,7 @@ def _assert_scout_available(snapshot: TrialEvaluationSnapshot, evidence: TrialRuor marker.get("launch_id") != str(evidence.launch_id)or marker.get("context_id") != str(snapshot.context_id)or run.task_run.status != "completed"+ or run.task_run.task.stored_idempotency_key != f"scout-trial:{evidence.launch_id}"):raise TrialJudgeValidationError("The saved scout run is no longer available for this evaluation.")if ScoutTrialStore(run).invalid_reason() is not None:diff --git a/products/signals/backend/scout_harness/trial_launch.py b/products/signals/backend/scout_harness/trial_launch.pyindex bcd1425f4847..71a2105190f6 100644--- a/products/signals/backend/scout_harness/trial_launch.py+++ b/products/signals/backend/scout_harness/trial_launch.py@@ -484,6 +484,6 @@ def bound_trial_run(team_id: int, task_id: UUID | str | None) -> SignalScoutRunif run is None or run.metadata is None:return Nonelaunch_id = run.metadata["scout_trial"].get("launch_id")- if not isinstance(launch_id, str) or run.task_run.task.origin_key != f"scout-trial:{launch_id}":+ if not isinstance(launch_id, str) or run.task_run.task.stored_idempotency_key != f"scout-trial:{launch_id}":return Nonereturn rundiff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.pyindex c1af51c12904..3b783ab66b80 100644--- a/products/tasks/backend/facade/api.py+++ b/products/tasks/backend/facade/api.py@@ -39,11 +39,12 @@ from django.db.models import (QuerySet,Subquery,TextChoices,+ UUIDField,Value,When,)from django.db.models.fields.json import KeyTextTransform, KeyTransform-from django.db.models.functions import Coalesce+from django.db.models.functions import Cast, Coalescefrom django.utils import timezone as django_timezonefrom django.utils.dateparse import parse_datetimefrom django.utils.http import content_disposition_header@@ -109,6 +110,7 @@ from products.tasks.backend.facade.gateway import mint_private_gateway_token, refrom products.tasks.backend.facade.profiles import (GithubAccess,NetworkAccess,+ SourceKind,TaskProfile,TaskRunOptions,TaskSource,@@ -211,7 +213,7 @@ from products.tasks.backend.repository_config_analytics import (capture_space_context_changed,repositories_differ,)-from products.tasks.backend.task_sources import apply_task_source+from products.tasks.backend.task_sources import apply_task_source, idempotency_key_q, source_qfrom products.tasks.backend.visibility import (PRODUCT_PRIVATE_ORIGIN_PRODUCTS,TEAM_READABLE_ORIGIN_PRODUCTS,@@ -278,6 +280,7 @@ __all__ = ["create_imported_task_run","get_imported_task_run","get_task_by_origin_key",+ "get_task_id_by_idempotency_key","touch_imported_task","apply_task_run_model_config","task_run_model_outside_gateway_pin",@@ -1083,7 +1086,7 @@ def _task_detail_to_dto(repositories=task.repositories or ([task.repository] if task.repository else []),github_integration=task.github_integration_id,github_user_integration=task.github_user_integration_id,- signal_report=task.signal_report_id,+ signal_report=task.signal_report_source_id,json_schema=task.json_schema,internal=task.internal,archived=task.archived,@@ -1097,7 +1100,7 @@ def _task_detail_to_dto(latest_run_id=latest_run_id,channel=task.channel_id,slack_thread_references=_task_slack_thread_references(task),- origin_key=None if task.is_scout_experiment else task.origin_key,+ origin_key=None if task.is_scout_experiment else task.stored_idempotency_key,client_provenance=task.client_provenance,)@@ -1239,13 +1242,14 @@ def find_signal_implementation_run(task = run.task# Belt and braces: only signals' auto_start stamps the branch key today, but the ai_stage and# signal-report checks keep a future writer of the key from silently widening the carve-out.- if task.signal_report_id is None or (run.state or {}).get("ai_stage") != "implementation":+ signal_report_id = task.signal_report_source_id+ if signal_report_id is None or (run.state or {}).get("ai_stage") != "implementation":return Nonereturn contracts.SignalImplementationRunDTO(run_id=run.id,task_id=task.id,team_id=run.team_id,- signal_report_id=task.signal_report_id,+ signal_report_id=signal_report_id,task_created_by_id=task.created_by_id,)@@ -1431,32 +1435,46 @@ def task_exempt_from_code_access(task_id: str | UUID, team_id: int) -> bool:bypass. The report's own team is re-checked here even though the write serializeralready enforces it, so a future write path can't silently widen the exemption."""- return Task.objects.filter(- Q(- origin_product=Task.OriginProduct.SIGNAL_REPORT,- signal_report__team_id=team_id,- repository__isnull=True,- repositories=[],- github_integration__isnull=True,- github_user_integration__isnull=True,- )- | Q(- origin_product=Task.OriginProduct.SIGNALS_CHAT,- repository__isnull=True,- repositories=[],- github_integration__isnull=True,- github_user_integration__isnull=True,- )- | Q(- origin_product=Task.OriginProduct.SIGNAL_REPORT,- signal_report__team_id=team_id,- signal_report_tasks__team_id=team_id,- signal_report_tasks__report_id=F("signal_report_id"),- signal_report_tasks__relationship="implementation",+ linked_report_id = Case(+ When(+ source_kind=SourceKind.SIGNAL_REPORT.value,+ source_id__isnull=False,+ then=Cast("source_id", UUIDField()),),- id=task_id,- team_id=team_id,- ).exists()+ When(Q(source_kind__isnull=True) | Q(source_id__isnull=True), then=F("signal_report_id")),+ output_field=UUIDField(),+ )+ report_in_team = Exists(signal_report_queryset().filter(id=OuterRef("_linked_report_id"), team_id=team_id))+ return (+ Task.objects.annotate(_linked_report_id=linked_report_id)+ .filter(+ Q(+ report_in_team,+ origin_product=Task.OriginProduct.SIGNAL_REPORT,+ repository__isnull=True,+ repositories=[],+ github_integration__isnull=True,+ github_user_integration__isnull=True,+ )+ | Q(+ origin_product=Task.OriginProduct.SIGNALS_CHAT,+ repository__isnull=True,+ repositories=[],+ github_integration__isnull=True,+ github_user_integration__isnull=True,+ )+ | Q(+ report_in_team,+ origin_product=Task.OriginProduct.SIGNAL_REPORT,+ signal_report_tasks__team_id=team_id,+ signal_report_tasks__report_id=F("_linked_report_id"),+ signal_report_tasks__relationship="implementation",+ ),+ id=task_id,+ team_id=team_id,+ )+ .exists()+ )def get_in_progress_runs_for_github_integration(@@ -1520,10 +1538,10 @@ def get_signal_report_implementation_runs(return [_task_run_to_dto(run)for run in TaskRun.objects.filter(+ source_q(SourceKind.SIGNAL_REPORT, [report_id], via="task__"),team_id=team_id,task_id__in=task_ids,task__team_id=team_id,- task__signal_report_id=report_id,task__origin_product=Task.OriginProduct.SIGNAL_REPORT,task__deleted=False,)@@ -2020,11 +2038,13 @@ def link_slack_task_to_report(*, team_id: int, task_id: str, report_id: str, usetask = Task.objects.select_for_update().get(team_id=team_id, id=task_id, origin_product=Task.OriginProduct.SLACK, created_by_id=user_id)- if task.signal_report_id is not None:+ if task.signal_report_source_id is not None:returnrecord_slack_report_discussion(team_id=team_id, report_id=report_id, task_id=task_id, user_id=user_id)task.signal_report_id = UUID(report_id)- task.save(update_fields=["signal_report_id", "updated_at"])+ task.source_kind = SourceKind.SIGNAL_REPORT.value+ task.source_id = str(task.signal_report_id)+ task.save(update_fields=["signal_report_id", "source_kind", "source_id", "updated_at"])def create_and_run_task(@@ -4085,12 +4105,16 @@ def attach_task_run_input_files(*, team_id: int, run_id: UUID, files: Sequence[crun.save(update_fields=["artifacts", "updated_at"])+def get_task_id_by_idempotency_key(team_id: int, key: str) -> UUID | None:+ return Task.objects.filter(idempotency_key_q(key), team_id=team_id).values_list("id", flat=True).first()++def get_task_by_origin_key(team_id: int, origin_key: str, *, include_latest_run: bool = True) -> contracts.TaskDetailDTO | None:"""The live task carrying ``origin_key`` in ``team_id``, or None. Bypasses visibility: anorigin key is an idempotency handle for a system writer, not a user-facing lookup."""- task = Task.objects.filter(team_id=team_id, origin_key=origin_key, deleted=False).first()+ task = Task.objects.filter(idempotency_key_q(origin_key), team_id=team_id, deleted=False).first()return _task_detail_to_dto(task, include_latest_run=include_latest_run) if task is not None else None@@ -7312,14 +7336,25 @@ def list_workflow_last_runs(latest_run = TaskRun.objects.filter(task=OuterRef("pk"), team_id=team_id).order_by("-created_at", "-id")rows = (_visible_task_qs(team_id, user_id, for_list=True)- .filter(hog_flow_id__in=ids, internal=False)- .order_by("hog_flow_id", "-created_at", "-id")- .distinct("hog_flow_id")+ .filter(source_q(SourceKind.HOG_FLOW, ids), internal=False)+ .annotate(+ _workflow_id=Case(+ When(+ source_kind=SourceKind.HOG_FLOW.value,+ source_id__isnull=False,+ then=Cast("source_id", UUIDField()),+ ),+ default=F("hog_flow_id"),+ output_field=UUIDField(),+ ),+ )+ .order_by("_workflow_id", "-created_at", "-id")+ .distinct("_workflow_id").annotate(_run_status=Subquery(latest_run.values("status")[:1]),_run_created_at=Subquery(latest_run.values("created_at")[:1]),)- .values_list("hog_flow_id", "id", "created_at", "_run_status", "_run_created_at")+ .values_list("_workflow_id", "id", "created_at", "_run_status", "_run_created_at"))return {hog_flow_id: contracts.WorkflowLastRunDTO(@@ -7461,7 +7496,7 @@ def _list_tasks_queryset(hog_flow_id = filters.get("hog_flow_id")if hog_flow_id:- qs = qs.filter(hog_flow_id=hog_flow_id)+ qs = qs.filter(source_q(SourceKind.HOG_FLOW, [hog_flow_id]))if search:search_term = search.strip()@@ -8381,6 +8416,12 @@ def create_task()validated_data["github_integration"] = pilot_integration+ if report_ref:+ validated_data["source_kind"] = SourceKind.SIGNAL_REPORT.value+ validated_data["source_id"] = str(getattr(report_ref, "id", report_ref))+ if validated_data.get("origin_key") is not None:+ validated_data["idempotency_key"] = validated_data["origin_key"]+logger.info("Creating task with data: %s", validated_data)with transaction.atomic():if signal_report_id:@@ -8394,13 +8435,13 @@ def create_task(created_by_id=user_id,)task = Task.objects.create(**validated_data)- if task.signal_report_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT:+ if task.signal_report_source_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT:# Record the task↔report association + work-log artefact for the asserted relationship# (defaults to implementation, which also writes the auto-start spend gate row) so a# manually-started task matches autostarted ones.record_report_task(team_id=task.team_id,- report_id=str(task.signal_report_id),+ report_id=str(task.signal_report_source_id),task_id=str(task.id),relationship=signal_report_task_relationship,)@@ -8937,8 +8978,10 @@ def _find_idling_warm_run(return Nonenormalized_repositories = [repo.lower() for repo in (repositories or ([repository] if repository else []))]repository_filter = {"task__repository__iexact": repository} if repository else {"task__repository__isnull": True}+ report_filter = source_q(SourceKind.SIGNAL_REPORT, [signal_report_id], via="task__") if signal_report_id else Q()candidates = (TaskRun.objects.filter( # nosemgrep: idor-lookup-without-team — team_id filter applied via the task FK below+ report_filter,task__team_id=team_id,task__created_by_id=user_id,task__origin_product=origin_product,@@ -8948,7 +8991,6 @@ def _find_idling_warm_run(**({"state__billing_product": "posthog_ai"} if origin_product == Task.OriginProduct.SIGNAL_REPORT else {}),branch=branch or None,**repository_filter,- **({"task__signal_report_id": signal_report_id} if signal_report_id else {}),).exclude(status__in=_TERMINAL_TASK_RUN_STATUSES).select_related("task")@@ -9271,7 +9313,7 @@ def _reserve_report_warm_activation(record_report_task,)- report_id = str(task.signal_report_id)+ report_id = str(task.signal_report_source_id)if is_pull_request_run_request(description):enforce_self_driving_free_trial(team, report_id=report_id)with transaction.atomic():@@ -9288,7 +9330,7 @@ def _reserve_report_warm_activation(def _is_unlinked_report_warm_task(task: Task) -> bool:- if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or not task.signal_report_id:+ if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or not task.signal_report_source_id:return Falseif not task.runs.filter(team_id=task.team_id, state__has_key="prewarmed").exists():return False@@ -9296,7 +9338,7 @@ def _is_unlinked_report_warm_task(task: Task) -> bool:report_task_ids,)- return str(task.id) not in report_task_ids(team_id=task.team_id, report_id=str(task.signal_report_id))+ return str(task.id) not in report_task_ids(team_id=task.team_id, report_id=str(task.signal_report_source_id))def task_run_awaits_report_activation(run_id: str | UUID, task_id: str | UUID, team_id: int) -> bool:@@ -9754,8 +9796,8 @@ def run_task(error=contracts.TaskValidationError(kind="detail", detail=cap_detail, code=ACTIVE_SESSION_CAP_CODE))report_id_for_slot_check = (- str(task.signal_report_id)- if task.signal_report_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT+ str(task.signal_report_source_id)+ if task.signal_report_source_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORTelse None)is_implementation = Falsediff --git a/products/tasks/backend/facade/canvas_tasks.py b/products/tasks/backend/facade/canvas_tasks.pyindex faf87f53ff8d..8fc8dc5d6f0d 100644--- a/products/tasks/backend/facade/canvas_tasks.py+++ b/products/tasks/backend/facade/canvas_tasks.py@@ -6,6 +6,7 @@ from django.db import transactionfrom products.tasks.backend.facade import apifrom products.tasks.backend.facade.contracts import TaskRunDTOfrom products.tasks.backend.models import Channel, Task+from products.tasks.backend.task_sources import idempotency_key_qdef create_and_run_channel_task(@@ -32,7 +33,7 @@ def create_and_run_channel_task(raise ValueError("Space not found.")origin_key = f"canvas:{canvas_id}:{user_id}:{idempotency_key}"- existing = Task.objects.filter(team_id=team_id, origin_key=origin_key).first()+ existing = Task.objects.filter(idempotency_key_q(origin_key), team_id=team_id).first()if existing is not None:if existing.deleted or existing.channel_id != channel.id or existing.created_by_id != user_id:raise ValueError("The previous task is no longer available. Start a new request.")diff --git a/products/tasks/backend/facade/onboarding.py b/products/tasks/backend/facade/onboarding.pyindex 13bc7e5e9355..0bd252d4a13a 100644--- a/products/tasks/backend/facade/onboarding.py+++ b/products/tasks/backend/facade/onboarding.py@@ -38,6 +38,7 @@ from products.tasks.backend.facade.onboarding_prompt import ()from products.tasks.backend.logic.services.model_catalogue import filter_unsupported_effort, runtime_adapter_forfrom products.tasks.backend.models import Task, TaskClientProvenance+from products.tasks.backend.task_sources import idempotency_key_qfrom ee.billing.salesforce_enrichment.constants import PERSONAL_EMAIL_DOMAINS@@ -62,8 +63,8 @@ def _origin_key(user_id: int) -> str:def _started_session_id(team_id: int, user_id: int) -> UUID | None:return (Task.objects.filter(+ idempotency_key_q(_origin_key(user_id)),team_id=team_id,- origin_key=_origin_key(user_id),created_by_id=user_id,origin_product=Task.OriginProduct.USER_CREATED,)diff --git a/products/tasks/backend/facade/profiles.py b/products/tasks/backend/facade/profiles.pyindex 45a3ec8c946b..0ac3972a77d9 100644--- a/products/tasks/backend/facade/profiles.py+++ b/products/tasks/backend/facade/profiles.py@@ -75,6 +75,7 @@ class ContractCheck(StrEnum):class SourceKind(StrEnum):SIGNAL_REPORT = "signal_report"HOG_FLOW = "hog_flow"+ LOOP = "loop"@frozendiff --git a/products/tasks/backend/logic/services/task_analysis.py b/products/tasks/backend/logic/services/task_analysis.pyindex ce469510d481..d692e3e18049 100644--- a/products/tasks/backend/logic/services/task_analysis.py+++ b/products/tasks/backend/logic/services/task_analysis.py@@ -31,6 +31,7 @@ from products.tasks.backend.logic.services.staged_artifacts import (tag_task_artifact,)from products.tasks.backend.models import SandboxCustomImage, Task, TaskRun+from products.tasks.backend.task_sources import idempotency_key_startswith_qlogger = structlog.get_logger(__name__)@@ -121,9 +122,9 @@ def _next_analysis_attempt(*, team_id: int, target_run_id: str) -> int:one possible, which a bare per-run key would block forever."""return Task.objects.filter(+ idempotency_key_startswith_q(f"{TASK_ANALYSIS_ORIGIN_KEY_PREFIX}:{target_run_id}:"),team_id=team_id,origin_product=Task.OriginProduct.TASK_ANALYSIS,- origin_key__startswith=f"{TASK_ANALYSIS_ORIGIN_KEY_PREFIX}:{target_run_id}:",).count()diff --git a/products/tasks/backend/logic/services/workflow_step_resume.py b/products/tasks/backend/logic/services/workflow_step_resume.pyindex 54b10c94ffc4..f15fa47269ae 100644--- a/products/tasks/backend/logic/services/workflow_step_resume.py+++ b/products/tasks/backend/logic/services/workflow_step_resume.py@@ -80,9 +80,9 @@ def _result_for_run(task_run: TaskRun) -> dict[str, Any]:def _workflow_origin_key(task_run: TaskRun) -> str | None:task = task_run.task- if task.origin_product != Task.OriginProduct.WORKFLOW or not task.origin_key:+ if task.origin_product != Task.OriginProduct.WORKFLOW or not task.stored_idempotency_key:return None- return task.origin_key+ return task.stored_idempotency_keydef _emit(task_run: TaskRun, origin_key: str, status: WorkflowStepResumeStatus) -> None:diff --git a/products/tasks/backend/logic/services/workflow_tasks.py b/products/tasks/backend/logic/services/workflow_tasks.pyindex e1a7e3d57b65..772b05661d4a 100644--- a/products/tasks/backend/logic/services/workflow_tasks.py+++ b/products/tasks/backend/logic/services/workflow_tasks.py@@ -29,6 +29,7 @@ from products.slack_app.backend.facade.api import slack_artifact_delivery_state_from products.slack_app.backend.models import SlackThreadTaskMappingfrom products.slack_app.backend.slack_thread import SlackThreadContextfrom products.tasks.backend.facade import contracts+from products.tasks.backend.facade.profiles import SourceKindfrom products.tasks.backend.logic.services.ai_run_defaults import resolve_ai_run_selectionfrom products.tasks.backend.logic.services.code_usage_gate import usage_limit_responsefrom products.tasks.backend.logic.services.gateway_model_pin import model_allowed_by_product_pin@@ -46,6 +47,7 @@ from products.tasks.backend.logic.services.workflow_task_skills import (from products.tasks.backend.metrics import observe_workflow_task_createfrom products.tasks.backend.model_catalog import CLAUDEfrom products.tasks.backend.models import Channel, Task, TaskRun+from products.tasks.backend.task_sources import idempotency_key_q, source_qfrom products.tasks.backend.temporal.constants import WORKFLOW_RUN_IDLE_TIMEOUT_SECONDSfrom ee.billing.quota_limiting import QuotaResource, is_team_over_credit_budget@@ -331,7 +333,9 @@ def create_workflow_task(raise WorkflowTaskTeamRateCapped(team_rate_cap)in_flight = TaskRun.objects.filter(- task__team_id=team.id, task__hog_flow_id=hog_flow_id, status__in=ACTIVE_RUN_STATUSES+ source_q(SourceKind.HOG_FLOW, [hog_flow_id], via="task__"),+ task__team_id=team.id,+ status__in=ACTIVE_RUN_STATUSES,).count()if in_flight >= max_parallel_tasks:observe_workflow_task_create(reason="limit_reached")@@ -496,10 +500,10 @@ def _find_replayed_task() -> contracts.WorkflowTaskDTO | None:if origin_key is None:return None- existing = Task.objects.filter(team_id=team_id, origin_key=origin_key).first()+ existing = Task.objects.filter(idempotency_key_q(origin_key), team_id=team_id).first()if existing is None:return None- if existing.hog_flow_id != hog_flow_id:+ if existing.hog_flow_source_id != hog_flow_id:raise WorkflowTaskOriginKeyConflict(origin_key)return _task_dto(existing, created=False)@@ -516,7 +520,9 @@ def _daily_task_counts(team_id: int, hog_flow_id: uuid.UUID) -> _DailyTaskCounts"""One function so the unlocked pre-check and the authoritative locked recheck runthe identical query and can't drift apart."""since = django_timezone.now() - timedelta(hours=24)- workflow_count = Task.objects.filter(team_id=team_id, hog_flow_id=hog_flow_id, created_at__gte=since).count()+ workflow_count = Task.objects.filter(+ source_q(SourceKind.HOG_FLOW, [hog_flow_id]), team_id=team_id, created_at__gte=since+ ).count()team_count = Task.objects.filter(team_id=team_id, origin_product=Task.OriginProduct.WORKFLOW, created_at__gte=since).count()diff --git a/products/tasks/backend/management/commands/backfill_task_sources.py b/products/tasks/backend/management/commands/backfill_task_sources.pynew file mode 100644index 000000000000..568eba9f9a76--- /dev/null+++ b/products/tasks/backend/management/commands/backfill_task_sources.py@@ -0,0 +1,130 @@+import time+from collections.abc import Iterator+from typing import Any++from django.core.management.base import BaseCommand, CommandParser+from django.db import connection++from posthog.dataclasses import frozen++NIL_UUID = "00000000-0000-0000-0000-000000000000"+MAX_BATCH_SIZE = 5000++_LOOP_COLUMN_EXISTS_SQL = (+ "SELECT EXISTS ("+ "SELECT 1 FROM information_schema.columns "+ "WHERE table_schema = current_schema() AND table_name = 'posthog_task' AND column_name = 'loop_id'"+ ")"+)++_CANDIDATES_SQL = (+ "SELECT p.id FROM posthog_task p "+ "WHERE p.id > %(after)s::uuid "+ "AND (%(team_id)s::bigint IS NULL OR p.team_id = %(team_id)s::bigint) "+ "AND ("+ "((p.source_kind IS NULL OR p.source_id IS NULL) "+ "AND (p.signal_report_id IS NOT NULL OR p.hog_flow_id IS NOT NULL{loop_present})) "+ "OR (p.idempotency_key IS NULL AND p.origin_key IS NOT NULL)"+ ") "+ "ORDER BY p.id "+ "LIMIT %(batch_size)s"+)++_UPDATE_SQL = (+ "WITH batch AS ({candidates}) "+ "UPDATE posthog_task AS t "+ "SET source_kind = CASE WHEN t.source_kind IS NOT NULL AND t.source_id IS NOT NULL THEN t.source_kind "+ "ELSE COALESCE(CASE "+ "WHEN t.signal_report_id IS NOT NULL THEN 'signal_report' "+ "WHEN t.hog_flow_id IS NOT NULL THEN 'hog_flow'{loop_kind} "+ "END, t.source_kind) END, "+ "source_id = CASE WHEN t.source_kind IS NOT NULL AND t.source_id IS NOT NULL THEN t.source_id "+ "ELSE COALESCE(CASE "+ "WHEN t.signal_report_id IS NOT NULL THEN t.signal_report_id::text "+ "WHEN t.hog_flow_id IS NOT NULL THEN t.hog_flow_id::text{loop_id} "+ "END, t.source_id) END, "+ "idempotency_key = COALESCE(t.idempotency_key, t.origin_key) "+ "FROM batch WHERE t.id = batch.id "+ "RETURNING t.id"+)++_LOOP_PRESENT = " OR p.loop_id IS NOT NULL"+_LOOP_KIND = " WHEN t.loop_id IS NOT NULL THEN 'loop'"+_LOOP_ID = " WHEN t.loop_id IS NOT NULL THEN t.loop_id::text"+++@frozen+class BackfillBatch:+ tasks: int+ cursor: str+++def _loop_column_exists() -> bool:+ with connection.cursor() as cursor:+ cursor.execute(_LOOP_COLUMN_EXISTS_SQL)+ row = cursor.fetchone()+ return bool(row and row[0])+++def _candidates_sql(*, with_loop: bool) -> str:+ return _CANDIDATES_SQL.format(loop_present=_LOOP_PRESENT if with_loop else "")+++def _update_sql(*, with_loop: bool) -> str:+ return _UPDATE_SQL.format(+ candidates=_candidates_sql(with_loop=with_loop),+ loop_kind=_LOOP_KIND if with_loop else "",+ loop_id=_LOOP_ID if with_loop else "",+ )+++def backfill_task_sources(+ *,+ team_id: int | None = None,+ batch_size: int = 1000,+ after: str | None = None,+ dry_run: bool = False,+ sleep_seconds: float = 0.0,+) -> Iterator[BackfillBatch]:+ with_loop = _loop_column_exists()+ sql = _candidates_sql(with_loop=with_loop) if dry_run else _update_sql(with_loop=with_loop)+ params = {"team_id": team_id, "batch_size": min(max(batch_size, 1), MAX_BATCH_SIZE)}+ cursor_id = after or NIL_UUID+ while True:+ with connection.cursor() as cursor:+ cursor.execute(sql, {**params, "after": cursor_id})+ task_ids = [row[0] for row in cursor.fetchall()]+ if not task_ids:+ return+ cursor_id = str(max(task_ids))+ yield BackfillBatch(tasks=len(task_ids), cursor=cursor_id)+ if sleep_seconds:+ time.sleep(sleep_seconds)+++class Command(BaseCommand):+ help = (+ "Fill source_kind, source_id and idempotency_key on tasks created before those columns existed. "+ "Safe to rerun, and resumable with --after."+ )++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, metavar="ID", help="Only tasks in this team")+ parser.add_argument("--batch-size", type=int, default=1000)+ parser.add_argument("--after", help="Resume after this task id, as printed by an earlier run")+ parser.add_argument("--sleep-seconds", type=float, default=0.1, help="Pause between batches")+ parser.add_argument("--dry-run", action="store_true", help="Count the tasks without writing")++ def handle(self, *args: Any, **options: Any) -> None:+ verb = "Would fill" if options["dry_run"] else "Filled"+ total = 0+ for batch in backfill_task_sources(+ team_id=options["team_id"],+ batch_size=options["batch_size"],+ after=options["after"],+ dry_run=options["dry_run"],+ sleep_seconds=options["sleep_seconds"],+ ):+ total += batch.tasks+ self.stdout.write(f"{verb} {total} tasks; resume after {batch.cursor}")+ self.stdout.write(f"{verb} {total} tasks in all")diff --git a/products/tasks/backend/management/commands/test/test_backfill_task_sources.py b/products/tasks/backend/management/commands/test/test_backfill_task_sources.pynew file mode 100644index 000000000000..c1415f8ab0e5--- /dev/null+++ b/products/tasks/backend/management/commands/test/test_backfill_task_sources.py@@ -0,0 +1,145 @@+from io import StringIO+from uuid import UUID, uuid4++from django.core.management import call_command+from django.db import connection+from django.test import TestCase++from parameterized import parameterized++from posthog.models import Organization, Team++from products.signals.backend.models import SignalReport+from products.tasks.backend.management.commands.backfill_task_sources import backfill_task_sources+from products.tasks.backend.models import Task++REPORT_ID = UUID("11111111-1111-4111-8111-111111111111")+FLOW_ID = UUID("22222222-2222-4222-8222-222222222222")+LOOP_ID = UUID("33333333-3333-4333-8333-333333333333")+++class TestBackfillTaskSources(TestCase):+ def setUp(self) -> None:+ super().setUp()+ self.team = Team.objects.create(organization=Organization.objects.create(name="Org"), name="Team")+ SignalReport.objects.create(team=self.team, id=REPORT_ID)++ def _task(self, team: Team | None = None, **columns: object) -> Task:+ return Task.objects.create(+ team=team or self.team,+ title="Task",+ description="d",+ origin_product=Task.OriginProduct.USER_CREATED,+ **columns,+ )++ def _columns(self, task: Task) -> tuple[str | None, str | None, str | None]:+ task.refresh_from_db()+ return (task.source_kind, task.source_id, task.idempotency_key)++ @parameterized.expand(+ [+ ("nothing_to_fill", {}, (None, None, None)),+ ("report_only", {"signal_report_id": REPORT_ID}, ("signal_report", str(REPORT_ID), None)),+ ("flow_only", {"hog_flow_id": FLOW_ID}, ("hog_flow", str(FLOW_ID), None)),+ ("key_only", {"origin_key": "key"}, (None, None, "key")),+ (+ "report_and_key",+ {"signal_report_id": REPORT_ID, "origin_key": "key"},+ ("signal_report", str(REPORT_ID), "key"),+ ),+ (+ "source_filled_key_missing",+ {"hog_flow_id": FLOW_ID, "source_kind": "hog_flow", "source_id": str(FLOW_ID), "origin_key": "key"},+ ("hog_flow", str(FLOW_ID), "key"),+ ),+ (+ "key_filled_source_missing",+ {"hog_flow_id": FLOW_ID, "origin_key": "key", "idempotency_key": "key"},+ ("hog_flow", str(FLOW_ID), "key"),+ ),+ ("kind_without_id", {"hog_flow_id": FLOW_ID, "source_kind": "hog_flow"}, ("hog_flow", str(FLOW_ID), None)),+ (+ "id_without_kind",+ {"hog_flow_id": FLOW_ID, "source_id": str(FLOW_ID)},+ ("hog_flow", str(FLOW_ID), None),+ ),+ (+ "kind_without_id_over_another_old_source",+ {"signal_report_id": REPORT_ID, "source_kind": "hog_flow"},+ ("signal_report", str(REPORT_ID), None),+ ),+ ("kind_without_id_or_old_source", {"source_kind": "hog_flow"}, ("hog_flow", None, None)),+ (+ "new_columns_win_over_old",+ {+ "signal_report_id": REPORT_ID,+ "origin_key": "old",+ "source_kind": "loop",+ "source_id": "loop-1",+ "idempotency_key": "new",+ },+ ("loop", "loop-1", "new"),+ ),+ ]+ )+ def test_fills_the_columns_a_task_is_missing(+ self, _name: str, columns: dict, expected: tuple[str | None, str | None, str | None]+ ) -> None:+ task = self._task(**columns)++ list(backfill_task_sources())++ self.assertEqual(self._columns(task), expected)+ self.assertEqual(list(backfill_task_sources(dry_run=True)), [])++ def test_fills_the_source_of_a_legacy_loop_task(self) -> None:+ task = self._task()+ with connection.cursor() as cursor:+ cursor.execute("UPDATE posthog_task SET loop_id = %s WHERE id = %s", [LOOP_ID, task.id])++ list(backfill_task_sources())++ self.assertEqual(self._columns(task), ("loop", str(LOOP_ID), None))++ def test_dry_run_counts_the_tasks_without_writing(self) -> None:+ task = self._task(origin_key="key")++ batches = list(backfill_task_sources(dry_run=True))++ self.assertEqual([batch.tasks for batch in batches], [1])+ self.assertEqual(self._columns(task), (None, None, None))++ def test_only_fills_the_requested_team(self) -> None:+ other_team = Team.objects.create(organization=self.team.organization, name="Other")+ task = self._task(origin_key="key")+ other = self._task(team=other_team, origin_key="other-key")++ list(backfill_task_sources(team_id=self.team.id))++ self.assertEqual(self._columns(task), (None, None, "key"))+ self.assertEqual(self._columns(other), (None, None, None))++ def test_resumes_after_the_cursor_of_an_earlier_run(self) -> None:+ tasks = [self._task(origin_key=f"key-{uuid4()}") for _ in range(3)]+ ordered = sorted(tasks, key=lambda task: task.id)++ first_batch = next(backfill_task_sources(batch_size=1))++ self.assertEqual(first_batch.cursor, str(ordered[0].id))+ self.assertIsNotNone(self._columns(ordered[0])[2])+ self.assertIsNone(self._columns(ordered[1])[2])++ resumed = list(backfill_task_sources(batch_size=1, after=first_batch.cursor))++ self.assertEqual([batch.cursor for batch in resumed], [str(task.id) for task in ordered[1:]])+ self.assertTrue(all(self._columns(task)[2] is not None for task in tasks))++ def test_command_prints_the_cursor_to_resume_from(self) -> None:+ task = self._task(hog_flow_id=FLOW_ID)+ out = StringIO()++ call_command("backfill_task_sources", "--sleep-seconds", "0", "--batch-size", "10", stdout=out)++ self.assertIn(f"resume after {task.id}", out.getvalue())+ self.assertEqual(self._columns(task), ("hog_flow", str(FLOW_ID), None))diff --git a/products/tasks/backend/migrations/0147_task_source_columns.py b/products/tasks/backend/migrations/0147_task_source_columns.pynew file mode 100644index 000000000000..b3aacd50c4f1--- /dev/null+++ b/products/tasks/backend/migrations/0147_task_source_columns.py@@ -0,0 +1,25 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("tasks", "0146_sandboxsession_settle_indexes"),+ ]++ operations = [+ migrations.AddField(+ model_name="task",+ name="source_kind",+ field=models.CharField(blank=True, max_length=32, null=True),+ ),+ migrations.AddField(+ model_name="task",+ name="source_id",+ field=models.CharField(blank=True, max_length=64, null=True),+ ),+ migrations.AddField(+ model_name="task",+ name="idempotency_key",+ field=models.CharField(blank=True, max_length=128, null=True),+ ),+ ]diff --git a/products/tasks/backend/migrations/0148_task_source_indexes.py b/products/tasks/backend/migrations/0148_task_source_indexes.pynew file mode 100644index 000000000000..e002fc05d7bf--- /dev/null+++ b/products/tasks/backend/migrations/0148_task_source_indexes.py@@ -0,0 +1,43 @@+from django.db import migrations, models++from posthog.migration_helpers import CreateIndexConcurrently, SafeAddIndexConcurrently+++class Migration(migrations.Migration):+ atomic = False++ dependencies = [+ ("tasks", "0147_task_source_columns"),+ ]++ operations = [+ SafeAddIndexConcurrently(+ model_name="task",+ index=models.Index(+ fields=["source_kind", "source_id", "-created_at"],+ condition=models.Q(("source_kind__isnull", False)),+ name="posthog_task_source_idx",+ ),+ ),+ migrations.SeparateDatabaseAndState(+ state_operations=[+ migrations.AddConstraint(+ model_name="task",+ constraint=models.UniqueConstraint(+ condition=models.Q(("idempotency_key__isnull", False)),+ fields=("team", "idempotency_key"),+ name="posthog_task_idempotency_key_uniq",+ ),+ ),+ ],+ database_operations=[+ CreateIndexConcurrently(+ index_name="posthog_task_idempotency_key_uniq",+ table_name="posthog_task",+ columns="(team_id, idempotency_key)",+ unique=True,+ where="WHERE idempotency_key IS NOT NULL",+ ),+ ],+ ),+ ]diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txtindex eada7f0b5316..3113eb44471c 100644--- a/products/tasks/backend/migrations/max_migration.txt+++ b/products/tasks/backend/migrations/max_migration.txt@@ -1 +1 @@-0146_sandboxsession_settle_indexes+0148_task_source_indexesdiff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.pyindex 7cde9290cb74..70fb5a745518 100644--- a/products/tasks/backend/models.py+++ b/products/tasks/backend/models.py@@ -57,6 +57,7 @@ from products.tasks.backend.facade.profiles import (GithubAccess,GithubResolution,PrAuthorship,+ SourceKind,TaskProfile,TaskProfileError,TaskRunOptions,@@ -82,7 +83,7 @@ from products.tasks.backend.pr_urls import read_pr_urlsfrom products.tasks.backend.redis import dedicated_stream_configured, run_uses_dedicated_streamfrom products.tasks.backend.run_options import RunSettings, apply_run_options, run_options_record, validate_run_optionsfrom products.tasks.backend.storage import append_jsonl_object-from products.tasks.backend.task_sources import apply_task_source, source_from_columns+from products.tasks.backend.task_sources import apply_task_source, idempotency_key_startswith_qlogger = structlog.get_logger(__name__)@@ -579,6 +580,10 @@ class Task(Taggable, DeletedMetaFields, models.Model):# workflow engine redelivery) returns the existing task instead of making a second one.origin_key = models.CharField(max_length=128, null=True, blank=True)+ source_kind = models.CharField(max_length=32, null=True, blank=True)+ source_id = models.CharField(max_length=64, null=True, blank=True)+ idempotency_key = models.CharField(max_length=128, null=True, blank=True)+# DEPRECATED - do not usesignal_report = models.ForeignKey("signals.SignalReport",@@ -664,6 +669,11 @@ class Task(Taggable, DeletedMetaFields, models.Model):models.Index(fields=["team", "-last_activity_at", "-id"], name="posthog_task_team_activity_idx"),models.Index(fields=["channel", "-last_activity_at"], name="posthog_task_chan_activity_idx"),models.Index(fields=["hog_flow_id", "-created_at"], name="posthog_task_hog_flow_idx"),+ models.Index(+ fields=["source_kind", "source_id", "-created_at"],+ condition=models.Q(source_kind__isnull=False),+ name="posthog_task_source_idx",+ ),]constraints = [models.UniqueConstraint(@@ -671,6 +681,11 @@ class Task(Taggable, DeletedMetaFields, models.Model):condition=models.Q(origin_key__isnull=False),name="posthog_task_origin_key_uniq",),+ models.UniqueConstraint(+ fields=["team", "idempotency_key"],+ condition=models.Q(idempotency_key__isnull=False),+ name="posthog_task_idempotency_key_uniq",+ ),]def __str__(self):@@ -709,6 +724,32 @@ class Task(Taggable, DeletedMetaFields, models.Model):logger.warning("task_profile_not_registered", task_id=str(self.id), profile=self.profile)return None+ @property+ def stored_source(self) -> TaskSource | None:+ if self.source_kind is not None and self.source_id is not None:+ if self.source_kind == SourceKind.SIGNAL_REPORT and self.signal_report_id is None:+ return None+ return TaskSource(kind=SourceKind(self.source_kind), id=self.source_id)+ if self.signal_report_id is not None:+ return TaskSource(kind=SourceKind.SIGNAL_REPORT, id=str(self.signal_report_id))+ if self.hog_flow_id is not None:+ return TaskSource(kind=SourceKind.HOG_FLOW, id=str(self.hog_flow_id))+ return None++ @property+ def signal_report_source_id(self) -> uuid.UUID | None:+ source = self.stored_source+ return uuid.UUID(source.id) if source is not None and source.kind == SourceKind.SIGNAL_REPORT else None++ @property+ def hog_flow_source_id(self) -> uuid.UUID | None:+ source = self.stored_source+ return uuid.UUID(source.id) if source is not None and source.kind == SourceKind.HOG_FLOW else None++ @property+ def stored_idempotency_key(self) -> str | None:+ return self.idempotency_key if self.idempotency_key is not None else self.origin_key+def _resolves_pr_authorship_per_run(self) -> bool:profile = self.stored_profileif profile is None:@@ -780,15 +821,15 @@ class Task(Taggable, DeletedMetaFields, models.Model):def scout_experiment_q(cls, *, relation: Literal["", "task"] = "") -> models.Q:prefix = {"": "", "task": "task__"}[relation]return models.Q(**{f"{prefix}origin_product": cls.OriginProduct.SIGNALS_SCOUT}) & (- models.Q(**{f"{prefix}origin_key__startswith": SCOUT_TRIAL_ORIGIN_KEY_PREFIX})- | models.Q(**{f"{prefix}origin_key__startswith": SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX})+ idempotency_key_startswith_q(SCOUT_TRIAL_ORIGIN_KEY_PREFIX, via=prefix)+ | idempotency_key_startswith_q(SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX, via=prefix))@propertydef is_scout_experiment(self) -> bool:- return self.origin_product == self.OriginProduct.SIGNALS_SCOUT and (self.origin_key or "").startswith(- (SCOUT_TRIAL_ORIGIN_KEY_PREFIX, SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX)- )+ return self.origin_product == self.OriginProduct.SIGNALS_SCOUT and (+ self.stored_idempotency_key or ""+ ).startswith((SCOUT_TRIAL_ORIGIN_KEY_PREFIX, SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX))def capture_event(self, event: str, properties: dict | None = None, capture_fn: Callable[..., None] | None = None@@ -808,12 +849,12 @@ class Task(Taggable, DeletedMetaFields, models.Model):"repository": self.repository,"repositories": self.repositories or ([self.repository] if self.repository else []),}- if self.origin_key:- all_properties["origin_key"] = self.origin_key+ if self.stored_idempotency_key:+ all_properties["origin_key"] = self.stored_idempotency_keyif self.channel_id:all_properties["channel_id"] = str(self.channel_id)- if self.signal_report_id:- all_properties["signal_report_id"] = str(self.signal_report_id)+ if self.signal_report_source_id:+ all_properties["signal_report_id"] = str(self.signal_report_source_id)if properties:all_properties.update(properties)(capture_fn or posthoganalytics.capture)(@@ -1047,7 +1088,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):# and would otherwise land on the wider interactive cap.interactive_stage = INTERACTIVE_SIGNALS_AI_STAGE_BY_ORIGIN.get(task.origin_product)if interactive_stage and not state.get("ai_stage") and not task.internal:- if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or task.signal_report_id:+ if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or task.signal_report_source_id:state["ai_stage"] = interactive_stageinternal_stage = INTERNAL_SIGNALS_AI_STAGE_BY_ORIGIN.get(task.origin_product)if internal_stage and not state.get("ai_stage") and task.internal:@@ -1114,7 +1155,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):if effective_environment == TaskRun.Environment.CLOUD and scheduled_at is None:task.observe_run_started_beside_live_runs(task_run)- if task.origin_product == Task.OriginProduct.SIGNAL_REPORT and task.signal_report_id:+ if task.origin_product == Task.OriginProduct.SIGNAL_REPORT and task.signal_report_source_id:from products.signals.backend.facade.spend import ( # noqa: PLC0415 — avoids the task-model import cycleregister_report_task_spend,)@@ -1122,7 +1163,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):register_report_task_spend(team_id=task.team_id,run_id=str(task_run.id),- report_id=str(task.signal_report_id),+ report_id=str(task.signal_report_source_id),stage=state.get("ai_stage"),signal_id=spend_signal_id,implementation_kind=implementation_spend_kind,@@ -1355,6 +1396,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):signal_report_id: str | None = None,hog_flow_id: uuid.UUID | None = None,origin_key: str | None = None,+ source: TaskSource | None = None,ai_stage: str | None = None,ai_agent_name: str | None = None,sandbox_environment_id: str | None = None,@@ -1525,6 +1567,9 @@ class Task(Taggable, DeletedMetaFields, models.Model):state=initial_state,hog_flow_id=hog_flow_id,origin_key=origin_key,+ source_kind=source.kind.value if source is not None else None,+ source_id=source.id if source is not None else None,+ idempotency_key=origin_key,**({"signal_report_id": signal_report_id} if signal_report_id else {}),)@@ -1704,10 +1749,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):origin = resolve_task_origin(profile=profile, origin_product=origin_product, internal=internal)columns = apply_task_source(source, signal_report_id=signal_report_id, hog_flow_id=None)if profile is not None:- check_task_source(- profile=profile,- source=source_from_columns(signal_report_id=columns.signal_report_id, hog_flow_id=columns.hog_flow_id),- )+ check_task_source(profile=profile, source=columns.source)runtime = profile.runtimes[0] if profile is not None else Task.Runtime.ACP.valuevalidate_run_options(profile, options)if options is not None:@@ -1741,6 +1783,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):branch=branch,signal_report_id=columns.signal_report_id,hog_flow_id=columns.hog_flow_id,+ source=columns.source,sandbox_environment_id=sandbox_environment_id,internal=origin.internal,output_schema=output_schema,@@ -1844,9 +1887,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):signal_report_id = columns.signal_report_idhog_flow_id = columns.hog_flow_idif profile is not None:- check_task_source(- profile=profile, source=source_from_columns(signal_report_id=signal_report_id, hog_flow_id=hog_flow_id)- )+ check_task_source(profile=profile, source=columns.source)validate_run_options(profile, options)if options is not None:settings = apply_run_options(@@ -1903,6 +1944,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):signal_report_id=signal_report_id,hog_flow_id=hog_flow_id,origin_key=origin_key,+ source=columns.source,sandbox_environment_id=sandbox_environment_id,internal=internal,client_provenance=client_provenance,@@ -3308,7 +3350,7 @@ class TaskRun(models.Model):),"origin_product": self.task.origin_product,"title": self.task.title,- "signal_report_id": str(self.task.signal_report_id) if self.task.signal_report_id else None,+ "signal_report_id": str(self.task.signal_report_source_id) if self.task.signal_report_source_id else None,"environment": self.environment,# The bare `environment` property gets clobbered by the analytics# client's deployment-region super-property, so ship the run'sdiff --git a/products/tasks/backend/task_sources.py b/products/tasks/backend/task_sources.pyindex 79b538cc5954..ef88267de9d0 100644--- a/products/tasks/backend/task_sources.py+++ b/products/tasks/backend/task_sources.py@@ -1,5 +1,8 @@+from collections.abc import Iterablefrom uuid import UUID+from django.db.models import Q+from posthog.dataclasses import frozenfrom products.tasks.backend.facade.profiles import SourceKind, TaskSource, TaskSourceError@@ -9,18 +12,25 @@ from products.tasks.backend.facade.profiles import SourceKind, TaskSource, TaskSclass SourceColumns:signal_report_id: str | Nonehog_flow_id: UUID | None+ source: TaskSource | Nonedef apply_task_source(source: TaskSource | None, *, signal_report_id: str | None, hog_flow_id: UUID | None) -> SourceColumns:if source is None:- return SourceColumns(signal_report_id=signal_report_id, hog_flow_id=hog_flow_id)+ return SourceColumns(+ signal_report_id=signal_report_id,+ hog_flow_id=hog_flow_id,+ source=source_from_columns(signal_report_id=signal_report_id, hog_flow_id=hog_flow_id),+ )if signal_report_id is not None or hog_flow_id is not None:raise TaskSourceError("Pass the source as source or as signal_report_id and hog_flow_id, not both")if source.kind == SourceKind.SIGNAL_REPORT:- return SourceColumns(signal_report_id=source.id, hog_flow_id=None)- return SourceColumns(signal_report_id=None, hog_flow_id=UUID(source.id))+ return SourceColumns(signal_report_id=source.id, hog_flow_id=None, source=source)+ if source.kind == SourceKind.HOG_FLOW:+ return SourceColumns(signal_report_id=None, hog_flow_id=UUID(source.id), source=source)+ return SourceColumns(signal_report_id=None, hog_flow_id=None, source=source)def source_from_columns(*, signal_report_id: str | None, hog_flow_id: UUID | None) -> TaskSource | None:@@ -31,3 +41,45 @@ def source_from_columns(*, signal_report_id: str | None, hog_flow_id: UUID | Nonif hog_flow_id:return TaskSource(kind=SourceKind.HOG_FLOW, id=str(hog_flow_id))return None+++_RELATION_PREFIXES = frozenset({"", "task__"})+_IDEMPOTENCY_LOOKUPS = frozenset({"", "__startswith"})+++def _field_path(via: str, field: str) -> str:+ if via not in _RELATION_PREFIXES:+ raise ValueError(f"Unknown task relation prefix {via!r}")+ return via + field+++def source_q(kind: SourceKind, source_ids: Iterable[str | UUID], *, via: str = "") -> Q:+ ids = [str(source_id) for source_id in source_ids]+ on_new_columns = Q(**{_field_path(via, "source_kind"): kind.value, _field_path(via, "source_id__in"): ids})+ if kind == SourceKind.SIGNAL_REPORT:+ on_new_columns &= Q(**{_field_path(via, "signal_report_id__isnull"): False})+ legacy_field = "signal_report_id__in"+ elif kind == SourceKind.HOG_FLOW:+ legacy_field = "hog_flow_id__in"+ else:+ return on_new_columns+ new_columns_incomplete = Q(**{_field_path(via, "source_kind__isnull"): True}) | Q(+ **{_field_path(via, "source_id__isnull"): True}+ )+ return on_new_columns | (new_columns_incomplete & Q(**{_field_path(via, legacy_field): ids}))+++def idempotency_key_q(key: str, *, via: str = "") -> Q:+ return _idempotency_q("", key, via)+++def idempotency_key_startswith_q(key_prefix: str, *, via: str = "") -> Q:+ return _idempotency_q("__startswith", key_prefix, via)+++def _idempotency_q(lookup: str, value: str, via: str) -> Q:+ if lookup not in _IDEMPOTENCY_LOOKUPS:+ raise ValueError(f"Unknown idempotency key lookup {lookup!r}")+ on_new_column = Q(**{_field_path(via, "idempotency_key" + lookup): value})+ legacy = Q(**{_field_path(via, "idempotency_key__isnull"): True, _field_path(via, "origin_key" + lookup): value})+ return on_new_column | legacydiff --git a/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py b/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.pyindex e62090f6ccc3..1f4888d94b87 100644--- a/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py+++ b/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py@@ -91,7 +91,7 @@ def enforce_self_driving_run_quota(input: EnforceSelfDrivingRunQuotaInput) -> stif gate.enforced or not already_reported:capture_signal_report_quota_paused(team,- report_id=str(task.signal_report_id) if task.signal_report_id else None,+ report_id=str(task.signal_report_source_id) if task.signal_report_source_id else None,stage="implementation_run",enforced=gate.enforced,)diff --git a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.pyindex 5358648dc4d0..652c7a1fa9bd 100644--- a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py+++ b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py@@ -203,7 +203,7 @@ def complete_signal_spend(run_id: str) -> None:complete_task_spend(team_id=run.team_id,run_id=run_id,- report_id=str(run.task.signal_report_id) if run.task.signal_report_id else None,+ report_id=str(run.task.signal_report_source_id) if run.task.signal_report_source_id else None,)diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.pyindex 4c1836491c53..0fb82fd3a5d6 100644--- a/products/tasks/backend/tests/test_api.py+++ b/products/tasks/backend/tests/test_api.py@@ -3108,6 +3108,10 @@ class TestTaskAPI(BaseTaskAPITest):self.assertEqual(response.status_code, status.HTTP_201_CREATED)data = response.json()self.assertEqual(data["signal_report"], str(report.id))+ task = Task.objects.get(id=data["id"])+ self.assertEqual(+ (task.signal_report_id, task.source_kind, task.source_id), (report.id, "signal_report", str(report.id))+ )# A manual "start implementation" records the implementation task_run artefact (the# task↔report association and work-log entry) and writes the legacy SignalReportTask gate# row that blocks the auto-start pipeline from double-starting.@@ -17362,22 +17366,30 @@ class TestCloudUsageGate(BaseTaskAPITest):assert mock_feature_enabled.call_args.args[0] == GATED_MODEL_FLAGassert not Task.objects.filter(title="Gated").exists()- def _inbox_task(self, origin: Task.OriginProduct) -> Task:+ def _inbox_task(self, origin: Task.OriginProduct, *, source_columns: bool = False) -> Task:from products.signals.backend.models import SignalReporttask = self.create_task()task.origin_product = originif origin == Task.OriginProduct.SIGNAL_REPORT:- task.signal_report = SignalReport.objects.create(team=self.team)+ self._link_report(task, SignalReport.objects.create(team=self.team), source_columns=source_columns)task.save()return task+ def _link_report(self, task: Task, report, *, source_columns: bool) -> None:+ if source_columns:+ task.source_kind = "signal_report"+ task.source_id = str(report.id)+ else:+ task.signal_report = report+def _create_pr_task(self,*,relationship: str = "implementation",link_other_report: bool = False,report_in_other_team: bool = False,+ source_columns: bool = False,) -> Task:# `record_report_task` writes this row on the manual and the auto-start path.from products.signals.backend.models import SignalReport, SignalReportTask@@ -17390,7 +17402,7 @@ class TestCloudUsageGate(BaseTaskAPITest):report = SignalReport.objects.create(team=report_team)task = self.create_task()task.origin_product = Task.OriginProduct.SIGNAL_REPORT- task.signal_report = report+ self._link_report(task, report, source_columns=source_columns)task.repository = "posthog/posthog"task.save()SignalReportTask.objects.create(@@ -17612,19 +17624,39 @@ class TestCloudUsageGate(BaseTaskAPITest):self.assertEqual(response.status_code, expected_status)self.assertEqual(mock_warm.called, inbox)- def test_exemption_ignores_cross_team_report_link(self):+ @parameterized.expand([("old_columns", False), ("new_columns", True)])+ def test_exemption_ignores_cross_team_report_link(self, _name, source_columns):# The write serializer blocks cross-team report links, so forge one at the ORM level:- # the exemption query must re-check the report's team rather than trust the FK alone.+ # the exemption query must re-check the report's team rather than trust the link alone.from products.signals.backend.models import SignalReportother_team = Team.objects.create(organization=self.organization, name="Other Team")task = self.create_task()task.origin_product = Task.OriginProduct.SIGNAL_REPORT- task.signal_report = SignalReport.objects.create(team=other_team)+ self._link_report(task, SignalReport.objects.create(team=other_team), source_columns=source_columns)task.save()self.assertFalse(tasks_facade.task_exempt_from_code_access(task.id, self.team.id))+ @parameterized.expand(+ [+ ("report_kind_without_id", {"source_kind": "signal_report"}, True),+ ("flow_kind_without_id", {"source_kind": "hog_flow"}, True),+ (+ "complete_flow_source",+ {"source_kind": "hog_flow", "source_id": "33333333-3333-4333-8333-333333333333"},+ False,+ ),+ ]+ )+ def test_exemption_reads_the_old_report_when_the_new_columns_are_incomplete(self, _name, new_columns, exempt):+ task = self._inbox_task(Task.OriginProduct.SIGNAL_REPORT)+ for field, value in new_columns.items():+ setattr(task, field, value)+ task.save()++ self.assertEqual(tasks_facade.task_exempt_from_code_access(task.id, self.team.id), exempt)+@parameterized.expand([("legacy_repository", "repository", "posthog/posthog"),@@ -17640,8 +17672,9 @@ class TestCloudUsageGate(BaseTaskAPITest):self.assertFalse(tasks_facade.task_exempt_from_code_access(task.id, self.team.id))- def test_exemption_drops_for_signal_report_task_with_repository(self):- task = self._inbox_task(Task.OriginProduct.SIGNAL_REPORT)+ @parameterized.expand([("old_columns", False), ("new_columns", True)])+ def test_exemption_drops_for_signal_report_task_with_repository(self, _name, source_columns):+ task = self._inbox_task(Task.OriginProduct.SIGNAL_REPORT, source_columns=source_columns)self.assertTrue(tasks_facade.task_exempt_from_code_access(task.id, self.team.id))task.repository = "posthog/posthog"@@ -17651,19 +17684,24 @@ class TestCloudUsageGate(BaseTaskAPITest):@parameterized.expand([- ("implementation_link", "implementation", False, False, True),- ("discussion_label", "discussion", False, False, False),- ("link_to_another_report", "implementation", True, False, False),- ("report_in_another_team", "implementation", False, True, False),+ (f"{name}_{style}", style == "new_columns", relationship, link_other_report, report_in_other_team, expected)+ for style in ("old_columns", "new_columns")+ for name, relationship, link_other_report, report_in_other_team, expected in [+ ("implementation_link", "implementation", False, False, True),+ ("discussion_label", "discussion", False, False, False),+ ("link_to_another_report", "implementation", True, False, False),+ ("report_in_another_team", "implementation", False, True, False),+ ]])def test_exemption_for_create_pr_task_needs_a_team_scoped_implementation_link(- self, _name, relationship, link_other_report, report_in_other_team, expected+ self, _name, source_columns, relationship, link_other_report, report_in_other_team, expected):task = self._create_pr_task(relationship=relationship,link_other_report=link_other_report,report_in_other_team=report_in_other_team,+ source_columns=source_columns,)self.assertEqual(tasks_facade.task_exempt_from_code_access(task.id, self.team.id), expected)diff --git a/products/tasks/backend/tests/test_facade.py b/products/tasks/backend/tests/test_facade.pyindex a7c8283d7ce1..e533466feb3c 100644--- a/products/tasks/backend/tests/test_facade.py+++ b/products/tasks/backend/tests/test_facade.py@@ -483,11 +483,35 @@ class TestFacadeReadsAndMappers(TestCase):assert run.status == TaskRun.Status.COMPLETEDassert run.state == {}- def test_list_workflow_last_runs_uses_each_workflows_newest_listed_task(self):+ @parameterized.expand(+ [+ ("old_columns", "old", "old"),+ ("new_columns", "new", "new"),+ ("old_column_then_new_columns", "old", "new"),+ ("kind_without_id", "kind_only", "kind_only"),+ ]+ )+ def test_list_workflow_last_runs_uses_each_workflows_newest_listed_task(+ self, _name: str, older_columns: str, newer_columns: str+ ):+ def flow_columns(flow_id: UUID, columns: str) -> dict:+ columns_by_name: dict[str, dict] = {+ "old": {"hog_flow_id": flow_id},+ "new": {"source_kind": "hog_flow", "source_id": str(flow_id)},+ "kind_only": {"source_kind": "hog_flow", "hog_flow_id": flow_id},+ }+ return columns_by_name[columns]+now = django_timezone.now()flow_with_runs, flow_without_run, flow_not_asked = uuid4(), uuid4(), uuid4()- older = self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_with_runs)- newer = self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_with_runs, archived=True)+ older = self._make_task(+ origin_product=Task.OriginProduct.WORKFLOW, **flow_columns(flow_with_runs, older_columns)+ )+ newer = self._make_task(+ origin_product=Task.OriginProduct.WORKFLOW,+ archived=True,+ **flow_columns(flow_with_runs, newer_columns),+ )Task.objects.filter(pk=older.pk).update(created_at=now - timedelta(hours=2))Task.objects.filter(pk=newer.pk).update(created_at=now - timedelta(hours=1))TaskRun.objects.create(task=older, team=self.team, status=TaskRun.Status.COMPLETED)@@ -496,9 +520,15 @@ class TestFacadeReadsAndMappers(TestCase):TaskRun.objects.filter(pk=earlier_run.pk).update(created_at=now - timedelta(minutes=50))TaskRun.objects.filter(pk=newest_run.pk).update(created_at=now - timedelta(minutes=30))# Run history hides internal tasks, so the newest task here must not count as the last run.- self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_with_runs, internal=True)- not_started = self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_without_run)- self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_not_asked)+ self._make_task(+ origin_product=Task.OriginProduct.WORKFLOW,+ internal=True,+ **flow_columns(flow_with_runs, newer_columns),+ )+ not_started = self._make_task(+ origin_product=Task.OriginProduct.WORKFLOW, **flow_columns(flow_without_run, newer_columns)+ )+ self._make_task(origin_product=Task.OriginProduct.WORKFLOW, **flow_columns(flow_not_asked, newer_columns))last_runs = facade.list_workflow_last_runs(self.team.id, self.user.id, [flow_with_runs, flow_without_run])@@ -517,12 +547,22 @@ class TestFacadeReadsAndMappers(TestCase):),}- @parameterized.expand([("ordinary", False), ("scout_trial", True), ("scout_judge", True, "scout-trial-judge:")])- def test_task_exists_and_visibility(self, _name: str, is_trial: bool, prefix: str = "scout-trial:") -> None:+ @parameterized.expand(+ [+ ("ordinary", False),+ ("scout_trial", True),+ ("scout_judge", True, "scout-trial-judge:"),+ ("scout_trial_new_column", True, "scout-trial:", "idempotency_key"),+ ("scout_judge_new_column", True, "scout-trial-judge:", "idempotency_key"),+ ]+ )+ def test_task_exists_and_visibility(+ self, _name: str, is_trial: bool, prefix: str = "scout-trial:", key_column: str = "origin_key"+ ) -> None:task = self._make_task(**{"origin_product": Task.OriginProduct.SIGNALS_SCOUT,- "origin_key": f"{prefix}{uuid4()}:{uuid4()}",+ key_column: f"{prefix}{uuid4()}:{uuid4()}",}if is_trialelse {}@@ -552,13 +592,21 @@ class TestFacadeReadsAndMappers(TestCase):("bound", True, True, True, True),("sibling", True, True, False, False),("legacy", True, True, None, False),+ ("owner_new_column", True, False, False, True, "idempotency_key"),+ ("sibling_new_column", True, True, False, False, "idempotency_key"),])def test_run_authorization_classifies_trial_in_one_lookup(- self, _name: str, is_trial: bool, sandbox: bool, bound: bool | None, allowed: bool+ self,+ _name: str,+ is_trial: bool,+ sandbox: bool,+ bound: bool | None,+ allowed: bool,+ key_column: str = "origin_key",) -> None:task = self._make_task(- **{"origin_product": Task.OriginProduct.SIGNALS_SCOUT, "origin_key": f"scout-trial:{uuid4()}"}+ **{"origin_product": Task.OriginProduct.SIGNALS_SCOUT, key_column: f"scout-trial:{uuid4()}"}if is_trialelse {})@@ -2097,6 +2145,7 @@ class TestFacadeReadsAndMappers(TestCase):task = Task.objects.get(id=created if isinstance(created, UUID) else created.task_id)self.assertEqual(task.hog_flow_id, flow_id)+ self.assertEqual((task.source_kind, task.source_id), ("hog_flow", str(flow_id)))@parameterized.expand([diff --git a/products/tasks/backend/tests/test_models.py b/products/tasks/backend/tests/test_models.pyindex 042ee1194a83..49cec0a06ce4 100644--- a/products/tasks/backend/tests/test_models.py+++ b/products/tasks/backend/tests/test_models.py@@ -445,7 +445,7 @@ class TestTask(TestCase):self.assertFalse(Task.objects.filter(team=self.team).exists())- @parameterized.expand([(SourceKind.SIGNAL_REPORT,), (SourceKind.HOG_FLOW,)])+ @parameterized.expand([(SourceKind.SIGNAL_REPORT,), (SourceKind.HOG_FLOW,), (SourceKind.LOOP,)])@patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")def test_task_creation_stores_its_source(self, kind: SourceKind, _mock_execute_workflow) -> None:from products.signals.backend.models import SignalReport@@ -463,8 +463,62 @@ class TestTask(TestCase):)task.refresh_from_db()- expected = (source_id, None) if kind == SourceKind.SIGNAL_REPORT else (None, source_id)- self.assertEqual((task.signal_report_id, task.hog_flow_id), expected)+ expected_old_columns = {+ SourceKind.SIGNAL_REPORT: (source_id, None),+ SourceKind.HOG_FLOW: (None, source_id),+ SourceKind.LOOP: (None, None),+ }[kind]+ self.assertEqual((task.signal_report_id, task.hog_flow_id), expected_old_columns)+ self.assertEqual((task.source_kind, task.source_id), (kind.value, str(source_id)))++ @parameterized.expand(+ [+ ("signal_report_id", {"signal_report_id": "report"}, SourceKind.SIGNAL_REPORT),+ ("hog_flow_id", {"hog_flow_id": SOURCE_ID}, SourceKind.HOG_FLOW),+ ("no_source", {}, None),+ ]+ )+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")+ def test_task_creation_stores_the_source_given_as_old_columns(+ self, name: str, columns: dict, kind: SourceKind | None, _mock_execute_workflow+ ) -> None:+ from products.signals.backend.models import SignalReport++ user = User.objects.create(email=f"{name}@test.com")+ report_id = SignalReport.objects.create(team=self.team).id+ columns = {key: str(report_id) if value == "report" else value for key, value in columns.items()}++ task = Task.create_and_run(+ team=self.team,+ title="t",+ description="d",+ user_id=user.id,+ origin_product=Task.OriginProduct.USER_CREATED,+ **columns,+ )++ task.refresh_from_db()+ expected_id = {None: None, SourceKind.SIGNAL_REPORT: str(report_id), SourceKind.HOG_FLOW: str(SOURCE_ID)}[kind]+ self.assertEqual((task.source_kind, task.source_id), (kind.value if kind else None, expected_id))++ @parameterized.expand([("key", "scout-trial:1"), ("no_key", None)])+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")+ def test_task_creation_copies_the_origin_key_to_the_idempotency_key(+ self, name: str, origin_key: str | None, _mock_execute_workflow+ ) -> None:+ user = User.objects.create(email=f"{name}@test.com")++ task = Task.create_and_run(+ team=self.team,+ title="t",+ description="d",+ user_id=user.id,+ origin_product=Task.OriginProduct.USER_CREATED,+ origin_key=origin_key,+ )++ task.refresh_from_db()+ self.assertEqual((task.origin_key, task.idempotency_key), (origin_key, origin_key))@parameterized.expand([@@ -472,6 +526,14 @@ class TestTask(TestCase):"source_and_column",{"origin_product": Task.OriginProduct.USER_CREATED, "hog_flow_id": SOURCE_ID, "source": FLOW_SOURCE},),+ (+ "both_old_columns",+ {+ "origin_product": Task.OriginProduct.USER_CREATED,+ "signal_report_id": str(SOURCE_ID),+ "hog_flow_id": SOURCE_ID,+ },+ ),("profile_takes_no_source", {"profile": BUSINESS_KNOWLEDGE, "source": FLOW_SOURCE}),("profile_takes_no_legacy_source", {"profile": BUSINESS_KNOWLEDGE, "hog_flow_id": SOURCE_ID}),(diff --git a/products/tasks/backend/tests/test_task_sources.py b/products/tasks/backend/tests/test_task_sources.pynew file mode 100644index 000000000000..e818f9658466--- /dev/null+++ b/products/tasks/backend/tests/test_task_sources.py@@ -0,0 +1,181 @@+from collections.abc import Callable+from typing import Any+from uuid import UUID++from django.db.models import Q+from django.test import SimpleTestCase, TestCase++from parameterized import parameterized++from posthog.models import Organization, Team++from products.signals.backend.models import SignalReport+from products.tasks.backend.facade.profiles import SourceKind+from products.tasks.backend.models import Task, TaskRun+from products.tasks.backend.task_sources import idempotency_key_q, idempotency_key_startswith_q, source_q++REPORT_ID = UUID("11111111-1111-4111-8111-111111111111")+FLOW_ID = UUID("22222222-2222-4222-8222-222222222222")+++class TestStoredSource(SimpleTestCase):+ @parameterized.expand(+ [+ ("neither", {}, None, None),+ ("report_in_old_column", {"signal_report_id": REPORT_ID}, REPORT_ID, None),+ ("flow_in_old_column", {"hog_flow_id": FLOW_ID}, None, FLOW_ID),+ (+ "report_in_both_columns",+ {"source_kind": "signal_report", "source_id": str(REPORT_ID), "signal_report_id": REPORT_ID},+ REPORT_ID,+ None,+ ),+ ("report_deleted", {"source_kind": "signal_report", "source_id": str(REPORT_ID)}, None, None),+ ("flow_in_new_columns", {"source_kind": "hog_flow", "source_id": str(FLOW_ID)}, None, FLOW_ID),+ (+ "new_columns_win_over_old",+ {"source_kind": "hog_flow", "source_id": str(FLOW_ID), "signal_report_id": REPORT_ID},+ None,+ FLOW_ID,+ ),+ (+ "loop_hides_the_old_report",+ {"source_kind": "loop", "source_id": "loop-1", "signal_report_id": REPORT_ID},+ None,+ None,+ ),+ ("kind_without_id_falls_back", {"source_kind": "hog_flow", "hog_flow_id": FLOW_ID}, None, FLOW_ID),+ ]+ )+ def test_reads_the_new_columns_and_falls_back_to_the_old_ones(+ self, _name: str, columns: dict, report_id: UUID | None, flow_id: UUID | None+ ) -> None:+ task = Task(**columns)++ self.assertEqual((task.signal_report_source_id, task.hog_flow_source_id), (report_id, flow_id))++ @parameterized.expand(+ [+ ("neither", {}, None),+ ("old_column_only", {"origin_key": "old"}, "old"),+ ("new_column_only", {"idempotency_key": "new"}, "new"),+ ("new_column_wins", {"origin_key": "old", "idempotency_key": "new"}, "new"),+ ]+ )+ def test_reads_the_idempotency_key_and_falls_back_to_the_origin_key(+ self, _name: str, columns: dict, expected: str | None+ ) -> None:+ self.assertEqual(Task(**columns).stored_idempotency_key, expected)+++class TestSourceQueryPaths(SimpleTestCase):+ @parameterized.expand(+ [+ ("source_q", lambda: source_q(SourceKind.SIGNAL_REPORT, [REPORT_ID], via="team__")),+ ("idempotency_key_q", lambda: idempotency_key_q("key", via="team__")),+ ("idempotency_key_startswith_q", lambda: idempotency_key_startswith_q("key", via="created_by__")),+ ]+ )+ def test_refuses_an_unknown_relation_prefix(self, _name: str, build_q: Callable[[], Q]) -> None:+ with self.assertRaises(ValueError):+ build_q()+++class TestSourceQueries(TestCase):+ def setUp(self) -> None:+ super().setUp()+ self.team = Team.objects.create(organization=Organization.objects.create(name="Org"), name="Team")+ SignalReport.objects.create(team=self.team, id=REPORT_ID)+ rows: dict[str, dict[str, Any]] = {+ "neither": {},+ "report_old": {"signal_report_id": REPORT_ID, "origin_key": "scout-trial:report_old"},+ "report_deleted": {+ "source_kind": "signal_report",+ "source_id": str(REPORT_ID),+ "idempotency_key": "scout-trial:report_deleted",+ },+ "report_both": {+ "signal_report_id": REPORT_ID,+ "source_kind": "signal_report",+ "source_id": str(REPORT_ID),+ "origin_key": "scout-trial:report_both",+ "idempotency_key": "scout-trial:report_both",+ },+ "flow_old": {"hog_flow_id": FLOW_ID},+ "flow_new": {"source_kind": "hog_flow", "source_id": str(FLOW_ID)},+ "flow_over_old_report": {+ "signal_report_id": REPORT_ID,+ "source_kind": "hog_flow",+ "source_id": str(FLOW_ID),+ "origin_key": "scout-trial:stale",+ "idempotency_key": "other:fresh",+ },+ "flow_kind_without_id": {"source_kind": "hog_flow", "hog_flow_id": FLOW_ID},+ "flow_kind_without_id_over_old_report": {"source_kind": "hog_flow", "signal_report_id": REPORT_ID},+ }+ self.tasks = {+ name: Task.objects.create(+ team=self.team,+ title=name,+ description="d",+ origin_product=Task.OriginProduct.USER_CREATED,+ **columns,+ )+ for name, columns in rows.items()+ }++ def _names(self, queryset_filter) -> set[str]:+ return {task.title for task in Task.objects.filter(queryset_filter)}++ @parameterized.expand(+ [+ (+ "report",+ SourceKind.SIGNAL_REPORT,+ REPORT_ID,+ {"report_old", "report_both", "flow_kind_without_id_over_old_report"},+ ),+ (+ "flow",+ SourceKind.HOG_FLOW,+ FLOW_ID,+ {"flow_old", "flow_new", "flow_over_old_report", "flow_kind_without_id"},+ ),+ ("flow_id_under_the_report_kind", SourceKind.SIGNAL_REPORT, FLOW_ID, set()),+ ("loop_has_no_old_column", SourceKind.LOOP, REPORT_ID, set()),+ ]+ )+ def test_source_q_matches_new_and_old_columns_by_kind(+ self, _name: str, kind: SourceKind, source_id: UUID, expected: set[str]+ ) -> None:+ self.assertEqual(self._names(source_q(kind, [source_id])), expected)++ def test_source_q_follows_a_relation(self) -> None:+ for task in self.tasks.values():+ TaskRun.objects.create(task=task, team=self.team)++ runs = TaskRun.objects.filter(source_q(SourceKind.SIGNAL_REPORT, [REPORT_ID], via="task__"))++ self.assertEqual(+ {run.task.title for run in runs},+ {"report_old", "report_both", "flow_kind_without_id_over_old_report"},+ )++ @parameterized.expand(+ [+ ("old_column", "scout-trial:report_old", {"report_old"}),+ ("new_column", "scout-trial:report_deleted", {"report_deleted"}),+ ("both_columns", "scout-trial:report_both", {"report_both"}),+ ("old_value_hidden_by_new", "scout-trial:stale", set()),+ ("new_value", "other:fresh", {"flow_over_old_report"}),+ ("unknown", "missing", set()),+ ]+ )+ def test_idempotency_key_q_matches_new_and_old_columns(self, _name: str, key: str, expected: set[str]) -> None:+ self.assertEqual(self._names(idempotency_key_q(key)), expected)++ def test_idempotency_key_startswith_q_matches_new_and_old_columns(self) -> None:+ self.assertEqual(+ self._names(idempotency_key_startswith_q("scout-trial:")),+ {"report_old", "report_deleted", "report_both"},+ )diff --git a/products/tasks/backend/tests/test_workflow_tasks_api.py b/products/tasks/backend/tests/test_workflow_tasks_api.pyindex 76de7a22698e..2aae812427f9 100644--- a/products/tasks/backend/tests/test_workflow_tasks_api.py+++ b/products/tasks/backend/tests/test_workflow_tasks_api.py@@ -78,13 +78,18 @@ class TestWorkflowTasksAPI(APIBaseTest):HTTP_AUTHORIZATION=f"Bearer {token or _token(self.team.id, str(self.hog_flow.id))}",)- def _seed_workflow_task(self, run_status: str) -> Task:+ def _flow_columns(self, hog_flow_id: UUID | str, *, source_columns: bool) -> dict:+ if source_columns:+ return {"source_kind": "hog_flow", "source_id": str(hog_flow_id)}+ return {"hog_flow_id": hog_flow_id}++ def _seed_workflow_task(self, run_status: str, *, source_columns: bool = False) -> Task:task = Task.objects.create(team=self.team,title="existing",description="existing",origin_product=Task.OriginProduct.WORKFLOW,- hog_flow_id=self.hog_flow.id,+ **self._flow_columns(self.hog_flow.id, source_columns=source_columns),)TaskRun.objects.create(task=task, team=self.team, status=run_status)return task@@ -96,6 +101,7 @@ class TestWorkflowTasksAPI(APIBaseTest):hog_flow_id: UUID | None,origin_product: str = Task.OriginProduct.WORKFLOW,created_at: datetime | None = None,+ source_columns: bool = False,) -> None:# No TaskRun rows: the daily caps count Task rows only, and runless tasks keep the# in-flight count at 0, so a cap test cannot pass via the in-flight 409 instead.@@ -106,7 +112,11 @@ class TestWorkflowTasksAPI(APIBaseTest):title=f"seed-{i}",description="seed",origin_product=origin_product,- hog_flow_id=hog_flow_id,+ **(+ self._flow_columns(hog_flow_id, source_columns=source_columns)+ if hog_flow_id is not None+ else {}+ ),**({"created_at": created_at} if created_at is not None else {}),)for i in range(count)@@ -368,14 +378,15 @@ class TestWorkflowTasksAPI(APIBaseTest):assert not Task.objects.filter(hog_flow_id=flow.id).exists()usage_limit_response_mock.assert_not_called()- def test_skips_creation_at_the_in_flight_limit(self) -> None:+ @parameterized.expand([("old_columns", False), ("new_columns", True)])+ def test_skips_creation_at_the_in_flight_limit(self, _name: str, source_columns: bool) -> None:for _ in range(2):- self._seed_workflow_task(TaskRun.Status.IN_PROGRESS)+ self._seed_workflow_task(TaskRun.Status.IN_PROGRESS, source_columns=source_columns)response = self._post({"max_parallel_tasks": 2})assert response.status_code == status.HTTP_409_CONFLICT, response.json()- assert Task.objects.filter(hog_flow_id=self.hog_flow.id).count() == 2+ assert Task.objects.count() == 2def test_finished_runs_free_up_the_limit(self) -> None:self._seed_workflow_task(TaskRun.Status.COMPLETED)@@ -384,11 +395,13 @@ class TestWorkflowTasksAPI(APIBaseTest):assert response.status_code == status.HTTP_201_CREATED, response.json()- @parameterized.expand(["per_workflow", "team_wide"])+ @parameterized.expand([("per_workflow", False), ("per_workflow_new_columns", True), ("team_wide", False)])@patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response")- def test_skips_creation_at_the_daily_cap(self, scope: str, usage_limit_response_mock) -> None:- if scope == "per_workflow":- self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id))+ def test_skips_creation_at_the_daily_cap(self, scope: str, source_columns: bool, usage_limit_response_mock) -> None:+ if scope.startswith("per_workflow"):+ self._seed_created_tasks(+ WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id), source_columns=source_columns+ )expected_fragment = "This workflow reached its daily limit"else:# Two other workflows fill the team budget; this workflow is far under its own cap.@@ -518,14 +531,22 @@ class TestWorkflowTasksAPI(APIBaseTest):assert replay.status_code == status.HTTP_200_OK, replay.json()assert replay.json()["id"] == first.json()["id"]- def test_rejects_an_idempotency_key_used_by_another_workflow(self) -> None:+ @parameterized.expand(+ [+ ("old_columns", {"origin_key": "invocation-1"}, False),+ ("new_columns", {"idempotency_key": "invocation-1"}, True),+ ]+ )+ def test_rejects_an_idempotency_key_used_by_another_workflow(+ self, _name: str, key_columns: dict, source_columns: bool+ ) -> None:Task.objects.create(team=self.team,title="other",description="other",origin_product=Task.OriginProduct.WORKFLOW,- hog_flow_id=uuid4(),- origin_key="invocation-1",+ **self._flow_columns(uuid4(), source_columns=source_columns),+ **key_columns,)response = self._post({"idempotency_key": "invocation-1"})