Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(tasks): add task source columns (#127897)

Charles Vien1f3726bbb67f
diff --git a/products/posthog_ai/backend/tests/test_conversation_mirror.py b/products/posthog_ai/backend/tests/test_conversation_mirror.py
index f92e565c1fda..d9ac330cdf69 100644
--- a/products/posthog_ai/backend/tests/test_conversation_mirror.py
+++ b/products/posthog_ai/backend/tests/test_conversation_mirror.py
@@ -254,6 +254,7 @@ class TestMirrorConversation(APIBaseTest):
assert result.skipped_reason is None
task = Task.objects.get(id=result.task_id)
assert task.origin_key == origin_key_for_conversation(self.conversation.id)
+ assert task.idempotency_key == task.origin_key
assert task.origin_product == Task.OriginProduct.POSTHOG_AI
assert task.title == "Pageviews chat"
assert task.title_manually_set is False
diff --git a/products/review_hog/backend/receivers.py b/products/review_hog/backend/receivers.py
index 0e8a1cdc0c9b..46df955c6d55 100644
--- a/products/review_hog/backend/receivers.py
+++ b/products/review_hog/backend/receivers.py
@@ -142,7 +142,8 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:
if pr_url is None:
return
task = instance.task # first DB hit
- if task.signal_report_id is None:
+ signal_report_id = task.signal_report_source_id
+ if signal_report_id is None:
return
if task.deleted:
# A soft-deleted task's run can still save output. Its PR is disowned work, and the
@@ -154,7 +155,7 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:
if (instance.state or {}).get("ai_stage") != "implementation":
return
repository = (task.repository or "").strip() or None
- resolved = resolve_assigned_reviewers(instance.team_id, task.signal_report_id)
+ resolved = resolve_assigned_reviewers(instance.team_id, signal_report_id)
if not resolved:
return
settings_by_user = ReviewUserSettings.load_preferences_many(instance.team_id, resolved)
@@ -166,14 +167,14 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:
# The priority as it stood when the implementation task was created. The agent can append
# judgments through the artefact API, so a later one may be its own vote on its review.
signal_priority = persisted_report_priority(
- team_id=instance.team_id, report_id=str(task.signal_report_id), before=task.created_at
+ team_id=instance.team_id, report_id=str(signal_report_id), before=task.created_at
)
transaction.on_commit(
lambda: _start_review(
pr_url=pr_url,
team_id=instance.team_id,
user_id=acting_user_id,
- signal_report_id=str(task.signal_report_id),
+ signal_report_id=str(signal_report_id),
signal_priority=signal_priority,
repository=repository,
),
@@ -188,7 +189,7 @@ def handle_task_run_saved(sender: type, instance: Any, created: bool, **kwargs:
repository=stamphog_repository,
team_id=instance.team_id,
acting_user_id=stamphog_user_id,
- signal_report_id=str(task.signal_report_id),
+ signal_report_id=str(signal_report_id),
task_run_id=str(instance.id),
),
robust=True,
diff --git a/products/signals/backend/inbox_push_notifications.py b/products/signals/backend/inbox_push_notifications.py
index 9857e35437e7..78466e1edf48 100644
--- a/products/signals/backend/inbox_push_notifications.py
+++ b/products/signals/backend/inbox_push_notifications.py
@@ -191,7 +191,10 @@ def notify_report_task_update(
task = task_run.task
reports = list(
SignalReport.objects.filter(team_id=task.team_id)
- .filter(Q(id=task.signal_report_id) | SignalReport.reports_for_task_ids_filter([task.id], team_id=task.team_id))
+ .filter(
+ Q(id=task.signal_report_source_id)
+ | SignalReport.reports_for_task_ids_filter([task.id], team_id=task.team_id)
+ )
.select_related("team")
.order_by("created_at", "id")
)
diff --git a/products/signals/backend/scout_harness/runner.py b/products/signals/backend/scout_harness/runner.py
index 7e38bb845231..e8f8a18f8f8b 100644
--- a/products/signals/backend/scout_harness/runner.py
+++ b/products/signals/backend/scout_harness/runner.py
@@ -181,12 +181,10 @@ def _get_trial_config(trial: TrialLaunch) -> SignalScoutConfig:
def _existing_trial_result(team_id: int, trial: TrialLaunch) -> RunResult | None:
- run = (
- SignalScoutRun.objects.for_team(team_id)
- .select_related("task_run")
- .filter(task_run__task__origin_key=f"scout-trial:{trial.id}")
- .first()
- )
+ task_id = tasks_facade.get_task_id_by_idempotency_key(team_id, f"scout-trial:{trial.id}")
+ if task_id is None:
+ return None
+ run = SignalScoutRun.objects.for_team(team_id).select_related("task_run").filter(task_run__task_id=task_id).first()
if run is None:
return None
return RunResult(
diff --git a/products/signals/backend/scout_harness/trial_evaluation.py b/products/signals/backend/scout_harness/trial_evaluation.py
index eed236a20e22..df68322a88c5 100644
--- a/products/signals/backend/scout_harness/trial_evaluation.py
+++ b/products/signals/backend/scout_harness/trial_evaluation.py
@@ -351,7 +351,7 @@ def _bound_run(launch: TrialLaunch) -> SignalScoutRun | None:
or task.created_by_id != launch.user_id
or task.deleted
or task.origin_product != "signals_scout"
- or task.origin_key != f"scout-trial:{launch.id}"
+ or task.stored_idempotency_key != f"scout-trial:{launch.id}"
or not isinstance(marker, dict)
or marker.get("version") != 1
or marker.get("context_id") != str(launch.context_id)
diff --git a/products/signals/backend/scout_harness/trial_inspection.py b/products/signals/backend/scout_harness/trial_inspection.py
index 9c84ef5e009b..c58e2b0c2f17 100644
--- a/products/signals/backend/scout_harness/trial_inspection.py
+++ b/products/signals/backend/scout_harness/trial_inspection.py
@@ -176,7 +176,7 @@ class ScoutTrialInspection:
if (
not isinstance(marker.get("launch_id"), str)
or run.task_run.task.origin_product != "signals_scout"
- or run.task_run.task.origin_key != f"scout-trial:{marker['launch_id']}"
+ or run.task_run.task.stored_idempotency_key != f"scout-trial:{marker['launch_id']}"
):
continue
try:
diff --git a/products/signals/backend/scout_harness/trial_judge.py b/products/signals/backend/scout_harness/trial_judge.py
index 3cb222142f36..4ad804fc1460 100644
--- a/products/signals/backend/scout_harness/trial_judge.py
+++ b/products/signals/backend/scout_harness/trial_judge.py
@@ -78,7 +78,6 @@ def _assert_scout_available(snapshot: TrialEvaluationSnapshot, evidence: TrialRu
task_run__task__created_by_id=snapshot.user_id,
task_run__task__deleted=False,
task_run__task__origin_product="signals_scout",
- task_run__task__origin_key=f"scout-trial:{evidence.launch_id}",
)
.first()
)
@@ -90,6 +89,7 @@ def _assert_scout_available(snapshot: TrialEvaluationSnapshot, evidence: TrialRu
or marker.get("launch_id") != str(evidence.launch_id)
or marker.get("context_id") != str(snapshot.context_id)
or run.task_run.status != "completed"
+ or run.task_run.task.stored_idempotency_key != f"scout-trial:{evidence.launch_id}"
):
raise TrialJudgeValidationError("The saved scout run is no longer available for this evaluation.")
if ScoutTrialStore(run).invalid_reason() is not None:
diff --git a/products/signals/backend/scout_harness/trial_launch.py b/products/signals/backend/scout_harness/trial_launch.py
index bcd1425f4847..71a2105190f6 100644
--- a/products/signals/backend/scout_harness/trial_launch.py
+++ b/products/signals/backend/scout_harness/trial_launch.py
@@ -484,6 +484,6 @@ def bound_trial_run(team_id: int, task_id: UUID | str | None) -> SignalScoutRun
if run is None or run.metadata is None:
return None
launch_id = run.metadata["scout_trial"].get("launch_id")
- if not isinstance(launch_id, str) or run.task_run.task.origin_key != f"scout-trial:{launch_id}":
+ if not isinstance(launch_id, str) or run.task_run.task.stored_idempotency_key != f"scout-trial:{launch_id}":
return None
return run
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index c1af51c12904..3b783ab66b80 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -39,11 +39,12 @@ from django.db.models import (
QuerySet,
Subquery,
TextChoices,
+ UUIDField,
Value,
When,
)
from django.db.models.fields.json import KeyTextTransform, KeyTransform
-from django.db.models.functions import Coalesce
+from django.db.models.functions import Cast, Coalesce
from django.utils import timezone as django_timezone
from django.utils.dateparse import parse_datetime
from django.utils.http import content_disposition_header
@@ -109,6 +110,7 @@ from products.tasks.backend.facade.gateway import mint_private_gateway_token, re
from products.tasks.backend.facade.profiles import (
GithubAccess,
NetworkAccess,
+ SourceKind,
TaskProfile,
TaskRunOptions,
TaskSource,
@@ -211,7 +213,7 @@ from products.tasks.backend.repository_config_analytics import (
capture_space_context_changed,
repositories_differ,
)
-from products.tasks.backend.task_sources import apply_task_source
+from products.tasks.backend.task_sources import apply_task_source, idempotency_key_q, source_q
from products.tasks.backend.visibility import (
PRODUCT_PRIVATE_ORIGIN_PRODUCTS,
TEAM_READABLE_ORIGIN_PRODUCTS,
@@ -278,6 +280,7 @@ __all__ = [
"create_imported_task_run",
"get_imported_task_run",
"get_task_by_origin_key",
+ "get_task_id_by_idempotency_key",
"touch_imported_task",
"apply_task_run_model_config",
"task_run_model_outside_gateway_pin",
@@ -1083,7 +1086,7 @@ def _task_detail_to_dto(
repositories=task.repositories or ([task.repository] if task.repository else []),
github_integration=task.github_integration_id,
github_user_integration=task.github_user_integration_id,
- signal_report=task.signal_report_id,
+ signal_report=task.signal_report_source_id,
json_schema=task.json_schema,
internal=task.internal,
archived=task.archived,
@@ -1097,7 +1100,7 @@ def _task_detail_to_dto(
latest_run_id=latest_run_id,
channel=task.channel_id,
slack_thread_references=_task_slack_thread_references(task),
- origin_key=None if task.is_scout_experiment else task.origin_key,
+ origin_key=None if task.is_scout_experiment else task.stored_idempotency_key,
client_provenance=task.client_provenance,
)
@@ -1239,13 +1242,14 @@ def find_signal_implementation_run(
task = run.task
# Belt and braces: only signals' auto_start stamps the branch key today, but the ai_stage and
# signal-report checks keep a future writer of the key from silently widening the carve-out.
- if task.signal_report_id is None or (run.state or {}).get("ai_stage") != "implementation":
+ signal_report_id = task.signal_report_source_id
+ if signal_report_id is None or (run.state or {}).get("ai_stage") != "implementation":
return None
return contracts.SignalImplementationRunDTO(
run_id=run.id,
task_id=task.id,
team_id=run.team_id,
- signal_report_id=task.signal_report_id,
+ signal_report_id=signal_report_id,
task_created_by_id=task.created_by_id,
)
@@ -1431,32 +1435,46 @@ def task_exempt_from_code_access(task_id: str | UUID, team_id: int) -> bool:
bypass. The report's own team is re-checked here even though the write serializer
already enforces it, so a future write path can't silently widen the exemption.
"""
- return Task.objects.filter(
- Q(
- origin_product=Task.OriginProduct.SIGNAL_REPORT,
- signal_report__team_id=team_id,
- repository__isnull=True,
- repositories=[],
- github_integration__isnull=True,
- github_user_integration__isnull=True,
- )
- | Q(
- origin_product=Task.OriginProduct.SIGNALS_CHAT,
- repository__isnull=True,
- repositories=[],
- github_integration__isnull=True,
- github_user_integration__isnull=True,
- )
- | Q(
- origin_product=Task.OriginProduct.SIGNAL_REPORT,
- signal_report__team_id=team_id,
- signal_report_tasks__team_id=team_id,
- signal_report_tasks__report_id=F("signal_report_id"),
- signal_report_tasks__relationship="implementation",
+ linked_report_id = Case(
+ When(
+ source_kind=SourceKind.SIGNAL_REPORT.value,
+ source_id__isnull=False,
+ then=Cast("source_id", UUIDField()),
),
- id=task_id,
- team_id=team_id,
- ).exists()
+ When(Q(source_kind__isnull=True) | Q(source_id__isnull=True), then=F("signal_report_id")),
+ output_field=UUIDField(),
+ )
+ report_in_team = Exists(signal_report_queryset().filter(id=OuterRef("_linked_report_id"), team_id=team_id))
+ return (
+ Task.objects.annotate(_linked_report_id=linked_report_id)
+ .filter(
+ Q(
+ report_in_team,
+ origin_product=Task.OriginProduct.SIGNAL_REPORT,
+ repository__isnull=True,
+ repositories=[],
+ github_integration__isnull=True,
+ github_user_integration__isnull=True,
+ )
+ | Q(
+ origin_product=Task.OriginProduct.SIGNALS_CHAT,
+ repository__isnull=True,
+ repositories=[],
+ github_integration__isnull=True,
+ github_user_integration__isnull=True,
+ )
+ | Q(
+ report_in_team,
+ origin_product=Task.OriginProduct.SIGNAL_REPORT,
+ signal_report_tasks__team_id=team_id,
+ signal_report_tasks__report_id=F("_linked_report_id"),
+ signal_report_tasks__relationship="implementation",
+ ),
+ id=task_id,
+ team_id=team_id,
+ )
+ .exists()
+ )
def get_in_progress_runs_for_github_integration(
@@ -1520,10 +1538,10 @@ def get_signal_report_implementation_runs(
return [
_task_run_to_dto(run)
for run in TaskRun.objects.filter(
+ source_q(SourceKind.SIGNAL_REPORT, [report_id], via="task__"),
team_id=team_id,
task_id__in=task_ids,
task__team_id=team_id,
- task__signal_report_id=report_id,
task__origin_product=Task.OriginProduct.SIGNAL_REPORT,
task__deleted=False,
)
@@ -2020,11 +2038,13 @@ def link_slack_task_to_report(*, team_id: int, task_id: str, report_id: str, use
task = Task.objects.select_for_update().get(
team_id=team_id, id=task_id, origin_product=Task.OriginProduct.SLACK, created_by_id=user_id
)
- if task.signal_report_id is not None:
+ if task.signal_report_source_id is not None:
return
record_slack_report_discussion(team_id=team_id, report_id=report_id, task_id=task_id, user_id=user_id)
task.signal_report_id = UUID(report_id)
- task.save(update_fields=["signal_report_id", "updated_at"])
+ task.source_kind = SourceKind.SIGNAL_REPORT.value
+ task.source_id = str(task.signal_report_id)
+ task.save(update_fields=["signal_report_id", "source_kind", "source_id", "updated_at"])
def create_and_run_task(
@@ -4085,12 +4105,16 @@ def attach_task_run_input_files(*, team_id: int, run_id: UUID, files: Sequence[c
run.save(update_fields=["artifacts", "updated_at"])
+def get_task_id_by_idempotency_key(team_id: int, key: str) -> UUID | None:
+ return Task.objects.filter(idempotency_key_q(key), team_id=team_id).values_list("id", flat=True).first()
+
+
def get_task_by_origin_key(
team_id: int, origin_key: str, *, include_latest_run: bool = True
) -> contracts.TaskDetailDTO | None:
"""The live task carrying ``origin_key`` in ``team_id``, or None. Bypasses visibility: an
origin key is an idempotency handle for a system writer, not a user-facing lookup."""
- task = Task.objects.filter(team_id=team_id, origin_key=origin_key, deleted=False).first()
+ task = Task.objects.filter(idempotency_key_q(origin_key), team_id=team_id, deleted=False).first()
return _task_detail_to_dto(task, include_latest_run=include_latest_run) if task is not None else None
@@ -7312,14 +7336,25 @@ def list_workflow_last_runs(
latest_run = TaskRun.objects.filter(task=OuterRef("pk"), team_id=team_id).order_by("-created_at", "-id")
rows = (
_visible_task_qs(team_id, user_id, for_list=True)
- .filter(hog_flow_id__in=ids, internal=False)
- .order_by("hog_flow_id", "-created_at", "-id")
- .distinct("hog_flow_id")
+ .filter(source_q(SourceKind.HOG_FLOW, ids), internal=False)
+ .annotate(
+ _workflow_id=Case(
+ When(
+ source_kind=SourceKind.HOG_FLOW.value,
+ source_id__isnull=False,
+ then=Cast("source_id", UUIDField()),
+ ),
+ default=F("hog_flow_id"),
+ output_field=UUIDField(),
+ ),
+ )
+ .order_by("_workflow_id", "-created_at", "-id")
+ .distinct("_workflow_id")
.annotate(
_run_status=Subquery(latest_run.values("status")[:1]),
_run_created_at=Subquery(latest_run.values("created_at")[:1]),
)
- .values_list("hog_flow_id", "id", "created_at", "_run_status", "_run_created_at")
+ .values_list("_workflow_id", "id", "created_at", "_run_status", "_run_created_at")
)
return {
hog_flow_id: contracts.WorkflowLastRunDTO(
@@ -7461,7 +7496,7 @@ def _list_tasks_queryset(
hog_flow_id = filters.get("hog_flow_id")
if hog_flow_id:
- qs = qs.filter(hog_flow_id=hog_flow_id)
+ qs = qs.filter(source_q(SourceKind.HOG_FLOW, [hog_flow_id]))
if search:
search_term = search.strip()
@@ -8381,6 +8416,12 @@ def create_task(
)
validated_data["github_integration"] = pilot_integration
+ if report_ref:
+ validated_data["source_kind"] = SourceKind.SIGNAL_REPORT.value
+ validated_data["source_id"] = str(getattr(report_ref, "id", report_ref))
+ if validated_data.get("origin_key") is not None:
+ validated_data["idempotency_key"] = validated_data["origin_key"]
+
logger.info("Creating task with data: %s", validated_data)
with transaction.atomic():
if signal_report_id:
@@ -8394,13 +8435,13 @@ def create_task(
created_by_id=user_id,
)
task = Task.objects.create(**validated_data)
- if task.signal_report_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT:
+ if task.signal_report_source_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT:
# Record the task↔report association + work-log artefact for the asserted relationship
# (defaults to implementation, which also writes the auto-start spend gate row) so a
# manually-started task matches autostarted ones.
record_report_task(
team_id=task.team_id,
- report_id=str(task.signal_report_id),
+ report_id=str(task.signal_report_source_id),
task_id=str(task.id),
relationship=signal_report_task_relationship,
)
@@ -8937,8 +8978,10 @@ def _find_idling_warm_run(
return None
normalized_repositories = [repo.lower() for repo in (repositories or ([repository] if repository else []))]
repository_filter = {"task__repository__iexact": repository} if repository else {"task__repository__isnull": True}
+ report_filter = source_q(SourceKind.SIGNAL_REPORT, [signal_report_id], via="task__") if signal_report_id else Q()
candidates = (
TaskRun.objects.filter( # nosemgrep: idor-lookup-without-team — team_id filter applied via the task FK below
+ report_filter,
task__team_id=team_id,
task__created_by_id=user_id,
task__origin_product=origin_product,
@@ -8948,7 +8991,6 @@ def _find_idling_warm_run(
**({"state__billing_product": "posthog_ai"} if origin_product == Task.OriginProduct.SIGNAL_REPORT else {}),
branch=branch or None,
**repository_filter,
- **({"task__signal_report_id": signal_report_id} if signal_report_id else {}),
)
.exclude(status__in=_TERMINAL_TASK_RUN_STATUSES)
.select_related("task")
@@ -9271,7 +9313,7 @@ def _reserve_report_warm_activation(
record_report_task,
)
- report_id = str(task.signal_report_id)
+ report_id = str(task.signal_report_source_id)
if is_pull_request_run_request(description):
enforce_self_driving_free_trial(team, report_id=report_id)
with transaction.atomic():
@@ -9288,7 +9330,7 @@ def _reserve_report_warm_activation(
def _is_unlinked_report_warm_task(task: Task) -> bool:
- if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or not task.signal_report_id:
+ if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or not task.signal_report_source_id:
return False
if not task.runs.filter(team_id=task.team_id, state__has_key="prewarmed").exists():
return False
@@ -9296,7 +9338,7 @@ def _is_unlinked_report_warm_task(task: Task) -> bool:
report_task_ids,
)
- return str(task.id) not in report_task_ids(team_id=task.team_id, report_id=str(task.signal_report_id))
+ return str(task.id) not in report_task_ids(team_id=task.team_id, report_id=str(task.signal_report_source_id))
def task_run_awaits_report_activation(run_id: str | UUID, task_id: str | UUID, team_id: int) -> bool:
@@ -9754,8 +9796,8 @@ def run_task(
error=contracts.TaskValidationError(kind="detail", detail=cap_detail, code=ACTIVE_SESSION_CAP_CODE)
)
report_id_for_slot_check = (
- str(task.signal_report_id)
- if task.signal_report_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT
+ str(task.signal_report_source_id)
+ if task.signal_report_source_id and task.origin_product == Task.OriginProduct.SIGNAL_REPORT
else None
)
is_implementation = False
diff --git a/products/tasks/backend/facade/canvas_tasks.py b/products/tasks/backend/facade/canvas_tasks.py
index faf87f53ff8d..8fc8dc5d6f0d 100644
--- a/products/tasks/backend/facade/canvas_tasks.py
+++ b/products/tasks/backend/facade/canvas_tasks.py
@@ -6,6 +6,7 @@ from django.db import transaction
from products.tasks.backend.facade import api
from products.tasks.backend.facade.contracts import TaskRunDTO
from products.tasks.backend.models import Channel, Task
+from products.tasks.backend.task_sources import idempotency_key_q
def create_and_run_channel_task(
@@ -32,7 +33,7 @@ def create_and_run_channel_task(
raise ValueError("Space not found.")
origin_key = f"canvas:{canvas_id}:{user_id}:{idempotency_key}"
- existing = Task.objects.filter(team_id=team_id, origin_key=origin_key).first()
+ existing = Task.objects.filter(idempotency_key_q(origin_key), team_id=team_id).first()
if existing is not None:
if existing.deleted or existing.channel_id != channel.id or existing.created_by_id != user_id:
raise ValueError("The previous task is no longer available. Start a new request.")
diff --git a/products/tasks/backend/facade/onboarding.py b/products/tasks/backend/facade/onboarding.py
index 13bc7e5e9355..0bd252d4a13a 100644
--- a/products/tasks/backend/facade/onboarding.py
+++ b/products/tasks/backend/facade/onboarding.py
@@ -38,6 +38,7 @@ from products.tasks.backend.facade.onboarding_prompt import (
)
from products.tasks.backend.logic.services.model_catalogue import filter_unsupported_effort, runtime_adapter_for
from products.tasks.backend.models import Task, TaskClientProvenance
+from products.tasks.backend.task_sources import idempotency_key_q
from ee.billing.salesforce_enrichment.constants import PERSONAL_EMAIL_DOMAINS
@@ -62,8 +63,8 @@ def _origin_key(user_id: int) -> str:
def _started_session_id(team_id: int, user_id: int) -> UUID | None:
return (
Task.objects.filter(
+ idempotency_key_q(_origin_key(user_id)),
team_id=team_id,
- origin_key=_origin_key(user_id),
created_by_id=user_id,
origin_product=Task.OriginProduct.USER_CREATED,
)
diff --git a/products/tasks/backend/facade/profiles.py b/products/tasks/backend/facade/profiles.py
index 45a3ec8c946b..0ac3972a77d9 100644
--- a/products/tasks/backend/facade/profiles.py
+++ b/products/tasks/backend/facade/profiles.py
@@ -75,6 +75,7 @@ class ContractCheck(StrEnum):
class SourceKind(StrEnum):
SIGNAL_REPORT = "signal_report"
HOG_FLOW = "hog_flow"
+ LOOP = "loop"
@frozen
diff --git a/products/tasks/backend/logic/services/task_analysis.py b/products/tasks/backend/logic/services/task_analysis.py
index ce469510d481..d692e3e18049 100644
--- a/products/tasks/backend/logic/services/task_analysis.py
+++ b/products/tasks/backend/logic/services/task_analysis.py
@@ -31,6 +31,7 @@ from products.tasks.backend.logic.services.staged_artifacts import (
tag_task_artifact,
)
from products.tasks.backend.models import SandboxCustomImage, Task, TaskRun
+from products.tasks.backend.task_sources import idempotency_key_startswith_q
logger = structlog.get_logger(__name__)
@@ -121,9 +122,9 @@ def _next_analysis_attempt(*, team_id: int, target_run_id: str) -> int:
one possible, which a bare per-run key would block forever.
"""
return Task.objects.filter(
+ idempotency_key_startswith_q(f"{TASK_ANALYSIS_ORIGIN_KEY_PREFIX}:{target_run_id}:"),
team_id=team_id,
origin_product=Task.OriginProduct.TASK_ANALYSIS,
- origin_key__startswith=f"{TASK_ANALYSIS_ORIGIN_KEY_PREFIX}:{target_run_id}:",
).count()
diff --git a/products/tasks/backend/logic/services/workflow_step_resume.py b/products/tasks/backend/logic/services/workflow_step_resume.py
index 54b10c94ffc4..f15fa47269ae 100644
--- a/products/tasks/backend/logic/services/workflow_step_resume.py
+++ b/products/tasks/backend/logic/services/workflow_step_resume.py
@@ -80,9 +80,9 @@ def _result_for_run(task_run: TaskRun) -> dict[str, Any]:
def _workflow_origin_key(task_run: TaskRun) -> str | None:
task = task_run.task
- if task.origin_product != Task.OriginProduct.WORKFLOW or not task.origin_key:
+ if task.origin_product != Task.OriginProduct.WORKFLOW or not task.stored_idempotency_key:
return None
- return task.origin_key
+ return task.stored_idempotency_key
def _emit(task_run: TaskRun, origin_key: str, status: WorkflowStepResumeStatus) -> None:
diff --git a/products/tasks/backend/logic/services/workflow_tasks.py b/products/tasks/backend/logic/services/workflow_tasks.py
index e1a7e3d57b65..772b05661d4a 100644
--- a/products/tasks/backend/logic/services/workflow_tasks.py
+++ b/products/tasks/backend/logic/services/workflow_tasks.py
@@ -29,6 +29,7 @@ from products.slack_app.backend.facade.api import slack_artifact_delivery_state_
from products.slack_app.backend.models import SlackThreadTaskMapping
from products.slack_app.backend.slack_thread import SlackThreadContext
from products.tasks.backend.facade import contracts
+from products.tasks.backend.facade.profiles import SourceKind
from products.tasks.backend.logic.services.ai_run_defaults import resolve_ai_run_selection
from products.tasks.backend.logic.services.code_usage_gate import usage_limit_response
from products.tasks.backend.logic.services.gateway_model_pin import model_allowed_by_product_pin
@@ -46,6 +47,7 @@ from products.tasks.backend.logic.services.workflow_task_skills import (
from products.tasks.backend.metrics import observe_workflow_task_create
from products.tasks.backend.model_catalog import CLAUDE
from products.tasks.backend.models import Channel, Task, TaskRun
+from products.tasks.backend.task_sources import idempotency_key_q, source_q
from products.tasks.backend.temporal.constants import WORKFLOW_RUN_IDLE_TIMEOUT_SECONDS
from ee.billing.quota_limiting import QuotaResource, is_team_over_credit_budget
@@ -331,7 +333,9 @@ def create_workflow_task(
raise WorkflowTaskTeamRateCapped(team_rate_cap)
in_flight = TaskRun.objects.filter(
- task__team_id=team.id, task__hog_flow_id=hog_flow_id, status__in=ACTIVE_RUN_STATUSES
+ source_q(SourceKind.HOG_FLOW, [hog_flow_id], via="task__"),
+ task__team_id=team.id,
+ status__in=ACTIVE_RUN_STATUSES,
).count()
if in_flight >= max_parallel_tasks:
observe_workflow_task_create(reason="limit_reached")
@@ -496,10 +500,10 @@ def _find_replayed_task(
) -> contracts.WorkflowTaskDTO | None:
if origin_key is None:
return None
- existing = Task.objects.filter(team_id=team_id, origin_key=origin_key).first()
+ existing = Task.objects.filter(idempotency_key_q(origin_key), team_id=team_id).first()
if existing is None:
return None
- if existing.hog_flow_id != hog_flow_id:
+ if existing.hog_flow_source_id != hog_flow_id:
raise WorkflowTaskOriginKeyConflict(origin_key)
return _task_dto(existing, created=False)
@@ -516,7 +520,9 @@ def _daily_task_counts(team_id: int, hog_flow_id: uuid.UUID) -> _DailyTaskCounts
"""One function so the unlocked pre-check and the authoritative locked recheck run
the identical query and can't drift apart."""
since = django_timezone.now() - timedelta(hours=24)
- workflow_count = Task.objects.filter(team_id=team_id, hog_flow_id=hog_flow_id, created_at__gte=since).count()
+ workflow_count = Task.objects.filter(
+ source_q(SourceKind.HOG_FLOW, [hog_flow_id]), team_id=team_id, created_at__gte=since
+ ).count()
team_count = Task.objects.filter(
team_id=team_id, origin_product=Task.OriginProduct.WORKFLOW, created_at__gte=since
).count()
diff --git a/products/tasks/backend/management/commands/backfill_task_sources.py b/products/tasks/backend/management/commands/backfill_task_sources.py
new file mode 100644
index 000000000000..568eba9f9a76
--- /dev/null
+++ b/products/tasks/backend/management/commands/backfill_task_sources.py
@@ -0,0 +1,130 @@
+import time
+from collections.abc import Iterator
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandParser
+from django.db import connection
+
+from posthog.dataclasses import frozen
+
+NIL_UUID = "00000000-0000-0000-0000-000000000000"
+MAX_BATCH_SIZE = 5000
+
+_LOOP_COLUMN_EXISTS_SQL = (
+ "SELECT EXISTS ("
+ "SELECT 1 FROM information_schema.columns "
+ "WHERE table_schema = current_schema() AND table_name = 'posthog_task' AND column_name = 'loop_id'"
+ ")"
+)
+
+_CANDIDATES_SQL = (
+ "SELECT p.id FROM posthog_task p "
+ "WHERE p.id > %(after)s::uuid "
+ "AND (%(team_id)s::bigint IS NULL OR p.team_id = %(team_id)s::bigint) "
+ "AND ("
+ "((p.source_kind IS NULL OR p.source_id IS NULL) "
+ "AND (p.signal_report_id IS NOT NULL OR p.hog_flow_id IS NOT NULL{loop_present})) "
+ "OR (p.idempotency_key IS NULL AND p.origin_key IS NOT NULL)"
+ ") "
+ "ORDER BY p.id "
+ "LIMIT %(batch_size)s"
+)
+
+_UPDATE_SQL = (
+ "WITH batch AS ({candidates}) "
+ "UPDATE posthog_task AS t "
+ "SET source_kind = CASE WHEN t.source_kind IS NOT NULL AND t.source_id IS NOT NULL THEN t.source_kind "
+ "ELSE COALESCE(CASE "
+ "WHEN t.signal_report_id IS NOT NULL THEN 'signal_report' "
+ "WHEN t.hog_flow_id IS NOT NULL THEN 'hog_flow'{loop_kind} "
+ "END, t.source_kind) END, "
+ "source_id = CASE WHEN t.source_kind IS NOT NULL AND t.source_id IS NOT NULL THEN t.source_id "
+ "ELSE COALESCE(CASE "
+ "WHEN t.signal_report_id IS NOT NULL THEN t.signal_report_id::text "
+ "WHEN t.hog_flow_id IS NOT NULL THEN t.hog_flow_id::text{loop_id} "
+ "END, t.source_id) END, "
+ "idempotency_key = COALESCE(t.idempotency_key, t.origin_key) "
+ "FROM batch WHERE t.id = batch.id "
+ "RETURNING t.id"
+)
+
+_LOOP_PRESENT = " OR p.loop_id IS NOT NULL"
+_LOOP_KIND = " WHEN t.loop_id IS NOT NULL THEN 'loop'"
+_LOOP_ID = " WHEN t.loop_id IS NOT NULL THEN t.loop_id::text"
+
+
+@frozen
+class BackfillBatch:
+ tasks: int
+ cursor: str
+
+
+def _loop_column_exists() -> bool:
+ with connection.cursor() as cursor:
+ cursor.execute(_LOOP_COLUMN_EXISTS_SQL)
+ row = cursor.fetchone()
+ return bool(row and row[0])
+
+
+def _candidates_sql(*, with_loop: bool) -> str:
+ return _CANDIDATES_SQL.format(loop_present=_LOOP_PRESENT if with_loop else "")
+
+
+def _update_sql(*, with_loop: bool) -> str:
+ return _UPDATE_SQL.format(
+ candidates=_candidates_sql(with_loop=with_loop),
+ loop_kind=_LOOP_KIND if with_loop else "",
+ loop_id=_LOOP_ID if with_loop else "",
+ )
+
+
+def backfill_task_sources(
+ *,
+ team_id: int | None = None,
+ batch_size: int = 1000,
+ after: str | None = None,
+ dry_run: bool = False,
+ sleep_seconds: float = 0.0,
+) -> Iterator[BackfillBatch]:
+ with_loop = _loop_column_exists()
+ sql = _candidates_sql(with_loop=with_loop) if dry_run else _update_sql(with_loop=with_loop)
+ params = {"team_id": team_id, "batch_size": min(max(batch_size, 1), MAX_BATCH_SIZE)}
+ cursor_id = after or NIL_UUID
+ while True:
+ with connection.cursor() as cursor:
+ cursor.execute(sql, {**params, "after": cursor_id})
+ task_ids = [row[0] for row in cursor.fetchall()]
+ if not task_ids:
+ return
+ cursor_id = str(max(task_ids))
+ yield BackfillBatch(tasks=len(task_ids), cursor=cursor_id)
+ if sleep_seconds:
+ time.sleep(sleep_seconds)
+
+
+class Command(BaseCommand):
+ help = (
+ "Fill source_kind, source_id and idempotency_key on tasks created before those columns existed. "
+ "Safe to rerun, and resumable with --after."
+ )
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, metavar="ID", help="Only tasks in this team")
+ parser.add_argument("--batch-size", type=int, default=1000)
+ parser.add_argument("--after", help="Resume after this task id, as printed by an earlier run")
+ parser.add_argument("--sleep-seconds", type=float, default=0.1, help="Pause between batches")
+ parser.add_argument("--dry-run", action="store_true", help="Count the tasks without writing")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ verb = "Would fill" if options["dry_run"] else "Filled"
+ total = 0
+ for batch in backfill_task_sources(
+ team_id=options["team_id"],
+ batch_size=options["batch_size"],
+ after=options["after"],
+ dry_run=options["dry_run"],
+ sleep_seconds=options["sleep_seconds"],
+ ):
+ total += batch.tasks
+ self.stdout.write(f"{verb} {total} tasks; resume after {batch.cursor}")
+ self.stdout.write(f"{verb} {total} tasks in all")
diff --git a/products/tasks/backend/management/commands/test/test_backfill_task_sources.py b/products/tasks/backend/management/commands/test/test_backfill_task_sources.py
new file mode 100644
index 000000000000..c1415f8ab0e5
--- /dev/null
+++ b/products/tasks/backend/management/commands/test/test_backfill_task_sources.py
@@ -0,0 +1,145 @@
+from io import StringIO
+from uuid import UUID, uuid4
+
+from django.core.management import call_command
+from django.db import connection
+from django.test import TestCase
+
+from parameterized import parameterized
+
+from posthog.models import Organization, Team
+
+from products.signals.backend.models import SignalReport
+from products.tasks.backend.management.commands.backfill_task_sources import backfill_task_sources
+from products.tasks.backend.models import Task
+
+REPORT_ID = UUID("11111111-1111-4111-8111-111111111111")
+FLOW_ID = UUID("22222222-2222-4222-8222-222222222222")
+LOOP_ID = UUID("33333333-3333-4333-8333-333333333333")
+
+
+class TestBackfillTaskSources(TestCase):
+ def setUp(self) -> None:
+ super().setUp()
+ self.team = Team.objects.create(organization=Organization.objects.create(name="Org"), name="Team")
+ SignalReport.objects.create(team=self.team, id=REPORT_ID)
+
+ def _task(self, team: Team | None = None, **columns: object) -> Task:
+ return Task.objects.create(
+ team=team or self.team,
+ title="Task",
+ description="d",
+ origin_product=Task.OriginProduct.USER_CREATED,
+ **columns,
+ )
+
+ def _columns(self, task: Task) -> tuple[str | None, str | None, str | None]:
+ task.refresh_from_db()
+ return (task.source_kind, task.source_id, task.idempotency_key)
+
+ @parameterized.expand(
+ [
+ ("nothing_to_fill", {}, (None, None, None)),
+ ("report_only", {"signal_report_id": REPORT_ID}, ("signal_report", str(REPORT_ID), None)),
+ ("flow_only", {"hog_flow_id": FLOW_ID}, ("hog_flow", str(FLOW_ID), None)),
+ ("key_only", {"origin_key": "key"}, (None, None, "key")),
+ (
+ "report_and_key",
+ {"signal_report_id": REPORT_ID, "origin_key": "key"},
+ ("signal_report", str(REPORT_ID), "key"),
+ ),
+ (
+ "source_filled_key_missing",
+ {"hog_flow_id": FLOW_ID, "source_kind": "hog_flow", "source_id": str(FLOW_ID), "origin_key": "key"},
+ ("hog_flow", str(FLOW_ID), "key"),
+ ),
+ (
+ "key_filled_source_missing",
+ {"hog_flow_id": FLOW_ID, "origin_key": "key", "idempotency_key": "key"},
+ ("hog_flow", str(FLOW_ID), "key"),
+ ),
+ ("kind_without_id", {"hog_flow_id": FLOW_ID, "source_kind": "hog_flow"}, ("hog_flow", str(FLOW_ID), None)),
+ (
+ "id_without_kind",
+ {"hog_flow_id": FLOW_ID, "source_id": str(FLOW_ID)},
+ ("hog_flow", str(FLOW_ID), None),
+ ),
+ (
+ "kind_without_id_over_another_old_source",
+ {"signal_report_id": REPORT_ID, "source_kind": "hog_flow"},
+ ("signal_report", str(REPORT_ID), None),
+ ),
+ ("kind_without_id_or_old_source", {"source_kind": "hog_flow"}, ("hog_flow", None, None)),
+ (
+ "new_columns_win_over_old",
+ {
+ "signal_report_id": REPORT_ID,
+ "origin_key": "old",
+ "source_kind": "loop",
+ "source_id": "loop-1",
+ "idempotency_key": "new",
+ },
+ ("loop", "loop-1", "new"),
+ ),
+ ]
+ )
+ def test_fills_the_columns_a_task_is_missing(
+ self, _name: str, columns: dict, expected: tuple[str | None, str | None, str | None]
+ ) -> None:
+ task = self._task(**columns)
+
+ list(backfill_task_sources())
+
+ self.assertEqual(self._columns(task), expected)
+ self.assertEqual(list(backfill_task_sources(dry_run=True)), [])
+
+ def test_fills_the_source_of_a_legacy_loop_task(self) -> None:
+ task = self._task()
+ with connection.cursor() as cursor:
+ cursor.execute("UPDATE posthog_task SET loop_id = %s WHERE id = %s", [LOOP_ID, task.id])
+
+ list(backfill_task_sources())
+
+ self.assertEqual(self._columns(task), ("loop", str(LOOP_ID), None))
+
+ def test_dry_run_counts_the_tasks_without_writing(self) -> None:
+ task = self._task(origin_key="key")
+
+ batches = list(backfill_task_sources(dry_run=True))
+
+ self.assertEqual([batch.tasks for batch in batches], [1])
+ self.assertEqual(self._columns(task), (None, None, None))
+
+ def test_only_fills_the_requested_team(self) -> None:
+ other_team = Team.objects.create(organization=self.team.organization, name="Other")
+ task = self._task(origin_key="key")
+ other = self._task(team=other_team, origin_key="other-key")
+
+ list(backfill_task_sources(team_id=self.team.id))
+
+ self.assertEqual(self._columns(task), (None, None, "key"))
+ self.assertEqual(self._columns(other), (None, None, None))
+
+ def test_resumes_after_the_cursor_of_an_earlier_run(self) -> None:
+ tasks = [self._task(origin_key=f"key-{uuid4()}") for _ in range(3)]
+ ordered = sorted(tasks, key=lambda task: task.id)
+
+ first_batch = next(backfill_task_sources(batch_size=1))
+
+ self.assertEqual(first_batch.cursor, str(ordered[0].id))
+ self.assertIsNotNone(self._columns(ordered[0])[2])
+ self.assertIsNone(self._columns(ordered[1])[2])
+
+ resumed = list(backfill_task_sources(batch_size=1, after=first_batch.cursor))
+
+ self.assertEqual([batch.cursor for batch in resumed], [str(task.id) for task in ordered[1:]])
+ self.assertTrue(all(self._columns(task)[2] is not None for task in tasks))
+
+ def test_command_prints_the_cursor_to_resume_from(self) -> None:
+ task = self._task(hog_flow_id=FLOW_ID)
+ out = StringIO()
+
+ call_command("backfill_task_sources", "--sleep-seconds", "0", "--batch-size", "10", stdout=out)
+
+ self.assertIn(f"resume after {task.id}", out.getvalue())
+ self.assertEqual(self._columns(task), ("hog_flow", str(FLOW_ID), None))
diff --git a/products/tasks/backend/migrations/0147_task_source_columns.py b/products/tasks/backend/migrations/0147_task_source_columns.py
new file mode 100644
index 000000000000..b3aacd50c4f1
--- /dev/null
+++ b/products/tasks/backend/migrations/0147_task_source_columns.py
@@ -0,0 +1,25 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("tasks", "0146_sandboxsession_settle_indexes"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="task",
+ name="source_kind",
+ field=models.CharField(blank=True, max_length=32, null=True),
+ ),
+ migrations.AddField(
+ model_name="task",
+ name="source_id",
+ field=models.CharField(blank=True, max_length=64, null=True),
+ ),
+ migrations.AddField(
+ model_name="task",
+ name="idempotency_key",
+ field=models.CharField(blank=True, max_length=128, null=True),
+ ),
+ ]
diff --git a/products/tasks/backend/migrations/0148_task_source_indexes.py b/products/tasks/backend/migrations/0148_task_source_indexes.py
new file mode 100644
index 000000000000..e002fc05d7bf
--- /dev/null
+++ b/products/tasks/backend/migrations/0148_task_source_indexes.py
@@ -0,0 +1,43 @@
+from django.db import migrations, models
+
+from posthog.migration_helpers import CreateIndexConcurrently, SafeAddIndexConcurrently
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("tasks", "0147_task_source_columns"),
+ ]
+
+ operations = [
+ SafeAddIndexConcurrently(
+ model_name="task",
+ index=models.Index(
+ fields=["source_kind", "source_id", "-created_at"],
+ condition=models.Q(("source_kind__isnull", False)),
+ name="posthog_task_source_idx",
+ ),
+ ),
+ migrations.SeparateDatabaseAndState(
+ state_operations=[
+ migrations.AddConstraint(
+ model_name="task",
+ constraint=models.UniqueConstraint(
+ condition=models.Q(("idempotency_key__isnull", False)),
+ fields=("team", "idempotency_key"),
+ name="posthog_task_idempotency_key_uniq",
+ ),
+ ),
+ ],
+ database_operations=[
+ CreateIndexConcurrently(
+ index_name="posthog_task_idempotency_key_uniq",
+ table_name="posthog_task",
+ columns="(team_id, idempotency_key)",
+ unique=True,
+ where="WHERE idempotency_key IS NOT NULL",
+ ),
+ ],
+ ),
+ ]
diff --git a/products/tasks/backend/migrations/max_migration.txt b/products/tasks/backend/migrations/max_migration.txt
index eada7f0b5316..3113eb44471c 100644
--- a/products/tasks/backend/migrations/max_migration.txt
+++ b/products/tasks/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0146_sandboxsession_settle_indexes
+0148_task_source_indexes
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index 7cde9290cb74..70fb5a745518 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -57,6 +57,7 @@ from products.tasks.backend.facade.profiles import (
GithubAccess,
GithubResolution,
PrAuthorship,
+ SourceKind,
TaskProfile,
TaskProfileError,
TaskRunOptions,
@@ -82,7 +83,7 @@ from products.tasks.backend.pr_urls import read_pr_urls
from products.tasks.backend.redis import dedicated_stream_configured, run_uses_dedicated_stream
from products.tasks.backend.run_options import RunSettings, apply_run_options, run_options_record, validate_run_options
from products.tasks.backend.storage import append_jsonl_object
-from products.tasks.backend.task_sources import apply_task_source, source_from_columns
+from products.tasks.backend.task_sources import apply_task_source, idempotency_key_startswith_q
logger = structlog.get_logger(__name__)
@@ -579,6 +580,10 @@ class Task(Taggable, DeletedMetaFields, models.Model):
# workflow engine redelivery) returns the existing task instead of making a second one.
origin_key = models.CharField(max_length=128, null=True, blank=True)
+ source_kind = models.CharField(max_length=32, null=True, blank=True)
+ source_id = models.CharField(max_length=64, null=True, blank=True)
+ idempotency_key = models.CharField(max_length=128, null=True, blank=True)
+
# DEPRECATED - do not use
signal_report = models.ForeignKey(
"signals.SignalReport",
@@ -664,6 +669,11 @@ class Task(Taggable, DeletedMetaFields, models.Model):
models.Index(fields=["team", "-last_activity_at", "-id"], name="posthog_task_team_activity_idx"),
models.Index(fields=["channel", "-last_activity_at"], name="posthog_task_chan_activity_idx"),
models.Index(fields=["hog_flow_id", "-created_at"], name="posthog_task_hog_flow_idx"),
+ models.Index(
+ fields=["source_kind", "source_id", "-created_at"],
+ condition=models.Q(source_kind__isnull=False),
+ name="posthog_task_source_idx",
+ ),
]
constraints = [
models.UniqueConstraint(
@@ -671,6 +681,11 @@ class Task(Taggable, DeletedMetaFields, models.Model):
condition=models.Q(origin_key__isnull=False),
name="posthog_task_origin_key_uniq",
),
+ models.UniqueConstraint(
+ fields=["team", "idempotency_key"],
+ condition=models.Q(idempotency_key__isnull=False),
+ name="posthog_task_idempotency_key_uniq",
+ ),
]
def __str__(self):
@@ -709,6 +724,32 @@ class Task(Taggable, DeletedMetaFields, models.Model):
logger.warning("task_profile_not_registered", task_id=str(self.id), profile=self.profile)
return None
+ @property
+ def stored_source(self) -> TaskSource | None:
+ if self.source_kind is not None and self.source_id is not None:
+ if self.source_kind == SourceKind.SIGNAL_REPORT and self.signal_report_id is None:
+ return None
+ return TaskSource(kind=SourceKind(self.source_kind), id=self.source_id)
+ if self.signal_report_id is not None:
+ return TaskSource(kind=SourceKind.SIGNAL_REPORT, id=str(self.signal_report_id))
+ if self.hog_flow_id is not None:
+ return TaskSource(kind=SourceKind.HOG_FLOW, id=str(self.hog_flow_id))
+ return None
+
+ @property
+ def signal_report_source_id(self) -> uuid.UUID | None:
+ source = self.stored_source
+ return uuid.UUID(source.id) if source is not None and source.kind == SourceKind.SIGNAL_REPORT else None
+
+ @property
+ def hog_flow_source_id(self) -> uuid.UUID | None:
+ source = self.stored_source
+ return uuid.UUID(source.id) if source is not None and source.kind == SourceKind.HOG_FLOW else None
+
+ @property
+ def stored_idempotency_key(self) -> str | None:
+ return self.idempotency_key if self.idempotency_key is not None else self.origin_key
+
def _resolves_pr_authorship_per_run(self) -> bool:
profile = self.stored_profile
if profile is None:
@@ -780,15 +821,15 @@ class Task(Taggable, DeletedMetaFields, models.Model):
def scout_experiment_q(cls, *, relation: Literal["", "task"] = "") -> models.Q:
prefix = {"": "", "task": "task__"}[relation]
return models.Q(**{f"{prefix}origin_product": cls.OriginProduct.SIGNALS_SCOUT}) & (
- models.Q(**{f"{prefix}origin_key__startswith": SCOUT_TRIAL_ORIGIN_KEY_PREFIX})
- | models.Q(**{f"{prefix}origin_key__startswith": SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX})
+ idempotency_key_startswith_q(SCOUT_TRIAL_ORIGIN_KEY_PREFIX, via=prefix)
+ | idempotency_key_startswith_q(SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX, via=prefix)
)
@property
def is_scout_experiment(self) -> bool:
- return self.origin_product == self.OriginProduct.SIGNALS_SCOUT and (self.origin_key or "").startswith(
- (SCOUT_TRIAL_ORIGIN_KEY_PREFIX, SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX)
- )
+ return self.origin_product == self.OriginProduct.SIGNALS_SCOUT and (
+ self.stored_idempotency_key or ""
+ ).startswith((SCOUT_TRIAL_ORIGIN_KEY_PREFIX, SCOUT_TRIAL_JUDGE_ORIGIN_KEY_PREFIX))
def capture_event(
self, event: str, properties: dict | None = None, capture_fn: Callable[..., None] | None = None
@@ -808,12 +849,12 @@ class Task(Taggable, DeletedMetaFields, models.Model):
"repository": self.repository,
"repositories": self.repositories or ([self.repository] if self.repository else []),
}
- if self.origin_key:
- all_properties["origin_key"] = self.origin_key
+ if self.stored_idempotency_key:
+ all_properties["origin_key"] = self.stored_idempotency_key
if self.channel_id:
all_properties["channel_id"] = str(self.channel_id)
- if self.signal_report_id:
- all_properties["signal_report_id"] = str(self.signal_report_id)
+ if self.signal_report_source_id:
+ all_properties["signal_report_id"] = str(self.signal_report_source_id)
if properties:
all_properties.update(properties)
(capture_fn or posthoganalytics.capture)(
@@ -1047,7 +1088,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
# and would otherwise land on the wider interactive cap.
interactive_stage = INTERACTIVE_SIGNALS_AI_STAGE_BY_ORIGIN.get(task.origin_product)
if interactive_stage and not state.get("ai_stage") and not task.internal:
- if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or task.signal_report_id:
+ if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or task.signal_report_source_id:
state["ai_stage"] = interactive_stage
internal_stage = INTERNAL_SIGNALS_AI_STAGE_BY_ORIGIN.get(task.origin_product)
if internal_stage and not state.get("ai_stage") and task.internal:
@@ -1114,7 +1155,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
if effective_environment == TaskRun.Environment.CLOUD and scheduled_at is None:
task.observe_run_started_beside_live_runs(task_run)
- if task.origin_product == Task.OriginProduct.SIGNAL_REPORT and task.signal_report_id:
+ if task.origin_product == Task.OriginProduct.SIGNAL_REPORT and task.signal_report_source_id:
from products.signals.backend.facade.spend import ( # noqa: PLC0415 — avoids the task-model import cycle
register_report_task_spend,
)
@@ -1122,7 +1163,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
register_report_task_spend(
team_id=task.team_id,
run_id=str(task_run.id),
- report_id=str(task.signal_report_id),
+ report_id=str(task.signal_report_source_id),
stage=state.get("ai_stage"),
signal_id=spend_signal_id,
implementation_kind=implementation_spend_kind,
@@ -1355,6 +1396,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
signal_report_id: str | None = None,
hog_flow_id: uuid.UUID | None = None,
origin_key: str | None = None,
+ source: TaskSource | None = None,
ai_stage: str | None = None,
ai_agent_name: str | None = None,
sandbox_environment_id: str | None = None,
@@ -1525,6 +1567,9 @@ class Task(Taggable, DeletedMetaFields, models.Model):
state=initial_state,
hog_flow_id=hog_flow_id,
origin_key=origin_key,
+ source_kind=source.kind.value if source is not None else None,
+ source_id=source.id if source is not None else None,
+ idempotency_key=origin_key,
**({"signal_report_id": signal_report_id} if signal_report_id else {}),
)
@@ -1704,10 +1749,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
origin = resolve_task_origin(profile=profile, origin_product=origin_product, internal=internal)
columns = apply_task_source(source, signal_report_id=signal_report_id, hog_flow_id=None)
if profile is not None:
- check_task_source(
- profile=profile,
- source=source_from_columns(signal_report_id=columns.signal_report_id, hog_flow_id=columns.hog_flow_id),
- )
+ check_task_source(profile=profile, source=columns.source)
runtime = profile.runtimes[0] if profile is not None else Task.Runtime.ACP.value
validate_run_options(profile, options)
if options is not None:
@@ -1741,6 +1783,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
branch=branch,
signal_report_id=columns.signal_report_id,
hog_flow_id=columns.hog_flow_id,
+ source=columns.source,
sandbox_environment_id=sandbox_environment_id,
internal=origin.internal,
output_schema=output_schema,
@@ -1844,9 +1887,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
signal_report_id = columns.signal_report_id
hog_flow_id = columns.hog_flow_id
if profile is not None:
- check_task_source(
- profile=profile, source=source_from_columns(signal_report_id=signal_report_id, hog_flow_id=hog_flow_id)
- )
+ check_task_source(profile=profile, source=columns.source)
validate_run_options(profile, options)
if options is not None:
settings = apply_run_options(
@@ -1903,6 +1944,7 @@ class Task(Taggable, DeletedMetaFields, models.Model):
signal_report_id=signal_report_id,
hog_flow_id=hog_flow_id,
origin_key=origin_key,
+ source=columns.source,
sandbox_environment_id=sandbox_environment_id,
internal=internal,
client_provenance=client_provenance,
@@ -3308,7 +3350,7 @@ class TaskRun(models.Model):
),
"origin_product": self.task.origin_product,
"title": self.task.title,
- "signal_report_id": str(self.task.signal_report_id) if self.task.signal_report_id else None,
+ "signal_report_id": str(self.task.signal_report_source_id) if self.task.signal_report_source_id else None,
"environment": self.environment,
# The bare `environment` property gets clobbered by the analytics
# client's deployment-region super-property, so ship the run's
diff --git a/products/tasks/backend/task_sources.py b/products/tasks/backend/task_sources.py
index 79b538cc5954..ef88267de9d0 100644
--- a/products/tasks/backend/task_sources.py
+++ b/products/tasks/backend/task_sources.py
@@ -1,5 +1,8 @@
+from collections.abc import Iterable
from uuid import UUID
+from django.db.models import Q
+
from posthog.dataclasses import frozen
from products.tasks.backend.facade.profiles import SourceKind, TaskSource, TaskSourceError
@@ -9,18 +12,25 @@ from products.tasks.backend.facade.profiles import SourceKind, TaskSource, TaskS
class SourceColumns:
signal_report_id: str | None
hog_flow_id: UUID | None
+ source: TaskSource | None
def apply_task_source(
source: TaskSource | None, *, signal_report_id: str | None, hog_flow_id: UUID | None
) -> SourceColumns:
if source is None:
- return SourceColumns(signal_report_id=signal_report_id, hog_flow_id=hog_flow_id)
+ return SourceColumns(
+ signal_report_id=signal_report_id,
+ hog_flow_id=hog_flow_id,
+ source=source_from_columns(signal_report_id=signal_report_id, hog_flow_id=hog_flow_id),
+ )
if signal_report_id is not None or hog_flow_id is not None:
raise TaskSourceError("Pass the source as source or as signal_report_id and hog_flow_id, not both")
if source.kind == SourceKind.SIGNAL_REPORT:
- return SourceColumns(signal_report_id=source.id, hog_flow_id=None)
- return SourceColumns(signal_report_id=None, hog_flow_id=UUID(source.id))
+ return SourceColumns(signal_report_id=source.id, hog_flow_id=None, source=source)
+ if source.kind == SourceKind.HOG_FLOW:
+ return SourceColumns(signal_report_id=None, hog_flow_id=UUID(source.id), source=source)
+ return SourceColumns(signal_report_id=None, hog_flow_id=None, source=source)
def source_from_columns(*, signal_report_id: str | None, hog_flow_id: UUID | None) -> TaskSource | None:
@@ -31,3 +41,45 @@ def source_from_columns(*, signal_report_id: str | None, hog_flow_id: UUID | Non
if hog_flow_id:
return TaskSource(kind=SourceKind.HOG_FLOW, id=str(hog_flow_id))
return None
+
+
+_RELATION_PREFIXES = frozenset({"", "task__"})
+_IDEMPOTENCY_LOOKUPS = frozenset({"", "__startswith"})
+
+
+def _field_path(via: str, field: str) -> str:
+ if via not in _RELATION_PREFIXES:
+ raise ValueError(f"Unknown task relation prefix {via!r}")
+ return via + field
+
+
+def source_q(kind: SourceKind, source_ids: Iterable[str | UUID], *, via: str = "") -> Q:
+ ids = [str(source_id) for source_id in source_ids]
+ on_new_columns = Q(**{_field_path(via, "source_kind"): kind.value, _field_path(via, "source_id__in"): ids})
+ if kind == SourceKind.SIGNAL_REPORT:
+ on_new_columns &= Q(**{_field_path(via, "signal_report_id__isnull"): False})
+ legacy_field = "signal_report_id__in"
+ elif kind == SourceKind.HOG_FLOW:
+ legacy_field = "hog_flow_id__in"
+ else:
+ return on_new_columns
+ new_columns_incomplete = Q(**{_field_path(via, "source_kind__isnull"): True}) | Q(
+ **{_field_path(via, "source_id__isnull"): True}
+ )
+ return on_new_columns | (new_columns_incomplete & Q(**{_field_path(via, legacy_field): ids}))
+
+
+def idempotency_key_q(key: str, *, via: str = "") -> Q:
+ return _idempotency_q("", key, via)
+
+
+def idempotency_key_startswith_q(key_prefix: str, *, via: str = "") -> Q:
+ return _idempotency_q("__startswith", key_prefix, via)
+
+
+def _idempotency_q(lookup: str, value: str, via: str) -> Q:
+ if lookup not in _IDEMPOTENCY_LOOKUPS:
+ raise ValueError(f"Unknown idempotency key lookup {lookup!r}")
+ on_new_column = Q(**{_field_path(via, "idempotency_key" + lookup): value})
+ legacy = Q(**{_field_path(via, "idempotency_key__isnull"): True, _field_path(via, "origin_key" + lookup): value})
+ return on_new_column | legacy
diff --git a/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py b/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py
index e62090f6ccc3..1f4888d94b87 100644
--- a/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py
+++ b/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py
@@ -91,7 +91,7 @@ def enforce_self_driving_run_quota(input: EnforceSelfDrivingRunQuotaInput) -> st
if gate.enforced or not already_reported:
capture_signal_report_quota_paused(
team,
- report_id=str(task.signal_report_id) if task.signal_report_id else None,
+ report_id=str(task.signal_report_source_id) if task.signal_report_source_id else None,
stage="implementation_run",
enforced=gate.enforced,
)
diff --git a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
index 5358648dc4d0..652c7a1fa9bd 100644
--- a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
+++ b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
@@ -203,7 +203,7 @@ def complete_signal_spend(run_id: str) -> None:
complete_task_spend(
team_id=run.team_id,
run_id=run_id,
- report_id=str(run.task.signal_report_id) if run.task.signal_report_id else None,
+ report_id=str(run.task.signal_report_source_id) if run.task.signal_report_source_id else None,
)
diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py
index 4c1836491c53..0fb82fd3a5d6 100644
--- a/products/tasks/backend/tests/test_api.py
+++ b/products/tasks/backend/tests/test_api.py
@@ -3108,6 +3108,10 @@ class TestTaskAPI(BaseTaskAPITest):
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
data = response.json()
self.assertEqual(data["signal_report"], str(report.id))
+ task = Task.objects.get(id=data["id"])
+ self.assertEqual(
+ (task.signal_report_id, task.source_kind, task.source_id), (report.id, "signal_report", str(report.id))
+ )
# A manual "start implementation" records the implementation task_run artefact (the
# task↔report association and work-log entry) and writes the legacy SignalReportTask gate
# row that blocks the auto-start pipeline from double-starting.
@@ -17362,22 +17366,30 @@ class TestCloudUsageGate(BaseTaskAPITest):
assert mock_feature_enabled.call_args.args[0] == GATED_MODEL_FLAG
assert not Task.objects.filter(title="Gated").exists()
- def _inbox_task(self, origin: Task.OriginProduct) -> Task:
+ def _inbox_task(self, origin: Task.OriginProduct, *, source_columns: bool = False) -> Task:
from products.signals.backend.models import SignalReport
task = self.create_task()
task.origin_product = origin
if origin == Task.OriginProduct.SIGNAL_REPORT:
- task.signal_report = SignalReport.objects.create(team=self.team)
+ self._link_report(task, SignalReport.objects.create(team=self.team), source_columns=source_columns)
task.save()
return task
+ def _link_report(self, task: Task, report, *, source_columns: bool) -> None:
+ if source_columns:
+ task.source_kind = "signal_report"
+ task.source_id = str(report.id)
+ else:
+ task.signal_report = report
+
def _create_pr_task(
self,
*,
relationship: str = "implementation",
link_other_report: bool = False,
report_in_other_team: bool = False,
+ source_columns: bool = False,
) -> Task:
# `record_report_task` writes this row on the manual and the auto-start path.
from products.signals.backend.models import SignalReport, SignalReportTask
@@ -17390,7 +17402,7 @@ class TestCloudUsageGate(BaseTaskAPITest):
report = SignalReport.objects.create(team=report_team)
task = self.create_task()
task.origin_product = Task.OriginProduct.SIGNAL_REPORT
- task.signal_report = report
+ self._link_report(task, report, source_columns=source_columns)
task.repository = "posthog/posthog"
task.save()
SignalReportTask.objects.create(
@@ -17612,19 +17624,39 @@ class TestCloudUsageGate(BaseTaskAPITest):
self.assertEqual(response.status_code, expected_status)
self.assertEqual(mock_warm.called, inbox)
- def test_exemption_ignores_cross_team_report_link(self):
+ @parameterized.expand([("old_columns", False), ("new_columns", True)])
+ def test_exemption_ignores_cross_team_report_link(self, _name, source_columns):
# The write serializer blocks cross-team report links, so forge one at the ORM level:
- # the exemption query must re-check the report's team rather than trust the FK alone.
+ # the exemption query must re-check the report's team rather than trust the link alone.
from products.signals.backend.models import SignalReport
other_team = Team.objects.create(organization=self.organization, name="Other Team")
task = self.create_task()
task.origin_product = Task.OriginProduct.SIGNAL_REPORT
- task.signal_report = SignalReport.objects.create(team=other_team)
+ self._link_report(task, SignalReport.objects.create(team=other_team), source_columns=source_columns)
task.save()
self.assertFalse(tasks_facade.task_exempt_from_code_access(task.id, self.team.id))
+ @parameterized.expand(
+ [
+ ("report_kind_without_id", {"source_kind": "signal_report"}, True),
+ ("flow_kind_without_id", {"source_kind": "hog_flow"}, True),
+ (
+ "complete_flow_source",
+ {"source_kind": "hog_flow", "source_id": "33333333-3333-4333-8333-333333333333"},
+ False,
+ ),
+ ]
+ )
+ def test_exemption_reads_the_old_report_when_the_new_columns_are_incomplete(self, _name, new_columns, exempt):
+ task = self._inbox_task(Task.OriginProduct.SIGNAL_REPORT)
+ for field, value in new_columns.items():
+ setattr(task, field, value)
+ task.save()
+
+ self.assertEqual(tasks_facade.task_exempt_from_code_access(task.id, self.team.id), exempt)
+
@parameterized.expand(
[
("legacy_repository", "repository", "posthog/posthog"),
@@ -17640,8 +17672,9 @@ class TestCloudUsageGate(BaseTaskAPITest):
self.assertFalse(tasks_facade.task_exempt_from_code_access(task.id, self.team.id))
- def test_exemption_drops_for_signal_report_task_with_repository(self):
- task = self._inbox_task(Task.OriginProduct.SIGNAL_REPORT)
+ @parameterized.expand([("old_columns", False), ("new_columns", True)])
+ def test_exemption_drops_for_signal_report_task_with_repository(self, _name, source_columns):
+ task = self._inbox_task(Task.OriginProduct.SIGNAL_REPORT, source_columns=source_columns)
self.assertTrue(tasks_facade.task_exempt_from_code_access(task.id, self.team.id))
task.repository = "posthog/posthog"
@@ -17651,19 +17684,24 @@ class TestCloudUsageGate(BaseTaskAPITest):
@parameterized.expand(
[
- ("implementation_link", "implementation", False, False, True),
- ("discussion_label", "discussion", False, False, False),
- ("link_to_another_report", "implementation", True, False, False),
- ("report_in_another_team", "implementation", False, True, False),
+ (f"{name}_{style}", style == "new_columns", relationship, link_other_report, report_in_other_team, expected)
+ for style in ("old_columns", "new_columns")
+ for name, relationship, link_other_report, report_in_other_team, expected in [
+ ("implementation_link", "implementation", False, False, True),
+ ("discussion_label", "discussion", False, False, False),
+ ("link_to_another_report", "implementation", True, False, False),
+ ("report_in_another_team", "implementation", False, True, False),
+ ]
]
)
def test_exemption_for_create_pr_task_needs_a_team_scoped_implementation_link(
- self, _name, relationship, link_other_report, report_in_other_team, expected
+ self, _name, source_columns, relationship, link_other_report, report_in_other_team, expected
):
task = self._create_pr_task(
relationship=relationship,
link_other_report=link_other_report,
report_in_other_team=report_in_other_team,
+ source_columns=source_columns,
)
self.assertEqual(tasks_facade.task_exempt_from_code_access(task.id, self.team.id), expected)
diff --git a/products/tasks/backend/tests/test_facade.py b/products/tasks/backend/tests/test_facade.py
index a7c8283d7ce1..e533466feb3c 100644
--- a/products/tasks/backend/tests/test_facade.py
+++ b/products/tasks/backend/tests/test_facade.py
@@ -483,11 +483,35 @@ class TestFacadeReadsAndMappers(TestCase):
assert run.status == TaskRun.Status.COMPLETED
assert run.state == {}
- def test_list_workflow_last_runs_uses_each_workflows_newest_listed_task(self):
+ @parameterized.expand(
+ [
+ ("old_columns", "old", "old"),
+ ("new_columns", "new", "new"),
+ ("old_column_then_new_columns", "old", "new"),
+ ("kind_without_id", "kind_only", "kind_only"),
+ ]
+ )
+ def test_list_workflow_last_runs_uses_each_workflows_newest_listed_task(
+ self, _name: str, older_columns: str, newer_columns: str
+ ):
+ def flow_columns(flow_id: UUID, columns: str) -> dict:
+ columns_by_name: dict[str, dict] = {
+ "old": {"hog_flow_id": flow_id},
+ "new": {"source_kind": "hog_flow", "source_id": str(flow_id)},
+ "kind_only": {"source_kind": "hog_flow", "hog_flow_id": flow_id},
+ }
+ return columns_by_name[columns]
+
now = django_timezone.now()
flow_with_runs, flow_without_run, flow_not_asked = uuid4(), uuid4(), uuid4()
- older = self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_with_runs)
- newer = self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_with_runs, archived=True)
+ older = self._make_task(
+ origin_product=Task.OriginProduct.WORKFLOW, **flow_columns(flow_with_runs, older_columns)
+ )
+ newer = self._make_task(
+ origin_product=Task.OriginProduct.WORKFLOW,
+ archived=True,
+ **flow_columns(flow_with_runs, newer_columns),
+ )
Task.objects.filter(pk=older.pk).update(created_at=now - timedelta(hours=2))
Task.objects.filter(pk=newer.pk).update(created_at=now - timedelta(hours=1))
TaskRun.objects.create(task=older, team=self.team, status=TaskRun.Status.COMPLETED)
@@ -496,9 +520,15 @@ class TestFacadeReadsAndMappers(TestCase):
TaskRun.objects.filter(pk=earlier_run.pk).update(created_at=now - timedelta(minutes=50))
TaskRun.objects.filter(pk=newest_run.pk).update(created_at=now - timedelta(minutes=30))
# Run history hides internal tasks, so the newest task here must not count as the last run.
- self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_with_runs, internal=True)
- not_started = self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_without_run)
- self._make_task(origin_product=Task.OriginProduct.WORKFLOW, hog_flow_id=flow_not_asked)
+ self._make_task(
+ origin_product=Task.OriginProduct.WORKFLOW,
+ internal=True,
+ **flow_columns(flow_with_runs, newer_columns),
+ )
+ not_started = self._make_task(
+ origin_product=Task.OriginProduct.WORKFLOW, **flow_columns(flow_without_run, newer_columns)
+ )
+ self._make_task(origin_product=Task.OriginProduct.WORKFLOW, **flow_columns(flow_not_asked, newer_columns))
last_runs = facade.list_workflow_last_runs(self.team.id, self.user.id, [flow_with_runs, flow_without_run])
@@ -517,12 +547,22 @@ class TestFacadeReadsAndMappers(TestCase):
),
}
- @parameterized.expand([("ordinary", False), ("scout_trial", True), ("scout_judge", True, "scout-trial-judge:")])
- def test_task_exists_and_visibility(self, _name: str, is_trial: bool, prefix: str = "scout-trial:") -> None:
+ @parameterized.expand(
+ [
+ ("ordinary", False),
+ ("scout_trial", True),
+ ("scout_judge", True, "scout-trial-judge:"),
+ ("scout_trial_new_column", True, "scout-trial:", "idempotency_key"),
+ ("scout_judge_new_column", True, "scout-trial-judge:", "idempotency_key"),
+ ]
+ )
+ def test_task_exists_and_visibility(
+ self, _name: str, is_trial: bool, prefix: str = "scout-trial:", key_column: str = "origin_key"
+ ) -> None:
task = self._make_task(
**{
"origin_product": Task.OriginProduct.SIGNALS_SCOUT,
- "origin_key": f"{prefix}{uuid4()}:{uuid4()}",
+ key_column: f"{prefix}{uuid4()}:{uuid4()}",
}
if is_trial
else {}
@@ -552,13 +592,21 @@ class TestFacadeReadsAndMappers(TestCase):
("bound", True, True, True, True),
("sibling", True, True, False, False),
("legacy", True, True, None, False),
+ ("owner_new_column", True, False, False, True, "idempotency_key"),
+ ("sibling_new_column", True, True, False, False, "idempotency_key"),
]
)
def test_run_authorization_classifies_trial_in_one_lookup(
- self, _name: str, is_trial: bool, sandbox: bool, bound: bool | None, allowed: bool
+ self,
+ _name: str,
+ is_trial: bool,
+ sandbox: bool,
+ bound: bool | None,
+ allowed: bool,
+ key_column: str = "origin_key",
) -> None:
task = self._make_task(
- **{"origin_product": Task.OriginProduct.SIGNALS_SCOUT, "origin_key": f"scout-trial:{uuid4()}"}
+ **{"origin_product": Task.OriginProduct.SIGNALS_SCOUT, key_column: f"scout-trial:{uuid4()}"}
if is_trial
else {}
)
@@ -2097,6 +2145,7 @@ class TestFacadeReadsAndMappers(TestCase):
task = Task.objects.get(id=created if isinstance(created, UUID) else created.task_id)
self.assertEqual(task.hog_flow_id, flow_id)
+ self.assertEqual((task.source_kind, task.source_id), ("hog_flow", str(flow_id)))
@parameterized.expand(
[
diff --git a/products/tasks/backend/tests/test_models.py b/products/tasks/backend/tests/test_models.py
index 042ee1194a83..49cec0a06ce4 100644
--- a/products/tasks/backend/tests/test_models.py
+++ b/products/tasks/backend/tests/test_models.py
@@ -445,7 +445,7 @@ class TestTask(TestCase):
self.assertFalse(Task.objects.filter(team=self.team).exists())
- @parameterized.expand([(SourceKind.SIGNAL_REPORT,), (SourceKind.HOG_FLOW,)])
+ @parameterized.expand([(SourceKind.SIGNAL_REPORT,), (SourceKind.HOG_FLOW,), (SourceKind.LOOP,)])
@patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
def test_task_creation_stores_its_source(self, kind: SourceKind, _mock_execute_workflow) -> None:
from products.signals.backend.models import SignalReport
@@ -463,8 +463,62 @@ class TestTask(TestCase):
)
task.refresh_from_db()
- expected = (source_id, None) if kind == SourceKind.SIGNAL_REPORT else (None, source_id)
- self.assertEqual((task.signal_report_id, task.hog_flow_id), expected)
+ expected_old_columns = {
+ SourceKind.SIGNAL_REPORT: (source_id, None),
+ SourceKind.HOG_FLOW: (None, source_id),
+ SourceKind.LOOP: (None, None),
+ }[kind]
+ self.assertEqual((task.signal_report_id, task.hog_flow_id), expected_old_columns)
+ self.assertEqual((task.source_kind, task.source_id), (kind.value, str(source_id)))
+
+ @parameterized.expand(
+ [
+ ("signal_report_id", {"signal_report_id": "report"}, SourceKind.SIGNAL_REPORT),
+ ("hog_flow_id", {"hog_flow_id": SOURCE_ID}, SourceKind.HOG_FLOW),
+ ("no_source", {}, None),
+ ]
+ )
+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
+ def test_task_creation_stores_the_source_given_as_old_columns(
+ self, name: str, columns: dict, kind: SourceKind | None, _mock_execute_workflow
+ ) -> None:
+ from products.signals.backend.models import SignalReport
+
+ user = User.objects.create(email=f"{name}@test.com")
+ report_id = SignalReport.objects.create(team=self.team).id
+ columns = {key: str(report_id) if value == "report" else value for key, value in columns.items()}
+
+ task = Task.create_and_run(
+ team=self.team,
+ title="t",
+ description="d",
+ user_id=user.id,
+ origin_product=Task.OriginProduct.USER_CREATED,
+ **columns,
+ )
+
+ task.refresh_from_db()
+ expected_id = {None: None, SourceKind.SIGNAL_REPORT: str(report_id), SourceKind.HOG_FLOW: str(SOURCE_ID)}[kind]
+ self.assertEqual((task.source_kind, task.source_id), (kind.value if kind else None, expected_id))
+
+ @parameterized.expand([("key", "scout-trial:1"), ("no_key", None)])
+ @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
+ def test_task_creation_copies_the_origin_key_to_the_idempotency_key(
+ self, name: str, origin_key: str | None, _mock_execute_workflow
+ ) -> None:
+ user = User.objects.create(email=f"{name}@test.com")
+
+ task = Task.create_and_run(
+ team=self.team,
+ title="t",
+ description="d",
+ user_id=user.id,
+ origin_product=Task.OriginProduct.USER_CREATED,
+ origin_key=origin_key,
+ )
+
+ task.refresh_from_db()
+ self.assertEqual((task.origin_key, task.idempotency_key), (origin_key, origin_key))
@parameterized.expand(
[
@@ -472,6 +526,14 @@ class TestTask(TestCase):
"source_and_column",
{"origin_product": Task.OriginProduct.USER_CREATED, "hog_flow_id": SOURCE_ID, "source": FLOW_SOURCE},
),
+ (
+ "both_old_columns",
+ {
+ "origin_product": Task.OriginProduct.USER_CREATED,
+ "signal_report_id": str(SOURCE_ID),
+ "hog_flow_id": SOURCE_ID,
+ },
+ ),
("profile_takes_no_source", {"profile": BUSINESS_KNOWLEDGE, "source": FLOW_SOURCE}),
("profile_takes_no_legacy_source", {"profile": BUSINESS_KNOWLEDGE, "hog_flow_id": SOURCE_ID}),
(
diff --git a/products/tasks/backend/tests/test_task_sources.py b/products/tasks/backend/tests/test_task_sources.py
new file mode 100644
index 000000000000..e818f9658466
--- /dev/null
+++ b/products/tasks/backend/tests/test_task_sources.py
@@ -0,0 +1,181 @@
+from collections.abc import Callable
+from typing import Any
+from uuid import UUID
+
+from django.db.models import Q
+from django.test import SimpleTestCase, TestCase
+
+from parameterized import parameterized
+
+from posthog.models import Organization, Team
+
+from products.signals.backend.models import SignalReport
+from products.tasks.backend.facade.profiles import SourceKind
+from products.tasks.backend.models import Task, TaskRun
+from products.tasks.backend.task_sources import idempotency_key_q, idempotency_key_startswith_q, source_q
+
+REPORT_ID = UUID("11111111-1111-4111-8111-111111111111")
+FLOW_ID = UUID("22222222-2222-4222-8222-222222222222")
+
+
+class TestStoredSource(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("neither", {}, None, None),
+ ("report_in_old_column", {"signal_report_id": REPORT_ID}, REPORT_ID, None),
+ ("flow_in_old_column", {"hog_flow_id": FLOW_ID}, None, FLOW_ID),
+ (
+ "report_in_both_columns",
+ {"source_kind": "signal_report", "source_id": str(REPORT_ID), "signal_report_id": REPORT_ID},
+ REPORT_ID,
+ None,
+ ),
+ ("report_deleted", {"source_kind": "signal_report", "source_id": str(REPORT_ID)}, None, None),
+ ("flow_in_new_columns", {"source_kind": "hog_flow", "source_id": str(FLOW_ID)}, None, FLOW_ID),
+ (
+ "new_columns_win_over_old",
+ {"source_kind": "hog_flow", "source_id": str(FLOW_ID), "signal_report_id": REPORT_ID},
+ None,
+ FLOW_ID,
+ ),
+ (
+ "loop_hides_the_old_report",
+ {"source_kind": "loop", "source_id": "loop-1", "signal_report_id": REPORT_ID},
+ None,
+ None,
+ ),
+ ("kind_without_id_falls_back", {"source_kind": "hog_flow", "hog_flow_id": FLOW_ID}, None, FLOW_ID),
+ ]
+ )
+ def test_reads_the_new_columns_and_falls_back_to_the_old_ones(
+ self, _name: str, columns: dict, report_id: UUID | None, flow_id: UUID | None
+ ) -> None:
+ task = Task(**columns)
+
+ self.assertEqual((task.signal_report_source_id, task.hog_flow_source_id), (report_id, flow_id))
+
+ @parameterized.expand(
+ [
+ ("neither", {}, None),
+ ("old_column_only", {"origin_key": "old"}, "old"),
+ ("new_column_only", {"idempotency_key": "new"}, "new"),
+ ("new_column_wins", {"origin_key": "old", "idempotency_key": "new"}, "new"),
+ ]
+ )
+ def test_reads_the_idempotency_key_and_falls_back_to_the_origin_key(
+ self, _name: str, columns: dict, expected: str | None
+ ) -> None:
+ self.assertEqual(Task(**columns).stored_idempotency_key, expected)
+
+
+class TestSourceQueryPaths(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("source_q", lambda: source_q(SourceKind.SIGNAL_REPORT, [REPORT_ID], via="team__")),
+ ("idempotency_key_q", lambda: idempotency_key_q("key", via="team__")),
+ ("idempotency_key_startswith_q", lambda: idempotency_key_startswith_q("key", via="created_by__")),
+ ]
+ )
+ def test_refuses_an_unknown_relation_prefix(self, _name: str, build_q: Callable[[], Q]) -> None:
+ with self.assertRaises(ValueError):
+ build_q()
+
+
+class TestSourceQueries(TestCase):
+ def setUp(self) -> None:
+ super().setUp()
+ self.team = Team.objects.create(organization=Organization.objects.create(name="Org"), name="Team")
+ SignalReport.objects.create(team=self.team, id=REPORT_ID)
+ rows: dict[str, dict[str, Any]] = {
+ "neither": {},
+ "report_old": {"signal_report_id": REPORT_ID, "origin_key": "scout-trial:report_old"},
+ "report_deleted": {
+ "source_kind": "signal_report",
+ "source_id": str(REPORT_ID),
+ "idempotency_key": "scout-trial:report_deleted",
+ },
+ "report_both": {
+ "signal_report_id": REPORT_ID,
+ "source_kind": "signal_report",
+ "source_id": str(REPORT_ID),
+ "origin_key": "scout-trial:report_both",
+ "idempotency_key": "scout-trial:report_both",
+ },
+ "flow_old": {"hog_flow_id": FLOW_ID},
+ "flow_new": {"source_kind": "hog_flow", "source_id": str(FLOW_ID)},
+ "flow_over_old_report": {
+ "signal_report_id": REPORT_ID,
+ "source_kind": "hog_flow",
+ "source_id": str(FLOW_ID),
+ "origin_key": "scout-trial:stale",
+ "idempotency_key": "other:fresh",
+ },
+ "flow_kind_without_id": {"source_kind": "hog_flow", "hog_flow_id": FLOW_ID},
+ "flow_kind_without_id_over_old_report": {"source_kind": "hog_flow", "signal_report_id": REPORT_ID},
+ }
+ self.tasks = {
+ name: Task.objects.create(
+ team=self.team,
+ title=name,
+ description="d",
+ origin_product=Task.OriginProduct.USER_CREATED,
+ **columns,
+ )
+ for name, columns in rows.items()
+ }
+
+ def _names(self, queryset_filter) -> set[str]:
+ return {task.title for task in Task.objects.filter(queryset_filter)}
+
+ @parameterized.expand(
+ [
+ (
+ "report",
+ SourceKind.SIGNAL_REPORT,
+ REPORT_ID,
+ {"report_old", "report_both", "flow_kind_without_id_over_old_report"},
+ ),
+ (
+ "flow",
+ SourceKind.HOG_FLOW,
+ FLOW_ID,
+ {"flow_old", "flow_new", "flow_over_old_report", "flow_kind_without_id"},
+ ),
+ ("flow_id_under_the_report_kind", SourceKind.SIGNAL_REPORT, FLOW_ID, set()),
+ ("loop_has_no_old_column", SourceKind.LOOP, REPORT_ID, set()),
+ ]
+ )
+ def test_source_q_matches_new_and_old_columns_by_kind(
+ self, _name: str, kind: SourceKind, source_id: UUID, expected: set[str]
+ ) -> None:
+ self.assertEqual(self._names(source_q(kind, [source_id])), expected)
+
+ def test_source_q_follows_a_relation(self) -> None:
+ for task in self.tasks.values():
+ TaskRun.objects.create(task=task, team=self.team)
+
+ runs = TaskRun.objects.filter(source_q(SourceKind.SIGNAL_REPORT, [REPORT_ID], via="task__"))
+
+ self.assertEqual(
+ {run.task.title for run in runs},
+ {"report_old", "report_both", "flow_kind_without_id_over_old_report"},
+ )
+
+ @parameterized.expand(
+ [
+ ("old_column", "scout-trial:report_old", {"report_old"}),
+ ("new_column", "scout-trial:report_deleted", {"report_deleted"}),
+ ("both_columns", "scout-trial:report_both", {"report_both"}),
+ ("old_value_hidden_by_new", "scout-trial:stale", set()),
+ ("new_value", "other:fresh", {"flow_over_old_report"}),
+ ("unknown", "missing", set()),
+ ]
+ )
+ def test_idempotency_key_q_matches_new_and_old_columns(self, _name: str, key: str, expected: set[str]) -> None:
+ self.assertEqual(self._names(idempotency_key_q(key)), expected)
+
+ def test_idempotency_key_startswith_q_matches_new_and_old_columns(self) -> None:
+ self.assertEqual(
+ self._names(idempotency_key_startswith_q("scout-trial:")),
+ {"report_old", "report_deleted", "report_both"},
+ )
diff --git a/products/tasks/backend/tests/test_workflow_tasks_api.py b/products/tasks/backend/tests/test_workflow_tasks_api.py
index 76de7a22698e..2aae812427f9 100644
--- a/products/tasks/backend/tests/test_workflow_tasks_api.py
+++ b/products/tasks/backend/tests/test_workflow_tasks_api.py
@@ -78,13 +78,18 @@ class TestWorkflowTasksAPI(APIBaseTest):
HTTP_AUTHORIZATION=f"Bearer {token or _token(self.team.id, str(self.hog_flow.id))}",
)
- def _seed_workflow_task(self, run_status: str) -> Task:
+ def _flow_columns(self, hog_flow_id: UUID | str, *, source_columns: bool) -> dict:
+ if source_columns:
+ return {"source_kind": "hog_flow", "source_id": str(hog_flow_id)}
+ return {"hog_flow_id": hog_flow_id}
+
+ def _seed_workflow_task(self, run_status: str, *, source_columns: bool = False) -> Task:
task = Task.objects.create(
team=self.team,
title="existing",
description="existing",
origin_product=Task.OriginProduct.WORKFLOW,
- hog_flow_id=self.hog_flow.id,
+ **self._flow_columns(self.hog_flow.id, source_columns=source_columns),
)
TaskRun.objects.create(task=task, team=self.team, status=run_status)
return task
@@ -96,6 +101,7 @@ class TestWorkflowTasksAPI(APIBaseTest):
hog_flow_id: UUID | None,
origin_product: str = Task.OriginProduct.WORKFLOW,
created_at: datetime | None = None,
+ source_columns: bool = False,
) -> None:
# No TaskRun rows: the daily caps count Task rows only, and runless tasks keep the
# in-flight count at 0, so a cap test cannot pass via the in-flight 409 instead.
@@ -106,7 +112,11 @@ class TestWorkflowTasksAPI(APIBaseTest):
title=f"seed-{i}",
description="seed",
origin_product=origin_product,
- hog_flow_id=hog_flow_id,
+ **(
+ self._flow_columns(hog_flow_id, source_columns=source_columns)
+ if hog_flow_id is not None
+ else {}
+ ),
**({"created_at": created_at} if created_at is not None else {}),
)
for i in range(count)
@@ -368,14 +378,15 @@ class TestWorkflowTasksAPI(APIBaseTest):
assert not Task.objects.filter(hog_flow_id=flow.id).exists()
usage_limit_response_mock.assert_not_called()
- def test_skips_creation_at_the_in_flight_limit(self) -> None:
+ @parameterized.expand([("old_columns", False), ("new_columns", True)])
+ def test_skips_creation_at_the_in_flight_limit(self, _name: str, source_columns: bool) -> None:
for _ in range(2):
- self._seed_workflow_task(TaskRun.Status.IN_PROGRESS)
+ self._seed_workflow_task(TaskRun.Status.IN_PROGRESS, source_columns=source_columns)
response = self._post({"max_parallel_tasks": 2})
assert response.status_code == status.HTTP_409_CONFLICT, response.json()
- assert Task.objects.filter(hog_flow_id=self.hog_flow.id).count() == 2
+ assert Task.objects.count() == 2
def test_finished_runs_free_up_the_limit(self) -> None:
self._seed_workflow_task(TaskRun.Status.COMPLETED)
@@ -384,11 +395,13 @@ class TestWorkflowTasksAPI(APIBaseTest):
assert response.status_code == status.HTTP_201_CREATED, response.json()
- @parameterized.expand(["per_workflow", "team_wide"])
+ @parameterized.expand([("per_workflow", False), ("per_workflow_new_columns", True), ("team_wide", False)])
@patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response")
- def test_skips_creation_at_the_daily_cap(self, scope: str, usage_limit_response_mock) -> None:
- if scope == "per_workflow":
- self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id))
+ def test_skips_creation_at_the_daily_cap(self, scope: str, source_columns: bool, usage_limit_response_mock) -> None:
+ if scope.startswith("per_workflow"):
+ self._seed_created_tasks(
+ WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id), source_columns=source_columns
+ )
expected_fragment = "This workflow reached its daily limit"
else:
# Two other workflows fill the team budget; this workflow is far under its own cap.
@@ -518,14 +531,22 @@ class TestWorkflowTasksAPI(APIBaseTest):
assert replay.status_code == status.HTTP_200_OK, replay.json()
assert replay.json()["id"] == first.json()["id"]
- def test_rejects_an_idempotency_key_used_by_another_workflow(self) -> None:
+ @parameterized.expand(
+ [
+ ("old_columns", {"origin_key": "invocation-1"}, False),
+ ("new_columns", {"idempotency_key": "invocation-1"}, True),
+ ]
+ )
+ def test_rejects_an_idempotency_key_used_by_another_workflow(
+ self, _name: str, key_columns: dict, source_columns: bool
+ ) -> None:
Task.objects.create(
team=self.team,
title="other",
description="other",
origin_product=Task.OriginProduct.WORKFLOW,
- hog_flow_id=uuid4(),
- origin_key="invocation-1",
+ **self._flow_columns(uuid4(), source_columns=source_columns),
+ **key_columns,
)
response = self._post({"idempotency_key": "invocation-1"})