Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): finalize spend per unit of work (#123858)

Oliver Browne57ec4982551b
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index 592e882f1910..212b0afc496e 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -418,6 +418,8 @@ rules:
|SignalScoutSuggestionSet
|SignalScoutBackgroundBand
|SignalSpend
+ |LegacySignalSpend
+
|AnomalyScoringState
|SignalScratchpad
|SignalProjectProfile
@@ -902,6 +904,8 @@ rules:
|SignalScoutSuggestionSet
|SignalScoutBackgroundBand
|SignalSpend
+ |LegacySignalSpend
+
|AnomalyScoringState
|SignalScratchpad
|SignalProjectProfile
diff --git a/posthog/api/test/test_comments.py b/posthog/api/test/test_comments.py
index fea4d8f9d167..707143c2eaf7 100644
--- a/posthog/api/test/test_comments.py
+++ b/posthog/api/test/test_comments.py
@@ -20,7 +20,13 @@ from posthog.models.comment import Comment
from posthog.models.comment.utils import build_comment_item_url, extract_plain_text_from_rich_content
from posthog.models.oauth import OAuthAccessToken, OAuthApplication
from posthog.redis import get_client
-from posthog.temporal.oauth import ARRAY_APP_CLIENT_ID_DEV, POSTHOG_AI_APP_CLIENT_ID_DEV, resolve_scopes
+from posthog.temporal.oauth import (
+ ARRAY_APP_CLIENT_ID_DEV,
+ POSTHOG_AI_APP_CLIENT_ID_DEV,
+ SIGNALS_APP_CLIENT_ID_DEV,
+ create_oauth_access_token_for_user,
+ resolve_scopes,
+)
from products.access_control.backend.models.access_control import AccessControl
from products.canvas.backend.facade import testing as canvas_testing
@@ -238,7 +244,8 @@ class TestComments(APIBaseTest, QueryMatchingTest):
)
assert saved.status_code == status.HTTP_201_CREATED
- def test_task_comments_list_artifacts_comments_and_one_comment(self) -> None:
+ @parameterized.expand([ARRAY_APP_CLIENT_ID_DEV, SIGNALS_APP_CLIENT_ID_DEV, POSTHOG_AI_APP_CLIENT_ID_DEV])
+ def test_task_comments_list_artifacts_comments_and_one_comment(self, client_id: str) -> None:
task = self._task_artifact_target()
task_run_model = apps.get_model("tasks", "TaskRun")
task_run_model.objects.create(
@@ -287,7 +294,19 @@ class TestComments(APIBaseTest, QueryMatchingTest):
source_comment=root,
content="Malformed state is still a reply",
)
- client = self._sandbox_task_comment_client(task.id)
+ client = self._sandbox_task_comment_client(task.id, client_id=client_id)
+ if client_id == POSTHOG_AI_APP_CLIENT_ID_DEV:
+ task.origin_product = "signal_report"
+ task.save(update_fields=["origin_product"])
+ with mock.patch("posthog.temporal.oauth.get_instance_region", return_value=None):
+ token = create_oauth_access_token_for_user(
+ task.created_by,
+ task.team_id,
+ scopes=["task:read", "comment:read"],
+ application="posthog_ai",
+ sandbox_task_id=task.id,
+ )
+ client.credentials(HTTP_AUTHORIZATION=f"Bearer {token}")
artifacts = client.get(f"/api/projects/{self.team.id}/tasks/{task.id}/artifacts/")
assert artifacts.status_code == status.HTTP_200_OK
@@ -623,9 +642,10 @@ class TestComments(APIBaseTest, QueryMatchingTest):
assert response.status_code == status.HTTP_403_FORBIDDEN
- def test_task_comments_ask_legacy_sandboxes_to_restart(self) -> None:
+ @parameterized.expand([ARRAY_APP_CLIENT_ID_DEV, POSTHOG_AI_APP_CLIENT_ID_DEV])
+ def test_task_comments_ask_legacy_sandboxes_to_restart(self, client_id: str) -> None:
task = self._task_artifact_target()
- client = self._sandbox_task_comment_client()
+ client = self._sandbox_task_comment_client(client_id=client_id)
response = client.get(
f"/api/projects/{self.team.id}/tasks/{task.id}/artifacts/",
@@ -634,10 +654,11 @@ class TestComments(APIBaseTest, QueryMatchingTest):
assert response.status_code == status.HTTP_403_FORBIDDEN
assert "Restart the task" in str(response.json())
- def test_task_comments_reject_an_alternate_task_url_for_the_same_user(self) -> None:
+ @parameterized.expand([ARRAY_APP_CLIENT_ID_DEV, POSTHOG_AI_APP_CLIENT_ID_DEV])
+ def test_task_comments_reject_an_alternate_task_url_for_the_same_user(self, client_id: str) -> None:
bound_task = self._task_artifact_target()
other_task = self._task_artifact_target()
- client = self._sandbox_task_comment_client(bound_task.id)
+ client = self._sandbox_task_comment_client(bound_task.id, client_id=client_id)
response = client.get(
f"/api/projects/{self.team.id}/tasks/{other_task.id}/artifacts/",
@@ -645,9 +666,9 @@ class TestComments(APIBaseTest, QueryMatchingTest):
assert response.status_code == status.HTTP_404_NOT_FOUND
- def test_task_comments_reject_a_posthog_ai_sandbox_token(self) -> None:
+ def test_task_comments_reject_an_unrecognized_oauth_application(self) -> None:
task = self._task_artifact_target()
- client = self._sandbox_task_comment_client(task.id, client_id=POSTHOG_AI_APP_CLIENT_ID_DEV)
+ client = self._sandbox_task_comment_client(task.id, client_id="unrelated-client")
response = client.get(
f"/api/projects/{self.team.id}/tasks/{task.id}/artifacts/",
@@ -655,9 +676,10 @@ class TestComments(APIBaseTest, QueryMatchingTest):
assert response.status_code == status.HTTP_403_FORBIDDEN
- def test_task_comments_require_comment_read_scope(self) -> None:
+ @parameterized.expand([ARRAY_APP_CLIENT_ID_DEV, POSTHOG_AI_APP_CLIENT_ID_DEV])
+ def test_task_comments_require_comment_read_scope(self, client_id: str) -> None:
task = self._task_artifact_target()
- client = self._sandbox_task_comment_client(task.id, scopes="task:read")
+ client = self._sandbox_task_comment_client(task.id, client_id=client_id, scopes="task:read")
response = client.get(f"/api/projects/{self.team.id}/tasks/{task.id}/comments/")
diff --git a/posthog/temporal/oauth.py b/posthog/temporal/oauth.py
index 9bb13daff2e7..a97d6c4d9db9 100644
--- a/posthog/temporal/oauth.py
+++ b/posthog/temporal/oauth.py
@@ -73,17 +73,6 @@ SIGNALS_OAUTH_APP_CLIENT_IDS = frozenset(
}
)
-# Apps that mint tokens for a cloud task's own coding agent, which is what the task-comment
-# channel is restricted to. Signals joined when its runs moved off the Array app; PostHog AI is
-# deliberately absent, as it was before. The `sandbox_task_id` binding on the token is what
-# actually scopes access to one task — this set only keeps unrelated first-party tokens out.
-TASK_AGENT_OAUTH_APP_CLIENT_IDS = frozenset(
- {
- *POSTHOG_CODE_OAUTH_APP_CLIENT_IDS,
- *SIGNALS_OAUTH_APP_CLIENT_IDS,
- }
-)
-
SANDBOX_OAUTH_APP_CLIENT_IDS = frozenset(
{
*POSTHOG_CODE_OAUTH_APP_CLIENT_IDS,
diff --git a/posthog/temporal/tests/ai/test_module_integrity.py b/posthog/temporal/tests/ai/test_module_integrity.py
index df98918167e0..cd99dfd01d88 100644
--- a/posthog/temporal/tests/ai/test_module_integrity.py
+++ b/posthog/temporal/tests/ai/test_module_integrity.py
@@ -129,6 +129,7 @@ class TestSignalsProductModuleIntegrity:
"update_signal_spend_activity",
"confirm_signal_spend_activity",
"publish_report_spend_activity",
+ "report_spend_owner_activity",
"generate_scout_rubrics_activity",
"fail_scout_rubrics_activity",
"dispatch_inbox_slack_notifications_activity",
@@ -138,6 +139,7 @@ class TestSignalsProductModuleIntegrity:
"emit_backfill_signal_activity",
"fetch_error_tracking_issues_activity",
"assign_and_emit_signal_activity",
+ "complete_signal_grouping_activity",
"capture_signal_dropped_activity",
"check_signals_quota_limited_activity",
"delete_report_activity",
@@ -154,6 +156,7 @@ class TestSignalsProductModuleIntegrity:
"mark_report_failed_activity",
"read_signals_from_s3_activity",
"check_report_quota_gate_activity",
+ "check_report_task_settlement_activity",
"mark_report_in_progress_activity",
"mark_report_pending_input_activity",
"mark_report_ready_activity",
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txt
index aa85a5b175a6..bf65addbdff3 100644
--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt
+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt
@@ -257,8 +257,6 @@ post_save:products.signals.backend.receivers.sync_report_latest_actionability_on
post_save:products.signals.backend.receivers.sync_suggested_reviewer_index_on_save
post_save:products.signals.backend.receivers.sync_task_run_pr_to_assignments
post_save:products.signals.backend.receivers.tombstone_report_embedding_on_delete
-post_save:products.signals.backend.spend_receivers.scout_run_created
-post_save:products.signals.backend.spend_receivers.task_run_created
post_save:products.slack_app.backend.signals.invalidate_repo_list_on_user_github_change
post_save:products.slack_app.backend.signals.onboard_slack_inbox_on_install
post_save:products.subscriptions.backend.models.subscription.subscription_saved
diff --git a/products/signals/backend/apps.py b/products/signals/backend/apps.py
index d4d111d4f50a..1b18a761959c 100644
--- a/products/signals/backend/apps.py
+++ b/products/signals/backend/apps.py
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING
+from uuid import UUID
from django.apps import AppConfig
@@ -20,15 +21,19 @@ class SignalsConfig(AppConfig):
from . import (
activity_logging, # noqa: F401
receivers, # noqa: F401
- spend_receivers,
)
receivers.connect_task_run_assignment_sync()
- spend_receivers.connect_spend_receivers()
+ self._register_task_spend_updates()
self._register_signal_emission_gate()
self._register_scout_skill_archive_guard()
self._register_scout_skill_rename_hook()
+ def _register_task_spend_updates(self) -> None:
+ from products.tasks.backend.facade.task_run_signals import connect_task_run_cost_updated
+
+ connect_task_run_cost_updated(_record_task_spend_delta, dispatch_uid="signals.task_spend_delta")
+
def _register_signal_emission_gate(self) -> None:
"""Let the data-import pipeline ask whether to emit signals for a source without
importing this product (it depends on warehouse_sources). The gate impl is imported
@@ -89,3 +94,16 @@ class SignalsConfig(AppConfig):
raise SkillRenameConflict(str(err)) from err
register_skill_rename_hook(_hook)
+
+
+def _record_task_spend_delta(
+ *, team_id: int, run_id: UUID, cost_delta_microusd: int, origin_product: str, **kwargs: object
+) -> None:
+ if origin_product not in {"signal_report", "signals_scout"}:
+ return
+
+ from products.signals.backend.spend import (
+ record_task_cost_delta, # noqa: PLC0415 — keeps accounting off Django startup
+ )
+
+ record_task_cost_delta(team_id=team_id, run_id=run_id, cost_delta_microusd=cost_delta_microusd)
diff --git a/products/signals/backend/auto_start.py b/products/signals/backend/auto_start.py
index e8028245720c..dc2c974b970c 100644
--- a/products/signals/backend/auto_start.py
+++ b/products/signals/backend/auto_start.py
@@ -749,6 +749,7 @@ def _create_implementation_task_if_absent(
*,
team_id: int,
report_id: str,
+ implementation_spend_kind: str = "reviewer_triggered_implementation",
title: str,
description: str,
expected_content: ImplementationReportContent,
@@ -906,6 +907,7 @@ def _create_implementation_task_if_absent(
repository=repository,
branch=base_branch,
signal_report_id=report_id,
+ implementation_spend_kind=implementation_spend_kind,
# Resolved by the caller outside this lock, like `agent_runtime` above, so the
# create-time free-trial gate makes no flag request while the report row is locked.
free_trial_enabled=free_trial_enabled,
@@ -1351,6 +1353,7 @@ async def maybe_autostart_implementation_task(
*,
team_id: int,
report_id: str,
+ implementation_spend_kind: str = "reviewer_triggered_implementation",
repository: str,
title: str,
summary: str,
@@ -1460,6 +1463,7 @@ async def maybe_autostart_implementation_task(
team_id=team_id,
report_id=report_id,
remaining_retries=remaining_retries - 1,
+ implementation_spend_kind=implementation_spend_kind,
dispatch=dispatch,
ch_user=ch_user,
)
@@ -1641,6 +1645,7 @@ async def maybe_autostart_implementation_task(
created = await database_sync_to_async(_create_implementation_task_if_absent, thread_sensitive=False)(
team_id=team_id,
report_id=report_id,
+ implementation_spend_kind=implementation_spend_kind,
title=title,
expected_content=expected_content,
expected_reviewers=assignment.state,
@@ -1675,6 +1680,7 @@ async def maybe_autostart_implementation_task(
team_id=team_id,
report_id=report_id,
remaining_retries=remaining_retries - 1,
+ implementation_spend_kind=implementation_spend_kind,
dispatch=dispatch,
ch_user=ch_user,
)
@@ -1738,6 +1744,7 @@ async def maybe_autostart_from_report_artefacts(
*,
team_id: int,
report_id: str,
+ implementation_spend_kind: str = "reviewer_triggered_implementation",
remaining_retries: int = 2,
dispatch: ImplementationDispatch | None = None,
ch_user: ClickHouseUser = ClickHouseUser.DEFAULT,
@@ -1830,6 +1837,7 @@ async def maybe_autostart_from_report_artefacts(
return await maybe_autostart_implementation_task(
team_id=team_id,
report_id=report_id,
+ implementation_spend_kind=implementation_spend_kind,
repository=repository,
title=report.title,
summary=report.summary,
diff --git a/products/signals/backend/billing.py b/products/signals/backend/billing.py
index 99bb4e56895d..8ea1f3d9fdcb 100644
--- a/products/signals/backend/billing.py
+++ b/products/signals/backend/billing.py
@@ -66,7 +66,7 @@ SIGNALS_CREDITS_PER_DOLLAR = 100 # 1 credit = $0.01, matching ai_credits
SIGNALS_CREDITS_PER_REPORT_WITH_PR = 15 * SIGNALS_CREDITS_PER_DOLLAR # 1500
-def _bridges_with_pr_run(**run_created_at: datetime) -> QuerySet[SignalReportTask]:
+def _bridges_with_pr_run(*, include_manual: bool = False, **run_created_at: datetime) -> QuerySet[SignalReportTask]:
"""Implementation bridges whose task shipped a billable PR run matching `run_created_at`.
Rooted on the signals-owned `SignalReportTask` bridge and traversing to runs via the
@@ -86,6 +86,7 @@ def _bridges_with_pr_run(**run_created_at: datetime) -> QuerySet[SignalReportTas
against the same `TaskRun` row.
"""
return SignalReportTask.objects.filter(
+ **({} if include_manual else {"task__runs__state__billing_product__isnull": True}),
relationship=_IMPLEMENTATION,
task__runs__output__pr_url__startswith=GITHUB_PR_URL_PREFIX,
# Fail closed on team disagreement across run / task / bridge / report.
@@ -121,7 +122,7 @@ BILLING_EXEMPT_SOURCE_PRODUCTS: dict[str, str] = {
}
-def _scout_skill_is_canonical(team_id: int, skill_name: str) -> bool:
+def _scout_skill_is_canonical(team_id: int, skill_name: str, *, version: int | None = None) -> bool:
"""Whether the team's row for `skill_name` is still the PostHog-shipped scout, byte for byte.
`skill_name` alone doesn't attest PostHog-system origin: a team can edit a seeded scout in
@@ -129,7 +130,8 @@ def _scout_skill_is_canonical(team_id: int, skill_name: str) -> bool:
permanent exemptions. `scout_skill_row_is_proven_canonical` settles it by content hash against
the fingerprint stamped at seed time. Fails closed — no row, no hash, no match → billable.
"""
- skill = LLMSkill.objects.filter(team_id=team_id, name=skill_name, deleted=False, is_latest=True).first()
+ skills = LLMSkill.objects.filter(team_id=team_id, name=skill_name, deleted=False)
+ skill = (skills.filter(is_latest=True) if version is None else skills.filter(version=version)).first()
return skill is not None and scout_skill_row_is_proven_canonical(skill)
@@ -167,7 +169,7 @@ def mark_report_billing_exempt(report: SignalReport, reason: str) -> bool:
raise ValueError(f"Unknown billing exemption reason: {reason!r}")
if report.billing_exempt_reason:
return False
- if first_billable_pr_run_at(report.id) is not None:
+ if first_billable_pr_run_at(report.id, team_id=report.team_id) is not None:
raise BillingExemptionError(
f"Report {report.id} already has a billable PR run; use a refund instead of a billing exemption."
)
@@ -176,12 +178,16 @@ def mark_report_billing_exempt(report: SignalReport, reason: str) -> bool:
return True
+def report_has_implementation_pr(report_id: str | uuid.UUID, *, team_id: int) -> bool:
+ return _bridges_with_pr_run(include_manual=True).filter(report_id=report_id, team_id=team_id).exists()
+
+
class FirstBillablePrRun(NamedTuple):
created_at: datetime
pr_url: str
-def first_billable_pr_run(report_id: str | uuid.UUID) -> FirstBillablePrRun | None:
+def first_billable_pr_run(report_id: str | uuid.UUID, *, team_id: int) -> FirstBillablePrRun | None:
"""The report's billable moment: its first implementation run with a GitHub PR URL (with that
run's `created_at` and `pr_url`), or None if it has never shipped one.
@@ -192,7 +198,7 @@ def first_billable_pr_run(report_id: str | uuid.UUID) -> FirstBillablePrRun | No
"""
row = (
_bridges_with_pr_run()
- .filter(report_id=report_id)
+ .filter(report_id=report_id, team_id=team_id)
# Both columns resolve against the TaskRun join the filter established, so the earliest
# billable run's timestamp and URL come from the same row.
.order_by("task__runs__created_at")
@@ -204,9 +210,9 @@ def first_billable_pr_run(report_id: str | uuid.UUID) -> FirstBillablePrRun | No
return FirstBillablePrRun(created_at=row[0], pr_url=row[1])
-def first_billable_pr_run_at(report_id: str | uuid.UUID) -> datetime | None:
+def first_billable_pr_run_at(report_id: str | uuid.UUID, *, team_id: int) -> datetime | None:
"""`first_billable_pr_run`, timestamp only."""
- run = first_billable_pr_run(report_id)
+ run = first_billable_pr_run(report_id, team_id=team_id)
return run.created_at if run else None
@@ -242,7 +248,7 @@ def annotate_first_billable_pr_run_at(queryset: QuerySet[SignalReport]) -> Query
return queryset.annotate(first_billable_pr_run_at=Subquery(earliest))
-def first_billable_pr_run_at_by_report(report_ids: Sequence[str | uuid.UUID]) -> dict[str, datetime]:
+def first_billable_pr_run_at_by_report(report_ids: Sequence[str | uuid.UUID], *, team_id: int) -> dict[str, datetime]:
"""`annotate_first_billable_pr_run_at` for a known set of reports, in one grouped query.
The annotation is a correlated subquery, so it costs one bridge-and-run walk per row the
@@ -251,7 +257,7 @@ def first_billable_pr_run_at_by_report(report_ids: Sequence[str | uuid.UUID]) ->
"""
rows = (
_bridges_with_pr_run()
- .filter(report_id__in=report_ids)
+ .filter(report_id__in=report_ids, team_id=team_id)
.values("report_id")
.annotate(first_run_at=Min("task__runs__created_at"))
)
diff --git a/products/signals/backend/emission/pipeline.py b/products/signals/backend/emission/pipeline.py
index a7a01068156f..9ded74f98ebe 100644
--- a/products/signals/backend/emission/pipeline.py
+++ b/products/signals/backend/emission/pipeline.py
@@ -32,7 +32,8 @@ from products.signals.backend.emission.steering import SourceSteering, apply_ste
from products.signals.backend.facade.api import emit_signal
from products.signals.backend.llm_spend import generate_with_spend
from products.signals.backend.models import SignalEmissionRecord
-from products.signals.backend.spend import signal_id_for
+from products.signals.backend.spend import complete_signal_pipeline, finalized_signal_ids, signal_id_for
+from products.signals.backend.spend_billing import system_source_exempt
from products.signals.backend.system_one_decision import run_model_decision
from products.signals.backend.system_one_prompts import SystemOnePrompt, bundled_prompt, current_prompt
from products.signals.backend.temporal import metrics
@@ -681,6 +682,15 @@ async def run_signal_pipeline(
)
logger.info(f"Built {len(outputs)} signal outputs from {len(records)} records for {source_label}", **extra)
+ # An activity retry must not rerun filters for a signal whose pipeline already finished.
+ finished = await database_sync_to_async(finalized_signal_ids)(
+ team_id=team.id, signal_ids=[output.signal_id for output in outputs if output.signal_id]
+ )
+ if config.record_processed_outputs and finished:
+ # Dispatch may have succeeded before the emission ledger write failed.
+ await _record_processed_outputs(team, [output for output in outputs if output.signal_id in finished])
+ outputs = [output for output in outputs if output.signal_id not in finished]
+
organization = await database_sync_to_async(lambda: team.organization)()
for output in outputs:
capture_pipeline_stage("signal_data_source_entered", team, organization, output)
@@ -725,6 +735,14 @@ async def run_signal_pipeline(
threshold=threshold,
extra=extra,
)
+ summarized_ids = {output.signal_id for output in outputs}
+ for original in pre_summary_by_id.values():
+ if original.signal_id and original.signal_id not in summarized_ids:
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=team.id,
+ signal_id=original.signal_id,
+ billing_exempt=system_source_exempt(original.source_product),
+ )
for output in outputs:
pre = pre_summary_by_id.get(output.source_id)
if pre is not None and len(pre.description) > threshold:
@@ -756,6 +774,12 @@ async def run_signal_pipeline(
)
for source_id, output in pre_filter_by_id.items():
if source_id not in post_filter_ids:
+ if output.signal_id:
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=team.id,
+ signal_id=output.signal_id,
+ billing_exempt=system_source_exempt(output.source_product),
+ )
capture_pipeline_stage(
"signal_data_source_filtered", team, organization, output, {"steering_applied": steering.active}
)
diff --git a/products/signals/backend/emission/tests/test_direct_gate.py b/products/signals/backend/emission/tests/test_direct_gate.py
index 4b2e5b8f7f42..0eaeea4d54df 100644
--- a/products/signals/backend/emission/tests/test_direct_gate.py
+++ b/products/signals/backend/emission/tests/test_direct_gate.py
@@ -172,6 +172,8 @@ class TestEmitSignalWiring:
patch(f"{FACADE_MODULE_PATH}.SignalSourceConfig") as source_config,
patch(f"{FACADE_MODULE_PATH}.posthoganalytics"),
patch(f"{FACADE_MODULE_PATH}.async_connect", connect),
+ patch(f"{FACADE_MODULE_PATH}.complete_signal_pipeline"),
+ patch(f"{FACADE_MODULE_PATH}.finalized_signal_ids", return_value=set()),
patch(f"{GATE_MODULE_PATH}.steering_filters_signal", AsyncMock(return_value=True)) as gate,
):
source_config.is_source_enabled.return_value = True
diff --git a/products/signals/backend/emission/tests/test_emit_signals.py b/products/signals/backend/emission/tests/test_emit_signals.py
index 74db1f8c8ad1..046f0d262009 100644
--- a/products/signals/backend/emission/tests/test_emit_signals.py
+++ b/products/signals/backend/emission/tests/test_emit_signals.py
@@ -9,6 +9,7 @@ import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from django.test import override_settings
+from django.utils import timezone
import temporalio.worker
from temporalio import activity
@@ -17,6 +18,8 @@ from temporalio.worker import Worker
from posthog.hogql import ast
+from posthog.sync import database_sync_to_async
+
from products.ml_inference.backend.facade.contracts import DecisionResult, NoulAnswer
from products.signals.backend.emission._prompts import ISSUE_ACTIONABILITY_PROMPT
from products.signals.backend.emission.emit_signals import (
@@ -37,6 +40,8 @@ from products.signals.backend.emission.pipeline import (
summarize_long_descriptions,
)
from products.signals.backend.emission.registry import SignalEmitterOutput, SignalSourceTableConfig
+from products.signals.backend.models import SignalEmissionRecord, SignalSpend
+from products.signals.backend.spend import signal_id_for
PIPELINE_MODULE_PATH = "products.signals.backend.emission.pipeline"
FETCHER_MODULE_PATH = "products.signals.backend.emission.fetchers.data_warehouse"
@@ -368,11 +373,72 @@ class TestRunSignalPipelineEmitterFailures:
assert result == {"status": "success", "reason": "no_actionable_records", "signals_emitted": 0}
[email protected]_db(transaction=True)
class TestEmissionRetryIdentity:
@pytest.mark.asyncio
- async def test_activity_retries_keep_signal_id_but_later_imports_do_not(self):
+ @pytest.mark.parametrize("record_processed_outputs", [False, True])
+ async def test_retry_repairs_ledger_for_finalized_signal(self, team, record_processed_outputs: bool) -> None:
+ output = replace(_make_output(source_id="record-1"), idempotency_key="record-1")
+ signal_id = signal_id_for(
+ team_id=team.id,
+ source_product=output.source_product,
+ source_type=output.source_type,
+ idempotency_key="record-1",
+ )
+ await database_sync_to_async(
+ lambda: SignalSpend.objects.for_team(team.id).create(
+ team_id=team.id, signal_id=signal_id, kind=SignalSpend.Kind.AD_HOC, spend=1, finalized_at=timezone.now()
+ )
+ )()
+ config = _make_config(emitter=lambda team_id, record: output, record_processed_outputs=record_processed_outputs)
+ with patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit:
+ for _ in range(2):
+ result = await run_signal_pipeline(team=team, config=config, records=[{"id": "record-1"}], extra={})
+ assert result["signals_emitted"] == 0
+ emit.assert_not_awaited()
+ assert await SignalEmissionRecord.objects.filter(team=team, source_id="record-1").acount() == int(
+ record_processed_outputs
+ )
+
+ @pytest.mark.asyncio
+ async def test_retry_skips_filtered_signal_after_sibling_dispatch_failure(self, team):
+ checked = []
+
+ async def filter_outputs(*, outputs, **kwargs):
+ checked.append([output.source_id for output in outputs])
+ for output in outputs:
+ if output.source_id == "filtered":
+ await database_sync_to_async(
+ lambda output=output: SignalSpend.objects.for_team(team.id).create(
+ team_id=team.id, signal_id=output.signal_id, kind=SignalSpend.Kind.AD_HOC, spend=1
+ )
+ )()
+ return [output for output in outputs if output.source_id != "filtered"]
+
+ with (
+ patch(f"{PIPELINE_MODULE_PATH}.activity") as temporal_activity,
+ patch(f"{PIPELINE_MODULE_PATH}.filter_actionable", side_effect=filter_outputs),
+ patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit,
+ patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture"),
+ ):
+ temporal_activity.in_activity.return_value = True
+ temporal_activity.info.return_value.workflow_run_id = "import"
+ temporal_activity.info.return_value.activity_id = "emit"
+ records = [{"id": "filtered"}, {"id": "accepted"}]
+ config = _make_config(actionability_prompt="Filter {description}")
+ emit.side_effect = RuntimeError("dispatch failed")
+ with pytest.raises(RuntimeError):
+ await run_signal_pipeline(team=team, config=config, records=records, extra={})
+ filtered = await database_sync_to_async(lambda: SignalSpend.objects.for_team(team.id).get())()
+ assert filtered.finalized_at is not None
+ emit.side_effect = None
+ await run_signal_pipeline(team=team, config=config, records=records, extra={})
+ assert checked == [["filtered", "accepted"], ["accepted"]]
+ assert [call.kwargs["source_id"] for call in emit.call_args_list] == ["accepted", "accepted"]
+
+ @pytest.mark.asyncio
+ async def test_activity_retries_keep_signal_id_but_later_imports_do_not(self, team):
config = _make_config(record_processed_outputs=False)
- team = MagicMock(id=1)
with (
patch(f"{PIPELINE_MODULE_PATH}.activity") as temporal_activity,
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit,
@@ -395,12 +461,11 @@ class TestEmissionRetryIdentity:
assert emit.call_args.kwargs["signal_id"] != first
@pytest.mark.asyncio
- async def test_material_versions_dedupe_across_imports(self):
+ async def test_material_versions_dedupe_across_imports(self, team):
def emitter(team_id, record):
return replace(_make_output(source_id=record["id"]), idempotency_key=f"{record['id']}:{record['version']}")
config = _make_config(emitter=emitter, sync_with_source=True)
- team = MagicMock(id=1)
with (
patch(f"{PIPELINE_MODULE_PATH}.activity") as temporal_activity,
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit,
@@ -617,7 +682,7 @@ class TestFilterActionable:
with (
patch(f"{PIPELINE_MODULE_PATH}.build_async_anthropic_client", return_value=mock_client),
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
result = await filter_actionable(team, outputs, "prompt {description}", extra={})
@@ -714,7 +779,7 @@ class TestSummarizeLongDescriptions:
with (
patch(f"{PIPELINE_MODULE_PATH}.build_async_anthropic_client", return_value=mock_client),
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
result = await summarize_long_descriptions(team, [short, long], self.PROMPT, self.THRESHOLD, extra={})
@@ -739,7 +804,7 @@ class TestEmitSignals:
with (
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as mock_emit,
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
count = await _emit_signals(team=team, organization=MagicMock(), outputs=[output], extra={})
@@ -769,7 +834,7 @@ class TestEmitSignals:
with (
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit,
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
await _emit_signals(
team=MagicMock(),
@@ -795,7 +860,7 @@ class TestEmitSignals:
with (
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", side_effect=mock_emit),
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture") as capture,
):
count = await _emit_signals(team=MagicMock(), organization=MagicMock(), outputs=outputs, extra={})
@@ -822,7 +887,7 @@ class TestEmitSignals:
with (
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as mock_emit,
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
count = await _emit_signals(team=MagicMock(), organization=MagicMock(), outputs=[output], extra={})
@@ -837,7 +902,7 @@ class TestEmitSignals:
with (
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as mock_emit,
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
with pytest.raises(RuntimeError, match="All 1 signal emissions failed"):
await _emit_signals(team=MagicMock(), organization=MagicMock(), outputs=[output], extra={})
@@ -850,23 +915,22 @@ class TestEmitSignals:
with (
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", side_effect=Exception("boom")),
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
with pytest.raises(RuntimeError, match="All 2 signal emissions failed"):
await _emit_signals(team=MagicMock(), organization=MagicMock(), outputs=outputs, extra={})
[email protected]_db(transaction=True)
class TestExistingSourceOnlyOutputs:
@pytest.mark.asyncio
- async def test_terminal_source_only_emits_when_it_already_has_a_report(self):
+ async def test_terminal_source_only_emits_when_it_already_has_a_report(self, team):
config = _make_config(sync_with_source=True, actionability_prompt="Actionable? {description}")
records: list[dict[str, Any]] = [
{"id": "open", "state": "open"},
{"id": "known-closed", "state": "closed", "existing_source_only": True},
{"id": "new-closed", "state": "closed", "existing_source_only": True},
]
- team = MagicMock(id=1)
- team.uuid = uuid.UUID("00000000-0000-0000-0000-000000000001")
with (
patch(
f"{PIPELINE_MODULE_PATH}.fetch_source_signal_matches",
@@ -879,7 +943,7 @@ class TestExistingSourceOnlyOutputs:
side_effect=lambda **kwargs: kwargs["outputs"],
) as filter_actionable,
patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture") as capture,
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
):
result = await run_signal_pipeline(team=team, config=config, records=records, extra={})
@@ -895,11 +959,10 @@ class TestExistingSourceOnlyOutputs:
assert filtered[0].kwargs["properties"]["filter_reason"] == "source_state_without_report"
[email protected]_db(transaction=True)
class TestPipelineStageTelemetry:
@pytest.mark.asyncio
- async def test_captures_each_stage_per_signal(self):
- team = MagicMock(id=1)
- team.uuid = uuid.UUID("00000000-0000-0000-0000-000000000001")
+ async def test_captures_each_stage_per_signal(self, team):
short_description = "short actionable bug"
long_description = "x" * 500
@@ -945,7 +1008,7 @@ class TestPipelineStageTelemetry:
with (
patch(f"{PIPELINE_MODULE_PATH}.build_async_anthropic_client", return_value=mock_llm_client),
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock),
patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture") as capture,
):
@@ -969,11 +1032,10 @@ class TestPipelineStageTelemetry:
]
[email protected]_db(transaction=True)
class TestRunSignalPipelineSteering:
@pytest.mark.asyncio
- async def test_source_config_steering_customizes_actionability_prompt(self):
- team = MagicMock(id=1)
- team.uuid = uuid.UUID("00000000-0000-0000-0000-000000000001")
+ async def test_source_config_steering_customizes_actionability_prompt(self, team):
config = _make_config(actionability_prompt=ISSUE_ACTIONABILITY_PROMPT)
records = [{"id": "chore_1", "description": "Bump lodash to 4.17.21"}]
@@ -988,7 +1050,7 @@ class TestRunSignalPipelineSteering:
with (
patch(f"{PIPELINE_MODULE_PATH}.build_async_anthropic_client", return_value=mock_llm_client),
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock),
patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture") as capture,
):
@@ -1016,9 +1078,7 @@ class TestRunSignalPipelineSteering:
assert [call.kwargs["properties"]["steering_applied"] for call in filtered_calls] == [True]
@pytest.mark.asyncio
- async def test_without_source_config_prompt_is_unchanged(self):
- team = MagicMock(id=1)
- team.uuid = uuid.UUID("00000000-0000-0000-0000-000000000001")
+ async def test_without_source_config_prompt_is_unchanged(self, team):
config = _make_config(actionability_prompt=ISSUE_ACTIONABILITY_PROMPT)
records = [{"id": "issue_1", "description": "Export button broken"}]
@@ -1033,7 +1093,7 @@ class TestRunSignalPipelineSteering:
with (
patch(f"{PIPELINE_MODULE_PATH}.build_async_anthropic_client", return_value=mock_llm_client),
- patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ patch(f"{PIPELINE_MODULE_PATH}.activity.in_activity", return_value=False),
patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock),
patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture"),
):
diff --git a/products/signals/backend/facade/api.py b/products/signals/backend/facade/api.py
index 7eb64ac3f5df..314a5883193c 100644
--- a/products/signals/backend/facade/api.py
+++ b/products/signals/backend/facade/api.py
@@ -117,7 +117,8 @@ from products.signals.backend.scout_harness.workflow_runs import (
start_workflow_scout_run as start_workflow_scout_run,
)
from products.signals.backend.signal_metadata import SourceSliceSignalStats, fetch_signal_stats_for_source_slice
-from products.signals.backend.spend import signal_id_for
+from products.signals.backend.spend import complete_signal_pipeline, finalized_signal_ids, signal_id_for
+from products.signals.backend.spend_billing import system_source_exempt
# Re-exported for external products (tasks presentation catches it around facade create_task).
from products.signals.backend.task_run_artefacts import ReportTaskCapExceeded as ReportTaskCapExceeded
@@ -865,12 +866,20 @@ async def emit_signal(
organization = await database_sync_to_async(lambda: team.organization)()
if not organization.is_ai_data_processing_approved:
+ if signal_id:
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=team.id, signal_id=signal_id, billing_exempt=system_source_exempt(source_product)
+ )
return
is_enabled = await database_sync_to_async(SignalSourceConfig.is_source_enabled, thread_sensitive=False)(
team.id, source_product, source_type
)
if not is_enabled:
+ if signal_id:
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=team.id, signal_id=signal_id, billing_exempt=system_source_exempt(source_product)
+ )
return
description_tokens = _token_count(description)
@@ -930,6 +939,8 @@ async def emit_signal(
signal_id = signal_id or signal_id_for(
team_id=team.id, source_product=source_product, source_type=source_type, idempotency_key=idempotency_key
)
+ if await database_sync_to_async(finalized_signal_ids)(team_id=team.id, signal_ids=[signal_id]):
+ return
# Below the started event on purpose: a filtered signal then has a top-of-funnel event to be
# counted against, so a steering drop reads apart from a dispatch failure rather than as one.
@@ -949,6 +960,9 @@ async def emit_signal(
weight=weight,
extra=extra or {},
):
+ await database_sync_to_async(complete_signal_pipeline, thread_sensitive=False)(
+ team_id=team.id, signal_id=signal_id, billing_exempt=system_source_exempt(source_product)
+ )
return
client = await async_connect()
diff --git a/products/signals/backend/facade/spend.py b/products/signals/backend/facade/spend.py
index 714766d22d31..1822884d2042 100644
--- a/products/signals/backend/facade/spend.py
+++ b/products/signals/backend/facade/spend.py
@@ -1 +1,4 @@
-from products.signals.backend.spend import complete_task_spend as complete_task_spend
+from products.signals.backend.spend import (
+ complete_task_spend as complete_task_spend,
+ register_report_task_spend as register_report_task_spend,
+)
diff --git a/products/signals/backend/implementation_dispatch.py b/products/signals/backend/implementation_dispatch.py
index ab438f022d5c..7a1a99b46799 100644
--- a/products/signals/backend/implementation_dispatch.py
+++ b/products/signals/backend/implementation_dispatch.py
@@ -180,7 +180,15 @@ class ImplementationDispatcher:
team_id=team_id, report_id=str(decision.report_id), attribution=ArtefactAttribution.system()
)
outcome = async_to_sync(maybe_autostart_from_report_artefacts)(
- team_id=team_id, report_id=str(decision.report_id), dispatch=reservation, ch_user=ClickHouseUser.SIGNALS
+ team_id=team_id,
+ report_id=str(decision.report_id),
+ dispatch=reservation,
+ ch_user=ClickHouseUser.SIGNALS,
+ implementation_spend_kind=(
+ "scout_triggered_implementation"
+ if reservation.source_skill
+ else "reviewer_triggered_implementation"
+ ),
)
except Exception:
logger.exception("signals_implementation_dispatch_failed", decision_id=decision_id)
diff --git a/products/signals/backend/llm_spend.py b/products/signals/backend/llm_spend.py
index 390b15db9149..2279cbd48b3d 100644
--- a/products/signals/backend/llm_spend.py
+++ b/products/signals/backend/llm_spend.py
@@ -75,7 +75,6 @@ async def _settle_generation(
return None
finally:
await database_sync_to_async(record_llm_spend)(
- request_id,
token_cost_microusd=cost_microusd,
team_id=team_id,
signal_id=signal_id,
@@ -107,7 +106,7 @@ async def generate_with_spend(
await _settle_generation(request_id, gateway=gateway, team_id=team_id, signal_id=signal_id, stage=stage)
else:
await database_sync_to_async(record_llm_spend)(
- None, token_cost_microusd=None, team_id=team_id, signal_id=signal_id, stage=stage
+ token_cost_microusd=None, team_id=team_id, signal_id=signal_id, stage=stage
)
raise
spend = await _settle_generation(
diff --git a/products/signals/backend/management/commands/seed_refund_test_data.py b/products/signals/backend/management/commands/seed_refund_test_data.py
index 8de1a88d42ab..6ec8ec7a935d 100644
--- a/products/signals/backend/management/commands/seed_refund_test_data.py
+++ b/products/signals/backend/management/commands/seed_refund_test_data.py
@@ -131,6 +131,7 @@ class Command(BaseCommand):
user_id=user_id,
repository=repository,
create_pr=False,
+ ai_stage="implementation",
start_workflow=False,
signal_report_id=str(report.id),
internal=False,
@@ -162,7 +163,7 @@ class Command(BaseCommand):
if run_created_at is not None:
tasks_facade.set_task_run_created_at_for_seeding(run_id, task_id, team.id, created_at=run_created_at)
- billable = first_billable_pr_run(report.id)
+ billable = first_billable_pr_run(report.id, team_id=report.team_id)
if billable is None:
raise CommandError(f"Report {report.id} is not billable after seeding — bridge/run wiring broke")
self.stdout.write(f" · billable at {billable.created_at.isoformat()} ({billable.pr_url})")
diff --git a/products/signals/backend/migrations/0208_reset_signal_spend.py b/products/signals/backend/migrations/0208_reset_signal_spend.py
new file mode 100644
index 000000000000..d0f5adb4cb58
--- /dev/null
+++ b/products/signals/backend/migrations/0208_reset_signal_spend.py
@@ -0,0 +1,188 @@
+# Generated by Django 5.2.18 on 2026-10-26 14:46
+
+import django.db.models.deletion
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+from posthog.migration_helpers import untrack_field
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "1413_event_filter_pipeline_override"),
+ ("signals", "0207_backfill_report_link_targets"),
+ ]
+
+ # Receipt history is intentionally discarded for the new spend model.
+ operations = [
+ migrations.RunSQL("TRUNCATE TABLE signals_signalspend"),
+ migrations.RemoveConstraint(
+ model_name="signalspend",
+ name="signals_spend_source_unique",
+ ),
+ migrations.RemoveConstraint(
+ model_name="signalspend",
+ name="signals_spend_one_owner",
+ ),
+ migrations.RemoveConstraint(
+ model_name="signalspend",
+ name="signals_spend_nonnegative",
+ ),
+ # Physical column drops must follow a deployment with these fields absent from Django.
+ migrations.AlterField(
+ model_name="signalspend", name="is_task", field=models.BooleanField(default=False, null=True)
+ ),
+ migrations.AlterField(
+ model_name="signalspend", name="source_id", field=models.CharField(max_length=255, null=True)
+ ),
+ migrations.AlterField(
+ model_name="signalspend", name="stage", field=models.CharField(default="unknown", max_length=100, null=True)
+ ),
+ untrack_field("signalspend", "is_task", "source_id", "stage", "task_id", "token_cost_microusd"),
+ migrations.AddField(
+ model_name="signalspend",
+ name="billing_exempt",
+ field=models.BooleanField(default=False, db_default=False),
+ ),
+ migrations.AddField(
+ model_name="signalspend",
+ name="finalized_at",
+ field=models.DateTimeField(blank=True, db_index=True, null=True),
+ ),
+ migrations.AddField(
+ model_name="signalspend",
+ name="kind",
+ field=models.CharField(
+ choices=[
+ ("research", "Research"),
+ ("repo_selection", "Repo Selection"),
+ ("ad-hoc", "Ad Hoc"),
+ ("scout_run", "Scout Run"),
+ (
+ "reviewer_triggered_implementation",
+ "Reviewer Triggered Implementation",
+ ),
+ (
+ "scout_triggered_implementation",
+ "Scout Triggered Implementation",
+ ),
+ ],
+ null=True,
+ max_length=40,
+ ),
+ ),
+ migrations.AddField(
+ model_name="signalspend",
+ name="llm_failed_stages",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.CharField(max_length=100), default=list, db_default=[], size=None
+ ),
+ ),
+ migrations.AddField(
+ model_name="signalspend",
+ name="report",
+ field=models.ForeignKey(
+ blank=True,
+ db_constraint=False,
+ null=True,
+ on_delete=django.db.models.deletion.DO_NOTHING,
+ related_name="+",
+ to="signals.signalreport",
+ ),
+ ),
+ migrations.AddField(
+ model_name="signalspend",
+ name="spend",
+ field=models.DecimalField(
+ decimal_places=4,
+ default=0,
+ db_default=0,
+ help_text="Customer spend in USD cents",
+ max_digits=20,
+ ),
+ ),
+ migrations.AddField(
+ model_name="signalspend",
+ name="task_run_id",
+ field=models.UUIDField(blank=True, null=True, unique=True),
+ ),
+ migrations.AlterField(
+ model_name="signalspend",
+ name="scout_run",
+ field=models.ForeignKey(
+ blank=True,
+ db_constraint=False,
+ null=True,
+ on_delete=django.db.models.deletion.DO_NOTHING,
+ related_name="+",
+ to="signals.signalscoutrun",
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="signalspend",
+ constraint=models.UniqueConstraint(
+ condition=models.Q(("kind", "ad-hoc")),
+ fields=("team", "signal_id"),
+ name="signals_owner_spend_ad_hoc",
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="signalspend",
+ constraint=models.CheckConstraint(
+ condition=(
+ models.Q(kind__isnull=False)
+ & models.Q(
+ models.Q(
+ ("report__isnull", True),
+ ("scout_run__isnull", True),
+ ("signal_id__isnull", False),
+ models.Q(
+ models.Q(("kind", "ad-hoc"), ("task_run_id__isnull", True)),
+ models.Q(
+ ("kind__in", ["research", "repo_selection"]),
+ ("task_run_id__isnull", False),
+ ),
+ _connector="OR",
+ ),
+ ),
+ models.Q(
+ ("kind", "scout_run"),
+ ("report__isnull", True),
+ ("scout_run__isnull", False),
+ ("signal_id__isnull", True),
+ ("task_run_id__isnull", False),
+ ),
+ models.Q(
+ (
+ "kind__in",
+ [
+ "reviewer_triggered_implementation",
+ "scout_triggered_implementation",
+ ],
+ ),
+ ("report__isnull", False),
+ ("scout_run__isnull", True),
+ ("signal_id__isnull", True),
+ ("task_run_id__isnull", False),
+ ),
+ _connector="OR",
+ )
+ | models.Q(
+ kind__isnull=True,
+ task_run_id__isnull=True,
+ report__isnull=True,
+ finalized_at__isnull=True,
+ spend=0,
+ )
+ ),
+ name="signals_owner_spend_kind_owner",
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="signalspend",
+ constraint=models.CheckConstraint(
+ condition=models.Q(("spend__gte", 0)),
+ name="signals_owner_spend_nonnegative",
+ ),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 815a011c5b13..552dce6c4f51 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0207_backfill_report_link_targets
+0208_reset_signal_spend
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 8cbaf6f03d8d..68519c750780 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -3958,34 +3958,77 @@ class SignalScoutBackgroundBand(TeamScopedRootMixin, UUIDModel):
class SignalSpend(TeamScopedRootMixin, UUIDModel):
- """One absolute cost observation per gateway request or task run."""
+ class Kind(models.TextChoices):
+ RESEARCH = "research"
+ REPO_SELECTION = "repo_selection"
+ AD_HOC = "ad-hoc"
+ SCOUT_RUN = "scout_run"
+ REVIEWER_TRIGGERED_IMPLEMENTATION = "reviewer_triggered_implementation"
+ SCOUT_TRIGGERED_IMPLEMENTATION = "scout_triggered_implementation"
team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")
+ # Old workers omit kind during rolling deploys; their receipts remain unbillable.
+ kind = models.CharField(max_length=40, choices=Kind.choices, null=True)
signal_id = models.UUIDField(null=True, blank=True)
- scout_run = models.ForeignKey(SignalScoutRun, on_delete=models.CASCADE, null=True, blank=True)
- task_id = models.UUIDField(null=True, blank=True, db_index=True)
- source_id = models.CharField(max_length=255)
- stage = models.CharField(max_length=100, default="unknown")
+ scout_run = models.ForeignKey(
+ SignalScoutRun, on_delete=models.DO_NOTHING, db_constraint=False, null=True, blank=True, related_name="+"
+ )
+ # Only implementation spend is report-owned; signals and scout runs drive all other spend.
+ report = models.ForeignKey(
+ SignalReport, on_delete=models.DO_NOTHING, db_constraint=False, null=True, blank=True, related_name="+"
+ )
+ task_run_id = models.UUIDField(null=True, blank=True, unique=True)
+ llm_failed_stages = ArrayField(models.CharField(max_length=100), default=list, db_default=[])
+ spend = models.DecimalField(
+ max_digits=20, decimal_places=4, default=0, db_default=0, help_text="Customer spend in USD cents"
+ )
accounting_failed = models.BooleanField(default=False)
- is_task = models.BooleanField(default=False)
- token_cost_microusd = models.BigIntegerField(null=True, blank=True)
- updated_at = models.DateTimeField(auto_now=True)
+ finalized_at = models.DateTimeField(null=True, blank=True, db_index=True)
+ billing_exempt = models.BooleanField(default=False, db_default=False)
created_at = models.DateTimeField(auto_now_add=True)
+ updated_at = models.DateTimeField(auto_now=True)
class Meta:
+ indexes = [models.Index(fields=["team", "signal_id"], name="signals_spend_signal_idx")]
constraints = [
- models.UniqueConstraint(fields=["team", "source_id", "is_task"], name="signals_spend_source_unique"),
+ models.UniqueConstraint(
+ fields=["team", "signal_id"], condition=models.Q(kind="ad-hoc"), name="signals_owner_spend_ad_hoc"
+ ),
models.CheckConstraint(
condition=(
- models.Q(signal_id__isnull=False, scout_run__isnull=True)
- | models.Q(signal_id__isnull=True, scout_run__isnull=False)
+ models.Q(kind__isnull=False)
+ & (
+ models.Q(signal_id__isnull=False, scout_run__isnull=True, report__isnull=True)
+ & (
+ models.Q(kind="ad-hoc", task_run_id__isnull=True)
+ | models.Q(kind__in=["research", "repo_selection"], task_run_id__isnull=False)
+ )
+ | models.Q(
+ kind="scout_run",
+ scout_run__isnull=False,
+ signal_id__isnull=True,
+ report__isnull=True,
+ task_run_id__isnull=False,
+ )
+ | models.Q(
+ kind__in=["reviewer_triggered_implementation", "scout_triggered_implementation"],
+ report__isnull=False,
+ signal_id__isnull=True,
+ scout_run__isnull=True,
+ task_run_id__isnull=False,
+ )
+ )
+ | models.Q(
+ kind__isnull=True,
+ task_run_id__isnull=True,
+ report__isnull=True,
+ finalized_at__isnull=True,
+ spend=0,
+ )
),
- name="signals_spend_one_owner",
+ name="signals_owner_spend_kind_owner",
),
- models.CheckConstraint(condition=models.Q(token_cost_microusd__gte=0), name="signals_spend_nonnegative"),
- ]
- indexes = [
- models.Index(fields=["team", "signal_id"], name="signals_spend_signal_idx"),
+ models.CheckConstraint(condition=models.Q(spend__gte=0), name="signals_owner_spend_nonnegative"),
]
diff --git a/products/signals/backend/report_generation/research.py b/products/signals/backend/report_generation/research.py
index ab063570f40f..07ba43d8bf3c 100644
--- a/products/signals/backend/report_generation/research.py
+++ b/products/signals/backend/report_generation/research.py
@@ -1605,6 +1605,7 @@ async def run_multi_turn_research(
verbose: bool = False,
output_fn: OutputFn = None,
signal_report_id: str | None = None,
+ spend_signal_id: str | None = None,
has_business_knowledge: bool = False,
resolved_report_title: str | None = None,
resolved_report_summary: str | None = None,
@@ -1668,6 +1669,7 @@ async def run_multi_turn_research(
output_fn=output_fn,
origin_product=tasks_facade.TaskOriginProduct.SIGNAL_REPORT,
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
ai_stage=AI_STAGE_RESEARCH,
internal=True,
analytics_query_context=(
diff --git a/products/signals/backend/report_generation/select_repo.py b/products/signals/backend/report_generation/select_repo.py
index d4e7975556c2..1e6a6f03d6e9 100644
--- a/products/signals/backend/report_generation/select_repo.py
+++ b/products/signals/backend/report_generation/select_repo.py
@@ -74,6 +74,7 @@ async def select_repository_for_team(
*,
step_name: str = "repo_selection",
signal_report_id: str | None = None,
+ spend_signal_id: str | None = None,
sandbox_environment_id: str | None = None,
verbose: bool = False,
output_fn: OutputFn = None,
@@ -108,6 +109,7 @@ async def select_repository_for_team(
priority=Priority.BATCH,
step_name=step_name,
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
sandbox_environment_id=sandbox_environment_id,
verbose=verbose,
output_fn=output_fn,
@@ -144,6 +146,7 @@ async def select_repository_for_report(
signals: list[SignalData],
*,
signal_report_id: str | None = None,
+ spend_signal_id: str | None = None,
sandbox_environment_id: str | None = None,
verbose: bool = False,
output_fn: OutputFn = None,
@@ -163,6 +166,7 @@ async def select_repository_for_report(
request_section,
step_name="repo_selection",
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
sandbox_environment_id=sandbox_environment_id,
verbose=verbose,
output_fn=output_fn,
diff --git a/products/signals/backend/report_task_actions.py b/products/signals/backend/report_task_actions.py
index 249b4f6fe2af..d4597891cca3 100644
--- a/products/signals/backend/report_task_actions.py
+++ b/products/signals/backend/report_task_actions.py
@@ -29,7 +29,6 @@ from posthog.utils import absolute_uri
from products.signals.backend.artefact_schemas import SIGNALS_PRODUCT, TASK_RUN_TYPE_DISCUSSION
from products.signals.backend.enums import ReportTaskActionKind
-from products.signals.backend.free_trial import FREE_TRIAL_PR_MESSAGE, self_driving_free_trial_enabled
from products.signals.backend.implementation_pr import ImplementationPr, fetch_implementation_prs_for_reports
from products.signals.backend.models import SignalReport, SignalReportArtefact
from products.signals.backend.pr_ci_status import pr_ci_statuses_for_references, rollup_ci_status
@@ -238,9 +237,6 @@ def get_it_merged_action(
disabled_reason = GET_IT_MERGED_NO_PR_DISABLED_REASON
if disabled_reason is None and not context.pr_actions_flag_enabled():
disabled_reason = GET_IT_MERGED_FLAG_DISABLED_REASON
- # The merge pushes to the PR, so a trial org gets the same refusal as Create PR.
- if disabled_reason is None and self_driving_free_trial_enabled(context.team):
- disabled_reason = FREE_TRIAL_PR_MESSAGE
url = action_report_url(report)
return ReportTaskAction(
kind=ReportTaskActionKind.GET_IT_MERGED,
@@ -262,9 +258,6 @@ def make_it_green_action(
disabled_reason = MAKE_IT_GREEN_FLAG_DISABLED_REASON
if disabled_reason is None and not _open_pull_requests_ci_failing(report.team_id, list(state.pull_requests)):
disabled_reason = MAKE_IT_GREEN_CI_NOT_FAILING_DISABLED_REASON
- # The fix pushes to the PR, so a trial org gets the same refusal as Create PR.
- if disabled_reason is None and self_driving_free_trial_enabled(context.team):
- disabled_reason = FREE_TRIAL_PR_MESSAGE
return ReportTaskAction(
kind=ReportTaskActionKind.MAKE_IT_GREEN,
disabled_reason=disabled_reason,
diff --git a/products/signals/backend/scout_harness/limits.py b/products/signals/backend/scout_harness/limits.py
index 10ffc737c82c..7bd0a9053fe8 100644
--- a/products/signals/backend/scout_harness/limits.py
+++ b/products/signals/backend/scout_harness/limits.py
@@ -1,34 +1,15 @@
from __future__ import annotations
-# A regular scout run's hard runtime cap. Enforced via the Temporal activity's
-# `start_to_close_timeout` in `scout_scheduler.py` — if the agent is still going
-# at this point, the activity is killed and the run row is marked failed by the
-# bridge. Also passed to `MultiTurnSession` as the per-turn poll budget
-# (`max_poll_seconds`) so the dropped-finalization salvage fires before the activity's
-# timeout — keep it below `WORKFLOW_HARD_CEILING_S`. Tuning this is a config decision,
-# not a per-run override knob.
+# Keep the agent polling budget below the activity timeout so teardown can finish.
DEFAULT_MAX_RUNTIME_S = 20 * 60
TRIGGERED_BY_ONBOARDING = "onboarding"
-# Slack added on top of `DEFAULT_MAX_RUNTIME_S` for the Temporal activity
-# `start_to_close_timeout`, so heartbeat-based failures get a chance to surface
-# before Temporal's own timeout fires.
-ACTIVITY_SLACK_S = 60
+# Task teardown can wait up to 15 minutes for spend settlement after the agent stops.
+ACTIVITY_SLACK_S = 15 * 60 + 60
-# Hard ceiling on how long a regular scout activity can actually be running. The
-# workflow sets `start_to_close_timeout = DEFAULT_MAX_RUNTIME_S + ACTIVITY_SLACK_S`,
-# providing a heartbeat window before Temporal's own timeout fires. The stale-RUNNING
-# self-heal in `runner.py` uses this as the staleness base.
WORKFLOW_HARD_CEILING_S = DEFAULT_MAX_RUNTIME_S + ACTIVITY_SLACK_S
-# Age past which an in-flight scout run is treated as orphaned and reaped by the
-# stale-run self-heal in `runner.py`. A run older than the activity's hard ceiling cannot
-# still be legitimately executing — Temporal kills the activity at `WORKFLOW_HARD_CEILING_S`
-# — so a `QUEUED`/`IN_PROGRESS` TaskRun past this cutoff is an orphan left behind by a
-# crashed worker/sandbox that never wrote a terminal status. Set to a generous multiple of
-# the ceiling so a run merely at the wall (about to fail or finish) is never reaped out from
-# under itself; a lane blocked by an orphan then self-clears within one or two coordinator
-# ticks.
+# Leave a margin beyond the activity timeout before reaping an unfinished run.
STALE_RUN_CUTOFF_S = 2 * WORKFLOW_HARD_CEILING_S
# How many of a lane's newest runs the self-heal checks for orphans. An orphan is a run stuck in
diff --git a/products/signals/backend/scout_harness/runner.py b/products/signals/backend/scout_harness/runner.py
index 326b60e04e0a..bcbcd71ab973 100644
--- a/products/signals/backend/scout_harness/runner.py
+++ b/products/signals/backend/scout_harness/runner.py
@@ -7,7 +7,7 @@ from collections import deque
from contextlib import nullcontext
from dataclasses import dataclass, replace
from datetime import UTC, datetime, timedelta
-from functools import lru_cache
+from functools import lru_cache, partial
from typing import Any
from uuid import UUID
@@ -86,6 +86,7 @@ from products.signals.backend.scout_harness.trial_launch import (
from products.signals.backend.scout_harness.trial_result import export_trial_result, validate_trial_runtime
from products.signals.backend.scout_harness.trial_state import SCOUT_TRIAL_METADATA_KEY, initial_trial_state
from products.signals.backend.scout_report import scout_report_tags_in_use
+from products.signals.backend.spend import record_task_spend
from products.signals.backend.temporal.agentic import (
SIGNALS_REPORT_RESEARCH_ENV_NAME,
get_or_create_signals_sandbox_env,
@@ -1367,6 +1368,10 @@ def _self_heal_stale_runs(team_id: int, skill_name: str) -> None:
)
if not stale_runs:
return
+ from products.signals.backend.tasks import (
+ settle_scout_spend, # noqa: PLC0415 - avoids loading the Celery task registry when importing the runner
+ )
+
# Resolve the team once, only when there is actually something to reap, so the reaped
# event carries the same team / groups shape as the other scout lifecycle events.
team = _get_team(team_id)
@@ -1414,6 +1419,7 @@ def _self_heal_stale_runs(team_id: int, skill_name: str) -> None:
status_before=status_before,
age_seconds=age_seconds,
)
+ transaction.on_commit(partial(settle_scout_spend.delay, team_id=team_id, run_id=str(task_run.id)))
except Exception:
logger.exception(
"signals_scout: failed to reap stale in-progress run; continuing",
@@ -1575,16 +1581,19 @@ def _create_run_row(
# by it. Absent when no pre-check ran.
if precheck_query_source:
metadata["precheck_query_source"] = precheck_query_source
- return SignalScoutRun.objects.unscoped().create(
- id=run_id,
- task_run_id=task_run_id,
- team=team,
- scout_config=config,
- skill_name=skill.name,
- skill_version=skill.version,
- metadata=metadata,
- trial_state=initial_trial_state() if trial is not None else None,
- )
+ with transaction.atomic():
+ run = SignalScoutRun.objects.for_team(team.id).create(
+ id=run_id,
+ task_run_id=task_run_id,
+ team=team,
+ scout_config=config,
+ skill_name=skill.name,
+ skill_version=skill.version,
+ metadata=metadata,
+ trial_state=initial_trial_state() if trial is not None else None,
+ )
+ record_task_spend(team_id=team.id, run_id=str(task_run_id), scout_run_id=str(run.id), stage="scout")
+ return run
@dataclass(frozen=True)
diff --git a/products/signals/backend/scout_harness/tools/report.py b/products/signals/backend/scout_harness/tools/report.py
index 6f288228867c..0c8b01923e97 100644
--- a/products/signals/backend/scout_harness/tools/report.py
+++ b/products/signals/backend/scout_harness/tools/report.py
@@ -1567,7 +1567,9 @@ async def _maybe_autostart_report(*, team_id: int, report_id: str) -> None:
)
try:
- await maybe_autostart_from_report_artefacts(team_id=team_id, report_id=report_id)
+ await maybe_autostart_from_report_artefacts(
+ team_id=team_id, report_id=report_id, implementation_spend_kind="scout_triggered_implementation"
+ )
except Exception:
logger.exception("signals_scout.emit_report: autostart failed", extra={"report_id": report_id})
diff --git a/products/signals/backend/scout_harness/trial_state.py b/products/signals/backend/scout_harness/trial_state.py
index 7a2d57ed9a4e..cc4d32d29622 100644
--- a/products/signals/backend/scout_harness/trial_state.py
+++ b/products/signals/backend/scout_harness/trial_state.py
@@ -135,12 +135,14 @@ class ScoutTrialStore:
) -> _Result:
invalid_reason: str | None = None
with transaction.atomic():
- locked = (
+ runs = (
SignalScoutRun.objects.for_team(self.run.team_id)
.select_related("task_run")
- .select_for_update(of=("self", "task_run"))
- .get(pk=self.run.pk, task_run_id=self.run.task_run_id)
+ .filter(pk=self.run.pk, task_run_id=self.run.task_run_id)
)
+ # Gateway accounting locks the task run before updating scout totals; match its order to avoid deadlocks.
+ runs.select_for_update(of=("task_run",)).get()
+ locked = runs.select_for_update(of=("self",)).get()
if not is_scout_trial(locked) or locked.task_run.team_id != self.run.team_id:
raise ScoutTrialStateError("The scout run has no private state.")
if not allow_terminal and locked.task_run.status != "in_progress":
diff --git a/products/signals/backend/spend.py b/products/signals/backend/spend.py
index ede997b3e69f..48333693a0ae 100644
--- a/products/signals/backend/spend.py
+++ b/products/signals/backend/spend.py
@@ -1,31 +1,26 @@
+from __future__ import annotations
+
import asyncio
+from collections.abc import Sequence
from dataclasses import field
from datetime import timedelta
+from decimal import Decimal
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
from django.conf import settings
-from django.contrib.postgres.aggregates import ArrayAgg
from django.db import transaction
-from django.db.models import Count, Q, Sum
+from django.db.models import F, Func, JSONField, Value
+from django.db.models.functions import Coalesce
from django.utils import timezone
-import structlog
from asgiref.sync import async_to_sync
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.dataclasses import frozen
-from posthog.sync import database_sync_to_async
-
-from products.signals.backend.models import (
- SignalReport,
- SignalReportArtefact,
- SignalReportTask,
- SignalScoutRun,
- SignalSpend,
-)
-from products.signals.backend.pricing import cost_to_spend
-logger = structlog.get_logger(__name__)
+from products.signals.backend.models import SignalReport, SignalScoutRun, SignalSpend
+from products.signals.backend.pricing import cost_to_spend
+from products.signals.backend.spend_billing import apply_report_billing, scout_spend_exempt, trial_spend_exempt
@frozen
@@ -42,13 +37,21 @@ def signal_id_for(*, team_id: int, source_product: str, source_type: str, idempo
)
+def finalized_signal_ids(*, team_id: int, signal_ids: Sequence[str]) -> set[str]:
+ if not signal_ids:
+ return set()
+ return {
+ str(signal_id)
+ for signal_id in SignalSpend.objects.for_team(team_id)
+ .filter(kind=SignalSpend.Kind.AD_HOC, signal_id__in=signal_ids, finalized_at__isnull=False)
+ .values_list("signal_id", flat=True)
+ }
+
+
def report_triggering_signal(
*, team_id: int, report_id: str, ch_user: ClickHouseUser = ClickHouseUser.DEFAULT
) -> str | None:
- with transaction.atomic():
- report = (
- SignalReport.objects.filter(team_id=team_id, id=report_id).only("triggering_signal_id", "team_id").first()
- )
+ report = SignalReport.objects.filter(team_id=team_id, id=report_id).only("triggering_signal_id", "team_id").first()
if report is None:
return None
if report.triggering_signal_id is not None:
@@ -59,7 +62,7 @@ def report_triggering_signal(
)
signals = fetch_signals_for_report_sync(report.team, report_id, newest=1, ch_user=ch_user)
- if not signals:
+ if not signals or finalized_signal_ids(team_id=team_id, signal_ids=[signals[0]["signal_id"]]):
return None
with transaction.atomic():
reports = SignalReport.objects.filter(team_id=team_id, id=report_id)
@@ -95,7 +98,6 @@ async def submit_signal_metadata_updates(
team_id: int,
signal_ids: list[str],
change: SignalMetadataChange | None = None,
- wait_for_completion: bool = False,
) -> None:
from temporalio.common import WorkflowIDReusePolicy # noqa: PLC0415 — keeps Temporal off Django startup
@@ -105,7 +107,7 @@ async def submit_signal_metadata_updates(
return
client = await asyncio.wait_for(async_connect(), timeout=5)
for start in range(0, len(signal_ids), 20):
- handles = await asyncio.gather(
+ await asyncio.gather(
*(
client.start_workflow(
"update-signal-spend",
@@ -115,27 +117,33 @@ async def submit_signal_metadata_updates(
id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,
start_signal="update_metadata" if change else "refresh",
start_signal_args=[change] if change else [],
- execution_timeout=timedelta(minutes=15),
rpc_timeout=timedelta(seconds=5),
)
for signal_id in signal_ids[start : start + 20]
)
)
- if wait_for_completion:
- await asyncio.gather(*(handle.result() for handle in handles))
-async def publish_signal_spend(*, team_id: int, signal_id: str) -> None:
- try:
- await submit_signal_metadata_updates(team_id=team_id, signal_ids=[signal_id], wait_for_completion=True)
- except Exception:
- logger.exception(
- "signals.spend.accounting_failed", stage="persist_metadata", team_id=team_id, signal_id=signal_id
- )
+def _locked_ad_hoc_spend(*, team_id: int, signal_id: str) -> SignalSpend:
+ spend, _ = SignalSpend.objects.for_team(team_id).get_or_create(
+ team_id=team_id,
+ signal_id=signal_id,
+ kind=SignalSpend.Kind.AD_HOC,
+ defaults={"billing_exempt": lambda: trial_spend_exempt(team_id)},
+ )
+ return SignalSpend.objects.for_team(team_id).select_for_update().get(id=spend.id)
+
+
+def _save_ad_hoc_spend(spend: SignalSpend, *, finalize: bool = False) -> None:
+ if spend.finalized_at is not None:
+ return
+ spend.accounting_failed = bool(spend.llm_failed_stages)
+ if finalize and not spend.accounting_failed:
+ spend.finalized_at = timezone.now()
+ spend.save()
def record_llm_spend(
- request_id: str | None,
*,
token_cost_microusd: int | None,
team_id: int | None,
@@ -144,106 +152,56 @@ def record_llm_spend(
) -> None:
if team_id is None or signal_id is None:
return
- try:
- with transaction.atomic():
- spend, created = SignalSpend.objects.for_team(team_id).get_or_create(
- team_id=team_id,
- source_id=request_id or f"unknown-{uuid4()}",
- is_task=False,
- defaults={
- "signal_id": signal_id,
- "stage": stage,
- "token_cost_microusd": token_cost_microusd,
- "accounting_failed": token_cost_microusd is None,
- },
- )
- if not created and token_cost_microusd is not None:
- SignalSpend.objects.for_team(team_id).filter(id=spend.id).update(
- token_cost_microusd=token_cost_microusd,
- accounting_failed=False,
- updated_at=timezone.now(),
- )
+ with transaction.atomic():
+ spend = _locked_ad_hoc_spend(team_id=team_id, signal_id=signal_id)
+ if spend.finalized_at is not None:
+ raise ValueError("Cannot add generation costs to finalized spend")
if token_cost_microusd is None:
- logger.warning("signals.spend.generation_unaccounted", stage=stage, team_id=team_id, signal_id=signal_id)
- except Exception:
- logger.exception(
- "signals.spend.accounting_failed", stage=stage, team_id=team_id, signal_id=signal_id, request_id=request_id
- )
+ spend.llm_failed_stages = sorted(set(spend.llm_failed_stages) | {stage})
+ else:
+ spend.spend += cost_to_spend(token_cost_microusd)
+ _save_ad_hoc_spend(spend)
def signal_spend_summaries(*, team_id: int, signal_ids: list[str]) -> dict[str, SignalSpendSummary]:
- valid_ids = []
- for signal_id in signal_ids:
+ ids = []
+ for signal_id in dict.fromkeys(signal_ids):
try:
- valid_ids.append(UUID(signal_id))
+ ids.append(UUID(signal_id))
except ValueError:
continue
- if not valid_ids:
- return {}
- try:
- with transaction.atomic():
- rows = (
- SignalSpend.objects.for_team(team_id)
- .filter(signal_id__in=valid_ids)
- .values("signal_id")
- .annotate(
- cost=Sum("token_cost_microusd"),
- tasks=Count("id", filter=Q(is_task=True)),
- failed_stages=ArrayAgg("stage", distinct=True, filter=Q(accounting_failed=True)),
- )
- )
- return {
- str(row["signal_id"]): SignalSpendSummary(
- total_spend=float(cost_to_spend(row["cost"] or 0, task_count=row["tasks"])),
- failed_stages=sorted(row["failed_stages"] or []),
- )
- for row in rows
- }
- except Exception:
- logger.exception("signals.spend.accounting_failed", stage="read_spend", team_id=team_id, signal_ids=signal_ids)
- return {}
+ amounts: dict[str, Decimal] = {}
+ failures: dict[str, set[str]] = {}
+ for spend in SignalSpend.objects.for_team(team_id).filter(signal_id__in=ids):
+ if spend.kind is None:
+ continue
+ key = str(spend.signal_id)
+ amounts[key] = amounts.get(key, Decimal(0)) + spend.spend
+ if spend.accounting_failed:
+ failures.setdefault(key, set()).update(spend.llm_failed_stages or [spend.kind])
+ return {
+ key: SignalSpendSummary(total_spend=float(amount), failed_stages=sorted(failures.get(key, set())))
+ for key, amount in amounts.items()
+ }
def signal_spend_totals(*, team_id: int, signal_ids: list[str]) -> dict[str, float]:
return {
- signal_id: summary.total_spend
- for signal_id, summary in signal_spend_summaries(team_id=team_id, signal_ids=signal_ids).items()
+ key: value.total_spend for key, value in signal_spend_summaries(team_id=team_id, signal_ids=signal_ids).items()
}
-def refresh_task_spend(*, spend_id: UUID, team_id: int) -> None:
- try:
- from products.tasks.backend.facade.billing import (
- get_task_run_token_usage, # noqa: PLC0415 — keeps task cost services off startup
- )
-
- with transaction.atomic():
- spend = SignalSpend.objects.for_team(team_id).select_for_update().get(id=spend_id)
- usage = get_task_run_token_usage(team_id=team_id, run_id=UUID(spend.source_id))
- if usage.cost_microusd is not None:
- spend.token_cost_microusd = usage.cost_microusd
- spend.accounting_failed = not usage.complete
- spend.save(update_fields=["token_cost_microusd", "accounting_failed", "updated_at"])
- if not usage.complete:
- logger.warning(
- "signals.spend.generation_unaccounted",
- stage=spend.stage,
- team_id=team_id,
- signal_id=str(spend.signal_id) if spend.signal_id else None,
- scout_run_id=str(spend.scout_run_id) if spend.scout_run_id else None,
- request_id=spend.source_id,
- )
- if spend.scout_run_id:
- scout_run = SignalScoutRun.objects.for_team(team_id).select_for_update().get(id=spend.scout_run_id)
- scout_run.total_spend = cost_to_spend(spend.token_cost_microusd or 0, task_count=1)
- scout_run.metadata = {
- **(scout_run.metadata or {}),
- "spend_accounting_failed_stages": [spend.stage] if spend.accounting_failed else [],
- }
- scout_run.save(update_fields=["total_spend", "metadata"])
- except Exception:
- logger.exception(
- "signals.spend.accounting_failed", stage="task_cost_update", team_id=team_id, spend_id=str(spend_id)
+def _save_scout_total(spend: SignalSpend) -> None:
+ if spend.scout_run_id is not None:
+ SignalScoutRun.objects.for_team(spend.team_id).filter(id=spend.scout_run_id).update(
+ total_spend=spend.spend,
+ metadata=Func(
+ Coalesce(F("metadata"), Value({}, output_field=JSONField())),
+ Value("{spend_accounting_failed_stages}"),
+ Value(["scout"] if spend.accounting_failed else [], output_field=JSONField()),
+ function="jsonb_set",
+ output_field=JSONField(),
+ ),
)
@@ -254,99 +212,187 @@ def record_task_spend(
stage: str,
signal_id: str | None = None,
scout_run_id: str | None = None,
- task_id: str | None = None,
+ report_id: str | None = None,
+ implementation_kind: str | None = None,
) -> None:
- if signal_id is None and scout_run_id is None:
+ from products.tasks.backend.facade.billing import locked_task_run_spend # noqa: PLC0415
+
+ if scout_run_id:
+ kind = SignalSpend.Kind.SCOUT_RUN
+ signal_id, report_id = None, None
+ elif stage == "implementation":
+ kind = SignalSpend.Kind(implementation_kind or SignalSpend.Kind.REVIEWER_TRIGGERED_IMPLEMENTATION)
+ if kind not in {
+ SignalSpend.Kind.REVIEWER_TRIGGERED_IMPLEMENTATION,
+ SignalSpend.Kind.SCOUT_TRIGGERED_IMPLEMENTATION,
+ }:
+ raise ValueError("Implementation requires an implementation spend kind")
+ if report_id is None:
+ raise ValueError("Implementation spend requires its report")
+ signal_id = None
+ elif stage in {SignalSpend.Kind.RESEARCH, SignalSpend.Kind.REPO_SELECTION}:
+ kind = SignalSpend.Kind(stage)
+ if signal_id is None:
+ raise ValueError("Research and repository selection require their triggering signal")
+ else:
return
- if (signal_id is None) == (scout_run_id is None):
- raise ValueError("A task's spend must have exactly one owner")
- spend, _ = SignalSpend.objects.for_team(team_id).get_or_create(
- team_id=team_id,
- source_id=run_id,
- is_task=True,
- defaults={"signal_id": signal_id, "scout_run_id": scout_run_id, "task_id": task_id, "stage": stage},
- )
- refresh_task_spend(spend_id=spend.id, team_id=team_id)
+ report = SignalReport.objects.filter(team_id=team_id, id=report_id).first() if report_id else None
+ owner_report_id = report_id if stage == "implementation" and scout_run_id is None else None
+ with locked_task_run_spend(team_id=team_id, run_id=UUID(run_id)) as run:
+ spend, created = SignalSpend.objects.for_team(team_id).get_or_create(
+ team_id=team_id,
+ task_run_id=run.run_id,
+ defaults={"kind": kind, "signal_id": signal_id, "scout_run_id": scout_run_id, "report_id": owner_report_id},
+ )
+ spend = SignalSpend.objects.for_team(team_id).select_for_update().get(id=spend.id)
+ if (str(spend.signal_id), str(spend.scout_run_id), str(spend.report_id)) != (
+ str(signal_id),
+ str(scout_run_id),
+ str(owner_report_id),
+ ):
+ raise ValueError("Task run is already owned by another spend owner")
+ if spend.finalized_at is not None:
+ return
+ if created:
+ spend.created_at = run.created_at
+ spend.billing_exempt = trial_spend_exempt(team_id)
+ if scout_run_id:
+ scout = SignalScoutRun.objects.for_team(team_id).get(id=scout_run_id)
+ spend.created_at = min(spend.created_at, scout.created_at)
+ spend.billing_exempt = spend.billing_exempt or scout_spend_exempt(scout)
+ if report:
+ apply_report_billing(spend, report)
+ spend.spend = cost_to_spend(run.cost_microusd or 0, task_count=1)
+ spend.accounting_failed = not run.complete
+ spend.save()
+ _save_scout_total(spend)
+
+
+def record_task_cost_delta(*, team_id: int, run_id: UUID, cost_delta_microusd: int) -> None:
+ # The sender holds the run lock and commits this delta together with gateway request deduplication.
+ with transaction.atomic():
+ spend = SignalSpend.objects.for_team(team_id).select_for_update().filter(task_run_id=run_id).first()
+ if spend is None or spend.finalized_at is not None:
+ return
+ spend.spend += cost_to_spend(cost_delta_microusd)
+ spend.save(update_fields=["spend", "updated_at"])
+ _save_scout_total(spend)
-def _settle_task_spend(spend: SignalSpend) -> None:
- from products.tasks.backend.facade.billing import ( # noqa: PLC0415 — keeps task services off startup
- process_pending_gateway_usage,
+def register_report_task_spend(
+ *,
+ team_id: int,
+ run_id: str,
+ report_id: str,
+ stage: str | None,
+ signal_id: str | None = None,
+ implementation_kind: str | None = None,
+) -> None:
+ if stage not in {"research", "implementation", "repo_selection"}:
+ return
+ if SignalSpend.objects.for_team(team_id).filter(task_run_id=run_id).exists():
+ return
+ if stage != "implementation" and signal_id is None:
+ signal_id = report_triggering_signal(team_id=team_id, report_id=report_id)
+ record_task_spend(
+ team_id=team_id,
+ run_id=run_id,
+ report_id=report_id,
+ signal_id=signal_id,
+ stage=stage,
+ implementation_kind=implementation_kind,
)
- try:
- process_pending_gateway_usage(run_id=UUID(spend.source_id), team_id=spend.team_id, limit=1000)
- refresh_task_spend(spend_id=spend.id, team_id=spend.team_id)
- except Exception:
- with transaction.atomic():
- SignalSpend.objects.for_team(spend.team_id).filter(id=spend.id).update(accounting_failed=True)
- logger.exception(
- "signals.spend.accounting_failed", stage=spend.stage, team_id=spend.team_id, source_id=spend.source_id
- )
+def complete_signal_pipeline(
+ *, team_id: int, signal_id: str, report_id: str | None = None, billing_exempt: bool = False
+) -> None:
+ with transaction.atomic():
+ spend = _locked_ad_hoc_spend(team_id=team_id, signal_id=signal_id)
+ if spend.finalized_at is not None:
+ return
+ spend.billing_exempt = spend.billing_exempt or billing_exempt
+ report = SignalReport.objects.filter(team_id=team_id, id=report_id).first() if report_id else None
+ if report:
+ apply_report_billing(spend, report)
+ # The summary can still make one-shot calls after grouping hands off this signal.
+ handed_off = (
+ report
+ and UUID(signal_id) in {report.triggering_signal_id, report.pending_triggering_signal_id}
+ and report.status != SignalReport.Status.POTENTIAL
+ )
+ _save_ad_hoc_spend(spend, finalize=not handed_off)
-def settle_signal_task_spend(*, team_id: int, signal_id: str) -> None:
- for spend in SignalSpend.objects.for_team(team_id).filter(signal_id=signal_id, is_task=True):
- _settle_task_spend(spend)
+def begin_report_spend(*, team_id: int, report_id: str, signal_id: str, billing_exempt: bool = False) -> None:
+ with transaction.atomic():
+ spend = _locked_ad_hoc_spend(team_id=team_id, signal_id=signal_id)
+ if spend.finalized_at is not None:
+ return
+ spend.billing_exempt = spend.billing_exempt or billing_exempt
+ report = SignalReport.objects.get(team_id=team_id, id=report_id)
+ apply_report_billing(spend, report)
+ _save_ad_hoc_spend(spend)
-def _report_spend_signal_ids(*, team_id: int, report_id: str) -> list[str]:
- from products.tasks.backend.facade.api import ( # noqa: PLC0415 — keeps the task facade off startup
- TaskRunStatus,
- get_signal_report_implementation_runs,
- )
- report = SignalReport.objects.filter(team_id=team_id, id=report_id).first()
- if report is None or report.status == SignalReport.Status.IN_PROGRESS:
- return []
- task_ids = list(
- SignalReportTask.objects.filter(
- team_id=team_id, report_id=report_id, relationship="implementation"
- ).values_list("task_id", flat=True)
- )
- runs = get_signal_report_implementation_runs(team_id, report_id, task_ids)
- if any(run.status not in {TaskRunStatus.COMPLETED, TaskRunStatus.FAILED, TaskRunStatus.CANCELLED} for run in runs):
- return []
- task_ids.extend(
- SignalReportArtefact.objects.filter(team_id=team_id, report_id=report_id, task_id__isnull=False).values_list(
- "task_id", flat=True
- )
- )
- owners = set(
- SignalSpend.objects.for_team(team_id)
- .filter(task_id__in=task_ids, signal_id__isnull=False)
- .values_list("signal_id", flat=True)
+def finish_report_spend(*, team_id: int, report_id: str, signal_id: str) -> None:
+ with transaction.atomic():
+ report = SignalReport.objects.select_for_update().filter(team_id=team_id, id=report_id).first()
+ spend = _locked_ad_hoc_spend(team_id=team_id, signal_id=signal_id)
+ if spend.finalized_at is None:
+ if report:
+ apply_report_billing(spend, report)
+ _save_ad_hoc_spend(spend, finalize=True)
+ if (
+ report
+ and report.triggering_signal_id == UUID(signal_id)
+ and report.pending_triggering_signal_id not in {None, report.triggering_signal_id}
+ ):
+ pending = _locked_ad_hoc_spend(team_id=team_id, signal_id=str(report.pending_triggering_signal_id))
+ if pending.finalized_at is None:
+ apply_report_billing(pending, report)
+ _save_ad_hoc_spend(pending, finalize=True)
+ report.pending_triggering_signal_id = None
+ report.save(update_fields=["pending_triggering_signal_id"])
+ if report and report.status == SignalReport.Status.CANDIDATE and report.triggering_signal_id == UUID(signal_id):
+ report.triggering_signal_id = None
+ report.save(update_fields=["triggering_signal_id"])
+
+
+def complete_task_spend(*, team_id: int, run_id: str, report_id: str | None = None) -> None:
+ from products.tasks.backend.facade.billing import ( # noqa: PLC0415
+ TaskTokenUsageUnavailable,
+ locked_task_run_spend,
+ process_pending_gateway_usage,
)
- if report.triggering_signal_id:
- owners.add(report.triggering_signal_id)
- return sorted(str(owner) for owner in owners)
-
-
-async def publish_report_spend(*, team_id: int, report_id: str) -> None:
- try:
- signal_ids = await database_sync_to_async(_report_spend_signal_ids)(team_id=team_id, report_id=report_id)
- for signal_id in signal_ids:
- await publish_signal_spend(team_id=team_id, signal_id=signal_id)
- except Exception:
- logger.exception(
- "signals.spend.accounting_failed", stage="report_completion", team_id=team_id, report_id=report_id
- )
-
-def complete_task_spend(*, team_id: int, run_id: str) -> None:
- try:
- spend = SignalSpend.objects.for_team(team_id).filter(source_id=run_id, is_task=True).first()
- if spend is None:
- return
- if spend.scout_run_id:
- _settle_task_spend(spend)
- else:
- refresh_task_spend(spend_id=spend.id, team_id=team_id)
- if spend.stage == "implementation" and spend.task_id:
- report_ids = SignalReportTask.objects.filter(
- team_id=team_id, task_id=spend.task_id, relationship="implementation"
- ).values_list("report_id", flat=True)
- for report_id in report_ids:
- async_to_sync(publish_report_spend)(team_id=team_id, report_id=str(report_id))
- except Exception:
- logger.exception("signals.spend.accounting_failed", stage="task_completion", team_id=team_id, run_id=run_id)
+ spend = SignalSpend.objects.for_team(team_id).filter(task_run_id=UUID(str(run_id))).first()
+ if spend is None:
+ return
+ if spend.finalized_at is None:
+ with locked_task_run_spend(team_id=team_id, run_id=UUID(run_id)) as run:
+ if not run.is_terminal:
+ return
+ process_pending_gateway_usage(run_id=UUID(run_id), team_id=team_id, limit=1000)
+ with locked_task_run_spend(team_id=team_id, run_id=UUID(run_id)) as run:
+ spend = SignalSpend.objects.for_team(team_id).select_for_update().get(id=spend.id)
+ if spend.finalized_at is None:
+ billing_report_id = spend.report_id or report_id
+ if billing_report_id is not None:
+ report = (
+ SignalReport.objects.filter(team_id=team_id, id=billing_report_id)
+ .only("billing_exempt_reason")
+ .first()
+ )
+ if report:
+ apply_report_billing(spend, report)
+ spend.spend = cost_to_spend(run.cost_microusd or 0, task_count=1)
+ spend.accounting_failed = not run.complete
+ if run.is_terminal and run.complete:
+ spend.finalized_at = timezone.now()
+ spend.save()
+ _save_scout_total(spend)
+ if spend.accounting_failed:
+ raise TaskTokenUsageUnavailable("Task usage is not settled yet")
+ if spend.signal_id:
+ async_to_sync(submit_signal_metadata_updates)(team_id=team_id, signal_ids=[str(spend.signal_id)])
diff --git a/products/signals/backend/spend_billing.py b/products/signals/backend/spend_billing.py
new file mode 100644
index 000000000000..6cc91f0a92e2
--- /dev/null
+++ b/products/signals/backend/spend_billing.py
@@ -0,0 +1,43 @@
+import posthoganalytics
+
+from posthog.models import Team
+
+from products.signals.backend.enums import SignalSourceProduct
+from products.signals.backend.free_trial import SELF_DRIVING_FREE_TRIAL_FLAG
+from products.signals.backend.models import SignalReport, SignalScoutRun, SignalSpend
+
+
+def system_source_exempt(source_product: str | None) -> bool:
+ return source_product in {SignalSourceProduct.HEALTH_CHECKS, SignalSourceProduct.ENGINEERING_ANALYTICS}
+
+
+def scout_spend_exempt(scout: SignalScoutRun) -> bool:
+ from products.signals.backend.billing import ( # noqa: PLC0415 — billing uses this module for aggregation
+ BILLING_EXEMPT_SCOUT_SKILLS,
+ _scout_skill_is_canonical,
+ )
+
+ return scout.trial_state is not None or (
+ scout.skill_name in BILLING_EXEMPT_SCOUT_SKILLS
+ and _scout_skill_is_canonical(scout.team_id, scout.skill_name, version=scout.skill_version)
+ )
+
+
+def trial_spend_exempt(team_id: int) -> bool:
+ org_id = str(Team.objects.values_list("organization_id", flat=True).get(id=team_id))
+ return (
+ posthoganalytics.feature_enabled(
+ SELF_DRIVING_FREE_TRIAL_FLAG,
+ org_id,
+ groups={"organization": org_id},
+ group_properties={"organization": {"id": org_id}},
+ only_evaluate_locally=True,
+ send_feature_flag_events=False,
+ )
+ is True
+ )
+
+
+def apply_report_billing(spend: SignalSpend, report: SignalReport) -> None:
+ if report.billing_exempt_reason:
+ spend.billing_exempt = True
diff --git a/products/signals/backend/spend_receivers.py b/products/signals/backend/spend_receivers.py
deleted file mode 100644
index 2054a9ec6c88..000000000000
--- a/products/signals/backend/spend_receivers.py
+++ /dev/null
@@ -1,104 +0,0 @@
-from typing import TYPE_CHECKING
-from uuid import UUID
-
-from django.db import transaction
-from django.db.models.signals import post_save
-
-import structlog
-
-from products.signals.backend.models import SignalReportTask, SignalScoutRun, SignalSpend
-from products.signals.backend.spend import record_task_spend, refresh_task_spend, report_triggering_signal
-from products.tasks.backend.facade.task_run_signals import (
- TaskOriginProduct,
- connect_task_run_cost_updated,
- connect_task_run_post_save,
-)
-
-logger = structlog.get_logger(__name__)
-
-
-if TYPE_CHECKING:
- from products.tasks.backend.facade.task_run_signals import TaskRun
-
-
-def task_run_created(*, instance: "TaskRun", created: bool, **kwargs: object) -> None:
- if not created:
- return
- stage = (instance.state or {}).get("ai_stage") or "implementation"
- try:
- with transaction.atomic():
- task = instance.task
- if task.origin_product != TaskOriginProduct.SIGNAL_REPORT or not task.signal_report_id:
- return
- previous = SignalSpend.objects.for_team(instance.team_id).filter(task_id=task.id, is_task=True).first()
- if (
- previous is None
- and (instance.state or {}).get("ai_stage")
- not in {
- "research",
- "implementation",
- "repo_selection",
- }
- and not SignalReportTask.objects.filter(
- team_id=instance.team_id, task_id=task.id, relationship="implementation"
- ).exists()
- ):
- return
- stage = (instance.state or {}).get("ai_stage") or (previous.stage if previous else "implementation")
- signal_id = (
- str(previous.signal_id)
- if previous and previous.signal_id
- else report_triggering_signal(team_id=instance.team_id, report_id=str(task.signal_report_id))
- )
- record_task_spend(
- team_id=instance.team_id,
- run_id=str(instance.id),
- stage=stage,
- signal_id=signal_id,
- task_id=str(task.id),
- )
- except Exception:
- logger.exception(
- "signals.spend.accounting_failed", stage=stage, team_id=instance.team_id, run_id=str(instance.id)
- )
-
-
-def scout_run_created(*, instance: SignalScoutRun, created: bool, **kwargs: object) -> None:
- if not created:
- return
- try:
- with transaction.atomic():
- record_task_spend(
- team_id=instance.team_id,
- run_id=str(instance.task_run_id),
- stage="scout",
- scout_run_id=str(instance.id),
- task_id=str(instance.task_run.task_id),
- )
- except Exception:
- logger.exception(
- "signals.spend.accounting_failed", stage="scout", team_id=instance.team_id, scout_run_id=str(instance.id)
- )
-
-
-def task_cost_updated(*, run_id: UUID, team_id: int, **kwargs: object) -> None:
- try:
- with transaction.atomic():
- spend_id = (
- SignalSpend.objects.for_team(team_id)
- .filter(source_id=str(run_id), is_task=True)
- .values_list("id", flat=True)
- .first()
- )
- if spend_id is not None:
- refresh_task_spend(spend_id=spend_id, team_id=team_id)
- except Exception:
- logger.exception(
- "signals.spend.accounting_failed", stage="task_cost_update", team_id=team_id, run_id=str(run_id)
- )
-
-
-def connect_spend_receivers() -> None:
- connect_task_run_cost_updated(task_cost_updated, dispatch_uid="signals.task_cost_updated")
- connect_task_run_post_save(task_run_created, dispatch_uid="signals.spend_task_created")
- post_save.connect(scout_run_created, sender=SignalScoutRun, dispatch_uid="signals.spend_scout_created")
diff --git a/products/signals/backend/task_run_artefacts.py b/products/signals/backend/task_run_artefacts.py
index 2835c7b91b1f..ca21ba1ee2a7 100644
--- a/products/signals/backend/task_run_artefacts.py
+++ b/products/signals/backend/task_run_artefacts.py
@@ -29,7 +29,7 @@ from products.signals.backend.artefact_schemas import (
NoteArtefact,
TaskRunArtefact,
)
-from products.signals.backend.billing import first_billable_pr_run_at, mark_report_billing_exempt
+from products.signals.backend.billing import mark_report_billing_exempt, report_has_implementation_pr
from products.signals.backend.models import ArtefactAttribution, SignalReport, SignalReportArtefact, SignalReportTask
from products.signals.backend.report_assignments import ReportClaimConflict, claim_report_for_task, release_claim
from products.signals.backend.report_claims import get_active_claim, is_user_claim_of, is_user_claim_of_task_creator
@@ -416,22 +416,10 @@ def record_report_task(
def release_quota_cancelled_implementation(*, team_id: int, task_id: str) -> list[str]:
- """Remove a quota-cancelled implementation's auto-start records so its report can be
- implemented again by a later cycle.
-
- A run the quota gate cancels mid-flight never shipped its PR, but its `SignalReportTask` gate
- row and implementation `task_run` artefact would keep `associated_task_runs` reporting a
- started implementation and permanently block re-implementation. Deleting both restores the
- report to "never implemented"; a system `note` artefact records for the report timeline why
- the run stopped. Locks each report row (the same lock auto-start creation takes) so the
- removal serializes with a concurrent auto-start evaluation. Returns the affected report ids
- (empty when the task has no implementation link).
-
- Reports that already shipped a billable PR are skipped entirely: the cancel decision is
- run-scoped but this delete is task-scoped, and a sibling run of the same task may have
- shipped the PR that billed the report. Its `SignalReportTask` row is billing's evidence —
- the `billed_earlier` dedup and refund eligibility both resolve through it — so deleting it
- would re-bill the report on its next implementation and strand the paid charge unrefundable.
+ """Remove implementation gates so a quota-cancelled report can run again.
+
+ Preserve shipped PRs, including those from sibling runs. Lock reports to serialize
+ cleanup with auto-start, and return the affected report IDs.
"""
report_ids = list(
SignalReportTask.objects.filter(
@@ -444,10 +432,7 @@ def release_quota_cancelled_implementation(*, team_id: int, task_id: str) -> lis
report = SignalReport.objects.select_for_update().filter(id=report_id, team_id=team_id).first()
if report is None:
continue
- if first_billable_pr_run_at(report_id) is not None:
- # A sibling run already shipped this report's billable PR. These records are
- # billing's evidence for that charge — deleting them would double-bill the next
- # implementation — and the report needs no release: it *is* implemented.
+ if report_has_implementation_pr(report_id, team_id=team_id):
continue
assignment = get_active_claim(team_id=team_id, report_id=report_id)
if assignment is not None and assignment.actor_kind == "task" and str(assignment.actor_task_id) == task_id:
diff --git a/products/signals/backend/tasks.py b/products/signals/backend/tasks.py
index 0015f362cbb4..e4452c079d9f 100644
--- a/products/signals/backend/tasks.py
+++ b/products/signals/backend/tasks.py
@@ -143,6 +143,24 @@ def reconcile_inbox_onboarding(team_id: int, failed_checks: int = 0) -> None:
reconcile_inbox_onboarding.apply_async(args=[team_id], countdown=60)
+@shared_task(
+ ignore_result=True,
+ queue=CeleryQueue.LONG_RUNNING.value,
+ autoretry_for=(Exception,),
+ retry_backoff=True,
+ max_retries=10,
+ acks_late=True,
+ reject_on_worker_lost=True,
+)
+@with_team_scope(canonical=True)
+def settle_scout_spend(team_id: int, run_id: str) -> None:
+ from products.signals.backend.spend import (
+ complete_task_spend, # noqa: PLC0415 - keeps spend accounting off the task registration import path
+ )
+
+ complete_task_spend(team_id=team_id, run_id=run_id)
+
+
@shared_task(
name="products.signals.backend.tasks.refresh_pull_request_participants",
ignore_result=True,
diff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.py
index b0950c52fb9b..e89c12f7a56f 100644
--- a/products/signals/backend/temporal/__init__.py
+++ b/products/signals/backend/temporal/__init__.py
@@ -74,6 +74,7 @@ from products.signals.backend.temporal.emitter import SignalEmitterWorkflow
from products.signals.backend.temporal.grouping import (
TeamSignalGroupingWorkflow,
assign_and_emit_signal_activity,
+ complete_signal_grouping_activity,
fetch_report_contexts_activity,
generate_search_queries_activity,
get_embedding_activity,
@@ -115,6 +116,7 @@ from products.signals.backend.temporal.spend import (
from products.signals.backend.temporal.summary import (
SignalReportSummaryWorkflow,
check_report_quota_gate_activity,
+ check_report_task_settlement_activity,
dispatch_inbox_slack_notifications_activity,
implementation_buffer_seconds_activity,
mark_report_failed_activity,
@@ -126,6 +128,7 @@ from products.signals.backend.temporal.summary import (
publish_report_spend_activity,
report_has_assigned_signals_activity,
report_is_candidate_activity,
+ report_spend_owner_activity,
reset_report_to_potential_activity,
revert_report_to_candidate_activity,
)
@@ -177,6 +180,7 @@ ACTIVITIES = [
stamp_dispatched_signals_scout_runs_activity,
run_due_signal_report_checks_activity,
assign_and_emit_signal_activity,
+ complete_signal_grouping_activity,
capture_signal_dropped_activity,
check_signals_quota_limited_activity,
delete_report_activity,
@@ -193,6 +197,7 @@ ACTIVITIES = [
mark_report_failed_activity,
read_signals_from_s3_activity,
check_report_quota_gate_activity,
+ check_report_task_settlement_activity,
mark_report_in_progress_activity,
mark_report_pending_input_activity,
mark_report_ready_activity,
@@ -201,6 +206,7 @@ ACTIVITIES = [
report_is_candidate_activity,
publish_report_completed_activity,
publish_report_spend_activity,
+ report_spend_owner_activity,
report_has_assigned_signals_activity,
revert_report_to_candidate_activity,
delete_team_reports_activity,
diff --git a/products/signals/backend/temporal/agentic/report.py b/products/signals/backend/temporal/agentic/report.py
index 569663f8feb0..00bf2841b435 100644
--- a/products/signals/backend/temporal/agentic/report.py
+++ b/products/signals/backend/temporal/agentic/report.py
@@ -101,6 +101,7 @@ class RunAgenticReportInput:
# correcting or clearing it) so the run does not bury that newer row. Defaults to None so an
# older workflow history that predates this field replays cleanly (guard off).
repo_selection_as_of: datetime | None = None
+ signal_id: str | None = None
@frozen
@@ -1214,6 +1215,7 @@ async def run_agentic_report_activity(input: RunAgenticReportInput) -> RunAgenti
previous_checks=previous_checks,
implementation_context=implementation_context,
signal_report_id=input.report_id,
+ spend_signal_id=input.signal_id,
has_business_knowledge=has_bk,
resolved_report_title=resolved_report_title,
resolved_report_summary=resolved_report_summary,
diff --git a/products/signals/backend/temporal/agentic/select_repository.py b/products/signals/backend/temporal/agentic/select_repository.py
index d4d0c57f6c5d..92adbe2bc0ef 100644
--- a/products/signals/backend/temporal/agentic/select_repository.py
+++ b/products/signals/backend/temporal/agentic/select_repository.py
@@ -40,11 +40,12 @@ GITHUB_ONLY_DOMAINS = [
logger = structlog.get_logger(__name__)
-@dataclass
+@dataclass(frozen=False)
class SelectRepositoryInput:
team_id: int
report_id: str
signals: list[SignalData]
+ signal_id: str | None = None
def _resolve_sandbox_user_id(team_id: int) -> int | None:
@@ -158,6 +159,7 @@ async def select_repository_activity(input: SelectRepositoryInput) -> RepoSelect
user_id=user_id,
signals=input.signals,
signal_report_id=input.report_id,
+ spend_signal_id=input.signal_id,
sandbox_environment_id=sandbox_env_id,
)
logger.info(
diff --git a/products/signals/backend/temporal/buffer.py b/products/signals/backend/temporal/buffer.py
index 74a6546688b6..f8b94107d5e6 100644
--- a/products/signals/backend/temporal/buffer.py
+++ b/products/signals/backend/temporal/buffer.py
@@ -25,6 +25,8 @@ from posthog.temporal.common.utils import close_db_connections
from products.signals.backend.daily_limit import capture_signal_report_daily_limit_paused, daily_report_limit_gate
from products.signals.backend.quota import is_team_signals_quota_limited
+from products.signals.backend.spend import complete_signal_pipeline
+from products.signals.backend.spend_billing import system_source_exempt
from products.signals.backend.temporal import metrics
from products.signals.backend.temporal.drop_telemetry import capture_signal_dropped_event, signal_dropped_input
from products.signals.backend.temporal.grouping_v2 import TeamSignalGroupingV2Workflow
@@ -99,23 +101,21 @@ async def check_signals_quota_limited_activity(input: CheckSignalsQuotaInput) ->
# single-reason drop metric — so the daily-limit event stream stays complete on co-bound days.
if daily_gate.limited:
capture_signal_report_daily_limit_paused(team, report_id=None, stage="ingestion", gate=daily_gate)
- if quota_limited:
- logger.info(
- "signals_buffer.dropped_batch_quota_limited",
- team_id=input.team_id,
- signal_count=input.signal_count,
- )
- metrics.increment_dropped(stage="ingestion", reason="quota_limited", count=input.signal_count)
- _capture_ingestion_drops(team, input.signals, "quota_limited", "Signals credits quota exceeded")
- return True
- if daily_gate.limited:
+ if quota_limited or daily_gate.limited:
+ reason = "quota_limited" if quota_limited else "daily_report_limit"
+ message = "Signals credits quota exceeded" if quota_limited else "Daily report limit reached"
+ for signal in input.signals:
+ if signal.signal_id:
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=input.team_id,
+ signal_id=signal.signal_id,
+ billing_exempt=system_source_exempt(signal.source_product),
+ )
logger.info(
- "signals_buffer.dropped_batch_daily_limit",
- team_id=input.team_id,
- signal_count=input.signal_count,
+ "signals_buffer.dropped_batch", team_id=input.team_id, signal_count=input.signal_count, reason=reason
)
- metrics.increment_dropped(stage="ingestion", reason="daily_report_limit", count=input.signal_count)
- _capture_ingestion_drops(team, input.signals, "daily_report_limit", "Daily report limit reached")
+ metrics.increment_dropped(stage="ingestion", reason=reason, count=input.signal_count)
+ _capture_ingestion_drops(team, input.signals, reason, message)
return True
return False
@@ -292,7 +292,8 @@ class BufferSignalsWorkflow:
retry_policy=RetryPolicy(maximum_attempts=3),
)
for s in batch
- ]
+ ],
+ return_exceptions=workflow.patched("signals-retry-failed-safety-only"),
)
except ActivityError:
self._signal_buffer[:0] = batch
@@ -304,8 +305,13 @@ class BufferSignalsWorkflow:
await workflow.sleep(timedelta(seconds=SAFETY_FILTER_RETRY_DELAY_SECONDS))
continue
safe_signals: list[EmitSignalInputs] = []
+ failed_signals: list[EmitSignalInputs] = []
for signal, result in zip(batch, safety_results):
- if result.safe:
+ if isinstance(result, ActivityError):
+ failed_signals.append(signal)
+ elif isinstance(result, BaseException):
+ raise result
+ elif result.safe:
safe_signals.append(signal)
else:
logger.warning(
@@ -316,6 +322,9 @@ class BufferSignalsWorkflow:
source_type=signal.source_type,
source_id=signal.source_id,
)
+ if failed_signals:
+ self._signal_buffer[:0] = failed_signals
+ await workflow.sleep(timedelta(seconds=SAFETY_FILTER_RETRY_DELAY_SECONDS))
batch = safe_signals
if not batch:
diff --git a/products/signals/backend/temporal/custom_agent.py b/products/signals/backend/temporal/custom_agent.py
index 751f6c6c8404..0ffdfc82cccd 100644
--- a/products/signals/backend/temporal/custom_agent.py
+++ b/products/signals/backend/temporal/custom_agent.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import uuid
import importlib
from datetime import timedelta
+from typing import TYPE_CHECKING
from django.conf import settings
@@ -19,7 +20,6 @@ from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.scoped import scoped_temporal
from posthog.temporal.common.utils import close_db_connections
-from products.signals.backend.custom_agent.base import AIDataProcessingNotApprovedError, CustomSignalAgent
from products.signals.backend.custom_agent.loader import import_agent_class, validate_agent_class_identity
from products.signals.backend.custom_agent.schemas import (
CustomAgentRunHandle,
@@ -29,6 +29,9 @@ from products.signals.backend.custom_agent.schemas import (
validated_identifier,
)
+if TYPE_CHECKING:
+ from products.signals.backend.custom_agent.base import CustomSignalAgent
+
logger = structlog.get_logger(__name__)
@@ -101,6 +104,9 @@ async def arun_agent(
# without org-level consent. `team.organization` may be a deferred FK access.
organization = await database_sync_to_async(lambda: team.organization)()
if not organization.is_ai_data_processing_approved:
+ # The agent base imports Temporal helpers, so importing it at module load creates a cycle.
+ from products.signals.backend.custom_agent.base import AIDataProcessingNotApprovedError
+
raise AIDataProcessingNotApprovedError(
f"Organization {organization.id} has not approved AI data processing; "
f"refusing to launch custom signal agent for team {team_id}"
diff --git a/products/signals/backend/temporal/drop_telemetry.py b/products/signals/backend/temporal/drop_telemetry.py
index 7a08566ad4a3..0e42ec45ecd2 100644
--- a/products/signals/backend/temporal/drop_telemetry.py
+++ b/products/signals/backend/temporal/drop_telemetry.py
@@ -24,7 +24,7 @@ _PATCH_SIGNAL_DROPPED = "signal-dropped-telemetry-v1"
_MAX_ERROR_LENGTH = 500
-@dataclass
+@dataclass(frozen=True)
class CaptureSignalDroppedInput:
team_id: int
source_product: str
@@ -35,6 +35,7 @@ class CaptureSignalDroppedInput:
error_type: str
error: str
extra: dict = field(default_factory=dict)
+ signal_id: str | None = None
def summarize_drop_error(error: BaseException) -> tuple[str, str]:
@@ -67,6 +68,7 @@ def signal_dropped_input(
# Flatten before scheduling: the raw `extra` can nest large customer-derived
# payloads, and the activity input is recorded verbatim in workflow history.
extra=_telemetry_props_from_extra(signal.extra),
+ signal_id=signal.signal_id,
)
diff --git a/products/signals/backend/temporal/grouping.py b/products/signals/backend/temporal/grouping.py
index 18f3f81f7130..559f4fc8740f 100644
--- a/products/signals/backend/temporal/grouping.py
+++ b/products/signals/backend/temporal/grouping.py
@@ -41,7 +41,8 @@ from products.signals.backend.receivers import _is_safety_suppressed
from products.signals.backend.recurrence import fixed_dismissal_at
from products.signals.backend.report_merge import signal_target_report
from products.signals.backend.signal_metadata import EMBEDDING_MODEL
-from products.signals.backend.spend import signal_spend_summaries
+from products.signals.backend.spend import begin_report_spend, complete_signal_pipeline, signal_spend_summaries
+from products.signals.backend.spend_billing import system_source_exempt
from products.signals.backend.temporal import metrics
from products.signals.backend.temporal.drop_telemetry import capture_signal_dropped
from products.signals.backend.temporal.llm import MAX_QUERY_TOKENS, call_llm, truncate_query_to_token_limit
@@ -793,6 +794,7 @@ class AssignAndEmitSignalOutput:
# The title the report holds after assignment. It differs from `updated_title` when assignment keeps a
# researched title. None when no title is known, which includes results recorded before this field existed.
report_title: Optional[str] = None
+ triggering_signal_id: str | None = None
@frozen
@@ -813,6 +815,7 @@ class AssignAndEmitDbResult:
# What the report's last completed pass covered, for the same reason.
report_signals_researched: int = 0
report_title: Optional[str] = None
+ triggering_signal_id: str | None = None
def _link_check_follow_up(*, team_id: int, report_id: str, source_product: str, extra: dict) -> None:
@@ -966,6 +969,15 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
billing_exempt_reason=BILLING_EXEMPT_SOURCE_PRODUCTS.get(input.source_product),
)
+ # Persist source exemptions before a summary can see and finalize this trigger.
+ if system_source_exempt(input.source_product):
+ begin_report_spend(
+ team_id=input.team_id,
+ report_id=str(report.id),
+ signal_id=input.signal_id,
+ billing_exempt=True,
+ )
+
# Promotion rules by status:
# - SUPPRESSED: never promoted. A report dismissed as fixed never receives new signals
# either (a recurrence spawns a fresh report above).
@@ -1005,14 +1017,15 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
# first matching signal after the limit lifts re-evaluates promotion under the
# same rules.
promotion_suppressed = True
- # If candidate got here - it usually means CH issue down the way
- # (e.g. CH wait raised before start_child_workflow)
- elif report.status != SignalReport.Status.CANDIDATE:
- updated_fields = report.transition_to(SignalReport.Status.CANDIDATE)
- report.triggering_signal_id = input.signal_id
- report.save(update_fields=[*updated_fields, "triggering_signal_id"])
- promoted = True
else:
+ updated_fields = (
+ report.transition_to(SignalReport.Status.CANDIDATE)
+ if report.status != SignalReport.Status.CANDIDATE
+ else []
+ )
+ if updated_fields or report.triggering_signal_id is None:
+ report.triggering_signal_id = input.signal_id
+ report.save(update_fields=[*updated_fields, "triggering_signal_id"])
promoted = True
pending_bucket = next_research_bucket(report.researching_signal_count or 0)
@@ -1074,6 +1087,7 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
return AssignAndEmitDbResult(
report_id=report_id,
promoted=promoted,
+ triggering_signal_id=str(report.triggering_signal_id) if report.triggering_signal_id else None,
timestamp=ts,
matched_deleted_report=False,
run_count=report.run_count,
@@ -1238,6 +1252,7 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
return AssignAndEmitSignalOutput(
report_id=db_result.report_id,
promoted=db_result.promoted,
+ triggering_signal_id=db_result.triggering_signal_id,
timestamp=db_result.timestamp,
run_count=db_result.run_count,
research_debounce_seconds=RESEARCH_DEBOUNCE_SECONDS,
@@ -1316,6 +1331,51 @@ def _augment_candidates_with_batch(
return augmented
+@frozen
+class SignalGroupingCompletion:
+ team_id: int
+ signal_id: str
+ report_id: str | None
+ billing_exempt: bool
+
+
+@scoped_temporal()
+@close_db_connections
+async def complete_signal_grouping_activity(signals: list[SignalGroupingCompletion]) -> None:
+ for signal in signals:
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=signal.team_id,
+ signal_id=signal.signal_id,
+ report_id=signal.report_id,
+ billing_exempt=signal.billing_exempt,
+ )
+
+
+async def _complete_signal_batch(batch: list[EmitSignalInputs], reports: dict[str, str]) -> None:
+ if not workflow.patched("signals-owner-spend-grouping-completion"):
+ return
+ try:
+ await workflow.execute_activity(
+ complete_signal_grouping_activity,
+ [
+ SignalGroupingCompletion(
+ team_id=signal.team_id,
+ signal_id=signal.signal_id,
+ report_id=reports.get(signal.signal_id),
+ billing_exempt=system_source_exempt(signal.source_product),
+ )
+ for signal in batch
+ if signal.signal_id
+ ],
+ start_to_close_timeout=timedelta(minutes=1),
+ schedule_to_close_timeout=timedelta(minutes=20),
+ retry_policy=RetryPolicy(initial_interval=timedelta(seconds=5), maximum_interval=timedelta(seconds=30)),
+ )
+ except temporalio.exceptions.ActivityError:
+ workflow.logger.exception("signals.spend.accounting_failed", extra={"stage": "grouping_completion"})
+
+
async def _process_signal_batch(
batch: list[EmitSignalInputs],
cached_type_examples: Optional[FetchSignalTypeExamplesOutput] = None,
@@ -1330,6 +1390,7 @@ async def _process_signal_batch(
within a batch.
"""
team_id = batch[0].team_id
+ original_batch = batch
# Purely defensive
if not all(signal.team_id == team_id for signal in batch):
raise ValueError("All signals in a batch must belong to the same team")
@@ -1413,6 +1474,7 @@ async def _process_signal_batch(
signal_embeddings = [signal_embeddings[i] for i in retained_indices]
query_gen_results = [query_gen_results[i] for i in retained_indices]
if not batch:
+ await _complete_signal_batch(original_batch, {})
return dropped, type_examples_result
# Step 3: Embed all queries across all signals (flatten → parallel embed)
@@ -1661,6 +1723,7 @@ async def _process_signal_batch(
team_id=signal.team_id,
report_id=assign_result.report_id,
debounce_seconds=assign_result.research_debounce_seconds,
+ signal_id=assign_result.triggering_signal_id,
),
assign_result.run_count,
)
@@ -1730,6 +1793,7 @@ async def _process_signal_batch(
report_id=report_input.report_id,
)
+ await _complete_signal_batch(original_batch, {signal_id: result.report_id for signal_id, result in emitted_signals})
return dropped, type_examples_result
diff --git a/products/signals/backend/temporal/parallel_grouping.py b/products/signals/backend/temporal/parallel_grouping.py
index 25945cc36a4c..2595b88c1052 100644
--- a/products/signals/backend/temporal/parallel_grouping.py
+++ b/products/signals/backend/temporal/parallel_grouping.py
@@ -406,6 +406,7 @@ async def _process_parallel_batch(
team_id=signal.team_id,
report_id=result.assign_result.report_id,
debounce_seconds=result.assign_result.research_debounce_seconds,
+ signal_id=result.assign_result.triggering_signal_id,
),
result.assign_result.run_count,
)
diff --git a/products/signals/backend/temporal/report_safety_judge.py b/products/signals/backend/temporal/report_safety_judge.py
index 1c3271720ca1..cdedc779c0f5 100644
--- a/products/signals/backend/temporal/report_safety_judge.py
+++ b/products/signals/backend/temporal/report_safety_judge.py
@@ -239,11 +239,12 @@ async def judge_report_safety(
return SafetyJudgeResponse(choice=True)
-@dataclass
+@dataclass(frozen=False)
class SafetyJudgeInput:
team_id: int
report_id: str
signals: list[SignalData]
+ signal_id: str | None = None
@dataclass
@@ -258,7 +259,7 @@ class SafetyJudgeOutput:
async def report_safety_judge_activity(input: SafetyJudgeInput) -> SafetyJudgeOutput:
"""Assess report for prompt injection attacks and store result as artefact."""
try:
- signal_id = await database_sync_to_async(report_triggering_signal)(
+ signal_id = input.signal_id or await database_sync_to_async(report_triggering_signal)(
team_id=input.team_id, report_id=input.report_id, ch_user=ClickHouseUser.SIGNALS
)
except Exception:
diff --git a/products/signals/backend/temporal/safety_filter.py b/products/signals/backend/temporal/safety_filter.py
index bc504c7f0dcb..98772f323f02 100644
--- a/products/signals/backend/temporal/safety_filter.py
+++ b/products/signals/backend/temporal/safety_filter.py
@@ -17,10 +17,13 @@ from temporalio import activity
from posthog.event_usage import groups
from posthog.models import Team
from posthog.redis import get_async_client
+from posthog.sync import database_sync_to_async
from posthog.temporal.common.scoped import scoped_temporal
from posthog.temporal.common.utils import close_db_connections
from products.signals.backend.facade.api import _telemetry_props_from_extra
+from products.signals.backend.spend import complete_signal_pipeline, finalized_signal_ids
+from products.signals.backend.spend_billing import system_source_exempt
from products.signals.backend.system_one_decision import SAFETY_CATEGORIES, ModelMode, model_mode, run_model_decision
from products.signals.backend.system_one_prompts import SystemOnePrompt, bundled_prompt, current_prompt
from products.signals.backend.temporal import metrics
@@ -344,6 +347,9 @@ async def _capture_signal_blocked_event(input: SafetyFilterInput, result: Safety
@close_db_connections
async def safety_filter_activity(input: SafetyFilterInput) -> SafetyFilterOutput:
"""Filter out unsafe signals before passing them through the pipeline."""
+ if input.signal_id and input.team_id is not None:
+ if await database_sync_to_async(finalized_signal_ids)(team_id=input.team_id, signal_ids=[input.signal_id]):
+ return SafetyFilterOutput(safe=False, threat_type="", explanation="Signal already processed")
try:
result = await safety_filter(
input.team_id,
@@ -358,6 +364,12 @@ async def safety_filter_activity(input: SafetyFilterInput) -> SafetyFilterOutput
raise
if not result.safe:
+ if input.signal_id and input.team_id is not None:
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=input.team_id,
+ signal_id=input.signal_id,
+ billing_exempt=system_source_exempt(input.source_product),
+ )
metrics.increment_safety_blocked(input.source_product or "unknown")
await _capture_signal_blocked_event(input, result)
diff --git a/products/signals/backend/temporal/spend.py b/products/signals/backend/temporal/spend.py
index a221e85ea292..9d169dd16eb0 100644
--- a/products/signals/backend/temporal/spend.py
+++ b/products/signals/backend/temporal/spend.py
@@ -10,19 +10,13 @@ from posthog.temporal.common.base import PostHogWorkflow
with workflow.unsafe.imports_passed_through():
from posthog.temporal.common.utils import asyncify
- from products.signals.backend.spend import (
- SignalMetadataChange,
- SignalSpendPublication,
- SignalSpendUpdate,
- settle_signal_task_spend,
- )
+ from products.signals.backend.spend import SignalMetadataChange, SignalSpendPublication, SignalSpendUpdate
from products.signals.backend.spend_metadata import confirm_signal_spend_metadata, publish_signal_spend_metadata
@activity.defn
@asyncify
def update_signal_spend_activity(input: SignalSpendUpdate) -> SignalSpendPublication:
- settle_signal_task_spend(team_id=input.team_id, signal_id=input.signal_id)
return publish_signal_spend_metadata(input)
@@ -65,13 +59,17 @@ class UpdateSignalSpendWorkflow(PostHogWorkflow):
schedule_to_close_timeout=timedelta(minutes=10),
retry_policy=RetryPolicy(initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=10)),
)
- await workflow.execute_activity(
- confirm_signal_spend_activity,
- publication,
- start_to_close_timeout=timedelta(minutes=1),
- schedule_to_close_timeout=timedelta(minutes=10),
- retry_policy=RetryPolicy(initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=10)),
- )
+ if changes or not workflow.patched("signals-spend-refresh-without-confirmation"):
+ # A later whole-row refresh must see acknowledged moves/deletions before writing.
+ await workflow.execute_activity(
+ confirm_signal_spend_activity,
+ publication,
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(
+ initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=10)
+ ),
+ )
+
if updates == self._updates:
return
if workflow.info().is_continue_as_new_suggested():
diff --git a/products/signals/backend/temporal/summary.py b/products/signals/backend/temporal/summary.py
index d6e6916be873..cafc660aaa3b 100644
--- a/products/signals/backend/temporal/summary.py
+++ b/products/signals/backend/temporal/summary.py
@@ -3,7 +3,7 @@
import json
import asyncio
from collections.abc import Mapping
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from datetime import timedelta
from typing import Any
@@ -16,6 +16,7 @@ import temporalio
import posthoganalytics
from structlog.types import FilteringBoundLogger
from temporalio import workflow
+from temporalio.client import WorkflowExecutionStatus
from temporalio.common import RetryPolicy, WorkflowIDReusePolicy
from temporalio.exceptions import (
ActivityError,
@@ -31,6 +32,7 @@ from posthog.kafka_client.routing import get_producer
from posthog.kafka_client.topics import KAFKA_SIGNALS_REPORT_COMPLETED
from posthog.models import Organization, Team
from posthog.sync import database_sync_to_async
+from posthog.temporal.common.client import async_connect
from posthog.temporal.common.scoped import scoped_temporal
from posthog.temporal.common.utils import close_db_connections
@@ -46,6 +48,7 @@ from products.signals.backend.models import (
SIGNALS_AT_RUN_INCREMENT,
SignalReport,
SignalReportArtefact,
+ SignalSpend,
SignalTeamConfig,
)
from products.signals.backend.quota import (
@@ -57,7 +60,12 @@ from products.signals.backend.report_generation.research import ActionabilityCho
from products.signals.backend.report_generation.select_repo import RepoSelectionResult
from products.signals.backend.report_metric_query_access import query_filter_shape_allows_read
from products.signals.backend.report_metrics import REPORT_METRIC_GOAL_FIELDS
-from products.signals.backend.spend import publish_report_spend
+from products.signals.backend.spend import (
+ begin_report_spend,
+ finish_report_spend,
+ report_triggering_signal,
+ submit_signal_metadata_updates,
+)
from products.signals.backend.stack_plan import create_layer_reports, start_unblocked_layers_of_plan
from products.signals.backend.temporal import metrics
from products.signals.backend.temporal.agentic.report import (
@@ -87,6 +95,7 @@ from products.signals.backend.temporal.types import (
SignalReportSummaryWorkflowInputs,
next_research_bucket,
)
+from products.tasks.backend.facade.api import get_task_run
logger = structlog.get_logger(__name__)
@@ -215,6 +224,33 @@ class ReportDecision:
pending_reason: str | None = None
+async def check_report_task_settlement_activity(input: SignalReportSummaryWorkflowInputs) -> None:
+ @database_sync_to_async
+ def task_run_ids() -> list[str]:
+ return [
+ str(run_id)
+ for run_id in SignalSpend.objects.for_team(input.team_id)
+ .filter(signal_id=input.signal_id, task_run_id__isnull=False)
+ .values_list("task_run_id", flat=True)
+ ]
+
+ run_ids = await task_run_ids()
+ if not run_ids:
+ return
+ client = await async_connect()
+ for run_id in run_ids:
+ run = await database_sync_to_async(get_task_run)(run_id, team_id=input.team_id)
+ if run is None:
+ raise ApplicationError("Spend task run no longer exists", non_retryable=True)
+ execution = await client.get_workflow_handle(run.workflow_id).describe(rpc_timeout=timedelta(seconds=5))
+ if execution.status == WorkflowExecutionStatus.RUNNING:
+ # Retry backoff frees the shared activity slot while the task settles.
+ raise ApplicationError("Task spend is still settling", type="TaskSpendPending")
+ if execution.status != WorkflowExecutionStatus.COMPLETED:
+ raise ApplicationError("Task workflow did not complete settlement", non_retryable=True)
+
+
RESEARCH_SANDBOX_RETRY_DELAYS = (timedelta(minutes=10), timedelta(minutes=20), timedelta(minutes=40))
RESEARCH_SANDBOX_RETRY_BUDGET = sum(RESEARCH_SANDBOX_RETRY_DELAYS, timedelta())
@@ -229,19 +265,6 @@ def _is_research_sandbox_unavailable(error: BaseException) -> bool:
@temporalio.workflow.defn(name="signal-report-summary")
class SignalReportSummaryWorkflow:
- """
- Workflow that runs when a signal report is promoted to candidate status.
-
- Flow:
- 1. Fetch all signals for the report from ClickHouse
- 2. Mark report as in_progress
- 3. Run safety judge to filter unsafe reports
- 4. Select a repository for the agentic research
- 5. Run the agentic report research flow
- 6. Apply the resulting actionability decision to transition the report
- 7. If new signals arrived during the run, loop back to step 1
- """
-
@staticmethod
def parse_inputs(inputs: list[str]) -> SignalReportSummaryWorkflowInputs:
loaded = json.loads(inputs[0])
@@ -251,6 +274,34 @@ class SignalReportSummaryWorkflow:
def workflow_id_for(team_id: int, report_id: str) -> str:
return f"signals-report:{team_id}:{report_id}"
+ async def _wait_for_task_settlement(self, inputs: SignalReportSummaryWorkflowInputs) -> None:
+ if inputs.signal_id and workflow.patched("signals-wait-for-task-settlement"):
+ try:
+ await workflow.execute_activity(
+ check_report_task_settlement_activity,
+ inputs,
+ start_to_close_timeout=timedelta(minutes=1),
+ schedule_to_close_timeout=timedelta(minutes=20),
+ retry_policy=RetryPolicy(
+ initial_interval=timedelta(seconds=5), maximum_interval=timedelta(seconds=30)
+ ),
+ )
+ except ActivityError:
+ # Billing failure must not rerun paid research or discard its result.
+ workflow.logger.exception("signals.spend.accounting_failed", extra={"signal_id": inputs.signal_id})
+
+ async def _publish_spend(self, inputs: SignalReportSummaryWorkflowInputs) -> None:
+ try:
+ await workflow.execute_activity(
+ publish_report_spend_activity,
+ inputs,
+ start_to_close_timeout=timedelta(minutes=20),
+ schedule_to_close_timeout=timedelta(minutes=20),
+ retry_policy=RetryPolicy(initial_interval=timedelta(seconds=5), maximum_interval=timedelta(seconds=30)),
+ )
+ except Exception:
+ workflow.logger.exception("Failed to publish completed signal spend")
+
@temporalio.workflow.run
async def run(self, inputs: SignalReportSummaryWorkflowInputs) -> None:
with posthoganalytics.new_context(capture_exceptions=False):
@@ -260,16 +311,10 @@ class SignalReportSummaryWorkflow:
try:
await self._run_impl(inputs)
finally:
- if workflow.patched("signals-publish-spend-on-completion"):
- try:
- await workflow.execute_activity(
- publish_report_spend_activity,
- inputs,
- start_to_close_timeout=timedelta(minutes=20),
- retry_policy=RetryPolicy(maximum_attempts=1),
- )
- except Exception:
- workflow.logger.exception("Failed to publish completed report spend")
+ if workflow.patched("signals-publish-spend-on-completion") and not workflow.patched(
+ "signals-spend-explicit-owner"
+ ):
+ await self._publish_spend(inputs)
async def _run_impl(self, inputs: SignalReportSummaryWorkflowInputs) -> None:
# Bind team_id + report_id so all logs flow to the log_entries sink (the Temporal
@@ -287,7 +332,24 @@ class SignalReportSummaryWorkflow:
for _ in range(max_iterations):
# Loop internally rather than spawning new workflows because summary workflows are
# fire-and-forget (ParentClosePolicy.ABANDON), so there's no external caller to wait/restart them.
- should_loop = await self._run_once(inputs, log)
+ if workflow.patched("signals-spend-explicit-owner"):
+ signal_id = inputs.signal_id or await workflow.execute_activity(
+ report_spend_owner_activity,
+ inputs,
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ if signal_id is None:
+ log.info("Report has no open spend owner, waiting for a new triggering signal")
+ return
+ try:
+ should_loop = await self._run_once(replace(inputs, signal_id=signal_id), log)
+ finally:
+ if signal_id:
+ await self._publish_spend(replace(inputs, signal_id=signal_id))
+ inputs = replace(inputs, signal_id=None)
+ else:
+ should_loop = await self._run_once(inputs, log)
if not should_loop:
return
log.warning(
@@ -414,57 +476,68 @@ class SignalReportSummaryWorkflow:
):
log.info("Report run paused: org over self-driving credits quota", stage="summary_entry")
return False
- # 1. Fetch signals for the report
- fetch_result = await self._fetch_signals(inputs, log)
- if not fetch_result.signals:
- # patched(): same marker as the retry loop, so pre-patch histories replay straight to failure.
- if workflow.patched("signals-empty-fetch-retry") and await workflow.execute_activity(
- report_has_assigned_signals_activity,
- ReportHasAssignedSignalsInput(team_id=inputs.team_id, report_id=inputs.report_id),
- start_to_close_timeout=timedelta(minutes=1),
- retry_policy=RetryPolicy(maximum_attempts=3),
- ):
- # Signals are assigned in Postgres but not yet readable in ClickHouse. Failing here
- # would be terminal, so leave the report as it is: grouping re-promotes a candidate
- # on the next matching signal, and that run sees the rows.
- log.warning("Report has assigned signals that are not yet visible in ClickHouse, deferring run")
- metrics.increment_report_run_deferred("signals_not_visible")
- return False
- log.error("No signals found for report, marking as failed")
- await workflow.execute_activity(
- mark_report_failed_activity,
- MarkReportFailedInput(
- team_id=inputs.team_id,
- report_id=inputs.report_id,
- error="No signals found",
- failure_reason="no_signals_found",
- ),
- start_to_close_timeout=timedelta(minutes=1),
- retry_policy=RetryPolicy(maximum_attempts=3),
- )
- # No loop, as no signals to process
- return False
- signal_count = len(fetch_result.signals)
- source_products = sorted({s.source_product for s in fetch_result.signals})
- # 2. Mark report as in_progress to prevent duplicate runs while this workflow is active
- await workflow.execute_activity(
- mark_report_in_progress_activity,
- MarkReportInProgressInput(
- team_id=inputs.team_id,
- report_id=inputs.report_id,
- signal_count=signal_count,
- source_products=source_products,
- ),
- start_to_close_timeout=timedelta(minutes=1),
- retry_policy=RetryPolicy(maximum_attempts=3),
- )
+ claim_before_fetch = workflow.patched("signals-spend-explicit-owner")
try:
+ if claim_before_fetch:
+ # Grouping must reserve arrivals for the next pass before we take the ClickHouse snapshot.
+ await workflow.execute_activity(
+ mark_report_in_progress_activity,
+ MarkReportInProgressInput(
+ team_id=inputs.team_id, report_id=inputs.report_id, signal_id=inputs.signal_id
+ ),
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ fetch_result = await self._fetch_signals(inputs, log)
+ if not fetch_result.signals:
+ # patched(): same marker as the retry loop, so pre-patch histories replay straight to failure.
+ if workflow.patched("signals-empty-fetch-retry") and await workflow.execute_activity(
+ report_has_assigned_signals_activity,
+ ReportHasAssignedSignalsInput(team_id=inputs.team_id, report_id=inputs.report_id),
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ ):
+ # Allow grouping to retry when assigned signals reach ClickHouse.
+ log.warning("Report has assigned signals that are not yet visible in ClickHouse, deferring run")
+ metrics.increment_report_run_deferred("signals_not_visible")
+ if claim_before_fetch:
+ await self._revert_report_to_candidate(inputs)
+ return False
+ log.error("No signals found for report, marking as failed")
+ await workflow.execute_activity(
+ mark_report_failed_activity,
+ MarkReportFailedInput(
+ team_id=inputs.team_id,
+ report_id=inputs.report_id,
+ error="No signals found",
+ failure_reason="no_signals_found",
+ ),
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ return False
+ signal_count = len(fetch_result.signals)
+ source_products = sorted({s.source_product for s in fetch_result.signals})
+ if not claim_before_fetch:
+ await workflow.execute_activity(
+ mark_report_in_progress_activity,
+ MarkReportInProgressInput(
+ team_id=inputs.team_id,
+ report_id=inputs.report_id,
+ signal_id=inputs.signal_id,
+ signal_count=signal_count,
+ source_products=source_products,
+ ),
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
# 3. Run safety judge first to avoid passing unsafe report into the agentic research
safety_result = await workflow.execute_activity(
report_safety_judge_activity,
SafetyJudgeInput(
team_id=inputs.team_id,
report_id=inputs.report_id,
+ signal_id=inputs.signal_id,
signals=fetch_result.signals,
),
start_to_close_timeout=timedelta(minutes=5),
@@ -509,6 +582,7 @@ class SignalReportSummaryWorkflow:
SelectRepositoryInput(
team_id=inputs.team_id,
report_id=inputs.report_id,
+ signal_id=inputs.signal_id,
signals=fetch_result.signals,
),
# Budget = heavy-cache warmup (cold, ≤1000 repos, rate-limit backoffs) +
@@ -518,6 +592,7 @@ class SignalReportSummaryWorkflow:
# Retry once: the sandbox agent turn can time out (poll_for_turn) or hit a transient LLM 429.
retry_policy=RetryPolicy(maximum_attempts=2),
)
+ await self._wait_for_task_settlement(inputs)
if repo_result.repository is None:
log.warning(
"Report has no repository selected",
@@ -547,11 +622,13 @@ class SignalReportSummaryWorkflow:
RunAgenticReportInput(
team_id=inputs.team_id,
report_id=inputs.report_id,
+ signal_id=inputs.signal_id,
signals=fetch_result.signals,
repo_selection=repo_result,
repo_selection_as_of=repo_selection_as_of,
),
)
+ await self._wait_for_task_settlement(inputs)
decision = ReportDecision(
title=agentic_result.title,
summary=agentic_result.summary,
@@ -862,12 +939,13 @@ async def revert_report_to_candidate_activity(input: RevertReportToCandidateInpu
)
-@dataclass
+@dataclass(frozen=False)
class MarkReportInProgressInput:
team_id: int
report_id: str
- signal_count: int
+ signal_count: int | None = None
source_products: list[str] = field(default_factory=list)
+ signal_id: str | None = None
@temporalio.activity.defn
@@ -887,10 +965,15 @@ async def mark_report_in_progress_activity(input: MarkReportInProgressInput) ->
report = SignalReport.objects.select_for_update().get(id=input.report_id, team_id=input.team_id)
if report.status == SignalReport.Status.IN_PROGRESS:
return report.run_count, True
+ signal_id = input.signal_id or (str(report.triggering_signal_id) if report.triggering_signal_id else None)
+ if signal_id:
+ begin_report_spend(team_id=input.team_id, report_id=input.report_id, signal_id=signal_id)
updated_fields = report.transition_to(
SignalReport.Status.IN_PROGRESS, signals_at_run_increment=SIGNALS_AT_RUN_INCREMENT
)
report.pending_triggering_signal_id = None
+ if input.signal_count is None:
+ input.signal_count = report.signal_count
report.researching_signal_count = input.signal_count
report.save(update_fields=[*updated_fields, "pending_triggering_signal_id", "researching_signal_count"])
return report.run_count, False
@@ -910,6 +993,7 @@ async def mark_report_in_progress_activity(input: MarkReportInProgressInput) ->
)
return
+ assert input.signal_count is not None
team = await Team.objects.select_related("organization").aget(pk=input.team_id)
_capture_report_event(
event="signal_report_started",
@@ -1556,4 +1640,21 @@ async def publish_report_completed_activity(input: PublishReportCompletedInput)
@temporalio.activity.defn
@scoped_temporal()
async def publish_report_spend_activity(input: SignalReportSummaryWorkflowInputs) -> None:
- await publish_report_spend(team_id=input.team_id, report_id=input.report_id)
+ # Older histories did not snapshot their owner. New passes always supply it explicitly.
+ signal_id = input.signal_id or await database_sync_to_async(report_triggering_signal)(
+ team_id=input.team_id, report_id=input.report_id
+ )
+ if signal_id:
+ await database_sync_to_async(finish_report_spend)(
+ team_id=input.team_id, report_id=input.report_id, signal_id=signal_id
+ )
+ await submit_signal_metadata_updates(team_id=input.team_id, signal_ids=[signal_id])
+
+
+@scoped_temporal()
+@close_db_connections
+async def report_spend_owner_activity(input: SignalReportSummaryWorkflowInputs) -> str | None:
+ return input.signal_id or await database_sync_to_async(report_triggering_signal)(
+ team_id=input.team_id, report_id=input.report_id
+ )
diff --git a/products/signals/backend/temporal/types.py b/products/signals/backend/temporal/types.py
index 49d3fece50d3..b08f0da32725 100644
--- a/products/signals/backend/temporal/types.py
+++ b/products/signals/backend/temporal/types.py
@@ -202,6 +202,7 @@ class SignalReportSummaryWorkflowInputs:
requested_implementation_user_id: int | None = None
requested_implementation_task_id: str | None = None
requested_after_run_count: int | None = None
+ signal_id: str | None = None
@dataclass
diff --git a/products/signals/backend/test/test_api.py b/products/signals/backend/test/test_api.py
index 5b366c8e8a90..c729b5e51fd9 100644
--- a/products/signals/backend/test/test_api.py
+++ b/products/signals/backend/test/test_api.py
@@ -20,7 +20,8 @@ from products.signals.backend.temporal.emitter import SignalEmitterWorkflow
@pytest.fixture
-def team_stub() -> MagicMock:
+def team_stub(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+ monkeypatch.setattr("products.signals.backend.facade.api.finalized_signal_ids", MagicMock(return_value=set()))
org = MagicMock()
org.is_ai_data_processing_approved = True
team = MagicMock()
@@ -84,6 +85,32 @@ GITHUB_ISSUE_EXTRA = {
@pytest.mark.asyncio
class TestEmitSignalValidation:
+ @pytest.mark.parametrize("ai_approved, source_enabled", [(False, True), (True, False)])
+ @pytest.mark.parametrize("has_spend_owner", [False, True])
+ async def test_disabled_emission_completes_existing_spend(
+ self, team_stub: MagicMock, ai_approved: bool, source_enabled: bool, has_spend_owner: bool
+ ) -> None:
+ team_stub.organization.is_ai_data_processing_approved = ai_approved
+ signal_id = str(uuid.uuid4()) if has_spend_owner else None
+ with (
+ patch.object(SignalSourceConfig, "is_source_enabled", return_value=source_enabled),
+ patch("products.signals.backend.facade.api.complete_signal_pipeline") as complete,
+ patch("products.signals.backend.facade.api.async_connect") as connect,
+ ):
+ await emit_signal(
+ team=team_stub,
+ source_product="zendesk",
+ source_type="ticket",
+ source_id="disabled-source",
+ description="An existing signal cannot enter the disabled pipeline",
+ signal_id=signal_id,
+ )
+ if signal_id:
+ complete.assert_called_once_with(team_id=team_stub.id, signal_id=signal_id, billing_exempt=False)
+ else:
+ complete.assert_not_called()
+ connect.assert_not_called()
+
@pytest.mark.parametrize(
"source_product, source_type, extra",
[
diff --git a/products/signals/backend/test/test_assign_and_emit_signal.py b/products/signals/backend/test/test_assign_and_emit_signal.py
index 9baf4011e0a3..0f4b8b801d8b 100644
--- a/products/signals/backend/test/test_assign_and_emit_signal.py
+++ b/products/signals/backend/test/test_assign_and_emit_signal.py
@@ -17,10 +17,10 @@ from products.signals.backend.artefact_attribution import ArtefactAttribution
from products.signals.backend.artefact_schemas import Dismissal, RelatedTo, ReportLink
from products.signals.backend.daily_limit import DailyReportLimitGate
from products.signals.backend.enums import ReportLinkKind
-from products.signals.backend.models import SignalEmissionRecord, SignalReport, SignalReportArtefact
+from products.signals.backend.models import SignalEmissionRecord, SignalReport, SignalReportArtefact, SignalSpend
from products.signals.backend.quota import SelfDrivingQuotaGate
from products.signals.backend.report_merge import MERGE_DISMISSAL_REASON
-from products.signals.backend.spend import record_llm_spend
+from products.signals.backend.spend import complete_signal_pipeline, finish_report_spend, record_llm_spend
from products.signals.backend.temporal.grouping import (
WEIGHT_THRESHOLD,
AssignAndEmitSignalInput,
@@ -32,6 +32,7 @@ from products.signals.backend.temporal.grouping import (
verify_match_specificity_activity,
)
from products.signals.backend.temporal.signal_queries import SourceSignalMatch
+from products.signals.backend.temporal.summary import MarkReportInProgressInput, mark_report_in_progress_activity
from products.signals.backend.temporal.types import (
ExistingReportMatch,
MatchedMetadata,
@@ -132,7 +133,6 @@ async def test_new_match_creates_potential_report_below_threshold(ateam, patch_s
input_ = _build_input(ateam.id, _new_match(), weight=WEIGHT_THRESHOLD * 0.5)
await database_sync_to_async(record_llm_spend)(
- "example-generation",
token_cost_microusd=123_456,
team_id=ateam.id,
signal_id=input_.signal_id,
@@ -285,7 +285,8 @@ async def test_potential_does_not_promote_when_signals_at_run_gate_holds(ateam):
@pytest.mark.asyncio
@pytest.mark.django_db
-async def test_candidate_returns_promoted_true_without_changing_status(ateam):
[email protected]("previous_pass", ["active", "completed", "accounting_failed"])
+async def test_candidate_returns_promoted_true_without_changing_status(ateam, previous_pass):
"""A new signal arriving at an already-CANDIDATE report must:
- return promoted=True so the caller spawns a recovery workflow
- leave status at CANDIDATE
@@ -299,9 +300,31 @@ async def test_candidate_returns_promoted_true_without_changing_status(ateam):
total_weight=2.0,
signal_count=3,
)
+ await database_sync_to_async(record_llm_spend)(
+ team_id=ateam.id,
+ signal_id=str(original_driver),
+ stage="grouping",
+ token_cost_microusd=None if previous_pass == "accounting_failed" else 10000,
+ )
+ if previous_pass != "active":
+ await database_sync_to_async(finish_report_spend)(
+ team_id=ateam.id, report_id=str(report.id), signal_id=str(original_driver)
+ )
+ old_spend = await database_sync_to_async(
+ lambda: SignalSpend.objects.for_team(ateam.id).filter(signal_id=original_driver).values().get()
+ )()
input_ = _build_input(ateam.id, _existing_match(str(report.id)), weight=0.5)
+ await database_sync_to_async(record_llm_spend)(
+ team_id=ateam.id, signal_id=input_.signal_id, stage="grouping", token_cost_microusd=10_000
+ )
result = await assign_and_emit_signal_activity(input_)
+ assert (
+ await database_sync_to_async(lambda: SignalSpend.objects.for_team(ateam.id).get(signal_id=input_.signal_id))()
+ ).finalized_at is None
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=ateam.id, signal_id=input_.signal_id, report_id=result.report_id
+ )
assert result.promoted is True
refreshed = await database_sync_to_async(SignalReport.objects.get)(id=report.id)
@@ -309,7 +332,18 @@ async def test_candidate_returns_promoted_true_without_changing_status(ateam):
assert refreshed.total_weight == pytest.approx(2.5)
assert refreshed.signal_count == 4
- assert refreshed.triggering_signal_id == original_driver
+ expected_owner = str(original_driver) if previous_pass == "active" else input_.signal_id
+ assert str(refreshed.triggering_signal_id) == result.triggering_signal_id == expected_owner
+ assert (
+ await database_sync_to_async(
+ lambda: SignalSpend.objects.for_team(ateam.id).filter(signal_id=original_driver).values().get()
+ )()
+ == old_spend
+ )
+ incoming = await database_sync_to_async(
+ lambda: SignalSpend.objects.for_team(ateam.id).get(signal_id=input_.signal_id)
+ )()
+ assert (incoming.finalized_at is None) is (previous_pass != "active")
@pytest.mark.asyncio
@@ -1254,7 +1288,6 @@ async def test_deleted_report_skips_counter_updates_and_marks_signal_deleted(ate
input_ = _build_input(ateam.id, _existing_match(str(report.id)), weight=0.5)
await database_sync_to_async(record_llm_spend)(
- "example-generation",
token_cost_microusd=123_456,
team_id=ateam.id,
signal_id=input_.signal_id,
@@ -1431,18 +1464,26 @@ async def test_daily_limit_gate_emits_no_event_when_signal_would_not_promote(ate
@pytest.mark.asyncio
@pytest.mark.django_db
-async def test_signal_crossing_next_bucket_owns_next_research_pass(ateam):
+ "source_product,exempt", [("health_checks", True), ("engineering_analytics", True), ("error_tracking", False)]
+)
+async def test_signal_crossing_next_bucket_owns_next_research_pass(ateam, source_product, exempt):
original_driver = uuid.uuid4()
report = await database_sync_to_async(SignalReport.objects.create)(
team=ateam,
- status=SignalReport.Status.IN_PROGRESS,
+ status=SignalReport.Status.CANDIDATE,
signal_count=2,
total_weight=2,
- researching_signal_count=2,
triggering_signal_id=original_driver,
)
+ await mark_report_in_progress_activity(
+ MarkReportInProgressInput(team_id=ateam.id, report_id=str(report.id), signal_id=str(original_driver))
+ )
for _ in range(3):
- signal = _build_input(ateam.id, _existing_match(str(report.id)))
+ signal = _build_input(ateam.id, _existing_match(str(report.id)), source_product=source_product)
+ await database_sync_to_async(record_llm_spend)(
+ team_id=ateam.id, signal_id=signal.signal_id, stage="grouping", token_cost_microusd=100
+ )
await assign_and_emit_signal_activity(signal)
await database_sync_to_async(report.refresh_from_db)()
assert report.triggering_signal_id == original_driver
@@ -1453,3 +1494,17 @@ async def test_signal_crossing_next_bucket_owns_next_research_pass(ateam):
assert str(report.pending_triggering_signal_id) == next_driver
else:
assert str(report.pending_triggering_signal_id) == next_driver
+
+ await database_sync_to_async(finish_report_spend)(
+ team_id=ateam.id, report_id=str(report.id), signal_id=str(original_driver)
+ )
+ pending = await database_sync_to_async(lambda: SignalSpend.objects.for_team(ateam.id).get(signal_id=next_driver))()
+ finalized_at = pending.finalized_at
+ assert finalized_at is not None
+ assert pending.billing_exempt is exempt
+ await database_sync_to_async(complete_signal_pipeline)(
+ team_id=ateam.id, report_id=str(report.id), signal_id=next_driver, billing_exempt=exempt
+ )
+ await database_sync_to_async(pending.refresh_from_db)()
+ assert pending.finalized_at == finalized_at
+ assert pending.billing_exempt is exempt
diff --git a/products/signals/backend/test/test_auto_start.py b/products/signals/backend/test/test_auto_start.py
index a0515582d16f..40faffce8902 100644
--- a/products/signals/backend/test/test_auto_start.py
+++ b/products/signals/backend/test/test_auto_start.py
@@ -665,7 +665,8 @@ def test_human_pin_does_not_authorize_automatic_execution(
@pytest.mark.parametrize(
"concurrent_change", [None, {"summary": "A newer fix"}, {"run_count": 2}, {"content_revision_count": 1}]
)
-def test_create_implementation_task_if_absent_is_idempotent(organization, team, concurrent_change):
[email protected]("spend_kind", ["reviewer_triggered_implementation", "scout_triggered_implementation"])
+def test_create_implementation_task_if_absent_is_idempotent(organization, team, concurrent_change, spend_kind):
# The locked create guards against duplicate auto-start tasks: a second evaluation that
# observes the link row must no-op rather than spawn another Task / draft PR. It also asserts
# the facade is invoked with the SIGNAL_REPORT origin and ai_stage="implementation" so the
@@ -694,6 +695,7 @@ def test_create_implementation_task_if_absent_is_idempotent(organization, team,
kwargs = {
"team_id": team.id,
"report_id": str(report.id),
+ "implementation_spend_kind": spend_kind,
"title": "t",
"description": "d",
"expected_content": ImplementationReportContent.from_report(report),
@@ -725,6 +727,7 @@ def test_create_implementation_task_if_absent_is_idempotent(organization, team,
call_kwargs = mock_create.call_args.kwargs
assert call_kwargs["origin_product"] == tasks_facade.TaskOriginProduct.SIGNAL_REPORT
assert call_kwargs["ai_stage"] == "implementation"
+ assert call_kwargs["implementation_spend_kind"] == spend_kind
assert call_kwargs["internal"] is True
# The description's memory protocol is rendered from this same posture, so a posture that stops
# granting the scratchpad would leave the run told to remember with no tool to remember with.
diff --git a/products/signals/backend/test/test_billing.py b/products/signals/backend/test/test_billing.py
index c1ee2caadb00..5bbf9922613c 100644
--- a/products/signals/backend/test/test_billing.py
+++ b/products/signals/backend/test/test_billing.py
@@ -1,7 +1,10 @@
from datetime import UTC, datetime
from typing import TYPE_CHECKING
+from uuid import uuid4
+import time_machine
from posthog.test.base import BaseTest
+from unittest.mock import patch
from django.apps import apps
@@ -20,7 +23,13 @@ from products.signals.backend.billing import (
mark_report_billing_exempt,
system_billing_exempt_reason,
)
-from products.signals.backend.models import SignalReport, SignalReportRefund, SignalReportTask, SignalScoutRun
+from products.signals.backend.models import (
+ SignalReport,
+ SignalReportRefund,
+ SignalReportTask,
+ SignalScoutRun,
+ SignalSpend,
+)
from products.signals.backend.scout_harness.lazy_seed import HARNESS_SEEDED_BY, _compute_row_hash
from products.skills.backend.models.skills import LLMSkill
@@ -129,23 +138,46 @@ class TestSignalsBilling(BaseTest):
# 1 credit = $0.01, so $15 == 1500 credits.
self.assertEqual(SIGNALS_CREDITS_PER_REPORT_WITH_PR, 1500)
- @parameterized.expand([("organization_id", False), ("organization_ids", True)])
- def test_organization_filter_excludes_other_organizations(self, _name: str, as_list: bool) -> None:
+ @parameterized.expand([(False,), (True,)])
+ def test_organization_filter_excludes_other_organizations(self, as_list: bool) -> None:
other_team = Team.objects.create(organization=Organization.objects.create(name="Other org"))
- for team in (self.team, other_team):
- self._pr_run(self._report(team=team), created_at=_at(10), team=team)
-
+ with time_machine.travel(_at(10), tick=False):
+ for team in (self.team, other_team):
+ self._pr_run(self._report(team=team), created_at=_at(10), team=team)
+ SignalSpend.objects.for_team(team.id).create(
+ team=team,
+ kind=SignalSpend.Kind.AD_HOC,
+ signal_id=uuid4(),
+ spend=20,
+ finalized_at=_at(10),
+ )
org_filter = (
{"organization_ids": [self.organization.id]} if as_list else {"organization_id": self.organization.id}
)
- self.assertEqual(
- dict(get_signals_billing_credits_by_team(PERIOD_START, PERIOD_END, **org_filter)), {self.team.id: 1500}
- )
-
- def test_report_with_pr_billed_flat(self) -> None:
+ with patch("posthoganalytics.get_feature_flag", return_value="2026-06-01") as flag:
+ self.assertEqual(
+ dict(get_signals_billing_credits_by_team(PERIOD_START, PERIOD_END, **org_filter)),
+ {self.team.id: 1500},
+ )
+ flag.assert_not_called()
+
+ @parameterized.expand([(None, 1500), ("posthog_ai", 0)])
+ def test_report_with_pr_billed_only_when_not_billed_as_phai(self, product: str | None, credits: int) -> None:
report = self._report()
- self._pr_run(report, created_at=_at(10))
- self.assertEqual(self._credits(), {self.team.id: 1500})
+ run = self._pr_run(report, created_at=_at(10))
+ if product:
+ run.state = {"billing_product": product}
+ run.save(update_fields=["state"])
+ self.assertEqual(self._credits(), {self.team.id: credits} if credits else {})
+ self.assertEqual(first_billable_pr_run(report.id, team_id=report.team_id) is not None, bool(credits))
+ _task_run_model().objects.create(
+ team=self.team,
+ task=run.task,
+ created_at=_at(11),
+ output={"pr_url": "https://github.com/x/y/pull/2"},
+ state={"billing_product": "posthog_ai"},
+ )
+ self.assertEqual(self._credits(), {self.team.id: credits} if credits else {})
def test_report_without_pr_not_billed(self) -> None:
self._report()
@@ -389,14 +421,15 @@ class TestSignalsBilling(BaseTest):
self._pr_run(report, created_at=_at(5), pr_url="https://github.com/x/y/pull/1")
self._pr_run(report, created_at=_at(12), pr_url="https://github.com/x/y/pull/2")
self.assertEqual(
- first_billable_pr_run(report.id),
+ first_billable_pr_run(report.id, team_id=report.team_id),
FirstBillablePrRun(created_at=_at(5), pr_url="https://github.com/x/y/pull/1"),
)
+ self.assertIsNone(first_billable_pr_run(report.id, team_id=report.team_id + 1))
def test_first_billable_pr_run_none_without_billable_run(self) -> None:
report = self._report()
self._pr_run(report, created_at=_at(3), pr_url=None)
- self.assertIsNone(first_billable_pr_run(report.id))
+ self.assertIsNone(first_billable_pr_run(report.id, team_id=report.team_id))
def test_aggregates_across_teams_and_reports(self) -> None:
team_b = Team.objects.create(organization=self.organization, name="team-b")
diff --git a/products/signals/backend/test/test_buffer_workflow.py b/products/signals/backend/test/test_buffer_workflow.py
index 0d17999ff9c1..a7d5d34cd14e 100644
--- a/products/signals/backend/test/test_buffer_workflow.py
+++ b/products/signals/backend/test/test_buffer_workflow.py
@@ -38,9 +38,11 @@ def _signal(team_id: int) -> EmitSignalInputs:
class _Recorder:
- def __init__(self, over_quota: bool, safety_failures: int = 0) -> None:
+ def __init__(self, over_quota: bool, safety_failures: int = 0, mixed_batch: bool = False) -> None:
self.over_quota = over_quota
self.safety_failures = safety_failures
+ self.mixed_batch = mixed_batch
+ self.safety_sources: list[str | None] = []
self.quota_checks = 0
self.safety_checks = 0
self.flushes = 0
@@ -61,6 +63,9 @@ async def _drive(recorder: _Recorder) -> None:
@activity.defn(name="safety_filter_activity")
async def fake_safety(_input: SafetyFilterInput) -> SafetyFilterOutput:
recorder.safety_checks += 1
+ recorder.safety_sources.append(_input.source_id)
+ if _input.source_id == "unsafe":
+ return SafetyFilterOutput(safe=False, threat_type="test", explanation=None)
if recorder.safety_failures > 0:
recorder.safety_failures -= 1
raise ApplicationError("safety provider unavailable", non_retryable=True)
@@ -84,13 +89,16 @@ async def _drive(recorder: _Recorder) -> None:
activities=[fake_quota, fake_safety, fake_flush, fake_grouping],
workflow_runner=UnsandboxedWorkflowRunner(),
):
+ unsafe = _signal(1)
+ unsafe.source_id = "unsafe"
handle = await env.client.start_workflow(
BufferSignalsWorkflow.run,
- BufferSignalsInput(team_id=1),
+ BufferSignalsInput(team_id=1, pending_signals=[unsafe, _signal(1)] if recorder.mixed_batch else []),
id=f"buffer-{uuid.uuid4()}",
task_queue=TASK_QUEUE,
)
- await handle.signal(BufferSignalsWorkflow.submit_signal, _signal(1))
+ if not recorder.mixed_batch:
+ await handle.signal(BufferSignalsWorkflow.submit_signal, _signal(1))
terminal = recorder.gate_reached if recorder.over_quota else recorder.flow_done
await asyncio.wait_for(terminal.wait(), timeout=30)
with suppress(RPCError):
@@ -119,10 +127,12 @@ async def test_under_quota_batch_flows_through():
@pytest.mark.asyncio
-async def test_safety_failure_requeues_the_drained_batch():
- recorder = _Recorder(over_quota=False, safety_failures=1)
[email protected]("mixed_batch", [False, True])
+async def test_safety_failure_requeues_only_failed_signals(mixed_batch: bool):
+ recorder = _Recorder(over_quota=False, safety_failures=1, mixed_batch=mixed_batch)
await _drive(recorder)
- assert recorder.safety_checks == 2
+ assert recorder.safety_checks == 2 + int(mixed_batch)
+ assert recorder.safety_sources.count("unsafe") == int(mixed_batch)
assert recorder.flushes == 1
assert recorder.grouping_starts == 1
diff --git a/products/signals/backend/test/test_grouping_refusal_batch.py b/products/signals/backend/test/test_grouping_refusal_batch.py
index a0d6abf8055c..2f90070f61fd 100644
--- a/products/signals/backend/test/test_grouping_refusal_batch.py
+++ b/products/signals/backend/test/test_grouping_refusal_batch.py
@@ -1,7 +1,9 @@
from datetime import UTC, datetime
import pytest
-from unittest.mock import patch
+from unittest.mock import Mock, patch
+
+from temporalio.exceptions import ActivityError, RetryState
from products.signals.backend.temporal import grouping
from products.signals.backend.temporal.signal_queries import (
@@ -19,20 +21,30 @@ from products.signals.backend.temporal.types import (
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "quarantined,isolate_refusals,duplicate",
+ "quarantined,isolate_refusals,duplicate,failure_at",
[
- ((True, False), True, False),
- ((False, True), True, False),
- ((True, True), True, False),
- ((False, False), False, False),
- ((True, False), True, True),
+ ((True, False), True, False, None),
+ ((False, True), True, False, None),
+ ((True, True), True, False, None),
+ ((False, False), False, False, None),
+ ((True, False), True, True, None),
+ ((False, False), True, False, "get_embedding_activity"),
+ ((False, False), True, False, "wait_for_signal_in_clickhouse_activity"),
+ ((False, False), True, False, "complete_signal_grouping_activity"),
],
)
async def test_quarantined_signals_do_not_block_healthy_batch_members(
- quarantined: tuple[bool, bool], isolate_refusals: bool, duplicate: bool
+ quarantined: tuple[bool, bool], isolate_refusals: bool, duplicate: bool, failure_at: str | None
) -> None:
batch = [
- EmitSignalInputs(team_id=1, source_product="test", source_type="issue", source_id=str(i), description=str(i))
+ EmitSignalInputs(
+ team_id=1,
+ signal_id=str(i),
+ source_product="test",
+ source_type="issue",
+ source_id=str(i),
+ description=str(i),
+ )
for i in range(2)
]
if duplicate:
@@ -43,6 +55,7 @@ async def test_quarantined_signals_do_not_block_healthy_batch_members(
source_product="github",
source_type="issue",
source_id="source-sync",
+ signal_id="source-sync",
description="2",
sync_with_source=True,
source_match_signal_id="previous-signal",
@@ -52,7 +65,26 @@ async def test_quarantined_signals_do_not_block_healthy_batch_members(
query_requests: list[str] = []
emitted: list[grouping.AssignAndEmitSignalInput] = []
+ completions: list[grouping.SignalGroupingCompletion] = []
+ visibility_confirmed = False
+
async def execute(activity: object, input: object, **kwargs: object) -> object:
+ nonlocal visibility_confirmed
+ if failure_at is not None and getattr(activity, "__name__", None) == failure_at:
+ raise ActivityError(
+ "failed",
+ scheduled_event_id=1,
+ started_event_id=2,
+ identity="test",
+ activity_type=failure_at,
+ activity_id="test",
+ retry_state=RetryState.TIMEOUT,
+ )
+ if activity == grouping.complete_signal_grouping_activity:
+ assert visibility_confirmed
+ assert isinstance(input, list)
+ completions.extend(input)
+ return None
if activity == grouping.get_embedding_activity:
assert isinstance(input, grouping.GenerateEmbeddingInput)
return grouping.GenerateEmbeddingOutput(embedding=[float(input.content)])
@@ -95,6 +127,7 @@ async def test_quarantined_signals_do_not_block_healthy_batch_members(
report_id="report", promoted=False, timestamp=datetime.now(UTC), run_count=0
)
if activity == grouping.wait_for_signal_in_clickhouse_activity:
+ visibility_confirmed = True
return None
raise AssertionError(f"Unexpected activity: {activity}")
@@ -103,10 +136,21 @@ async def test_quarantined_signals_do_not_block_healthy_batch_members(
patch.object(
grouping.workflow,
"patched",
- side_effect=lambda name: isolate_refusals and name == "signals-query-refusal-review-v1",
+ side_effect=lambda name: (
+ name == "signals-owner-spend-grouping-completion"
+ or (isolate_refusals and name == "signals-query-refusal-review-v1")
+ ),
),
):
- dropped, _ = await grouping._process_signal_batch(batch, FetchSignalTypeExamplesOutput(examples=[]))
+ if failure_at in {"get_embedding_activity", "wait_for_signal_in_clickhouse_activity"}:
+ with pytest.raises(ActivityError):
+ await grouping._process_signal_batch(batch, FetchSignalTypeExamplesOutput(examples=[]))
+ assert completions == []
+ return
+ with patch.object(grouping.workflow, "logger", Mock()):
+ dropped, _ = await grouping._process_signal_batch(batch, FetchSignalTypeExamplesOutput(examples=[]))
+ if failure_at is None:
+ assert [item.signal_id for item in completions] == [signal.signal_id for signal in batch]
assert dropped == sum(quarantined[int(signal.source_id)] for signal in batch if signal.source_id != "source-sync")
assert [signal.source_id for signal in emitted] == [
diff --git a/products/signals/backend/test/test_implementation_dispatch.py b/products/signals/backend/test/test_implementation_dispatch.py
index 1472db6adde7..1d0a255fd530 100644
--- a/products/signals/backend/test/test_implementation_dispatch.py
+++ b/products/signals/backend/test/test_implementation_dispatch.py
@@ -78,8 +78,9 @@ class TestImplementationDispatch(BaseTest):
@parameterized.expand([("quota",), ("disabled",), ("no_runner",)])
def test_policy_block_needs_a_new_trigger(self, reason: str) -> None:
- with patch(AUTOSTART, new=AsyncMock(return_value=AutostartOutcome(status="blocked", reason=reason))):
+ with patch(AUTOSTART, new=AsyncMock(return_value=AutostartOutcome(status="blocked", reason=reason))) as start:
self.start()
+ assert start.call_args.kwargs["implementation_spend_kind"] == "scout_triggered_implementation"
assert self.progress().status == "blocked"
with patch(DISPATCH_TASK) as enqueue:
self.dispatcher.enqueue_page()
@@ -126,6 +127,7 @@ class TestImplementationDispatch(BaseTest):
with travel(timezone.now() + timedelta(seconds=301), tick=False):
self.start()
start.assert_awaited_once()
+ assert start.call_args.kwargs["implementation_spend_kind"] == "reviewer_triggered_implementation"
assert self.progress().attempt == 2
@parameterized.expand([("revision",), ("research",), ("dismissal",), ("new_decision",)])
diff --git a/products/signals/backend/test/test_scout_harness.py b/products/signals/backend/test/test_scout_harness.py
index 02381ac90e86..f845633ab38b 100644
--- a/products/signals/backend/test/test_scout_harness.py
+++ b/products/signals/backend/test/test_scout_harness.py
@@ -2059,7 +2059,7 @@ async def test_trial_runs_keep_runtime_and_state_separate_from_the_production_sc
assert captured[0]["max_poll_seconds"] == 30 * 60
sandbox_context = captured[0]["context"]
assert isinstance(sandbox_context, CustomPromptSandboxContext)
- assert sandbox_context.sandbox_timeout_seconds == 37 * 60
+ assert sandbox_context.sandbox_timeout_seconds == 52 * 60
assert sandbox_context.model == "gpt-5.6-sol"
assert sandbox_context.reasoning_effort == "high"
assert sandbox_context.posthog_mcp_scopes == "signals_scout_experiment"
@@ -3374,6 +3374,7 @@ async def test_stale_in_progress_run_is_reaped_and_unblocks_dispatch(ateam, aerr
with (
patch("products.signals.backend.temporal.agentic.resolve_acting_user_id_for_team", return_value=42),
patch("products.signals.backend.scout_harness.runner._spawn_and_run", side_effect=fake_spawn),
+ patch("products.signals.backend.tasks.settle_scout_spend.delay"),
):
result = await arun_signals_scout(team_id=ateam.id, skill_name="signals-scout-errors")
@@ -3466,6 +3467,7 @@ async def test_stale_run_reap_captures_run_reaped_event(ateam, aerrors_skill, st
with (
patch("products.signals.backend.scout_harness.runner._spawn_and_run", side_effect=fake_spawn),
patch("products.signals.backend.scout_harness.runner.posthoganalytics.capture") as capture,
+ patch("products.signals.backend.tasks.settle_scout_spend.delay"),
):
await arun_signals_scout(team_id=ateam.id, skill_name="signals-scout-errors")
# A second dispatch must not report the same run again.
@@ -3849,6 +3851,21 @@ _ROUTED_MODEL_KEYS = ("model", "runtime_adapter", "reasoning_effort", "service_t
class TestRunRowProvenanceStamps(BaseTest):
+ def test_accounting_failure_rolls_back_scout_registration(self) -> None:
+ config, _ = SignalScoutConfig.objects.get_or_create(team=self.team, skill_name="signals-scout-general")
+ run_id = uuid7()
+ task_run = _make_task_run(self.team)
+ with patch("products.signals.backend.scout_harness.runner.record_task_spend", side_effect=RuntimeError("db")):
+ with self.assertRaises(RuntimeError):
+ _create_run_row(
+ run_id=run_id,
+ task_run_id=task_run.id,
+ team=self.team,
+ config=config,
+ skill=self._skill(allowed_tools=["emit_report"], origin="custom"),
+ )
+ assert not SignalScoutRun.objects.for_team(self.team.id).filter(id=run_id).exists()
+
# The three dimensions an eval or A/B has to hold constant. Each is unrecoverable after the
# fact, so a regression that hardcodes one (rather than reading it off the loaded skill)
# silently mislabels every run from then on instead of failing loudly.
@@ -3986,7 +4003,7 @@ async def test_activity_wakes_the_workflow_step_that_started_the_run(ateam, work
)
@pytest.mark.parametrize(
"workflow_origin_key,trial_launch_id,timeout_minutes",
- [("job:step:1", None, 21), (None, None, 21), (None, "11111111-1111-1111-1111-111111111111", 36)],
+ [("job:step:1", None, 36), (None, None, 36), (None, "11111111-1111-1111-1111-111111111111", 51)],
)
async def test_workflow_delivers_scout_outcomes_even_when_the_run_activity_cannot(
outcome, workflow_origin_key, trial_launch_id, timeout_minutes
diff --git a/products/signals/backend/test/test_scout_harness_tools.py b/products/signals/backend/test/test_scout_harness_tools.py
index 430a3563d233..26741c8ef71c 100644
--- a/products/signals/backend/test/test_scout_harness_tools.py
+++ b/products/signals/backend/test/test_scout_harness_tools.py
@@ -203,7 +203,6 @@ class TestSearchRecentRuns(BaseTest):
assert len(hits) == MAX_RUN_SEARCH_LIMIT
def test_summary_surfaces_run_metadata(self) -> None:
- # New runs also carry accounting metadata; legacy NULL rows must still project as {}.
routed = _create_run(self.team, metadata={"model": "@cf/zai-org/glm-5.2", "runtime_adapter": "claude"})
legacy = _create_run(self.team, metadata=None)
SignalScoutRun.objects.for_team(self.team.id).filter(pk=legacy.pk).update(metadata=None)
@@ -213,7 +212,6 @@ class TestSearchRecentRuns(BaseTest):
assert by_run_id[str(routed.id)].metadata == {
"model": "@cf/zai-org/glm-5.2",
"runtime_adapter": "claude",
- "spend_accounting_failed_stages": ["scout"],
}
assert by_run_id[str(legacy.id)].metadata == {}
diff --git a/products/signals/backend/test/test_scout_report_api.py b/products/signals/backend/test/test_scout_report_api.py
index 9cd8c32004a6..3f3425823df0 100644
--- a/products/signals/backend/test/test_scout_report_api.py
+++ b/products/signals/backend/test/test_scout_report_api.py
@@ -259,6 +259,7 @@ class TestScoutReportAPI(APIBaseTest):
judge.assert_awaited_once()
# The report already surfaced, so its draft PR must not be started a second time.
autostart.assert_awaited_once()
+ assert autostart.call_args.kwargs["implementation_spend_kind"] == "scout_triggered_implementation"
event = next(c for c in capture.call_args_list if c.kwargs["event"] == "signals_scout_report_emitted")
assert event.kwargs["properties"]["outcome"] == "idempotent_replay"
@@ -550,7 +551,8 @@ class TestScoutReportAPI(APIBaseTest):
resolved = SignalReport.objects.create(team=self.team, status=SignalReport.Status.RESOLVED, title="fixed")
gate_at_autostart: list[AutostartSkip | None] = []
- async def _read_gates(*, team_id: int, report_id: str) -> None:
+ async def _read_gates(*, team_id: int, report_id: str, implementation_spend_kind: str) -> None:
+ assert implementation_spend_kind == "scout_triggered_implementation"
gate_at_autostart.append(await sync_to_async(_evaluate_link_gates)(team_id, report_id))
payload = self._payload(
diff --git a/products/signals/backend/test/test_scout_report_persistence.py b/products/signals/backend/test/test_scout_report_persistence.py
index a74bd4f5f9a7..3f8932017a45 100644
--- a/products/signals/backend/test/test_scout_report_persistence.py
+++ b/products/signals/backend/test/test_scout_report_persistence.py
@@ -1,6 +1,8 @@
import json
from contextlib import AbstractContextManager, nullcontext
from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from uuid import UUID
import pytest
from posthog.test.base import BaseTest
@@ -33,6 +35,7 @@ from products.signals.backend.models import (
SignalReportArtefact,
SignalScoutConfig,
SignalScoutRun,
+ SignalSpend,
)
from products.signals.backend.report_charts import MAX_REPORT_CHARTS, ReportChart
from products.signals.backend.report_metrics import ReportMetric
@@ -59,7 +62,7 @@ from products.signals.backend.scout_report import (
from products.signals.backend.scout_report.judge import resolve_authored_report_status
from products.signals.backend.scout_report.persistence import _report_metrics_unchanged
from products.signals.backend.slack_inbox_notifications import resolve_suggested_reviewer_user_ids
-from products.signals.backend.spend import signal_spend_totals
+from products.signals.backend.spend import register_report_task_spend, signal_spend_totals
from products.tasks.backend.models import Task, TaskRun
PERSISTENCE_MODULE = "products.signals.backend.scout_report.persistence"
@@ -246,19 +249,28 @@ class TestScoutReportPersistence(BaseTest):
patch("posthog.temporal.common.client.async_connect", new=AsyncMock()),
self.captureOnCommitCallbacks(execute=True),
):
- TaskRun.objects.create(
- team=self.team,
- task=task,
+ implementation_run = task.create_run(
environment=TaskRun.Environment.CLOUD,
- state={
+ extra_state={
"ai_stage": "implementation",
"unprocessed_request_ids": [],
"token_cost": {"model": {"provider": {"cost_microusd": 10_001}}},
},
)
- assert signal_spend_totals(team_id=self.team.id, signal_ids=doc_ids) == {doc_ids[0]: 5.0001}
+ register_report_task_spend(
+ team_id=self.team.id,
+ run_id=str(implementation_run.id),
+ report_id=str(report.id),
+ stage="implementation",
+ implementation_kind=SignalSpend.Kind.SCOUT_TRIGGERED_IMPLEMENTATION,
+ )
+ spend = SignalSpend.objects.for_team(self.team.id).get(task_run_id=implementation_run.id)
+ assert spend.report_id == report.id
+ assert spend.signal_id is None
+ assert spend.spend == Decimal("5.0001")
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=doc_ids) == {}
report.refresh_from_db()
- assert str(report.triggering_signal_id) == doc_ids[0]
+ assert report.triggering_signal_id == (None if legacy else UUID(doc_ids[0]))
# Provenance: a note artefact marks the report scout-authored and is attributed to the task.
note = SignalReportArtefact.objects.get(report_id=result.report_id, type=SignalReportArtefact.ArtefactType.NOTE)
diff --git a/products/signals/backend/test/test_scout_trial_comparison.py b/products/signals/backend/test/test_scout_trial_comparison.py
index c376805ca748..9a3b18a95b8a 100644
--- a/products/signals/backend/test/test_scout_trial_comparison.py
+++ b/products/signals/backend/test/test_scout_trial_comparison.py
@@ -166,7 +166,7 @@ class TestScoutTrialComparisonWorkflow(SimpleTestCase):
assert start_trial_comparison(2, comparison_id) == first
for call in client.start_workflow.await_args_list:
assert call.kwargs["id"] == first
- assert call.kwargs["execution_timeout"] == timedelta(minutes=4522)
+ assert call.kwargs["execution_timeout"] == timedelta(minutes=4537)
assert call.kwargs["id_conflict_policy"] == WorkflowIDConflictPolicy.USE_EXISTING
assert call.kwargs["id_reuse_policy"] == WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY
assert call.args[1] == TrialComparisonInput(team_id=2, comparison_id=str(comparison_id))
diff --git a/products/signals/backend/test/test_scout_trial_state.py b/products/signals/backend/test/test_scout_trial_state.py
index d40980964e76..b752be99132c 100644
--- a/products/signals/backend/test/test_scout_trial_state.py
+++ b/products/signals/backend/test/test_scout_trial_state.py
@@ -1,7 +1,11 @@
from __future__ import annotations
import json
+from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from queue import Queue
+from time import monotonic
from typing import Any
from uuid import uuid4
@@ -10,6 +14,7 @@ import time_machine
from posthog.test.base import APIBaseTest
from unittest.mock import AsyncMock, patch
+from django.db import connection, transaction
from django.test import override_settings
from django.utils import timezone
@@ -28,6 +33,7 @@ from products.signals.backend.models import (
SignalReportArtefact,
SignalScoutRun,
SignalScratchpad,
+ SignalSpend,
)
from products.signals.backend.repo_corrections import SCOUT_REPOSITORY_REASON
from products.signals.backend.scout_harness.tools.report import (
@@ -50,6 +56,7 @@ from products.signals.backend.scout_harness.trial_state import (
memory_snapshot,
)
from products.signals.backend.scout_report.persistence import MAX_REPORT_SIGNALS, set_scout_report_inferred_repository
+from products.signals.backend.spend import record_task_cost_delta
from products.signals.backend.suggested_reviewer_index import report_ids_naming_reviewers
from products.signals.backend.temporal.report_safety_judge import SafetyJudgeResponse
from products.signals.backend.test.test_scout_harness_api import _authenticate_as_scout, _make_run
@@ -57,6 +64,54 @@ from products.skills.backend.models.skills import LLMSkill
from products.tasks.backend.models import TaskRun
[email protected]_db(transaction=True)
+def test_trial_memory_write_and_accounting_both_commit(team: Team) -> None:
+ with team_scope(team.id):
+ run = _make_run(team, metadata={"scout_trial": {"version": 1, "context_id": str(uuid4())}})
+ spend = SignalSpend.objects.for_team(team.id).create(
+ team_id=team.id,
+ kind=SignalSpend.Kind.SCOUT_RUN,
+ scout_run_id=run.id,
+ task_run_id=run.task_run_id,
+ spend=Decimal("4"),
+ billing_exempt=True,
+ )
+ trial_pid: Queue[int] = Queue()
+
+ def remember() -> None:
+ try:
+ with transaction.atomic():
+ with connection.cursor() as cursor:
+ cursor.execute("SET LOCAL lock_timeout = '10s'")
+ cursor.execute("SELECT pg_backend_pid()")
+ trial_pid.put(cursor.fetchone()[0])
+ ScoutTrialStore(run, initial_memory=[]).remember(key="finding", content="Synthetic finding")
+ finally:
+ connection.close()
+
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ with transaction.atomic():
+ TaskRun.objects.select_for_update().get(id=run.task_run_id, team_id=team.id)
+ with connection.cursor() as cursor:
+ cursor.execute("SET LOCAL lock_timeout = '10s'")
+ future = pool.submit(remember)
+ pid = trial_pid.get(timeout=10)
+ deadline = monotonic() + 10
+ while monotonic() < deadline:
+ cursor.execute("SELECT pg_backend_pid() = ANY(pg_blocking_pids(%s))", [pid])
+ if cursor.fetchone()[0]:
+ break
+ else:
+ pytest.fail("Trial writer did not reach the held task run lock")
+ record_task_cost_delta(team_id=team.id, run_id=run.task_run_id, cost_delta_microusd=10_000)
+ future.result(timeout=15)
+
+ spend.refresh_from_db()
+ run.refresh_from_db()
+ assert spend.spend == run.total_spend == Decimal("5")
+ assert ScoutTrialStore(run, initial_memory=[]).search_memory(key="finding")[0].content == "Synthetic finding"
+
+
class TestScoutTrialState(APIBaseTest):
def setUp(self) -> None:
super().setUp()
diff --git a/products/signals/backend/test/test_signal_queries.py b/products/signals/backend/test/test_signal_queries.py
index d803c70c6979..4beea3423c0f 100644
--- a/products/signals/backend/test/test_signal_queries.py
+++ b/products/signals/backend/test/test_signal_queries.py
@@ -51,7 +51,7 @@ from products.signals.backend.temporal.signal_queries import (
retract_source_signals,
)
-_MODEL_TABLE = f"distributed_posthog_document_embeddings_{EMBEDDING_MODEL.value.replace('-', '_')}"
+_MODEL_TABLE = f"sharded_posthog_document_embeddings_{EMBEDDING_MODEL.value.replace('-', '_')}"
class _SignalEmbeddingsTestBase(ClickhouseTestMixin, APIBaseTest):
@@ -521,9 +521,7 @@ class TestSpendPublicationVersions(_SignalEmbeddingsTestBase):
time_machine.travel(now, tick=False) as clock,
patch("products.signals.backend.spend_metadata.producer_scope", return_value=producer),
):
- record_llm_spend(
- "first", token_cost_microusd=100, team_id=self.team.id, signal_id=driver, stage="actionability"
- )
+ record_llm_spend(token_cost_microusd=100, team_id=self.team.id, signal_id=driver, stage="actionability")
first = publish_signal_spend_metadata(signal)
first_payloads = [call.kwargs["data"] for call in producer.__enter__.return_value.produce.call_args_list]
for payload in first_payloads:
@@ -531,23 +529,19 @@ class TestSpendPublicationVersions(_SignalEmbeddingsTestBase):
assert confirm_signal_spend_metadata(first)
producer.__enter__.return_value.produce.reset_mock()
clock.shift(timedelta(seconds=2))
- record_llm_spend(
- "second", token_cost_microusd=200, team_id=self.team.id, signal_id=driver, stage="summarization"
- )
+ record_llm_spend(token_cost_microusd=200, team_id=self.team.id, signal_id=driver, stage="summarization")
second = publish_signal_spend_metadata(
SignalSpendUpdate(team_id=self.team.id, signal_id=driver, changes=[change])
)
assert all(second.versions[key] > version for key, version in first.versions.items())
- for call in producer.__enter__.return_value.produce.call_args_list:
- payload = call.kwargs["data"]
+ second_payloads = [call.kwargs["data"] for call in producer.__enter__.return_value.produce.call_args_list]
+ assert not confirm_signal_spend_metadata(second)
+ for payload in second_payloads:
deliver(payload, second.versions[payload["model_name"]])
assert confirm_signal_spend_metadata(second)
- for payload in first_payloads:
- deliver(payload, first.versions[payload["model_name"]])
- assert confirm_signal_spend_metadata(second)
producer.__enter__.return_value.produce.reset_mock()
clock.shift(timedelta(seconds=2))
- record_llm_spend("third", token_cost_microusd=300, team_id=self.team.id, signal_id=driver, stage="research")
+ record_llm_spend(token_cost_microusd=300, team_id=self.team.id, signal_id=driver, stage="research")
third = publish_signal_spend_metadata(
SignalSpendUpdate(
team_id=self.team.id,
@@ -569,6 +563,9 @@ class TestSpendPublicationVersions(_SignalEmbeddingsTestBase):
assert set(payload["embedding"]) == ({0.0} if change.clear_content else {0.125})
deliver(payload, third.versions[payload["model_name"]])
assert confirm_signal_spend_metadata(third)
+ for payload in second_payloads:
+ deliver(payload, second.versions[payload["model_name"]])
+ assert confirm_signal_spend_metadata(third)
class TestFetchLiveSourceSignalMatches(_SignalEmbeddingsTestBase):
diff --git a/products/signals/backend/test/test_signal_report_api.py b/products/signals/backend/test/test_signal_report_api.py
index 12dcbc3150c4..427074aff1a4 100644
--- a/products/signals/backend/test/test_signal_report_api.py
+++ b/products/signals/backend/test/test_signal_report_api.py
@@ -1775,10 +1775,10 @@ class TestSignalReportListAPI(APIBaseTest):
with (
self.settings(CLOUD_DEPLOYMENT=cloud_deployment),
- patch(
- "products.signals.backend.ranking.inbox_sort.posthoganalytics.get_feature_flag", **flag_result
- ) as get_flag,
+ patch("products.signals.backend.ranking.inbox_sort.posthoganalytics") as analytics,
):
+ get_flag = analytics.get_feature_flag
+ get_flag.configure_mock(**flag_result)
response = self.client.get(self._list_url(status="ready", sort="recommended"))
if cloud_deployment == "EU":
@@ -1827,9 +1827,9 @@ class TestSignalReportListAPI(APIBaseTest):
content=json.dumps({"old_title": "Old", "new_title": "Stale P0"}),
)
- with patch(
- "products.signals.backend.ranking.inbox_sort.posthoganalytics.get_feature_flag", return_value=outcome
- ) as get_flag:
+ with patch("products.signals.backend.ranking.inbox_sort.posthoganalytics") as analytics:
+ get_flag = analytics.get_feature_flag
+ get_flag.return_value = outcome
response = self.client.get(self._list_url(status="ready", sort="recommended"))
assert response.status_code == status.HTTP_200_OK
diff --git a/products/signals/backend/test/test_spend.py b/products/signals/backend/test/test_spend.py
index ffd57a71b6b7..a962ee1bc7c3 100644
--- a/products/signals/backend/test/test_spend.py
+++ b/products/signals/backend/test/test_spend.py
@@ -1,17 +1,23 @@
import json
import asyncio
from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
from decimal import Decimal
+from importlib import import_module
+from threading import Barrier
from types import SimpleNamespace
-from uuid import uuid4
+from uuid import UUID, uuid4
+import pytest
import time_machine
from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, MagicMock, Mock, patch
-from django.db import connection, transaction
+from django.db import IntegrityError, connection, connections, transaction
+from django.db.migrations.loader import MigrationLoader
from django.test import SimpleTestCase, override_settings
+from django.test.utils import CaptureQueriesContext
from django.utils import timezone
import httpx
@@ -19,38 +25,49 @@ from anthropic import APIResponseValidationError, AsyncAnthropic
from asgiref.sync import async_to_sync
from fakeredis import FakeRedis
from parameterized import parameterized
-from structlog.testing import capture_logs
+from temporalio.client import WorkflowExecutionStatus
+from temporalio.exceptions import ApplicationError
from posthog.schema import EmbeddingModelName
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.llm.gateway_client import AIGatewayConfig
from posthog.llm.gateway_usage import GatewayRequestCost
-from posthog.models import Team
+from posthog.models import Organization, Team
from posthog.sync import database_sync_to_async
from posthog.token_bucket import TEST_reset_scripts
from products.ml_inference.backend.facade.contracts import DecisionResult, NoulAnswer
from products.signals.backend.emission.pipeline import check_actionability
from products.signals.backend.emission.registry import SignalEmitterOutput
+from products.signals.backend.facade.api import emit_signal
from products.signals.backend.llm_spend import generate_with_spend
from products.signals.backend.models import SignalReport, SignalReportTask, SignalScoutRun, SignalSpend
from products.signals.backend.pricing import cost_to_spend
+from products.signals.backend.scout_harness.limits import STALE_RUN_CUTOFF_S
+from products.signals.backend.scout_harness.runner import _self_heal_stale_runs
from products.signals.backend.signal_metadata import fetch_signals_for_report_sync
from products.signals.backend.spend import (
+ SignalSpendSummary,
SignalSpendUpdate,
+ begin_report_spend,
+ complete_signal_pipeline,
complete_task_spend,
- publish_report_spend,
+ finish_report_spend,
record_llm_spend,
+ record_task_spend,
+ register_report_task_spend,
report_triggering_signal,
- settle_signal_task_spend,
signal_spend_summaries,
signal_spend_totals,
)
from products.signals.backend.spend_metadata import confirm_signal_spend_metadata, publish_signal_spend_metadata
from products.signals.backend.system_one_prompts import bundled_prompt
+from products.signals.backend.tasks import settle_scout_spend
from products.signals.backend.temporal.llm import call_llm
-from products.tasks.backend.facade.task_run_signals import task_run_cost_updated
+from products.signals.backend.temporal.summary import check_report_task_settlement_activity
+from products.signals.backend.temporal.types import SignalReportSummaryWorkflowInputs
+from products.tasks.backend.facade.billing import TaskTokenUsageUnavailable, process_pending_gateway_usage
from products.tasks.backend.models import Task, TaskRun
@@ -60,6 +77,63 @@ class TestSpendPricing(SimpleTestCase):
assert cost_to_spend(cost, task_count=tasks) == Decimal(expected)
[email protected]_db(transaction=True)
+def test_concurrent_generation_costs_preserve_both_writers() -> None:
+ team = Team.objects.create(organization=Organization.objects.create(name="Spend test"))
+ signal_id = str(uuid4())
+ barrier = Barrier(2)
+
+ def record(worker: int) -> None:
+ try:
+ barrier.wait(timeout=10)
+ for _ in range(5):
+ record_llm_spend(
+ token_cost_microusd=worker,
+ team_id=team.id,
+ signal_id=signal_id,
+ stage="grouping",
+ )
+ finally:
+ connections.close_all()
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ futures = [pool.submit(record, worker) for worker in (1, 2)]
+ for future in futures:
+ future.result(timeout=30)
+ spend = SignalSpend.objects.for_team(team.id).get(signal_id=signal_id)
+ assert spend.spend == Decimal("0.0015")
+ assert spend.finalized_at is None
+
+
[email protected]_db(transaction=True)
+def test_direct_steering_drop_finalizes_its_known_cost() -> None:
+ team = Team.objects.create(
+ organization=Organization.objects.create(name="Steering spend test", is_ai_data_processing_approved=True)
+ )
+ driver = str(uuid4())
+ with (
+ patch("products.signals.backend.facade.api.SignalSourceConfig.is_source_enabled", return_value=True),
+ patch(
+ "products.signals.backend.emission.direct_gate.steering_filters_signal", new=AsyncMock(return_value=True)
+ ),
+ patch("products.signals.backend.facade.api.async_connect", new=AsyncMock()) as connect,
+ ):
+ record_llm_spend(team_id=team.id, signal_id=driver, stage="actionability", token_cost_microusd=100)
+ async_to_sync(emit_signal)(
+ team=team,
+ source_product="error_tracking",
+ source_type="issue_created",
+ source_id="synthetic-issue",
+ description="Synthetic error excluded by source steering",
+ extra={"fingerprint": "synthetic-fingerprint"},
+ signal_id=driver,
+ )
+ spend = SignalSpend.objects.for_team(team.id).get(signal_id=driver)
+ assert spend.finalized_at is not None
+ assert spend.spend == Decimal("0.01")
+ connect.assert_not_called()
+
+
class TestSignalSpend(BaseTest):
def setUp(self) -> None:
super().setUp()
@@ -72,9 +146,7 @@ class TestSignalSpend(BaseTest):
def test_clickhouse_spend_retries_until_visible_and_tracks_later_costs(self, source: str) -> None:
driver = str(uuid4())
signal = SignalSpendUpdate(team_id=self.team.id, signal_id=driver)
- record_llm_spend(
- "first", token_cost_microusd=123, team_id=self.team.id, signal_id=driver, stage="actionability"
- )
+ record_llm_spend(token_cost_microusd=123, team_id=self.team.id, signal_id=driver, stage="actionability")
now = timezone.now()
metadata = {"report_id": "moved-report", "deleted": True, "extra": {"example": 1}, "total_spend": 0}
rows: dict[str, list] = {}
@@ -97,6 +169,11 @@ class TestSignalSpend(BaseTest):
with self.assertRaises(RuntimeError):
publish_signal_spend_metadata(signal)
produce_scope.side_effect = None
+ producer.produce.return_value.get.side_effect = RuntimeError("Kafka delivery failed")
+ with self.assertRaisesRegex(RuntimeError, "Kafka delivery failed"):
+ publish_signal_spend_metadata(signal)
+ producer.produce.return_value.get.side_effect = None
+ producer.reset_mock()
publication = publish_signal_spend_metadata(signal)
assert producer.produce.call_count == len(EmbeddingModelName)
for _ in range(5):
@@ -122,7 +199,6 @@ class TestSignalSpend(BaseTest):
producer.produce.assert_not_called()
if source == "generation":
record_llm_spend(
- "later",
token_cost_microusd=123,
team_id=self.team.id,
signal_id=driver,
@@ -148,9 +224,7 @@ class TestSignalSpend(BaseTest):
(driver, (payload["content"], payload["metadata"], now, [0.5], timezone.now()))
]
assert confirm_signal_spend_metadata(newer)
- record_llm_spend(
- None, token_cost_microusd=None, team_id=self.team.id, signal_id=driver, stage="summarization"
- )
+ record_llm_spend(token_cost_microusd=None, team_id=self.team.id, signal_id=driver, stage="summarization")
producer.reset_mock()
clock.shift(2)
publish_signal_spend_metadata(signal)
@@ -168,26 +242,28 @@ class TestSignalSpend(BaseTest):
signal_row = fetch_signals_for_report_sync(self.team, str(uuid4()))[0]
assert signal_row["total_spend"] == metadata["total_spend"]
- @parameterized.expand([("web", None), ("pipeline", ClickHouseUser.SIGNALS)])
+ @parameterized.expand([("web", None, False), ("pipeline", ClickHouseUser.SIGNALS, False), ("settled", None, True)])
def test_triggering_signal_lookup_preserves_query_identity(
- self, _name: str, ch_user: ClickHouseUser | None
+ self, _name: str, ch_user: ClickHouseUser | None, finalized: bool
) -> None:
report = SignalReport.objects.create(team=self.team)
driver = str(uuid4())
now = timezone.now()
+ if finalized:
+ record_llm_spend(team_id=self.team.id, signal_id=driver, stage="grouping", token_cost_microusd=100)
+ complete_signal_pipeline(team_id=self.team.id, signal_id=driver)
with patch(
"products.signals.backend.signal_metadata.execute_hogql_query",
return_value=Mock(results=[(driver, "Example finding", "{}", now, now)]),
) as query:
- assert (
- report_triggering_signal(
- team_id=self.team.id, report_id=str(report.id), **({"ch_user": ch_user} if ch_user else {})
- )
- == driver
- )
+ assert report_triggering_signal(
+ team_id=self.team.id, report_id=str(report.id), **({"ch_user": ch_user} if ch_user else {})
+ ) == (None if finalized else driver)
assert query.call_args.kwargs["ch_user"] == (ch_user or ClickHouseUser.DEFAULT)
report.refresh_from_db()
- assert str(report.triggering_signal_id) == driver
+ assert (str(report.triggering_signal_id) if report.triggering_signal_id else None) == (
+ None if finalized else driver
+ )
@parameterized.expand([("priced", 12_345), ("free", 0), ("cancelled", None), ("missing_id", None)])
async def test_generation_waits_for_spend_without_regenerating(self, outcome: str, cost: int | None) -> None:
@@ -231,9 +307,9 @@ class TestSignalSpend(BaseTest):
assert lookup.await_count == 3
generate.assert_awaited_once()
spend = await database_sync_to_async(lambda: SignalSpend.objects.for_team(self.team.id).get(signal_id=driver))()
- assert spend.token_cost_microusd == cost
+ assert spend.spend == cost_to_spend(cost or 0)
assert spend.accounting_failed is (cost is None)
- assert spend.stage == "summarization"
+ assert spend.llm_failed_stages == (["summarization"] if cost is None else [])
@parameterized.expand([("unavailable",), ("hung",), ("activity_deadline",)])
async def test_usage_timeout_preserves_completed_generation(self, outcome: str) -> None:
@@ -274,9 +350,8 @@ class TestSignalSpend(BaseTest):
receipt = await database_sync_to_async(
lambda: SignalSpend.objects.for_team(self.team.id).get(signal_id=driver)
)()
- assert receipt.source_id == "generation-1"
assert receipt.accounting_failed is True
- assert receipt.stage == "actionability"
+ assert receipt.llm_failed_stages == ["actionability"]
@parameterized.expand(
[(gateway, failure) for gateway in (True, False) for failure in ("timeout", "transport", "cancelled")]
@@ -301,7 +376,7 @@ class TestSignalSpend(BaseTest):
)
spend = await database_sync_to_async(lambda: SignalSpend.objects.for_team(self.team.id).get(signal_id=driver))()
assert spend.accounting_failed is True
- assert spend.stage == "actionability"
+ assert spend.llm_failed_stages == ["actionability"]
generate.assert_awaited_once()
@parameterized.expand([("invalid_json",), ("invalid_message",)])
@@ -344,12 +419,8 @@ class TestSignalSpend(BaseTest):
ai_product="signals",
)
spend = await database_sync_to_async(lambda: SignalSpend.objects.for_team(self.team.id).get(signal_id=driver))()
- assert spend.stage == "safety"
+ assert spend.llm_failed_stages == (["safety"] if failure == "invalid_json" else [])
assert spend.accounting_failed is (failure == "invalid_json")
- if failure == "invalid_json":
- assert spend.source_id.startswith("unknown-")
- else:
- assert spend.source_id == "generation-1"
@parameterized.expand([("gateway",), ("legacy",)])
async def test_parallel_generations_record_every_attempt_for_the_explicit_owner(self, mode: str) -> None:
@@ -408,29 +479,13 @@ class TestSignalSpend(BaseTest):
results = await asyncio.gather(call(drivers[0], "request-a"), call(drivers[1], "request-b"))
assert list(results) == ["ok", "ok"]
assert await call(None, "unattributed") == "ok"
- rows = await database_sync_to_async(
- lambda: list(
- SignalSpend.objects.for_team(self.team.id).values(
- "source_id", "signal_id", "stage", "accounting_failed"
- )
- )
- )()
+ rows = await database_sync_to_async(lambda: list(SignalSpend.objects.for_team(self.team.id).values()))()
assert await database_sync_to_async(
lambda: signal_spend_totals(team_id=self.team.id, signal_ids=drivers)
)() == {drivers[0]: 0.0246 if mode == "gateway" else 0, drivers[1]: 0.0123 if mode == "gateway" else 0}
- assert len(rows) == 3
- assert sorted(str(row["signal_id"]) for row in rows) == sorted([drivers[0], drivers[0], drivers[1]])
- assert all(row["stage"] == "actionability" for row in rows)
- if mode == "gateway":
- assert {row["source_id"]: str(row["signal_id"]) for row in rows} == {
- "request-a-1": drivers[0],
- "request-a-2": drivers[0],
- "request-b-1": drivers[1],
- }
- assert not any(row["accounting_failed"] for row in rows)
- else:
- assert all(row["accounting_failed"] for row in rows)
- assert all(row["source_id"].startswith("unknown-") for row in rows)
+ assert len(rows) == 2
+ assert {str(row["signal_id"]) for row in rows} == set(drivers)
+ assert all(row["accounting_failed"] is (mode != "gateway") for row in rows)
@parameterized.expand(
[
@@ -496,27 +551,25 @@ class TestSignalSpend(BaseTest):
)
finally:
TEST_reset_scripts()
- rows = await database_sync_to_async(
- lambda: list(
- SignalSpend.objects.for_team(self.team.id).values(
- "source_id", "signal_id", "stage", "accounting_failed"
- )
- )
- )()
+ rows = await database_sync_to_async(lambda: list(SignalSpend.objects.for_team(self.team.id).values()))()
if mode == "system-one-only":
assert rows == []
else:
assert len(rows) == 1
assert str(rows[0]["signal_id"]) == driver
- if request_id_available:
- assert rows[0]["source_id"] == "traditional-request"
- else:
- assert rows[0]["source_id"].startswith("unknown-")
- assert rows[0]["stage"] == "actionability"
+ assert rows[0]["spend"] == (Decimal("0.0123") if request_id_available else 0)
assert rows[0]["accounting_failed"] is (not request_id_available)
client.messages.create.assert_awaited_once()
- def _run(self, *, report: SignalReport | None = None, task: Task | None = None, stage: str = "research") -> TaskRun:
+ def _run(
+ self,
+ *,
+ report: SignalReport | None = None,
+ task: Task | None = None,
+ stage: str = "research",
+ signal_id: str | None = None,
+ implementation_kind: str | None = None,
+ ) -> TaskRun:
task = task or Task.objects.create(
team=self.team,
title="Example research",
@@ -524,56 +577,444 @@ class TestSignalSpend(BaseTest):
origin_product=Task.OriginProduct.SIGNAL_REPORT,
signal_report=report,
)
- return TaskRun.objects.create(
- team=self.team,
- task=task,
+ run = task.create_run(
environment=TaskRun.Environment.CLOUD,
- status=TaskRun.Status.IN_PROGRESS,
- state={"ai_stage": stage, "unprocessed_request_ids": [], "token_cost": {}},
+ spend_signal_id=signal_id,
+ implementation_spend_kind=implementation_kind,
+ extra_state={"ai_stage": stage, "unprocessed_request_ids": [], "token_cost": {}},
+ )
+ run.status = TaskRun.Status.IN_PROGRESS
+ run.save(update_fields=["status"])
+ return run
+
+ def _complete_task_spend(self, *, team_id: int, run_id: str) -> None:
+ run = TaskRun.objects.select_related("task").get(team_id=team_id, id=run_id)
+ complete_task_spend(
+ team_id=team_id,
+ run_id=run_id,
+ report_id=str(run.task.signal_report_id) if run.task.signal_report_id else None,
)
+ def test_settlement_waits_for_only_the_triggering_signals_task_workflows(self) -> None:
+ driver = str(uuid4())
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
+ self._run(report=report, signal_id=str(uuid4()))
+ run = self._run(report=report, signal_id=driver)
+ handle = AsyncMock()
+ self.temporal.get_workflow_handle = Mock(return_value=handle)
+ inputs = SignalReportSummaryWorkflowInputs(team_id=self.team.id, report_id=str(report.id), signal_id=driver)
+ with patch(
+ "products.signals.backend.temporal.summary.async_connect", new=AsyncMock(return_value=self.temporal)
+ ):
+ handle.describe.return_value.status = WorkflowExecutionStatus.RUNNING
+ with self.assertRaisesRegex(ApplicationError, "still settling"):
+ async_to_sync(check_report_task_settlement_activity)(inputs)
+ self.temporal.get_workflow_handle.assert_called_once_with(run.workflow_id)
+ handle.describe.return_value.status = WorkflowExecutionStatus.COMPLETED
+ async_to_sync(check_report_task_settlement_activity)(inputs)
+
+ def _refresh_task_spend(self, *, run_id: str, team_id: int) -> None:
+ run = TaskRun.objects.get(id=run_id, team_id=team_id)
+ spend = SignalSpend.objects.for_team(team_id).filter(task_run_id=run.id).first()
+ if spend:
+ assert spend.kind is not None
+ record_task_spend(
+ team_id=team_id,
+ run_id=run_id,
+ signal_id=str(spend.signal_id) if spend.signal_id else None,
+ scout_run_id=str(spend.scout_run_id) if spend.scout_run_id else None,
+ report_id=str(spend.report_id) if spend.report_id else None,
+ stage="scout" if spend.scout_run_id else "implementation" if spend.report_id else spend.kind,
+ implementation_kind=spend.kind if spend.report_id else None,
+ )
+
def _set_task_cost(self, run: TaskRun, cost: int | None, *, pending: bool = False) -> None:
run.refresh_from_db()
- state = dict(run.state or {})
- state["unprocessed_request_ids"] = ["pending-request"] if cost is None or pending else []
- if cost is not None:
- state["token_cost"] = {"model-a": {"provider-a": {"cost_microusd": cost}}}
- run.state = state
+ previous = sum(
+ bucket.get("cost_microusd", 0)
+ for providers in run.state.get("token_cost", {}).values()
+ for bucket in providers.values()
+ )
+ request_id = str(uuid4())
+ run.state["unprocessed_request_ids"] = [request_id]
+ run.state.setdefault("token_cost", {})
run.save(update_fields=["state"])
- with self.captureOnCommitCallbacks(execute=True):
- task_run_cost_updated.send(sender=TaskRun, run_id=run.id, team_id=self.team.id)
+ with patch(
+ "products.tasks.backend.logic.services.gateway_usage._fetch_gateway_cost",
+ new=AsyncMock(
+ return_value=GatewayRequestCost(model="model-a", provider="provider-a", cost_microusd=cost - previous)
+ if cost is not None
+ else None
+ ),
+ ):
+ process_pending_gateway_usage(run_id=run.id, team_id=run.team_id)
+ if pending:
+ run.refresh_from_db()
+ run.state["unprocessed_request_ids"] = ["pending-request"]
+ run.save(update_fields=["state"])
+ self._refresh_task_spend(run_id=str(run.id), team_id=self.team.id)
- @parameterized.expand([("research",), ("repo_selection",), ("implementation",)])
- def test_late_task_cost_updates_charge_driver_once_and_preserve_fractional_cents(self, resumed_stage: str) -> None:
+ @parameterized.expand([("research",), ("repo_selection",)])
+ def test_task_cost_updates_charge_exact_run_once(self, stage: str) -> None:
driver, other = str(uuid4()), str(uuid4())
report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
- research = self._run(report=report, stage=resumed_stage)
- implementation = self._run(report=report, stage="implementation")
- self._set_task_cost(research, 10_001)
- self._set_task_cost(implementation, 25_002)
- assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver, other]) == {driver: 11.5003}
-
+ run = self._run(report=report, stage=stage)
+ self._set_task_cost(run, 10_001)
report.triggering_signal_id = other
report.save(update_fields=["triggering_signal_id"])
- self._set_task_cost(research, 10_004)
- assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver, other]) == {driver: 11.5006}
- assert SignalSpend.objects.for_team(self.team.id).count() == 2
- self.temporal.reset_mock()
- with self.captureOnCommitCallbacks(execute=True):
- try:
- with transaction.atomic():
- task_run_cost_updated.send(sender=TaskRun, run_id=research.id, team_id=self.team.id)
- raise RuntimeError("roll back task cost")
- except RuntimeError:
- pass
- self.temporal.start_workflow.assert_not_called()
-
- rerun = self._run(task=research.task, stage="")
+ self._set_task_cost(run, 10_004)
+ register_report_task_spend(
+ team_id=self.team.id, run_id=str(run.id), report_id=str(report.id), stage=stage, signal_id=other
+ )
+ rerun = self._run(task=run.task, stage=stage, signal_id=other)
self._set_task_cost(rerun, 1)
- assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver, other]) == {driver: 15.5007}
- assert SignalSpend.objects.for_team(self.team.id).get(source_id=str(rerun.id)).stage == resumed_stage
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver, other]) == {driver: 5.0004, other: 4.0001}
+ rows = SignalSpend.objects.for_team(self.team.id)
+ assert rows.get(task_run_id=run.id).signal_id == UUID(driver)
+ assert rows.get(task_run_id=rerun.id).signal_id == UUID(other)
+ assert set(rows.values_list("kind", flat=True)) == {stage}
- def test_task_cost_callback_updates_spend_synchronously_without_publishing(self) -> None:
+ @parameterized.expand([("reviewer_triggered_implementation",), ("scout_triggered_implementation",)])
+ def test_each_kind_finalizes_independently_and_new_implementation_keeps_settled_rows(
+ self, implementation_kind: str
+ ) -> None:
+ driver = str(uuid4())
+ report = SignalReport.objects.create(
+ team=self.team, triggering_signal_id=driver, status=SignalReport.Status.CANDIDATE
+ )
+ record_llm_spend(token_cost_microusd=100, team_id=self.team.id, signal_id=driver, stage="grouping")
+ complete_signal_pipeline(team_id=self.team.id, signal_id=driver, report_id=str(report.id))
+ research = self._run(report=report)
+ repo = self._run(report=report, stage="repo_selection")
+ for run in (research, repo):
+ self._set_task_cost(run, 10_000)
+ run.status = TaskRun.Status.COMPLETED
+ run.save(update_fields=["status"])
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ rows = SignalSpend.objects.for_team(self.team.id)
+ assert rows.filter(signal_id=driver, finalized_at__isnull=False).count() == 2
+ implementation = self._run(report=report, stage="implementation", implementation_kind=implementation_kind)
+ finish_report_spend(team_id=self.team.id, report_id=str(report.id), signal_id=driver)
+ assert rows.filter(signal_id=driver, finalized_at__isnull=False).count() == 3
+ before = list(rows.filter(signal_id=driver).order_by("id").values())
+ for run in (
+ implementation,
+ self._run(task=implementation.task, stage="implementation", implementation_kind=implementation_kind),
+ ):
+ self._set_task_cost(run, 20_000)
+ run.status = TaskRun.Status.COMPLETED
+ run.save(update_fields=["status"])
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ row = rows.get(task_run_id=run.id)
+ assert row.report_id == report.id
+ assert row.signal_id is None
+ assert row.kind == implementation_kind
+ assert row.spend == Decimal(6)
+ assert row.finalized_at is not None
+ assert list(rows.filter(signal_id=driver).order_by("id").values()) == before
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver]) == {driver: 10.01}
+ assert rows.filter(report=report).count() == 2
+
+ @parameterized.expand(
+ [
+ (Task.OriginProduct.SIGNAL_REPORT, "inbox"),
+ (Task.OriginProduct.SIGNAL_REPORT, ""),
+ (Task.OriginProduct.SIGNALS_SCOUT, ""),
+ ]
+ )
+ def test_manual_report_run_leaves_finalized_signal_spend_unchanged(self, origin: str, stage: str) -> None:
+ driver = str(uuid4())
+ report = SignalReport.objects.create(
+ team=self.team, triggering_signal_id=driver, status=SignalReport.Status.READY
+ )
+ if origin == Task.OriginProduct.SIGNALS_SCOUT:
+ task = Task.objects.create(team=self.team, title="Scout", origin_product=origin)
+ run = self._run(task=task, stage="scout")
+ scout = SignalScoutRun.objects.for_team(self.team.id).create(
+ team=self.team, task_run=run, skill_name="example-scout", skill_version=1
+ )
+ record_task_spend(team_id=self.team.id, run_id=str(run.id), scout_run_id=str(scout.id), stage="scout")
+ else:
+ run = self._run(report=report, stage="implementation")
+ SignalReportTask.objects.create(team=self.team, report=report, task=run.task, relationship="implementation")
+ self._set_task_cost(run, 100)
+ run.status = TaskRun.Status.COMPLETED
+ run.save(update_fields=["status"])
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ before = SignalSpend.objects.for_team(self.team.id).values().get()
+ assert before["finalized_at"] is not None
+ resumed = self._run(task=run.task, stage=stage)
+ assert resumed.state["billing_product"] == "posthog_ai"
+ self._set_task_cost(resumed, 200)
+ resumed.status = TaskRun.Status.COMPLETED
+ resumed.save(update_fields=["status"])
+ self._complete_task_spend(team_id=self.team.id, run_id=str(resumed.id))
+ assert SignalSpend.objects.for_team(self.team.id).values().get() == before
+
+ @parameterized.expand([(Task.OriginProduct.SIGNAL_REPORT, "research"), (Task.OriginProduct.SIGNALS_SCOUT, "scout")])
+ def test_prelaunch_failure_settles_with_zero_tokens(self, origin: str, stage: str) -> None:
+ driver = str(uuid4())
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
+ task = Task.objects.create(
+ team=self.team, title="Fails before launch", origin_product=origin, signal_report=report
+ )
+ run = task.create_run(
+ environment=TaskRun.Environment.CLOUD, extra_state={"ai_stage": stage}, spend_signal_id=driver
+ )
+ if origin == Task.OriginProduct.SIGNALS_SCOUT:
+ scout = SignalScoutRun.objects.for_team(self.team.id).create(
+ team=self.team, task_run=run, skill_name="example-scout", skill_version=1
+ )
+ record_task_spend(team_id=self.team.id, run_id=str(run.id), scout_run_id=str(scout.id), stage="scout")
+ run.status = TaskRun.Status.FAILED
+ run.save(update_fields=["status"])
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ spend = SignalSpend.objects.for_team(self.team.id).get(task_run_id=run.id)
+ assert spend.finalized_at is not None
+ assert not spend.accounting_failed
+ assert spend.spend == cost_to_spend(0, task_count=1)
+
+ @parameterized.expand([("research",), ("repo_selection",)])
+ def test_resumed_task_belongs_to_new_signal_and_preserves_finalized_owner(self, stage: str) -> None:
+ driver, other = str(uuid4()), str(uuid4())
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
+ run = self._run(report=report, stage=stage)
+ self._set_task_cost(run, 100)
+ run.status = TaskRun.Status.COMPLETED
+ run.save(update_fields=["status"])
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ before = SignalSpend.objects.for_team(self.team.id).get(task_run_id=run.id)
+ resumed = self._run(task=run.task, stage=stage, signal_id=other)
+ self._set_task_cost(resumed, 200)
+ resumed.status = TaskRun.Status.FAILED
+ resumed.save(update_fields=["status"])
+ self.temporal.start_workflow.side_effect = RuntimeError("Temporal unavailable")
+ with self.assertRaisesRegex(RuntimeError, "Temporal unavailable"):
+ self._complete_task_spend(team_id=self.team.id, run_id=str(resumed.id))
+ settled = SignalSpend.objects.for_team(self.team.id).get(task_run_id=resumed.id)
+ assert settled.finalized_at is not None
+ assert settled.spend == Decimal("4.02")
+ self.temporal.start_workflow.side_effect = None
+ self._complete_task_spend(team_id=self.team.id, run_id=str(resumed.id))
+ register_report_task_spend(
+ team_id=self.team.id, run_id=str(run.id), report_id=str(report.id), stage=stage, signal_id=other
+ )
+ after = SignalSpend.objects.for_team(self.team.id).get(task_run_id=run.id)
+ assert (after.signal_id, after.spend, after.finalized_at) == (
+ before.signal_id,
+ before.spend,
+ before.finalized_at,
+ )
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver, other]) == {driver: 4.01, other: 4.02}
+ assert SignalSpend.objects.for_team(self.team.id).get(task_run_id=resumed.id).finalized_at is not None
+
+ @parameterized.expand([(SignalReport.Status.READY,), (SignalReport.Status.CANDIDATE,)])
+ def test_pipeline_handoff_stays_open_until_summary_completion(self, status: str) -> None:
+ driver = str(uuid4())
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver, status=status)
+ record_llm_spend(token_cost_microusd=100, team_id=self.team.id, signal_id=driver, stage="grouping")
+ complete_signal_pipeline(team_id=self.team.id, signal_id=driver, report_id=str(report.id))
+ spend = SignalSpend.objects.for_team(self.team.id).get(signal_id=driver)
+ self.assertIsNone(spend.finalized_at)
+ finish_report_spend(team_id=self.team.id, signal_id=driver, report_id=str(report.id))
+ spend.refresh_from_db()
+ self.assertIsNotNone(spend.finalized_at)
+
+ @parameterized.expand([("terminal", False, False), ("next_pass", True, False), ("newer_pass", True, True)])
+ def test_summary_completion_settles_only_abandoned_pending_trigger(
+ self, _name: str, next_pass: bool, newer_pass: bool
+ ) -> None:
+ driver, pending = str(uuid4()), str(uuid4())
+ report = SignalReport.objects.create(
+ team=self.team,
+ status=SignalReport.Status.CANDIDATE if next_pass else SignalReport.Status.POTENTIAL,
+ triggering_signal_id=str(uuid4()) if newer_pass else pending if next_pass else driver,
+ pending_triggering_signal_id=pending,
+ )
+ for signal_id in (driver, pending):
+ record_llm_spend(token_cost_microusd=100, team_id=self.team.id, signal_id=signal_id, stage="grouping")
+ finish_report_spend(team_id=self.team.id, report_id=str(report.id), signal_id=driver)
+ rows = SignalSpend.objects.for_team(self.team.id)
+ assert rows.get(signal_id=driver).finalized_at is not None
+ assert (rows.get(signal_id=pending).finalized_at is None) == next_pass
+ report.refresh_from_db()
+ assert (report.pending_triggering_signal_id is not None) == next_pass
+
+ def test_filtered_signal_finalizes_only_when_every_source_is_priced(self) -> None:
+ driver = str(uuid4())
+ record_llm_spend(token_cost_microusd=100, team_id=self.team.id, signal_id=driver, stage="summarization")
+ record_llm_spend(token_cost_microusd=None, team_id=self.team.id, signal_id=driver, stage="actionability")
+ complete_signal_pipeline(team_id=self.team.id, signal_id=driver)
+ spend = SignalSpend.objects.for_team(self.team.id).get(signal_id=driver)
+ self.assertIsNone(spend.finalized_at)
+ assert spend.accounting_failed
+ assert spend.spend == Decimal("0.01")
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver, str(uuid4())]) == {driver: 0.01}
+ with self.assertRaises(IntegrityError), transaction.atomic():
+ SignalSpend.objects.for_team(self.team.id).create(
+ team_id=self.team.id, signal_id=driver, kind=SignalSpend.Kind.AD_HOC
+ )
+
+ def test_reset_discards_receipts_and_keeps_the_existing_table(self) -> None:
+ migration = import_module("products.signals.backend.migrations.0208_reset_signal_spend")
+ with override_settings(MIGRATION_MODULES={}):
+ state = MigrationLoader(connection).project_state([("signals", "0207_backfill_report_link_targets")])
+ with connection.cursor() as cursor:
+ cursor.execute("CREATE SCHEMA test_signal_spend_reset")
+ cursor.execute("SET LOCAL search_path TO test_signal_spend_reset, public")
+ try:
+ old_spend = state.apps.get_model("signals", "SignalSpend")
+ with connection.schema_editor() as editor:
+ editor.create_model(old_spend)
+ old_spend.objects.create(
+ team_id=self.team.id,
+ signal_id=uuid4(),
+ source_id="discarded-call",
+ token_cost_microusd=10_000,
+ )
+ with connection.cursor() as cursor:
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute("SELECT 'signals_signalspend'::regclass::oid")
+ original_table = cursor.fetchone()[0]
+ with connection.schema_editor() as editor:
+ state = migration.Migration("0208_reset_signal_spend", "signals").apply(state, editor)
+ spend = state.apps.get_model("signals", "SignalSpend")
+ assert not spend.objects.exists()
+ row = spend.objects.create(
+ team_id=self.team.id,
+ signal_id=uuid4(),
+ kind="research",
+ task_run_id=uuid4(),
+ spend=Decimal("4.0123"),
+ finalized_at=timezone.now(),
+ )
+ assert spend.objects.get(id=row.id).spend == Decimal("4.0123")
+ legacy_signal = old_spend.objects.create(
+ team_id=self.team.id,
+ signal_id=row.signal_id,
+ source_id="overlapping-call",
+ token_cost_microusd=20_000,
+ accounting_failed=True,
+ )
+ legacy_scout = old_spend.objects.create(
+ team_id=self.team.id,
+ scout_run_id=uuid4(),
+ source_id=str(uuid4()),
+ is_task=True,
+ )
+ legacy_scout.token_cost_microusd = 30_000
+ legacy_scout.save(update_fields=["token_cost_microusd"])
+ assert old_spend.objects.get(id=legacy_scout.id).token_cost_microusd == 30_000
+ assert list(
+ spend.objects.filter(id__in=[legacy_signal.id, legacy_scout.id]).values_list(
+ "kind", "spend", "finalized_at"
+ )
+ ) == [(None, Decimal(0), None), (None, Decimal(0), None)]
+ assert signal_spend_summaries(team_id=self.team.id, signal_ids=[str(row.signal_id)]) == {
+ str(row.signal_id): SignalSpendSummary(total_spend=4.0123, failed_stages=[])
+ }
+ with self.assertRaises(IntegrityError), transaction.atomic():
+ spend.objects.filter(id=legacy_signal.id).update(finalized_at=timezone.now())
+ with connection.cursor() as cursor:
+ cursor.execute("SELECT 'signals_signalspend'::regclass::oid")
+ assert cursor.fetchone()[0] == original_table
+ finally:
+ with connection.cursor() as cursor:
+ cursor.execute("SET LOCAL search_path TO public")
+ cursor.execute("DROP SCHEMA test_signal_spend_reset CASCADE")
+
+ @parameterized.expand([("research",), ("repo_selection",), ("implementation",), ("scout",)])
+ def test_completion_does_not_recreate_discarded_spend(self, stage: str) -> None:
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=uuid4())
+ run = self._run(report=report, stage=stage)
+ run.status = TaskRun.Status.COMPLETED
+ run.save(update_fields=["status"])
+ if stage == "scout":
+ scout = SignalScoutRun.objects.for_team(self.team.id).create(
+ team=self.team, task_run=run, skill_name="example-scout", skill_version=1
+ )
+ record_task_spend(team_id=self.team.id, run_id=str(run.id), scout_run_id=str(scout.id), stage="scout")
+ SignalSpend.objects.for_team(self.team.id).filter(task_run_id=run.id).delete()
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ assert not SignalSpend.objects.for_team(self.team.id).exists()
+
+ def test_task_binding_cannot_charge_a_second_owner(self) -> None:
+ driver, other = str(uuid4()), str(uuid4())
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
+ run = self._run(report=report)
+ self._set_task_cost(run, 1)
+ with self.assertRaisesRegex(ValueError, "already owned"):
+ record_task_spend(team_id=self.team.id, run_id=str(run.id), signal_id=other, stage="research")
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver, other]) == {driver: 4.0001}
+
+ def test_gateway_request_and_owner_delta_roll_back_together(self) -> None:
+ driver = str(uuid4())
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
+ run = self._run(report=report)
+ run.state["unprocessed_request_ids"] = ["example-request"]
+ run.save(update_fields=["state"])
+
+ def refuse_owner_write(
+ execute: Callable[..., object], sql: str, params: object, many: bool, context: object
+ ) -> object:
+ if sql.startswith('UPDATE "signals_signalspend"'):
+ raise RuntimeError("Owner write failed")
+ return execute(sql, params, many, context)
+
+ with patch(
+ "products.tasks.backend.logic.services.gateway_usage._fetch_gateway_cost",
+ new=AsyncMock(return_value=GatewayRequestCost(model="model", provider="provider", cost_microusd=123)),
+ ):
+ with (
+ connection.execute_wrapper(refuse_owner_write),
+ self.assertRaisesRegex(RuntimeError, "Owner write failed"),
+ ):
+ process_pending_gateway_usage(run_id=run.id, team_id=self.team.id)
+ run.refresh_from_db()
+ assert run.state["unprocessed_request_ids"] == ["example-request"]
+ assert run.state["token_cost"] == {}
+ for _ in range(2):
+ process_pending_gateway_usage(run_id=run.id, team_id=self.team.id)
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver]) == {driver: 4.0123}
+
+ def test_spend_reads_do_not_reopen_or_bill_a_completed_owner(self) -> None:
+ driver = str(uuid4())
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
+ record_llm_spend(token_cost_microusd=0, team_id=self.team.id, signal_id=driver, stage="grouping")
+ run = self._run(report=report)
+ self._set_task_cost(run, 10_000)
+ SignalSpend.objects.for_team(self.team.id).filter(signal_id=driver).update(finalized_at=timezone.now())
+ before = list(SignalSpend.objects.for_team(self.team.id).filter(signal_id=driver).order_by("id").values())
+ run.state = {"ai_stage": "research", "token_cost": {"model": {"provider": {"cost_microusd": 20_000}}}}
+ run.save(update_fields=["state"])
+ with CaptureQueriesContext(connection) as queries:
+ assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver]) == {driver: 5}
+ assert all(query["sql"].startswith("SELECT") and "FOR UPDATE" not in query["sql"] for query in queries)
+ self._refresh_task_spend(team_id=self.team.id, run_id=str(run.id))
+ begin_report_spend(team_id=self.team.id, report_id=str(report.id), signal_id=driver)
+ with self.assertRaises(ValueError):
+ record_llm_spend(token_cost_microusd=100, team_id=self.team.id, signal_id=driver, stage="grouping")
+ complete_signal_pipeline(team_id=self.team.id, signal_id=driver)
+ assert (
+ list(SignalSpend.objects.for_team(self.team.id).filter(signal_id=driver).order_by("id").values()) == before
+ )
+
+ def test_report_reassignment_and_deletion_do_not_leave_spend_open(self) -> None:
+ driver = str(uuid4())
+ first, second = [
+ SignalReport.objects.create(team=self.team, triggering_signal_id=driver, status=SignalReport.Status.READY)
+ for _ in range(2)
+ ]
+ for report in [first, second]:
+ begin_report_spend(team_id=self.team.id, report_id=str(report.id), signal_id=driver)
+ report_id = str(second.id)
+ second.delete()
+ finish_report_spend(team_id=self.team.id, signal_id=driver, report_id=report_id)
+ spend = SignalSpend.objects.for_team(self.team.id).get(signal_id=driver)
+ self.assertIsNotNone(spend.finalized_at)
+
+ def test_task_cost_updates_spend_synchronously_without_publishing(self) -> None:
driver = str(uuid4())
report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
run = self._run(report=report)
@@ -591,44 +1032,22 @@ class TestSignalSpend(BaseTest):
assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver]) == {driver: 5.0001}
self.temporal.start_workflow.assert_not_called()
- @parameterized.expand(
- [
- (SignalReport.Status.CANDIDATE, None),
- (SignalReport.Status.READY, None),
- (SignalReport.Status.READY, TaskRun.Status.COMPLETED),
- (SignalReport.Status.READY, TaskRun.Status.FAILED),
- (SignalReport.Status.READY, TaskRun.Status.CANCELLED),
- ]
- )
- def test_report_publishes_at_research_or_implementation_completion(
- self, report_status: str, implementation_status: str | None
- ) -> None:
+ @parameterized.expand([("research", True), ("repo_selection", True), ("implementation", False)])
+ def test_only_signal_owned_tasks_publish_signal_metadata(self, stage: str, publishes: bool) -> None:
driver = str(uuid4())
- report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver, status=report_status)
- research = self._run(report=report)
- self._set_task_cost(research, 10_001)
- implementation = None
- if implementation_status:
- implementation = self._run(report=report, stage="implementation")
- SignalReportTask.objects.create(
- team=self.team, report=report, task_id=implementation.task_id, relationship="implementation"
- )
- self._set_task_cost(implementation, 20_002)
- async_to_sync(publish_report_spend)(team_id=self.team.id, report_id=str(report.id))
- if implementation is not None:
- self.temporal.start_workflow.assert_not_called()
- assert implementation_status is not None
- implementation.status = implementation_status
- implementation.save(update_fields=["status"])
- complete_task_spend(team_id=self.team.id, run_id=str(implementation.id))
- self.temporal.start_workflow.assert_awaited_once()
- assert self.temporal.start_workflow.call_args.args[1] == SignalSpendUpdate(
- team_id=self.team.id, signal_id=driver
- )
- self.temporal.start_workflow.return_value.result.assert_awaited_once()
- assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver]) == {
- driver: 11.0003 if implementation else 5.0001
- }
+ report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
+ run = self._run(report=report, stage=stage)
+ self._set_task_cost(run, 10_001)
+ report.billing_exempt_reason = SignalReport.BillingExemptReason.POSTHOG_SYSTEM
+ report.save(update_fields=["billing_exempt_reason"])
+ run.status = TaskRun.Status.COMPLETED
+ run.save(update_fields=["status"])
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ assert self.temporal.start_workflow.await_count == int(publishes)
+ spend = SignalSpend.objects.for_team(self.team.id).get(task_run_id=run.id)
+ assert spend.spend == Decimal("5.0001")
+ assert spend.finalized_at is not None
+ assert spend.billing_exempt
@parameterized.expand([(False,), (True,)])
@override_settings(SANDBOX_AI_GATEWAY_URL="https://gateway.example", SANDBOX_AI_GATEWAY_MINT_KEY="test-key")
@@ -643,17 +1062,26 @@ class TestSignalSpend(BaseTest):
if is_scout
else None
)
+ if scout:
+ record_task_spend(team_id=self.team.id, run_id=str(run.id), scout_run_id=str(scout.id), stage="scout")
run.state = {"ai_stage": "research", "unprocessed_request_ids": ["pending-request"], "token_cost": {}}
run.status = TaskRun.Status.COMPLETED
run.save(update_fields=["state", "status"])
with patch(
"products.tasks.backend.logic.services.gateway_usage._fetch_gateway_cost",
- new=AsyncMock(return_value=GatewayRequestCost(model="example", provider="example", cost_microusd=123_456)),
+ new=AsyncMock(return_value=None),
) as fetch:
- if scout:
- complete_task_spend(team_id=self.team.id, run_id=str(run.id))
- else:
- settle_signal_task_spend(team_id=self.team.id, signal_id=driver)
+ with self.assertRaises(TaskTokenUsageUnavailable):
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ owner = SignalSpend.objects.for_team(self.team.id).get(
+ **({"scout_run_id": scout.id} if scout else {"signal_id": driver})
+ )
+ assert owner.accounting_failed
+ assert owner.finalized_at is None
+ self.temporal.start_workflow.assert_not_called()
+ fetch.reset_mock()
+ fetch.return_value = GatewayRequestCost(model="example", provider="example", cost_microusd=123_456)
+ self._complete_task_spend(team_id=self.team.id, run_id=str(run.id))
fetch.assert_awaited_once_with("pending-request")
if scout:
scout.refresh_from_db()
@@ -664,25 +1092,22 @@ class TestSignalSpend(BaseTest):
summary = signal_spend_summaries(team_id=self.team.id, signal_ids=[driver])[driver]
assert summary.total_spend == 16.3456
assert summary.failed_stages == []
- self.temporal.start_workflow.assert_not_called()
+ assert self.temporal.start_workflow.await_count == (0 if is_scout else 1)
- def test_one_shot_receipts_preserve_known_spend_and_record_failed_stage(self) -> None:
+ def test_one_shot_calls_preserve_known_spend_and_record_failed_stage(self) -> None:
driver = str(uuid4())
- for request_id, cost in [("one-shot-1", None), ("one-shot-2", 9), ("one-shot-2", None)]:
- record_llm_spend(
- request_id, token_cost_microusd=cost, team_id=self.team.id, signal_id=driver, stage="actionability"
- )
+ for cost in (None, 9, None):
+ record_llm_spend(token_cost_microusd=cost, team_id=self.team.id, signal_id=driver, stage="actionability")
summary = signal_spend_summaries(team_id=self.team.id, signal_ids=[driver])[driver]
assert summary.total_spend == 0.0009
assert summary.failed_stages == ["actionability"]
- for _ in range(2):
- record_llm_spend(
- "one-shot-1", token_cost_microusd=1, team_id=self.team.id, signal_id=driver, stage="actionability"
- )
- assert SignalSpend.objects.for_team(self.team.id).count() == 2
+ record_llm_spend(token_cost_microusd=1, team_id=self.team.id, signal_id=driver, stage="actionability")
+ assert SignalSpend.objects.for_team(self.team.id).count() == 1
assert signal_spend_totals(team_id=self.team.id, signal_ids=[driver]) == {driver: 0.001}
- assert signal_spend_summaries(team_id=self.team.id, signal_ids=[driver])[driver].failed_stages == []
- record_llm_spend(None, token_cost_microusd=None, team_id=self.team.id, signal_id=driver, stage="summarization")
+ assert signal_spend_summaries(team_id=self.team.id, signal_ids=[driver])[driver].failed_stages == [
+ "actionability"
+ ]
+ record_llm_spend(token_cost_microusd=None, team_id=self.team.id, signal_id=driver, stage="summarization")
other_team = Team.objects.create(organization=self.organization, name="Other project")
assert signal_spend_totals(team_id=other_team.id, signal_ids=[driver]) == {}
@@ -698,6 +1123,10 @@ class TestSignalSpend(BaseTest):
skill_name="example-scout",
skill_version=1,
)
+ scout.created_at -= timedelta(days=1)
+ scout.save(update_fields=["created_at"])
+ record_task_spend(team_id=self.team.id, run_id=str(task_run.id), scout_run_id=str(scout.id), stage="scout")
+ assert SignalSpend.objects.for_team(self.team.id).get(scout_run=scout).created_at == scout.created_at
assert scout.total_spend is None
scout.refresh_from_db()
assert scout.total_spend == (Decimal("16.3456") if settled_before_owner else Decimal("4"))
@@ -719,35 +1148,65 @@ class TestSignalSpend(BaseTest):
assert scout.metadata is not None
assert scout.metadata["spend_accounting_failed_stages"] == []
- @parameterized.expand([("generation", "actionability"), ("task", "research"), ("task_cost", "task_cost_update")])
- def test_accounting_database_failure_does_not_abort_pipeline_transaction(self, source: str, stage: str) -> None:
+ @parameterized.expand([(TaskRun.Status.IN_PROGRESS,), (TaskRun.Status.COMPLETED,)])
+ def test_reaped_scout_retries_accounting_until_gateway_cost_settles(self, status: str) -> None:
+ run = self._run()
+ if status == TaskRun.Status.COMPLETED:
+ run.state["timed_out_inactivity"] = True
+ run.state["unprocessed_request_ids"] = ["late-scout-request"]
+ run.status = status
+ run.created_at = timezone.now() - timedelta(seconds=STALE_RUN_CUTOFF_S + 60)
+ run.save(update_fields=["state", "status", "created_at"])
+ scout = SignalScoutRun.objects.for_team(self.team.id).create(
+ team=self.team, task_run=run, skill_name="example-scout", skill_version=1
+ )
+ record_task_spend(team_id=self.team.id, run_id=str(run.id), scout_run_id=str(scout.id), stage="scout")
+ with (
+ patch("products.signals.backend.tasks.settle_scout_spend.delay") as enqueue,
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ _self_heal_stale_runs(self.team.id, "example-scout")
+ _self_heal_stale_runs(self.team.id, "example-scout")
+ enqueue.assert_called_once_with(team_id=self.team.id, run_id=str(run.id))
+ with patch(
+ "products.tasks.backend.logic.services.gateway_usage._fetch_gateway_cost", new=AsyncMock(return_value=None)
+ ):
+ with self.assertRaises(TaskTokenUsageUnavailable):
+ settle_scout_spend(team_id=self.team.id, run_id=str(run.id))
+ spend = SignalSpend.objects.for_team(self.team.id).get(scout_run=scout)
+ assert spend.finalized_at is None
+ with patch(
+ "products.tasks.backend.logic.services.gateway_usage._fetch_gateway_cost",
+ new=AsyncMock(return_value=GatewayRequestCost(model="example", provider="example", cost_microusd=123_456)),
+ ):
+ settle_scout_spend(team_id=self.team.id, run_id=str(run.id))
+ spend = SignalSpend.objects.for_team(self.team.id).get(scout_run=scout)
+ assert spend.finalized_at is not None
+ assert spend.spend == Decimal("16.3456")
+
+ @parameterized.expand([("generation",), ("task",), ("pipeline",), ("report_start",)])
+ def test_accounting_failure_does_not_silently_create_untracked_work(self, source: str) -> None:
driver = str(uuid4())
report = SignalReport.objects.create(team=self.team, triggering_signal_id=driver)
- existing_run = self._run(report=report) if source == "task_cost" else None
def fail_spend_query(
execute: Callable[..., object], sql: str, params: object, many: bool, context: object
) -> object:
- if sql.startswith('INSERT INTO "signals_signalspend"') or (
- source == "task_cost" and sql.startswith("SELECT") and '"signals_signalspend"' in sql
- ):
- return execute("SELECT 1 / 0", (), False, context)
+ if sql.startswith('INSERT INTO "signals_signalspend"'):
+ raise RuntimeError("Accounting unavailable")
return execute(sql, params, many, context)
- with capture_logs() as logs, connection.execute_wrapper(fail_spend_query):
- if existing_run is not None:
- with self.captureOnCommitCallbacks(execute=True):
- task_run_cost_updated.send(sender=TaskRun, run_id=existing_run.id, team_id=self.team.id)
- elif source == "task":
- run = self._run(report=report)
- assert TaskRun.objects.filter(id=run.id).exists()
+ with (
+ connection.execute_wrapper(fail_spend_query),
+ self.assertRaisesRegex(RuntimeError, "Accounting unavailable"),
+ ):
+ if source == "task":
+ self._run(report=report)
+ elif source == "pipeline":
+ complete_signal_pipeline(team_id=self.team.id, signal_id=driver)
+ elif source == "report_start":
+ begin_report_spend(team_id=self.team.id, signal_id=driver, report_id=str(report.id))
else:
- record_llm_spend(
- "example-request",
- token_cost_microusd=123,
- team_id=self.team.id,
- signal_id=driver,
- stage="actionability",
- )
- assert SignalReport.objects.filter(id=report.id).exists()
- assert any(log["event"] == "signals.spend.accounting_failed" and log["stage"] == stage for log in logs)
+ record_llm_spend(token_cost_microusd=123, team_id=self.team.id, signal_id=driver, stage="actionability")
+ assert not SignalSpend.objects.for_team(self.team.id).exists()
+ assert not TaskRun.objects.filter(team_id=self.team.id).exists()
diff --git a/products/signals/backend/test/test_spend_workflow.py b/products/signals/backend/test/test_spend_workflow.py
index 5f583fb27aa1..1670b1fb95c5 100644
--- a/products/signals/backend/test/test_spend_workflow.py
+++ b/products/signals/backend/test/test_spend_workflow.py
@@ -1,12 +1,17 @@
import asyncio
+import logging
+from datetime import timedelta
+from types import SimpleNamespace
from uuid import uuid4
import pytest
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
from django.test import override_settings
-from temporalio import activity
+from temporalio import activity, workflow
+from temporalio.client import WorkflowFailureError
+from temporalio.common import RetryPolicy
from temporalio.exceptions import ApplicationError
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
@@ -16,21 +21,73 @@ from products.signals.backend.spend import (
SignalSpendPublication,
SignalSpendSummary,
SignalSpendUpdate,
- publish_signal_spend,
submit_signal_metadata_updates,
)
from products.signals.backend.temporal.spend import UpdateSignalSpendWorkflow
+from products.signals.backend.temporal.summary import (
+ SignalReportSummaryWorkflow,
+ check_report_task_settlement_activity,
+ publish_report_spend_activity,
+)
+from products.signals.backend.temporal.types import SignalReportSummaryWorkflowInputs
+from products.tasks.backend.facade.agents import MultiTurnSession
+from products.tasks.backend.models import Task
+
+
+class SettlingTask:
+ def __init__(self) -> None:
+ self.completed = False
+
+ @workflow.signal
+ def complete_task(self, status: str, error: str | None) -> None:
+ self.completed = True
+
+ @workflow.run
+ async def run(self, input: SignalSpendUpdate) -> None:
+ await workflow.wait_condition(lambda: self.completed)
+ await workflow.execute_activity(
+ "publish_task_spend",
+ input,
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+
+
+class ResearchAndSettle:
+ @workflow.run
+ async def run(self, input: SignalReportSummaryWorkflowInputs) -> str:
+ await workflow.execute_activity("research_and_end_session", input, start_to_close_timeout=timedelta(minutes=1))
+ await SignalReportSummaryWorkflow()._wait_for_task_settlement(input)
+ await workflow.execute_activity(
+ publish_report_spend_activity,
+ input,
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(initial_interval=timedelta(seconds=1), maximum_attempts=2),
+ )
+ return "research retained"
@pytest.mark.asyncio
-async def test_spend_updates_retry_and_serialize_updates_arriving_during_and_after_a_write() -> None:
[email protected]("publication_fails", [False, True])
+async def test_spend_updates_retry_and_serialize_updates_arriving_during_and_after_a_write(
+ publication_fails: bool,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ caplog.set_level(logging.INFO, logger="temporalio.workflow")
+ caplog.set_level(logging.INFO, logger="temporalio.activity")
queue = f"spend-test-{uuid4()}"
driver = str(uuid4())
write_started, release_write = asyncio.Event(), asyncio.Event()
+ confirmation_started, release_confirmation = asyncio.Event(), asyncio.Event()
+ confirmation_finished = asyncio.Event()
+ writes_before_confirmation: list[int] = []
latest_spend = 1
stored: list[int] = []
changes: list[SignalMetadataChange] = []
attempts = 0
+ expected_initial = [2] if publication_fails else [1, 2]
@activity.defn(name="update_signal_spend_activity")
async def write_metadata(input: SignalSpendUpdate) -> SignalSpendPublication:
@@ -38,11 +95,15 @@ async def test_spend_updates_retry_and_serialize_updates_arriving_during_and_aft
assert (input.team_id, input.signal_id) == (1, driver)
attempts += 1
if attempts == 1:
- raise ApplicationError("ClickHouse row not visible yet", type="SignalSpendNotVisible")
+ raise ApplicationError("Kafka delivery failed")
spend = latest_spend
if attempts == 2:
write_started.set()
await release_write.wait()
+ if publication_fails:
+ raise ApplicationError("Kafka delivery failed permanently", non_retryable=True)
+ if confirmation_started.is_set() and not confirmation_finished.is_set():
+ writes_before_confirmation.append(spend)
stored.append(spend)
changes.extend(input.changes)
return SignalSpendPublication(
@@ -56,7 +117,10 @@ async def test_spend_updates_retry_and_serialize_updates_arriving_during_and_aft
nonlocal confirmation_attempts
confirmation_attempts += 1
if confirmation_attempts == 1:
+ confirmation_started.set()
+ await release_confirmation.wait()
raise ApplicationError("ClickHouse ingestion delayed", type="SignalSpendNotVisible")
+ confirmation_finished.set()
async with await WorkflowEnvironment.start_time_skipping() as env:
async with Worker(
@@ -70,20 +134,148 @@ async def test_spend_updates_retry_and_serialize_updates_arriving_during_and_aft
override_settings(VIDEO_EXPORT_TASK_QUEUE=queue),
patch("posthog.temporal.common.client.async_connect", new=AsyncMock(return_value=env.client)),
):
- publication = asyncio.create_task(publish_signal_spend(team_id=1, signal_id=driver))
- await asyncio.wait_for(write_started.wait(), timeout=20)
- assert not publication.done()
+ await submit_signal_metadata_updates(team_id=1, signal_ids=[driver])
+ await asyncio.wait_for(write_started.wait(), timeout=30)
+ handle = env.client.get_workflow_handle(f"signal-spend-1-{driver}")
latest_spend = 2
+ await submit_signal_metadata_updates(team_id=1, signal_ids=[driver])
+ release_write.set()
+ if publication_fails:
+ with pytest.raises(WorkflowFailureError):
+ await asyncio.wait_for(handle.result(), timeout=30)
+ assert stored == []
+ publication_fails = False
+ await submit_signal_metadata_updates(team_id=1, signal_ids=[driver])
+ await asyncio.wait_for(handle.result(), timeout=30)
+ assert stored == expected_initial
+ assert confirmation_attempts == 0
+
+ latest_spend = 3
change = SignalMetadataChange(report_id="new-report", expected_report_id="old-report")
await submit_signal_metadata_updates(team_id=1, signal_ids=[driver], change=change)
- handle = env.client.get_workflow_handle(f"signal-spend-1-{driver}")
+ await asyncio.wait_for(confirmation_started.wait(), timeout=30)
+ latest_spend = 4
+ await submit_signal_metadata_updates(team_id=1, signal_ids=[driver])
await handle.signal("refresh")
- release_write.set()
- await asyncio.wait_for(publication, timeout=20)
- assert stored == [1, 2]
+ release_confirmation.set()
+ await asyncio.wait_for(handle.result(), timeout=30)
+ assert stored[-2:] == [3, 4]
assert changes == [change]
+ assert confirmation_attempts == 2
+ assert writes_before_confirmation == []
- latest_spend = 3
- await publish_signal_spend(team_id=1, signal_id=driver)
- assert stored == [1, 2, 3]
- assert confirmation_attempts == 4
+ latest_spend = 5
+ await submit_signal_metadata_updates(team_id=1, signal_ids=[driver])
+ await asyncio.wait_for(handle.result(), timeout=30)
+ assert stored[-1] == 5
+ assert confirmation_attempts == 2
+
+
[email protected]("publication_fails", [False, True])
+async def test_research_releases_shared_slot_before_waiting_for_task_settlement(
+ publication_fails: bool, caplog: pytest.LogCaptureFixture
+) -> None:
+ caplog.set_level(logging.INFO, logger="temporalio.workflow")
+ caplog.set_level(logging.INFO, logger="temporalio.activity")
+ queue = f"settlement-test-{uuid4()}"
+ task_queue = f"task-test-{uuid4()}"
+ task_workflow_id = str(uuid4())
+ signal_id = str(uuid4())
+ researches = 0
+ published = False
+ confirmations = 0
+
+ @activity.defn(name="publish_task_spend")
+ async def publish_task_spend(input: SignalSpendUpdate) -> None:
+ await submit_signal_metadata_updates(team_id=input.team_id, signal_ids=[input.signal_id])
+
+ @activity.defn(name="research_and_end_session")
+ async def research_and_end_session(input: SignalReportSummaryWorkflowInputs) -> None:
+ nonlocal researches
+ researches += 1
+ session = MultiTurnSession(
+ task=MagicMock(origin_product=Task.OriginProduct.SIGNAL_REPORT),
+ task_run=MagicMock(),
+ _workflow_handle=env.client.get_workflow_handle(task_workflow_id),
+ )
+ await session.end()
+
+ @activity.defn(name="update_signal_spend_activity")
+ async def write_metadata(input: SignalSpendUpdate) -> SignalSpendPublication:
+ nonlocal published
+ if publication_fails:
+ raise ApplicationError("Kafka unavailable", non_retryable=True)
+ published = True
+ return SignalSpendPublication(
+ signal=input, summary=SignalSpendSummary(total_spend=4, failed_stages=[]), versions={}
+ )
+
+ @activity.defn(name="confirm_signal_spend_activity")
+ async def confirm_metadata(input: SignalSpendPublication) -> None:
+ nonlocal confirmations
+ confirmations += 1
+
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with (
+ Worker(
+ env.client,
+ task_queue=queue,
+ workflows=[ResearchAndSettle, UpdateSignalSpendWorkflow],
+ activities=[
+ research_and_end_session,
+ check_report_task_settlement_activity,
+ publish_report_spend_activity,
+ write_metadata,
+ confirm_metadata,
+ ],
+ max_concurrent_activities=1,
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ),
+ Worker(
+ env.client,
+ task_queue=task_queue,
+ workflows=[SettlingTask],
+ activities=[publish_task_spend],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ),
+ ):
+ with (
+ override_settings(VIDEO_EXPORT_TASK_QUEUE=queue),
+ patch("posthog.temporal.common.client.async_connect", new=AsyncMock(return_value=env.client)),
+ patch(
+ "products.signals.backend.temporal.summary.async_connect", new=AsyncMock(return_value=env.client)
+ ),
+ patch("products.signals.backend.temporal.summary.SignalSpend") as spend_model,
+ patch("products.signals.backend.temporal.summary.finish_report_spend"),
+ patch(
+ "products.signals.backend.temporal.summary.get_task_run",
+ return_value=SimpleNamespace(workflow_id=task_workflow_id),
+ ),
+ ):
+ spend_model.objects.for_team.return_value.filter.return_value.values_list.return_value = [uuid4()]
+ await env.client.start_workflow(
+ SettlingTask.run,
+ SignalSpendUpdate(team_id=1, signal_id=signal_id),
+ id=task_workflow_id,
+ task_queue=task_queue,
+ )
+ result = await asyncio.wait_for(
+ env.client.execute_workflow(
+ ResearchAndSettle.run,
+ SignalReportSummaryWorkflowInputs(team_id=1, report_id=str(uuid4()), signal_id=signal_id),
+ id=str(uuid4()),
+ task_queue=queue,
+ ),
+ timeout=30,
+ )
+ assert result == "research retained"
+ assert researches == 1
+ handle = env.client.get_workflow_handle(f"signal-spend-1-{signal_id}")
+ if publication_fails:
+ with pytest.raises(WorkflowFailureError):
+ await asyncio.wait_for(handle.result(), timeout=30)
+ else:
+ await asyncio.wait_for(handle.result(), timeout=30)
+ assert published is not publication_fails
+ assert confirmations == 0
diff --git a/products/signals/backend/test/test_summary_settle_autostart.py b/products/signals/backend/test/test_summary_settle_autostart.py
index 3aab69f663fc..00860def990c 100644
--- a/products/signals/backend/test/test_summary_settle_autostart.py
+++ b/products/signals/backend/test/test_summary_settle_autostart.py
@@ -124,6 +124,8 @@ class _Recorder:
self.candidate_on_first_check = candidate_on_first_check
self.researches = 0
self.autostarts = 0
+ self.spend_owners: list[str | None] = []
+ self.work_owners: list[str | None] = []
self.candidate_checks = 0
# Auto-starts completed by the time the notification child made its first state check. The
# child only waits for the implementation PR if the task already exists then, so a 0 here
@@ -169,20 +171,27 @@ async def _run_summary_workflow(
async def fake_fetch(input: FetchSignalsForReportInput) -> FetchSignalsForReportOutput:
return FetchSignalsForReportOutput(signals=[_signal_data()])
+ @activity.defn(name="report_spend_owner_activity")
+ async def fake_spend_owner(input: SignalReportSummaryWorkflowInputs) -> str:
+ return str(uuid.UUID(int=recorder.researches + 1))
+
@activity.defn(name="mark_report_in_progress_activity")
async def fake_mark_in_progress(input: MarkReportInProgressInput) -> None:
return None
@activity.defn(name="report_safety_judge_activity")
async def fake_safety(input: SafetyJudgeInput) -> SafetyJudgeOutput:
+ recorder.work_owners.append(input.signal_id)
return SafetyJudgeOutput(safe=True, explanation="ok")
@activity.defn(name="select_repository_activity")
async def fake_select_repo(input: SelectRepositoryInput) -> RepoSelectionResult:
+ recorder.work_owners.append(input.signal_id)
return RepoSelectionResult(repository="owner/repo", reason="selected")
@activity.defn(name="run_agentic_report_activity")
async def fake_research(input: RunAgenticReportInput) -> RunAgenticReportOutput:
+ recorder.work_owners.append(input.signal_id)
recorder.researches += 1
return RunAgenticReportOutput(
title="t",
@@ -203,6 +212,10 @@ async def _run_summary_workflow(
async def fake_publish(input: PublishReportCompletedInput) -> None:
return None
+ @activity.defn(name="publish_report_spend_activity")
+ async def fake_publish_spend(input: SignalReportSummaryWorkflowInputs) -> None:
+ recorder.spend_owners.append(input.signal_id)
+
@activity.defn(name="implementation_buffer_seconds_activity")
async def fake_buffer(input: ImplementationBufferInput) -> int:
return recorder.buffer_seconds
@@ -232,11 +245,13 @@ async def _run_summary_workflow(
fake_quota,
fake_fetch,
fake_mark_in_progress,
+ fake_spend_owner,
fake_safety,
fake_select_repo,
fake_research,
fake_mark_ready,
fake_publish,
+ fake_publish_spend,
fake_buffer,
fake_candidate,
fake_autostart,
@@ -308,6 +323,8 @@ async def test_signal_during_buffer_re_researches_instead_of_implementing():
# A signal landed during the buffer (report back to candidate), so the run loops to re-research
# rather than implementing the now-stale summary; implementation only starts after it settles again.
assert recorder.researches == 2
+ assert recorder.spend_owners == [str(uuid.UUID(int=1)), str(uuid.UUID(int=2))]
+ assert recorder.work_owners == [str(uuid.UUID(int=1))] * 3 + [str(uuid.UUID(int=2))] * 3
assert recorder.candidate_checks == 2
assert recorder.autostarts == 1
# Only the settled run notifies, so no card goes out for the summary the re-research replaced.
diff --git a/products/signals/backend/test/test_summary_workflow.py b/products/signals/backend/test/test_summary_workflow.py
index 26713ce490c4..1a5be7154ac7 100644
--- a/products/signals/backend/test/test_summary_workflow.py
+++ b/products/signals/backend/test/test_summary_workflow.py
@@ -6,6 +6,8 @@ from datetime import UTC, datetime
import pytest
from unittest.mock import patch
+from django.db import OperationalError
+
import pytest_asyncio
from asgiref.sync import sync_to_async
from temporalio import activity
@@ -189,6 +191,8 @@ class _Recorder:
research_metrics: list[dict[str, object]] | None = None,
checks_snapshot: dict[str, str] | None = None,
selected_repository: str | None = "owner/repo",
+ spend_failures: int = 0,
+ spend_owner_available: bool = True,
research_errors: list[Exception] | None = None,
) -> None:
self.gate_answers = gate_answers or {}
@@ -204,6 +208,7 @@ class _Recorder:
self.failure_error_types: list[str | None] = []
self.gate_checks: list[str] = []
self.fetches = 0
+ self.claimed_before_fetch: list[bool] = []
self.assigned_signal_checks = 0
self.failure_reasons: list[str | None] = []
self.marks_in_progress = 0
@@ -211,6 +216,8 @@ class _Recorder:
self.repo_selections = 0
self.researches = 0
self.spend_publications = 0
+ self.spend_failures = spend_failures
+ self.spend_owner_available = spend_owner_available
self.reverts = 0
self.resets = 0
self.failures = 0
@@ -260,6 +267,7 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
async def fake_fetch(input: FetchSignalsForReportInput) -> FetchSignalsForReportOutput:
signals = recorder.fetch_results[min(recorder.fetches, len(recorder.fetch_results) - 1)]
recorder.fetches += 1
+ recorder.claimed_before_fetch.append(recorder.marks_in_progress > 0)
return FetchSignalsForReportOutput(signals=signals)
@activity.defn(name="report_has_assigned_signals_activity")
@@ -267,6 +275,12 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
recorder.assigned_signal_checks += 1
return recorder.has_assigned_signals
+ @activity.defn(name="report_spend_owner_activity")
+ async def fake_spend_owner(input: SignalReportSummaryWorkflowInputs) -> str | None:
+ if not recorder.spend_owner_available:
+ return None
+ return input.report_id
+
@activity.defn(name="mark_report_in_progress_activity")
async def fake_mark_in_progress(input: MarkReportInProgressInput) -> None:
recorder.marks_in_progress += 1
@@ -313,9 +327,15 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
async def fake_publish(input: PublishReportCompletedInput) -> None:
return None
+ @activity.defn(name="check_report_task_settlement_activity")
+ async def fake_task_settlement(input: SignalReportSummaryWorkflowInputs) -> None:
+ return None
+
@activity.defn(name="publish_report_spend_activity")
async def fake_publish_spend(input: SignalReportSummaryWorkflowInputs) -> None:
recorder.spend_publications += 1
+ if recorder.spend_publications <= recorder.spend_failures:
+ raise OperationalError("Transient accounting failure")
@activity.defn(name="revert_report_to_candidate_activity")
async def fake_revert(input: RevertReportToCandidateInput) -> None:
@@ -344,6 +364,7 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
fake_fetch,
fake_has_assigned_signals,
fake_mark_in_progress,
+ fake_spend_owner,
fake_safety,
fake_select_repo,
fake_research,
@@ -351,6 +372,7 @@ async def _run_summary_workflow(recorder: _Recorder) -> None:
fake_mark_ready,
fake_publish,
fake_publish_spend,
+ fake_task_settlement,
fake_revert,
fake_reset,
fake_failed,
@@ -394,12 +416,13 @@ async def test_pre_research_gate_reverts_report_and_skips_research():
@pytest.mark.asyncio
-async def test_open_gates_let_the_run_flow_through():
- recorder = _Recorder(gate_answers={})
[email protected]("spend_failures", [0, 3])
+async def test_open_gates_let_the_run_flow_through(spend_failures: int) -> None:
+ recorder = _Recorder(gate_answers={}, spend_failures=spend_failures)
await _run_summary_workflow(recorder)
assert recorder.gate_checks == ["summary_entry", "pre_repo_selection", "pre_research"]
assert recorder.researches == 1
- assert recorder.spend_publications == 1
+ assert recorder.spend_publications == spend_failures + 1
assert recorder.reverts == 0
assert recorder.failures == 0
@@ -493,11 +516,22 @@ async def test_no_repository_keeps_report_content_and_logs_the_blocker():
# ---------------------------------------------------------------------------
+async def test_summary_without_an_open_spend_owner_makes_no_paid_calls():
+ recorder = _Recorder(spend_owner_available=False)
+ await _run_summary_workflow(recorder)
+ assert recorder.fetches == 0
+ assert recorder.safety_checks == 0
+ assert recorder.researches == 0
+ assert recorder.spend_publications == 0
+
+
@pytest.mark.asyncio
async def test_empty_first_fetch_is_retried_and_run_proceeds():
recorder = _Recorder(fetch_results=[[], [], [_signal_data()]])
await _run_summary_workflow(recorder)
assert recorder.fetches == 3
+ assert all(recorder.claimed_before_fetch)
assert recorder.researches == 1
assert recorder.spend_publications == 1
assert recorder.failures == 0
@@ -510,8 +544,8 @@ async def test_persistently_empty_fetch_leaves_report_for_repromotion_when_signa
await _run_summary_workflow(recorder)
assert recorder.fetches == 1 + EMPTY_FETCH_RETRY_ATTEMPTS
assert recorder.assigned_signal_checks == 1
- # Neither researched nor failed: the report stays where it is and grouping re-promotes it.
- assert recorder.marks_in_progress == 0
+ assert recorder.marks_in_progress == 1
+ assert recorder.reverts == 1
assert recorder.researches == 0
assert recorder.failures == 0
diff --git a/products/signals/backend/test/test_task_run_artefacts.py b/products/signals/backend/test/test_task_run_artefacts.py
index d2ee724b29c9..cee0a31e3ed2 100644
--- a/products/signals/backend/test/test_task_run_artefacts.py
+++ b/products/signals/backend/test/test_task_run_artefacts.py
@@ -603,7 +603,8 @@ class TestReleaseQuotaCancelledImplementation(BaseTest):
task = self._task()
assert release_quota_cancelled_implementation(team_id=self.team.id, task_id=str(task.id)) == []
- def test_release_skips_report_whose_billable_pr_shipped_from_a_sibling_run(self):
+ @parameterized.expand([("legacy", None), ("manual", "posthog_ai")])
+ def test_release_skips_report_whose_billable_pr_shipped_from_a_sibling_run(self, _name, billing_product):
# The cancel decision is run-scoped but this delete is task-scoped: a sibling run of the
# same task can have shipped the report's billable PR in an earlier period. The bridge row
# is billing's evidence for that charge (billed-earlier dedup, refund eligibility), so the
@@ -615,14 +616,15 @@ class TestReleaseQuotaCancelledImplementation(BaseTest):
team=self.team,
task=task,
output={"pr_url": "https://github.com/x/y/pull/1"},
+ state={"billing_product": billing_product} if billing_product else {},
created_at=datetime(2026, 6, 10, tzinfo=UTC),
)
released = release_quota_cancelled_implementation(team_id=self.team.id, task_id=str(task.id))
+ assert (first_billable_pr_run_at(report.id, team_id=report.team_id) is not None) == (billing_product is None)
assert released == []
assert SignalReportTask.objects.filter(task_id=task.id, relationship=TASK_RUN_TYPE_IMPLEMENTATION).exists()
- assert first_billable_pr_run_at(report.id) is not None
# Auto-start must still see a started implementation — the report is implemented, not stuck.
assert SignalReport.associated_task_runs(
report_id=str(report.id), team_id=self.team.id, product=SIGNALS_PRODUCT, type=TASK_RUN_TYPE_IMPLEMENTATION
diff --git a/products/signals/backend/views.py b/products/signals/backend/views.py
index 10221e69c42e..8e833cea7b49 100644
--- a/products/signals/backend/views.py
+++ b/products/signals/backend/views.py
@@ -132,11 +132,6 @@ from products.signals.backend.dismissal_notes import forward_dismissal_note
from products.signals.backend.enums import ReportPriority, ReportTaskActionKind
from products.signals.backend.facade.api import emit_signal
from products.signals.backend.feedback_notes import forward_feedback_note
-from products.signals.backend.free_trial import (
- FREE_TRIAL_PR_MESSAGE,
- FreeTrialPullRequestRefused,
- capture_signal_report_free_trial_paused,
-)
from products.signals.backend.implementation_pr import (
ImplementationPr,
fetch_implementation_prs_for_reports,
@@ -2921,7 +2916,7 @@ class SignalReportViewSet(
# One grouped query for the whole page, in place of the per-row annotation the other
# actions carry, for the serializer's refund_ineligibility_reason field.
with tracer.start_as_current_span("signals.reports.list.fetch_billable_pr_runs"):
- first_billable_pr_run_at_map = first_billable_pr_run_at_by_report(report_ids)
+ first_billable_pr_run_at_map = first_billable_pr_run_at_by_report(report_ids, team_id=self.team_id)
# Same trade for the two artefact-derived fields the row renders: one query each over the
# page, rather than a correlated subquery the sort makes Postgres run for every report in
@@ -3432,9 +3427,6 @@ class SignalReportViewSet(
if report_action.disabled_reason is not None:
capture("blocked")
- if report_action.disabled_reason == FREE_TRIAL_PR_MESSAGE:
- capture_signal_report_free_trial_paused(self.team, report_id=str(report.id), stage="report_action")
- raise FreeTrialPullRequestRefused()
error_status: int = (
status.HTTP_403_FORBIDDEN
if report_action.disabled_reason == AI_CONSENT_DISABLED_REASON
@@ -4278,7 +4270,7 @@ class SignalReportViewSet(
# Same decision the serializer's `refund_ineligibility_reason` field renders from,
# so the UI's button state and this endpoint's 400s can never disagree.
- billable_run = first_billable_pr_run(report.id)
+ billable_run = first_billable_pr_run(report.id, team_id=report.team_id)
# Computed once and frozen onto the refund row below: the credited-path sync must
# report the period the refund was accepted in, not whatever period is current when
# the Celery task eventually reaches billing.
diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py
index ec7fbe017587..ccaa5e2cd046 100644
--- a/products/tasks/backend/facade/api.py
+++ b/products/tasks/backend/facade/api.py
@@ -3055,6 +3055,7 @@ _PROTECTED_RUN_STATE_KEYS = frozenset(
# is_interactive_signals_run reads it the same way, so forging it would move the run off
# the interactive budget and out of its per-run spend ceiling.
"ai_stage",
+ "billing_product",
# A removed stamp lets a model change leave the token's pin, and the gateway denies every turn.
GATEWAY_PRODUCT_STATE_KEY,
# Names the agent (scout, custom agent, workflow) the run executes, lifted onto its
@@ -3451,7 +3452,7 @@ def _refresh_self_driving_quota_for_pr(run: TaskRun, old_pr_url: str | None) ->
and a refresh hiccup must not abort those or turn an already-committed run write into a 500.
"""
try:
- if old_pr_url:
+ if old_pr_url or (run.state or {}).get("billing_product") == "posthog_ai":
return
new_pr_url = (run.output or {}).get("pr_url") if isinstance(run.output, dict) else None
if not new_pr_url or run.task.origin_product != Task.OriginProduct.SIGNAL_REPORT:
@@ -3485,16 +3486,9 @@ def _refresh_self_driving_quota_for_pr(run: TaskRun, old_pr_url: str | None) ->
def enforce_self_driving_free_trial(
team: Team, *, report_id: str | None = None, stage: str = "manual_create", enabled: bool | None = None
) -> None:
- """Refuse to create or start a PR-opening self-driving task while the team's org is on a
- Self-driving free trial: a trial org gets reports, not pull requests, on any path (the
- auto-start gate in products/signals/backend/auto_start.py holds the pipeline back the same
- way). Emits `signal_report_free_trial_paused` at ``stage``. Raises
- ``FreeTrialPullRequestRefused`` (402, code ``self_driving_free_trial``) so clients can show
- the trial message.
-
- ``enabled`` takes a verdict the caller already resolved, and then the gate reads no flag of
- its own. A caller that holds a database lock resolves the flag before it takes the lock,
- because the read does network I/O.
+ """Keep PR-opening report work unavailable during the self-driving trial.
+
+ Resolve enabled before taking database locks, since flag evaluation can use the network.
"""
from products.signals.backend.free_trial import ( # noqa: PLC0415 — cross-product read kept off the api import path
FreeTrialPullRequestRefused,
@@ -3509,13 +3503,6 @@ def enforce_self_driving_free_trial(
def enforce_self_driving_pr_quota(team: Team, *, report_id: str | None = None, stage: str = "manual_create") -> None:
- """Refuse to create a PR-opening self-driving task while the team's org is over its self-driving
- credits quota with enforcement on. The implementation task is the step that leads to the
- billable PR, so the manual create-from-report path must respect the same limit as the pipeline
- auto-start gate (products/signals/backend/auto_start.py). Emits `signal_report_quota_paused`
- at ``stage`` whenever the org is limited, so each caller's gate stays measurable during the
- dark launch like every other gate. Raises ``QuotaLimitExceeded`` (402).
- """
from posthog.exceptions import QuotaLimitExceeded # noqa: PLC0415 — keep billing deps off the api import path
from products.signals.backend.quota import ( # noqa: PLC0415 — cross-product read kept off the api import path
@@ -6775,6 +6762,12 @@ def resume_task_run_in_cloud(
if is_cloud_active:
return "already_active", None, None
+ if (
+ task.origin_product in {Task.OriginProduct.SIGNAL_REPORT, Task.OriginProduct.SIGNALS_SCOUT}
+ and (run.state or {}).get("billing_product") != "posthog_ai"
+ ):
+ raise ValidationError("Start a new run to continue automated report or scout work.")
+
if get_pr_authorship_mode(run.task, run.state) == PrAuthorshipMode.USER:
pr_authorship_mode, validation_error = _resolve_cloud_pr_authorship_mode(
run.task,
@@ -8138,23 +8131,17 @@ def create_task(
# The write serializer binds the report as `signal_report` (a PK field); direct callers may
# pass `signal_report_id`. Either way this is the manual "start work from a report" path.
- # Gated regardless of the relationship label: the label is client-selected and manually
- # created tasks run PR-capable by default, so a "discussion" label must not dodge the limit.
report_ref = validated_data.get("signal_report") or validated_data.get("signal_report_id")
signal_report_id = (
str(getattr(report_ref, "id", report_ref))
if report_ref and validated_data.get("origin_product") == Task.OriginProduct.SIGNAL_REPORT
else None
)
- if signal_report_id:
- enforce_self_driving_pr_quota(team, report_id=signal_report_id)
- # Create PR and the discussions that push to the report's PR are held back on a trial. Other
- # discussions keep working, and one can still open a PR that billing never counts. That is
- # an accepted risk of a sales trial.
- if signal_report_task_relationship in (None, "implementation") or is_pull_request_run_request(
- validated_data.get("description")
- ):
- enforce_self_driving_free_trial(team, report_id=signal_report_id)
+ if signal_report_id and (
+ signal_report_task_relationship in (None, "implementation")
+ or is_pull_request_run_request(validated_data.get("description"))
+ ):
+ enforce_self_driving_free_trial(team, report_id=signal_report_id)
if refuse_without_repository and signal_report is not None:
_capture_report_task_refused_without_repository(team=team, report_id=str(signal_report.id))
raise ValidationError({"repository": REPORT_IMPLEMENTATION_REPOSITORY_UNRESOLVED})
@@ -8687,6 +8674,7 @@ def _find_idling_warm_run(
task__deleted=False,
task__github_integration_id=github_integration_id,
state__await_user_message=True,
+ **({"state__billing_product": "posthog_ai"} if origin_product == Task.OriginProduct.SIGNAL_REPORT else {}),
branch=branch or None,
**repository_filter,
**({"task__signal_report_id": signal_report_id} if signal_report_id else {}),
@@ -8761,6 +8749,11 @@ def _idling_warm_run_for_task(task: Task) -> TaskRun | None:
run = task.latest_run
if run is None or run.is_terminal or (run.state or {}).get("cancel_requested_at"):
return None
+ if (
+ task.origin_product == Task.OriginProduct.SIGNAL_REPORT
+ and (run.state or {}).get("billing_product") != "posthog_ai"
+ ):
+ return None
if not (run.state or {}).get("await_user_message"):
return None
return run
@@ -8986,7 +8979,6 @@ def _reserve_report_warm_activation(
)
report_id = str(task.signal_report_id)
- enforce_self_driving_pr_quota(team, report_id=report_id)
if is_pull_request_run_request(description):
enforce_self_driving_free_trial(team, report_id=report_id)
with transaction.atomic():
@@ -9398,14 +9390,9 @@ def run_task(
pipeline_rerun: bool = False,
free_trial_enabled: bool | None = None,
) -> contracts.TaskRunResult | None:
- """Create a run for a task and kick off its workflow, mirroring ``TaskViewSet.run``.
-
- Returns ``None`` if the task isn't found/visible (the view raises 404). Otherwise a
- ``TaskRunResult`` carrying the refreshed task detail DTO or a structured error. The usage
- gate (429) is applied by the view before calling this. A report implementation raises
- ``FreeTrialPullRequestRefused`` (402) while the team's org is on a self-driving free trial.
- ``pipeline_rerun`` is reserved for a server-requested Signals research rerun. It creates a
- fresh run and stamps the protected implementation stage from the verified report-task link.
+ """Return None for an inaccessible task, otherwise return its refreshed detail or a structured error.
+
+ The caller applies the usage gate. Only server-requested Signals research may set pipeline_rerun.
"""
from products.signals.backend.task_run_artefacts import ( # noqa: PLC0415 — cross-product read kept off the api import path
enforce_report_implementation_rerun_cap,
@@ -9447,11 +9434,7 @@ def run_task(
)
is_implementation = False
if report_id_for_slot_check is not None:
- # Free trial gate: the create-time gate refuses a new implementation, but a task created
- # before sales turned the flag on can still be started or retried from here, and its pull
- # request bills the trial org. Only the implementation relationship opens one, so a
- # discussion keeps running. Outside the transaction below, because the flag read does
- # network I/O and must not hold the report row lock.
+ # Resolve the trial flag outside the transaction: network I/O must not hold the report lock.
is_implementation = is_report_implementation_task(
team_id=team_id, report_id=report_id_for_slot_check, task_id=str(task.id)
)
@@ -9739,7 +9722,7 @@ def run_task(
prev_stack_base_branch = (previous_run.state or {}).get("stack_base_branch")
if prev_stack_base_branch:
extra_state["stack_base_branch"] = prev_stack_base_branch
- if pipeline_rerun and task.internal and is_implementation:
+ if pipeline_rerun and is_implementation:
extra_state["ai_stage"] = "implementation"
# A read-only GitHub grant describes how the task was created, not one run — without the
diff --git a/products/tasks/backend/facade/billing.py b/products/tasks/backend/facade/billing.py
index cbbc8ac8824b..f0c94d6fc301 100644
--- a/products/tasks/backend/facade/billing.py
+++ b/products/tasks/backend/facade/billing.py
@@ -10,6 +10,8 @@ from products.tasks.backend.logic.services.gateway_usage import (
get_task_run_cost,
get_task_run_token_cost_microusd,
get_task_run_token_usage,
+ get_task_runs_spend,
+ locked_task_run_spend,
process_pending_gateway_usage,
)
from products.tasks.backend.logic.services.sandbox_pricing import ComputeRateCardConfigurationError
@@ -48,5 +50,7 @@ __all__ = [
"get_task_cost",
"get_task_run_token_cost_microusd",
"get_task_run_token_usage",
+ "get_task_runs_spend",
+ "locked_task_run_spend",
"process_pending_gateway_usage",
]
diff --git a/products/tasks/backend/facade/contracts.py b/products/tasks/backend/facade/contracts.py
index 032ff7c4138e..cf6e874a1b34 100644
--- a/products/tasks/backend/facade/contracts.py
+++ b/products/tasks/backend/facade/contracts.py
@@ -138,6 +138,17 @@ class TaskRunTokenUsage:
complete: bool
+@dataclass(frozen=True, kw_only=True)
+class TaskRunSpend:
+ created_at: datetime
+ run_id: UUID
+ task_id: UUID
+ cost_microusd: int | None
+ complete: bool
+ is_terminal: bool
+ stage: str
+
+
@dataclass(frozen=True)
class TaskRunCost:
"""Provider costs in integer USD cents, or None while a source is unavailable or incomplete."""
diff --git a/products/tasks/backend/logic/repo_selection/agent.py b/products/tasks/backend/logic/repo_selection/agent.py
index c7998865625a..fac52e05ffb3 100644
--- a/products/tasks/backend/logic/repo_selection/agent.py
+++ b/products/tasks/backend/logic/repo_selection/agent.py
@@ -627,6 +627,7 @@ async def select_repository(
candidate_repos: list[str] | None = None,
step_name: str = "repo_selection",
signal_report_id: str | None = None,
+ spend_signal_id: str | None = None,
sandbox_environment_id: str | None = None,
verbose: bool = False,
output_fn: OutputFn = None,
@@ -766,6 +767,7 @@ async def select_repository(
output_fn=output_fn,
origin_product=origin_product,
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
ai_stage="repo_selection",
internal=True,
# An unreadable end turn used to fail the whole selection, and the caller then dropped the
diff --git a/products/tasks/backend/logic/services/custom_prompt_internals.py b/products/tasks/backend/logic/services/custom_prompt_internals.py
index 092e2efad760..7d7abb0ccbc5 100644
--- a/products/tasks/backend/logic/services/custom_prompt_internals.py
+++ b/products/tasks/backend/logic/services/custom_prompt_internals.py
@@ -344,6 +344,7 @@ async def _create_task_and_trigger(
output_schema: dict[str, Any] | None = None,
analytics_query_context: list[dict[str, object]] | None = None,
files: Sequence[SessionFile] = (),
+ spend_signal_id: str | None = None,
) -> tuple[Task, TaskRun]:
title = f"[sandbox_prompt:{step_name}] {description[:80]}" if step_name else description[:100]
team = await sync_to_async(Team.objects.get)(id=context.team_id)
@@ -383,6 +384,7 @@ async def _create_task_and_trigger(
mode="background",
branch=branch,
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
ai_stage=ai_stage,
ai_agent_name=ai_agent_name,
posthog_mcp_scopes=posthog_mcp_scopes,
@@ -432,6 +434,7 @@ async def create_task_and_trigger(
mcp_gateway_server_ids: list[str] | None = None,
output_schema: dict[str, Any] | None = None,
analytics_query_context: list[dict[str, object]] | None = None,
+ spend_signal_id: str | None = None,
) -> AgentTaskRunDTO:
task, task_run = await _create_task_and_trigger(
description,
@@ -440,6 +443,7 @@ async def create_task_and_trigger(
step_name=step_name,
origin_product=origin_product,
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
ai_stage=ai_stage,
ai_agent_name=ai_agent_name,
internal=internal,
diff --git a/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py b/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py
index a5eba43be910..975009385b21 100644
--- a/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py
+++ b/products/tasks/backend/logic/services/custom_prompt_multi_turn_runner.py
@@ -90,6 +90,7 @@ class MultiTurnSession:
output_fn: OutputFn = None,
origin_product: Task.OriginProduct | None = None,
signal_report_id: str | None = None,
+ spend_signal_id: str | None = None,
ai_stage: str | None = None,
ai_agent_name: str | None = None,
internal: bool = False,
@@ -138,6 +139,7 @@ class MultiTurnSession:
output_fn=output_fn,
origin_product=origin_product,
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
ai_stage=ai_stage,
ai_agent_name=ai_agent_name,
internal=internal,
@@ -219,6 +221,7 @@ class MultiTurnSession:
output_fn: OutputFn = None,
origin_product: Task.OriginProduct | None = None,
signal_report_id: str | None = None,
+ spend_signal_id: str | None = None,
ai_stage: str | None = None,
ai_agent_name: str | None = None,
internal: bool = False,
@@ -267,6 +270,7 @@ class MultiTurnSession:
step_name=step_name,
origin_product=origin_product,
signal_report_id=signal_report_id,
+ spend_signal_id=spend_signal_id,
ai_stage=ai_stage,
ai_agent_name=ai_agent_name,
internal=internal,
diff --git a/products/tasks/backend/logic/services/gateway_usage.py b/products/tasks/backend/logic/services/gateway_usage.py
index 70d40434930e..9e8bf9c46b61 100644
--- a/products/tasks/backend/logic/services/gateway_usage.py
+++ b/products/tasks/backend/logic/services/gateway_usage.py
@@ -1,7 +1,8 @@
import re
import time
import asyncio
-from collections.abc import Iterable
+from collections.abc import Iterable, Iterator
+from contextlib import contextmanager
from datetime import timedelta
from decimal import ROUND_HALF_EVEN, Decimal
from functools import lru_cache
@@ -18,7 +19,7 @@ from asgiref.sync import async_to_sync
from posthog.dataclasses import frozen
from posthog.llm.gateway_usage import GatewayRequestCost, fetch_gateway_cost
-from products.tasks.backend.facade.contracts import TaskRunCost, TaskRunTokenUsage
+from products.tasks.backend.facade.contracts import TaskRunCost, TaskRunSpend, TaskRunTokenUsage
from products.tasks.backend.facade.task_run_signals import task_run_cost_updated
from products.tasks.backend.logic.services.sandbox_pricing import COMPUTE_RATE_CARDS, calculate_sandbox_compute_cost
from products.tasks.backend.models import SandboxSession, TaskRun
@@ -61,8 +62,16 @@ def gateway_usage_enabled(run: TaskRun) -> bool:
def _save_accounting_state(run: TaskRun) -> None:
# Accounting can finish after completion; it must not emit completion signals again.
# It also leaves `updated_at` alone, because recency gauges and stale-run sweeps read it.
+ previous = TaskRun.objects.select_related("task").get(id=run.id, team_id=run.team_id)
+ delta = (_token_usage(run).cost_microusd or 0) - (_token_usage(previous).cost_microusd or 0)
TaskRun.objects.filter(id=run.id, team_id=run.team_id).update(state=run.state)
- task_run_cost_updated.send(sender=TaskRun, run_id=run.id, team_id=run.team_id)
+ task_run_cost_updated.send(
+ sender=TaskRun,
+ run_id=run.id,
+ team_id=run.team_id,
+ cost_delta_microusd=delta,
+ origin_product=previous.task.origin_product,
+ )
def record_gateway_routing(*, run_id: UUID | str, team_id: int, uses_gateway: bool) -> None:
@@ -227,6 +236,29 @@ def get_task_run_token_usage(*, run_id: UUID, team_id: int) -> TaskRunTokenUsage
return _token_usage(TaskRun.objects.get(id=run_id, team_id=team_id))
+def _run_spend(run: TaskRun) -> TaskRunSpend:
+ usage = _token_usage(run)
+ return TaskRunSpend(
+ created_at=run.created_at,
+ run_id=run.id,
+ task_id=run.task_id,
+ cost_microusd=usage.cost_microusd,
+ complete=usage.complete,
+ is_terminal=run.is_terminal,
+ stage=(run.state or {}).get("ai_stage") or "task",
+ )
+
+
+@contextmanager
+def locked_task_run_spend(*, team_id: int, run_id: UUID) -> Iterator[TaskRunSpend]:
+ with transaction.atomic():
+ yield _run_spend(_locked_run(run_id, team_id))
+
+
+def get_task_runs_spend(*, team_id: int, run_ids: Iterable[UUID]) -> list[TaskRunSpend]:
+ return [_run_spend(run) for run in TaskRun.objects.filter(team_id=team_id, id__in=run_ids)]
+
+
def _token_cost_microusd(run: TaskRun) -> int | None:
usage = _token_usage(run)
return usage.cost_microusd if usage.complete else None
diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py
index 3d9071bfb7a8..6ef5a40232b9 100644
--- a/products/tasks/backend/models.py
+++ b/products/tasks/backend/models.py
@@ -856,6 +856,8 @@ class Task(Taggable, DeletedMetaFields, models.Model):
scheduled_at: datetime | None = None,
startup_requested_at: datetime | None = None,
subscription_owner_id: int | None = None,
+ spend_signal_id: str | None = None,
+ implementation_spend_kind: str | None = None,
) -> "TaskRun":
startup_requested_at = startup_requested_at or django_timezone.now()
if scheduled_at is not None and django_timezone.is_naive(scheduled_at):
@@ -953,6 +955,30 @@ class Task(Taggable, DeletedMetaFields, models.Model):
if task.origin_product != Task.OriginProduct.SIGNAL_REPORT or task.signal_report_id:
state["ai_stage"] = interactive_stage
+ # A manual resume can share an internal Task with an automated run. Freeze billing
+ # on this run so historical PR charges and pipeline attribution do not change.
+ if (
+ task.origin_product == Task.OriginProduct.SIGNAL_REPORT
+ and state.get("ai_stage")
+ not in {
+ "research",
+ "implementation",
+ "repo_selection",
+ "custom_agent",
+ }
+ ) or (task.origin_product == Task.OriginProduct.SIGNALS_SCOUT and not state.get("ai_stage")):
+ state["billing_product"] = "posthog_ai"
+
+ if (
+ environment == TaskRun.Environment.CLOUD
+ and task.origin_product in (Task.OriginProduct.SIGNAL_REPORT, Task.OriginProduct.SIGNALS_SCOUT)
+ and state.get("billing_product") != "posthog_ai"
+ ):
+ # A new automated run has no usage yet, including when provisioning fails.
+ # Gateway fallback marks coverage incomplete before any agent can start.
+ state.setdefault("unprocessed_request_ids", [])
+ state.setdefault("token_cost", {})
+
resume_from_run_id = (extra_state or {}).get("resume_from_run_id")
if resume_from_run_id is not None:
resume_source = TaskRun.objects.filter(id=resume_from_run_id, task_id=task.id).only("state").first()
@@ -990,6 +1016,20 @@ class Task(Taggable, DeletedMetaFields, models.Model):
if effective_environment == TaskRun.Environment.CLOUD and scheduled_at is None:
task.observe_run_started_beside_live_runs(task_run)
+ if task.origin_product == Task.OriginProduct.SIGNAL_REPORT and task.signal_report_id:
+ from products.signals.backend.facade.spend import ( # noqa: PLC0415 — avoids the task-model import cycle
+ register_report_task_spend,
+ )
+
+ register_report_task_spend(
+ team_id=task.team_id,
+ run_id=str(task_run.id),
+ report_id=str(task.signal_report_id),
+ stage=state.get("ai_stage"),
+ signal_id=spend_signal_id,
+ implementation_kind=implementation_spend_kind,
+ )
+
def emit_created_events() -> None:
task_run.publish_stream_state_event()
observe_task_run_created(task_run)
@@ -1569,6 +1609,8 @@ class Task(Taggable, DeletedMetaFields, models.Model):
posthog_mcp_scopes: PosthogMcpScopes = "full",
branch: str | None = None,
signal_report_id: str | None = None,
+ spend_signal_id: str | None = None,
+ implementation_spend_kind: str | None = None,
hog_flow_id: uuid.UUID | None = None,
origin_key: str | None = None,
extra_run_state: dict[str, Any] | None = None,
@@ -1699,6 +1741,8 @@ class Task(Taggable, DeletedMetaFields, models.Model):
acting_user_id=user_id,
scheduled_at=scheduled_at,
startup_requested_at=startup_requested_at,
+ spend_signal_id=spend_signal_id,
+ implementation_spend_kind=implementation_spend_kind,
)
if before_task_dispatch is not None:
initial_state = before_task_dispatch(task_run.id)
diff --git a/products/tasks/backend/presentation/views/api.py b/products/tasks/backend/presentation/views/api.py
index 5a50e9266b72..4a00ab238f37 100644
--- a/products/tasks/backend/presentation/views/api.py
+++ b/products/tasks/backend/presentation/views/api.py
@@ -60,7 +60,7 @@ from posthog.rate_limit import TaskRunChartRenderThrottle
from posthog.renderers import ServerSentEventRenderer
from posthog.schema_migrations.upgrade import upgrade
from posthog.security.outbound_proxy import internal_requests_session
-from posthog.temporal.oauth import TASK_AGENT_OAUTH_APP_CLIENT_IDS
+from posthog.temporal.oauth import SANDBOX_OAUTH_APP_CLIENT_IDS
from posthog.token_bucket import (
BucketDecision,
BucketUnavailable,
@@ -550,7 +550,7 @@ class TaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
if not isinstance(authenticator, OAuthAccessTokenAuthentication):
raise PermissionDenied("Task comments are available only to the current task agent.")
application = authenticator.access_token.application
- if application is None or application.client_id not in TASK_AGENT_OAUTH_APP_CLIENT_IDS:
+ if application is None or application.client_id not in SANDBOX_OAUTH_APP_CLIENT_IDS:
raise PermissionDenied("Task comments are available only to the current task agent.")
try:
parsed_task_id = UUID(task_id)
diff --git a/products/tasks/backend/temporal/oauth.py b/products/tasks/backend/temporal/oauth.py
index c79453aaaa69..3d3b067d907e 100644
--- a/products/tasks/backend/temporal/oauth.py
+++ b/products/tasks/backend/temporal/oauth.py
@@ -76,10 +76,15 @@ INTERACTIVE_SIGNALS_ORIGIN_PRODUCTS = frozenset(
)
-def _oauth_application_for_task(task: Task) -> SandboxOAuthApplication:
+def _oauth_application_for_task(task: Task, state: dict[str, Any] | None) -> SandboxOAuthApplication:
# A metrics import bills the `posthog_ai` gateway product, which accepts only this application.
if task.origin_product in (Task.OriginProduct.POSTHOG_AI, Task.OriginProduct.METRICS_IMPORT):
return "posthog_ai"
+ if (
+ task.origin_product in (Task.OriginProduct.SIGNAL_REPORT, Task.OriginProduct.SIGNALS_SCOUT)
+ and (state or {}).get("billing_product") == "posthog_ai"
+ ):
+ return "posthog_ai"
if task.origin_product in SIGNALS_ORIGIN_PRODUCTS:
return "signals"
return "array"
@@ -203,7 +208,7 @@ def create_oauth_access_token(
token_options: dict[str, Any] = {
"scopes": scopes,
- "application": _oauth_application_for_task(task),
+ "application": _oauth_application_for_task(task, run_state),
"sandbox_task_id": task.id,
}
if task.origin_product in {
@@ -219,7 +224,7 @@ def create_oauth_access_token(
# task needs a member-capable token, or every member-proxy call it was
# just granted would 403.
token_options["include_mcp_builtin_agent_scope"] = True
- if is_interactive_signals_run(task, run_state):
+ if token_options["application"] == "signals" and is_interactive_signals_run(task, run_state):
token_options["include_interactive_run_scope"] = True
if task.origin_product == Task.OriginProduct.SLACK:
token_options["include_slack_run_scope"] = True
diff --git a/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py b/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py
index 80880155ae30..c2e24dbfa1f8 100644
--- a/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py
+++ b/products/tasks/backend/temporal/process_task/activities/enforce_self_driving_quota.py
@@ -72,7 +72,10 @@ def enforce_self_driving_run_quota(input: EnforceSelfDrivingRunQuotaInput) -> st
if run is None or run.is_terminal:
return SELF_DRIVING_QUOTA_STOP_CHECKING
task = run.task
- if task.origin_product != Task.OriginProduct.SIGNAL_REPORT:
+ if (
+ task.origin_product != Task.OriginProduct.SIGNAL_REPORT
+ or (run.state or {}).get("billing_product") == "posthog_ai"
+ ):
return SELF_DRIVING_QUOTA_STOP_CHECKING
if _has_billable_pr_url(run.output):
return SELF_DRIVING_QUOTA_STOP_CHECKING
diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_enforce_self_driving_quota.py b/products/tasks/backend/temporal/process_task/activities/tests/test_enforce_self_driving_quota.py
index e5036123fc8f..f8674c5901cb 100644
--- a/products/tasks/backend/temporal/process_task/activities/tests/test_enforce_self_driving_quota.py
+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_enforce_self_driving_quota.py
@@ -41,8 +41,14 @@ def _make_self_driving_run(test_task, test_task_run):
@pytest.mark.requires_secrets
class TestEnforceSelfDrivingRunQuotaActivity:
@pytest.mark.django_db(transaction=True)
- def test_non_self_driving_origin_stops_checking_without_gating(self, activity_environment, test_task_run):
- # The gate must never touch user or AI tasks: a wrong-origin run exits before any quota read.
+ @pytest.mark.parametrize("manual_report", [False, True])
+ def test_non_self_driving_origin_stops_checking_without_gating(
+ self, activity_environment, test_task, test_task_run, manual_report
+ ):
+ if manual_report:
+ _make_self_driving_run(test_task, test_task_run)
+ test_task_run.state = {"billing_product": "posthog_ai"}
+ test_task_run.save(update_fields=["state"])
with patch(f"{MODULE}.self_driving_quota_gate") as gate_mock:
assert _run_activity(activity_environment, test_task_run) == SELF_DRIVING_QUOTA_STOP_CHECKING
gate_mock.assert_not_called()
diff --git a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
index 95cc62162c64..de5866a96db6 100644
--- a/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
+++ b/products/tasks/backend/temporal/process_task/activities/update_task_run_status.py
@@ -190,9 +190,13 @@ def update_task_run_status(input: UpdateTaskRunStatusInput) -> None:
@activity.defn
@asyncify
def complete_signal_spend(run_id: str) -> None:
- run = TaskRun.objects.only("team_id", "status").get(id=run_id)
+ run = TaskRun.objects.select_related("task").get(id=run_id)
if run.status in _TERMINAL_STATUSES:
- complete_task_spend(team_id=run.team_id, run_id=run_id)
+ complete_task_spend(
+ team_id=run.team_id,
+ run_id=run_id,
+ report_id=str(run.task.signal_report_id) if run.task.signal_report_id else None,
+ )
def _capture_posthog_ai_chat_analytics(
diff --git a/products/tasks/backend/temporal/process_task/ai_gateway_token.py b/products/tasks/backend/temporal/process_task/ai_gateway_token.py
index a9fc08b8cd70..bb54abe734c0 100644
--- a/products/tasks/backend/temporal/process_task/ai_gateway_token.py
+++ b/products/tasks/backend/temporal/process_task/ai_gateway_token.py
@@ -129,8 +129,12 @@ _MINT_ATTEMPTS = 2
_MINT_TIMEOUT_SECONDS = 3
-def resolve_sandbox_ai_product(origin_product: str | None, ai_stage: str | None, *, internal: bool = False) -> str:
+def resolve_sandbox_ai_product(
+ origin_product: str | None, ai_stage: str | None, *, internal: bool = False, billing_product: str | None = None
+) -> str:
"""The `ai_product` the agent server will resolve for this run."""
+ if origin_product in ("signal_report", "signals_scout") and billing_product == "posthog_ai":
+ return "posthog_ai"
gateway_product = _ORIGIN_TO_GATEWAY_PRODUCT.get(origin_product or "")
# Stored rows may carry a caller-set review_hog origin predating its
# reservation; only the server-stamped `internal` flag admits the mintable product.
@@ -250,7 +254,9 @@ def mint_refusal(
refusal = _posthog_code_refusal(team_id, model)
if refusal:
return refusal
- elif runtime == "pi" and not (ai_product == "signals_scout" and origin_product == "signals_scout" and internal):
+ elif runtime == "pi" and not (
+ ai_product in {"signals_scout", "posthog_ai"} and origin_product == "signals_scout" and internal
+ ):
return "pi_runtime"
# The gateway denies an off-pin model with no fallback.
if not model_allowed_by_product_pin(ai_product, model):
diff --git a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
index bf9604a9bbc1..dc9175980864 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_ai_gateway_token.py
@@ -132,13 +132,21 @@ class TestSharedRoutingContract:
)
def test_resolve_matches_contract(self, case):
assert (
- resolve_sandbox_ai_product(case["origin_product"], case["ai_stage"], internal=case["internal"])
+ resolve_sandbox_ai_product(
+ case["origin_product"],
+ case["ai_stage"],
+ internal=case["internal"],
+ )
== case["expected"]
)
@pytest.mark.parametrize("case", _CASES["routed"], ids=lambda c: f"{c['origin_product']}/{c['ai_stage']}")
def test_routed_matches_contract(self, case):
- ai_product = resolve_sandbox_ai_product(case["origin_product"], case["ai_stage"], internal=case["internal"])
+ ai_product = resolve_sandbox_ai_product(
+ case["origin_product"],
+ case["ai_stage"],
+ internal=case["internal"],
+ )
assert sandbox_product_routed(ai_product, case["ai_stage"], case["allowlist"]) == case["expected"]
@@ -455,21 +463,44 @@ class TestAiGatewayEnvVars:
assert "AI_GATEWAY_AI_STAGE" not in env
mint.assert_not_called()
- def test_workflow_run_gets_a_pinned_token_when_routed(self, mint_settings):
- mint_settings.SANDBOX_AI_GATEWAY_PRODUCTS = "workflows"
+ @pytest.mark.parametrize(
+ "origin, state, product",
+ [
+ ("workflow", None, "workflows"),
+ ("signal_report", {"ai_stage": "inbox", "billing_product": "posthog_ai"}, "posthog_ai"),
+ ("signal_report", {"billing_product": "posthog_ai"}, "posthog_ai"),
+ ("signals_scout", {"billing_product": "posthog_ai"}, "posthog_ai"),
+ ],
+ )
+ @pytest.mark.parametrize("over_quota", [False, True])
+ def test_user_billed_run_uses_its_credit_bucket(self, mint_settings, origin, state, product, over_quota):
+ mint_settings.SANDBOX_AI_GATEWAY_PRODUCTS = product
+ runtime = "pi" if origin == "signals_scout" else None
with (
patch(
- "products.tasks.backend.temporal.process_task.utils.mint_scoped_token",
- return_value="phe_abc",
+ "products.tasks.backend.temporal.process_task.utils.mint_scoped_token", return_value="phe_abc"
) as mint,
patch(
"products.tasks.backend.temporal.process_task.ai_gateway_token._team_credit_refusal",
- return_value=None,
- ),
+ return_value="ai_credits_exhausted" if over_quota else None,
+ ) as quota,
):
- env = ai_gateway_env_vars(team_id=123, origin_product="workflow")
- assert env["AI_GATEWAY_TOKEN"] == "phe_abc"
- mint.assert_called_once_with(ai_product="workflows", team_id=123, user=None)
+ if product == "workflows" and over_quota:
+ with pytest.raises(GatewayTokenUnavailableError):
+ ai_gateway_env_vars(team_id=123, origin_product=origin, state=state, runtime=runtime, internal=True)
+ mint.assert_not_called()
+ else:
+ env = ai_gateway_env_vars(
+ team_id=123, origin_product=origin, state=state, runtime=runtime, internal=True
+ )
+ if over_quota:
+ assert "AI_GATEWAY_TOKEN" not in env
+ mint.assert_not_called()
+ else:
+ assert env["AI_GATEWAY_TOKEN"] == "phe_abc"
+ assert env["AI_GATEWAY_PRODUCT"] == product
+ mint.assert_called_once_with(ai_product=product, team_id=123, user=None)
+ quota.assert_called_once_with(123, "ai_credits")
# The agent trusts these as the worker's word, so the API must refuse a run-supplied value.
def test_reserved_keys_cover_the_pinned_product_env(self):
diff --git a/products/tasks/backend/temporal/process_task/tests/test_workflow.py b/products/tasks/backend/temporal/process_task/tests/test_workflow.py
index da01958c6d13..675d1147f9e4 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_workflow.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_workflow.py
@@ -3,6 +3,7 @@ import json
import uuid
import random
import asyncio
+import logging
import dataclasses
from collections.abc import Awaitable, Callable
from concurrent.futures import ThreadPoolExecutor
@@ -1658,13 +1659,11 @@ async def test_sandbox_started_carries_run_attribution(origin_product, team_id,
@pytest.mark.django_db
class TestProcessTaskWorkflowUnit:
- def test_quota_recheck_not_scheduled_for_non_pr_runs(self):
- # Research / repo-selection sessions run as SIGNAL_REPORT-origin tasks with
- # create_pr=False; scheduling the recheck for them would let the quota gate cancel
- # in-flight research.
+ @pytest.mark.parametrize("create_pr, state", [(False, {}), (True, {"billing_product": "posthog_ai"})])
+ def test_quota_recheck_not_scheduled_outside_automatic_implementation(self, create_pr, state):
wf = ProcessTaskWorkflow()
wf._context = dataclasses.replace(
- _build_context(github_integration_id=None), origin_product="signal_report", create_pr=False
+ _build_context(github_integration_id=None), origin_product="signal_report", create_pr=create_pr, state=state
)
assert wf._self_driving_quota_recheck_scheduled() is False
@@ -2591,40 +2590,57 @@ class TestProcessTaskWorkflowUnit:
assert forward_mock.called is forwarded
@pytest.mark.parametrize(
- "origin_product, pr_progress_emitted, ci_repetitions, end_of_turn_received, expected_status",
+ "origin_product, billing_product, pr_progress_emitted, ci_repetitions, end_of_turn_received, expected_status, spend_fails, cleanup_fails",
[
- (None, False, 1, None, "completed"),
- (None, False, 1, True, "completed"),
+ (None, None, False, 1, None, "completed", False, False),
+ (None, None, False, 1, True, "completed", False, False),
# The agent was mid-turn when the sandbox vanished, so the workflow step waiting on
# this run has no work to continue from.
- ("workflow", False, 1, False, "failed"),
+ ("workflow", None, False, 1, False, "failed", False, False),
# Every other origin keeps completing: the run is the snapshot its reader or its resume
# flow picks up, and a red run in the task list would report breakage that never was.
- (None, False, 1, False, "completed"),
- ("slack", False, 1, False, "completed"),
- ("user_created", False, 1, None, "completed"),
- ("signal_report", False, 1, None, "completed"),
- ("signals_scout", False, 1, None, "completed"),
+ (None, None, False, 1, False, "completed", False, False),
+ ("slack", None, False, 1, False, "completed", False, False),
+ ("user_created", None, False, 1, None, "completed", False, False),
+ ("signal_report", None, False, 1, None, "completed", False, False),
+ ("signal_report", "posthog_ai", False, 1, None, "completed", False, False),
+ ("signals_scout", None, False, 1, None, "completed", False, False),
+ ("signal_report", None, False, 1, None, "completed", True, False),
+ ("signals_scout", None, False, 1, None, "completed", True, False),
+ ("signals_scout", None, False, 1, None, "completed", False, True),
# Onboarding runs are one-shot, so a vanished sandbox is a failed setup rather than a
# resumable snapshot.
- ("onboarding", False, 1, None, "failed"),
+ ("onboarding", None, False, 1, None, "failed", False, False),
# Unless the PR is already open: the wizard reads the terminal status, so a downgrade
# would report a failed install over a PR the user can merge.
- ("onboarding", True, 1, None, "completed"),
+ ("onboarding", None, True, 1, None, "completed", False, False),
# No follow-up round ever ran, so the empty PR latch is unobserved rather than evidence
# of no PR. Downgrading here would fail a run whose PR the loop never got to look at.
- ("onboarding", False, 0, None, "completed"),
+ ("onboarding", None, False, 0, None, "completed", False, False),
],
)
async def test_run_completes_when_credential_refresh_detects_sandbox_gone(
- self, monkeypatch, origin_product, pr_progress_emitted, ci_repetitions, end_of_turn_received, expected_status
+ self,
+ monkeypatch,
+ origin_product,
+ billing_product,
+ pr_progress_emitted,
+ ci_repetitions,
+ end_of_turn_received,
+ expected_status,
+ spend_fails,
+ cleanup_fails,
+ caplog,
):
workflow = ProcessTaskWorkflow()
workflow._pr_progress_emitted = pr_progress_emitted
workflow._ci_repetitions = ci_repetitions
- context = _build_context(github_integration_id=123, origin_product=origin_product)
+ context = dataclasses.replace(
+ _build_context(github_integration_id=123, origin_product=origin_product),
+ state={"billing_product": billing_product} if billing_product else {},
+ )
update_task_run_status_mock = AsyncMock()
- cleanup_sandbox_mock = AsyncMock()
+ cleanup_sandbox_mock = AsyncMock(side_effect=RuntimeError("cleanup failed") if cleanup_fails else None)
monkeypatch.setattr(workflow, "_get_task_processing_context", AsyncMock(return_value=context))
monkeypatch.setattr(workflow, "_update_task_run_status", update_task_run_status_mock)
@@ -2677,18 +2693,38 @@ class TestProcessTaskWorkflowUnit:
await asyncio.sleep(0)
monkeypatch.setattr(process_task_workflow_module.workflow, "wait_condition", fake_wait_condition)
+ monkeypatch.setattr(process_task_workflow_module.workflow, "logger", logging.getLogger("temporalio.workflow"))
async def finish_spend(activity_fn, run_id, **kwargs):
assert activity_fn is process_task_workflow_module.complete_signal_spend
assert run_id == "run-id"
cleanup_sandbox_mock.assert_awaited_once()
+ if spend_fails:
+ raise ActivityError(
+ "Task usage is not settled",
+ scheduled_event_id=1,
+ started_event_id=2,
+ identity="test",
+ activity_type="complete_signal_spend",
+ activity_id="billing",
+ retry_state=RetryState.TIMEOUT,
+ )
finish_spend_mock = AsyncMock(side_effect=finish_spend)
monkeypatch.setattr(process_task_workflow_module.workflow, "execute_activity", finish_spend_mock)
+ if cleanup_fails:
+ with pytest.raises(RuntimeError, match="cleanup failed"):
+ await workflow.run(ProcessTaskInput(run_id="run-id"))
+ finish_spend_mock.assert_awaited_once()
+ return
result = await workflow.run(ProcessTaskInput(run_id="run-id"))
- assert finish_spend_mock.await_count == (1 if origin_product in {"signal_report", "signals_scout"} else 0)
+ assert finish_spend_mock.await_count == (
+ 1 if origin_product in {"signal_report", "signals_scout"} and billing_product != "posthog_ai" else 0
+ )
assert result.success is True
+ if spend_fails:
+ assert "signals.spend.accounting_failed" in caplog.text
assert workflow._completion_status == expected_status
update_task_run_status_mock.assert_any_await(
expected_status,
@@ -2888,10 +2924,13 @@ class TestProcessTaskWorkflowUnit:
# loop-free unit test doesn't have.
monkeypatch.setattr(process_task_workflow_module.workflow, "logger", Mock())
+ finish_spend = AsyncMock()
+ monkeypatch.setattr(process_task_workflow_module.workflow, "execute_activity", finish_spend)
result = await workflow.run(ProcessTaskInput(run_id="run-id"))
assert result.success is True
update_task_run_status_mock.assert_any_await(expected_status, **expected_kwargs)
+ assert finish_spend.await_count == (1 if origin_product == "signal_report" else 0)
@pytest.mark.parametrize(
"create_pr, pr_progress_emitted, ci_repetitions, expected",
diff --git a/products/tasks/backend/temporal/process_task/utils.py b/products/tasks/backend/temporal/process_task/utils.py
index 67b22e577c25..ed70fb0b3af8 100644
--- a/products/tasks/backend/temporal/process_task/utils.py
+++ b/products/tasks/backend/temporal/process_task/utils.py
@@ -1638,16 +1638,26 @@ def run_gateway_env_vars(ctx: TaskProcessingContext, task: Task) -> dict[str, st
"AI_GATEWAY_PRODUCT": "signals_scout",
"AI_GATEWAY_AI_STAGE": (ctx.state or {}).get("ai_stage") or "scout",
}
+ requires_pi_scout = (
+ ctx.task_runtime == "pi"
+ and ctx.origin_product == "signals_scout"
+ and (ctx.state or {}).get("billing_product") != "posthog_ai"
+ )
if "own-subscription" in (ctx.claude_model_access, ctx.codex_model_access):
- if ctx.task_runtime == "pi" and ctx.origin_product == "signals_scout":
+ if requires_pi_scout:
raise GatewayNotConfiguredError("Pi scouts require the AI gateway")
record_gateway_routing(run_id=ctx.run_id, team_id=ctx.team_id, uses_gateway=False)
return {}
ai_stage = (ctx.state or {}).get("ai_stage")
- requires_go = resolve_sandbox_ai_product(ctx.origin_product, ai_stage, internal=task.internal) == (
- POSTHOG_CODE_PRODUCT
+ requires_go = (
+ resolve_sandbox_ai_product(
+ ctx.origin_product,
+ ai_stage,
+ internal=task.internal,
+ billing_product=(ctx.state or {}).get("billing_product"),
+ )
+ == POSTHOG_CODE_PRODUCT
)
- requires_pi_scout = ctx.task_runtime == "pi" and ctx.origin_product == "signals_scout"
try:
env_vars = ai_gateway_env_vars(
team_id=ctx.team_id,
@@ -1694,7 +1704,7 @@ def run_gateway_env_vars(ctx: TaskProcessingContext, task: Task) -> dict[str, st
exc_info=True,
)
env_vars = {}
- if ctx.task_runtime == "pi" and ctx.origin_product == "signals_scout" and not env_vars.get("AI_GATEWAY_TOKEN"):
+ if requires_pi_scout and not env_vars.get("AI_GATEWAY_TOKEN"):
raise GatewayTokenUnavailableError(
"Pi scouts require a scoped AI gateway token", {"team_id": ctx.team_id, "run_id": ctx.run_id}
)
@@ -1780,10 +1790,16 @@ def ai_gateway_env_vars(
also raises, because the Python gateway refuses it too.
"""
ai_product = (
- resolve_sandbox_ai_product(origin_product, ai_stage, internal=internal) if team_id is not None else None
+ resolve_sandbox_ai_product(
+ origin_product, ai_stage, internal=internal, billing_product=(state or {}).get("billing_product")
+ )
+ if team_id is not None
+ else None
)
requires_go = ai_product == POSTHOG_CODE_PRODUCT
- requires_pi_scout = runtime == "pi" and origin_product == "signals_scout"
+ requires_pi_scout = (
+ runtime == "pi" and origin_product == "signals_scout" and (state or {}).get("billing_product") != "posthog_ai"
+ )
if requires_pi_scout and not (
ai_product == "signals_scout"
and settings.SANDBOX_AI_GATEWAY_URL
diff --git a/products/tasks/backend/temporal/process_task/workflow.py b/products/tasks/backend/temporal/process_task/workflow.py
index 064af72e4ee4..4b655181abf6 100644
--- a/products/tasks/backend/temporal/process_task/workflow.py
+++ b/products/tasks/backend/temporal/process_task/workflow.py
@@ -656,6 +656,7 @@ class ProcessTaskWorkflow(PostHogWorkflow):
self._self_driving_quota_checks_active
and self._context is not None
and self.context.origin_product == _ORIGIN_PRODUCT_SIGNAL_REPORT
+ and (self.context.state or {}).get("billing_product") != "posthog_ai"
# Research / repo-selection / custom-agent sessions run with create_pr=False and can
# never open the billable PR; rechecking them would let the quota gate cancel
# in-flight research.
@@ -1611,61 +1612,68 @@ class ProcessTaskWorkflow(PostHogWorkflow):
# Skip teardown on a continue_as_new hand-off (a `return` here would swallow the
# ContinueAsNewError); the loops are already stopped and the sandbox lives on.
if not continuing_as_new:
- if self._active_followup_task is not None:
- await self._cancel_relay(self._active_followup_task)
- self._active_followup_task = None
- if credential_refresh_task is not None:
- await self._cancel_relay(credential_refresh_task)
- if permission_response_task is not None:
- await self._cancel_relay(permission_response_task)
- if preview_task is not None:
- await self._cancel_relay(preview_task)
- if agent_shadow_task is not None:
- await self._cancel_relay(agent_shadow_task)
- await self._flush_boot_telemetry_tasks()
- await self._flush_pending_progress_activities()
- await self._close_dev_stack_preview_progress()
-
- cleanup_sandbox_id = sandbox_id or self._sandbox_id_for_cleanup
- if cleanup_sandbox_id:
+ try:
+ if self._active_followup_task is not None:
+ await self._cancel_relay(self._active_followup_task)
+ self._active_followup_task = None
+ if credential_refresh_task is not None:
+ await self._cancel_relay(credential_refresh_task)
+ if permission_response_task is not None:
+ await self._cancel_relay(permission_response_task)
+ if preview_task is not None:
+ await self._cancel_relay(preview_task)
+ if agent_shadow_task is not None:
+ await self._cancel_relay(agent_shadow_task)
+ await self._flush_boot_telemetry_tasks()
+ await self._flush_pending_progress_activities()
+ await self._close_dev_stack_preview_progress()
+
+ cleanup_sandbox_id = sandbox_id or self._sandbox_id_for_cleanup
+ if cleanup_sandbox_id:
+ if (
+ self._context
+ and self._context.mode == "interactive"
+ and self._context.use_modal_resume_snapshots
+ ):
+ await self._create_resume_snapshot(
+ cleanup_sandbox_id,
+ reason="teardown",
+ allow_pruning=True,
+ )
+
+ await self._read_sandbox_logs(cleanup_sandbox_id)
+ await self._cleanup_sandbox(
+ cleanup_sandbox_id,
+ complete_stream=True,
+ )
+ sandbox_cleaned = True
+ self._sandbox_id_for_cleanup = None
+ elif not run_stream_completed and (self._context is None or self._context.environment == "cloud"):
+ await self._complete_run_stream(run_id)
+
+ if sandbox_cleaned and self._slack_thread_context and self._context:
+ await self._post_slack_update(sandbox_cleaned=True)
+ finally:
if (
self._context
- and self._context.mode == "interactive"
- and self._context.use_modal_resume_snapshots
+ and self.context.origin_product in {_ORIGIN_PRODUCT_SIGNAL_REPORT, "signals_scout"}
+ and (self.context.state or {}).get("billing_product") != "posthog_ai"
+ and workflow.patched("signals-publish-spend-on-completion")
):
- await self._create_resume_snapshot(
- cleanup_sandbox_id,
- reason="teardown",
- allow_pruning=True,
- )
-
- await self._read_sandbox_logs(cleanup_sandbox_id)
- await self._cleanup_sandbox(
- cleanup_sandbox_id,
- complete_stream=True,
- )
- sandbox_cleaned = True
- self._sandbox_id_for_cleanup = None
- elif not run_stream_completed and (self._context is None or self._context.environment == "cloud"):
- await self._complete_run_stream(run_id)
-
- if sandbox_cleaned and self._slack_thread_context and self._context:
- await self._post_slack_update(sandbox_cleaned=True)
-
- if (
- self._context
- and self.context.origin_product in {_ORIGIN_PRODUCT_SIGNAL_REPORT, "signals_scout"}
- and workflow.patched("signals-publish-spend-on-completion")
- ):
- try:
- await workflow.execute_activity(
- complete_signal_spend,
- run_id,
- start_to_close_timeout=timedelta(minutes=20),
- retry_policy=RetryPolicy(maximum_attempts=1),
- )
- except Exception:
- workflow.logger.exception("Failed to finalize signal spend", extra={"run_id": run_id})
+ try:
+ await workflow.execute_activity(
+ complete_signal_spend,
+ run_id,
+ start_to_close_timeout=timedelta(minutes=15),
+ schedule_to_close_timeout=timedelta(minutes=15),
+ retry_policy=RetryPolicy(
+ initial_interval=timedelta(seconds=5), maximum_interval=timedelta(seconds=30)
+ ),
+ )
+ except temporalio.exceptions.ActivityError:
+ workflow.logger.exception(
+ "signals.spend.accounting_failed", extra={"run_id": run_id, "stage": "task_completion"}
+ )
async def _provision_and_start_agent(self, input: ProcessTaskInput, run_id: str) -> tuple[str, str, str | None]:
"""Initial-run setup: resolve agent-design, provision the sandbox, start the agent
diff --git a/products/tasks/backend/temporal/tests/test_oauth.py b/products/tasks/backend/temporal/tests/test_oauth.py
index e68a738dff59..5aca9d253ee5 100644
--- a/products/tasks/backend/temporal/tests/test_oauth.py
+++ b/products/tasks/backend/temporal/tests/test_oauth.py
@@ -60,27 +60,39 @@ def test_built_in_agent_origins_use_restricted_oauth_scope(
)
+ "origin, state",
+ [
+ (Task.OriginProduct.POSTHOG_AI, {}),
+ (Task.OriginProduct.SIGNAL_REPORT, {"billing_product": "posthog_ai"}),
+ (Task.OriginProduct.SIGNALS_SCOUT, {"billing_product": "posthog_ai"}),
+ ],
+)
@patch("products.tasks.backend.temporal.oauth.is_builtin_agent_enforcement_enabled", return_value=True)
@patch("products.tasks.backend.temporal.oauth._create_oauth_access_token_for_user", return_value="token")
def test_posthog_ai_task_keeps_member_token_and_posthog_ai_oauth_application(
mock_create: MagicMock,
mock_enforcement: MagicMock,
+ origin: Task.OriginProduct,
+ state: dict[str, str],
) -> None:
task = MagicMock(
id="task-id",
created_by=MagicMock(),
team_id=123,
- origin_product=Task.OriginProduct.POSTHOG_AI,
+ origin_product=origin,
)
- assert create_oauth_access_token(task) == "token"
+ assert create_oauth_access_token(task, run_state=state) == "token"
+ expected = {"include_mcp_builtin_agent_scope": True} if origin == Task.OriginProduct.SIGNALS_SCOUT else {}
mock_create.assert_called_once_with(
task.created_by,
123,
scopes="read_only",
application="posthog_ai",
sandbox_task_id=task.id,
+ **expected,
)
@@ -202,7 +214,7 @@ def test_signals_origins_mint_under_the_signals_app_and_mark_only_user_started_r
@pytest.mark.django_db
@patch("products.tasks.backend.temporal.oauth.is_builtin_agent_enforcement_enabled", return_value=False)
@patch("products.tasks.backend.temporal.oauth._create_oauth_access_token_for_user", return_value="token")
-def test_two_runs_on_one_auto_started_task_get_different_signals_budgets(
+def test_manual_resume_of_pipeline_task_uses_phai_oauth(
mock_create: MagicMock,
mock_enforcement: MagicMock,
) -> None:
@@ -236,7 +248,8 @@ def test_two_runs_on_one_auto_started_task_get_different_signals_budgets(
token.save()
mock_create.return_value = "second-token"
create_oauth_access_token_for_run(task, second.state, run_id=second.id)
- assert mock_create.call_args.kwargs["include_interactive_run_scope"] is True
+ assert mock_create.call_args.kwargs["application"] == "posthog_ai"
+ assert "include_interactive_run_scope" not in mock_create.call_args.kwargs
second.refresh_from_db()
assert second.state["sandbox_oauth_token_ids"] == [str(token.id)]
diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py
index d98e64510aa3..de584fc925bf 100644
--- a/products/tasks/backend/tests/test_api.py
+++ b/products/tasks/backend/tests/test_api.py
@@ -15239,21 +15239,22 @@ class TestTaskRunCommandAPI(BaseTaskAPITest):
task.refresh_from_db()
self.assertNotIn("mobile_push_user_ids", task.state or {})
+ @parameterized.expand([("ai_gateway_product", "slack_app"), ("billing_product", "posthog_ai")])
@patch("products.tasks.backend.models.TaskRun.publish_stream_state_event")
- def test_patch_cannot_forge_the_gateway_pin_stamp(self, _mock_publish):
+ def test_patch_cannot_forge_billing_or_gateway_product(self, key, original, _mock_publish):
task = self.create_task()
run = TaskRun.objects.create(
- task=task, team=self.team, status=TaskRun.Status.IN_PROGRESS, state={"ai_gateway_product": "slack_app"}
+ task=task, team=self.team, status=TaskRun.Status.IN_PROGRESS, state={key: original}
)
url = f"/api/projects/@current/tasks/{task.id}/runs/{run.id}/"
- response = self.client.patch(url, {"state": {}, "state_remove_keys": ["ai_gateway_product"]}, format="json")
+ response = self.client.patch(url, {"state": {}, "state_remove_keys": [key]}, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
- response = self.client.patch(url, {"state": {"ai_gateway_product": "review_hog"}}, format="json")
+ response = self.client.patch(url, {"state": {key: "review_hog"}}, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
run.refresh_from_db()
- assert run.state["ai_gateway_product"] == "slack_app"
+ assert run.state[key] == original
@override_settings(SANDBOX_JWT_PRIVATE_KEY=TEST_RSA_PRIVATE_KEY)
@patch("products.tasks.backend.presentation.views.api.http_requests.post")
diff --git a/products/tasks/backend/tests/test_facade.py b/products/tasks/backend/tests/test_facade.py
index ef42361f31b2..4a4a65ba0bbc 100644
--- a/products/tasks/backend/tests/test_facade.py
+++ b/products/tasks/backend/tests/test_facade.py
@@ -13,6 +13,7 @@ from django.test import SimpleTestCase, TestCase, TransactionTestCase, override_
from django.utils import timezone as django_timezone
from parameterized import parameterized
+from rest_framework.exceptions import ValidationError
from posthog.clickhouse.query_tagging import get_query_tags, reset_query_tags
from posthog.constants import AvailableFeature
@@ -451,6 +452,18 @@ class TestFacadeReadsAndMappers(TestCase):
run.refresh_from_db()
self.assertEqual(run.environment, TaskRun.Environment.CLOUD)
+ @parameterized.expand([(Task.OriginProduct.SIGNAL_REPORT,), (Task.OriginProduct.SIGNALS_SCOUT,)])
+ def test_resume_in_cloud_cannot_reopen_automated_spend(self, origin_product: str) -> None:
+ task = self._make_task(origin_product=origin_product)
+ run = TaskRun.objects.create(
+ task=task, team=self.team, environment=TaskRun.Environment.CLOUD, status=TaskRun.Status.COMPLETED
+ )
+ with self.assertRaisesMessage(ValidationError, "Start a new run"):
+ facade.resume_task_run_in_cloud(run.id, task.id, self.team.id, self.user.id)
+ run.refresh_from_db()
+ assert run.status == TaskRun.Status.COMPLETED
+ assert run.state == {}
+
def test_list_workflow_last_runs_uses_each_workflows_newest_listed_task(self):
now = django_timezone.now()
flow_with_runs, flow_without_run, flow_not_asked = uuid4(), uuid4(), uuid4()
@@ -1425,9 +1438,10 @@ class TestFacadeReadsAndMappers(TestCase):
@parameterized.expand(
[
- ("manual_implementation", False, "implementation", "implementation", None),
- ("pipeline_after_manual", True, "implementation", None, "implementation"),
- ("pipeline_discussion", True, "discussion", "implementation", None),
+ ("manual_implementation", False, "implementation", "implementation", None, True),
+ ("pipeline_after_manual", True, "implementation", None, "implementation", True),
+ ("pipeline_after_ui_task", True, "implementation", None, "implementation", False),
+ ("pipeline_discussion", True, "discussion", "implementation", None, True),
]
)
def test_run_task_resume_of_a_pipeline_task_stamps_only_requested_pipeline_runs(
@@ -1437,12 +1451,13 @@ class TestFacadeReadsAndMappers(TestCase):
relationship: str,
previous_stage: str | None,
expected_stage: str | None,
+ internal: bool,
):
- from products.signals.backend.models import SignalReport
+ from products.signals.backend.models import SignalReport, SignalSpend
from products.signals.backend.task_run_artefacts import record_report_task
report = SignalReport.objects.create(team=self.team)
- task = self._make_task(origin_product=Task.OriginProduct.SIGNAL_REPORT, signal_report=report, internal=True)
+ task = self._make_task(origin_product=Task.OriginProduct.SIGNAL_REPORT, signal_report=report, internal=internal)
record_report_task(
team_id=self.team.id, report_id=str(report.id), task_id=str(task.id), relationship=relationship
)
@@ -1466,6 +1481,17 @@ class TestFacadeReadsAndMappers(TestCase):
assert result is not None and result.error is None
new_run = task.runs.exclude(id=previous_run.id).get()
self.assertEqual(new_run.state.get("ai_stage"), expected_stage)
+ self.assertEqual(new_run.state.get("billing_product"), None if expected_stage else "posthog_ai")
+ spend = SignalSpend.objects.for_team(self.team.id).filter(task_run_id=new_run.id).first()
+ if expected_stage:
+ assert spend is not None
+ assert spend.kind == SignalSpend.Kind.REVIEWER_TRIGGERED_IMPLEMENTATION
+ assert spend.report_id == report.id
+ assert spend.signal_id is None
+ else:
+ assert spend is None
+ task.refresh_from_db()
+ self.assertEqual(task.origin_product, Task.OriginProduct.SIGNAL_REPORT)
@parameterized.expand(
[
@@ -2319,36 +2345,27 @@ class TestSelfDrivingQuotaFacadeGates(TestCase):
self.assertTrue(Task.objects.filter(id=created.task_id).exists())
@parameterized.expand([(None,), ("implementation",), ("discussion",)])
- def test_create_task_blocked_for_manual_report_creation_when_enforced(self, relationship):
- # The inbox "start work from report" path (write serializer binds `signal_report`).
- # Every relationship label is gated: the label is client-selected and manual tasks run
- # PR-capable by default, so a "discussion" label must not dodge the limit.
- from django.apps import apps
-
- from posthog.exceptions import QuotaLimitExceeded
-
- SignalReport = apps.get_model("signals", "SignalReport")
- report = SignalReport.objects.create(team=self.team, status="ready", title="t", summary="s")
- with (
- self._enforced_gate(),
- patch("products.signals.backend.quota.capture_signal_report_quota_paused") as capture_mock,
- self.assertRaises(QuotaLimitExceeded),
- ):
- facade.create_task(
+ def test_manual_report_creation_uses_phai_instead_of_self_driving_quota(self, relationship):
+ report = apps.get_model("signals", "SignalReport").objects.create(team=self.team, status="ready")
+ apps.get_model("signals", "SignalReportArtefact").objects.create(
+ team=self.team,
+ report=report,
+ type="repo_selection",
+ content='{"repository": "example/repository", "reason": "Matches the example application"}',
+ )
+ with self._enforced_gate():
+ dto = facade.create_task(
self.team.id,
self.user.id,
validated_data={
- "title": "Implementation: t",
+ "title": "Implementation",
"description": "d",
"origin_product": Task.OriginProduct.SIGNAL_REPORT,
"signal_report": report,
"signal_report_task_relationship": relationship,
},
)
- self.assertFalse(Task.objects.filter(team=self.team).exists())
- # Genuinely manual creations keep the `manual_create` stage, distinct from the facade
- # backstop's `task_create`.
- self.assertEqual(capture_mock.call_args.kwargs["stage"], "manual_create")
+ self.assertEqual(Task.objects.get(id=dto.id).signal_report_id, report.id)
class TestSelfDrivingQuotaRefreshDispatch(TestCase):
@@ -2682,8 +2699,6 @@ class TestSelfDrivingFreeTrialFacadeGates(TestCase):
@parameterized.expand([(None,), ("implementation",)])
def test_create_task_refuses_create_pr_from_report_on_trial(self, relationship):
- # A trial org gets reports, not pull requests: the manual Create PR path is refused with
- # the trial code, so a client can show the trial message, and the held-back PR is counted.
from products.signals.backend.free_trial import FreeTrialPullRequestRefused
report = self._report()
@@ -2708,7 +2723,6 @@ class TestSelfDrivingFreeTrialFacadeGates(TestCase):
self.assertEqual(capture_mock.call_args.kwargs, {"report_id": str(report.id), "stage": "manual_create"})
def test_create_task_allows_discussion_from_report_on_trial(self):
- # Discuss answers questions about a report and stays open on a trial; only Create PR is held back.
report = self._report()
with self._on_trial():
dto = facade.create_task(
@@ -2724,11 +2738,10 @@ class TestSelfDrivingFreeTrialFacadeGates(TestCase):
)
self.assertTrue(Task.objects.filter(id=dto.id).exists())
- @parameterized.expand([("implementation", True), ("discussion", False)])
- def test_run_task_refuses_an_existing_report_implementation_on_trial(self, relationship, expect_refusal):
- # The create-time gate cannot see a task that already existed when sales turned the flag
- # on. Starting or retrying that task opens the pull request the trial promises to withhold,
- # and bills the trial org for it. A discussion opens none, so it keeps running.
+ @parameterized.expand(
+ [("implementation", True, True), ("implementation", False, True), ("discussion", False, False)]
+ )
+ def test_run_task_refuses_report_implementation_on_trial(self, relationship, pipeline_rerun, expect_refusal):
from products.signals.backend.free_trial import FreeTrialPullRequestRefused
from products.signals.backend.task_run_artefacts import record_report_task
@@ -2752,11 +2765,23 @@ class TestSelfDrivingFreeTrialFacadeGates(TestCase):
):
if expect_refusal:
with self.assertRaises(FreeTrialPullRequestRefused) as raised:
- facade.run_task(task.id, self.team.id, self.user.id, validated_data={"mode": "background"})
+ facade.run_task(
+ task.id,
+ self.team.id,
+ self.user.id,
+ validated_data={"mode": "background"},
+ pipeline_rerun=pipeline_rerun,
+ )
self.assertEqual(raised.exception.get_codes(), "self_driving_free_trial")
self.assertEqual(capture_mock.call_args.kwargs, {"report_id": str(report.id), "stage": "task_run"})
else:
- result = facade.run_task(task.id, self.team.id, self.user.id, validated_data={"mode": "background"})
+ result = facade.run_task(
+ task.id,
+ self.team.id,
+ self.user.id,
+ validated_data={"mode": "background"},
+ pipeline_rerun=pipeline_rerun,
+ )
assert result is not None and result.error is None
self.assertEqual(task.runs.exists(), not expect_refusal)
diff --git a/products/tasks/backend/tests/test_gateway_usage_workflow.py b/products/tasks/backend/tests/test_gateway_usage_workflow.py
index 5aa939b3ea6d..193b825e9377 100644
--- a/products/tasks/backend/tests/test_gateway_usage_workflow.py
+++ b/products/tasks/backend/tests/test_gateway_usage_workflow.py
@@ -1,24 +1,38 @@
import asyncio
+import inspect
import logging
from datetime import datetime, timedelta
+from decimal import Decimal
from uuid import uuid4
import pytest
+from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, Mock, patch
from django.test import override_settings
from asgiref.sync import sync_to_async
+from parameterized import parameterized
from temporalio import activity
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+from posthog.llm.gateway_usage import GatewayRequestCost
+
+from products.signals.backend.models import SignalScoutRun, SignalSpend
+from products.signals.backend.spend import complete_task_spend, record_task_spend
+from products.tasks.backend.facade.billing import TaskTokenUsageUnavailable
from products.tasks.backend.logic.services.gateway_usage import (
_gateway_usage_client,
_schedule_gateway_usage,
schedule_gateway_usage,
)
-from products.tasks.backend.temporal.gateway_usage import GatewayUsageInput, TaskRunGatewayUsageWorkflow
+from products.tasks.backend.models import Task, TaskRun
+from products.tasks.backend.temporal.gateway_usage import (
+ GatewayUsageInput,
+ TaskRunGatewayUsageWorkflow,
+ reconcile_gateway_usage,
+)
@pytest.mark.parametrize("failed_connection", [False, True])
@@ -214,3 +228,47 @@ async def test_callback_extends_deadline_during_activity_retries() -> None:
execution_timeout=timedelta(minutes=5),
)
assert settled
+
+
+class TestScoutSpendSettlement(BaseTest):
+ @parameterized.expand([(TaskRun.Status.COMPLETED,), (TaskRun.Status.FAILED,), (TaskRun.Status.CANCELLED,)])
+ def test_scout_finalization_waits_for_late_usage(self, status: str) -> None:
+ task = Task.objects.create(
+ team=self.team, title="Example scout", description="", origin_product=Task.OriginProduct.SIGNAL_REPORT
+ )
+ run = task.create_run(environment=TaskRun.Environment.CLOUD)
+ run.state = {
+ "unprocessed_request_ids": ["pending-request"],
+ "token_cost": {"model-a": {"provider-a": {"cost_microusd": 100}}},
+ }
+ run.status = status
+ run.save(update_fields=["state", "status"])
+ scout = SignalScoutRun.objects.for_team(self.team.id).create(
+ team=self.team, task_run=run, skill_name="example-scout", skill_version=1
+ )
+ record_task_spend(team_id=self.team.id, run_id=str(run.id), scout_run_id=str(scout.id), stage="scout")
+ with (
+ patch("products.tasks.backend.facade.billing.process_pending_gateway_usage"),
+ self.assertRaises(TaskTokenUsageUnavailable),
+ ):
+ complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ spend = SignalSpend.objects.for_team(self.team.id).get(scout_run=scout)
+ self.assertIsNone(spend.finalized_at)
+ self.assertTrue(spend.accounting_failed)
+ with patch(
+ "products.tasks.backend.logic.services.gateway_usage._fetch_gateway_cost",
+ new=AsyncMock(return_value=GatewayRequestCost(model="example", provider="example", cost_microusd=50)),
+ ):
+ assert (
+ inspect.unwrap(reconcile_gateway_usage)(GatewayUsageInput(run_id=str(run.id), team_id=self.team.id))
+ == 0
+ )
+ spend.refresh_from_db()
+ self.assertIsNone(spend.finalized_at)
+ complete_task_spend(team_id=self.team.id, run_id=str(run.id))
+ spend.refresh_from_db()
+ scout.refresh_from_db()
+ self.assertIsNotNone(spend.finalized_at)
+ assert not spend.accounting_failed
+ assert spend.spend == Decimal("4.015")
+ assert scout.total_spend == Decimal("4.0150")
diff --git a/products/tasks/backend/tests/test_multi_turn_session.py b/products/tasks/backend/tests/test_multi_turn_session.py
index 3bf3d3919f96..c1cc4f5c9429 100644
--- a/products/tasks/backend/tests/test_multi_turn_session.py
+++ b/products/tasks/backend/tests/test_multi_turn_session.py
@@ -13,6 +13,7 @@ from django.db import OperationalError, transaction
from asgiref.sync import sync_to_async
from parameterized import parameterized
from pydantic import BaseModel, Field, JsonValue
+from temporalio.service import RPCError, RPCStatusCode
from posthog.models import Integration, Organization, Team
from posthog.models.user import User
@@ -2086,3 +2087,18 @@ class TestPollForTurnConnectionDrop:
await poll_for_turn(FakeTaskRun(), skip_lines=0)
assert get_mock.call_count == 2
+
+
[email protected]("origin", [Task.OriginProduct.SIGNAL_REPORT, Task.OriginProduct.SIGNALS_SCOUT])
[email protected]("rpc_status", [None, RPCStatusCode.NOT_FOUND, RPCStatusCode.UNAVAILABLE])
+async def test_pipeline_session_releases_activity_slot_after_signalling(
+ origin: str, rpc_status: RPCStatusCode | None
+) -> None:
+ handle = AsyncMock()
+ if rpc_status is not None:
+ handle.signal.side_effect = RPCError("completion signal failed", rpc_status, b"")
+ session = MultiTurnSession(task=MagicMock(origin_product=origin), task_run=MagicMock(), _workflow_handle=handle)
+ await session.end()
+ handle.signal.assert_awaited_once()
+ handle.result.assert_not_awaited()
diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.py
index ffb1111b2a71..4dbfe06dd1ca 100644
--- a/products/tasks/backend/tests/test_warm_facade.py
+++ b/products/tasks/backend/tests/test_warm_facade.py
@@ -835,19 +835,22 @@ class TestCreateTaskWarmReuse(APIBaseTest):
@parameterized.expand(
[
- ("credential_does_not_match_the_gate", "discussion", True, False),
- ("implementation_never_takes_a_discussion_warm", "implementation", False, False),
+ ("credential_does_not_match_the_gate", "discussion", True, False, False),
+ ("implementation_never_takes_a_discussion_warm", "implementation", False, False, False),
+ ("old_warm_has_signals_credentials", "discussion", False, False, True),
]
)
def test_report_create_cold_starts_when_the_warm_does_not_fit(
- self, _name, relationship, warm_has_credential, code_access_allowed
+ self, _name, relationship, warm_has_credential, code_access_allowed, legacy_billing
):
from products.signals.backend.models import SignalReport
report = SignalReport.objects.create(team=self.team)
- warm_task, _run = self._report_warm_run(
+ warm_task, run = self._report_warm_run(
report, github_integration=self.integration if warm_has_credential else None
)
+ if legacy_billing:
+ TaskRun.update_state_atomic(run.id, remove_keys=["billing_product"])
with patch(f"{FACADE}.signal_task_run_user_message", return_value=True) as mock_signal:
dto = self._create_report_task(report, relationship=relationship, code_access_allowed=code_access_allowed)