Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): skip scout runs whose skill is deleted before they start (#121026)

posthog[bot]cfc8ffa3bda0
diff --git a/products/signals/backend/temporal/agentic/scout_scheduler.py b/products/signals/backend/temporal/agentic/scout_scheduler.py
index 37c241b943f7..9057baee94a7 100644
--- a/products/signals/backend/temporal/agentic/scout_scheduler.py
+++ b/products/signals/backend/temporal/agentic/scout_scheduler.py
@@ -42,7 +42,9 @@ from products.signals.backend.scout_harness.limits import (
WORKFLOW_HARD_CEILING_S,
)
from products.signals.backend.scout_harness.precheck import PRECHECK_TIMEOUT_S, evaluate_scout_precheck
+from products.signals.backend.scout_harness.skill_loader import SkillNotFoundError
from products.signals.backend.temporal import metrics
+from products.skills.backend.models.skills import LLMSkill
if TYPE_CHECKING:
# Type-only: importing the harness runner at module load would close the cycle
@@ -162,12 +164,12 @@ async def evaluate_signals_scout_precheck_activity(input: RunSignalsScoutInput)
def _scout_config_runnable(team_id: int, skill_name: str) -> bool:
"""Whether the coordinator would still dispatch this scout: enabled, or paused by the failure
- breaker, which the coordinator probes. A deleted project is never runnable."""
+ breaker, which the coordinator probes, and its skill still exists. A deleted project is never runnable."""
try:
configs = SignalScoutConfig.objects.for_team(team_id)
except TeamScopeError:
return False
- return (
+ config_runnable = (
configs.filter(skill_name=skill_name)
.filter(
Q(enabled=True)
@@ -178,6 +180,10 @@ def _scout_config_runnable(team_id: int, skill_name: str) -> bool:
)
.exists()
)
+ return (
+ config_runnable
+ and LLMSkill.objects.filter(team_id=team_id, name=skill_name, is_latest=True, deleted=False).exists()
+ )
@temporalio.activity.defn
@@ -320,6 +326,23 @@ async def _run_signals_scout(input: RunSignalsScoutInput) -> RunSignalsScoutOutp
precheck_rows=input.precheck_rows,
precheck_query_source=input.precheck_query_source,
)
+ except SkillNotFoundError:
+ # A person deleted or renamed the skill after the run was dispatched. Nothing is left to run.
+ metrics.increment_scout_run("skill_missing")
+ logger.info(
+ "signals_scout: skill is gone, skipping the run",
+ team_id=input.team_id,
+ skill_name=input.skill_name,
+ )
+ return RunSignalsScoutOutput(
+ run_id=None,
+ task_run_id=None,
+ status=None,
+ runtime_s=0.0,
+ skill_name=input.skill_name,
+ skill_version=input.skill_version or 0,
+ skip_reason="skill_missing",
+ )
except (OperationalError, InterfaceError):
# Transient DB connection drop (pgbouncer pool recycle / failover / deploy). Stay
# fail-safe and silent: report a failed run for this tick rather than raising. The
diff --git a/products/signals/backend/test/test_scout_harness.py b/products/signals/backend/test/test_scout_harness.py
index c6b8351e5f87..00136357316a 100644
--- a/products/signals/backend/test/test_scout_harness.py
+++ b/products/signals/backend/test/test_scout_harness.py
@@ -3460,12 +3460,19 @@ async def test_activity_runs_when_team_under_signals_quota(ateam):
@pytest.mark.asyncio
@pytest.mark.django_db
-async def test_activity_swallows_transient_db_connection_drop(ateam):
- # A pgbouncer pool recycle / failover can surface as OperationalError from the runner's
- # synchronous DB access, outside the run-row try/except. The activity's "never raises"
- # contract must hold: report a failed run instead of letting it escape.
+ "error,expected_status,expected_skip_reason",
+ [
+ # A pgbouncer pool recycle / failover can surface as OperationalError from the runner's
+ # synchronous DB access, outside the run-row try/except.
+ (OperationalError("server closed the connection unexpectedly"), "failed", None),
+ # The skill was deleted while a scheduled run slept its start delay.
+ (SkillNotFoundError("No skill named 'signals-scout-errors'"), None, "skill_missing"),
+ ],
+)
+async def test_activity_does_not_raise_on_runner_error(ateam, error, expected_status, expected_skip_reason):
async def fake_arun(**_kwargs):
- raise OperationalError("server closed the connection unexpectedly")
+ raise error
with patch(
"products.signals.backend.scout_harness.runner.arun_signals_scout",
@@ -3479,9 +3486,33 @@ async def test_activity_swallows_transient_db_connection_drop(ateam):
assert output.run_id is None
assert output.task_run_id is None
- assert output.status == "failed"
+ assert output.status == expected_status
assert output.skill_name == "signals-scout-errors"
- assert output.skip_reason is None
+ assert output.skip_reason == expected_skip_reason
+
+
[email protected]("skill_state,expected", [("live", True), ("deleted", False), ("absent", False)])
+async def test_config_runnable_check_requires_a_live_skill(ateam, skill_state, expected):
+ await sync_to_async(SignalScoutConfig.objects.create)(team=ateam, skill_name="signals-scout-errors", enabled=True)
+ if skill_state != "absent":
+ await sync_to_async(LLMSkill.objects.create)(
+ team=ateam,
+ name="signals-scout-errors",
+ description="Errors scout",
+ body="scout",
+ deleted=skill_state == "deleted",
+ is_latest=skill_state == "live",
+ )
+
+ env = ActivityEnvironment()
+ runnable = await env.run(
+ check_signals_scout_config_runnable_activity,
+ RunSignalsScoutInput(team_id=ateam.id, skill_name="signals-scout-errors"),
+ )
+
+ assert runnable is expected
# ── Tasks-UI cross-link: SignalScoutRun ─→ TaskRun ────────────────────────────