Skip to content

Automate browser based workflows with AI

Log why a failed run went unattributed on the terminal run log (#8656)

Shuchang Zheng8fae26bf0536
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 26de84d10..ac549c9a2 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -1105,10 +1105,16 @@ def _start_hold(workflow_run: WorkflowRun) -> str:
return "none"
-def _failure_infra_component(workflow_run: WorkflowRun, status: WorkflowRunStatus) -> str | None:
+def _failure_attribution_log_fields(workflow_run: WorkflowRun, status: WorkflowRunStatus) -> dict[str, str | None]:
if status == WorkflowRunStatus.completed:
- return None
- return derive_failure_attribution(workflow_run.failure_category)["primary_infra_component"]
+ return {"primary_infra_component": None, "primary_failure_category": None, "attribution_evidence_source": None}
+ attribution = derive_failure_attribution(workflow_run.failure_category)
+ # An unattributed run keeps its category and evidence so the abstention reason stays countable.
+ return {
+ "primary_infra_component": attribution["primary_infra_component"],
+ "primary_failure_category": attribution["failure_category"],
+ "attribution_evidence_source": attribution["evidence_source"],
+ }
def _workflow_block_engine_attribution_for_duration_log(workflow_run_id: str) -> WorkflowBlockEngineArmAttribution:
@@ -12787,7 +12793,7 @@ class WorkflowService:
duration_seconds=duration_seconds,
recorded_seconds=recorded_seconds,
workflow_run_status=workflow_run.status,
- primary_infra_component=_failure_infra_component(workflow_run, status),
+ **_failure_attribution_log_fields(workflow_run, status),
# A run with a retry policy reaches a terminal status once per attempt. Consumers that
# derive one outcome per run keep only the events without a pending retry.
attempt_number=attempt_number
diff --git a/tests/unit/test_workflow_run_minutes_gate.py b/tests/unit/test_workflow_run_minutes_gate.py
index 78f754f69..6004bfe3d 100644
--- a/tests/unit/test_workflow_run_minutes_gate.py
+++ b/tests/unit/test_workflow_run_minutes_gate.py
@@ -632,6 +632,30 @@ async def test_conditional_cancel_duration_log_carries_the_pinned_arm_and_its_de
assert duration_logs[0]["billing_tier"] == BillingTier.ENTERPRISE.value
+async def test_an_unattributed_run_logs_why_attribution_abstained(record_run_duration: AsyncMock) -> None:
+ # The unattributed share is read by abstention reason: a keyword-only anti-bot guess and a run with no
+ # category at all both land in `unattributed`, and only these two fields tell them apart.
+ row = _make_row(started=True)
+ row.status = WorkflowRunStatus.terminated
+ row.failure_category = [
+ {
+ "category": "ANTI_BOT_DETECTION",
+ "confidence_float": 0.9,
+ "reasoning": "keyword match",
+ "evidence_source": "keyword_only",
+ }
+ ]
+
+ with capture_logs() as logs:
+ await WorkflowService()._after_workflow_run_status_write(row, WorkflowRunStatus.terminated)
+
+ [event] = [e for e in logs if e.get("event") == "Workflow run duration metrics"]
+ assert event["primary_infra_component"] == "unattributed"
+ assert event["primary_failure_category"] == "ANTI_BOT_DETECTION"
+ assert event["attribution_evidence_source"] == "keyword_only"
+
+
@pytest.mark.asyncio
async def test_duration_log_reads_unknown_when_no_context_is_current(
record_run_duration: AsyncMock,