Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log why a failed run went unattributed on the terminal run log (#8656)
Shuchang Zheng8fae26bf0536
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 26de84d10..ac549c9a2 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -1105,10 +1105,16 @@ def _start_hold(workflow_run: WorkflowRun) -> str:return "none"-def _failure_infra_component(workflow_run: WorkflowRun, status: WorkflowRunStatus) -> str | None:+def _failure_attribution_log_fields(workflow_run: WorkflowRun, status: WorkflowRunStatus) -> dict[str, str | None]:if status == WorkflowRunStatus.completed:- return None- return derive_failure_attribution(workflow_run.failure_category)["primary_infra_component"]+ return {"primary_infra_component": None, "primary_failure_category": None, "attribution_evidence_source": None}+ attribution = derive_failure_attribution(workflow_run.failure_category)+ # An unattributed run keeps its category and evidence so the abstention reason stays countable.+ return {+ "primary_infra_component": attribution["primary_infra_component"],+ "primary_failure_category": attribution["failure_category"],+ "attribution_evidence_source": attribution["evidence_source"],+ }def _workflow_block_engine_attribution_for_duration_log(workflow_run_id: str) -> WorkflowBlockEngineArmAttribution:@@ -12787,7 +12793,7 @@ class WorkflowService:duration_seconds=duration_seconds,recorded_seconds=recorded_seconds,workflow_run_status=workflow_run.status,- primary_infra_component=_failure_infra_component(workflow_run, status),+ **_failure_attribution_log_fields(workflow_run, status),# A run with a retry policy reaches a terminal status once per attempt. Consumers that# derive one outcome per run keep only the events without a pending retry.attempt_number=attempt_numberdiff --git a/tests/unit/test_workflow_run_minutes_gate.py b/tests/unit/test_workflow_run_minutes_gate.pyindex 78f754f69..6004bfe3d 100644--- a/tests/unit/test_workflow_run_minutes_gate.py+++ b/tests/unit/test_workflow_run_minutes_gate.py@@ -632,6 +632,30 @@ async def test_conditional_cancel_duration_log_carries_the_pinned_arm_and_its_deassert duration_logs[0]["billing_tier"] == BillingTier.ENTERPRISE.value+async def test_an_unattributed_run_logs_why_attribution_abstained(record_run_duration: AsyncMock) -> None:+ # The unattributed share is read by abstention reason: a keyword-only anti-bot guess and a run with no+ # category at all both land in `unattributed`, and only these two fields tell them apart.+ row = _make_row(started=True)+ row.status = WorkflowRunStatus.terminated+ row.failure_category = [+ {+ "category": "ANTI_BOT_DETECTION",+ "confidence_float": 0.9,+ "reasoning": "keyword match",+ "evidence_source": "keyword_only",+ }+ ]++ with capture_logs() as logs:+ await WorkflowService()._after_workflow_run_status_write(row, WorkflowRunStatus.terminated)++ [event] = [e for e in logs if e.get("event") == "Workflow run duration metrics"]+ assert event["primary_infra_component"] == "unattributed"+ assert event["primary_failure_category"] == "ANTI_BOT_DETECTION"+ assert event["attribution_evidence_source"] == "keyword_only"++@pytest.mark.asyncioasync def test_duration_log_reads_unknown_when_no_context_is_current(record_run_duration: AsyncMock,