Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Remove the biased workflow_status log field in favour of workflow_run_status (#8355)
Aaron Perezd9dbcba2a769
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 5820e57e8..277ec6996 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -7904,7 +7904,7 @@ class WorkflowService:block_type=block.block_type,block_label=block.label,block_status=block_result.status,- workflow_status=target_status,+ resolved_terminal_status=target_status,deferred=defer_status_write,)if defer_status_write:@@ -9498,7 +9498,6 @@ class WorkflowService:LOG.info(f"Marking workflow run {workflow_run_id} as completed",workflow_run_id=workflow_run_id,- workflow_status="completed",)# Add workflow completion tag to trace@@ -9695,7 +9694,6 @@ class WorkflowService:LOG.info(f"Marking workflow run {workflow_run_id} as failed",workflow_run_id=workflow_run_id,- workflow_status="failed",failure_reason=failure_reason,)@@ -9713,7 +9711,6 @@ class WorkflowService:LOG.info("Workflow run failure classified",workflow_run_id=workflow_run_id,- workflow_status="failed",failure_category=failure_category,primary_failure_category=failure_category[0].get("category") if failure_category else None,failure_category_source=failure_category_source,@@ -9767,7 +9764,6 @@ class WorkflowService:LOG.info(f"Marked workflow run {workflow_run_id} as failed (conditional)",workflow_run_id=workflow_run_id,- workflow_status="failed",failure_category=failure_category,)if cascade_children:@@ -9816,7 +9812,6 @@ class WorkflowService:LOG.info(f"Marked workflow run {workflow_run_id} as running",workflow_run_id=workflow_run_id,- workflow_status="running",run_with=run_with,queued_seconds=queued_seconds,)@@ -9832,7 +9827,6 @@ class WorkflowService:LOG.info(f"Marking workflow run {workflow_run_id} as terminated",workflow_run_id=workflow_run_id,- workflow_status="terminated",failure_reason=failure_reason,)@@ -9852,7 +9846,6 @@ class WorkflowService:LOG.info("Workflow run failure classified",workflow_run_id=workflow_run_id,- workflow_status="terminated",failure_category=failure_category,primary_failure_category=failure_category[0].get("category") if failure_category else None,failure_category_source=failure_category_source,@@ -9880,7 +9873,6 @@ class WorkflowService:LOG.info(f"Marking workflow run {workflow_run_id} as canceled",workflow_run_id=workflow_run_id,- workflow_status="canceled",)updated = await self.mark_workflow_run_as_canceled_if_not_final(@@ -9920,7 +9912,6 @@ class WorkflowService:LOG.info(f"Marked workflow run {workflow_run_id} as canceled (conditional)",workflow_run_id=workflow_run_id,- workflow_status="canceled",)otel_trace.get_current_span().set_attribute("task.completion_status", WorkflowRunStatus.canceled)@@ -9990,7 +9981,6 @@ class WorkflowService:LOG.info(f"Marking workflow run {workflow_run_id} as timed out",workflow_run_id=workflow_run_id,- workflow_status="timed_out",)failure_category = self._classify_workflow_terminal_failure(@@ -10000,7 +9990,6 @@ class WorkflowService:LOG.info("Workflow run failure classified",workflow_run_id=workflow_run_id,- workflow_status="timed_out",failure_category=failure_category,primary_failure_category=failure_category[0].get("category") if failure_category else None,failure_category_source="code_level",diff --git a/tests/unit/test_workflow_status_field_guard.py b/tests/unit/test_workflow_status_field_guard.pynew file mode 100644index 000000000..d758d03f5--- /dev/null+++ b/tests/unit/test_workflow_status_field_guard.py@@ -0,0 +1,71 @@+"""AST guard against reintroducing the ``workflow_status`` log field.++``workflow_status`` was a redundant duplicate of ``workflow_run_status``, emitted only by the+``mark_workflow_run_as_*`` wrappers. The unified finalizer ``_finalize_workflow_run_status``+bypasses those wrappers for completed/terminated runs while failures still route through+``mark_workflow_run_as_failed``, so the field was populated on some terminal paths and not+others -- a biased sample that reads as a ~99% failure rate (measured 7 completed vs 1,882+failed over 3h, against a true 4,548 / 949).++``workflow_run_status`` on the ``"Workflow run duration metrics"`` log is the canonical outcome+facet: ``_after_workflow_run_status_write`` emits exactly one per final status write, and both+status-write helpers call it.+"""++from __future__ import annotations++import ast+from pathlib import Path++_REPO_ROOT = Path(__file__).resolve().parents[2]+_SCANNED_ROOTS = ("skyvern", "cloud")+_BANNED_KEYWORD = "workflow_status"+++def _log_call_keywords(tree: ast.AST) -> list[tuple[int, str]]:+ """Keyword names passed to ``LOG.<method>(...)`` calls, with line numbers."""+ found: list[tuple[int, str]] = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call):+ continue+ func = node.func+ if not isinstance(func, ast.Attribute):+ continue+ receiver = func.value+ if not (isinstance(receiver, ast.Name) and receiver.id == "LOG"):+ continue+ for keyword in node.keywords:+ if keyword.arg is not None:+ found.append((node.lineno, keyword.arg))+ return found+++def _python_files() -> list[Path]:+ files: list[Path] = []+ for root in _SCANNED_ROOTS:+ files.extend((_REPO_ROOT / root).rglob("*.py"))+ return files+++def test_no_production_log_emits_workflow_status() -> None:+ offenders: list[str] = []+ for path in _python_files():+ try:+ tree = ast.parse(path.read_text(encoding="utf-8"))+ except (SyntaxError, UnicodeDecodeError): # pragma: no cover - not our source+ continue+ for lineno, keyword in _log_call_keywords(tree):+ if keyword == _BANNED_KEYWORD:+ offenders.append(f"{path.relative_to(_REPO_ROOT)}:{lineno}")++ assert not offenders, (+ "`workflow_status` is a biased half-sample of terminal outcomes -- the unified finalizer "+ "bypasses the mark_* wrappers that emit it. Use `workflow_run_status` on the "+ '"Workflow run duration metrics" log instead. Offenders: ' + ", ".join(sorted(offenders))+ )+++def test_guard_detects_a_planted_violation() -> None:+ """The guard must fail on a real violation, not merely pass on clean source."""+ tree = ast.parse('LOG.info("x", workflow_run_id="wr_1", workflow_status="completed")\n')+ assert (1, _BANNED_KEYWORD) in _log_call_keywords(tree)