Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: widen 'access denied' failure classification with auth-context guard (#5427)
LawyZheng87d270dbc010
diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.pyindex a8dbd6a21..327552883 100644--- a/skyvern/forge/failure_classifier.py+++ b/skyvern/forge/failure_classifier.py@@ -31,20 +31,28 @@ def classify_from_failure_reason(categories: list[dict] = []# Bot detection / CAPTCHA — use specific phrases to avoid false positives- if any(- kw in reason- for kw in [- "captcha",- "cloudflare",- "bot detect",- "bot block",- "ip block",- "request block",- "access denied by",- "anti-bot",- "human verification",- ]- ):+ _auth_context_keywords = ["login", "auth", "password", "permission", "credential"]+ _has_auth_context = any(kw in reason for kw in _auth_context_keywords)+ _antibot_keywords = [+ "captcha",+ "cloudflare",+ "bot detect",+ "bot block",+ "ip block",+ "request block",+ "anti-bot",+ "human verification",+ ]+ # "access denied" is ambiguous: it can be bot blocking OR auth failure.+ # Only treat it as bot detection when there are no auth-related keywords nearby.+ # Note: in Skyvern's context, failure_reason is LLM-generated from page observations,+ # so RBAC-style messages like "Access denied: insufficient privileges" are unlikely.+ # If this becomes a false-positive source, consider further narrowing (e.g. requiring+ # "access denied" appears without ANY qualifier, or adding more exclusion keywords).+ if not _has_auth_context:+ _antibot_keywords.append("access denied")++ if any(kw in reason for kw in _antibot_keywords):categories.append({"category": "ANTI_BOT_DETECTION",@@ -87,8 +95,10 @@ def classify_from_failure_reason(})- # Auth failure- if any(kw in reason for kw in ["login fail", "authentication fail", "auth fail", "mfa", "password"]):+ # Auth failure — also catches "access denied" when auth context is present+ if any(kw in reason for kw in ["login fail", "authentication fail", "auth fail", "mfa", "password"]) or (+ "access denied" in reason and _has_auth_context+ ):categories.append({"category": "AUTH_FAILURE",diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex 642383e33..d956b0324 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -31,6 +31,8 @@ class TestAntiBotDetection:"Access denied by WAF","Anti-bot system triggered","Human verification required",+ "The current page shows an 'Access Denied' error",+ "Access Denied - website blocked the request",],)def test_keywords_match(self, reason: str) -> None:@@ -39,6 +41,22 @@ class TestAntiBotDetection:categories = [r["category"] for r in result]assert "ANTI_BOT_DETECTION" in categories+ def test_access_denied_with_auth_context_is_not_antibot(self) -> None:+ """'Access denied' + auth keywords → AUTH_FAILURE, not ANTI_BOT_DETECTION."""+ result = classify_from_failure_reason("Access denied after login - user does not have permission")+ assert result is not None+ categories = [r["category"] for r in result]+ assert "AUTH_FAILURE" in categories+ assert "ANTI_BOT_DETECTION" not in categories++ def test_access_denied_with_password_context_is_not_antibot(self) -> None:+ """'Access denied' when password is mentioned → AUTH_FAILURE."""+ result = classify_from_failure_reason("The page shows 'Access Denied'. Unable to enter your password.")+ assert result is not None+ categories = [r["category"] for r in result]+ assert "AUTH_FAILURE" in categories+ assert "ANTI_BOT_DETECTION" not in categories+def test_broad_blocked_does_not_match(self) -> None:"""'blocked' alone should NOT trigger ANTI_BOT_DETECTION (narrowed keywords)."""result = classify_from_failure_reason("UI element blocked by overlay")