Skip to content

Automate browser based workflows with AI

fix: widen 'access denied' failure classification with auth-context guard (#5427)

LawyZheng87d270dbc010
diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.py
index a8dbd6a21..327552883 100644
--- a/skyvern/forge/failure_classifier.py
+++ b/skyvern/forge/failure_classifier.py
@@ -31,20 +31,28 @@ def classify_from_failure_reason(
categories: list[dict] = []
# Bot detection / CAPTCHA — use specific phrases to avoid false positives
- if any(
- kw in reason
- for kw in [
- "captcha",
- "cloudflare",
- "bot detect",
- "bot block",
- "ip block",
- "request block",
- "access denied by",
- "anti-bot",
- "human verification",
- ]
- ):
+ _auth_context_keywords = ["login", "auth", "password", "permission", "credential"]
+ _has_auth_context = any(kw in reason for kw in _auth_context_keywords)
+ _antibot_keywords = [
+ "captcha",
+ "cloudflare",
+ "bot detect",
+ "bot block",
+ "ip block",
+ "request block",
+ "anti-bot",
+ "human verification",
+ ]
+ # "access denied" is ambiguous: it can be bot blocking OR auth failure.
+ # Only treat it as bot detection when there are no auth-related keywords nearby.
+ # Note: in Skyvern's context, failure_reason is LLM-generated from page observations,
+ # so RBAC-style messages like "Access denied: insufficient privileges" are unlikely.
+ # If this becomes a false-positive source, consider further narrowing (e.g. requiring
+ # "access denied" appears without ANY qualifier, or adding more exclusion keywords).
+ if not _has_auth_context:
+ _antibot_keywords.append("access denied")
+
+ if any(kw in reason for kw in _antibot_keywords):
categories.append(
{
"category": "ANTI_BOT_DETECTION",
@@ -87,8 +95,10 @@ def classify_from_failure_reason(
}
)
- # Auth failure
- if any(kw in reason for kw in ["login fail", "authentication fail", "auth fail", "mfa", "password"]):
+ # Auth failure — also catches "access denied" when auth context is present
+ if any(kw in reason for kw in ["login fail", "authentication fail", "auth fail", "mfa", "password"]) or (
+ "access denied" in reason and _has_auth_context
+ ):
categories.append(
{
"category": "AUTH_FAILURE",
diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py
index 642383e33..d956b0324 100644
--- a/tests/unit/test_failure_classifier.py
+++ b/tests/unit/test_failure_classifier.py
@@ -31,6 +31,8 @@ class TestAntiBotDetection:
"Access denied by WAF",
"Anti-bot system triggered",
"Human verification required",
+ "The current page shows an 'Access Denied' error",
+ "Access Denied - website blocked the request",
],
)
def test_keywords_match(self, reason: str) -> None:
@@ -39,6 +41,22 @@ class TestAntiBotDetection:
categories = [r["category"] for r in result]
assert "ANTI_BOT_DETECTION" in categories
+ def test_access_denied_with_auth_context_is_not_antibot(self) -> None:
+ """'Access denied' + auth keywords → AUTH_FAILURE, not ANTI_BOT_DETECTION."""
+ result = classify_from_failure_reason("Access denied after login - user does not have permission")
+ assert result is not None
+ categories = [r["category"] for r in result]
+ assert "AUTH_FAILURE" in categories
+ assert "ANTI_BOT_DETECTION" not in categories
+
+ def test_access_denied_with_password_context_is_not_antibot(self) -> None:
+ """'Access denied' when password is mentioned → AUTH_FAILURE."""
+ result = classify_from_failure_reason("The page shows 'Access Denied'. Unable to enter your password.")
+ assert result is not None
+ categories = [r["category"] for r in result]
+ assert "AUTH_FAILURE" in categories
+ assert "ANTI_BOT_DETECTION" not in categories
+
def test_broad_blocked_does_not_match(self) -> None:
"""'blocked' alone should NOT trigger ANTI_BOT_DETECTION (narrowed keywords)."""
result = classify_from_failure_reason("UI element blocked by overlay")