Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: tell the model when a site's bot protection blocks a page fetch (#13760)

Nilay42735fcf21bb
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 4df7afab1f7..4f0b2dabcaf 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -16647,7 +16647,12 @@ def _is_bot_check(status: int, headers) -> bool:
return True
# Rate limits and outages behind these CDNs carry the same Server header.
server = (headers.get("Server") or "").lower()
- return status == 403 and ("cloudflare" in server or "akamaighost" in server)
+ if status != 403:
+ return False
+ # Cloudflare names itself on every response it proxies; its own blocks skip the cache and send no CF-Cache-Status.
+ if "cloudflare" in server:
+ return headers.get("cf-cache-status") is None
+ return "akamaighost" in server
def _fetch_url_raw(
@@ -17724,14 +17729,21 @@ def _fetch_page_text(
if err is None and body.strip():
return _truncate_page_text(body.strip(), max_chars)
+ meta: dict = {}
err, body, content_type = _fetch_url_raw(
url,
timeout = timeout,
deadline = deadline,
cancel_event = cancel_event,
+ meta_out = meta,
**policy_kwargs,
)
if err is not None:
+ if meta.get("bot_check"):
+ return (
+ f"{err.rstrip()}. The site's bot protection blocked this automated request; "
+ "fetching this site again will not help, so use another source."
+ )
return err
# Trust a declared HTML type, and otherwise sniff the body: servers with a missing or wrong Content-Type still get
diff --git a/studio/backend/tests/test_browser_routes.py b/studio/backend/tests/test_browser_routes.py
index c0b5fd72f90..feb09305dac 100644
--- a/studio/backend/tests/test_browser_routes.py
+++ b/studio/backend/tests/test_browser_routes.py
@@ -494,6 +494,7 @@ def test_bot_checks_are_told_apart_from_plain_refusals():
assert not _is_bot_check(503, headers(Server = "cloudflare"))
assert not _is_bot_check(404, headers(Server = "cloudflare"))
assert not _is_bot_check(403, headers(Server = "nginx"))
+ assert not _is_bot_check(403, headers(Server = "cloudflare", cf_cache_status = "DYNAMIC"))
def test_a_byte_order_mark_decides_the_encoding():
diff --git a/studio/backend/tests/test_web_fetch_bot_check.py b/studio/backend/tests/test_web_fetch_bot_check.py
new file mode 100644
index 00000000000..399b1250a7d
--- /dev/null
+++ b/studio/backend/tests/test_web_fetch_bot_check.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import socket
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference import tools
+
+
+def _serve_403(monkeypatch, headers):
+ server = socket.create_server(("127.0.0.1", 0))
+ server.settimeout(10)
+ body = b"<html>Just a moment...</html>"
+ head = "".join(f"{name}: {value}\r\n" for name, value in headers.items())
+ response = (
+ f"HTTP/1.1 403 Forbidden\r\nContent-Type: text/html\r\nContent-Length: {len(body)}\r\n{head}\r\n"
+ ).encode() + body
+
+ def respond():
+ with server:
+ while True:
+ try:
+ conn, _ = server.accept()
+ except OSError:
+ return
+ with conn:
+ conn.recv(65536)
+ conn.sendall(response)
+
+ threading.Thread(target = respond, daemon = True).start()
+ monkeypatch.setattr(
+ tools, "_validate_and_resolve_host", lambda host, port: (True, "", ["127.0.0.1"])
+ )
+ return f"http://example.com:{server.getsockname()[1]}/page"
+
+
+ "headers",
+ [
+ {"Server": "cloudflare", "cf-mitigated": "challenge"},
+ {"Server": "cloudflare"},
+ {"Server": "DataDome", "X-DataDome": "protected"},
+ {"Server": "AkamaiGHost"},
+ ],
+)
+def test_the_model_is_told_a_bot_check_blocked_the_fetch(monkeypatch, headers):
+ url = _serve_403(monkeypatch, headers)
+ result = tools.execute_tool("web_search", {"url": url}, timeout = 10)
+ assert result.startswith("Failed to fetch URL: HTTP 403 Forbidden")
+ assert "bot protection" in result
+ assert "fetching this site again will not help" in result
+
+
+ "headers",
+ [
+ {"Server": "nginx"},
+ # an origin's own 403 passed through Cloudflare carries a cache status
+ {"Server": "cloudflare", "CF-Cache-Status": "DYNAMIC"},
+ ],
+)
+def test_a_plain_403_keeps_its_message(monkeypatch, headers):
+ url = _serve_403(monkeypatch, headers)
+ result = tools.execute_tool("web_search", {"url": url}, timeout = 10)
+ assert result == "Failed to fetch URL: HTTP 403 Forbidden"
diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.py
index 644d4f8b96b..4bc9c68206e 100644
--- a/studio/backend/tests/test_web_fetch_extraction.py
+++ b/studio/backend/tests/test_web_fetch_extraction.py
@@ -1114,6 +1114,7 @@ def test_fetch_page_text_falls_back_to_html_when_readme_api_fails(monkeypatch):
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
if url.startswith("https://api.github.com/"):
return "Failed to fetch URL: HTTP 403 rate limited", "", ""
@@ -1137,6 +1138,7 @@ def test_fetch_page_text_reads_github_file_link_as_raw_file(monkeypatch):
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
calls.append(url)
if url.startswith("https://raw.githubusercontent.com/"):
@@ -1162,6 +1164,7 @@ def test_fetch_page_text_falls_back_to_github_file_page_when_raw_fails(monkeypat
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
calls.append(url)
if url.startswith("https://raw.githubusercontent.com/"):
@@ -1186,6 +1189,7 @@ def test_fetch_page_text_keeps_semicolon_in_github_file_name(monkeypatch):
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
calls.append(url)
return None, "file body", "text/plain"
@@ -1214,6 +1218,7 @@ def test_fetch_page_text_leaves_other_github_urls_on_their_page(monkeypatch, url
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
calls.append(url)
return "Failed to fetch URL: HTTP 403", "", ""
@@ -1237,6 +1242,7 @@ def _page_text(monkeypatch, url, body, content_type):
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
return None, body, content_type
@@ -1310,6 +1316,7 @@ def test_fetch_page_text_propagates_fetch_errors(monkeypatch):
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
return "Failed to fetch URL: HTTP 404 Not Found", "", ""
@@ -1766,6 +1773,7 @@ def test_fetch_page_text_keeps_a_markdown_readme_verbatim(
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
assert url == "https://api.github.com/repos/unslothai/unsloth/readme"
return (None, md_readme, "text/plain")
@@ -2334,6 +2342,7 @@ def test_fetch_page_text_missing_content_type_html_sniffed(monkeypatch):
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
return None, _GITHUB_PAGE, ""
@@ -2669,6 +2678,7 @@ def test_fetch_page_text_shares_one_deadline_across_readme_and_fallback(monkeypa
extra_headers = None,
deadline = None,
cancel_event = None,
+ meta_out = None,
):
seen_deadlines.append(deadline)
# Fail the README API so the HTML fallback also runs.