Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: tell the model when a site's bot protection blocks a page fetch (#13760)
Nilay42735fcf21bb
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 4df7afab1f7..4f0b2dabcaf 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -16647,7 +16647,12 @@ def _is_bot_check(status: int, headers) -> bool:return True# Rate limits and outages behind these CDNs carry the same Server header.server = (headers.get("Server") or "").lower()- return status == 403 and ("cloudflare" in server or "akamaighost" in server)+ if status != 403:+ return False+ # Cloudflare names itself on every response it proxies; its own blocks skip the cache and send no CF-Cache-Status.+ if "cloudflare" in server:+ return headers.get("cf-cache-status") is None+ return "akamaighost" in serverdef _fetch_url_raw(@@ -17724,14 +17729,21 @@ def _fetch_page_text(if err is None and body.strip():return _truncate_page_text(body.strip(), max_chars)+ meta: dict = {}err, body, content_type = _fetch_url_raw(url,timeout = timeout,deadline = deadline,cancel_event = cancel_event,+ meta_out = meta,**policy_kwargs,)if err is not None:+ if meta.get("bot_check"):+ return (+ f"{err.rstrip()}. The site's bot protection blocked this automated request; "+ "fetching this site again will not help, so use another source."+ )return err# Trust a declared HTML type, and otherwise sniff the body: servers with a missing or wrong Content-Type still getdiff --git a/studio/backend/tests/test_browser_routes.py b/studio/backend/tests/test_browser_routes.pyindex c0b5fd72f90..feb09305dac 100644--- a/studio/backend/tests/test_browser_routes.py+++ b/studio/backend/tests/test_browser_routes.py@@ -494,6 +494,7 @@ def test_bot_checks_are_told_apart_from_plain_refusals():assert not _is_bot_check(503, headers(Server = "cloudflare"))assert not _is_bot_check(404, headers(Server = "cloudflare"))assert not _is_bot_check(403, headers(Server = "nginx"))+ assert not _is_bot_check(403, headers(Server = "cloudflare", cf_cache_status = "DYNAMIC"))def test_a_byte_order_mark_decides_the_encoding():diff --git a/studio/backend/tests/test_web_fetch_bot_check.py b/studio/backend/tests/test_web_fetch_bot_check.pynew file mode 100644index 00000000000..399b1250a7d--- /dev/null+++ b/studio/backend/tests/test_web_fetch_bot_check.py@@ -0,0 +1,73 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import socket+import sys+import threading+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from core.inference import tools+++def _serve_403(monkeypatch, headers):+ server = socket.create_server(("127.0.0.1", 0))+ server.settimeout(10)+ body = b"<html>Just a moment...</html>"+ head = "".join(f"{name}: {value}\r\n" for name, value in headers.items())+ response = (+ f"HTTP/1.1 403 Forbidden\r\nContent-Type: text/html\r\nContent-Length: {len(body)}\r\n{head}\r\n"+ ).encode() + body++ def respond():+ with server:+ while True:+ try:+ conn, _ = server.accept()+ except OSError:+ return+ with conn:+ conn.recv(65536)+ conn.sendall(response)++ threading.Thread(target = respond, daemon = True).start()+ monkeypatch.setattr(+ tools, "_validate_and_resolve_host", lambda host, port: (True, "", ["127.0.0.1"])+ )+ return f"http://example.com:{server.getsockname()[1]}/page"+++ "headers",+ [+ {"Server": "cloudflare", "cf-mitigated": "challenge"},+ {"Server": "cloudflare"},+ {"Server": "DataDome", "X-DataDome": "protected"},+ {"Server": "AkamaiGHost"},+ ],+)+def test_the_model_is_told_a_bot_check_blocked_the_fetch(monkeypatch, headers):+ url = _serve_403(monkeypatch, headers)+ result = tools.execute_tool("web_search", {"url": url}, timeout = 10)+ assert result.startswith("Failed to fetch URL: HTTP 403 Forbidden")+ assert "bot protection" in result+ assert "fetching this site again will not help" in result+++ "headers",+ [+ {"Server": "nginx"},+ # an origin's own 403 passed through Cloudflare carries a cache status+ {"Server": "cloudflare", "CF-Cache-Status": "DYNAMIC"},+ ],+)+def test_a_plain_403_keeps_its_message(monkeypatch, headers):+ url = _serve_403(monkeypatch, headers)+ result = tools.execute_tool("web_search", {"url": url}, timeout = 10)+ assert result == "Failed to fetch URL: HTTP 403 Forbidden"diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.pyindex 644d4f8b96b..4bc9c68206e 100644--- a/studio/backend/tests/test_web_fetch_extraction.py+++ b/studio/backend/tests/test_web_fetch_extraction.py@@ -1114,6 +1114,7 @@ def test_fetch_page_text_falls_back_to_html_when_readme_api_fails(monkeypatch):extra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):if url.startswith("https://api.github.com/"):return "Failed to fetch URL: HTTP 403 rate limited", "", ""@@ -1137,6 +1138,7 @@ def test_fetch_page_text_reads_github_file_link_as_raw_file(monkeypatch):extra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):calls.append(url)if url.startswith("https://raw.githubusercontent.com/"):@@ -1162,6 +1164,7 @@ def test_fetch_page_text_falls_back_to_github_file_page_when_raw_fails(monkeypatextra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):calls.append(url)if url.startswith("https://raw.githubusercontent.com/"):@@ -1186,6 +1189,7 @@ def test_fetch_page_text_keeps_semicolon_in_github_file_name(monkeypatch):extra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):calls.append(url)return None, "file body", "text/plain"@@ -1214,6 +1218,7 @@ def test_fetch_page_text_leaves_other_github_urls_on_their_page(monkeypatch, urlextra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):calls.append(url)return "Failed to fetch URL: HTTP 403", "", ""@@ -1237,6 +1242,7 @@ def _page_text(monkeypatch, url, body, content_type):extra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):return None, body, content_type@@ -1310,6 +1316,7 @@ def test_fetch_page_text_propagates_fetch_errors(monkeypatch):extra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):return "Failed to fetch URL: HTTP 404 Not Found", "", ""@@ -1766,6 +1773,7 @@ def test_fetch_page_text_keeps_a_markdown_readme_verbatim(extra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):assert url == "https://api.github.com/repos/unslothai/unsloth/readme"return (None, md_readme, "text/plain")@@ -2334,6 +2342,7 @@ def test_fetch_page_text_missing_content_type_html_sniffed(monkeypatch):extra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):return None, _GITHUB_PAGE, ""@@ -2669,6 +2678,7 @@ def test_fetch_page_text_shares_one_deadline_across_readme_and_fallback(monkeypaextra_headers = None,deadline = None,cancel_event = None,+ meta_out = None,):seen_deadlines.append(deadline)# Fail the README API so the HTML fallback also runs.