RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): prevent path traversal and SSRF in Browser URL file upload (#18711)
陈志谦4870e83aa7a6
diff --git a/agent/component/browser.py b/agent/component/browser.pyindex 67116ed781..276b426ccd 100644--- a/agent/component/browser.py+++ b/agent/component/browser.py@@ -26,9 +26,7 @@ import tempfilefrom abc import ABCfrom pathlib import Pathfrom typing import Any-from urllib.error import HTTPError, URLError-from urllib.parse import unquote, urlparse-from urllib.request import Request, urlopen+from urllib.parse import unquote, urljoin, urlparsefrom agent.component.base import ComponentBasefrom agent.component.llm import LLMParam@@ -42,6 +40,10 @@ from common.connection_utils import timeoutfrom common.misc_utils import get_uuidfrom rag.llm import FACTORY_DEFAULT_BASE_URL+# Mirrors MAX_IMAGE_REDIRECTS in rag/app/naive.py: cap the manually followed+# redirect hops when fetching URL-sourced upload files.+MAX_UPLOAD_URL_REDIRECTS = 5+class BrowserParam(LLMParam):"""@@ -171,6 +173,26 @@ class Browser(ComponentBase, ABC):parsed = urlparse(token)return parsed.scheme in {"http", "https"} and bool(parsed.netloc)+ @staticmethod+ def _safe_url_filename(candidate: Any) -> str:+ """Return a sanitized single path segment, or "" when none can be derived.++ Percent-escapes are decoded BEFORE splitting path segments: "%2e%2e%2f"+ is not a separator for os.path.basename(), but decodes to "../", which+ previously escaped the upload directory on join (CWE-22). Backslashes+ are normalized to forward slashes first so Windows-style traversal is+ split the same way while legitimate names survive.+ """+ token = str(candidate or "").strip()+ if not token:+ return ""+ token = os.path.basename(unquote(token).strip().replace("\\", "/")).strip()+ if not token or token in {".", ".."}:+ return ""+ if "/" in token or "\\" in token or os.sep in token or (os.altsep and os.altsep in token):+ return ""+ return token+@staticmethoddef _extract_url_filename(url: str, headers: Any) -> str:content_disposition = str(getattr(headers, "get", lambda *_args, **_kwargs: "")("Content-Disposition", "") or "")@@ -178,25 +200,25 @@ class Browser(ComponentBase, ABC):# Prefer RFC 5987 encoded filename*=UTF-8''... when present.m = re.search(r"filename\*\s*=\s*(?:UTF-8''|utf-8'')?([^;]+)", content_disposition)if m:- name = unquote(m.group(1).strip().strip('"'))+ name = Browser._safe_url_filename(m.group(1).strip().strip('"'))if name:- return os.path.basename(name)+ return namem = re.search(r'filename\s*=\s*"([^"]+)"', content_disposition)if m:- name = m.group(1).strip()+ name = Browser._safe_url_filename(m.group(1).strip())if name:- return os.path.basename(name)+ return namem = re.search(r"filename\s*=\s*([^;]+)", content_disposition)if m:- name = m.group(1).strip().strip('"')+ name = Browser._safe_url_filename(m.group(1).strip().strip('"'))if name:- return os.path.basename(name)+ return name- parsed = urlparse(url)- raw_name = os.path.basename(parsed.path or "")- name = unquote(raw_name).strip()+ name = Browser._safe_url_filename(urlparse(url).path or "")if name:return name+ # No safe name could be derived: fall back to a UUID with a fixed,+ # whitelisted suffix so the result can never leave upload_dir.return f"url_file_{get_uuid()[:8]}.bin"@staticmethod@@ -219,32 +241,73 @@ class Browser(ComponentBase, ABC):os.environ[key] = valuedef _prepare_upload_url_file(self, url: str, upload_dir: str) -> dict[str, Any] | None:+ import requests++ from common.ssrf_guard import assert_url_is_safe, pin_dns+max_bytes = self._resolve_upload_url_max_bytes()local_path = ""local_name = ""total_size = 0+ response = None+ # Isolated session: upload URLs are untrusted, so the fetch must not+ # honor ambient environment proxies (HTTP_PROXY/HTTPS_PROXY would let+ # the proxy resolve the hostname, bypassing the pinned IP) and must not+ # load .netrc credentials for the target.+ session = requests.Session()+ session.trust_env = Falsetry:- req = Request(url, headers={"User-Agent": "RAGFlow-Browser-Node/1.0"})- with urlopen(req, timeout=30) as response:- local_name = self._extract_url_filename(url, response.headers)-- local_path = os.path.join(upload_dir, local_name)- index = 1- while os.path.exists(local_path):- stem, ext = os.path.splitext(local_name)- local_path = os.path.join(upload_dir, f"{stem}_{index}{ext}")- index += 1-- with open(local_path, "wb") as f:- while True:- chunk = response.read(1024 * 1024)- if not chunk:- break- total_size += len(chunk)- if total_size > max_bytes:- raise ValueError(f"upload url file exceeds max size limit: {max_bytes}")- f.write(chunk)- except (HTTPError, URLError, OSError, TimeoutError, ValueError) as e:+ # SSRF guard: upload URLs can come from untrusted canvas variables,+ # so validate and DNS-pin every hop before connecting. Redirects are+ # followed manually so each hop is re-validated, mirroring+ # rag/app/naive.py load_images_from_urls. The thread-local pin_dns+ # (not pin_dns_global) matches this synchronous fetch path, which+ # resolves DNS in the calling thread.+ current_hostname, current_ip = assert_url_is_safe(url)+ current_url = url+ for _ in range(MAX_UPLOAD_URL_REDIRECTS + 1):+ # Release the previous hop before opening the next: with+ # stream=True the connection isn't returned to the pool until+ # the body is read or the response is closed.+ if response is not None:+ response.close()+ with pin_dns(current_hostname, current_ip):+ response = session.get(+ current_url,+ stream=True,+ timeout=30,+ allow_redirects=False,+ headers={"User-Agent": "RAGFlow-Browser-Node/1.0"},+ )+ if response.status_code not in (301, 302, 303, 307, 308):+ break+ location = response.headers.get("Location")+ if not location:+ break+ current_url = urljoin(current_url, location)+ current_hostname, current_ip = assert_url_is_safe(current_url)+ else:+ raise ValueError(f"Exceeded {MAX_UPLOAD_URL_REDIRECTS} redirects fetching {url!r}")+ response.raise_for_status()++ local_name = self._extract_url_filename(current_url, response.headers)++ local_path = os.path.join(upload_dir, local_name)+ index = 1+ while os.path.exists(local_path):+ stem, ext = os.path.splitext(local_name)+ local_path = os.path.join(upload_dir, f"{stem}_{index}{ext}")+ index += 1++ with open(local_path, "wb") as f:+ for chunk in response.iter_content(chunk_size=1024 * 1024):+ if not chunk:+ continue+ total_size += len(chunk)+ if total_size > max_bytes:+ raise ValueError(f"upload url file exceeds max size limit: {max_bytes}")+ f.write(chunk)+ except (requests.RequestException, OSError, TimeoutError, ValueError) as e:if local_path and os.path.exists(local_path):try:os.remove(local_path)@@ -252,6 +315,12 @@ class Browser(ComponentBase, ABC):passlogging.warning("Browser failed to fetch upload url. url=%s, error=%s", url, e)return None+ finally:+ # Always release the final/streamed response, including the+ # error paths where the body is never read.+ if response is not None:+ response.close()+ session.close()if total_size <= 0:if local_path and os.path.exists(local_path):diff --git a/test/unit_test/agent/component/test_browser_use_component.py b/test/unit_test/agent/component/test_browser_use_component.pyindex b2e6144f9a..f827baa669 100644--- a/test/unit_test/agent/component/test_browser_use_component.py+++ b/test/unit_test/agent/component/test_browser_use_component.py@@ -14,10 +14,12 @@# limitations under the License.#+import reimport sysimport typesfrom pathlib import Pathfrom types import SimpleNamespace+from urllib.parse import urlparsefrom api.db import FileType@@ -46,6 +48,29 @@ def _install_cv2_stub_if_unavailable():_install_cv2_stub_if_unavailable()++def _install_xgboost_stub_if_unavailable():+ # deepdoc/parser/pdf_parser.py imports xgboost unconditionally. Its macOS+ # wheels need libomp, which slim test environments may not have; none of+ # these tests touch the layout model, so a stub keeps them collectable.+ try:+ import xgboost # noqa: F401++ return+ except Exception:+ pass+ stub = types.ModuleType("xgboost")++ def _module_getattr(name):+ raise RuntimeError(f"xgboost runtime call ({name}) is unavailable in this test environment")++ stub.__getattr__ = _module_getattr+ sys.modules["xgboost"] = stub+++_install_xgboost_stub_if_unavailable()++from agent.component import browser as browser_use_module # noqa: E402@@ -149,32 +174,105 @@ def test_extract_ids_does_not_split_http_url_by_comma():assert refs == ["https://example.com/download?name=a,b.txt"]-def test_prepare_upload_files_supports_http_url(monkeypatch, tmp_path):- component = _build_component()- component._param.upload_sources = "https://example.com/files/demo.txt"+class _FakeRequestsResponse:+ def __init__(self, status_code=200, headers=None, data=b""):+ self.status_code = status_code+ self.headers = dict(headers or {})+ self._data = data+ self.closed = False++ def iter_content(self, chunk_size=1024 * 1024):+ for i in range(0, len(self._data), max(chunk_size, 1)):+ yield self._data[i : i + chunk_size]++ def raise_for_status(self):+ if self.status_code >= 400:+ raise AssertionError(f"unexpected HTTP status: {self.status_code}")++ def close(self):+ self.closed = True+++class _FakeRequestsSession:+ """Stands in for requests.Session: records calls and lifecycle state."""++ def __init__(self, handler):+ # Production must set trust_env = False right after construction.+ self.trust_env = True+ self.closed = False+ self.calls = []+ self._handler = handler++ def get(self, url, **kwargs):+ self.calls.append(url)+ return self._handler(url, **kwargs)++ def close(self):+ self.closed = True+- class _FakeResponse:- def __init__(self):- self.headers = {"Content-Disposition": 'attachment; filename="remote_demo.txt"'}- self._data = b"hello from url"- self._pos = 0-- def read(self, size=-1):- if size <= 0:- chunk = self._data[self._pos :]- self._pos = len(self._data)- return chunk- chunk = self._data[self._pos : self._pos + size]- self._pos += len(chunk)- return chunk+def _patch_requests_session(monkeypatch, handler):+ """Redirect requests.Session() to a recording fake; returns created sessions."""+ import requests++ created = []++ def _factory():+ session = _FakeRequestsSession(handler)+ created.append(session)+ return session++ monkeypatch.setattr(requests, "Session", _factory)+ return created+++def _record_dns_pins(monkeypatch):+ """Record pin_dns(host, ip) activations; exposes the active stack."""+ import common.ssrf_guard as ssrf++ active = []+ recorded = []++ class _pin:+ def __init__(self, host, ip):+ self._host, self._ip = host, ipdef __enter__(self):- return self+ active.append((self._host, self._ip))+ recorded.append((self._host, self._ip))- def __exit__(self, exc_type, exc_val, exc_tb):- return False+ def __exit__(self, *_exc):+ active.pop()- monkeypatch.setattr(browser_use_module, "urlopen", lambda *_args, **_kwargs: _FakeResponse())+ monkeypatch.setattr(ssrf, "pin_dns", _pin)+ return recorded, active+++def _allow_public_hosts(monkeypatch, allowed_substrings=("example.com", "example.net")):+ import common.ssrf_guard as ssrf++ def _fake_assert(url):+ host = urlparse(url).hostname or ""+ for marker in allowed_substrings:+ if marker in host:+ return (host, "93.184.216.34")+ raise ValueError(f"blocked in test: {url}")++ monkeypatch.setattr(ssrf, "assert_url_is_safe", _fake_assert)+++def test_prepare_upload_files_supports_http_url(monkeypatch, tmp_path):+ component = _build_component()+ component._param.upload_sources = "https://example.com/files/demo.txt"++ _allow_public_hosts(monkeypatch)+ _patch_requests_session(+ monkeypatch,+ lambda _url, **_kwargs: _FakeRequestsResponse(+ headers={"Content-Disposition": 'attachment; filename="remote_demo.txt"'},+ data=b"hello from url",+ ),+ )prepared = component._prepare_upload_files(str(tmp_path))@@ -186,6 +284,228 @@ def test_prepare_upload_files_supports_http_url(monkeypatch, tmp_path):assert Path(prepared[0]["local_path"]).read_bytes() == b"hello from url"+def test_extract_url_filename_decodes_percent_escaped_traversal():+ # Regression (CWE-22): "%2e%2e%2f" is not a separator for os.path.basename()+ # before decoding, so basename-then-unquote left "../" sequences in the+ # name and the joined path escaped the upload directory.+ name = browser_use_module.Browser._extract_url_filename(+ "https://example.com/download/%2e%2e%2f..%2f..%2fetc%2fpasswd",+ headers={},+ )++ assert name == "passwd"+++def test_extract_url_filename_sanitizes_content_disposition_traversal():+ headers = {"Content-Disposition": "attachment; filename*=UTF-8''..%2f..%2fowned.txt"}++ name = browser_use_module.Browser._extract_url_filename("https://example.com/dl", headers=headers)++ assert name == "owned.txt"+++def test_extract_url_filename_falls_back_to_uuid_when_no_safe_name():+ # The whole path decodes to a traversal segment, so no safe name remains.+ name = browser_use_module.Browser._extract_url_filename("https://example.com/%2e%2e%2f", headers={})++ assert re.fullmatch(r"url_file_[0-9a-f]{8}\.bin", name)+++def test_prepare_upload_url_file_keeps_percent_traversal_inside_upload_dir(monkeypatch, tmp_path):+ component = _build_component()++ _allow_public_hosts(monkeypatch)+ _patch_requests_session(monkeypatch, lambda _url, **_kwargs: _FakeRequestsResponse(data=b"payload"))++ prepared = component._prepare_upload_url_file("https://example.com/files/%2e%2e%2f..%2fvictim.txt", str(tmp_path))++ assert prepared is not None+ local_path = Path(prepared["local_path"])+ assert local_path.parent == tmp_path+ assert local_path.name == "victim.txt"+ assert local_path.read_bytes() == b"payload"+++def test_prepare_upload_url_file_rejects_private_and_metadata_urls(monkeypatch, tmp_path):+ # Uses the real assert_url_is_safe (no monkeypatch): loopback, link-local+ # metadata and RFC1918 hosts must be refused before any connection.+ component = _build_component()+ monkeypatch.delenv("ALLOW_ANY_HOST", raising=False)++ connections = []+ sessions = _patch_requests_session(monkeypatch, lambda url, **_kwargs: connections.append(url))++ for url in (+ "http://127.0.0.1:8080/admin",+ "http://169.254.169.254/latest/meta-data/",+ "http://10.1.2.3/internal.txt",+ "http://[::1]/x.txt",+ ):+ assert component._prepare_upload_url_file(url, str(tmp_path)) is None++ assert connections == []+ assert list(tmp_path.iterdir()) == []+ # The isolated session is still constructed and released on each attempt.+ assert all(session.closed for session in sessions)+++def test_prepare_upload_url_file_rejects_redirect_to_intranet(monkeypatch, tmp_path):+ # First hop is a public URL answering 302 -> cloud metadata: the redirect+ # target must be re-validated and rejected before connecting to it.+ component = _build_component()++ _allow_public_hosts(monkeypatch)+ connections = []++ def _redirect_to_metadata(url, **_kwargs):+ connections.append(url)+ return _FakeRequestsResponse(+ status_code=302,+ headers={"Location": "http://169.254.169.254/latest/meta-data/iam/security-credentials/"},+ )++ sessions = _patch_requests_session(monkeypatch, _redirect_to_metadata)++ prepared = component._prepare_upload_url_file("https://example.com/file.bin", str(tmp_path))++ assert prepared is None+ assert connections == ["https://example.com/file.bin"]+ assert list(tmp_path.iterdir()) == []+ # The rejected fetch still releases its isolated session.+ assert sessions[0].closed is True+++def test_prepare_upload_url_file_follows_safe_redirects(monkeypatch, tmp_path):+ component = _build_component()++ _allow_public_hosts(monkeypatch)+ responses = []++ def _get(url, **_kwargs):+ if url == "https://example.com/download":+ response = _FakeRequestsResponse(status_code=301, headers={"Location": "https://cdn.example.net/final/report.pdf"})+ elif url == "https://cdn.example.net/final/report.pdf":+ response = _FakeRequestsResponse(data=b"PDF")+ else:+ raise AssertionError(f"unexpected url: {url}")+ responses.append(response)+ return response++ sessions = _patch_requests_session(monkeypatch, _get)++ prepared = component._prepare_upload_url_file("https://example.com/download", str(tmp_path))++ assert prepared is not None+ assert prepared["name"] == "report.pdf"+ assert prepared["source_url"] == "https://example.com/download"+ assert Path(prepared["local_path"]).read_bytes() == b"PDF"+ # The isolated session ignores ambient proxy/netrc config and is closed,+ # and every hop's response (including intermediate redirects) is closed.+ assert sessions[0].trust_env is False+ assert sessions[0].closed is True+ assert [r.closed for r in responses] == [True, True]+++def test_prepare_upload_url_file_redirect_limit_boundary(monkeypatch, tmp_path):+ # MAX_UPLOAD_URL_REDIRECTS is 5: five hops to the final 200 are accepted,+ # a chain that needs a sixth redirect is refused.+ component = _build_component()+ _allow_public_hosts(monkeypatch)++ def _chain(hop_count):+ def _get(url, **_kwargs):+ hop = int(url.rsplit("/", 1)[-1])+ if hop < hop_count:+ return _FakeRequestsResponse(status_code=302, headers={"Location": f"https://example.com/hop/{hop + 1}"})+ return _FakeRequestsResponse(data=b"ok")++ return _get++ _patch_requests_session(monkeypatch, _chain(5))+ prepared = component._prepare_upload_url_file("https://example.com/hop/0", str(tmp_path))+ assert prepared is not None+ assert Path(prepared["local_path"]).read_bytes() == b"ok"++ _patch_requests_session(monkeypatch, _chain(6))+ prepared = component._prepare_upload_url_file("https://example.com/hop/0", str(tmp_path / "sub"))+ assert prepared is None+++def test_prepare_upload_url_file_cleans_partial_file_on_size_limit(monkeypatch, tmp_path):+ component = _build_component()+ monkeypatch.setattr(browser_use_module.Browser, "_resolve_upload_url_max_bytes", lambda _self: 4)+ _allow_public_hosts(monkeypatch)+ _patch_requests_session(monkeypatch, lambda _url, **_kwargs: _FakeRequestsResponse(data=b"0123456789"))++ prepared = component._prepare_upload_url_file("https://example.com/big.bin", str(tmp_path))++ assert prepared is None+ assert list(tmp_path.iterdir()) == []+++def test_prepare_upload_url_file_cleans_partial_file_when_stream_fails(monkeypatch, tmp_path):+ import requests++ component = _build_component()+ _allow_public_hosts(monkeypatch)++ class _ExplodingResponse(_FakeRequestsResponse):+ def iter_content(self, chunk_size=1024 * 1024):+ yield b"partial-"+ raise requests.RequestException("connection reset mid-stream")++ _patch_requests_session(monkeypatch, lambda _url, **_kwargs: _ExplodingResponse(data=b"partial-"))++ prepared = component._prepare_upload_url_file("https://example.com/dies.bin", str(tmp_path))++ assert prepared is None+ assert list(tmp_path.iterdir()) == []+++def test_prepare_upload_url_file_pins_dns_for_every_hop(monkeypatch, tmp_path):+ component = _build_component()++ import common.ssrf_guard as ssrf++ def _assert(url):+ host = urlparse(url).hostname or ""+ return (host, {"example.com": "93.184.216.34", "cdn.example.net": "198.51.100.7"}[host])++ monkeypatch.setattr(ssrf, "assert_url_is_safe", _assert)+ recorded, active = _record_dns_pins(monkeypatch)++ def _get(url, **_kwargs):+ if url == "https://example.com/download":+ assert active[-1] == ("example.com", "93.184.216.34"), active+ return _FakeRequestsResponse(status_code=302, headers={"Location": "https://cdn.example.net/report.pdf"})+ assert active[-1] == ("cdn.example.net", "198.51.100.7"), active+ return _FakeRequestsResponse(data=b"ok")++ _patch_requests_session(monkeypatch, _get)++ prepared = component._prepare_upload_url_file("https://example.com/download", str(tmp_path))++ assert prepared is not None+ assert recorded == [+ ("example.com", "93.184.216.34"),+ ("cdn.example.net", "198.51.100.7"),+ ]+ # No pin leaks past the fetch: the stack unwinds to empty.+ assert active == []+++def test_extract_url_filename_sanitizes_all_content_disposition_forms():+ cases = [+ ("attachment; filename*=UTF-8''..%2f..%2fstar.txt", "star.txt"),+ ('attachment; filename="..\\..\\quoted.txt"', "quoted.txt"),+ ("attachment; filename=../../unquoted.txt", "unquoted.txt"),+ ('attachment; filename="..%2F..%2Fmixed.txt"', "mixed.txt"),+ ]+ for header, expected in cases:+ name = browser_use_module.Browser._extract_url_filename("https://example.com/dl", headers={"Content-Disposition": header})+ assert name == expected, (header, name)++def test_save_downloads_persists_file_records(monkeypatch, tmp_path):component = _build_component()component._canvas = _FakeCanvas()