Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): prevent path traversal and SSRF in Browser URL file upload (#18711)

陈志谦4870e83aa7a6
diff --git a/agent/component/browser.py b/agent/component/browser.py
index 67116ed781..276b426ccd 100644
--- a/agent/component/browser.py
+++ b/agent/component/browser.py
@@ -26,9 +26,7 @@ import tempfile
from abc import ABC
from pathlib import Path
from typing import Any
-from urllib.error import HTTPError, URLError
-from urllib.parse import unquote, urlparse
-from urllib.request import Request, urlopen
+from urllib.parse import unquote, urljoin, urlparse
from agent.component.base import ComponentBase
from agent.component.llm import LLMParam
@@ -42,6 +40,10 @@ from common.connection_utils import timeout
from common.misc_utils import get_uuid
from rag.llm import FACTORY_DEFAULT_BASE_URL
+# Mirrors MAX_IMAGE_REDIRECTS in rag/app/naive.py: cap the manually followed
+# redirect hops when fetching URL-sourced upload files.
+MAX_UPLOAD_URL_REDIRECTS = 5
+
class BrowserParam(LLMParam):
"""
@@ -171,6 +173,26 @@ class Browser(ComponentBase, ABC):
parsed = urlparse(token)
return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+ @staticmethod
+ def _safe_url_filename(candidate: Any) -> str:
+ """Return a sanitized single path segment, or "" when none can be derived.
+
+ Percent-escapes are decoded BEFORE splitting path segments: "%2e%2e%2f"
+ is not a separator for os.path.basename(), but decodes to "../", which
+ previously escaped the upload directory on join (CWE-22). Backslashes
+ are normalized to forward slashes first so Windows-style traversal is
+ split the same way while legitimate names survive.
+ """
+ token = str(candidate or "").strip()
+ if not token:
+ return ""
+ token = os.path.basename(unquote(token).strip().replace("\\", "/")).strip()
+ if not token or token in {".", ".."}:
+ return ""
+ if "/" in token or "\\" in token or os.sep in token or (os.altsep and os.altsep in token):
+ return ""
+ return token
+
@staticmethod
def _extract_url_filename(url: str, headers: Any) -> str:
content_disposition = str(getattr(headers, "get", lambda *_args, **_kwargs: "")("Content-Disposition", "") or "")
@@ -178,25 +200,25 @@ class Browser(ComponentBase, ABC):
# Prefer RFC 5987 encoded filename*=UTF-8''... when present.
m = re.search(r"filename\*\s*=\s*(?:UTF-8''|utf-8'')?([^;]+)", content_disposition)
if m:
- name = unquote(m.group(1).strip().strip('"'))
+ name = Browser._safe_url_filename(m.group(1).strip().strip('"'))
if name:
- return os.path.basename(name)
+ return name
m = re.search(r'filename\s*=\s*"([^"]+)"', content_disposition)
if m:
- name = m.group(1).strip()
+ name = Browser._safe_url_filename(m.group(1).strip())
if name:
- return os.path.basename(name)
+ return name
m = re.search(r"filename\s*=\s*([^;]+)", content_disposition)
if m:
- name = m.group(1).strip().strip('"')
+ name = Browser._safe_url_filename(m.group(1).strip().strip('"'))
if name:
- return os.path.basename(name)
+ return name
- parsed = urlparse(url)
- raw_name = os.path.basename(parsed.path or "")
- name = unquote(raw_name).strip()
+ name = Browser._safe_url_filename(urlparse(url).path or "")
if name:
return name
+ # No safe name could be derived: fall back to a UUID with a fixed,
+ # whitelisted suffix so the result can never leave upload_dir.
return f"url_file_{get_uuid()[:8]}.bin"
@staticmethod
@@ -219,32 +241,73 @@ class Browser(ComponentBase, ABC):
os.environ[key] = value
def _prepare_upload_url_file(self, url: str, upload_dir: str) -> dict[str, Any] | None:
+ import requests
+
+ from common.ssrf_guard import assert_url_is_safe, pin_dns
+
max_bytes = self._resolve_upload_url_max_bytes()
local_path = ""
local_name = ""
total_size = 0
+ response = None
+ # Isolated session: upload URLs are untrusted, so the fetch must not
+ # honor ambient environment proxies (HTTP_PROXY/HTTPS_PROXY would let
+ # the proxy resolve the hostname, bypassing the pinned IP) and must not
+ # load .netrc credentials for the target.
+ session = requests.Session()
+ session.trust_env = False
try:
- req = Request(url, headers={"User-Agent": "RAGFlow-Browser-Node/1.0"})
- with urlopen(req, timeout=30) as response:
- local_name = self._extract_url_filename(url, response.headers)
-
- local_path = os.path.join(upload_dir, local_name)
- index = 1
- while os.path.exists(local_path):
- stem, ext = os.path.splitext(local_name)
- local_path = os.path.join(upload_dir, f"{stem}_{index}{ext}")
- index += 1
-
- with open(local_path, "wb") as f:
- while True:
- chunk = response.read(1024 * 1024)
- if not chunk:
- break
- total_size += len(chunk)
- if total_size > max_bytes:
- raise ValueError(f"upload url file exceeds max size limit: {max_bytes}")
- f.write(chunk)
- except (HTTPError, URLError, OSError, TimeoutError, ValueError) as e:
+ # SSRF guard: upload URLs can come from untrusted canvas variables,
+ # so validate and DNS-pin every hop before connecting. Redirects are
+ # followed manually so each hop is re-validated, mirroring
+ # rag/app/naive.py load_images_from_urls. The thread-local pin_dns
+ # (not pin_dns_global) matches this synchronous fetch path, which
+ # resolves DNS in the calling thread.
+ current_hostname, current_ip = assert_url_is_safe(url)
+ current_url = url
+ for _ in range(MAX_UPLOAD_URL_REDIRECTS + 1):
+ # Release the previous hop before opening the next: with
+ # stream=True the connection isn't returned to the pool until
+ # the body is read or the response is closed.
+ if response is not None:
+ response.close()
+ with pin_dns(current_hostname, current_ip):
+ response = session.get(
+ current_url,
+ stream=True,
+ timeout=30,
+ allow_redirects=False,
+ headers={"User-Agent": "RAGFlow-Browser-Node/1.0"},
+ )
+ if response.status_code not in (301, 302, 303, 307, 308):
+ break
+ location = response.headers.get("Location")
+ if not location:
+ break
+ current_url = urljoin(current_url, location)
+ current_hostname, current_ip = assert_url_is_safe(current_url)
+ else:
+ raise ValueError(f"Exceeded {MAX_UPLOAD_URL_REDIRECTS} redirects fetching {url!r}")
+ response.raise_for_status()
+
+ local_name = self._extract_url_filename(current_url, response.headers)
+
+ local_path = os.path.join(upload_dir, local_name)
+ index = 1
+ while os.path.exists(local_path):
+ stem, ext = os.path.splitext(local_name)
+ local_path = os.path.join(upload_dir, f"{stem}_{index}{ext}")
+ index += 1
+
+ with open(local_path, "wb") as f:
+ for chunk in response.iter_content(chunk_size=1024 * 1024):
+ if not chunk:
+ continue
+ total_size += len(chunk)
+ if total_size > max_bytes:
+ raise ValueError(f"upload url file exceeds max size limit: {max_bytes}")
+ f.write(chunk)
+ except (requests.RequestException, OSError, TimeoutError, ValueError) as e:
if local_path and os.path.exists(local_path):
try:
os.remove(local_path)
@@ -252,6 +315,12 @@ class Browser(ComponentBase, ABC):
pass
logging.warning("Browser failed to fetch upload url. url=%s, error=%s", url, e)
return None
+ finally:
+ # Always release the final/streamed response, including the
+ # error paths where the body is never read.
+ if response is not None:
+ response.close()
+ session.close()
if total_size <= 0:
if local_path and os.path.exists(local_path):
diff --git a/test/unit_test/agent/component/test_browser_use_component.py b/test/unit_test/agent/component/test_browser_use_component.py
index b2e6144f9a..f827baa669 100644
--- a/test/unit_test/agent/component/test_browser_use_component.py
+++ b/test/unit_test/agent/component/test_browser_use_component.py
@@ -14,10 +14,12 @@
# limitations under the License.
#
+import re
import sys
import types
from pathlib import Path
from types import SimpleNamespace
+from urllib.parse import urlparse
from api.db import FileType
@@ -46,6 +48,29 @@ def _install_cv2_stub_if_unavailable():
_install_cv2_stub_if_unavailable()
+
+def _install_xgboost_stub_if_unavailable():
+ # deepdoc/parser/pdf_parser.py imports xgboost unconditionally. Its macOS
+ # wheels need libomp, which slim test environments may not have; none of
+ # these tests touch the layout model, so a stub keeps them collectable.
+ try:
+ import xgboost # noqa: F401
+
+ return
+ except Exception:
+ pass
+ stub = types.ModuleType("xgboost")
+
+ def _module_getattr(name):
+ raise RuntimeError(f"xgboost runtime call ({name}) is unavailable in this test environment")
+
+ stub.__getattr__ = _module_getattr
+ sys.modules["xgboost"] = stub
+
+
+_install_xgboost_stub_if_unavailable()
+
+
from agent.component import browser as browser_use_module # noqa: E402
@@ -149,32 +174,105 @@ def test_extract_ids_does_not_split_http_url_by_comma():
assert refs == ["https://example.com/download?name=a,b.txt"]
-def test_prepare_upload_files_supports_http_url(monkeypatch, tmp_path):
- component = _build_component()
- component._param.upload_sources = "https://example.com/files/demo.txt"
+class _FakeRequestsResponse:
+ def __init__(self, status_code=200, headers=None, data=b""):
+ self.status_code = status_code
+ self.headers = dict(headers or {})
+ self._data = data
+ self.closed = False
+
+ def iter_content(self, chunk_size=1024 * 1024):
+ for i in range(0, len(self._data), max(chunk_size, 1)):
+ yield self._data[i : i + chunk_size]
+
+ def raise_for_status(self):
+ if self.status_code >= 400:
+ raise AssertionError(f"unexpected HTTP status: {self.status_code}")
+
+ def close(self):
+ self.closed = True
+
+
+class _FakeRequestsSession:
+ """Stands in for requests.Session: records calls and lifecycle state."""
+
+ def __init__(self, handler):
+ # Production must set trust_env = False right after construction.
+ self.trust_env = True
+ self.closed = False
+ self.calls = []
+ self._handler = handler
+
+ def get(self, url, **kwargs):
+ self.calls.append(url)
+ return self._handler(url, **kwargs)
+
+ def close(self):
+ self.closed = True
+
- class _FakeResponse:
- def __init__(self):
- self.headers = {"Content-Disposition": 'attachment; filename="remote_demo.txt"'}
- self._data = b"hello from url"
- self._pos = 0
-
- def read(self, size=-1):
- if size <= 0:
- chunk = self._data[self._pos :]
- self._pos = len(self._data)
- return chunk
- chunk = self._data[self._pos : self._pos + size]
- self._pos += len(chunk)
- return chunk
+def _patch_requests_session(monkeypatch, handler):
+ """Redirect requests.Session() to a recording fake; returns created sessions."""
+ import requests
+
+ created = []
+
+ def _factory():
+ session = _FakeRequestsSession(handler)
+ created.append(session)
+ return session
+
+ monkeypatch.setattr(requests, "Session", _factory)
+ return created
+
+
+def _record_dns_pins(monkeypatch):
+ """Record pin_dns(host, ip) activations; exposes the active stack."""
+ import common.ssrf_guard as ssrf
+
+ active = []
+ recorded = []
+
+ class _pin:
+ def __init__(self, host, ip):
+ self._host, self._ip = host, ip
def __enter__(self):
- return self
+ active.append((self._host, self._ip))
+ recorded.append((self._host, self._ip))
- def __exit__(self, exc_type, exc_val, exc_tb):
- return False
+ def __exit__(self, *_exc):
+ active.pop()
- monkeypatch.setattr(browser_use_module, "urlopen", lambda *_args, **_kwargs: _FakeResponse())
+ monkeypatch.setattr(ssrf, "pin_dns", _pin)
+ return recorded, active
+
+
+def _allow_public_hosts(monkeypatch, allowed_substrings=("example.com", "example.net")):
+ import common.ssrf_guard as ssrf
+
+ def _fake_assert(url):
+ host = urlparse(url).hostname or ""
+ for marker in allowed_substrings:
+ if marker in host:
+ return (host, "93.184.216.34")
+ raise ValueError(f"blocked in test: {url}")
+
+ monkeypatch.setattr(ssrf, "assert_url_is_safe", _fake_assert)
+
+
+def test_prepare_upload_files_supports_http_url(monkeypatch, tmp_path):
+ component = _build_component()
+ component._param.upload_sources = "https://example.com/files/demo.txt"
+
+ _allow_public_hosts(monkeypatch)
+ _patch_requests_session(
+ monkeypatch,
+ lambda _url, **_kwargs: _FakeRequestsResponse(
+ headers={"Content-Disposition": 'attachment; filename="remote_demo.txt"'},
+ data=b"hello from url",
+ ),
+ )
prepared = component._prepare_upload_files(str(tmp_path))
@@ -186,6 +284,228 @@ def test_prepare_upload_files_supports_http_url(monkeypatch, tmp_path):
assert Path(prepared[0]["local_path"]).read_bytes() == b"hello from url"
+def test_extract_url_filename_decodes_percent_escaped_traversal():
+ # Regression (CWE-22): "%2e%2e%2f" is not a separator for os.path.basename()
+ # before decoding, so basename-then-unquote left "../" sequences in the
+ # name and the joined path escaped the upload directory.
+ name = browser_use_module.Browser._extract_url_filename(
+ "https://example.com/download/%2e%2e%2f..%2f..%2fetc%2fpasswd",
+ headers={},
+ )
+
+ assert name == "passwd"
+
+
+def test_extract_url_filename_sanitizes_content_disposition_traversal():
+ headers = {"Content-Disposition": "attachment; filename*=UTF-8''..%2f..%2fowned.txt"}
+
+ name = browser_use_module.Browser._extract_url_filename("https://example.com/dl", headers=headers)
+
+ assert name == "owned.txt"
+
+
+def test_extract_url_filename_falls_back_to_uuid_when_no_safe_name():
+ # The whole path decodes to a traversal segment, so no safe name remains.
+ name = browser_use_module.Browser._extract_url_filename("https://example.com/%2e%2e%2f", headers={})
+
+ assert re.fullmatch(r"url_file_[0-9a-f]{8}\.bin", name)
+
+
+def test_prepare_upload_url_file_keeps_percent_traversal_inside_upload_dir(monkeypatch, tmp_path):
+ component = _build_component()
+
+ _allow_public_hosts(monkeypatch)
+ _patch_requests_session(monkeypatch, lambda _url, **_kwargs: _FakeRequestsResponse(data=b"payload"))
+
+ prepared = component._prepare_upload_url_file("https://example.com/files/%2e%2e%2f..%2fvictim.txt", str(tmp_path))
+
+ assert prepared is not None
+ local_path = Path(prepared["local_path"])
+ assert local_path.parent == tmp_path
+ assert local_path.name == "victim.txt"
+ assert local_path.read_bytes() == b"payload"
+
+
+def test_prepare_upload_url_file_rejects_private_and_metadata_urls(monkeypatch, tmp_path):
+ # Uses the real assert_url_is_safe (no monkeypatch): loopback, link-local
+ # metadata and RFC1918 hosts must be refused before any connection.
+ component = _build_component()
+ monkeypatch.delenv("ALLOW_ANY_HOST", raising=False)
+
+ connections = []
+ sessions = _patch_requests_session(monkeypatch, lambda url, **_kwargs: connections.append(url))
+
+ for url in (
+ "http://127.0.0.1:8080/admin",
+ "http://169.254.169.254/latest/meta-data/",
+ "http://10.1.2.3/internal.txt",
+ "http://[::1]/x.txt",
+ ):
+ assert component._prepare_upload_url_file(url, str(tmp_path)) is None
+
+ assert connections == []
+ assert list(tmp_path.iterdir()) == []
+ # The isolated session is still constructed and released on each attempt.
+ assert all(session.closed for session in sessions)
+
+
+def test_prepare_upload_url_file_rejects_redirect_to_intranet(monkeypatch, tmp_path):
+ # First hop is a public URL answering 302 -> cloud metadata: the redirect
+ # target must be re-validated and rejected before connecting to it.
+ component = _build_component()
+
+ _allow_public_hosts(monkeypatch)
+ connections = []
+
+ def _redirect_to_metadata(url, **_kwargs):
+ connections.append(url)
+ return _FakeRequestsResponse(
+ status_code=302,
+ headers={"Location": "http://169.254.169.254/latest/meta-data/iam/security-credentials/"},
+ )
+
+ sessions = _patch_requests_session(monkeypatch, _redirect_to_metadata)
+
+ prepared = component._prepare_upload_url_file("https://example.com/file.bin", str(tmp_path))
+
+ assert prepared is None
+ assert connections == ["https://example.com/file.bin"]
+ assert list(tmp_path.iterdir()) == []
+ # The rejected fetch still releases its isolated session.
+ assert sessions[0].closed is True
+
+
+def test_prepare_upload_url_file_follows_safe_redirects(monkeypatch, tmp_path):
+ component = _build_component()
+
+ _allow_public_hosts(monkeypatch)
+ responses = []
+
+ def _get(url, **_kwargs):
+ if url == "https://example.com/download":
+ response = _FakeRequestsResponse(status_code=301, headers={"Location": "https://cdn.example.net/final/report.pdf"})
+ elif url == "https://cdn.example.net/final/report.pdf":
+ response = _FakeRequestsResponse(data=b"PDF")
+ else:
+ raise AssertionError(f"unexpected url: {url}")
+ responses.append(response)
+ return response
+
+ sessions = _patch_requests_session(monkeypatch, _get)
+
+ prepared = component._prepare_upload_url_file("https://example.com/download", str(tmp_path))
+
+ assert prepared is not None
+ assert prepared["name"] == "report.pdf"
+ assert prepared["source_url"] == "https://example.com/download"
+ assert Path(prepared["local_path"]).read_bytes() == b"PDF"
+ # The isolated session ignores ambient proxy/netrc config and is closed,
+ # and every hop's response (including intermediate redirects) is closed.
+ assert sessions[0].trust_env is False
+ assert sessions[0].closed is True
+ assert [r.closed for r in responses] == [True, True]
+
+
+def test_prepare_upload_url_file_redirect_limit_boundary(monkeypatch, tmp_path):
+ # MAX_UPLOAD_URL_REDIRECTS is 5: five hops to the final 200 are accepted,
+ # a chain that needs a sixth redirect is refused.
+ component = _build_component()
+ _allow_public_hosts(monkeypatch)
+
+ def _chain(hop_count):
+ def _get(url, **_kwargs):
+ hop = int(url.rsplit("/", 1)[-1])
+ if hop < hop_count:
+ return _FakeRequestsResponse(status_code=302, headers={"Location": f"https://example.com/hop/{hop + 1}"})
+ return _FakeRequestsResponse(data=b"ok")
+
+ return _get
+
+ _patch_requests_session(monkeypatch, _chain(5))
+ prepared = component._prepare_upload_url_file("https://example.com/hop/0", str(tmp_path))
+ assert prepared is not None
+ assert Path(prepared["local_path"]).read_bytes() == b"ok"
+
+ _patch_requests_session(monkeypatch, _chain(6))
+ prepared = component._prepare_upload_url_file("https://example.com/hop/0", str(tmp_path / "sub"))
+ assert prepared is None
+
+
+def test_prepare_upload_url_file_cleans_partial_file_on_size_limit(monkeypatch, tmp_path):
+ component = _build_component()
+ monkeypatch.setattr(browser_use_module.Browser, "_resolve_upload_url_max_bytes", lambda _self: 4)
+ _allow_public_hosts(monkeypatch)
+ _patch_requests_session(monkeypatch, lambda _url, **_kwargs: _FakeRequestsResponse(data=b"0123456789"))
+
+ prepared = component._prepare_upload_url_file("https://example.com/big.bin", str(tmp_path))
+
+ assert prepared is None
+ assert list(tmp_path.iterdir()) == []
+
+
+def test_prepare_upload_url_file_cleans_partial_file_when_stream_fails(monkeypatch, tmp_path):
+ import requests
+
+ component = _build_component()
+ _allow_public_hosts(monkeypatch)
+
+ class _ExplodingResponse(_FakeRequestsResponse):
+ def iter_content(self, chunk_size=1024 * 1024):
+ yield b"partial-"
+ raise requests.RequestException("connection reset mid-stream")
+
+ _patch_requests_session(monkeypatch, lambda _url, **_kwargs: _ExplodingResponse(data=b"partial-"))
+
+ prepared = component._prepare_upload_url_file("https://example.com/dies.bin", str(tmp_path))
+
+ assert prepared is None
+ assert list(tmp_path.iterdir()) == []
+
+
+def test_prepare_upload_url_file_pins_dns_for_every_hop(monkeypatch, tmp_path):
+ component = _build_component()
+
+ import common.ssrf_guard as ssrf
+
+ def _assert(url):
+ host = urlparse(url).hostname or ""
+ return (host, {"example.com": "93.184.216.34", "cdn.example.net": "198.51.100.7"}[host])
+
+ monkeypatch.setattr(ssrf, "assert_url_is_safe", _assert)
+ recorded, active = _record_dns_pins(monkeypatch)
+
+ def _get(url, **_kwargs):
+ if url == "https://example.com/download":
+ assert active[-1] == ("example.com", "93.184.216.34"), active
+ return _FakeRequestsResponse(status_code=302, headers={"Location": "https://cdn.example.net/report.pdf"})
+ assert active[-1] == ("cdn.example.net", "198.51.100.7"), active
+ return _FakeRequestsResponse(data=b"ok")
+
+ _patch_requests_session(monkeypatch, _get)
+
+ prepared = component._prepare_upload_url_file("https://example.com/download", str(tmp_path))
+
+ assert prepared is not None
+ assert recorded == [
+ ("example.com", "93.184.216.34"),
+ ("cdn.example.net", "198.51.100.7"),
+ ]
+ # No pin leaks past the fetch: the stack unwinds to empty.
+ assert active == []
+
+
+def test_extract_url_filename_sanitizes_all_content_disposition_forms():
+ cases = [
+ ("attachment; filename*=UTF-8''..%2f..%2fstar.txt", "star.txt"),
+ ('attachment; filename="..\\..\\quoted.txt"', "quoted.txt"),
+ ("attachment; filename=../../unquoted.txt", "unquoted.txt"),
+ ('attachment; filename="..%2F..%2Fmixed.txt"', "mixed.txt"),
+ ]
+ for header, expected in cases:
+ name = browser_use_module.Browser._extract_url_filename("https://example.com/dl", headers={"Content-Disposition": header})
+ assert name == expected, (header, name)
+
+
def test_save_downloads_persists_file_records(monkeypatch, tmp_path):
component = _build_component()
component._canvas = _FakeCanvas()