Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: isolate extract html request headers (#18322)

ump45nose15aad785183f
diff --git a/rag/utils/file_utils.py b/rag/utils/file_utils.py
index 8a1006fbf4..ef3617dbb4 100644
--- a/rag/utils/file_utils.py
+++ b/rag/utils/file_utils.py
@@ -202,14 +202,12 @@ def extract_links_from_pdf(pdf_bytes: bytes):
_GLOBAL_SESSION: Optional[requests.Session] = None
-def _get_session(headers: Optional[Dict[str, str]] = None) -> requests.Session:
+def _get_session() -> requests.Session:
"""Get or create a global reusable session."""
global _GLOBAL_SESSION
if _GLOBAL_SESSION is None:
_GLOBAL_SESSION = requests.Session()
_GLOBAL_SESSION.headers.update({"User-Agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0 Safari/537.36")})
- if headers:
- _GLOBAL_SESSION.headers.update(headers)
return _GLOBAL_SESSION
@@ -234,12 +232,12 @@ def extract_html(
- html_bytes: Raw HTML content (or None if failed)
- metadata: HTTP info (status_code, content_type, final_url, error if any)
"""
- session = _get_session(headers=headers)
+ session = _get_session()
metadata = {"final_url": url, "status_code": "", "content_type": "", "error": ""}
for attempt in range(1, max_retries + 1):
try:
- resp = session.get(url, timeout=timeout)
+ resp = session.get(url, timeout=timeout, headers=headers)
resp.raise_for_status()
html_bytes = resp.content
diff --git a/test/unit_test/rag/utils/test_extract_html_headers.py b/test/unit_test/rag/utils/test_extract_html_headers.py
new file mode 100644
index 0000000000..41a1def8b5
--- /dev/null
+++ b/test/unit_test/rag/utils/test_extract_html_headers.py
@@ -0,0 +1,46 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from rag.utils import file_utils
+
+
+class _Response:
+ content = b"<html />"
+ url = "https://example.test/"
+ status_code = 200
+
+ def __init__(self):
+ self.headers = {"Content-Type": "text/html"}
+
+ def raise_for_status(self):
+ pass
+
+
+def test_extract_html_does_not_reuse_request_headers(monkeypatch):
+ monkeypatch.setattr(file_utils, "_GLOBAL_SESSION", None)
+ request_headers = []
+
+ def fake_get(self, url, timeout, headers=None):
+ request_headers.append({**self.headers, **(headers or {})})
+ return _Response()
+
+ monkeypatch.setattr(file_utils.requests.Session, "get", fake_get)
+
+ file_utils.extract_html("https://example.test/one", headers={"Authorization": "Bearer request-one"})
+ file_utils.extract_html("https://example.test/two")
+
+ assert request_headers[0]["Authorization"] == "Bearer request-one"
+ assert "Authorization" not in request_headers[1]