Skip to content

Personal AI, On Personal Devices

Python

fix(tools): separate Firecrawl Basic Auth from logged URLs

Elliot Slusky9c16dc74def4
diff --git a/src/openjarvis/tools/web_search.py b/src/openjarvis/tools/web_search.py
index f7a0e3c6..0e69d25e 100644
--- a/src/openjarvis/tools/web_search.py
+++ b/src/openjarvis/tools/web_search.py
@@ -484,6 +484,20 @@ class WebSearchTool(BaseTool):
headers["Authorization"] = f"Bearer {self._firecrawl_api_key}"
return headers
+ def _firecrawl_endpoint(self, operation: str) -> tuple[str, tuple[str, str] | None]:
+ """Keep self-hosted Basic Auth out of URLs exposed by HTTPX logging."""
+ import httpx
+
+ url = httpx.URL(f"{self._firecrawl_api_url}/v2/{operation}")
+ auth = None
+ if url.username or url.password:
+ # HTTPX already uses URL userinfo as Basic Auth, overriding a
+ # Bearer header when both are present. Preserve that behavior while
+ # ensuring its INFO request log contains only the endpoint URL.
+ auth = (url.username, url.password)
+ url = url.copy_with(username=None, password=None)
+ return str(url), auth
+
@staticmethod
def _format_firecrawl_results(payload: dict[str, Any]) -> tuple[str, int]:
"""Render a Firecrawl search payload in the shared result format.
@@ -515,14 +529,16 @@ class WebSearchTool(BaseTool):
if not self._firecrawl_api_key and not self._firecrawl_self_hosted:
raise _WebSearchEngineError(f"{FIRECRAWL_API_KEY_ENV} is not set")
try:
+ endpoint, auth = self._firecrawl_endpoint("search")
response = httpx.post(
- f"{self._firecrawl_api_url}/v2/search",
+ endpoint,
json={
"query": query,
"limit": max_results,
"origin": FIRECRAWL_ORIGIN,
},
headers=self._firecrawl_headers(),
+ auth=auth,
timeout=30.0,
)
response.raise_for_status()
@@ -580,8 +596,9 @@ class WebSearchTool(BaseTool):
import httpx
try:
+ endpoint, auth = self._firecrawl_endpoint("scrape")
response = httpx.post(
- f"{self._firecrawl_api_url}/v2/scrape",
+ endpoint,
json={
"url": url,
"formats": ["markdown"],
@@ -591,6 +608,7 @@ class WebSearchTool(BaseTool):
"origin": FIRECRAWL_ORIGIN,
},
headers=self._firecrawl_headers(),
+ auth=auth,
timeout=60.0,
)
response.raise_for_status()
diff --git a/tests/tools/test_web_search.py b/tests/tools/test_web_search.py
index 1efb430b..9b33529f 100644
--- a/tests/tools/test_web_search.py
+++ b/tests/tools/test_web_search.py
@@ -1519,6 +1519,65 @@ class TestFirecrawlScrapeExtraction:
class TestFirecrawlCredentialSafety:
+ @pytest.mark.parametrize("mode", ["search", "scrape"])
+ @pytest.mark.parametrize("api_key", [None, "fc-test-key"])
+ @pytest.mark.parametrize(
+ "username,password",
+ [("private-user", "private-password"), ("private@user", "private:pass/word")],
+ )
+ def test_successful_basic_auth_never_logs_url_credentials(
+ self, monkeypatch, caplog, mode, api_key, username, password
+ ):
+ import base64
+ from urllib.parse import quote
+
+ import httpx
+
+ requests = []
+ encoded_user = quote(username, safe="")
+ encoded_password = quote(password, safe="")
+ tool = WebSearchTool(
+ engine="firecrawl",
+ firecrawl_api_key=api_key,
+ firecrawl_api_url=(
+ f"https://{encoded_user}:{encoded_password}@example.test/firecrawl"
+ ),
+ )
+
+ def respond(request):
+ requests.append(request)
+ data = {"web": []} if mode == "search" else {"markdown": "Page text"}
+ return httpx.Response(200, json={"success": True, "data": data})
+
+ monkeypatch.setattr("openjarvis.tools.web_search.check_ssrf", lambda url: None)
+ query = "https://example.test/article" if mode == "scrape" else "test query"
+ with httpx.Client(
+ transport=httpx.MockTransport(respond), trust_env=False
+ ) as client:
+ monkeypatch.setattr(httpx, "post", client.post)
+ with caplog.at_level("INFO", logger="httpx"):
+ result = tool.execute(query=query)
+
+ assert result.success is True
+ assert len(requests) == 1
+ request = requests[0]
+ endpoint = f"https://example.test/firecrawl/v2/{mode}"
+ assert str(request.url) == endpoint
+ assert endpoint in caplog.text
+ scheme, encoded_auth = request.headers["Authorization"].split(" ", 1)
+ assert scheme == "Basic"
+ assert base64.b64decode(encoded_auth).decode() == f"{username}:{password}"
+ for secret in (
+ username,
+ password,
+ encoded_user,
+ encoded_password,
+ encoded_auth,
+ ):
+ assert secret not in caplog.text
+ assert secret not in result.content
+ assert secret not in str(result.metadata)
+
@pytest.fixture
def http_client(self, monkeypatch):
"""Exercise real HTTPX/header validation with an in-memory connection."""