Personal AI, On Personal Devices
Python
fix(tools): separate Firecrawl Basic Auth from logged URLs
Elliot Slusky9c16dc74def4
diff --git a/src/openjarvis/tools/web_search.py b/src/openjarvis/tools/web_search.pyindex f7a0e3c6..0e69d25e 100644--- a/src/openjarvis/tools/web_search.py+++ b/src/openjarvis/tools/web_search.py@@ -484,6 +484,20 @@ class WebSearchTool(BaseTool):headers["Authorization"] = f"Bearer {self._firecrawl_api_key}"return headers+ def _firecrawl_endpoint(self, operation: str) -> tuple[str, tuple[str, str] | None]:+ """Keep self-hosted Basic Auth out of URLs exposed by HTTPX logging."""+ import httpx++ url = httpx.URL(f"{self._firecrawl_api_url}/v2/{operation}")+ auth = None+ if url.username or url.password:+ # HTTPX already uses URL userinfo as Basic Auth, overriding a+ # Bearer header when both are present. Preserve that behavior while+ # ensuring its INFO request log contains only the endpoint URL.+ auth = (url.username, url.password)+ url = url.copy_with(username=None, password=None)+ return str(url), auth+@staticmethoddef _format_firecrawl_results(payload: dict[str, Any]) -> tuple[str, int]:"""Render a Firecrawl search payload in the shared result format.@@ -515,14 +529,16 @@ class WebSearchTool(BaseTool):if not self._firecrawl_api_key and not self._firecrawl_self_hosted:raise _WebSearchEngineError(f"{FIRECRAWL_API_KEY_ENV} is not set")try:+ endpoint, auth = self._firecrawl_endpoint("search")response = httpx.post(- f"{self._firecrawl_api_url}/v2/search",+ endpoint,json={"query": query,"limit": max_results,"origin": FIRECRAWL_ORIGIN,},headers=self._firecrawl_headers(),+ auth=auth,timeout=30.0,)response.raise_for_status()@@ -580,8 +596,9 @@ class WebSearchTool(BaseTool):import httpxtry:+ endpoint, auth = self._firecrawl_endpoint("scrape")response = httpx.post(- f"{self._firecrawl_api_url}/v2/scrape",+ endpoint,json={"url": url,"formats": ["markdown"],@@ -591,6 +608,7 @@ class WebSearchTool(BaseTool):"origin": FIRECRAWL_ORIGIN,},headers=self._firecrawl_headers(),+ auth=auth,timeout=60.0,)response.raise_for_status()diff --git a/tests/tools/test_web_search.py b/tests/tools/test_web_search.pyindex 1efb430b..9b33529f 100644--- a/tests/tools/test_web_search.py+++ b/tests/tools/test_web_search.py@@ -1519,6 +1519,65 @@ class TestFirecrawlScrapeExtraction:class TestFirecrawlCredentialSafety:+ @pytest.mark.parametrize("mode", ["search", "scrape"])+ @pytest.mark.parametrize("api_key", [None, "fc-test-key"])+ @pytest.mark.parametrize(+ "username,password",+ [("private-user", "private-password"), ("private@user", "private:pass/word")],+ )+ def test_successful_basic_auth_never_logs_url_credentials(+ self, monkeypatch, caplog, mode, api_key, username, password+ ):+ import base64+ from urllib.parse import quote++ import httpx++ requests = []+ encoded_user = quote(username, safe="")+ encoded_password = quote(password, safe="")+ tool = WebSearchTool(+ engine="firecrawl",+ firecrawl_api_key=api_key,+ firecrawl_api_url=(+ f"https://{encoded_user}:{encoded_password}@example.test/firecrawl"+ ),+ )++ def respond(request):+ requests.append(request)+ data = {"web": []} if mode == "search" else {"markdown": "Page text"}+ return httpx.Response(200, json={"success": True, "data": data})++ monkeypatch.setattr("openjarvis.tools.web_search.check_ssrf", lambda url: None)+ query = "https://example.test/article" if mode == "scrape" else "test query"+ with httpx.Client(+ transport=httpx.MockTransport(respond), trust_env=False+ ) as client:+ monkeypatch.setattr(httpx, "post", client.post)+ with caplog.at_level("INFO", logger="httpx"):+ result = tool.execute(query=query)++ assert result.success is True+ assert len(requests) == 1+ request = requests[0]+ endpoint = f"https://example.test/firecrawl/v2/{mode}"+ assert str(request.url) == endpoint+ assert endpoint in caplog.text+ scheme, encoded_auth = request.headers["Authorization"].split(" ", 1)+ assert scheme == "Basic"+ assert base64.b64decode(encoded_auth).decode() == f"{username}:{password}"+ for secret in (+ username,+ password,+ encoded_user,+ encoded_password,+ encoded_auth,+ ):+ assert secret not in caplog.text+ assert secret not in result.content+ assert secret not in str(result.metadata)+@pytest.fixturedef http_client(self, monkeypatch):"""Exercise real HTTPX/header validation with an in-memory connection."""