Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): mask credentials in request URLs before logs and Sentry (#15417)

Nicholas Tindle89a07f8f1e3e
diff --git a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py
index 9f7a43c778..ae077bcb76 100644
--- a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py
+++ b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py
@@ -38,6 +38,7 @@ from backend.copilot.graphiti.client import derive_memory_group_id
from backend.copilot.graphiti.config import graphiti_config
from backend.copilot.graphiti.falkordb_driver import AutoGPTFalkorDriver
from backend.util.clients import get_scheduler_client
+from backend.util.url_redaction import redact_url
logger = logging.getLogger(__name__)
@@ -238,7 +239,7 @@ def _audit_cross_user_access(
# The URL is repr-escaped too: with path-scoped expert routes the
# expert_id is a path segment, so control characters would
# otherwise ride into the log line through the URL itself.
- f"for {request.method} {str(request.url)!r}"
+ f"for {request.method} {redact_url(str(request.url))!r}"
)
diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.py
index afda27afb5..19c093f4cd 100644
--- a/autogpt_platform/backend/backend/api/rest_api.py
+++ b/autogpt_platform/backend/backend/api/rest_api.py
@@ -107,6 +107,7 @@ from backend.util.exceptions import (
PreconditionFailed,
)
from backend.util.service import UnhealthyServiceError
+from backend.util.url_redaction import redact_url
from backend.util.workspace_storage import shutdown_workspace_storage
from .external.fastapi_app import external_api
@@ -301,7 +302,7 @@ def handle_internal_http_error(status_code: int = 500, log_error: bool = True):
logger.exception(
"%s %s failed. Investigate and resolve the underlying issue: %s",
request.method,
- request.url.path,
+ redact_url(request.url.path),
exc,
exc_info=exc,
)
@@ -309,7 +310,7 @@ def handle_internal_http_error(status_code: int = 500, log_error: bool = True):
logger.warning(
"%s %s failed with %d: %s",
request.method,
- request.url.path,
+ redact_url(request.url.path),
status_code,
exc,
)
@@ -337,7 +338,7 @@ async def validation_error_handler(
logger.error(
"Validation failed for %s %s: %s. Fix the request payload and try again.",
request.method,
- request.url.path,
+ redact_url(request.url.path),
exc,
)
errors: list | str
diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.py
index b0ddd127d7..727d8a2eb5 100644
--- a/autogpt_platform/backend/backend/data/user.py
+++ b/autogpt_platform/backend/backend/data/user.py
@@ -877,7 +877,7 @@ async def unsubscribe_user_by_token(token: str) -> None:
),
)
except Exception as e:
- raise DatabaseError(f"Failed to unsubscribe user by token {token}: {e}") from e
+ raise DatabaseError(f"Failed to unsubscribe user by token: {e}") from e
async def cleanup_user_managed_credentials(
diff --git a/autogpt_platform/backend/backend/data/user_test.py b/autogpt_platform/backend/backend/data/user_test.py
index 78381c9da3..d198683088 100644
--- a/autogpt_platform/backend/backend/data/user_test.py
+++ b/autogpt_platform/backend/backend/data/user_test.py
@@ -1,6 +1,7 @@
"""Unit tests for helpers in backend.data.user."""
import asyncio
+import base64
from collections.abc import AsyncIterator, Iterator
from contextlib import asynccontextmanager
from datetime import datetime, timezone
@@ -1854,3 +1855,15 @@ class TestFindOrphanedAuthIdentities:
assert found[0].email_owner_id == "user-old"
assert found[0].has_email_collision is True
+
+
+async def test_unsubscribe_error_does_not_repeat_the_token():
+ """The one-click route logs this error on any failure, and the token in it
+ never expires."""
+ token = base64.urlsafe_b64encode(b"user-1:not-its-signature").decode()
+
+ with pytest.raises(DatabaseError) as exc:
+ await user_module.unsubscribe_user_by_token(token)
+
+ assert token not in str(exc.value)
diff --git a/autogpt_platform/backend/backend/util/logging.py b/autogpt_platform/backend/backend/util/logging.py
index 61280d93f1..ce574e25f6 100644
--- a/autogpt_platform/backend/backend/util/logging.py
+++ b/autogpt_platform/backend/backend/util/logging.py
@@ -1,9 +1,12 @@
import logging
from backend.util.settings import AppEnvironment, BehaveAs, Settings
+from backend.util.url_redaction import RequestURLRedactionFilter
settings = Settings()
+_request_url_redaction = RequestURLRedactionFilter()
+
def configure_logging():
import autogpt_libs.logging.config
@@ -16,6 +19,14 @@ def configure_logging():
# Silence httpx logger
logging.getLogger("httpx").setLevel(logging.WARNING)
+ # These log request URLs we don't format: uvicorn the path and query of
+ # every request (the access log, and a line per WebSocket handshake), and
+ # autogpt_libs the URL of every request an admin makes as another user. A
+ # parent logger's filters never see records propagated from its children,
+ # so the filter goes on each logger directly.
+ for name in ("uvicorn.access", "uvicorn.error", "autogpt_libs.auth.dependencies"):
+ logging.getLogger(name).addFilter(_request_url_redaction)
+
def is_structured_logging_enabled() -> bool:
"""Check if structured logging (cloud logging) is enabled."""
diff --git a/autogpt_platform/backend/backend/util/metrics.py b/autogpt_platform/backend/backend/util/metrics.py
index c36304a7ac..c8f80645d1 100644
--- a/autogpt_platform/backend/backend/util/metrics.py
+++ b/autogpt_platform/backend/backend/util/metrics.py
@@ -20,6 +20,7 @@ except ImportError:
from backend.util.exceptions import get_execution_failure_reason
from backend.util.security import SENSITIVE_FIELD_NAMES
from backend.util.settings import BehaveAs, Settings
+from backend.util.url_redaction import REDACTED, redact_url
settings = Settings()
@@ -135,11 +136,33 @@ def _scrub_sentry_event(event: dict) -> None:
those here made the SDK drop the whole event as an internal error. ``_meta``
is the SDK's own annotation tree and is left alone.
"""
+ _redact_request_urls(event)
for key, value in event.items():
if key != "_meta":
event[key] = _scrub_secrets(value)
+def _redact_request_urls(event: dict) -> None:
+ """Mask credentials in the request URL the ASGI integration attaches.
+
+ EventScrubber only scrubs a request's headers, cookies and body, so the
+ query string, which for the WebSocket holds the user's session JWT, and
+ the path secrets in ``url`` would otherwise be sent with every event.
+ """
+ request = event.get("request") or {}
+ if isinstance(request.get("url"), str):
+ request["url"] = redact_url(request["url"])
+ # The SDK has already decoded the query, so a %26 inside a value reads as a
+ # separator and the rest of the value as a key. None of it can be kept.
+ if request.get("query_string"):
+ request["query_string"] = REDACTED
+ # A request no route matched is named after its raw URL.
+ transaction = event.get("transaction")
+ source = (event.get("transaction_info") or {}).get("source")
+ if source == "url" and isinstance(transaction, str):
+ event["transaction"] = redact_url(transaction)
+
+
# FalkorDB (Graphiti CoPilot memory) connection-teardown noise. graphiti-core's
# ``execute_query`` logs ERROR + re-raises whenever a query races the closing of
# its redis.asyncio connection — the fire-and-forget cache-eviction close or a
@@ -265,6 +288,13 @@ def _before_send(event, hint):
return event
+def _before_send_transaction(event, hint):
+ """Transactions skip ``before_send`` but carry the same request data, and
+ every WebSocket connection is one."""
+ _redact_request_urls(event)
+ return event
+
+
def sentry_init():
if _running_under_pytest():
return
@@ -281,6 +311,7 @@ def sentry_init():
profiles_sample_rate=1.0,
environment=f"app:{settings.config.app_env.value}-behave:{settings.config.behave_as.value}",
before_send=_before_send,
+ before_send_transaction=_before_send_transaction,
integrations=[
AsyncioIntegration(),
LoggingIntegration(),
diff --git a/autogpt_platform/backend/backend/util/request_url_logging_test.py b/autogpt_platform/backend/backend/util/request_url_logging_test.py
new file mode 100644
index 0000000000..8cfd901a0e
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/request_url_logging_test.py
@@ -0,0 +1,317 @@
+"""Credentials that ride in request URLs stay out of the logs and Sentry.
+
+The browser's WebSocket carries the user's session JWT as ``?token=``, and some
+paths are bearer secrets of their own. These start the REST and WebSocket
+servers the way their entrypoints do, under uvicorn with the logging every
+service process configures, and check what the requests leave behind.
+"""
+
+import asyncio
+import io
+import logging
+import re
+import sys
+from collections.abc import AsyncIterator, Iterator
+from contextlib import asynccontextmanager
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+import sentry_sdk
+import uvicorn
+from autogpt_libs.auth import dependencies as auth_dependencies
+from sentry_sdk.envelope import Envelope
+from sentry_sdk.integrations.asyncio import AsyncioIntegration
+from sentry_sdk.transport import Transport
+from starlette.datastructures import URL
+from starlette.requests import Request
+from websockets.asyncio.client import connect
+from websockets.exceptions import InvalidStatus
+
+from backend.api import rest_api, ws_api
+from backend.api.conn_manager import ConnectionManager
+from backend.api.test_helpers import override_config
+from backend.util import logging as service_logging
+from backend.util import metrics
+from backend.util.process import AppProcess
+from backend.util.url_redaction import REDACTED
+
+SESSION_TOKEN = "SECRET123"
+SHARE_TOKEN = "5ec2e7ab-0000-4000-8000-00000000c0de"
+WEBHOOK_ID = "0b5e55ed-0000-4000-8000-00000000beef"
+INVITATION_TOKEN = "invitation-SECRET456"
+CREDENTIALS = (SESSION_TOKEN, SHARE_TOKEN, WEBHOOK_ID, INVITATION_TOKEN)
+_FAKE_DSN = "https://[email protected]/1"
+
+
[email protected](params=[False, True], ids=["plain", "cloud"])
+def service_output(
+ request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
+) -> io.StringIO:
+ """Everything a service process writes to stdout and stderr at production's
+ log level. ``cloud`` is the structured JSON Cloud Logging collects."""
+ monkeypatch.setenv("LOG_LEVEL", "INFO")
+ monkeypatch.setattr(
+ service_logging, "is_structured_logging_enabled", lambda: request.param
+ )
+ root = logging.getLogger()
+ monkeypatch.setattr(root, "handlers", [])
+ service_logging.configure_logging()
+ output = io.StringIO()
+ for handler in root.handlers:
+ assert isinstance(handler, logging.StreamHandler)
+ handler.setStream(output)
+ return output
+
+
+def sentry(monkeypatch: pytest.MonkeyPatch) -> Iterator["_CapturingTransport"]:
+ """What a live SDK client, built from the options ``sentry_init()`` passes,
+ would send to Sentry while the test runs."""
+ calls: list[dict] = []
+ with monkeypatch.context() as during_init:
+ during_init.setattr(metrics, "_sentry_init", lambda **kw: calls.append(kw))
+ during_init.setattr(metrics.settings.secrets, "sentry_dsn", _FAKE_DSN)
+ during_init.delitem(sys.modules, "pytest")
+ during_init.delenv("PYTEST_CURRENT_TEST", raising=False)
+ metrics.sentry_init()
+ [options] = calls
+ # The profiler starts a sampler thread, and the asyncio integration patches
+ # the session's shared event loop; neither sees request data.
+ options["profiles_sample_rate"] = 0.0
+ options["integrations"] = [
+ integration
+ for integration in options["integrations"]
+ if not isinstance(integration, AsyncioIntegration)
+ ]
+ transport = _CapturingTransport()
+ client = sentry_sdk.Client(**options, transport=transport)
+ monkeypatch.setattr(sentry_sdk.get_global_scope(), "client", client)
+ yield transport
+ client.close()
+
+
+async def test_websocket_server_logs_no_session_token(
+ service_output: io.StringIO, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ async with _serve_websocket_server(monkeypatch) as host:
+ await _open_websockets(host)
+
+ logs = service_output.getvalue()
+ for credential in CREDENTIALS:
+ assert credential not in logs
+ assert logs.count(f"WebSocket /ws?token={REDACTED}") == 2
+ assert logs.count(f"WebSocket /ws?q={REDACTED}&token={REDACTED}") == 1
+ # The line keeps its shape, closing quote included (escaped in JSON).
+ assert len(re.findall(r'\[REDACTED\]\\?" \[accepted\]', logs)) == 2
+
+
+async def test_rest_server_logs_no_credentials_from_request_urls(
+ service_output: io.StringIO, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ async with (
+ _serve(rest_api.AgentServer(), monkeypatch) as host,
+ httpx.AsyncClient(base_url=f"http://{host}") as client,
+ ):
+ # Fails validation, so the 422 handler logs the request path.
+ messages = await client.get(
+ f"/api/public/shared/chats/{SHARE_TOKEN}/messages",
+ params={"limit": "many", "token": SESSION_TOKEN},
+ )
+ # No webhook manager handles this provider, so the request fails
+ # server-side and the 500 handler logs the request path.
+ ingress = await client.post(
+ f"/api/integrations/unregistered/webhooks/{WEBHOOK_ID}/ingress",
+ params={"token": SESSION_TOKEN},
+ json={},
+ )
+ assert (messages.status_code, ingress.status_code) == (422, 500)
+
+ logs = service_output.getvalue()
+ for credential in CREDENTIALS:
+ assert credential not in logs
+ shared_chat = f"/api/public/shared/chats/{REDACTED}/messages"
+ assert f"GET {shared_chat}?limit={REDACTED}&token={REDACTED} HTTP/1.1" in logs
+ assert f"Validation failed for GET {shared_chat}" in logs
+ webhook = f"/api/integrations/unregistered/webhooks/{REDACTED}/ingress"
+ assert f"POST {webhook}?token={REDACTED} HTTP/1.1" in logs
+ assert f"POST {webhook} failed." in logs
+
+
+async def test_admin_impersonation_audit_logs_no_credentials_from_the_url(
+ service_output: io.StringIO, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(auth_dependencies, "_heal_platform_user", AsyncMock())
+ request = Request(
+ {
+ "type": "http",
+ "method": "POST",
+ "scheme": "https",
+ "server": ("backend", 443),
+ "path": f"/api/invitations/{INVITATION_TOKEN}/accept",
+ "query_string": f"token={SESSION_TOKEN}".encode(),
+ "headers": [(b"host", b"backend"), (b"x-act-as-user-id", b"user-1")],
+ }
+ )
+ admin = {"sub": "admin-1", "role": "admin", "email": "[email protected]"}
+
+ assert await auth_dependencies.get_user_id(request, admin) == "user-1"
+
+ logs = service_output.getvalue()
+ for credential in CREDENTIALS:
+ assert credential not in logs
+ url = f"https://backend/api/invitations/{REDACTED}/accept?token={REDACTED}"
+ assert f"acting as user user-1 for requesting POST {url}" in logs
+
+
+ "msg, args",
+ [
+ ('%s - "WebSocket %s" [accepted]', ("1.2.3.4:5", f"/ws?token={SESSION_TOKEN}")),
+ ("GET /ws?token=%s", (SESSION_TOKEN,)),
+ ("GET %(target)s", ({"target": f"/ws?token={SESSION_TOKEN}"},)),
+ ("for requesting %s", (URL(f"https://backend/ws?token={SESSION_TOKEN}"),)),
+ ],
+ ids=["uvicorn", "split-across-format", "mapping", "url-object"],
+)
+def test_every_record_shape_on_a_filtered_logger_is_masked(
+ service_output: io.StringIO, msg: str, args: tuple
+) -> None:
+ logging.getLogger("uvicorn.error").info(msg, *args)
+
+ logs = service_output.getvalue()
+ assert SESSION_TOKEN not in logs
+ assert f"token={REDACTED}" in logs
+
+
+async def test_sentry_gets_no_credentials_from_request_urls(
+ sentry: "_CapturingTransport", monkeypatch: pytest.MonkeyPatch
+) -> None:
+ async with _serve_websocket_server(monkeypatch) as host:
+ await _open_websockets(host)
+ async with (
+ _serve(rest_api.AgentServer(), monkeypatch) as host,
+ httpx.AsyncClient(base_url=f"http://{host}") as client,
+ ):
+ query = {"token": SESSION_TOKEN}
+ await client.get(
+ f"/api/public/shared/chats/{SHARE_TOKEN}/messages",
+ params={"limit": "many", **query},
+ )
+ # Sent as %26, which Sentry decodes before before_send sees the query.
+ await client.get(
+ f"/api/public/shared/chats/{SHARE_TOKEN}/x",
+ params={"token": f"&{SESSION_TOKEN}"},
+ )
+ # Sentry builds request.url from the Host header, which the client sets.
+ await client.get(
+ f"/api/public/shared/chats/{SHARE_TOKEN}/messages",
+ params={"limit": "many"},
+ headers={"host": "evil?"},
+ )
+ await client.get(
+ f"/api/integrations/generic_webhook/webhooks/{WEBHOOK_ID}/ingress",
+ params=query,
+ )
+
+ for credential in CREDENTIALS:
+ assert credential not in sentry.payload
+ requests = [event["request"] for event in sentry.events if "request" in event]
+ assert len(requests) >= 6
+ assert {request.get("query_string") for request in requests} == {REDACTED}
+ assert any(
+ f"/api/public/shared/chats/{REDACTED}/messages" in request["url"]
+ for request in requests
+ )
+ # The request no route matched is named after its URL.
+ assert f"/api/public/shared/chats/{REDACTED}/x" in {
+ event.get("transaction", "").removeprefix(f"http://{host}")
+ for event in sentry.events
+ }
+
+
+@asynccontextmanager
+async def _serve(
+ process: AppProcess, monkeypatch: pytest.MonkeyPatch
+) -> AsyncIterator[str]:
+ """Serve what ``process.run()`` hands uvicorn, on a free local port and
+ without the lifespan's database and Redis connections."""
+ runs: list[tuple] = []
+ monkeypatch.setattr(
+ uvicorn, "run", lambda app, **kwargs: runs.append((app, kwargs))
+ )
+ # AgentServer.run() turns asyncio debug on for local runs; undo it after.
+ monkeypatch.setenv("PYTHONASYNCIODEBUG", "")
+ process.run()
+ [(app, kwargs)] = runs
+ server = uvicorn.Server(
+ uvicorn.Config(
+ app, **{**kwargs, "host": "127.0.0.1", "port": 0, "lifespan": "off"}
+ )
+ )
+ serving = asyncio.create_task(server.serve())
+ for _ in range(1000):
+ if server.started or serving.done():
+ break
+ await asyncio.sleep(0.01)
+ assert server.started, "uvicorn did not start"
+ port = server.servers[0].sockets[0].getsockname()[1]
+ try:
+ yield f"127.0.0.1:{port}"
+ finally:
+ server.should_exit = True
+ await serving
+
+
+@asynccontextmanager
+async def _serve_websocket_server(
+ monkeypatch: pytest.MonkeyPatch,
+) -> AsyncIterator[str]:
+ # Accepting a socket also subscribes it to the user's notifications in Redis.
+ monkeypatch.setattr(
+ ConnectionManager, "_start_notification_subscription", AsyncMock()
+ )
+ monkeypatch.setattr(ws_api, "_connection_manager", None)
+ async with _serve(ws_api.WebsocketServer(), monkeypatch) as host:
+ yield host
+
+
+async def _open_websockets(host: str) -> None:
+ url = f"ws://{host}/ws?token={SESSION_TOKEN}"
+ # Not a valid JWT, so the handshake is refused, as an expired one's is...
+ with pytest.raises(InvalidStatus):
+ async with connect(url):
+ pass
+ # ...and with auth off it is accepted, as a live session's is. A raw quote
+ # or # earlier in the query must not end the masking before the token.
+ with override_config(ws_api.settings, "enable_auth", False):
+ async with connect(url):
+ pass
+ target = f"/ws?q=it's#&token={SESSION_TOKEN}"
+ assert b" 101 " in await _raw_websocket_handshake(host, target)
+
+
+async def _raw_websocket_handshake(host: str, target: str) -> bytes:
+ """Sent as bytes, since a client library drops a # before sending."""
+ address, port = host.split(":")
+ reader, writer = await asyncio.open_connection(address, int(port))
+ writer.write(
+ f"GET {target} HTTP/1.1\r\nHost: {host}\r\nUpgrade: websocket\r\n"
+ "Connection: Upgrade\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"
+ "Sec-WebSocket-Version: 13\r\n\r\n".encode()
+ )
+ status_line = await reader.readline()
+ writer.close()
+ await writer.wait_closed()
+ return status_line
+
+
+class _CapturingTransport(Transport):
+ def __init__(self) -> None:
+ super().__init__()
+ self.events: list[dict] = []
+ self.payload = ""
+
+ def capture_envelope(self, envelope: Envelope) -> None:
+ self.payload += envelope.serialize().decode()
+ self.events += [item.payload.json or {} for item in envelope.items]
diff --git a/autogpt_platform/backend/backend/util/url_redaction.py b/autogpt_platform/backend/backend/util/url_redaction.py
new file mode 100644
index 0000000000..2eb58f78d8
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/url_redaction.py
@@ -0,0 +1,103 @@
+"""Masking for credentials that travel in request URLs.
+
+The browser opens its WebSocket with the user's session JWT in ``?token=``,
+email links carry one-click tokens in the query, and some path segments are
+bearer secrets themselves: whoever holds one can post to a webhook, claim a
+platform link, read a shared chat or run, or accept an org invitation. uvicorn
+logs the path and query of every request and Sentry attaches both to every
+event, so anything that records a request URL masks it here first.
+"""
+
+import logging
+import re
+from collections.abc import Mapping
+
+REDACTED = "[REDACTED]"
+
+# The segment after each of these prefixes is a credential, even on a route
+# that doesn't exist. /api/invitations/pending is the one static route among
+# them, so it alone is kept. The match is zero-width so that overlapping
+# prefixes, which a crafted path or Host header can stack, are all found.
+_CREDENTIAL_PATH_SEGMENT = re.compile(
+ r"(?=/api/(?:integrations/(?:[^/?\s]+/)?webhooks|platform-linking/(?:user-)?tokens"
+ r"|public/shared(?:/chats)?|invitations(?!/pending(?:[/?\s\"']|$)))/([^/?\s]+))"
+)
+# A request target can't contain whitespace, so in a longer message its query
+# runs to the next whitespace. Nothing else ends it: a raw quote can sit in a
+# value, and uvicorn's WebSocket scope keeps even a raw # in the query.
+_QUERY_IN_TEXT = re.compile(r"\?(\S+)")
+_QUERY_VALUE = re.compile(r"=[^&]*")
+_MENTIONS_URL = re.compile(r"\?|/api/")
+
+
+def redact_url(url: str) -> str:
+ """Mask the credentials in a URL or request target.
+
+ Every query value is masked and its key kept, so ``/ws?token=abc&page=2``
+ becomes ``/ws?token=[REDACTED]&page=[REDACTED]``, and so is every path
+ segment that is a credential in its own right. Paths are masked across the
+ whole string, after the query: Sentry builds a URL from the Host header,
+ which can hold a ``?`` of its own.
+ """
+ target, question_mark, query = url.partition("?")
+ return _redact_path(target + question_mark + _redact_query(query))
+
+
+def redact_urls_in_text(text: str) -> str:
+ """Mask the credentials in every request URL inside a longer message."""
+ return _redact_path(
+ _QUERY_IN_TEXT.sub(lambda match: "?" + _redact_query(match[1]), text)
+ )
+
+
+def _redact_path(text: str) -> str:
+ # Built once from the spans, so a URL stacked with prefixes stays linear.
+ parts: list[str] = []
+ done = 0
+ for match in _CREDENTIAL_PATH_SEGMENT.finditer(text):
+ start, end = match.span(1)
+ if end > done:
+ parts += [text[done : max(start, done)], REDACTED]
+ done = end
+ return "".join(parts) + text[done:]
+
+
+def _redact_query(query: str) -> str:
+ return _QUERY_VALUE.sub("=" + REDACTED, query)
+
+
+class RequestURLRedactionFilter(logging.Filter):
+ """Masks request URLs in a log record before any handler sees it.
+
+ Each argument is masked as a whole URL, which is how uvicorn passes the
+ request target. A format string that mentions a URL itself, or a message
+ that arrives already formatted, is rendered and scanned instead.
+ """
+
+ def filter(self, record: logging.LogRecord) -> bool:
+ if isinstance(record.args, Mapping):
+ record.args = {
+ key: _redact_argument(value) for key, value in record.args.items()
+ }
+ elif isinstance(record.args, tuple):
+ record.args = tuple(_redact_argument(arg) for arg in record.args)
+ if isinstance(record.msg, str) and _MENTIONS_URL.search(record.msg):
+ try:
+ message = record.getMessage()
+ except Exception:
+ # A malformed record fails again, and is reported, in the
+ # handler; raising here would fail the request that logged it.
+ return True
+ masked = redact_urls_in_text(message)
+ if masked != message:
+ record.msg, record.args = masked, ()
+ return True
+
+
+def _redact_argument(argument: object) -> object:
+ try:
+ text = argument if isinstance(argument, str) else str(argument)
+ except Exception:
+ return argument # the handler hits, and reports, the same failure
+ masked = redact_url(text)
+ return argument if masked == text else masked
diff --git a/autogpt_platform/backend/backend/util/url_redaction_test.py b/autogpt_platform/backend/backend/util/url_redaction_test.py
new file mode 100644
index 0000000000..652c230c0a
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/url_redaction_test.py
@@ -0,0 +1,218 @@
+"""What redact_url() and redact_urls_in_text() mask, and what they keep.
+
+Every credential route is covered and nothing else is touched, and a seeded
+randomized test buries the credential among junk chosen to end a match early.
+request_url_logging_test.py checks the same masking end to end.
+"""
+
+import random
+import re
+import time
+from collections.abc import Iterator
+
+import pytest
+from starlette.routing import BaseRoute, Mount, Route, WebSocketRoute
+
+from backend.api import rest_api
+from backend.util.url_redaction import REDACTED, redact_url, redact_urls_in_text
+
+
+ "url, expected",
+ [
+ ("/ws?token=a.b.c", f"/ws?token={REDACTED}"),
+ ("/x?token=a&page=2&&flag", f"/x?token={REDACTED}&page={REDACTED}&&flag"),
+ ("/x?token=a=b;c=d", f"/x?token={REDACTED}"),
+ ("/x?a=1?token=b", f"/x?a={REDACTED}"),
+ ("/x?=bare", f"/x?={REDACTED}"),
+ # uvicorn's WebSocket scope keeps a raw # in the query.
+ ("/ws?a=1#&token=b", f"/ws?a={REDACTED}&token={REDACTED}"),
+ # Sentry builds the URL from the Host header, which can hold a ?.
+ (
+ "http://evil?/api/public/shared/chats/s-1/messages",
+ f"http://evil?/api/public/shared/chats/{REDACTED}/messages",
+ ),
+ # A prefix stacked in front of a real one doesn't hide the real one.
+ (
+ "http://x/api/invitations/api/public/shared/chats/s-1/messages",
+ f"http://x/api/invitations/{REDACTED}/public/shared/chats/{REDACTED}"
+ "/messages",
+ ),
+ # The query is masked first, so its result is checked for paths too.
+ (
+ "/ws?/api/integrations/a=x/y&/webhooks/t",
+ f"/ws?/api/integrations/a={REDACTED}&/webhooks/{REDACTED}",
+ ),
+ # Characters that can end a URL in a message don't end this one.
+ ("/ws?q=it's&token=a", f"/ws?q={REDACTED}&token={REDACTED}"),
+ ('/ws?q="&token=a', f"/ws?q={REDACTED}&token={REDACTED}"),
+ ("/ws?q=a b&token=c", f"/ws?q={REDACTED}&token={REDACTED}"),
+ (
+ "https://h:1/api/email/unsubscribe?token=dXNlcjpzaWc%3D",
+ f"https://h:1/api/email/unsubscribe?token={REDACTED}",
+ ),
+ (
+ "/api/integrations/github/webhooks/w-1/ingress",
+ f"/api/integrations/github/webhooks/{REDACTED}/ingress",
+ ),
+ (
+ "/api/integrations/webhooks/w-1/ping",
+ f"/api/integrations/webhooks/{REDACTED}/ping",
+ ),
+ (
+ "/api/platform-linking/tokens/t-1/confirm",
+ f"/api/platform-linking/tokens/{REDACTED}/confirm",
+ ),
+ (
+ "/api/platform-linking/user-tokens/t-1/confirm",
+ f"/api/platform-linking/user-tokens/{REDACTED}/confirm",
+ ),
+ ("/api/public/shared/s-1", f"/api/public/shared/{REDACTED}"),
+ (
+ "/api/public/shared/s-1/files/f-1/download",
+ f"/api/public/shared/{REDACTED}/files/f-1/download",
+ ),
+ (
+ "/api/public/shared/chats/s-1/messages?limit=5",
+ f"/api/public/shared/chats/{REDACTED}/messages?limit={REDACTED}",
+ ),
+ ("/api/invitations/i-1/accept", f"/api/invitations/{REDACTED}/accept"),
+ # A route that doesn't exist still carries the token.
+ ("/api/invitations/i-1", f"/api/invitations/{REDACTED}"),
+ ("/api/invitations/pending-1/x", f"/api/invitations/{REDACTED}/x"),
+ # A path prefix in front of the app.
+ (
+ "/backend/api/invitations/i-1/accept",
+ f"/backend/api/invitations/{REDACTED}/accept",
+ ),
+ # Already masked stays the same, and IDs that grant nothing are kept.
+ (f"/ws?token={REDACTED}", f"/ws?token={REDACTED}"),
+ ("/api/orgs/o-1/invitations/i-1", "/api/orgs/o-1/invitations/i-1"),
+ ("/api/invitations/pending", "/api/invitations/pending"),
+ ("/api/integrations/github/webhooks", "/api/integrations/github/webhooks"),
+ ("/api/library/agents/a-1", "/api/library/agents/a-1"),
+ ],
+)
+def test_redact_url(url: str, expected: str) -> None:
+ assert redact_url(url) == expected
+
+
+ "text, expected",
+ [
+ (
+ '1.2.3.4:5 - "GET /api/invitations/i-1/decline?x=1 HTTP/1.1" 401',
+ f'1.2.3.4:5 - "GET /api/invitations/{REDACTED}/decline?x={REDACTED}'
+ ' HTTP/1.1" 401',
+ ),
+ # A raw quote is part of the query; only whitespace ends it.
+ (
+ '"GET /ws?q=it\'s&token=a HTTP/1.1"',
+ f'"GET /ws?q={REDACTED}&token={REDACTED} HTTP/1.1"',
+ ),
+ (
+ "acting as user u for requesting GET https://h/api/public/shared/s-1?x=1",
+ "acting as user u for requesting GET "
+ f"https://h/api/public/shared/{REDACTED}?x={REDACTED}",
+ ),
+ (
+ '"GET /ws?a=1#&token=t HTTP/1.1"',
+ f'"GET /ws?a={REDACTED}&token={REDACTED} HTTP/1.1"',
+ ),
+ ("Did it work? Yes.", "Did it work? Yes."),
+ ],
+)
+def test_redact_urls_in_text(text: str, expected: str) -> None:
+ assert redact_urls_in_text(text) == expected
+
+
+def test_every_route_with_a_credential_in_its_path_is_masked_and_no_other() -> None:
+ """Checks the masked prefixes against the REST app's real routes, so a new
+ route with a token in its path fails here until it is covered."""
+ for path in _route_paths(rest_api.app.routes):
+ sample = _PATH_PARAM.sub(
+ lambda match: (
+ f"secret-{match[1]}" if match[1] in _CREDENTIAL_PARAMS else "value"
+ ),
+ path,
+ )
+ if _CREDENTIAL_PARAMS & set(_PATH_PARAM.findall(path)):
+ assert "secret-" not in redact_url(sample), path
+ else:
+ assert redact_url(sample) == sample, path
+
+
[email protected]("seed", range(5))
+def test_no_request_target_keeps_its_credential_whatever_surrounds_it(
+ seed: int,
+) -> None:
+ """Random request targets with the credential in a query value, in a path
+ segment or both, among junk chosen to end a match early. Masked on their
+ own and inside an access log line, the credential never survives, and
+ masking again changes nothing."""
+ rng = random.Random(seed)
+ for _ in range(200):
+ target = _random_request_target(rng)
+ masked = redact_url(target)
+ assert _SECRET not in masked, target
+ assert redact_url(masked) == masked, target
+ line = f'1.2.3.4:5 - "GET {target} HTTP/1.1" 200'
+ masked_line = redact_urls_in_text(line)
+ assert _SECRET not in masked_line, line
+ assert redact_urls_in_text(masked_line) == masked_line, line
+
+
+def test_masking_stays_linear_in_a_url_stacked_with_prefixes() -> None:
+ """A client can send a URL made of thousands of prefixes, and the filter
+ masks it on the event loop; quadratic work took seconds here."""
+ url = "/api/invitations/x" * 50_000
+ started = time.monotonic()
+ assert redact_url(url) == f"/api/invitations/{REDACTED}" * 50_000
+ assert time.monotonic() - started < 2
+
+
+_PATH_PARAM = re.compile(r"{(\w+)(?::\w+)?}")
+# The path parameters that are bearer credentials.
+_CREDENTIAL_PARAMS = {"token", "share_token", "webhook_id"}
+
+
+def _route_paths(routes: list[BaseRoute], prefix: str = "") -> Iterator[str]:
+ for route in routes:
+ if isinstance(route, Mount):
+ yield from _route_paths(route.routes, prefix + route.path)
+ elif isinstance(route, (Route, WebSocketRoute)):
+ yield prefix + route.path
+
+
+_SECRET = "SeCrEt-7f3a"
+# A request target has no whitespace, and & always separates pairs; anything
+# else may sit next to the credential, a raw # included.
+_JUNK = ["", "a", "it's", '"', "?", "#", "=", ";", "/", "%26", "%3D", "[x]", "{"]
+_CREDENTIAL_PATHS = [
+ "/api/integrations/generic_webhook/webhooks/{}/ingress",
+ "/api/integrations/webhooks/{}/ping",
+ "/api/platform-linking/tokens/{}/confirm",
+ "/api/platform-linking/user-tokens/{}/confirm",
+ "/api/public/shared/{}",
+ "/api/public/shared/chats/{}/messages",
+ "/api/invitations/{}/accept",
+ "/api/invitations/{}",
+]
+
+
+def _random_request_target(rng: random.Random) -> str:
+ def junk() -> str:
+ return "".join(rng.choices(_JUNK, k=rng.randint(0, 3)))
+
+ pairs = [f"{junk()}={junk()}" for _ in range(rng.randint(0, 3))]
+ if rng.random() < 0.5:
+ pairs.insert(rng.randint(0, len(pairs)), f"token={junk()}{_SECRET}{junk()}")
+ path = rng.choice(["/ws", "/api/email/unsubscribe", "/"])
+ else:
+ # A crafted path or Host header can stack another prefix in front.
+ stacked = rng.choice(["", "", "/api/public/shared", "/api/invitations"])
+ path = stacked + rng.choice(_CREDENTIAL_PATHS).format(_SECRET)
+ # Sentry prefixes the scheme and the Host header, which the client controls.
+ origin = f"http://h{junk()}" if rng.random() < 0.3 else ""
+ query = "?" + "&".join(pairs) if pairs or rng.random() < 0.5 else ""
+ return origin + path + query