AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): refuse unknown webhook providers and bad email links with 404/400, not 500 (#15758)
Reinier van der Leer8f383d4a45e5
diff --git a/autogpt_platform/backend/backend/api/features/integrations/router.py b/autogpt_platform/backend/backend/api/features/integrations/router.pyindex 41500c44ce..0cbe9c9301 100644--- a/autogpt_platform/backend/backend/api/features/integrations/router.py+++ b/autogpt_platform/backend/backend/api/features/integrations/router.py@@ -85,7 +85,7 @@ from backend.integrations.service_identity import (service_for_credential,service_for_provider,)-from backend.integrations.webhooks import get_webhook_manager+from backend.integrations.webhooks import get_webhook_manager, supports_webhooksfrom backend.util import product_analyticsfrom backend.util.exceptions import (ExpertRunPausedError,@@ -1224,6 +1224,12 @@ async def webhook_ingress_generic(webhook_id: Annotated[str, Path(title="Our ID for the webhook")],):logger.debug(f"Received {provider.value} webhook ingress for ID {webhook_id}")+ if not supports_webhooks(provider):+ # Worded like the unknown-webhook 404 below, so the two can't be told apart.+ raise HTTPException(+ status_code=status.HTTP_404_NOT_FOUND,+ detail=f"Webhook #{webhook_id} not found",+ )webhook_manager = get_webhook_manager(provider)try:webhook = await get_webhook(webhook_id, include_relations=True)diff --git a/autogpt_platform/backend/backend/api/features/integrations/webhook_ingress_test.py b/autogpt_platform/backend/backend/api/features/integrations/webhook_ingress_test.pyindex 007ee1ab48..cb2b7b783f 100644--- a/autogpt_platform/backend/backend/api/features/integrations/webhook_ingress_test.py+++ b/autogpt_platform/backend/backend/api/features/integrations/webhook_ingress_test.py@@ -30,6 +30,7 @@ import pytestfrom backend.api.features.integrations.router import routerfrom backend.data.integrations import WebhookWithRelationsfrom backend.integrations.providers import ProviderName+from backend.util.exceptions import NotFoundErrorapp = fastapi.FastAPI()app.include_router(router)@@ -150,12 +151,7 @@ def _manager(provider: ProviderName):def _run(webhook, provider_path: str, **kwargs):"""Boilerplate: patch the router's deps, set the right manager, fire request."""- patches = _patch_ingress(webhook) + [- patch(- "backend.api.features.integrations.router.get_webhook_manager",- return_value=_manager(webhook.provider),- )- ]+ patches = _patch_ingress(webhook) + _stub_registry(_manager(webhook.provider))# ExitStack guarantees previously started patchers are stopped even if# a later one's __enter__ raises.with ExitStack() as stack:@@ -164,6 +160,21 @@ def _run(webhook, provider_path: str, **kwargs):return _post(provider_path, **kwargs)+def _stub_registry(manager):+ # SDK providers register their managers only at app startup, which these+ # tests skip; the real registry is also case-sensitive.+ return [+ patch(+ "backend.api.features.integrations.router.supports_webhooks",+ return_value=True,+ ),+ patch(+ "backend.api.features.integrations.router.get_webhook_manager",+ return_value=manager,+ ),+ ]++# ---------------------------------------------------------------------------# Unsigned providers: default no-op verify, ingress passes through.# ---------------------------------------------------------------------------@@ -497,12 +508,9 @@ class TestProviderPathConfusion:# Unlike `_run`, the manager comes from the URL path provider (what the# real router does) rather than the stored webhook's provider — that# divergence is the whole point of the attack being reproduced.- patches = _patch_ingress(webhook) + [- patch(- "backend.api.features.integrations.router.get_webhook_manager",- return_value=_manager(url_manager_provider),- )- ]+ patches = _patch_ingress(webhook) + _stub_registry(+ _manager(url_manager_provider)+ )with ExitStack() as stack:for p in patches:stack.enter_context(p)@@ -836,3 +844,20 @@ def test_a_banned_owners_delivery_is_acknowledged_and_dropped():publish.assert_not_awaited()onboarding.assert_not_awaited()node_trigger.assert_not_called()++[email protected]("provider", ["someotherprovider", "slack"])+def test_a_provider_without_webhooks_is_answered_like_an_unknown_webhook(provider):+ """Even if a row names that provider: no manager exists to verify it."""+ with patch(+ "backend.api.features.integrations.router.get_webhook",+ AsyncMock(side_effect=NotFoundError(f"Webhook #{WEBHOOK_ID} not found")),+ ):+ unknown_webhook = _post("github")+ with ExitStack() as stack:+ for p in _patch_ingress(_make_webhook(ProviderName(provider))):+ stack.enter_context(p)+ no_webhooks = _post(provider)++ assert unknown_webhook.status_code == no_webhooks.status_code == 404+ assert no_webhooks.json() == unknown_webhook.json()diff --git a/autogpt_platform/backend/backend/api/features/postmark/postmark.py b/autogpt_platform/backend/backend/api/features/postmark/postmark.pyindex 224e30fa9d..6567cb866f 100644--- a/autogpt_platform/backend/backend/api/features/postmark/postmark.py+++ b/autogpt_platform/backend/backend/api/features/postmark/postmark.py@@ -1,7 +1,7 @@import loggingfrom typing import Annotated-from fastapi import APIRouter, Body, HTTPException, Query, Security+from fastapi import APIRouter, Body, Query, Securityfrom fastapi.responses import JSONResponsefrom backend.api.utils.api_key_auth import APIKeyAuthenticator@@ -37,14 +37,7 @@ postmark_api_key_auth = APIKeyAuthenticator(@router.post("/unsubscribe", summary="One Click Email Unsubscribe")async def unsubscribe_via_one_click(token: Annotated[str, Query()]):logger.info("Received unsubscribe request from One Click Unsubscribe")- try:- await unsubscribe_user_by_token(token)- except Exception as e:- logger.exception("Unsubscribe failed: %s", e)- raise HTTPException(- status_code=500,- detail={"message": str(e), "hint": "Verify Postmark token settings."},- )+ await unsubscribe_user_by_token(token)return JSONResponse(status_code=200, content={"status": "ok"})diff --git a/autogpt_platform/backend/backend/api/features/postmark/postmark_test.py b/autogpt_platform/backend/backend/api/features/postmark/postmark_test.pynew file mode 100644index 0000000000..4afc1293fd--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/postmark/postmark_test.py@@ -0,0 +1,152 @@+"""One-click unsubscribe, answered through the app's own error handlers."""++import base64+import uuid+from datetime import datetime, timezone+from unittest.mock import AsyncMock+from urllib.parse import parse_qs, urlparse++import fastapi+import fastapi.testclient+import pytest+import pytest_mock+from prisma.actions import UserActions+from prisma.enums import BriefingFrequency, SubscriptionTier+from prisma.errors import DataError+from prisma.models import User as PrismaUser++from backend.api.rest_api import app as rest_app+from backend.data.user import generate_unsubscribe_link++from .postmark import router++app = fastapi.FastAPI(exception_handlers=rest_app.exception_handlers)+app.include_router(router)+client = fastapi.testclient.TestClient(app, raise_server_exceptions=False)+++def user_id() -> str:+ # Fresh per test: the user lookup is cached in the shared Redis for minutes.+ return str(uuid.uuid4())+++# The User table, beneath the real lookup and preference update.+def lookup(mocker: pytest_mock.MockFixture, user_id: str) -> AsyncMock:+ return mocker.patch.object(+ UserActions, "find_unique", AsyncMock(return_value=_user_row(user_id))+ )+++def update(mocker: pytest_mock.MockFixture, user_id: str) -> AsyncMock:+ return mocker.patch.object(+ UserActions, "update", AsyncMock(return_value=_user_row(user_id))+ )+++def test_a_valid_token_unsubscribes_its_user(+ lookup: AsyncMock, update: AsyncMock, user_id: str+):+ response = _unsubscribe(_token_for(user_id))++ assert response.status_code == 200+ update.assert_awaited_once()+ assert update.await_args.kwargs["where"] == {"id": user_id}+ assert (+ update.await_args.kwargs["data"]["briefingFrequency"] == BriefingFrequency.OFF+ )+++ "token",+ [+ "abc",+ "!!!!",+ base64.urlsafe_b64encode(b"no-separator").decode(),+ base64.urlsafe_b64encode(b"\xff\xfe:x").decode(),+ base64.urlsafe_b64encode(b"user-1:not-its-signature").decode(),+ base64.urlsafe_b64encode("user-1:é".encode()).decode(),+ ],+ ids=[+ "bad-padding",+ "not-base64",+ "no-separator",+ "not-utf8",+ "wrong-signature",+ "non-ascii-signature",+ ],+)+def test_a_token_we_did_not_issue_is_a_bad_request(+ token: str, lookup: AsyncMock, update: AsyncMock+):+ response = _unsubscribe(token)++ assert response.status_code == 400+ assert token not in response.text+ update.assert_not_awaited()+++def test_a_token_for_a_deleted_user_is_refused_like_a_forged_one(+ lookup: AsyncMock, update: AsyncMock, user_id: str+):+ lookup.return_value = None+ token = _token_for(user_id)++ response = _unsubscribe(token)++ assert response.status_code == 400+ assert response.json()["detail"] == "Invalid unsubscribe token"+ assert user_id not in response.text and token not in response.text+ update.assert_not_awaited()+++def test_a_user_deleted_before_the_update_is_refused_like_a_forged_one(+ lookup: AsyncMock, update: AsyncMock, user_id: str+):+ update.return_value = None+ token = _token_for(user_id)++ response = _unsubscribe(token)++ assert response.status_code == 400+ assert response.json()["detail"] == "Invalid unsubscribe token"+ assert user_id not in response.text and token not in response.text+++def test_a_database_failure_stays_a_server_error(+ lookup: AsyncMock, update: AsyncMock, user_id: str+):+ update.side_effect = DataError({"user_facing_error": {"message": "reset"}})++ response = _unsubscribe(_token_for(user_id))++ assert response.status_code == 500+++def _unsubscribe(token: str):+ return client.post("/unsubscribe", params={"token": token})+++def _token_for(user_id: str) -> str:+ return parse_qs(urlparse(generate_unsubscribe_link(user_id)).query)["token"][0]+++def _user_row(user_id: str) -> PrismaUser:+ now = datetime.now(timezone.utc)+ return PrismaUser(+ id=user_id,+ email="[email protected]",+ emailVerified=True,+ createdAt=now,+ updatedAt=now,+ metadata="{}",+ integrations="",+ maxEmailsPerDay=3,+ briefingFrequency=BriefingFrequency.DAILY,+ alertsEnabled=True,+ notifyOnStoreVerdict=True,+ timezone="UTC",+ subscriptionTier=SubscriptionTier.NO_TIER,+ )diff --git a/autogpt_platform/backend/backend/api/features/user/routes.py b/autogpt_platform/backend/backend/api/features/user/routes.pyindex 1929d4ca65..d20caabaf8 100644--- a/autogpt_platform/backend/backend/api/features/user/routes.py+++ b/autogpt_platform/backend/backend/api/features/user/routes.py@@ -24,6 +24,7 @@ from backend.data.user import (update_user_timezone,verify_preference_token,)+from backend.util.exceptions import NotFoundErrorfrom backend.util.settings import Settingssettings = Settings()@@ -180,11 +181,15 @@ async def apply_email_preference_choice(if user_id is None:raise HTTPException(status_code=400, detail="Invalid or expired link")- current = await get_user_notification_preference(user_id)- updated = _preference_with_choice(current, choice)- if updated is None:- raise HTTPException(status_code=400, detail="Unknown preference choice")- return await update_user_notification_preference(user_id, updated)+ try:+ current = await get_user_notification_preference(user_id)+ updated = _preference_with_choice(current, choice)+ if updated is None:+ raise HTTPException(status_code=400, detail="Unknown preference choice")+ return await update_user_notification_preference(user_id, updated)+ except NotFoundError:+ # A deleted user's link is refused like a forged one.+ raise HTTPException(status_code=400, detail="Invalid or expired link") from Nonedef _preference_with_choice(diff --git a/autogpt_platform/backend/backend/api/features/user/routes_test.py b/autogpt_platform/backend/backend/api/features/user/routes_test.pyindex 09b86adc70..d2618f37f0 100644--- a/autogpt_platform/backend/backend/api/features/user/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/user/routes_test.py@@ -1,5 +1,6 @@import jsonimport re+import urllib.parseimport uuidfrom collections.abc import AsyncIteratorfrom datetime import datetime, timezone@@ -14,6 +15,8 @@ import pytest_mockfrom autogpt_libs.auth.jwt_utils import get_jwt_payloadfrom fastapi.routing import APIRoutefrom prisma.actions import UserActions+from prisma.enums import BriefingFrequency, SubscriptionTier+from prisma.errors import DataError, RecordNotFoundErrorfrom prisma.models import User as PrismaUserfrom pytest_snapshot.plugin import Snapshot@@ -22,6 +25,7 @@ from backend.api.rest_api import app as real_appfrom backend.data.model import Userfrom backend.data.notifications import AudienceAction, NotificationResultfrom backend.data.user import (+ generate_preference_link,get_or_create_user,get_user_by_email,get_user_by_id,@@ -523,3 +527,98 @@ def test_auth_surface_has_no_other_operations():@pytest.mark.parametrize("method,path", sorted(AUTHENTICATED | UNAUTHENTICATED))def test_auth_operation_keeps_its_tags(method: str, path: str):assert real_app.openapi()["paths"][path][method]["tags"] == ["v1", "auth"]+++class TestEmailPreferenceLinkForAMissingUser:+ """The footer link names its user, so a deleted user's link must be refused+ like a forged one: no 500, and no user id in the answer."""++ @pytest.fixture+ def user_id(self) -> str:+ return str(uuid.uuid4())++ @pytest.fixture+ def lookup(self, mocker: pytest_mock.MockFixture, user_id: str) -> AsyncMock:+ return mocker.patch.object(+ UserActions,+ "find_unique_or_raise",+ AsyncMock(return_value=_preference_row(user_id)),+ )++ @pytest.fixture+ def update(self, mocker: pytest_mock.MockFixture, user_id: str) -> AsyncMock:+ return mocker.patch.object(+ UserActions, "update", AsyncMock(return_value=_preference_row(user_id))+ )++ def test_a_valid_link_applies_its_choice(+ self, lookup: AsyncMock, update: AsyncMock, user_id: str+ ):+ response = _apply_footer_choice(user_id, "off")++ assert response.status_code == 200+ assert update.await_args.kwargs["data"]["briefingFrequency"] == (+ BriefingFrequency.OFF+ )++ def test_a_deleted_users_link_is_refused_like_a_forged_one(+ self, lookup: AsyncMock, update: AsyncMock, user_id: str+ ):+ lookup.side_effect = RecordNotFoundError({}, message="no User row")++ response = _apply_footer_choice(user_id, "off")++ assert (response.status_code, response.json()) == (+ 400,+ {"detail": "Invalid or expired link"},+ )+ update.assert_not_awaited()++ def test_a_user_deleted_before_the_update_is_refused_like_a_forged_one(+ self, lookup: AsyncMock, update: AsyncMock, user_id: str+ ):+ update.return_value = None++ response = _apply_footer_choice(user_id, "off")++ assert (response.status_code, response.json()) == (+ 400,+ {"detail": "Invalid or expired link"},+ )++ def test_a_database_failure_stays_a_server_error(+ self, lookup: AsyncMock, update: AsyncMock, user_id: str+ ):+ lookup.side_effect = DataError({"user_facing_error": {"message": "reset"}})++ response = _apply_footer_choice(user_id, "off")++ assert response.status_code == 500+++def _apply_footer_choice(user_id: str, choice: str) -> httpx.Response:+ link = generate_preference_link(user_id, choice)+ token = urllib.parse.parse_qs(urllib.parse.urlparse(link).query)["t"][0]+ return fastapi.testclient.TestClient(real_app, raise_server_exceptions=False).post(+ "/api/auth/user/preferences/from-email",+ params={"choice": choice, "token": token},+ )+++def _preference_row(user_id: str) -> PrismaUser:+ now = datetime.now(timezone.utc)+ return PrismaUser(+ id=user_id,+ email="[email protected]",+ emailVerified=True,+ createdAt=now,+ updatedAt=now,+ metadata="{}",+ integrations="",+ maxEmailsPerDay=3,+ briefingFrequency=BriefingFrequency.DAILY,+ alertsEnabled=True,+ notifyOnStoreVerdict=True,+ timezone="UTC",+ subscriptionTier=SubscriptionTier.NO_TIER,+ )diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.pyindex c2f9de6b66..f1d1e51ec1 100644--- a/autogpt_platform/backend/backend/data/user.py+++ b/autogpt_platform/backend/backend/data/user.py@@ -56,6 +56,7 @@ settings = Settings()# Cache decorator alias for consistent user lookup cachingcache_user_lookup = cached(maxsize=1000, ttl_seconds=300, shared_cache=True)+_INVALID_UNSUBSCRIBE_TOKEN = "Invalid unsubscribe token"class UserCreationResult(BaseModel):@@ -678,6 +679,8 @@ async def get_user_notification_preference(user_id: str) -> NotificationPreferentry:user = await PrismaUser.prisma().find_unique_or_raise(where={"id": user_id})return _preference_from_user(user)+ except RecordNotFoundError as e:+ raise NotFoundError(f"User not found with ID: {user_id}") from eexcept Exception as e:raise DatabaseError(f"Failed to get user notification preference for user {user_id}: {e}"@@ -703,12 +706,14 @@ async def update_user_notification_preference(user = await PrismaUser.prisma().update(where={"id": user_id}, data=update_data)if not user:- raise ValueError(f"User not found with ID: {user_id}")+ raise NotFoundError(f"User not found with ID: {user_id}")# Invalidate cache for this user since notification preferences are# part of user dataget_user_by_id.cache_delete(user_id)return _preference_from_user(user)+ except NotFoundError:+ raiseexcept Exception as e:raise DatabaseError(f"Failed to update user notification preference for user {user_id}: {e}"@@ -801,36 +806,49 @@ def generate_unsubscribe_link(user_id: str) -> str:async def unsubscribe_user_by_token(token: str) -> None:"""Unsubscribe a user from all notifications using the token"""+ user_id = _verify_unsubscribe_token(token)+ # A deleted user's token gets a forged token's answer, which names no user.+ email = await get_user_email_by_id(user_id)+ if email is None:+ raise ValueError(_INVALID_UNSUBSCRIBE_TOKEN)try:- # Decode the token- decoded = base64.urlsafe_b64decode(token).decode("utf-8")- user_id, received_signature_hex = decoded.split(":", 1)-- # Verify the signature- secret_key = settings.secrets.unsubscribe_secret_key- expected_signature = hmac.new(- secret_key.encode("utf-8"), user_id.encode("utf-8"), hashlib.sha256- ).digest()-- if not hmac.compare_digest(expected_signature.hex(), received_signature_hex):- raise ValueError("Invalid token signature")-- user = await get_user_by_id(user_id)# One-click unsubscribe turns everything off, including store# verdicts — this is the trapdoor, and the volume knob in the Briefing# footer is what most people should be using instead.await update_user_notification_preference(- user.id,+ user_id,NotificationPreferenceDTO(- email=user.email,+ email=email,briefing_frequency=BriefingFrequency.OFF,alerts_enabled=False,store_verdicts_enabled=False,daily_limit=0,),)- except Exception as e:- raise DatabaseError(f"Failed to unsubscribe user by token: {e}") from e+ except NotFoundError as e:+ raise ValueError(_INVALID_UNSUBSCRIBE_TOKEN) from e+++def _verify_unsubscribe_token(token: str) -> str:+ """Return the token's user id; a token we did not issue is a ValueError+ whose message leaves the token out, because the token never expires."""+ try:+ decoded = base64.urlsafe_b64decode(token).decode("utf-8")+ user_id, received_signature_hex = decoded.split(":", 1)+ except ValueError as e:+ raise ValueError(_INVALID_UNSUBSCRIBE_TOKEN) from e++ secret_key = settings.secrets.unsubscribe_secret_key+ expected_signature_hex = hmac.new(+ secret_key.encode("utf-8"), user_id.encode("utf-8"), hashlib.sha256+ ).hexdigest()+ # Bytes, because compare_digest raises TypeError on a non-ASCII str.+ if not hmac.compare_digest(+ expected_signature_hex.encode("utf-8"),+ received_signature_hex.encode("utf-8"),+ ):+ raise ValueError(_INVALID_UNSUBSCRIBE_TOKEN)+ return user_idasync def cleanup_user_managed_credentials(diff --git a/autogpt_platform/backend/backend/data/user_test.py b/autogpt_platform/backend/backend/data/user_test.pyindex 27288e44b6..0caa987786 100644--- a/autogpt_platform/backend/backend/data/user_test.py+++ b/autogpt_platform/backend/backend/data/user_test.py@@ -1994,7 +1994,7 @@ async def test_unsubscribe_error_does_not_repeat_the_token():never expires."""token = base64.urlsafe_b64encode(b"user-1:not-its-signature").decode()- with pytest.raises(DatabaseError) as exc:+ with pytest.raises(ValueError) as exc:await user_module.unsubscribe_user_by_token(token)assert token not in str(exc.value)diff --git a/autogpt_platform/backend/backend/util/request_url_logging_test.py b/autogpt_platform/backend/backend/util/request_url_logging_test.pyindex 8cfd901a0e..62c4be82f2 100644--- a/autogpt_platform/backend/backend/util/request_url_logging_test.py+++ b/autogpt_platform/backend/backend/util/request_url_logging_test.py@@ -30,6 +30,7 @@ from websockets.exceptions import InvalidStatusfrom backend.api import rest_api, ws_apifrom backend.api.conn_manager import ConnectionManager+from backend.api.features.integrations import router as integrations_routerfrom backend.api.test_helpers import override_configfrom backend.util import logging as service_loggingfrom backend.util import metrics@@ -109,6 +110,11 @@ async def test_websocket_server_logs_no_session_token(async def test_rest_server_logs_no_credentials_from_request_urls(service_output: io.StringIO, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ integrations_router,+ "get_webhook",+ AsyncMock(side_effect=RuntimeError("database unavailable")),+ )async with (_serve(rest_api.AgentServer(), monkeypatch) as host,httpx.AsyncClient(base_url=f"http://{host}") as client,@@ -118,10 +124,9 @@ async def test_rest_server_logs_no_credentials_from_request_urls(f"/api/public/shared/chats/{SHARE_TOKEN}/messages",params={"limit": "many", "token": SESSION_TOKEN},)- # No webhook manager handles this provider, so the request fails- # server-side and the 500 handler logs the request path.+ # The webhook lookup fails server-side, so the 500 handler logs the path.ingress = await client.post(- f"/api/integrations/unregistered/webhooks/{WEBHOOK_ID}/ingress",+ f"/api/integrations/github/webhooks/{WEBHOOK_ID}/ingress",params={"token": SESSION_TOKEN},json={},)@@ -133,7 +138,7 @@ async def test_rest_server_logs_no_credentials_from_request_urls(shared_chat = f"/api/public/shared/chats/{REDACTED}/messages"assert f"GET {shared_chat}?limit={REDACTED}&token={REDACTED} HTTP/1.1" in logsassert f"Validation failed for GET {shared_chat}" in logs- webhook = f"/api/integrations/unregistered/webhooks/{REDACTED}/ingress"+ webhook = f"/api/integrations/github/webhooks/{REDACTED}/ingress"assert f"POST {webhook}?token={REDACTED} HTTP/1.1" in logsassert f"POST {webhook} failed." in logs