AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/integrations): store the scopes the provider granted, not the ones we asked for (#14391)
Reinier van der Leer0d06d080ce0d
diff --git a/autogpt_platform/backend/backend/blocks/airtable/_oauth.py b/autogpt_platform/backend/backend/blocks/airtable/_oauth.pyindex 9cd69ec3de..95db25a0c0 100644--- a/autogpt_platform/backend/backend/blocks/airtable/_oauth.py+++ b/autogpt_platform/backend/backend/blocks/airtable/_oauth.py@@ -7,7 +7,13 @@ from enum import Enumfrom logging import getLoggerfrom typing import Optional-from backend.sdk import BaseOAuthHandler, OAuth2Credentials, ProviderName, SecretStr+from backend.sdk import (+ BaseOAuthHandler,+ OAuth2Credentials,+ ProviderName,+ SecretStr,+ parse_granted_scopes,+)from ._api import (OAuthTokenResponse,@@ -129,7 +135,7 @@ class AirtableOAuthHandler(BaseOAuthHandler):access_token_expires_at=int(time.time()) + response.expires_in,refresh_token_expires_at=int(time.time()) + response.refresh_expires_in,provider=self.PROVIDER_NAME,- scopes=scopes,+ scopes=parse_granted_scopes(response.scope, fallback=scopes),)logger.debug(f"Access token expires in {response.expires_in} seconds")logger.debug(@@ -165,7 +171,9 @@ class AirtableOAuthHandler(BaseOAuthHandler):access_token_expires_at=int(time.time()) + response.expires_in,refresh_token_expires_at=int(time.time()) + response.refresh_expires_in,provider=self.PROVIDER_NAME,- scopes=self.scopes,+ # Not `self.scopes`: that is the full default set, which would+ # re-widen a credential the user granted narrowly.+ scopes=credentials.scopes,)logger.debug(f"New access token expires in {response.expires_in} seconds")logger.debug(diff --git a/autogpt_platform/backend/backend/blocks/mcp/oauth.py b/autogpt_platform/backend/backend/blocks/mcp/oauth.pyindex 2228336cd3..e0d2eecfdb 100644--- a/autogpt_platform/backend/backend/blocks/mcp/oauth.py+++ b/autogpt_platform/backend/backend/blocks/mcp/oauth.py@@ -14,7 +14,7 @@ from typing import ClassVar, Optionalfrom pydantic import SecretStrfrom backend.data.model import OAuth2Credentials-from backend.integrations.oauth.base import BaseOAuthHandler+from backend.integrations.oauth.base import BaseOAuthHandler, parse_granted_scopesfrom backend.integrations.providers import ProviderNamefrom backend.util.request import Requests@@ -126,7 +126,7 @@ class MCPOAuthHandler(BaseOAuthHandler):),access_token_expires_at=now + expires_in if expires_in else None,refresh_token_expires_at=None,- scopes=scopes,+ scopes=parse_granted_scopes(tokens.get("scope"), fallback=scopes),metadata={"mcp_token_url": self.token_url,"mcp_resource_url": self.resource_url,diff --git a/autogpt_platform/backend/backend/blocks/wordpress/_oauth.py b/autogpt_platform/backend/backend/blocks/wordpress/_oauth.pyindex 7dfb74107a..360d06ee52 100644--- a/autogpt_platform/backend/backend/blocks/wordpress/_oauth.py+++ b/autogpt_platform/backend/backend/blocks/wordpress/_oauth.py@@ -4,7 +4,13 @@ from logging import getLoggerfrom typing import Optionalfrom urllib.parse import quote-from backend.sdk import BaseOAuthHandler, OAuth2Credentials, ProviderName, SecretStr+from backend.sdk import (+ BaseOAuthHandler,+ OAuth2Credentials,+ ProviderName,+ SecretStr,+ parse_granted_scopes,+)from ._api import (OAuthTokenResponse,@@ -121,7 +127,9 @@ class WordPressOAuthHandler(BaseOAuthHandler):access_token_expires_at=None,refresh_token_expires_at=None,provider=self.PROVIDER_NAME,- scopes=scopes if scopes else [],+ # `scope` is only set on global tokens; a single-blog token+ # reports nothing, leaving the request as the only information.+ scopes=parse_granted_scopes(response.scope, fallback=scopes),metadata=metadata,)diff --git a/autogpt_platform/backend/backend/integrations/oauth/base.py b/autogpt_platform/backend/backend/integrations/oauth/base.pyindex f8e1babe26..e7e8b0bdf8 100644--- a/autogpt_platform/backend/backend/integrations/oauth/base.py+++ b/autogpt_platform/backend/backend/integrations/oauth/base.py@@ -1,4 +1,5 @@import logging+import reimport timefrom abc import ABC, abstractmethodfrom typing import ClassVar, Optional@@ -84,3 +85,13 @@ class BaseOAuthHandler(ABC):logger.debug(f"Using default scopes for provider {str(self.PROVIDER_NAME)}")scopes = self.DEFAULT_SCOPESreturn scopes+++def parse_granted_scopes(scope: str | None, fallback: list[str]) -> list[str]:+ """Scopes the provider says it granted, or `fallback` if it says nothing.++ Accepts either delimiter providers use: a wrong per-provider separator+ stores the whole string as one nonexistent scope.+ """+ granted = [s for s in re.split(r"[\s,]+", scope or "") if s]+ return granted or fallbackdiff --git a/autogpt_platform/backend/backend/integrations/oauth/github.py b/autogpt_platform/backend/backend/integrations/oauth/github.pyindex ebec116660..d6e6a9f7d7 100644--- a/autogpt_platform/backend/backend/integrations/oauth/github.py+++ b/autogpt_platform/backend/backend/integrations/oauth/github.py@@ -6,7 +6,7 @@ from backend.data.model import OAuth2Credentialsfrom backend.integrations.providers import ProviderNamefrom backend.util.request import Requests-from .base import BaseOAuthHandler+from .base import BaseOAuthHandler, parse_granted_scopes# --8<-- [start:GithubOAuthHandlerExample]@@ -79,7 +79,8 @@ class GitHubOAuthHandler(BaseOAuthHandler):{"refresh_token": credentials.refresh_token.get_secret_value(),"grant_type": "refresh_token",- }+ },+ current_credentials=credentials,)async def _request_tokens(@@ -106,11 +107,11 @@ class GitHubOAuthHandler(BaseOAuthHandler):title=current_credentials.title if current_credentials else None,username=username,access_token=token_data["access_token"],- # Token refresh responses have an empty `scope` property (see docs),- # so we have to get the scope from the existing credentials object.- scopes=(- token_data.get("scope", "").split(",")- or (current_credentials.scopes if current_credentials else [])+ # GitHub App refresh responses carry an empty `scope`, so on refresh+ # the existing record is the only description of the grant.+ scopes=parse_granted_scopes(+ token_data.get("scope"),+ fallback=current_credentials.scopes if current_credentials else [],),# Refresh token and expiration intervals are only given if token expiration# is enabled in the GitHub App's settings.diff --git a/autogpt_platform/backend/backend/integrations/oauth/granted_scopes_test.py b/autogpt_platform/backend/backend/integrations/oauth/granted_scopes_test.pynew file mode 100644index 0000000000..3023bf338d--- /dev/null+++ b/autogpt_platform/backend/backend/integrations/oauth/granted_scopes_test.py@@ -0,0 +1,265 @@+"""Every handler must persist the scopes the provider granted, not the ones we asked for.++A credential that claims a scope its token lacks passes the up-front coverage+check in `copilot/tools/utils.py` and then 403s at run time, and its inflated+scope list can win `_merge_or_create_credential`'s superset check and overwrite+a credential that really does hold the wider grant.+"""++from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from pydantic import SecretStr++from backend.blocks.airtable._api import OAuthTokenResponse as AirtableTokenResponse+from backend.blocks.airtable._oauth import AirtableOAuthHandler+from backend.blocks.mcp.oauth import MCPOAuthHandler+from backend.blocks.wordpress._api import OAuthTokenResponse as WordPressTokenResponse+from backend.blocks.wordpress._oauth import WordPressOAuthHandler+from backend.data.model import OAuth2Credentials+from backend.integrations.credentials_store import IntegrationCredentialsStore+from backend.integrations.oauth.base import parse_granted_scopes+from backend.integrations.oauth.github import GitHubOAuthHandler+from backend.integrations.oauth.reddit import RedditOAuthHandler+from backend.integrations.oauth.todoist import TodoistOAuthHandler+from backend.integrations.oauth.twitter import TwitterOAuthHandler++CLIENT = ("client-id", "client-secret", "https://localhost/callback")+++ "scope,fallback,expected",+ [+ ("identity read", ["a"], ["identity", "read"]), # RFC 6749 space-delimited+ ("repo,gist", ["a"], ["repo", "gist"]), # GitHub/Todoist comma-delimited+ ("", ["a", "b"], ["a", "b"]), # provider reports nothing+ (None, ["a"], ["a"]), # provider omits the field+ ],+)+def test_parse_granted_scopes(scope, fallback, expected):+ assert parse_granted_scopes(scope, fallback=fallback) == expected+++async def test_reddit_stores_the_narrower_granted_set():+ with mock_requests(+ "backend.integrations.oauth.reddit",+ post=[{"access_token": "at", "refresh_token": "rt", "scope": "identity read"}],+ get=[{"name": "alice"}],+ ):+ credentials = await RedditOAuthHandler(*CLIENT).exchange_code_for_tokens(+ "code", ["identity", "read", "submit"], None+ )++ assert credentials.scopes == ["identity", "read"]+++async def test_twitter_stores_the_narrower_granted_set():+ with mock_requests(+ "backend.integrations.oauth.twitter",+ post=[{"access_token": "at", "expires_in": 7200, "scope": "tweet.read"}],+ get=[{"data": {"username": "alice"}}],+ ):+ credentials = await TwitterOAuthHandler(*CLIENT).exchange_code_for_tokens(+ "code", ["tweet.read", "tweet.write"], "verifier"+ )++ assert credentials.scopes == ["tweet.read"]+++async def test_todoist_stores_granted_scopes_and_falls_back_when_absent():+ async def exchange(token_response: dict) -> OAuth2Credentials:+ with mock_requests(+ "backend.integrations.oauth.todoist",+ post=[token_response, {"user": {"email": "[email protected]"}}],+ ):+ return await TodoistOAuthHandler(*CLIENT).exchange_code_for_tokens(+ "code", ["data:read", "data:delete"], None+ )++ granted = await exchange({"access_token": "at", "scope": "data:read"})+ assert granted.scopes == ["data:read"]++ # Todoist's documented response carries no `scope` at all.+ silent = await exchange({"access_token": "at"})+ assert silent.scopes == ["data:read", "data:delete"]+++async def test_mcp_stores_the_narrower_granted_set():+ with mock_requests(+ "backend.blocks.mcp.oauth",+ post=[{"access_token": "at", "expires_in": 3600, "scope": "mcp:read"}],+ ):+ credentials = await MCPOAuthHandler(+ client_id="client-id",+ client_secret="client-secret",+ redirect_uri="https://localhost/callback",+ authorize_url="https://mcp.example.com/authorize",+ token_url="https://mcp.example.com/token",+ ).exchange_code_for_tokens("code", ["mcp:read", "mcp:write"], "verifier")++ assert credentials.scopes == ["mcp:read"]+++async def test_airtable_stores_the_narrower_granted_set():+ with patch(+ "backend.blocks.airtable._oauth.oauth_exchange_code_for_tokens",+ AsyncMock(return_value=airtable_token_response("data.records:read")),+ ):+ credentials = await AirtableOAuthHandler(*CLIENT).exchange_code_for_tokens(+ "code", ["data.records:read", "data.records:write"], "verifier"+ )++ assert credentials.scopes == ["data.records:read"]+++async def test_airtable_refresh_keeps_the_stored_scopes():+ """`self.scopes` is the full default set — using it re-widens a narrow grant."""+ stored = OAuth2Credentials(+ id="cred-1",+ provider="airtable",+ access_token=SecretStr("at"),+ refresh_token=SecretStr("rt"),+ scopes=["data.records:read"],+ )+ with patch(+ "backend.blocks.airtable._oauth.oauth_refresh_tokens",+ AsyncMock(return_value=airtable_token_response("data.records:read")),+ ):+ refreshed = await AirtableOAuthHandler(*CLIENT)._refresh_tokens(stored)++ assert refreshed.scopes == ["data.records:read"]+ assert set(refreshed.scopes) != set(AirtableOAuthHandler.DEFAULT_SCOPES)+++async def test_wordpress_stores_granted_scopes_and_falls_back_when_absent():+ async def exchange(scope: str | None) -> OAuth2Credentials:+ response = WordPressTokenResponse(access_token="at", scope=scope)+ with patch(+ "backend.blocks.wordpress._oauth.oauth_exchange_code_for_tokens",+ AsyncMock(return_value=response),+ ):+ return await WordPressOAuthHandler(*CLIENT).exchange_code_for_tokens(+ "code", ["posts", "comments"], None+ )++ granted = await exchange("posts")+ assert granted.scopes == ["posts"]++ # Single-blog tokens report no scope; the request is all we know.+ silent = await exchange(None)+ assert silent.scopes == ["posts", "comments"]+++async def test_github_refresh_keeps_the_existing_scopes_and_id():+ """GitHub App refreshes answer with `"scope": ""` — `"".split(",")` is `[""]`."""+ stored = OAuth2Credentials(+ id="cred-1",+ provider="github",+ access_token=SecretStr("old"),+ refresh_token=SecretStr("rt"),+ scopes=["repo", "gist"],+ )+ with mock_requests(+ "backend.integrations.oauth.github",+ post=[{"access_token": "new", "refresh_token": "rt2", "scope": ""}],+ get=[{"login": "alice"}],+ ):+ refreshed = await GitHubOAuthHandler(*CLIENT)._refresh_tokens(stored)++ assert refreshed.scopes == ["repo", "gist"]+ assert refreshed.id == "cred-1"+++async def test_github_exchange_stores_the_narrower_granted_set():+ with mock_requests(+ "backend.integrations.oauth.github",+ post=[{"access_token": "at", "scope": "repo"}],+ get=[{"login": "alice"}],+ ):+ credentials = await GitHubOAuthHandler(*CLIENT).exchange_code_for_tokens(+ "code", ["repo", "gist"], None+ )++ assert credentials.scopes == ["repo"]+++async def test_first_save_of_a_narrower_grant_is_a_create_not_an_update(mocker):+ """First save never reaches `update_creds`, so its narrowing refusal can't fire."""+ store = IntegrationCredentialsStore()+ persist = patch_store(mocker, store, persisted=[])++ await store.add_creds("user-a", oauth_credentials(["identity"]))++ persist.assert_awaited_once()+ assert persist.await_args.args[1][0].scopes == ["identity"]+++async def test_refresh_that_narrows_the_scopes_is_refused_by_the_store(mocker):+ """Why refresh carries the stored set forward instead of re-reading the grant."""+ store = IntegrationCredentialsStore()+ existing = oauth_credentials(["identity", "read"])+ patch_store(mocker, store, persisted=[existing])++ with pytest.raises(ValueError, match="more restrictive set of scopes"):+ await store.update_creds("user-a", oauth_credentials(["identity"]))++ await store.update_creds("user-a", oauth_credentials(["identity", "read"]))+++def mock_requests(+ module: str, *, post: list[dict] | None = None, get: list[dict] | None = None+):+ """Patch `Requests` in *module* to answer each call with the next payload."""++ def response(payload: dict) -> MagicMock:+ return MagicMock(+ ok=True, status=200, json=MagicMock(return_value=payload), text=""+ )++ client = MagicMock()+ client.post = AsyncMock(side_effect=[response(p) for p in post or []])+ client.get = AsyncMock(side_effect=[response(p) for p in get or []])+ return patch(f"{module}.Requests", return_value=client)+++def airtable_token_response(scope: str) -> AirtableTokenResponse:+ return AirtableTokenResponse(+ access_token="at",+ refresh_token="rt",+ token_type="Bearer",+ scope=scope,+ expires_in=3600,+ refresh_expires_in=86400,+ )+++def oauth_credentials(scopes: list[str]) -> OAuth2Credentials:+ return OAuth2Credentials(+ id="cred-1",+ provider="reddit",+ access_token=SecretStr("at"),+ refresh_token=SecretStr("rt"),+ scopes=scopes,+ username="alice",+ )+++def patch_store(mocker, store: IntegrationCredentialsStore, persisted: list):+ mocker.patch.object(+ store, "_get_persisted_user_creds_unlocked", AsyncMock(return_value=persisted)+ )+ mocker.patch.object(store, "locked_user_integrations", AsyncMock())+ return mocker.patch.object(+ store, "_set_user_integration_creds", new_callable=AsyncMock+ )diff --git a/autogpt_platform/backend/backend/integrations/oauth/reddit.py b/autogpt_platform/backend/backend/integrations/oauth/reddit.pyindex a69e1e62c7..33ecc866d7 100644--- a/autogpt_platform/backend/backend/integrations/oauth/reddit.py+++ b/autogpt_platform/backend/backend/integrations/oauth/reddit.py@@ -5,7 +5,7 @@ from typing import ClassVar, Optionalfrom pydantic import SecretStrfrom backend.data.model import OAuth2Credentials-from backend.integrations.oauth.base import BaseOAuthHandler+from backend.integrations.oauth.base import BaseOAuthHandler, parse_granted_scopesfrom backend.integrations.providers import ProviderNamefrom backend.util.request import Requestsfrom backend.util.settings import Settings@@ -110,7 +110,7 @@ class RedditOAuthHandler(BaseOAuthHandler):refresh_token=tokens.get("refresh_token"),access_token_expires_at=int(time.time()) + tokens.get("expires_in", 3600),refresh_token_expires_at=None, # Reddit refresh tokens don't expire- scopes=scopes,+ scopes=parse_granted_scopes(tokens.get("scope"), fallback=scopes),)async def _get_username(self, access_token: str) -> str:diff --git a/autogpt_platform/backend/backend/integrations/oauth/todoist.py b/autogpt_platform/backend/backend/integrations/oauth/todoist.pyindex 66d34f95e8..ff879cc40d 100644--- a/autogpt_platform/backend/backend/integrations/oauth/todoist.py+++ b/autogpt_platform/backend/backend/integrations/oauth/todoist.py@@ -2,7 +2,7 @@ import urllib.parsefrom typing import ClassVar, Optionalfrom backend.data.model import OAuth2Credentials, ProviderName-from backend.integrations.oauth.base import BaseOAuthHandler+from backend.integrations.oauth.base import BaseOAuthHandler, parse_granted_scopesfrom backend.util.request import Requests@@ -66,7 +66,9 @@ class TodoistOAuthHandler(BaseOAuthHandler):refresh_token=None,access_token_expires_at=None,refresh_token_expires_at=None,- scopes=scopes,+ # Todoist's token response documents no `scope`; the request is then+ # all we know about the grant.+ scopes=parse_granted_scopes(tokens.get("scope"), fallback=scopes),)async def _refresh_tokens(diff --git a/autogpt_platform/backend/backend/integrations/oauth/twitter.py b/autogpt_platform/backend/backend/integrations/oauth/twitter.pyindex 486c14a210..2e73674b4e 100644--- a/autogpt_platform/backend/backend/integrations/oauth/twitter.py+++ b/autogpt_platform/backend/backend/integrations/oauth/twitter.py@@ -3,7 +3,7 @@ import urllib.parsefrom typing import ClassVar, Optionalfrom backend.data.model import OAuth2Credentials, ProviderName-from backend.integrations.oauth.base import BaseOAuthHandler+from backend.integrations.oauth.base import BaseOAuthHandler, parse_granted_scopesfrom backend.util.request import Requests@@ -92,7 +92,7 @@ class TwitterOAuthHandler(BaseOAuthHandler):refresh_token=tokens.get("refresh_token"),access_token_expires_at=int(time.time()) + tokens["expires_in"],refresh_token_expires_at=None,- scopes=scopes,+ scopes=parse_granted_scopes(tokens.get("scope"), fallback=scopes),)async def _get_username(self, access_token: str) -> str:diff --git a/autogpt_platform/backend/backend/sdk/__init__.py b/autogpt_platform/backend/backend/sdk/__init__.pyindex dc7260d08f..5aab1bd22b 100644--- a/autogpt_platform/backend/backend/sdk/__init__.py+++ b/autogpt_platform/backend/backend/sdk/__init__.py@@ -104,9 +104,10 @@ except ImportError:# OAuth handlerstry:- from backend.integrations.oauth.base import BaseOAuthHandler+ from backend.integrations.oauth.base import BaseOAuthHandler, parse_granted_scopesexcept ImportError:BaseOAuthHandler = None+ parse_granted_scopes = None# Credential type with proper provider name@@ -151,6 +152,7 @@ __all__ = ["update_webhook",# Provider-Specific (when available)"BaseOAuthHandler",+ "parse_granted_scopes",# Utilities"json","store_media_file",