AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): Fix Youtube blocking our cloud ips (#11456)
Swifty7d53c0de2757
diff --git a/autogpt_platform/backend/backend/blocks/youtube.py b/autogpt_platform/backend/backend/blocks/youtube.pyindex a397557a7f..322cac35a8 100644--- a/autogpt_platform/backend/backend/blocks/youtube.py+++ b/autogpt_platform/backend/backend/blocks/youtube.py@@ -1,9 +1,13 @@+import logging+from typing import Literalfrom urllib.parse import parse_qs, urlparse+from pydantic import SecretStrfrom youtube_transcript_api._api import YouTubeTranscriptApifrom youtube_transcript_api._errors import NoTranscriptFoundfrom youtube_transcript_api._transcripts import FetchedTranscriptfrom youtube_transcript_api.formatters import TextFormatter+from youtube_transcript_api.proxies import WebshareProxyConfigfrom backend.data.block import (Block,@@ -12,7 +16,42 @@ from backend.data.block import (BlockSchemaInput,BlockSchemaOutput,)-from backend.data.model import SchemaField+from backend.data.model import (+ CredentialsField,+ CredentialsMetaInput,+ SchemaField,+ UserPasswordCredentials,+)+from backend.integrations.providers import ProviderName++logger = logging.getLogger(__name__)++TEST_CREDENTIALS = UserPasswordCredentials(+ id="01234567-89ab-cdef-0123-456789abcdef",+ provider="webshare_proxy",+ username=SecretStr("mock-webshare-username"),+ password=SecretStr("mock-webshare-password"),+ title="Mock Webshare Proxy credentials",+)++TEST_CREDENTIALS_INPUT = {+ "provider": TEST_CREDENTIALS.provider,+ "id": TEST_CREDENTIALS.id,+ "type": TEST_CREDENTIALS.type,+ "title": TEST_CREDENTIALS.title,+}++WebshareProxyCredentials = UserPasswordCredentials+WebshareProxyCredentialsInput = CredentialsMetaInput[+ Literal[ProviderName.WEBSHARE_PROXY],+ Literal["user_password"],+]+++def WebshareProxyCredentialsField() -> WebshareProxyCredentialsInput:+ return CredentialsField(+ description="Webshare proxy credentials for fetching YouTube transcripts",+ )class TranscribeYoutubeVideoBlock(Block):@@ -22,6 +61,7 @@ class TranscribeYoutubeVideoBlock(Block):description="The URL of the YouTube video to transcribe",placeholder="https://www.youtube.com/watch?v=dQw4w9WgXcQ",)+ credentials: WebshareProxyCredentialsInput = WebshareProxyCredentialsField()class Output(BlockSchemaOutput):video_id: str = SchemaField(description="The extracted YouTube video ID")@@ -35,9 +75,12 @@ class TranscribeYoutubeVideoBlock(Block):id="f3a8f7e1-4b1d-4e5f-9f2a-7c3d5a2e6b4c",input_schema=TranscribeYoutubeVideoBlock.Input,output_schema=TranscribeYoutubeVideoBlock.Output,- description="Transcribes a YouTube video.",+ description="Transcribes a YouTube video using a proxy.",categories={BlockCategory.SOCIAL},- test_input={"youtube_url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},+ test_input={+ "youtube_url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ "credentials": TEST_CREDENTIALS_INPUT,+ },test_output=[("video_id", "dQw4w9WgXcQ"),(@@ -45,8 +88,9 @@ class TranscribeYoutubeVideoBlock(Block):"Never gonna give you up\nNever gonna let you down",),],+ test_credentials=TEST_CREDENTIALS,test_mock={- "get_transcript": lambda video_id: [+ "get_transcript": lambda video_id, credentials: [{"text": "Never gonna give you up"},{"text": "Never gonna let you down"},],@@ -69,16 +113,27 @@ class TranscribeYoutubeVideoBlock(Block):return parsed_url.path.split("/")[2]raise ValueError(f"Invalid YouTube URL: {url}")- @staticmethod- def get_transcript(video_id: str) -> FetchedTranscript:+ def get_transcript(+ self, video_id: str, credentials: WebshareProxyCredentials+ ) -> FetchedTranscript:"""Get transcript for a video, preferring English but falling back to any available language.:param video_id: The YouTube video ID+ :param credentials: The Webshare proxy credentials:return: The fetched transcript:raises: Any exception except NoTranscriptFound for requested languages"""- api = YouTubeTranscriptApi()+ logger.warning(+ "Using Webshare proxy for YouTube transcript fetch (video_id=%s)",+ video_id,+ )+ proxy_config = WebshareProxyConfig(+ proxy_username=credentials.username.get_secret_value(),+ proxy_password=credentials.password.get_secret_value(),+ )++ api = YouTubeTranscriptApi(proxy_config=proxy_config)try:# Try to get English transcript first (default behavior)return api.fetch(video_id=video_id)@@ -101,11 +156,17 @@ class TranscribeYoutubeVideoBlock(Block):transcript_text = formatter.format_transcript(transcript)return transcript_text- async def run(self, input_data: Input, **kwargs) -> BlockOutput:+ async def run(+ self,+ input_data: Input,+ *,+ credentials: WebshareProxyCredentials,+ **kwargs,+ ) -> BlockOutput:video_id = self.extract_video_id(input_data.youtube_url)yield "video_id", video_id- transcript = self.get_transcript(video_id)+ transcript = self.get_transcript(video_id, credentials)transcript_text = self.format_transcript(transcript=transcript)yield "transcript", transcript_textdiff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.pyindex 75ae346d5d..b1d36c2010 100644--- a/autogpt_platform/backend/backend/integrations/credentials_store.py+++ b/autogpt_platform/backend/backend/integrations/credentials_store.py@@ -15,6 +15,7 @@ from backend.data.model import (OAuth2Credentials,OAuthState,UserIntegrations,+ UserPasswordCredentials,)from backend.data.redis_client import get_redis_asyncfrom backend.util.settings import Settings@@ -207,6 +208,14 @@ v0_credentials = APIKeyCredentials(expires_at=None,)+webshare_proxy_credentials = UserPasswordCredentials(+ id="a5b3c7d9-2e4f-4a6b-8c1d-9e0f1a2b3c4d",+ provider="webshare_proxy",+ username=SecretStr(settings.secrets.webshare_proxy_username),+ password=SecretStr(settings.secrets.webshare_proxy_password),+ title="Use Credits for Webshare Proxy",+)+DEFAULT_CREDENTIALS = [ollama_credentials,revid_credentials,@@ -233,6 +242,7 @@ DEFAULT_CREDENTIALS = [google_maps_credentials,llama_api_credentials,v0_credentials,+ webshare_proxy_credentials,]@@ -321,6 +331,11 @@ class IntegrationCredentialsStore:all_credentials.append(zerobounce_credentials)if settings.secrets.google_maps_api_key:all_credentials.append(google_maps_credentials)+ if (+ settings.secrets.webshare_proxy_username+ and settings.secrets.webshare_proxy_password+ ):+ all_credentials.append(webshare_proxy_credentials)return all_credentialsasync def get_creds_by_id(diff --git a/autogpt_platform/backend/backend/integrations/providers.py b/autogpt_platform/backend/backend/integrations/providers.pyindex 3564ad32a8..3af5006ca4 100644--- a/autogpt_platform/backend/backend/integrations/providers.py+++ b/autogpt_platform/backend/backend/integrations/providers.py@@ -49,6 +49,7 @@ class ProviderName(str, Enum):TODOIST = "todoist"UNREAL_SPEECH = "unreal_speech"V0 = "v0"+ WEBSHARE_PROXY = "webshare_proxy"ZEROBOUNCE = "zerobounce"@classmethoddiff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex fcd009f820..ada6d362fe 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -571,6 +571,12 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings):open_router_api_key: str = Field(default="", description="Open Router API Key")llama_api_key: str = Field(default="", description="Llama API Key")v0_api_key: str = Field(default="", description="v0 by Vercel API key")+ webshare_proxy_username: str = Field(+ default="", description="Webshare Proxy Username"+ )+ webshare_proxy_password: str = Field(+ default="", description="Webshare Proxy Password"+ )reddit_client_id: str = Field(default="", description="Reddit client ID")reddit_client_secret: str = Field(default="", description="Reddit client secret")diff --git a/autogpt_platform/backend/test/blocks/test_youtube.py b/autogpt_platform/backend/test/blocks/test_youtube.pyindex 82c9311ff3..1af7c31b9b 100644--- a/autogpt_platform/backend/test/blocks/test_youtube.py+++ b/autogpt_platform/backend/test/blocks/test_youtube.py@@ -1,10 +1,14 @@from unittest.mock import Mock, patchimport pytest+from pydantic import SecretStrfrom youtube_transcript_api._errors import NoTranscriptFoundfrom youtube_transcript_api._transcripts import FetchedTranscript, Transcript+from youtube_transcript_api.proxies import WebshareProxyConfig-from backend.blocks.youtube import TranscribeYoutubeVideoBlock+from backend.blocks.youtube import TEST_CREDENTIALS, TranscribeYoutubeVideoBlock+from backend.data.model import UserPasswordCredentials+from backend.integrations.providers import ProviderNameclass TestTranscribeYoutubeVideoBlock:@@ -13,6 +17,7 @@ class TestTranscribeYoutubeVideoBlock:def setup_method(self):"""Set up test fixtures."""self.youtube_block = TranscribeYoutubeVideoBlock()+ self.credentials = TEST_CREDENTIALSdef test_extract_video_id_standard_url(self):"""Test extracting video ID from standard YouTube URL."""@@ -42,10 +47,41 @@ class TestTranscribeYoutubeVideoBlock:mock_api.fetch.return_value = mock_transcript# Execute- result = TranscribeYoutubeVideoBlock.get_transcript("test_video_id")+ result = self.youtube_block.get_transcript("test_video_id", self.credentials)# Assertassert result == mock_transcript+ mock_api_class.assert_called_once()+ proxy_config = mock_api_class.call_args[1]["proxy_config"]+ assert isinstance(proxy_config, WebshareProxyConfig)+ mock_api.fetch.assert_called_once_with(video_id="test_video_id")+ mock_api.list.assert_not_called()++ @patch("backend.blocks.youtube.YouTubeTranscriptApi")+ def test_get_transcript_with_custom_credentials(self, mock_api_class):+ """Test getting transcript with custom proxy credentials."""+ # Setup mock+ mock_api = Mock()+ mock_api_class.return_value = mock_api+ mock_transcript = Mock(spec=FetchedTranscript)+ mock_api.fetch.return_value = mock_transcript++ credentials = UserPasswordCredentials(+ provider=ProviderName.WEBSHARE_PROXY,+ username=SecretStr("custom_user"),+ password=SecretStr("custom_pass"),+ )++ # Execute+ result = self.youtube_block.get_transcript("test_video_id", credentials)++ # Assert+ assert result == mock_transcript+ mock_api_class.assert_called_once()+ proxy_config = mock_api_class.call_args[1]["proxy_config"]+ assert isinstance(proxy_config, WebshareProxyConfig)+ assert proxy_config.proxy_username == "custom_user"+ assert proxy_config.proxy_password == "custom_pass"mock_api.fetch.assert_called_once_with(video_id="test_video_id")mock_api.list.assert_not_called()@@ -74,10 +110,11 @@ class TestTranscribeYoutubeVideoBlock:mock_api.list.return_value = mock_transcript_list# Execute- result = TranscribeYoutubeVideoBlock.get_transcript("test_video_id")+ result = self.youtube_block.get_transcript("test_video_id", self.credentials)# Assertassert result == mock_fetched_transcript+ mock_api_class.assert_called_once()mock_api.fetch.assert_called_once_with(video_id="test_video_id")mock_api.list.assert_called_once_with("test_video_id")mock_transcript_hu.fetch.assert_called_once()@@ -109,10 +146,11 @@ class TestTranscribeYoutubeVideoBlock:mock_api.list.return_value = mock_transcript_list# Execute- result = TranscribeYoutubeVideoBlock.get_transcript("test_video_id")+ result = self.youtube_block.get_transcript("test_video_id", self.credentials)# Assert - should use manually created transcript firstassert result == mock_fetched_manual+ mock_api_class.assert_called_once()mock_transcript_manual.fetch.assert_called_once()mock_transcript_generated.fetch.assert_not_called()@@ -137,4 +175,5 @@ class TestTranscribeYoutubeVideoBlock:# Execute and assert exception is raisedwith pytest.raises(NoTranscriptFound):- TranscribeYoutubeVideoBlock.get_transcript("test_video_id")+ self.youtube_block.get_transcript("test_video_id", self.credentials)+ mock_api_class.assert_called_once()