AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
Handle errors in Jina ExtractWebsiteContentBlock (#12048)
DEEVEN SERUb8f5c208d08e
diff --git a/autogpt_platform/backend/backend/blocks/jina/search.py b/autogpt_platform/backend/backend/blocks/jina/search.pyindex 22a883fa03..5e58ddcab4 100644--- a/autogpt_platform/backend/backend/blocks/jina/search.py+++ b/autogpt_platform/backend/backend/blocks/jina/search.py@@ -17,6 +17,7 @@ from backend.blocks.jina._auth import (from backend.blocks.search import GetRequestfrom backend.data.model import SchemaFieldfrom backend.util.exceptions import BlockExecutionError+from backend.util.request import HTTPClientError, HTTPServerError, validate_urlclass SearchTheWebBlock(Block, GetRequest):@@ -110,7 +111,12 @@ class ExtractWebsiteContentBlock(Block, GetRequest):self, input_data: Input, *, credentials: JinaCredentials, **kwargs) -> BlockOutput:if input_data.raw_content:- url = input_data.url+ try:+ parsed_url, _, _ = await validate_url(input_data.url, [])+ url = parsed_url.geturl()+ except ValueError as e:+ yield "error", f"Invalid URL: {e}"+ returnheaders = {}else:url = f"https://r.jina.ai/{input_data.url}"@@ -119,5 +125,20 @@ class ExtractWebsiteContentBlock(Block, GetRequest):"Authorization": f"Bearer {credentials.api_key.get_secret_value()}",}- content = await self.get_request(url, json=False, headers=headers)+ try:+ content = await self.get_request(url, json=False, headers=headers)+ except HTTPClientError as e:+ yield "error", f"Client error ({e.status_code}) fetching {input_data.url}: {e}"+ return+ except HTTPServerError as e:+ yield "error", f"Server error ({e.status_code}) fetching {input_data.url}: {e}"+ return+ except Exception as e:+ yield "error", f"Failed to fetch {input_data.url}: {e}"+ return++ if not content:+ yield "error", f"No content returned for {input_data.url}"+ return+yield "content", contentdiff --git a/autogpt_platform/backend/test/blocks/test_jina_extract_website.py b/autogpt_platform/backend/test/blocks/test_jina_extract_website.pynew file mode 100644index 0000000000..335c43f966--- /dev/null+++ b/autogpt_platform/backend/test/blocks/test_jina_extract_website.py@@ -0,0 +1,66 @@+from typing import cast++import pytest++from backend.blocks.jina._auth import (+ TEST_CREDENTIALS,+ TEST_CREDENTIALS_INPUT,+ JinaCredentialsInput,+)+from backend.blocks.jina.search import ExtractWebsiteContentBlock+from backend.util.request import HTTPClientError+++async def test_extract_website_content_returns_content(monkeypatch):+ block = ExtractWebsiteContentBlock()+ input_data = block.Input(+ url="https://example.com",+ credentials=cast(JinaCredentialsInput, TEST_CREDENTIALS_INPUT),+ raw_content=True,+ )++ async def fake_get_request(url, json=False, headers=None):+ assert url == "https://example.com"+ assert headers == {}+ return "page content"++ monkeypatch.setattr(block, "get_request", fake_get_request)++ results = [+ output+ async for output in block.run(+ input_data=input_data, credentials=TEST_CREDENTIALS+ )+ ]++ assert ("content", "page content") in results+ assert all(key != "error" for key, _ in results)+++async def test_extract_website_content_handles_http_error(monkeypatch):+ block = ExtractWebsiteContentBlock()+ input_data = block.Input(+ url="https://example.com",+ credentials=cast(JinaCredentialsInput, TEST_CREDENTIALS_INPUT),+ raw_content=False,+ )++ async def fake_get_request(url, json=False, headers=None):+ raise HTTPClientError("HTTP 400 Error: Bad Request", 400)++ monkeypatch.setattr(block, "get_request", fake_get_request)++ results = [+ output+ async for output in block.run(+ input_data=input_data, credentials=TEST_CREDENTIALS+ )+ ]++ assert ("content", "page content") not in results+ error_messages = [value for key, value in results if key == "error"]+ assert error_messages+ assert "Client error (400)" in error_messages[0]+ assert "https://example.com" in error_messages[0]