AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
chore(backend/deps): Bump firecrawl-py from 2.16.3 to 4.3.1 in /autogpt_platform/backend (#10809)
dependabot[bot]f5ee579ab203
diff --git a/autogpt_platform/backend/backend/blocks/firecrawl/_api.py b/autogpt_platform/backend/backend/blocks/firecrawl/_api.pyindex e69de29bb2..c4a5e8e1af 100644--- a/autogpt_platform/backend/backend/blocks/firecrawl/_api.py+++ b/autogpt_platform/backend/backend/blocks/firecrawl/_api.py@@ -0,0 +1,12 @@+from enum import Enum+++class ScrapeFormat(Enum):+ MARKDOWN = "markdown"+ HTML = "html"+ RAW_HTML = "rawHtml"+ LINKS = "links"+ SCREENSHOT = "screenshot"+ SCREENSHOT_FULL_PAGE = "screenshot@fullPage"+ JSON = "json"+ CHANGE_TRACKING = "changeTracking"diff --git a/autogpt_platform/backend/backend/blocks/firecrawl/_format_utils.py b/autogpt_platform/backend/backend/blocks/firecrawl/_format_utils.pynew file mode 100644index 0000000000..45ccefe9f4--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/firecrawl/_format_utils.py@@ -0,0 +1,28 @@+"""Utility functions for converting between our ScrapeFormat enum and firecrawl FormatOption types."""++from typing import List++from firecrawl.v2.types import FormatOption, ScreenshotFormat++from backend.blocks.firecrawl._api import ScrapeFormat+++def convert_to_format_options(+ formats: List[ScrapeFormat],+) -> List[FormatOption]:+ """Convert our ScrapeFormat enum values to firecrawl FormatOption types.++ Handles special cases like screenshot@fullPage which needs to be converted+ to a ScreenshotFormat object.+ """+ result: List[FormatOption] = []++ for format_enum in formats:+ if format_enum.value == "screenshot@fullPage":+ # Special case: convert to ScreenshotFormat with full_page=True+ result.append(ScreenshotFormat(type="screenshot", full_page=True))+ else:+ # Regular string literals+ result.append(format_enum.value)++ return resultdiff --git a/autogpt_platform/backend/backend/blocks/firecrawl/crawl.py b/autogpt_platform/backend/backend/blocks/firecrawl/crawl.pyindex 6b8e1e0ad8..b20452f777 100644--- a/autogpt_platform/backend/backend/blocks/firecrawl/crawl.py+++ b/autogpt_platform/backend/backend/blocks/firecrawl/crawl.py@@ -1,8 +1,9 @@-from enum import Enumfrom typing import Any-from firecrawl import FirecrawlApp, ScrapeOptions+from firecrawl import FirecrawlApp+from firecrawl.v2.types import ScrapeOptions+from backend.blocks.firecrawl._api import ScrapeFormatfrom backend.sdk import (APIKeyCredentials,Block,@@ -14,21 +15,10 @@ from backend.sdk import ()from ._config import firecrawl---class ScrapeFormat(Enum):- MARKDOWN = "markdown"- HTML = "html"- RAW_HTML = "rawHtml"- LINKS = "links"- SCREENSHOT = "screenshot"- SCREENSHOT_FULL_PAGE = "screenshot@fullPage"- JSON = "json"- CHANGE_TRACKING = "changeTracking"+from ._format_utils import convert_to_format_optionsclass FirecrawlCrawlBlock(Block):-class Input(BlockSchema):credentials: CredentialsMetaInput = firecrawl.credentials_field()url: str = SchemaField(description="The URL to crawl")@@ -78,18 +68,17 @@ class FirecrawlCrawlBlock(Block):async def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:-app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())# Sync call- crawl_result = app.crawl_url(+ crawl_result = app.crawl(input_data.url,limit=input_data.limit,scrape_options=ScrapeOptions(- formats=[format.value for format in input_data.formats],- onlyMainContent=input_data.only_main_content,- maxAge=input_data.max_age,- waitFor=input_data.wait_for,+ formats=convert_to_format_options(input_data.formats),+ only_main_content=input_data.only_main_content,+ max_age=input_data.max_age,+ wait_for=input_data.wait_for,),)yield "data", crawl_result.data@@ -101,7 +90,7 @@ class FirecrawlCrawlBlock(Block):elif f == ScrapeFormat.HTML:yield "html", data.htmlelif f == ScrapeFormat.RAW_HTML:- yield "raw_html", data.rawHtml+ yield "raw_html", data.raw_htmlelif f == ScrapeFormat.LINKS:yield "links", data.linkselif f == ScrapeFormat.SCREENSHOT:@@ -109,6 +98,6 @@ class FirecrawlCrawlBlock(Block):elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:yield "screenshot_full_page", data.screenshotelif f == ScrapeFormat.CHANGE_TRACKING:- yield "change_tracking", data.changeTracking+ yield "change_tracking", data.change_trackingelif f == ScrapeFormat.JSON:yield "json", data.jsondiff --git a/autogpt_platform/backend/backend/blocks/firecrawl/extract.py b/autogpt_platform/backend/backend/blocks/firecrawl/extract.pyindex 8f3b507bec..caef24d126 100755--- a/autogpt_platform/backend/backend/blocks/firecrawl/extract.py+++ b/autogpt_platform/backend/backend/blocks/firecrawl/extract.py@@ -20,7 +20,6 @@ from ._config import firecrawl@cost(BlockCost(2, BlockCostType.RUN))class FirecrawlExtractBlock(Block):-class Input(BlockSchema):credentials: CredentialsMetaInput = firecrawl.credentials_field()urls: list[str] = SchemaField(@@ -53,7 +52,6 @@ class FirecrawlExtractBlock(Block):async def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:-app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())extract_result = app.extract(diff --git a/autogpt_platform/backend/backend/blocks/firecrawl/map.py b/autogpt_platform/backend/backend/blocks/firecrawl/map.pyindex 7661377901..3e33f90461 100644--- a/autogpt_platform/backend/backend/blocks/firecrawl/map.py+++ b/autogpt_platform/backend/backend/blocks/firecrawl/map.py@@ -1,3 +1,5 @@+from typing import Any+from firecrawl import FirecrawlAppfrom backend.sdk import (@@ -14,14 +16,16 @@ from ._config import firecrawlclass FirecrawlMapWebsiteBlock(Block):-class Input(BlockSchema):credentials: CredentialsMetaInput = firecrawl.credentials_field()url: str = SchemaField(description="The website url to map")class Output(BlockSchema):- links: list[str] = SchemaField(description="The links of the website")+ links: list[str] = SchemaField(description="List of URLs found on the website")+ results: list[dict[str, Any]] = SchemaField(+ description="List of search results with url, title, and description"+ )def __init__(self):super().__init__(@@ -35,12 +39,22 @@ class FirecrawlMapWebsiteBlock(Block):async def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:-app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())# Sync call- map_result = app.map_url(+ map_result = app.map(url=input_data.url,)- yield "links", map_result.links+ # Convert SearchResult objects to dicts+ results_data = [+ {+ "url": link.url,+ "title": link.title,+ "description": link.description,+ }+ for link in map_result.links+ ]++ yield "links", [link.url for link in map_result.links]+ yield "results", results_datadiff --git a/autogpt_platform/backend/backend/blocks/firecrawl/scrape.py b/autogpt_platform/backend/backend/blocks/firecrawl/scrape.pyindex 65627ad954..2adde7e6d2 100644--- a/autogpt_platform/backend/backend/blocks/firecrawl/scrape.py+++ b/autogpt_platform/backend/backend/blocks/firecrawl/scrape.py@@ -1,8 +1,8 @@-from enum import Enumfrom typing import Anyfrom firecrawl import FirecrawlApp+from backend.blocks.firecrawl._api import ScrapeFormatfrom backend.sdk import (APIKeyCredentials,Block,@@ -14,21 +14,10 @@ from backend.sdk import ()from ._config import firecrawl---class ScrapeFormat(Enum):- MARKDOWN = "markdown"- HTML = "html"- RAW_HTML = "rawHtml"- LINKS = "links"- SCREENSHOT = "screenshot"- SCREENSHOT_FULL_PAGE = "screenshot@fullPage"- JSON = "json"- CHANGE_TRACKING = "changeTracking"+from ._format_utils import convert_to_format_optionsclass FirecrawlScrapeBlock(Block):-class Input(BlockSchema):credentials: CredentialsMetaInput = firecrawl.credentials_field()url: str = SchemaField(description="The URL to crawl")@@ -78,12 +67,11 @@ class FirecrawlScrapeBlock(Block):async def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:-app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())- scrape_result = app.scrape_url(+ scrape_result = app.scrape(input_data.url,- formats=[format.value for format in input_data.formats],+ formats=convert_to_format_options(input_data.formats),only_main_content=input_data.only_main_content,max_age=input_data.max_age,wait_for=input_data.wait_for,@@ -96,7 +84,7 @@ class FirecrawlScrapeBlock(Block):elif f == ScrapeFormat.HTML:yield "html", scrape_result.htmlelif f == ScrapeFormat.RAW_HTML:- yield "raw_html", scrape_result.rawHtml+ yield "raw_html", scrape_result.raw_htmlelif f == ScrapeFormat.LINKS:yield "links", scrape_result.linkselif f == ScrapeFormat.SCREENSHOT:@@ -104,6 +92,6 @@ class FirecrawlScrapeBlock(Block):elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:yield "screenshot_full_page", scrape_result.screenshotelif f == ScrapeFormat.CHANGE_TRACKING:- yield "change_tracking", scrape_result.changeTracking+ yield "change_tracking", scrape_result.change_trackingelif f == ScrapeFormat.JSON:yield "json", scrape_result.jsondiff --git a/autogpt_platform/backend/backend/blocks/firecrawl/search.py b/autogpt_platform/backend/backend/blocks/firecrawl/search.pyindex 521d0f7e35..7af8796111 100644--- a/autogpt_platform/backend/backend/blocks/firecrawl/search.py+++ b/autogpt_platform/backend/backend/blocks/firecrawl/search.py@@ -1,8 +1,9 @@-from enum import Enumfrom typing import Any-from firecrawl import FirecrawlApp, ScrapeOptions+from firecrawl import FirecrawlApp+from firecrawl.v2.types import ScrapeOptions+from backend.blocks.firecrawl._api import ScrapeFormatfrom backend.sdk import (APIKeyCredentials,Block,@@ -14,21 +15,10 @@ from backend.sdk import ()from ._config import firecrawl---class ScrapeFormat(Enum):- MARKDOWN = "markdown"- HTML = "html"- RAW_HTML = "rawHtml"- LINKS = "links"- SCREENSHOT = "screenshot"- SCREENSHOT_FULL_PAGE = "screenshot@fullPage"- JSON = "json"- CHANGE_TRACKING = "changeTracking"+from ._format_utils import convert_to_format_optionsclass FirecrawlSearchBlock(Block):-class Input(BlockSchema):credentials: CredentialsMetaInput = firecrawl.credentials_field()query: str = SchemaField(description="The query to search for")@@ -61,7 +51,6 @@ class FirecrawlSearchBlock(Block):async def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:-app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())# Sync call@@ -69,11 +58,12 @@ class FirecrawlSearchBlock(Block):input_data.query,limit=input_data.limit,scrape_options=ScrapeOptions(- formats=[format.value for format in input_data.formats],- maxAge=input_data.max_age,- waitFor=input_data.wait_for,+ formats=convert_to_format_options(input_data.formats) or None,+ max_age=input_data.max_age,+ wait_for=input_data.wait_for,),)yield "data", scrape_result- for site in scrape_result.data:- yield "site", site+ if hasattr(scrape_result, "web") and scrape_result.web:+ for site in scrape_result.web:+ yield "site", sitediff --git a/autogpt_platform/backend/poetry.lock b/autogpt_platform/backend/poetry.lockindex ce769d4272..833006f04f 100644--- a/autogpt_platform/backend/poetry.lock+++ b/autogpt_platform/backend/poetry.lock@@ -1339,20 +1339,21 @@ packaging = ">=20"[[package]]name = "firecrawl-py"-version = "2.16.3"+version = "4.3.6"description = "Python SDK for Firecrawl API"optional = falsepython-versions = ">=3.8"groups = ["main"]files = [- {file = "firecrawl_py-2.16.3-py3-none-any.whl", hash = "sha256:94bb46af5e0df6c8ec414ac999a5355c0f5a46f15fd1cf5a02a3b31062db0aa8"},- {file = "firecrawl_py-2.16.3.tar.gz", hash = "sha256:5fd063ef4acc4c4be62648f1e11467336bc127780b3afc28d39078a012e6a14c"},+ {file = "firecrawl_py-4.3.6-py3-none-any.whl", hash = "sha256:9b5dffdf5ed08fdbf0966f17e18c1a034d59f42a20b2bf9a6291a83190d7eb0f"},+ {file = "firecrawl_py-4.3.6.tar.gz", hash = "sha256:303827a86d0f6237a8ddcaa0bcdaa4c5ee11d9a4880b0685302b8d9a0e191ee0"},][package.dependencies]aiohttp = "*"+httpx = "*"nest-asyncio = "*"-pydantic = "*"+pydantic = ">=2.0"python-dotenv = "*"requests = "*"websockets = "*"@@ -7273,4 +7274,4 @@ cffi = ["cffi (>=1.11)"][metadata]lock-version = "2.1"python-versions = ">=3.10,<3.14"-content-hash = "a19fcce9d4ab88f14eb1e5baa83e1e5a90c9995b04b84dae7cfa6257cf19012c"+content-hash = "2c9e0e6e1b4b7a5293cc4c9296937479e7c52c7d8d13315c76e3ae0e7833465d"diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex a9ad38a639..5f24c65960 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -78,7 +78,7 @@ aioclamd = "^1.0.0"setuptools = "^80.9.0"gcloud-aio-storage = "^9.5.0"pandas = "^2.3.1"-firecrawl-py = "^2.16.3"+firecrawl-py = "^4.3.6"exa-py = "^1.14.20"croniter = "^6.0.0"stagehand = "^0.5.1"