Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(python-sdk): add agent parameter support to scrape_url method (#1919)

devin-ai-integration[bot]71829dbde33c
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 9119facd4..1213f98d2 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
-__version__ = "2.16.4"
+__version__ = "2.16.5"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py
index 84a1544b6..57c89773d 100644
--- a/apps/python-sdk/firecrawl/firecrawl.py
+++ b/apps/python-sdk/firecrawl/firecrawl.py
@@ -478,6 +478,7 @@ class FirecrawlApp:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
zero_data_retention: Optional[bool] = None,
+ agent: Optional[AgentOptions] = None,
**kwargs) -> ScrapeResponse[Any]:
"""
Scrape and extract content from a URL.
@@ -502,6 +503,7 @@ class FirecrawlApp:
actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings
zero_data_retention (Optional[bool]): Whether to delete data after scrape is done
+ agent (Optional[AgentOptions]): Agent configuration for FIRE-1 model
Returns:
@@ -574,6 +576,8 @@ class FirecrawlApp:
scrape_params['storeInCache'] = store_in_cache
if zero_data_retention is not None:
scrape_params['zeroDataRetention'] = zero_data_retention
+ if agent is not None:
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
scrape_params.update(kwargs)
@@ -2343,7 +2347,7 @@ class FirecrawlApp:
else:
error_message = f"Server returned empty response with status {response.status_code}"
error_details = "No additional details available"
-+ except ValueError:
+ except ValueError:
error_message = f"Server returned unreadable response with status {response.status_code}"
error_details = "No additional details available"
@@ -2606,7 +2610,7 @@ class FirecrawlApp:
method_params = {
"scrape_url": {"formats", "include_tags", "exclude_tags", "only_main_content", "wait_for",
"timeout", "location", "mobile", "skip_tls_verification", "remove_base64_images",
- "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "max_age", "integration"},
+ "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "max_age", "agent", "integration"},
"search": {"limit", "tbs", "filter", "lang", "country", "location", "timeout", "scrape_options", "integration"},
"crawl_url": {"include_paths", "exclude_paths", "max_depth", "max_discovery_depth", "limit",
"allow_backward_links", "allow_external_links", "ignore_sitemap", "scrape_options",
@@ -2992,6 +2996,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
extract: Optional[JsonConfig] = None,
json_options: Optional[JsonConfig] = None,
actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None,
+ agent: Optional[AgentOptions] = None,
**kwargs) -> ScrapeResponse[Any]:
"""
Scrape a single URL asynchronously.
@@ -3014,6 +3019,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
extract (Optional[JsonConfig]): Content extraction settings
json_options (Optional[JsonConfig]): JSON extraction settings
actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
+ agent (Optional[AgentOptions]): Agent configuration for FIRE-1 model
**kwargs: Additional parameters to pass to the API
Returns:
@@ -3083,6 +3089,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
if actions:
scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if agent is not None:
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:
scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])
if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']: