Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdk): added agent option (#2108)
Rafael Miller6aae67bd8c0c
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex c0d6df090..0fb6631fd 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.3.3",+ "version": "4.3.4","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts b/apps/js-sdk/firecrawl/src/v2/methods/extract.tsindex e6165698c..bf9bca09f 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts@@ -1,4 +1,4 @@-import { type ExtractResponse, type ScrapeOptions } from "../types";+import { type ExtractResponse, type ScrapeOptions, type AgentOptions } from "../types";import { HttpClient } from "../utils/httpClient";import { ensureValidScrapeOptions } from "../utils/validation";import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";@@ -16,6 +16,7 @@ function prepareExtractPayload(args: {scrapeOptions?: ScrapeOptions;ignoreInvalidURLs?: boolean;integration?: string;+ agent?: AgentOptions;}): Record<string, unknown> {const body: Record<string, unknown> = {};if (args.urls) body.urls = args.urls;@@ -31,6 +32,7 @@ function prepareExtractPayload(args: {if (args.showSources != null) body.showSources = args.showSources;if (args.ignoreInvalidURLs != null) body.ignoreInvalidURLs = args.ignoreInvalidURLs;if (args.integration && args.integration.trim()) body.integration = args.integration.trim();+ if (args.agent) body.agent = args.agent;if (args.scrapeOptions) {ensureValidScrapeOptions(args.scrapeOptions);body.scrapeOptions = args.scrapeOptions;diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex ab5ccf438..e22b91606 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -337,6 +337,10 @@ export interface ExtractResponse {expiresAt?: string;}+export interface AgentOptions {+ model: "FIRE-1";+}+export interface ConcurrencyCheck {concurrency: number;maxConcurrency: number;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 3defbd54a..e4d2c1fa5 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.3.4"+__version__ = "4.3.5"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex 329259311..2f5a43a73 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -82,6 +82,7 @@ from .v2.types import (# Configuration typesClientConfig,+ AgentOptions,)__all__ = [@@ -162,4 +163,5 @@ __all__ = [# Configuration types'ClientConfig',+ 'AgentOptions',]\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex dffc6e2b8..d61349be2 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -37,6 +37,7 @@ from .types import (PDFAction,Location,PaginationConfig,+ AgentOptions,)from .utils.http_client import HttpClientfrom .utils.error_handler import FirecrawlError@@ -495,6 +496,7 @@ class FirecrawlClient:scrape_options: Optional['ScrapeOptions'] = None,ignore_invalid_urls: Optional[bool] = None,integration: Optional[str] = None,+ agent: Optional[AgentOptions] = None,):"""Start an extract job (non-blocking).@@ -508,7 +510,8 @@ class FirecrawlClient:show_sources: Include per-field/source mapping when availablescrape_options: Scrape options applied prior to extractionignore_invalid_urls: Skip invalid URLs instead of failing-+ integration: Integration tag/name+ agent: Agent configurationReturns:Response payload with job id/status (poll with get_extract_status)"""@@ -524,6 +527,7 @@ class FirecrawlClient:scrape_options=scrape_options,ignore_invalid_urls=ignore_invalid_urls,integration=integration,+ agent=agent,)def extract(@@ -541,6 +545,7 @@ class FirecrawlClient:poll_interval: int = 2,timeout: Optional[int] = None,integration: Optional[str] = None,+ agent: Optional[AgentOptions] = None,):"""Extract structured data and wait until completion.@@ -556,7 +561,8 @@ class FirecrawlClient:ignore_invalid_urls: Skip invalid URLs instead of failingpoll_interval: Seconds between status checkstimeout: Maximum seconds to wait (None for no timeout)-+ integration: Integration tag/name+ agent: Agent configurationReturns:Final extract response when completed"""@@ -574,6 +580,7 @@ class FirecrawlClient:poll_interval=poll_interval,timeout=timeout,integration=integration,+ agent=agent,)def start_batch_scrape(diff --git a/apps/python-sdk/firecrawl/v2/methods/extract.py b/apps/python-sdk/firecrawl/v2/methods/extract.pyindex 25d123ed7..87e4712db 100644--- a/apps/python-sdk/firecrawl/v2/methods/extract.py+++ b/apps/python-sdk/firecrawl/v2/methods/extract.py@@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optionalimport timefrom ..types import ExtractResponse, ScrapeOptions+from ..types import AgentOptionsfrom ..utils.http_client import HttpClientfrom ..utils.validation import prepare_scrape_optionsfrom ..utils.error_handler import handle_response_error@@ -19,6 +20,7 @@ def _prepare_extract_request(scrape_options: Optional[ScrapeOptions] = None,ignore_invalid_urls: Optional[bool] = None,integration: Optional[str] = None,+ agent: Optional[AgentOptions] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -43,6 +45,11 @@ def _prepare_extract_request(body["scrapeOptions"] = preparedif integration is not None and str(integration).strip():body["integration"] = str(integration).strip()+ if agent is not None:+ try:+ body["agent"] = agent.model_dump(exclude_none=True) # type: ignore[attr-defined]+ except AttributeError:+ body["agent"] = agent # fallbackreturn body@@ -59,6 +66,7 @@ def start_extract(scrape_options: Optional[ScrapeOptions] = None,ignore_invalid_urls: Optional[bool] = None,integration: Optional[str] = None,+ agent: Optional[AgentOptions] = None,) -> ExtractResponse:body = _prepare_extract_request(urls,@@ -71,6 +79,7 @@ def start_extract(scrape_options=scrape_options,ignore_invalid_urls=ignore_invalid_urls,integration=integration,+ agent=agent,)resp = client.post("/v2/extract", body)if not resp.ok:@@ -117,6 +126,7 @@ def extract(poll_interval: int = 2,timeout: Optional[int] = None,integration: Optional[str] = None,+ agent: Optional[AgentOptions] = None,) -> ExtractResponse:started = start_extract(client,@@ -130,9 +140,9 @@ def extract(scrape_options=scrape_options,ignore_invalid_urls=ignore_invalid_urls,integration=integration,+ agent=agent,)job_id = getattr(started, "id", None)if not job_id:return started- return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)-+ return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex a3a50e8e5..10a436869 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -114,6 +114,10 @@ class DocumentMetadata(BaseModel):def coerce_status_code_to_int(cls, v):return cls._coerce_string_to_int(v)+class AgentOptions(BaseModel):+ """Configuration for the agent in extract operations."""+ model: Literal["FIRE-1"] = "FIRE-1"+class AttributeResult(BaseModel):"""Result of attribute extraction."""selector: str@@ -497,6 +501,7 @@ class ExtractRequest(BaseModel):scrape_options: Optional[ScrapeOptions] = Noneignore_invalid_urls: Optional[bool] = Noneintegration: Optional[str] = None+ agent: Optional[AgentOptions] = Noneclass ExtractResponse(BaseModel):"""Response for extract operations (start/status/final)."""