Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk): extract v2
rafaelmmiller10b720289801
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex 93f489718..77b038c1d 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -21,24 +21,24 @@ def main():firecrawl = Firecrawl(api_key=api_key, api_url=api_url)- # # Scrape- # doc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"])- # print("scrape:", doc.markdown)-- # # Crawl (waits until terminal state)- # crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)- # print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total)-- # # Batch scrape- # batch = firecrawl.batch_scrape([- # "https://docs.firecrawl.dev",- # "https://firecrawl.dev",- # ], formats=["markdown"], poll_interval=1, wait_timeout=120)- # print("batch:", batch.status, batch.completed, "/", batch.total)-- # # Search- # search_response = firecrawl.search(query="What is the capital of France?", limit=5)- # print("search web results:", len(getattr(search_response, "web", []) or []))+ # Scrape+ doc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"])+ print("scrape:", doc.markdown)++ # Crawl (waits until terminal state)+ crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)+ print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total)++ # Batch scrape+ batch = firecrawl.batch_scrape([+ "https://docs.firecrawl.dev",+ "https://firecrawl.dev",+ ], formats=["markdown"], poll_interval=1, wait_timeout=120)+ print("batch:", batch.status, batch.completed, "/", batch.total)++ # Search+ search_response = firecrawl.search(query="What is the capital of France?", limit=5)+ print("search web results:", len(getattr(search_response, "web", []) or []))# Mapmap_response = firecrawl.map("https://firecrawl.dev")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 08e90493c..c9512ff45 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -44,6 +44,7 @@ from .methods import search as search_modulefrom .methods import map as map_modulefrom .methods import batch as batch_methodsfrom .methods import usage as usage_methods+from .methods import extract as extract_modulefrom .watcher import Watcherclass FirecrawlClient:@@ -439,25 +440,55 @@ class FirecrawlClient:request = CrawlParamsRequest(url=url, prompt=prompt)return crawl_module.crawl_params_preview(self.http_client, request)+ def start_extract(+ self,+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional['ScrapeOptions'] = None,+ ignore_invalid_urls: Optional[bool] = None,+ ):+ """+ Start an extract job (non-blocking). Use get_extract_status to poll.+ """+ return extract_module.start_extract(+ self.http_client,+ urls,+ prompt=prompt,+ schema=schema,+ system_prompt=system_prompt,+ allow_external_links=allow_external_links,+ enable_web_search=enable_web_search,+ show_sources=show_sources,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ )+def extract(self,urls: Optional[List[str]] = None,*,prompt: Optional[str] = None,- schema: Optional[Any] = None,+ schema: Optional[Dict[str, Any]] = None,system_prompt: Optional[str] = None,- allow_external_links: Optional[bool] = False,- enable_web_search: Optional[bool] = False,- show_sources: Optional[bool] = False,- agent: Optional[Dict[str, Any]] = None,- **kwargs,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional['ScrapeOptions'] = None,+ ignore_invalid_urls: Optional[bool] = None,+ poll_interval: int = 2,+ timeout: Optional[int] = None,):"""- Extract data from a list of URLs.+ Extract structured data using the v2 client surface, waiting until completion."""- from ..v1.client import V1FirecrawlApp- v1 = V1FirecrawlApp(api_key=self.config.api_key, api_url=self.config.api_url)- return v1.extract(+ return extract_module.extract(+ self.http_client,urls,prompt=prompt,schema=schema,@@ -465,8 +496,10 @@ class FirecrawlClient:allow_external_links=allow_external_links,enable_web_search=enable_web_search,show_sources=show_sources,- agent=agent,- **kwargs,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ poll_interval=poll_interval,+ timeout=timeout,)def start_batch_scrape(@@ -553,10 +586,7 @@ class FirecrawlClient:return batch_methods.get_batch_scrape_errors(self.http_client, job_id)def get_extract_status(self, job_id: str):- """Proxy to v1 get_extract_status while v2 extract is not implemented."""- from ..v1.client import V1FirecrawlApp- v1 = V1FirecrawlApp(api_key=self.config.api_key, api_url=self.config.api_url)- return v1.get_extract_status(job_id)+ return extract_module.get_extract_status(self.http_client, job_id)def get_concurrency(self):"""Get current concurrency and maximum allowed for this team/key (v2)."""diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 8e53059a2..3a8102801 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -41,6 +41,7 @@ from .methods.aio import crawl as async_crawl # type: ignore[attr-defined]from .methods.aio import search as async_search # type: ignore[attr-defined]from .methods.aio import map as async_map # type: ignore[attr-defined]from .methods.aio import usage as async_usage # type: ignore[attr-defined]+from .methods.aio import extract as async_extract # type: ignore[attr-defined]from .watcher_async import AsyncWatcher@@ -169,17 +170,18 @@ class AsyncFirecrawlClient:urls: Optional[List[str]] = None,*,prompt: Optional[str] = None,- schema: Optional[Any] = None,+ schema: Optional[Dict[str, Any]] = None,system_prompt: Optional[str] = None,- allow_external_links: Optional[bool] = False,- enable_web_search: Optional[bool] = False,- show_sources: Optional[bool] = False,- agent: Optional[Dict[str, Any]] = None,- **kwargs,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional['ScrapeOptions'] = None,+ ignore_invalid_urls: Optional[bool] = None,+ poll_interval: int = 2,+ timeout: Optional[int] = None,):- from ..v1.client import AsyncV1FirecrawlApp- v1 = AsyncV1FirecrawlApp(api_key=self.http_client.api_key, api_url=self.http_client.api_url)- return await v1.extract(+ return await async_extract.extract(+ self.async_http_client,urls,prompt=prompt,schema=schema,@@ -187,14 +189,40 @@ class AsyncFirecrawlClient:allow_external_links=allow_external_links,enable_web_search=enable_web_search,show_sources=show_sources,- agent=agent,- **kwargs,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ poll_interval=poll_interval,+ timeout=timeout,)async def get_extract_status(self, job_id: str):- from ..v1.client import AsyncV1FirecrawlApp- v1 = AsyncV1FirecrawlApp(api_key=self.http_client.api_key, api_url=self.http_client.api_url)- return await v1.get_extract_status(job_id)+ return await async_extract.get_extract_status(self.async_http_client, job_id)++ async def start_extract(+ self,+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional['ScrapeOptions'] = None,+ ignore_invalid_urls: Optional[bool] = None,+ ):+ return await async_extract.start_extract(+ self.async_http_client,+ urls,+ prompt=prompt,+ schema=schema,+ system_prompt=system_prompt,+ allow_external_links=allow_external_links,+ enable_web_search=enable_web_search,+ show_sources=show_sources,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ )# Usage endpointsasync def get_concurrency(self):diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py b/apps/python-sdk/firecrawl/v2/methods/aio/__init__.pyindex 85f6a99e2..5cc6d202e 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py@@ -1,2 +1 @@-# Async (aio) method modules for v2-+# Async (aio) method modules for v2\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/aio/extract.py b/apps/python-sdk/firecrawl/v2/methods/aio/extract.pynew file mode 100644index 000000000..981829dca--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/aio/extract.py@@ -0,0 +1,126 @@+from typing import Any, Dict, List, Optional+import asyncio++from ...types import ExtractResponse, ScrapeOptions+from ...utils.http_client_async import AsyncHttpClient+from ...utils.validation import prepare_scrape_options+++def _prepare_extract_request(+ urls: Optional[List[str]],+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ ignore_invalid_urls: Optional[bool] = None,+) -> Dict[str, Any]:+ body: Dict[str, Any] = {}+ if urls is not None:+ body["urls"] = urls+ if prompt is not None:+ body["prompt"] = prompt+ if schema is not None:+ body["schema"] = schema+ if system_prompt is not None:+ body["systemPrompt"] = system_prompt+ if allow_external_links is not None:+ body["allowExternalLinks"] = allow_external_links+ if enable_web_search is not None:+ body["enableWebSearch"] = enable_web_search+ if show_sources is not None:+ body["showSources"] = show_sources+ if ignore_invalid_urls is not None:+ body["ignoreInvalidURLs"] = ignore_invalid_urls+ if scrape_options is not None:+ prepared = prepare_scrape_options(scrape_options)+ if prepared:+ body["scrapeOptions"] = prepared+ return body+++async def start_extract(+ client: AsyncHttpClient,+ urls: Optional[List[str]],+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ ignore_invalid_urls: Optional[bool] = None,+) -> ExtractResponse:+ body = _prepare_extract_request(+ urls,+ prompt=prompt,+ schema=schema,+ system_prompt=system_prompt,+ allow_external_links=allow_external_links,+ enable_web_search=enable_web_search,+ show_sources=show_sources,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ )+ resp = await client.post("/v2/extract", body)+ return ExtractResponse(**resp.json())+++async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse:+ resp = await client.get(f"/v2/extract/{job_id}")+ return ExtractResponse(**resp.json())+++async def wait_extract(+ client: AsyncHttpClient,+ job_id: str,+ *,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+) -> ExtractResponse:+ start_ts = asyncio.get_event_loop().time()+ while True:+ status = await get_extract_status(client, job_id)+ if status.status in ("completed", "failed", "cancelled"):+ return status+ if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:+ return status+ await asyncio.sleep(max(1, poll_interval))+++async def extract(+ client: AsyncHttpClient,+ urls: Optional[List[str]],+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ ignore_invalid_urls: Optional[bool] = None,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+) -> ExtractResponse:+ started = await start_extract(+ client,+ urls,+ prompt=prompt,+ schema=schema,+ system_prompt=system_prompt,+ allow_external_links=allow_external_links,+ enable_web_search=enable_web_search,+ show_sources=show_sources,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ )+ job_id = getattr(started, "id", None)+ if not job_id:+ return started+ return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)+diff --git a/apps/python-sdk/firecrawl/v2/methods/extract.py b/apps/python-sdk/firecrawl/v2/methods/extract.pynew file mode 100644index 000000000..d863b1e1e--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/extract.py@@ -0,0 +1,131 @@+from typing import Any, Dict, List, Optional+import time++from ..types import ExtractResponse, ScrapeOptions+from ..utils.http_client import HttpClient+from ..utils.validation import prepare_scrape_options+from ..utils.error_handler import handle_response_error+++def _prepare_extract_request(+ urls: Optional[List[str]],+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ ignore_invalid_urls: Optional[bool] = None,+) -> Dict[str, Any]:+ body: Dict[str, Any] = {}+ if urls is not None:+ body["urls"] = urls+ if prompt is not None:+ body["prompt"] = prompt+ if schema is not None:+ body["schema"] = schema+ if system_prompt is not None:+ body["systemPrompt"] = system_prompt+ if allow_external_links is not None:+ body["allowExternalLinks"] = allow_external_links+ if enable_web_search is not None:+ body["enableWebSearch"] = enable_web_search+ if show_sources is not None:+ body["showSources"] = show_sources+ if ignore_invalid_urls is not None:+ body["ignoreInvalidURLs"] = ignore_invalid_urls+ if scrape_options is not None:+ prepared = prepare_scrape_options(scrape_options)+ if prepared:+ body["scrapeOptions"] = prepared+ return body+++def start_extract(+ client: HttpClient,+ urls: Optional[List[str]],+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ ignore_invalid_urls: Optional[bool] = None,+) -> ExtractResponse:+ body = _prepare_extract_request(+ urls,+ prompt=prompt,+ schema=schema,+ system_prompt=system_prompt,+ allow_external_links=allow_external_links,+ enable_web_search=enable_web_search,+ show_sources=show_sources,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ )+ resp = client.post("/v2/extract", body)+ if not resp.ok:+ handle_response_error(resp, "extract")+ return ExtractResponse(**resp.json())+++def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:+ resp = client.get(f"/v2/extract/{job_id}")+ if not resp.ok:+ handle_response_error(resp, "extract-status")+ return ExtractResponse(**resp.json())+++def wait_extract(+ client: HttpClient,+ job_id: str,+ *,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+) -> ExtractResponse:+ start_ts = time.time()+ while True:+ status = get_extract_status(client, job_id)+ if status.status in ("completed", "failed", "cancelled"):+ return status+ if timeout is not None and (time.time() - start_ts) > timeout:+ return status+ time.sleep(max(1, poll_interval))+++def extract(+ client: HttpClient,+ urls: Optional[List[str]],+ *,+ prompt: Optional[str] = None,+ schema: Optional[Dict[str, Any]] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = None,+ enable_web_search: Optional[bool] = None,+ show_sources: Optional[bool] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ ignore_invalid_urls: Optional[bool] = None,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+) -> ExtractResponse:+ started = start_extract(+ client,+ urls,+ prompt=prompt,+ schema=schema,+ system_prompt=system_prompt,+ allow_external_links=allow_external_links,+ enable_web_search=enable_web_search,+ show_sources=show_sources,+ scrape_options=scrape_options,+ ignore_invalid_urls=ignore_invalid_urls,+ )+ job_id = getattr(started, "id", None)+ if not job_id:+ return started+ return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)+diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex ce786a3aa..eff1008df 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -341,6 +341,18 @@ class MapResponse(BaseResponse[MapData]):"""Response for map operations."""pass+# Extract types+class ExtractResponse(BaseModel):+ """Response for extract operations (start/status/final)."""+ success: Optional[bool] = None+ id: Optional[str] = None+ status: Optional[Literal["processing", "completed", "failed", "cancelled"]] = None+ data: Optional[Any] = None+ error: Optional[str] = None+ warning: Optional[str] = None+ sources: Optional[Dict[str, Any]] = None+ expires_at: Optional[datetime] = None+# Usage/limits typesclass ConcurrencyCheck(BaseModel):"""Current concurrency and limits for the team/API key."""