Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(python-sdk): extract v2

rafaelmmiller10b720289801
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.py
index 93f489718..77b038c1d 100644
--- a/apps/python-sdk/example.py
+++ b/apps/python-sdk/example.py
@@ -21,24 +21,24 @@ def main():
firecrawl = Firecrawl(api_key=api_key, api_url=api_url)
- # # Scrape
- # doc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"])
- # print("scrape:", doc.markdown)
-
- # # Crawl (waits until terminal state)
- # crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)
- # print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total)
-
- # # Batch scrape
- # batch = firecrawl.batch_scrape([
- # "https://docs.firecrawl.dev",
- # "https://firecrawl.dev",
- # ], formats=["markdown"], poll_interval=1, wait_timeout=120)
- # print("batch:", batch.status, batch.completed, "/", batch.total)
-
- # # Search
- # search_response = firecrawl.search(query="What is the capital of France?", limit=5)
- # print("search web results:", len(getattr(search_response, "web", []) or []))
+ # Scrape
+ doc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"])
+ print("scrape:", doc.markdown)
+
+ # Crawl (waits until terminal state)
+ crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)
+ print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total)
+
+ # Batch scrape
+ batch = firecrawl.batch_scrape([
+ "https://docs.firecrawl.dev",
+ "https://firecrawl.dev",
+ ], formats=["markdown"], poll_interval=1, wait_timeout=120)
+ print("batch:", batch.status, batch.completed, "/", batch.total)
+
+ # Search
+ search_response = firecrawl.search(query="What is the capital of France?", limit=5)
+ print("search web results:", len(getattr(search_response, "web", []) or []))
# Map
map_response = firecrawl.map("https://firecrawl.dev")
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 08e90493c..c9512ff45 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -44,6 +44,7 @@ from .methods import search as search_module
from .methods import map as map_module
from .methods import batch as batch_methods
from .methods import usage as usage_methods
+from .methods import extract as extract_module
from .watcher import Watcher
class FirecrawlClient:
@@ -439,25 +440,55 @@ class FirecrawlClient:
request = CrawlParamsRequest(url=url, prompt=prompt)
return crawl_module.crawl_params_preview(self.http_client, request)
+ def start_extract(
+ self,
+ urls: Optional[List[str]] = None,
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional['ScrapeOptions'] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+ ):
+ """
+ Start an extract job (non-blocking). Use get_extract_status to poll.
+ """
+ return extract_module.start_extract(
+ self.http_client,
+ urls,
+ prompt=prompt,
+ schema=schema,
+ system_prompt=system_prompt,
+ allow_external_links=allow_external_links,
+ enable_web_search=enable_web_search,
+ show_sources=show_sources,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ )
+
def extract(
self,
urls: Optional[List[str]] = None,
*,
prompt: Optional[str] = None,
- schema: Optional[Any] = None,
+ schema: Optional[Dict[str, Any]] = None,
system_prompt: Optional[str] = None,
- allow_external_links: Optional[bool] = False,
- enable_web_search: Optional[bool] = False,
- show_sources: Optional[bool] = False,
- agent: Optional[Dict[str, Any]] = None,
- **kwargs,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional['ScrapeOptions'] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
):
"""
- Extract data from a list of URLs.
+ Extract structured data using the v2 client surface, waiting until completion.
"""
- from ..v1.client import V1FirecrawlApp
- v1 = V1FirecrawlApp(api_key=self.config.api_key, api_url=self.config.api_url)
- return v1.extract(
+ return extract_module.extract(
+ self.http_client,
urls,
prompt=prompt,
schema=schema,
@@ -465,8 +496,10 @@ class FirecrawlClient:
allow_external_links=allow_external_links,
enable_web_search=enable_web_search,
show_sources=show_sources,
- agent=agent,
- **kwargs,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ poll_interval=poll_interval,
+ timeout=timeout,
)
def start_batch_scrape(
@@ -553,10 +586,7 @@ class FirecrawlClient:
return batch_methods.get_batch_scrape_errors(self.http_client, job_id)
def get_extract_status(self, job_id: str):
- """Proxy to v1 get_extract_status while v2 extract is not implemented."""
- from ..v1.client import V1FirecrawlApp
- v1 = V1FirecrawlApp(api_key=self.config.api_key, api_url=self.config.api_url)
- return v1.get_extract_status(job_id)
+ return extract_module.get_extract_status(self.http_client, job_id)
def get_concurrency(self):
"""Get current concurrency and maximum allowed for this team/key (v2)."""
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 8e53059a2..3a8102801 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -41,6 +41,7 @@ from .methods.aio import crawl as async_crawl # type: ignore[attr-defined]
from .methods.aio import search as async_search # type: ignore[attr-defined]
from .methods.aio import map as async_map # type: ignore[attr-defined]
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
+from .methods.aio import extract as async_extract # type: ignore[attr-defined]
from .watcher_async import AsyncWatcher
@@ -169,17 +170,18 @@ class AsyncFirecrawlClient:
urls: Optional[List[str]] = None,
*,
prompt: Optional[str] = None,
- schema: Optional[Any] = None,
+ schema: Optional[Dict[str, Any]] = None,
system_prompt: Optional[str] = None,
- allow_external_links: Optional[bool] = False,
- enable_web_search: Optional[bool] = False,
- show_sources: Optional[bool] = False,
- agent: Optional[Dict[str, Any]] = None,
- **kwargs,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional['ScrapeOptions'] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
):
- from ..v1.client import AsyncV1FirecrawlApp
- v1 = AsyncV1FirecrawlApp(api_key=self.http_client.api_key, api_url=self.http_client.api_url)
- return await v1.extract(
+ return await async_extract.extract(
+ self.async_http_client,
urls,
prompt=prompt,
schema=schema,
@@ -187,14 +189,40 @@ class AsyncFirecrawlClient:
allow_external_links=allow_external_links,
enable_web_search=enable_web_search,
show_sources=show_sources,
- agent=agent,
- **kwargs,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ poll_interval=poll_interval,
+ timeout=timeout,
)
async def get_extract_status(self, job_id: str):
- from ..v1.client import AsyncV1FirecrawlApp
- v1 = AsyncV1FirecrawlApp(api_key=self.http_client.api_key, api_url=self.http_client.api_url)
- return await v1.get_extract_status(job_id)
+ return await async_extract.get_extract_status(self.async_http_client, job_id)
+
+ async def start_extract(
+ self,
+ urls: Optional[List[str]] = None,
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional['ScrapeOptions'] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+ ):
+ return await async_extract.start_extract(
+ self.async_http_client,
+ urls,
+ prompt=prompt,
+ schema=schema,
+ system_prompt=system_prompt,
+ allow_external_links=allow_external_links,
+ enable_web_search=enable_web_search,
+ show_sources=show_sources,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ )
# Usage endpoints
async def get_concurrency(self):
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py b/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py
index 85f6a99e2..5cc6d202e 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/__init__.py
@@ -1,2 +1 @@
-# Async (aio) method modules for v2
-
+# Async (aio) method modules for v2
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/extract.py b/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
new file mode 100644
index 000000000..981829dca
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
@@ -0,0 +1,126 @@
+from typing import Any, Dict, List, Optional
+import asyncio
+
+from ...types import ExtractResponse, ScrapeOptions
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.validation import prepare_scrape_options
+
+
+def _prepare_extract_request(
+ urls: Optional[List[str]],
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional[ScrapeOptions] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+) -> Dict[str, Any]:
+ body: Dict[str, Any] = {}
+ if urls is not None:
+ body["urls"] = urls
+ if prompt is not None:
+ body["prompt"] = prompt
+ if schema is not None:
+ body["schema"] = schema
+ if system_prompt is not None:
+ body["systemPrompt"] = system_prompt
+ if allow_external_links is not None:
+ body["allowExternalLinks"] = allow_external_links
+ if enable_web_search is not None:
+ body["enableWebSearch"] = enable_web_search
+ if show_sources is not None:
+ body["showSources"] = show_sources
+ if ignore_invalid_urls is not None:
+ body["ignoreInvalidURLs"] = ignore_invalid_urls
+ if scrape_options is not None:
+ prepared = prepare_scrape_options(scrape_options)
+ if prepared:
+ body["scrapeOptions"] = prepared
+ return body
+
+
+async def start_extract(
+ client: AsyncHttpClient,
+ urls: Optional[List[str]],
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional[ScrapeOptions] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+) -> ExtractResponse:
+ body = _prepare_extract_request(
+ urls,
+ prompt=prompt,
+ schema=schema,
+ system_prompt=system_prompt,
+ allow_external_links=allow_external_links,
+ enable_web_search=enable_web_search,
+ show_sources=show_sources,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ )
+ resp = await client.post("/v2/extract", body)
+ return ExtractResponse(**resp.json())
+
+
+async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse:
+ resp = await client.get(f"/v2/extract/{job_id}")
+ return ExtractResponse(**resp.json())
+
+
+async def wait_extract(
+ client: AsyncHttpClient,
+ job_id: str,
+ *,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+) -> ExtractResponse:
+ start_ts = asyncio.get_event_loop().time()
+ while True:
+ status = await get_extract_status(client, job_id)
+ if status.status in ("completed", "failed", "cancelled"):
+ return status
+ if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
+ return status
+ await asyncio.sleep(max(1, poll_interval))
+
+
+async def extract(
+ client: AsyncHttpClient,
+ urls: Optional[List[str]],
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional[ScrapeOptions] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+) -> ExtractResponse:
+ started = await start_extract(
+ client,
+ urls,
+ prompt=prompt,
+ schema=schema,
+ system_prompt=system_prompt,
+ allow_external_links=allow_external_links,
+ enable_web_search=enable_web_search,
+ show_sources=show_sources,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ )
+ job_id = getattr(started, "id", None)
+ if not job_id:
+ return started
+ return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
+
diff --git a/apps/python-sdk/firecrawl/v2/methods/extract.py b/apps/python-sdk/firecrawl/v2/methods/extract.py
new file mode 100644
index 000000000..d863b1e1e
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/extract.py
@@ -0,0 +1,131 @@
+from typing import Any, Dict, List, Optional
+import time
+
+from ..types import ExtractResponse, ScrapeOptions
+from ..utils.http_client import HttpClient
+from ..utils.validation import prepare_scrape_options
+from ..utils.error_handler import handle_response_error
+
+
+def _prepare_extract_request(
+ urls: Optional[List[str]],
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional[ScrapeOptions] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+) -> Dict[str, Any]:
+ body: Dict[str, Any] = {}
+ if urls is not None:
+ body["urls"] = urls
+ if prompt is not None:
+ body["prompt"] = prompt
+ if schema is not None:
+ body["schema"] = schema
+ if system_prompt is not None:
+ body["systemPrompt"] = system_prompt
+ if allow_external_links is not None:
+ body["allowExternalLinks"] = allow_external_links
+ if enable_web_search is not None:
+ body["enableWebSearch"] = enable_web_search
+ if show_sources is not None:
+ body["showSources"] = show_sources
+ if ignore_invalid_urls is not None:
+ body["ignoreInvalidURLs"] = ignore_invalid_urls
+ if scrape_options is not None:
+ prepared = prepare_scrape_options(scrape_options)
+ if prepared:
+ body["scrapeOptions"] = prepared
+ return body
+
+
+def start_extract(
+ client: HttpClient,
+ urls: Optional[List[str]],
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional[ScrapeOptions] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+) -> ExtractResponse:
+ body = _prepare_extract_request(
+ urls,
+ prompt=prompt,
+ schema=schema,
+ system_prompt=system_prompt,
+ allow_external_links=allow_external_links,
+ enable_web_search=enable_web_search,
+ show_sources=show_sources,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ )
+ resp = client.post("/v2/extract", body)
+ if not resp.ok:
+ handle_response_error(resp, "extract")
+ return ExtractResponse(**resp.json())
+
+
+def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:
+ resp = client.get(f"/v2/extract/{job_id}")
+ if not resp.ok:
+ handle_response_error(resp, "extract-status")
+ return ExtractResponse(**resp.json())
+
+
+def wait_extract(
+ client: HttpClient,
+ job_id: str,
+ *,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+) -> ExtractResponse:
+ start_ts = time.time()
+ while True:
+ status = get_extract_status(client, job_id)
+ if status.status in ("completed", "failed", "cancelled"):
+ return status
+ if timeout is not None and (time.time() - start_ts) > timeout:
+ return status
+ time.sleep(max(1, poll_interval))
+
+
+def extract(
+ client: HttpClient,
+ urls: Optional[List[str]],
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Dict[str, Any]] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = None,
+ enable_web_search: Optional[bool] = None,
+ show_sources: Optional[bool] = None,
+ scrape_options: Optional[ScrapeOptions] = None,
+ ignore_invalid_urls: Optional[bool] = None,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+) -> ExtractResponse:
+ started = start_extract(
+ client,
+ urls,
+ prompt=prompt,
+ schema=schema,
+ system_prompt=system_prompt,
+ allow_external_links=allow_external_links,
+ enable_web_search=enable_web_search,
+ show_sources=show_sources,
+ scrape_options=scrape_options,
+ ignore_invalid_urls=ignore_invalid_urls,
+ )
+ job_id = getattr(started, "id", None)
+ if not job_id:
+ return started
+ return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
+
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index ce786a3aa..eff1008df 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -341,6 +341,18 @@ class MapResponse(BaseResponse[MapData]):
"""Response for map operations."""
pass
+# Extract types
+class ExtractResponse(BaseModel):
+ """Response for extract operations (start/status/final)."""
+ success: Optional[bool] = None
+ id: Optional[str] = None
+ status: Optional[Literal["processing", "completed", "failed", "cancelled"]] = None
+ data: Optional[Any] = None
+ error: Optional[str] = None
+ warning: Optional[str] = None
+ sources: Optional[Dict[str, Any]] = None
+ expires_at: Optional[datetime] = None
+
# Usage/limits types
class ConcurrencyCheck(BaseModel):
"""Current concurrency and limits for the team/API key."""