Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(python-sdk): wip - base structure and search endpoint

rafaelmmillerf22fba62959a
diff --git a/apps/python-sdk/.env.example b/apps/python-sdk/.env.example
new file mode 100644
index 000000000..7bea7e7f2
--- /dev/null
+++ b/apps/python-sdk/.env.example
@@ -0,0 +1 @@
+FIRECRAWL_API_KEY=<your api key>
\ No newline at end of file
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.py
index 9b28e17e7..76535654e 100644
--- a/apps/python-sdk/example.py
+++ b/apps/python-sdk/example.py
@@ -1,141 +1,35 @@
-from firecrawl import JsonConfig, FirecrawlApp
-from pydantic import BaseModel, Field
-from typing import List
-import time
-app = FirecrawlApp(api_url="https://api.firecrawl.dev")
+#!/usr/bin/env python3
+"""
+Example demonstrating the v2 search functionality with individual parameters.
+"""
-# Scrape a website:
-scrape_result = app.scrape_url('example.com', formats=["markdown", "html"])
-print(scrape_result.markdown)
+import os
+from dotenv import load_dotenv
+from firecrawl import Firecrawl
+from firecrawl.v2.types import ScrapeOptions, ScrapeFormats
-# # # Test batch scrape
-urls = ['https://example.com', 'https://docs.firecrawl.dev']
-# # Synchronous batch scrape
-batch_result = app.batch_scrape_urls(urls, formats=["markdown", "html"])
-print("Synchronous Batch Scrape Result:")
-print(batch_result.data[0].markdown)
+load_dotenv()
-# # # Asynchronous batch scrape
-async_batch_result = app.async_batch_scrape_urls(urls, formats=["markdown", "html"])
-print("\nAsynchronous Batch Scrape Result:")
-print(async_batch_result)
+def main():
+ api_key = os.getenv("FIRECRAWL_API_KEY")
+ if not api_key:
+ raise ValueError("FIRECRAWL_API_KEY is not set")
-# # Crawl a website:
-crawl_result = app.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
-print(crawl_result.data[0].markdown)
+ firecrawl = Firecrawl(api_key=api_key)
-# # Asynchronous Crawl a website:
-async_result = app.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
-print(async_result)
+ # search examples
+ search_response = firecrawl.search(
+ query="What is the capital of France?",
+ sources=[
+ { type: "web" },
+ { type: "news" },
+ { type: "images" }
+ ],
+ limit=10)
+ print(search_response)
-crawl_status = app.check_crawl_status(async_result.id)
-print(crawl_status)
-attempts = 15
-while attempts > 0 and crawl_status.status != 'completed':
- print(crawl_status)
- crawl_status = app.check_crawl_status(async_result.id)
- attempts -= 1
- time.sleep(1)
-crawl_status = app.check_crawl_status(async_result.id)
-print(crawl_status)
-
-# JSON format:
-# Define schema to extract contents into using json schema
-json_schema = {
- "type": "object",
- "properties": {
- "top": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "title": {"type": "string"},
- "points": {"type": "number"},
- "by": {"type": "string"},
- "commentsURL": {"type": "string"}
- },
- "required": ["title", "points", "by", "commentsURL"]
- },
- "description": "Top 5 stories on Hacker News"
- }
- },
- "required": ["top"]
-}
-
-extract_config = JsonConfig(schema=json_schema)
-llm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=["json"], json_options=extract_config)
-
-print(llm_extraction_result.json)
-
-# Map a website:
-map_result = app.map_url('https://firecrawl.dev', search="blog")
-print(map_result)
-
-# Extract URLs:
-class ExtractSchema(BaseModel):
- title: str
- description: str
- links: List[str]
-
-# Define the schema using Pydantic
-extract_schema = ExtractSchema.schema()
-
-# Perform the extraction
-extract_result = app.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)
-print(extract_result)
-
-
-# Deep research example
-research_result = app.deep_research(
- "What are the latest developments in large language models?",
- max_urls=4
-)
-print("Research Results:", research_result)
-
-# Generate LLMs.txt example
-llms_result = app.generate_llms_text(
- "https://firecrawl.dev")
-print("LLMs.txt Results:", llms_result)
-
-
-# Crawl a website with WebSockets:
-# inside an async function...
-import nest_asyncio
-nest_asyncio.apply()
-
-# Define event handlers
-def on_document(detail):
- print("DOC", detail)
-
-def on_error(detail):
- print("ERR", detail['error'])
-
-def on_done(detail):
- print("DONE", detail['status'])
-
- # Function to start the crawl and watch process
-async def start_crawl_and_watch():
- # Initiate the crawl job and get the watcher
- watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
-
- # Add event listeners
- watcher.add_event_listener("document", on_document)
- watcher.add_event_listener("error", on_error)
- watcher.add_event_listener("done", on_done)
-
- # Start the watcher
- await watcher.connect()
-
-
-class ExtractSchema(BaseModel):
- company_mission: str
- supports_sso: bool
- is_open_source: bool
- is_in_yc: bool
-
-extract_config = JsonConfig(schema=ExtractSchema.model_json_schema())
-data = app.scrape_url('https://docs.firecrawl.dev/', formats=['json'], json_options=extract_config)
-print(data.json)
\ No newline at end of file
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/apps/python-sdk/example_async.py b/apps/python-sdk/example_async.py
deleted file mode 100644
index 1ad022440..000000000
--- a/apps/python-sdk/example_async.py
+++ /dev/null
@@ -1,136 +0,0 @@
-import time
-import nest_asyncio
-import uuid
-import asyncio
-from firecrawl.firecrawl import AsyncFirecrawlApp, ScrapeOptions, JsonConfig
-from pydantic import BaseModel, Field
-from typing import List
-
-app = AsyncFirecrawlApp(api_url="https://api.firecrawl.dev")
-
-async def example_scrape():
- # Scrape a website:
- scrape_result = await app.scrape_url('example.com', formats=["markdown", "html"])
- print(scrape_result.markdown)
-
-async def example_batch_scrape():
- # Batch scrape
- urls = ['https://example.com', 'https://docs.firecrawl.dev']
-
- # Synchronous batch scrape
- batch_result = await app.batch_scrape_urls(urls, formats=["markdown", "html"])
- print("Synchronous Batch Scrape Result:")
- print(batch_result.data[0].markdown)
-
- # Asynchronous batch scrape
- async_batch_result = await app.async_batch_scrape_urls(urls, formats=["markdown", "html"])
- print("\nAsynchronous Batch Scrape Result:")
- print(async_batch_result)
-
-async def example_crawl():
- # Crawl a website:
- crawl_result = await app.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
- print(crawl_result.data[0].markdown)
-
- # Asynchronous Crawl a website:
- async_result = await app.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
- print(async_result)
-
- crawl_status = await app.check_crawl_status(async_result.id)
- print(crawl_status)
-
- attempts = 15
- while attempts > 0 and crawl_status.status != 'completed':
- print(crawl_status)
- crawl_status = await app.check_crawl_status(async_result.id)
- attempts -= 1
- await asyncio.sleep(1) # Use async sleep instead of time.sleep
-
- crawl_status = await app.check_crawl_status(async_result.id)
- print(crawl_status)
-
-async def example_llm_extraction():
- # Define schema to extract contents into using pydantic
- class ArticleSchema(BaseModel):
- title: str
- points: int
- by: str
- commentsURL: str
-
- class TopArticlesSchema(BaseModel):
- top: List[ArticleSchema] = Field(..., description="Top 5 stories")
-
- extract_config = JsonConfig(schema=TopArticlesSchema.model_json_schema())
-
- llm_extraction_result = await app.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)
-
- print(llm_extraction_result.extract)
-
-async def example_map_and_extract():
- # Map a website:
- map_result = await app.map_url('https://firecrawl.dev', search="blog")
- print(map_result)
-
- # Extract URLs:
- class ExtractSchema(BaseModel):
- title: str
- description: str
- links: List[str]
-
- # Define the schema using Pydantic
- extract_schema = ExtractSchema.schema()
-
- # Perform the extraction
- extract_result = await app.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)
- print(extract_result)
-
-async def example_deep_research():
- # Deep research example
- research_result = await app.deep_research(
- "What are the latest developments in large language models?",
- max_urls=4
- )
- print("Research Results:", research_result)
-
-async def example_generate_llms_text():
- # Generate LLMs.txt example
- llms_result = await app.generate_llms_text(
- "https://firecrawl.dev")
- print("LLMs.txt Results:", llms_result)
-
-# Define event handlers for websocket
-def on_document(detail):
- print("DOC", detail)
-
-def on_error(detail):
- print("ERR", detail['error'])
-
-def on_done(detail):
- print("DONE", detail['status'])
-
-async def example_websocket_crawl():
- # Initiate the crawl job and get the watcher
- watcher = await app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
-
- # Add event listeners
- watcher.add_event_listener("document", on_document)
- watcher.add_event_listener("error", on_error)
- watcher.add_event_listener("done", on_done)
-
- # Start the watcher
- await watcher.connect()
-
-async def main():
- nest_asyncio.apply()
-
- await example_scrape()
- await example_batch_scrape()
- await example_crawl()
- await example_llm_extraction()
- await example_map_and_extract()
- await example_websocket_crawl()
- await example_deep_research()
- await example_generate_llms_text()
-
-if __name__ == "__main__":
- asyncio.run(main())
diff --git a/apps/python-sdk/example_v1.py b/apps/python-sdk/example_v1.py
new file mode 100644
index 000000000..70a98b467
--- /dev/null
+++ b/apps/python-sdk/example_v1.py
@@ -0,0 +1,286 @@
+from firecrawl import V1JsonConfig, FirecrawlApp, AsyncFirecrawlApp, V1ScrapeOptions, V1JsonConfig
+from pydantic import BaseModel, Field
+from typing import List
+import time
+import os
+from dotenv import load_dotenv
+
+# -- Async example imports
+import nest_asyncio
+import asyncio
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Get API key from environment
+api_key = os.getenv("TEST_API_KEY")
+api_url = os.getenv("TEST_API_URL", "https://api.firecrawl.dev")
+
+app = FirecrawlApp(api_key=api_key, api_url=api_url)
+
+# Scrape a website:
+scrape_result = app.v1.scrape_url('example.com', formats=["markdown", "html"])
+print(scrape_result.markdown)
+
+# # # Test batch scrape
+urls = ['https://example.com', 'https://docs.firecrawl.dev']
+# # Synchronous batch scrape
+batch_result = app.v1.batch_scrape_urls(urls, formats=["markdown", "html"])
+print("Synchronous Batch Scrape Result:")
+print(batch_result.data[0].markdown)
+
+# # # Asynchronous batch scrape
+async_batch_result = app.v1.async_batch_scrape_urls(urls, formats=["markdown", "html"])
+print("\nAsynchronous Batch Scrape Result:")
+print(async_batch_result)
+
+# # Crawl a website:
+crawl_result = app.v1.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
+print(crawl_result.data[0].markdown)
+
+# # Asynchronous Crawl a website:
+async_result = app.v1.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
+print(async_result)
+
+crawl_status = app.v1.check_crawl_status(async_result.id)
+print(crawl_status)
+
+attempts = 15
+while attempts > 0 and crawl_status.status != 'completed':
+ print(crawl_status)
+ crawl_status = app.v1.check_crawl_status(async_result.id)
+ attempts -= 1
+ time.sleep(1)
+
+crawl_status = app.v1.check_crawl_status(async_result.id)
+print(crawl_status)
+
+# JSON format:
+# Define schema to extract contents into using json schema
+json_schema = {
+ "type": "object",
+ "properties": {
+ "top": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "title": {"type": "string"},
+ "points": {"type": "number"},
+ "by": {"type": "string"},
+ "commentsURL": {"type": "string"}
+ },
+ "required": ["title", "points", "by", "commentsURL"]
+ },
+ "description": "Top 5 stories on Hacker News"
+ }
+ },
+ "required": ["top"]
+}
+
+extract_config = V1JsonConfig(schema=json_schema)
+llm_extraction_result = app.v1.scrape_url('https://news.ycombinator.com', formats=["json"], json_options=extract_config)
+
+print(llm_extraction_result.json)
+
+# Map a website:
+map_result = app.v1.map_url('https://firecrawl.dev', search="blog")
+print(map_result)
+
+# Extract URLs:
+class ExtractSchema(BaseModel):
+ title: str
+ description: str
+ links: List[str]
+
+# Define the schema using Pydantic
+extract_schema = ExtractSchema.schema()
+
+# Perform the extraction
+extract_result = app.v1.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)
+print(extract_result)
+
+
+# Deep research example
+research_result = app.v1.deep_research(
+ "What are the latest developments in large language models?",
+ max_urls=4
+)
+print("Research Results:", research_result)
+
+# Generate LLMs.txt example
+llms_result = app.v1.generate_llms_text(
+ "https://firecrawl.dev")
+print("LLMs.txt Results:", llms_result)
+
+
+# Crawl a website with WebSockets:
+# inside an async function...
+import nest_asyncio
+nest_asyncio.apply()
+
+# Define event handlers
+def on_document(detail):
+ print("DOC", detail)
+
+def on_error(detail):
+ print("ERR", detail['error'])
+
+def on_done(detail):
+ print("DONE", detail['status'])
+
+ # Function to start the crawl and watch process
+async def start_crawl_and_watch():
+ # Initiate the crawl job and get the watcher
+ watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
+
+ # Add event listeners
+ watcher.add_event_listener("document", on_document)
+ watcher.add_event_listener("error", on_error)
+ watcher.add_event_listener("done", on_done)
+
+ # Start the watcher
+ await watcher.connect()
+
+
+class ExtractSchema(BaseModel):
+ company_mission: str
+ supports_sso: bool
+ is_open_source: bool
+ is_in_yc: bool
+
+extract_config = V1JsonConfig(schema=ExtractSchema.model_json_schema())
+data = app.scrape_url('https://docs.firecrawl.dev/', formats=['json'], json_options=extract_config)
+print(data.json)
+
+# --- Async
+
+async_app = AsyncFirecrawlApp(api_url="https://api.firecrawl.dev")
+
+async def example_scrape():
+ # Scrape a website:
+ scrape_result = await async_app.v1.scrape_url('example.com', formats=["markdown", "html"])
+ print(scrape_result.markdown)
+
+async def example_batch_scrape():
+ # Batch scrape
+ urls = ['https://example.com', 'https://docs.firecrawl.dev']
+
+ # Synchronous batch scrape
+ batch_result = await async_app.v1.batch_scrape_urls(urls, formats=["markdown", "html"])
+ print("Synchronous Batch Scrape Result:")
+ print(batch_result.data[0].markdown)
+
+ # Asynchronous batch scrape
+ async_batch_result = await async_app.v1.async_batch_scrape_urls(urls, formats=["markdown", "html"])
+ print("\nAsynchronous Batch Scrape Result:")
+ print(async_batch_result)
+
+async def example_crawl():
+ # Crawl a website:
+ crawl_result = await async_app.v1.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
+ print(crawl_result.data[0].markdown)
+
+ # Asynchronous Crawl a website:
+ async_result = await async_app.v1.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
+ print(async_result)
+
+ crawl_status = await async_app.v1.check_crawl_status(async_result.id)
+ print(crawl_status)
+
+ attempts = 15
+ while attempts > 0 and crawl_status.status != 'completed':
+ print(crawl_status)
+ crawl_status = await async_app.v1.check_crawl_status(async_result.id)
+ attempts -= 1
+ await asyncio.sleep(1) # Use async sleep instead of time.sleep
+
+ crawl_status = await async_app.v1.check_crawl_status(async_result.id)
+ print(crawl_status)
+
+async def example_llm_extraction():
+ # Define schema to extract contents into using pydantic
+ class ArticleSchema(BaseModel):
+ title: str
+ points: int
+ by: str
+ commentsURL: str
+
+ class TopArticlesSchema(BaseModel):
+ top: List[ArticleSchema] = Field(..., description="Top 5 stories")
+
+ extract_config = V1JsonConfig(schema=TopArticlesSchema.model_json_schema())
+
+ llm_extraction_result = await async_app.v1.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)
+
+ print(llm_extraction_result.extract)
+
+async def example_map_and_extract():
+ # Map a website:
+ map_result = await async_app.v1.map_url('https://firecrawl.dev', search="blog")
+ print(map_result)
+
+ # Extract URLs:
+ class ExtractSchema(BaseModel):
+ title: str
+ description: str
+ links: List[str]
+
+ # Define the schema using Pydantic
+ extract_schema = ExtractSchema.schema()
+
+ # Perform the extraction
+ extract_result = await async_app.v1.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)
+ print(extract_result)
+
+async def example_deep_research():
+ # Deep research example
+ research_result = await async_app.v1.deep_research(
+ "What are the latest developments in large language models?",
+ max_urls=4
+ )
+ print("Research Results:", research_result)
+
+async def example_generate_llms_text():
+ # Generate LLMs.txt example
+ llms_result = await async_app.v1.generate_llms_text(
+ "https://firecrawl.dev")
+ print("LLMs.txt Results:", llms_result)
+
+# Define event handlers for websocket
+def on_document(detail):
+ print("DOC", detail)
+
+def on_error(detail):
+ print("ERR", detail['error'])
+
+def on_done(detail):
+ print("DONE", detail['status'])
+
+async def example_websocket_crawl():
+ # Initiate the crawl job and get the watcher
+ watcher = await async_app.v1.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
+
+ # Add event listeners
+ watcher.add_event_listener("document", on_document)
+ watcher.add_event_listener("error", on_error)
+ watcher.add_event_listener("done", on_done)
+
+ # Start the watcher
+ await watcher.connect()
+
+async def main():
+ nest_asyncio.apply()
+
+ await example_scrape()
+ await example_batch_scrape()
+ await example_crawl()
+ await example_llm_extraction()
+ await example_map_and_extract()
+ await example_websocket_crawl()
+ await example_deep_research()
+ await example_generate_llms_text()
+
+if __name__ == "__main__":
+ asyncio.run(main())
+
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/DELETE_firecrawl.py
similarity index 99%
rename from apps/python-sdk/firecrawl/firecrawl.py
rename to apps/python-sdk/firecrawl/DELETE_firecrawl.py
index 1207c3be7..a9c98a502 100644
--- a/apps/python-sdk/firecrawl/firecrawl.py
+++ b/apps/python-sdk/firecrawl/DELETE_firecrawl.py
@@ -2345,7 +2345,7 @@ class FirecrawlApp:
else:
error_message = f"Server returned empty response with status {response.status_code}"
error_details = "No additional details available"
-+ except ValueError:
+ except ValueError:
error_message = f"Server returned unreadable response with status {response.status_code}"
error_details = "No additional details available"
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 9119facd4..e90565779 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -1,19 +1,15 @@
"""
-This is the Firecrawl package.
+Firecrawl Python SDK
-This package provides a Python SDK for interacting with the Firecrawl API.
-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,
-and check the status of these jobs.
-
-For more information visit https://github.com/firecrawl/
"""
import logging
import os
-from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
+from .client import Firecrawl, AsyncFirecrawl
+from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp, V1JsonConfig
-__version__ = "2.16.4"
+__version__ = "3.0.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
@@ -27,17 +23,14 @@ def _configure_logger() -> None:
format to the firecrawl logger.
"""
try:
- # Create the formatter
formatter = logging.Formatter(
"[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
- # Create the console handler and set the formatter
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
- # Add the console handler to the firecrawl logger
logger.addHandler(console_handler)
except Exception as e:
logger.error("Failed to configure logging: %s", e)
@@ -45,20 +38,15 @@ def _configure_logger() -> None:
def setup_logging() -> None:
"""Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
- # Check if the firecrawl logger already has a handler
if logger.hasHandlers():
- return # To prevent duplicate logging
+ return
- # Check if the FIRECRAWL_LOGGING_LEVEL environment variable is set
if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
- # Attach a no-op handler to prevent warnings about no handlers
logger.addHandler(logging.NullHandler())
return
- # Attach the console handler to the firecrawl logger
_configure_logger()
- # Set the logging level based on the FIRECRAWL_LOGGING_LEVEL environment variable
if env == "DEBUG":
logger.setLevel(logging.DEBUG)
elif env == "INFO":
@@ -73,7 +61,15 @@ def setup_logging() -> None:
logger.setLevel(logging.INFO)
logger.warning("Unknown logging level: %s, defaulting to INFO", env)
-
-# Initialize logging configuration when the module is imported
setup_logging()
logger.debug("Debugging logger setup")
+
+__all__ = [
+ 'Firecrawl',
+ 'AsyncFirecrawl',
+ 'V1FirecrawlApp',
+ 'AsyncV1FirecrawlApp',
+ 'V1JsonConfig',
+ 'V1ScrapeOptions',
+ 'V1ChangeTrackingOptions',
+]
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/.env.example b/apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/.env.example
similarity index 100%
rename from apps/python-sdk/firecrawl/__tests__/e2e_withAuth/.env.example
rename to apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/.env.example
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/__init__.py b/apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/__init__.py
similarity index 100%
rename from apps/python-sdk/firecrawl/__tests__/e2e_withAuth/__init__.py
rename to apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/__init__.py
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/test.py
similarity index 100%
rename from apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py
rename to apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/test.py
diff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/.env.example b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/.env.example
similarity index 100%
rename from apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/.env.example
rename to apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/.env.example
diff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/__init__.py b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/__init__.py
similarity index 100%
rename from apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/__init__.py
rename to apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/__init__.py
diff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/test.py
similarity index 97%
rename from apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py
rename to apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/test.py
index 615655651..586d700d9 100644
--- a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py
+++ b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/test.py
@@ -44,7 +44,7 @@ def test_scrape_url_invalid_api_key():
# def test_blocklisted_url():
# blocklisted_url = "https://facebook.com/fake-test"
-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
# with pytest.raises(Exception) as excinfo:
# app.scrape_url(blocklisted_url)
# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
@@ -155,7 +155,7 @@ def test_crawl_url_invalid_api_key():
assert response is not None
# def test_should_return_error_for_blocklisted_url():
-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
# blocklisted_url = "https://twitter.com/fake-test"
# with pytest.raises(Exception) as excinfo:
# app.crawl_url(blocklisted_url)
@@ -336,7 +336,7 @@ def test_invalid_api_key_on_map():
assert response is not None
# def test_blocklisted_url_on_map():
-# app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
+# app = V1FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
# blocklisted_url = "https://facebook.com/fake-test"
# with pytest.raises(Exception) as excinfo:
# app.map_url(blocklisted_url)
@@ -455,7 +455,7 @@ def test_search_with_invalid_params():
# def test_scrape_url_with_parse_pdf_true():
# if TEST_API_KEY:
-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
# response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf', parse_pdf=True)
# assert response is not None
# assert 'markdown' in response
@@ -463,7 +463,7 @@ def test_search_with_invalid_params():
# def test_scrape_url_with_parse_pdf_false():
# if TEST_API_KEY:
-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
# response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf', parse_pdf=False)
# assert response is not None
# assert 'markdown' in response
@@ -471,9 +471,9 @@ def test_search_with_invalid_params():
# def test_scrape_options_with_parse_pdf():
# if TEST_API_KEY:
-# from firecrawl.firecrawl import ScrapeOptions
-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
-# scrape_options = ScrapeOptions(parsePDF=False, formats=['markdown'])
+# from firecrawl.firecrawl import V1ScrapeOptions
+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
+# scrape_options = V1ScrapeOptions(parsePDF=False, formats=['markdown'])
# response = app.search("firecrawl", limit=1, scrape_options=scrape_options)
# assert response is not None
# assert 'data' in response
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.example b/apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.example
new file mode 100644
index 000000000..4e7633804
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.example
@@ -0,0 +1,2 @@
+API_URL=http://localhost:3002
+API_KEY=fc-YOUR_API_KEY
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
new file mode 100644
index 000000000..d85595397
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
@@ -0,0 +1,86 @@
+from firecrawl import Firecrawl
+import os
+import pytest
+from dotenv import load_dotenv
+# from pydantic import BaseModel
+# from firecrawl.types import ScrapeOptions, JsonFormat, Location, WaitAction, SourceOptions, FormatOptions
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+ raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+ raise ValueError("API_URL is not set")
+
+firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+
+def test_search_minimal_request():
+ search_response = firecrawl.search(
+ query="What is the capital of France?"
+ )
+
+ print(f"Success: {search_response.success}")
+ print(f"Data: {search_response.data}")
+ print(f"Warning: {search_response.warning}")
+# @pytest.mark.e2e
+# def test_search_images_request():
+# search_response = firecrawl.search(
+# query="What is the capital of France?",
+# sources=[ "images" ],
+# limit=5
+# )
+
+# print(search_response)
+
+# @pytest.mark.e2e
+# def test_search_all_options():
+# class Schema(BaseModel):
+# name: str
+# description: str
+# type: str
+# required: bool
+
+# search_response = firecrawl.search(
+# query="What is the capital of France?",
+# sources=[
+# { type: "web" },
+# { type: "news" },
+# { type: "images" }
+# ],
+# limit=10,
+# tbs="",
+# location="",
+# timeout=60000,
+# ignore_invalid_urls=False,
+# scrape_options=ScrapeOptions(
+# formats=[
+# 'markdown',
+# JsonFormat(
+# schema={},
+# prompt=""
+# )
+# ],
+# only_main_content=True,
+# # include_tags=["<string>"], # TODO: check this
+# # exclude_tags=["<string>"], # TODO: check this
+# max_age=0,
+# headers={},
+# wait_for=0,
+# mobile=False,
+# skip_tls_verification=False,
+# timeout=30000,
+# parsers=[ 'pdf' ],
+# actions=[
+# WaitAction(milliseconds=2000)
+# ],
+# location=Location(
+# country="US",
+# languages=["en-US"]
+# ),
+# remove_base64_images=True,
+# block_ads=True,
+# proxy="basic",
+# store_in_cache=True,
+# )
+# )
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
new file mode 100644
index 000000000..9e415fc15
--- /dev/null
+++ b/apps/python-sdk/firecrawl/client.py
@@ -0,0 +1,164 @@
+"""
+Firecrawl Client
+
+A Firecrawl client that enables you to scrape content from websites, crawl entire sites, search the web, and extract structured data using AI.
+
+The client supports both v1 and v2 API versions, providing access to features like:
+- Web scraping with advanced options (screenshots, markdown conversion, etc.)
+- Site crawling with configurable depth and limits
+- Web search with content extraction
+- Structured data extraction using AI models
+- Deep research capabilities
+
+Usage:
+ from firecrawl import Firecrawl
+ firecrawl = Firecrawl(api_key="your-api-key")
+ result = firecrawl.scrape("https://example.com")
+
+Check example.py for other usage examples.
+"""
+
+from typing import Any, Dict, Optional, List, Union
+import logging
+
+
+from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp
+from .v2 import FirecrawlClient as V2FirecrawlClient
+
+logger = logging.getLogger("firecrawl")
+
+class V1Proxy:
+ """Type-annotated proxy for v1 client methods."""
+ _client: Optional[V1FirecrawlApp]
+
+ def __init__(self, client_instance: Optional[V1FirecrawlApp]):
+ self._client = client_instance
+
+ if client_instance:
+ self.scrape_url = client_instance.scrape_url
+ self.crawl_url = client_instance.crawl_url
+ self.batch_scrape_urls = client_instance.batch_scrape_urls
+ self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls
+ self.async_crawl_url = client_instance.async_crawl_url
+ self.check_crawl_status = client_instance.check_crawl_status
+ self.map_url = client_instance.map_url
+ self.extract = client_instance.extract
+ self.deep_research = client_instance.deep_research
+ self.generate_llms_text = client_instance.generate_llms_text
+
+class V2Proxy:
+ """Proxy class that forwards method calls to the appropriate version client."""
+ _client: Optional[V2FirecrawlClient]
+
+ def __init__(self, client_instance: Optional[V2FirecrawlClient]):
+ self._client = client_instance
+
+ # TODO: Not implemented yet
+ if client_instance:
+ self.scrape = client_instance.scrape
+ self.search = client_instance.search
+ self.crawl = client_instance.crawl
+ self.batch_scrape = client_instance.batch_scrape
+ self.get_crawl_status = client_instance.get_crawl_status
+ self.cancel_crawl = client_instance.cancel_crawl
+
+ def __getattr__(self, name):
+ """Forward attribute access to the underlying client."""
+ return getattr(self._client, name)
+
+class AsyncV1Proxy:
+ """Type-annotated proxy for v1 client methods."""
+ _client: Optional[AsyncV1FirecrawlApp]
+
+ def __init__(self, client_instance: Optional[AsyncV1FirecrawlApp]):
+ self._client = client_instance
+
+ if client_instance:
+ self.scrape_url = client_instance.scrape_url
+ self.crawl_url = client_instance.crawl_url
+ self.batch_scrape_urls = client_instance.batch_scrape_urls
+ self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls
+ self.async_crawl_url = client_instance.async_crawl_url
+ self.check_crawl_status = client_instance.check_crawl_status
+ self.map_url = client_instance.map_url
+ self.extract = client_instance.extract
+ self.deep_research = client_instance.deep_research
+ self.generate_llms_text = client_instance.generate_llms_text
+
+class AsyncV2Proxy:
+ """Proxy class that forwards method calls to the appropriate version client."""
+ _client: Optional[Any] = None
+
+ def __init__(self, client_instance: Optional[Any] = None):
+ self._client = client_instance
+ # TODO: Async v2 client not implemented yet
+
+ def __getattr__(self, name):
+ """Forward attribute access to the underlying client."""
+ if self._client:
+ return getattr(self._client, name)
+ raise AttributeError(f"Async v2 client not implemented yet: {name}")
+
+
+class Firecrawl:
+ """
+ Unified Firecrawl client with version-specific access patterns.
+
+ This class provides a unified interface that allows users to access
+ different API versions through version-specific attributes (v1, v2)
+ while defaulting to v2 for direct method calls.
+ """
+
+ def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
+ """
+ Initialize the unified Firecrawl client.
+
+ Args:
+ api_key: Your Firecrawl API key
+ api_url: The Firecrawl API URL (defaults to production)
+ """
+ self.api_key = api_key
+ self.api_url = api_url
+
+ # Initialize version-specific clients
+ self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None
+ self._v2_client = V2FirecrawlClient(api_key=api_key, api_url=api_url) if V2FirecrawlClient else None
+
+ # Create version-specific proxies
+ self.v1 = V1Proxy(self._v1_client) if self._v1_client else None
+ self.v2 = V2Proxy(self._v2_client)
+
+ # Methods
+ # TODO: Not implemented yet
+ # self.scrape = self._v2_client.scrape
+ # self.crawl = self._v2_client.crawl
+ # self.batch_scrape = self._v2_client.batch_scrape
+ # self.get_crawl_status = self._v2_client.get_crawl_status
+ # self.cancel_crawl = self._v2_client.cancel_crawl
+ self.search = self._v2_client.search
+
+class AsyncFirecrawl:
+ """
+ Unified Firecrawl client with version-specific access patterns.
+
+ This class provides a unified interface that allows users to access
+ different API versions through version-specific attributes (v1, v2)
+ while defaulting to v2 for direct method calls.
+ """
+
+ def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
+ self.api_key = api_key
+ self.api_url = api_url
+
+ # Initialize version-specific clients
+ self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None
+ # TODO: Async v2 client not implemented yet
+ self._v2_client = None
+
+ # Create version-specific proxies
+ self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None
+ self.v2 = AsyncV2Proxy(self._v2_client)
+
+# Export Firecrawl as an alias for FirecrawlApp
+FirecrawlApp = Firecrawl
+AsyncFirecrawlApp = AsyncFirecrawl
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.py
new file mode 100644
index 000000000..66538f84d
--- /dev/null
+++ b/apps/python-sdk/firecrawl/types.py
@@ -0,0 +1,168 @@
+"""
+Unified Firecrawl Types
+
+This module provides unified access to Firecrawl types across all API versions.
+Currently exports v2 types as the primary interface.
+"""
+
+from .v2.types import (
+ # Base types
+ BaseResponse,
+
+ # Document types
+ Document,
+ DocumentMetadata,
+
+ # Scrape types
+ ScrapeFormats,
+ ScrapeOptions,
+ ScrapeRequest,
+ ScrapeResponse,
+
+ # Crawl types
+ CrawlOptions,
+ CrawlRequest,
+ CrawlJob,
+ CrawlResponse,
+ CrawlStatusData,
+ CrawlStatusResponse,
+
+ # Batch scrape types
+ BatchScrapeRequest,
+ BatchScrapeJob,
+ BatchScrapeResponse,
+ BatchScrapeStatusData,
+ BatchScrapeStatusResponse,
+
+ # Map types
+ MapOptions,
+ MapRequest,
+ MapData,
+ MapResponse,
+
+ # Search types
+ Source,
+ SourceOption,
+ Format,
+ FormatOption,
+ SearchRequest,
+ SearchResult,
+ SearchResponse,
+
+ # Action types
+ WaitAction,
+ ScreenshotAction,
+ ClickAction,
+ WriteAction,
+ PressAction,
+ ScrollAction,
+ ScrapeAction,
+ ExecuteJavascriptAction,
+ PDFAction,
+
+ # Location and format types
+ Location,
+ JsonFormat,
+
+ # Error types
+ ErrorDetails,
+ ErrorResponse,
+
+ # Job management types
+ JobStatus,
+
+ # Webhook types
+ WebhookData,
+
+ # Configuration types
+ ClientConfig,
+
+ # Union types
+ ScrapeResult,
+ CrawlResult,
+ BatchResult,
+ JobResult,
+ StatusResult,
+ AnyResponse,
+)
+
+__all__ = [
+ # Base types
+ 'BaseResponse',
+
+ # Document types
+ 'Document',
+ 'DocumentMetadata',
+
+ # Scrape types
+ 'ScrapeFormats',
+ 'ScrapeOptions',
+ 'ScrapeRequest',
+ 'ScrapeResponse',
+
+ # Crawl types
+ 'CrawlOptions',
+ 'CrawlRequest',
+ 'CrawlJob',
+ 'CrawlResponse',
+ 'CrawlStatusData',
+ 'CrawlStatusResponse',
+
+ # Batch scrape types
+ 'BatchScrapeRequest',
+ 'BatchScrapeJob',
+ 'BatchScrapeResponse',
+ 'BatchScrapeStatusData',
+ 'BatchScrapeStatusResponse',
+
+ # Map types
+ 'MapOptions',
+ 'MapRequest',
+ 'MapData',
+ 'MapResponse',
+
+ # Search types
+ 'Source',
+ 'SourceOption',
+ 'Format',
+ 'FormatOption',
+ 'SearchRequest',
+ 'SearchResult',
+ 'SearchResponse',
+
+ # Action types
+ 'WaitAction',
+ 'ScreenshotAction',
+ 'ClickAction',
+ 'WriteAction',
+ 'PressAction',
+ 'ScrollAction',
+ 'ScrapeAction',
+ 'ExecuteJavascriptAction',
+ 'PDFAction',
+
+ # Location and format types
+ 'Location',
+ 'JsonFormat',
+
+ # Error types
+ 'ErrorDetails',
+ 'ErrorResponse',
+
+ # Job management types
+ 'JobStatus',
+
+ # Webhook types
+ 'WebhookData',
+
+ # Configuration types
+ 'ClientConfig',
+
+ # Union types
+ 'ScrapeResult',
+ 'CrawlResult',
+ 'BatchResult',
+ 'JobResult',
+ 'StatusResult',
+ 'AnyResponse',
+]
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v1/__init__.py b/apps/python-sdk/firecrawl/v1/__init__.py
new file mode 100644
index 000000000..6c831cc46
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v1/__init__.py
@@ -0,0 +1,14 @@
+"""
+Firecrawl v1 API (Legacy)
+
+This module provides the legacy v1 API for backward compatibility.
+
+Usage:
+ from firecrawl.v1 import V1FirecrawlApp
+ app = V1FirecrawlApp(api_key="your-api-key")
+ result = app.scrape_url("https://example.com")
+"""
+
+from .client import V1FirecrawlApp, AsyncV1FirecrawlApp, V1JsonConfig, V1ScrapeOptions, V1ChangeTrackingOptions
+
+__all__ = ['V1FirecrawlApp', 'AsyncV1FirecrawlApp', 'V1JsonConfig', 'V1ScrapeOptions', 'V1ChangeTrackingOptions']
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
new file mode 100644
index 000000000..a48c5f5f5
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -0,0 +1,4645 @@
+"""
+Firecrawl v1 API Client - Legacy Implementation
+
+This module provides the legacy v1 implementation of the Firecrawl SDK.
+It contains the complete `V1FirecrawlApp` class with all v1 API methods and types
+for backward compatibility. This is used by the unified client to provide
+version-specific access patterns like app.v1.scrape_url().
+
+Classes:
+ - V1FirecrawlApp: Legacy v1 client for interacting with the Firecrawl API.
+ - AsyncV1FirecrawlApp: Async version of the v1 client.
+ - CrawlWatcher: WebSocket-based crawl monitoring for v1.
+"""
+import logging
+import os
+import time
+from typing import Any, Dict, Optional, List, Union, Callable, Literal, TypeVar, Generic
+import json
+from datetime import datetime
+import re
+import requests
+import pydantic
+import websockets
+import aiohttp
+import asyncio
+
+logger : logging.Logger = logging.getLogger("firecrawl")
+
+def get_version():
+ try:
+ from pathlib import Path
+ package_path = os.path.dirname(__file__)
+ version_file = Path(os.path.join(package_path, '__init__.py')).read_text()
+ version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
+ if version_match:
+ return version_match.group(1).strip()
+ except Exception:
+ print("Failed to get version from __init__.py")
+ return None
+
+version = get_version()
+
+T = TypeVar('T')
+
+# class V1FirecrawlDocumentMetadata(pydantic.BaseModel):
+# """Metadata for a Firecrawl document."""
+# title: Optional[str] = None
+# description: Optional[str] = None
+# language: Optional[str] = None
+# keywords: Optional[str] = None
+# robots: Optional[str] = None
+# ogTitle: Optional[str] = None
+# ogDescription: Optional[str] = None
+# ogUrl: Optional[str] = None
+# ogImage: Optional[str] = None
+# ogAudio: Optional[str] = None
+# ogDeterminer: Optional[str] = None
+# ogLocale: Optional[str] = None
+# ogLocaleAlternate: Optional[List[str]] = None
+# ogSiteName: Optional[str] = None
+# ogVideo: Optional[str] = None
+# dctermsCreated: Optional[str] = None
+# dcDateCreated: Optional[str] = None
+# dcDate: Optional[str] = None
+# dctermsType: Optional[str] = None
+# dcType: Optional[str] = None
+# dctermsAudience: Optional[str] = None
+# dctermsSubject: Optional[str] = None
+# dcSubject: Optional[str] = None
+# dcDescription: Optional[str] = None
+# dctermsKeywords: Optional[str] = None
+# modifiedTime: Optional[str] = None
+# publishedTime: Optional[str] = None
+# articleTag: Optional[str] = None
+# articleSection: Optional[str] = None
+# sourceURL: Optional[str] = None
+# statusCode: Optional[int] = None
+# error: Optional[str] = None
+
+class V1AgentOptions(pydantic.BaseModel):
+ """Configuration for the agent."""
+ model: Literal["FIRE-1"] = "FIRE-1"
+ prompt: Optional[str] = None
+
+class V1AgentOptionsExtract(pydantic.BaseModel):
+ """Configuration for the agent in extract operations."""
+ model: Literal["FIRE-1"] = "FIRE-1"
+
+class V1ActionsResult(pydantic.BaseModel):
+ """Result of actions performed during scraping."""
+ screenshots: List[str]
+ pdfs: List[str]
+
+class V1ChangeTrackingData(pydantic.BaseModel):
+ """
+ Data for the change tracking format.
+ """
+ previousScrapeAt: Optional[str] = None
+ changeStatus: str # "new" | "same" | "changed" | "removed"
+ visibility: str # "visible" | "hidden"
+ diff: Optional[Dict[str, Any]] = None
+ json_field: Optional[Any] = pydantic.Field(None, alias='json')
+
+class V1FirecrawlDocument(pydantic.BaseModel, Generic[T]):
+ """Document retrieved or processed by Firecrawl."""
+ url: Optional[str] = None
+ markdown: Optional[str] = None
+ html: Optional[str] = None
+ rawHtml: Optional[str] = None
+ links: Optional[List[str]] = None
+ extract: Optional[T] = None
+ json_field: Optional[T] = pydantic.Field(None, alias='json')
+ screenshot: Optional[str] = None
+ metadata: Optional[Any] = None
+ actions: Optional[V1ActionsResult] = None
+ title: Optional[str] = None # v1 search only
+ description: Optional[str] = None # v1 search only
+ changeTracking: Optional[V1ChangeTrackingData] = None
+
+class V1LocationConfig(pydantic.BaseModel):
+ """Location configuration for scraping."""
+ country: Optional[str] = None
+ languages: Optional[List[str]] = None
+
+class V1WebhookConfig(pydantic.BaseModel):
+ """Configuration for webhooks."""
+ url: str
+ headers: Optional[Dict[str, str]] = None
+ metadata: Optional[Dict[str, str]] = None
+ events: Optional[List[Literal["completed", "failed", "page", "started"]]] = None
+
+class V1ChangeTrackingOptions(pydantic.BaseModel):
+ """Configuration for change tracking."""
+ modes: Optional[List[Literal["git-diff", "json"]]] = None
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
+ prompt: Optional[str] = None
+ tag: Optional[str] = None
+
+class V1ScrapeOptions(pydantic.BaseModel):
+ """Parameters for scraping operations."""
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None
+ headers: Optional[Dict[str, str]] = None
+ includeTags: Optional[List[str]] = None
+ excludeTags: Optional[List[str]] = None
+ onlyMainContent: Optional[bool] = None
+ waitFor: Optional[int] = None
+ timeout: Optional[int] = 30000
+ location: Optional[V1LocationConfig] = None
+ mobile: Optional[bool] = None
+ skipTlsVerification: Optional[bool] = None
+ removeBase64Images: Optional[bool] = None
+ blockAds: Optional[bool] = None
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None
+ changeTrackingOptions: Optional[V1ChangeTrackingOptions] = None
+ maxAge: Optional[int] = None
+ storeInCache: Optional[bool] = None
+ parsePDF: Optional[bool] = None
+
+class V1WaitAction(pydantic.BaseModel):
+ """Wait action to perform during scraping."""
+ type: Literal["wait"]
+ milliseconds: Optional[int] = None
+ selector: Optional[str] = None
+
+class V1ScreenshotAction(pydantic.BaseModel):
+ """Screenshot action to perform during scraping."""
+ type: Literal["screenshot"]
+ fullPage: Optional[bool] = None
+ quality: Optional[int] = None
+
+class V1ClickAction(pydantic.BaseModel):
+ """Click action to perform during scraping."""
+ type: Literal["click"]
+ selector: str
+
+class V1WriteAction(pydantic.BaseModel):
+ """Write action to perform during scraping."""
+ type: Literal["write"]
+ text: str
+
+class V1PressAction(pydantic.BaseModel):
+ """Press action to perform during scraping."""
+ type: Literal["press"]
+ key: str
+
+class V1ScrollAction(pydantic.BaseModel):
+ """Scroll action to perform during scraping."""
+ type: Literal["scroll"]
+ direction: Literal["up", "down"]
+ selector: Optional[str] = None
+
+class V1ScrapeAction(pydantic.BaseModel):
+ """Scrape action to perform during scraping."""
+ type: Literal["scrape"]
+
+class V1ExecuteJavascriptAction(pydantic.BaseModel):
+ """Execute javascript action to perform during scraping."""
+ type: Literal["executeJavascript"]
+ script: str
+
+class V1PDFAction(pydantic.BaseModel):
+ """PDF action to perform during scraping."""
+ type: Literal["pdf"]
+ format: Optional[Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"]] = None
+ landscape: Optional[bool] = None
+ scale: Optional[float] = None
+
+class V1ExtractAgent(pydantic.BaseModel):
+ """Configuration for the agent in extract operations."""
+ model: Literal["FIRE-1"] = "FIRE-1"
+
+class V1JsonConfig(pydantic.BaseModel):
+ """Configuration for extraction."""
+ prompt: Optional[str] = None
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
+ systemPrompt: Optional[str] = None
+ agent: Optional[V1ExtractAgent] = None
+
+class V1ScrapeParams(V1ScrapeOptions):
+ """Parameters for scraping operations."""
+ extract: Optional[V1JsonConfig] = None
+ jsonOptions: Optional[V1JsonConfig] = None
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None
+ agent: Optional[V1AgentOptions] = None
+ webhook: Optional[V1WebhookConfig] = None
+
+class V1ScrapeResponse(V1FirecrawlDocument[T], Generic[T]):
+ """Response from scraping operations."""
+ success: bool = True
+ warning: Optional[str] = None
+ error: Optional[str] = None
+
+class V1BatchScrapeResponse(pydantic.BaseModel):
+ """Response from batch scrape operations."""
+ id: Optional[str] = None
+ url: Optional[str] = None
+ success: bool = True
+ error: Optional[str] = None
+ invalidURLs: Optional[List[str]] = None
+
+class V1BatchScrapeStatusResponse(pydantic.BaseModel):
+ """Response from batch scrape status checks."""
+ success: bool = True
+ status: Literal["scraping", "completed", "failed", "cancelled"]
+ completed: int
+ total: int
+ creditsUsed: int
+ expiresAt: datetime
+ next: Optional[str] = None
+ data: List[V1FirecrawlDocument]
+
+class V1CrawlParams(pydantic.BaseModel):
+ """Parameters for crawling operations."""
+ includePaths: Optional[List[str]] = None
+ excludePaths: Optional[List[str]] = None
+ maxDepth: Optional[int] = None
+ maxDiscoveryDepth: Optional[int] = None
+ limit: Optional[int] = None
+ allowBackwardLinks: Optional[bool] = None
+ crawlEntireDomain: Optional[bool] = None
+ allowExternalLinks: Optional[bool] = None
+ ignoreSitemap: Optional[bool] = None
+ scrapeOptions: Optional[V1ScrapeOptions] = None
+ webhook: Optional[Union[str, V1WebhookConfig]] = None
+ deduplicateSimilarURLs: Optional[bool] = None
+ ignoreQueryParameters: Optional[bool] = None
+ regexOnFullURL: Optional[bool] = None
+ delay: Optional[int] = None # Delay in seconds between scrapes
+ maxConcurrency: Optional[int] = None
+ allowSubdomains: Optional[bool] = None
+
+class V1CrawlResponse(pydantic.BaseModel):
+ """Response from crawling operations."""
+ id: Optional[str] = None
+ url: Optional[str] = None
+ success: bool = True
+ error: Optional[str] = None
+
+class V1CrawlStatusResponse(pydantic.BaseModel):
+ """Response from crawl status checks."""
+ success: bool = True
+ status: Literal["scraping", "completed", "failed", "cancelled"]
+ completed: int
+ total: int
+ creditsUsed: int
+ expiresAt: datetime
+ next: Optional[str] = None
+ data: List[V1FirecrawlDocument]
+
+class V1CrawlErrorsResponse(pydantic.BaseModel):
+ """Response from crawl/batch scrape error monitoring."""
+ errors: List[Dict[str, str]] # {id: str, timestamp: str, url: str, error: str}
+ robotsBlocked: List[str]
+
+class V1MapParams(pydantic.BaseModel):
+ """Parameters for mapping operations."""
+ search: Optional[str] = None
+ ignoreSitemap: Optional[bool] = None
+ includeSubdomains: Optional[bool] = None
+ sitemapOnly: Optional[bool] = None
+ limit: Optional[int] = None
+ timeout: Optional[int] = 30000
+ useIndex: Optional[bool] = None
+
+class V1MapResponse(pydantic.BaseModel):
+ """Response from mapping operations."""
+ success: bool = True
+ links: Optional[List[str]] = None
+ error: Optional[str] = None
+
+class V1ExtractParams(pydantic.BaseModel):
+ """Parameters for extracting information from URLs."""
+ prompt: Optional[str] = None
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
+ systemPrompt: Optional[str] = None
+ allowExternalLinks: Optional[bool] = None
+ enableWebSearch: Optional[bool] = None
+ includeSubdomains: Optional[bool] = None
+ origin: Optional[str] = None
+ showSources: Optional[bool] = None
+ scrapeOptions: Optional[V1ScrapeOptions] = None
+
+class V1ExtractResponse(pydantic.BaseModel, Generic[T]):
+ """Response from extract operations."""
+ id: Optional[str] = None
+ status: Optional[Literal["processing", "completed", "failed"]] = None
+ expiresAt: Optional[datetime] = None
+ success: bool = True
+ data: Optional[T] = None
+ error: Optional[str] = None
+ warning: Optional[str] = None
+ sources: Optional[Dict[Any, Any]] = None
+
+class V1SearchParams(pydantic.BaseModel):
+ query: str
+ limit: Optional[int] = 5
+ tbs: Optional[str] = None
+ filter: Optional[str] = None
+ lang: Optional[str] = "en"
+ country: Optional[str] = "us"
+ location: Optional[str] = None
+ origin: Optional[str] = "api"
+ timeout: Optional[int] = 60000
+ scrapeOptions: Optional[V1ScrapeOptions] = None
+
+class V1SearchResponse(pydantic.BaseModel):
+ """Response from search operations."""
+ success: bool = True
+ data: List[V1FirecrawlDocument]
+ warning: Optional[str] = None
+ error: Optional[str] = None
+
+class V1GenerateLLMsTextParams(pydantic.BaseModel):
+ """
+ Parameters for the LLMs.txt generation operation.
+ """
+ maxUrls: Optional[int] = 10
+ showFullText: Optional[bool] = False
+ cache: Optional[bool] = True
+ __experimental_stream: Optional[bool] = None
+
+class V1DeepResearchParams(pydantic.BaseModel):
+ """
+ Parameters for the deep research operation.
+ """
+ maxDepth: Optional[int] = 7
+ timeLimit: Optional[int] = 270
+ maxUrls: Optional[int] = 20
+ analysisPrompt: Optional[str] = None
+ systemPrompt: Optional[str] = None
+ __experimental_streamSteps: Optional[bool] = None
+
+class V1DeepResearchResponse(pydantic.BaseModel):
+ """
+ Response from the deep research operation.
+ """
+ success: bool
+ id: str
+ error: Optional[str] = None
+
+class V1DeepResearchStatusResponse(pydantic.BaseModel):
+ """
+ Status response from the deep research operation.
+ """
+ success: bool
+ data: Optional[Dict[str, Any]] = None
+ status: str
+ error: Optional[str] = None
+ expiresAt: str
+ currentDepth: int
+ maxDepth: int
+ activities: List[Dict[str, Any]]
+ sources: List[Dict[str, Any]]
+ summaries: List[str]
+
+class V1GenerateLLMsTextResponse(pydantic.BaseModel):
+ """Response from LLMs.txt generation operations."""
+ success: bool = True
+ id: str
+ error: Optional[str] = None
+
+class V1GenerateLLMsTextStatusResponseData(pydantic.BaseModel):
+ llmstxt: str
+ llmsfulltxt: Optional[str] = None
+
+class V1GenerateLLMsTextStatusResponse(pydantic.BaseModel):
+ """Status response from LLMs.txt generation operations."""
+ success: bool = True
+ data: Optional[V1GenerateLLMsTextStatusResponseData] = None
+ status: Literal["processing", "completed", "failed"]
+ error: Optional[str] = None
+ expiresAt: str
+
+class V1SearchResponse(pydantic.BaseModel):
+ """
+ Response from the search operation.
+ """
+ success: bool
+ data: List[Dict[str, Any]]
+ warning: Optional[str] = None
+ error: Optional[str] = None
+
+class V1ExtractParams(pydantic.BaseModel):
+ """
+ Parameters for the extract operation.
+ """
+ prompt: Optional[str] = None
+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')
+ system_prompt: Optional[str] = None
+ allow_external_links: Optional[bool] = False
+ enable_web_search: Optional[bool] = False
+ # Just for backwards compatibility
+ enableWebSearch: Optional[bool] = False
+ show_sources: Optional[bool] = False
+ agent: Optional[Dict[str, Any]] = None
+
+class V1FirecrawlApp:
+ """
+ Legacy v1 Firecrawl client for backward compatibility.
+
+ This class provides the complete v1 API implementation including:
+ - URL scraping with various formats and options
+ - Website crawling with monitoring capabilities
+ - Batch scraping operations
+ - Search functionality
+ - Data extraction with LLM integration
+ - Deep research capabilities
+ - LLMs.txt generation
+
+ This is used by the unified client to provide version-specific access
+ through app.v1.method_name() patterns.
+ """
+ def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:
+ """
+ Initialize the V1FirecrawlApp instance with API key, API URL.
+
+ Args:
+ api_key (Optional[str]): API key for authenticating with the Firecrawl API.
+ api_url (Optional[str]): Base URL for the Firecrawl API.
+ """
+ self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')
+ self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')
+
+ # Only require API key when using cloud service
+ if 'api.firecrawl.dev' in self.api_url and self.api_key is None:
+ logger.warning("No API key provided for cloud service")
+ raise ValueError('No API key provided')
+
+ logger.debug(f"Initialized V1FirecrawlApp with API URL: {self.api_url}")
+
+ def scrape_url(
+ self,
+ url: str,
+ *,
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
+ mobile: Optional[bool] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+ parse_pdf: Optional[bool] = None,
+ extract: Optional[V1JsonConfig] = None,
+ json_options: Optional[V1JsonConfig] = None,
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,
+ change_tracking_options: Optional[V1ChangeTrackingOptions] = None,
+ max_age: Optional[int] = None,
+ store_in_cache: Optional[bool] = None,
+ zero_data_retention: Optional[bool] = None,
+ **kwargs) -> V1ScrapeResponse[Any]:
+ """
+ Scrape and extract content from a URL.
+
+ Args:
+ url (str): Target URL to scrape
+ formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)
+ headers (Optional[Dict[str, str]]): Custom HTTP headers
+ include_tags (Optional[List[str]]): HTML tags to include
+ exclude_tags (Optional[List[str]]): HTML tags to exclude
+ only_main_content (Optional[bool]): Extract main content only
+ wait_for (Optional[int]): Wait for a specific element to appear
+ timeout (Optional[int]): Request timeout (ms)
+ location (Optional[LocationConfig]): Location configuration
+ mobile (Optional[bool]): Use mobile user agent
+ skip_tls_verification (Optional[bool]): Skip TLS verification
+ remove_base64_images (Optional[bool]): Remove base64 images
+ block_ads (Optional[bool]): Block ads
+ proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)
+ extract (Optional[JsonConfig]): Content extraction settings
+ json_options (Optional[JsonConfig]): JSON extraction settings
+ actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
+ change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings
+ zero_data_retention (Optional[bool]): Whether to delete data after scrape is done
+
+
+ Returns:
+ ScrapeResponse with:
+ * Requested content formats
+ * Page metadata
+ * Extraction results
+ * Success/error status
+
+ Raises:
+ Exception: If scraping fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "scrape_url")
+
+ _headers = self._prepare_headers()
+
+ # Build scrape parameters
+ scrape_params = {
+ 'url': url,
+ 'origin': f"python-sdk@{version}"
+ }
+
+ # Add optional parameters if provided
+ if formats:
+ scrape_params['formats'] = formats
+ if headers:
+ scrape_params['headers'] = headers
+ if include_tags:
+ scrape_params['includeTags'] = include_tags
+ if exclude_tags:
+ scrape_params['excludeTags'] = exclude_tags
+ if only_main_content is not None:
+ scrape_params['onlyMainContent'] = only_main_content
+ if wait_for:
+ scrape_params['waitFor'] = wait_for
+ if timeout:
+ scrape_params['timeout'] = timeout
+ if location:
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
+ if mobile is not None:
+ scrape_params['mobile'] = mobile
+ if skip_tls_verification is not None:
+ scrape_params['skipTlsVerification'] = skip_tls_verification
+ if remove_base64_images is not None:
+ scrape_params['removeBase64Images'] = remove_base64_images
+ if block_ads is not None:
+ scrape_params['blockAds'] = block_ads
+ if proxy:
+ scrape_params['proxy'] = proxy
+ if parse_pdf is not None:
+ scrape_params['parsePDF'] = parse_pdf
+ if extract is not None:
+ extract = self._ensure_schema_dict(extract)
+ if isinstance(extract, dict) and "schema" in extract:
+ extract["schema"] = self._ensure_schema_dict(extract["schema"])
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
+ if json_options is not None:
+ json_options = self._ensure_schema_dict(json_options)
+ if isinstance(json_options, dict) and "schema" in json_options:
+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions:
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if change_tracking_options:
+ scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(by_alias=True, exclude_none=True)
+ if max_age is not None:
+ scrape_params['maxAge'] = max_age
+ if store_in_cache is not None:
+ scrape_params['storeInCache'] = store_in_cache
+ if zero_data_retention is not None:
+ scrape_params['zeroDataRetention'] = zero_data_retention
+
+ scrape_params.update(kwargs)
+
+ if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:
+ scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])
+ if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:
+ scrape_params['jsonOptions']['schema'] = self._ensure_schema_dict(scrape_params['jsonOptions']['schema'])
+
+ # Make request
+ response = requests.post(
+ f'{self.api_url}/v1/scrape',
+ headers=_headers,
+ json=scrape_params,
+ timeout=(timeout / 1000.0 + 5 if timeout is not None else None)
+ )
+
+ if response.status_code == 200:
+ try:
+ response_json = response.json()
+ if response_json.get('success') and 'data' in response_json:
+ return V1ScrapeResponse(**response_json['data'])
+ elif "error" in response_json:
+ raise Exception(f'Failed to scrape URL. Error: {response_json["error"]}')
+ else:
+ raise Exception(f'Failed to scrape URL. Error: {response_json}')
+ except ValueError:
+ raise Exception('Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'scrape URL')
+
+ def search(
+ self,
+ query: str,
+ *,
+ limit: Optional[int] = None,
+ tbs: Optional[str] = None,
+ filter: Optional[str] = None,
+ lang: Optional[str] = None,
+ country: Optional[str] = None,
+ location: Optional[str] = None,
+ timeout: Optional[int] = 30000,
+ scrape_options: Optional[V1ScrapeOptions] = None,
+ **kwargs) -> V1SearchResponse:
+ """
+ Search for content using Firecrawl.
+
+ Args:
+ query (str): Search query string
+ limit (Optional[int]): Max results (default: 5)
+ tbs (Optional[str]): Time filter (e.g. "qdr:d")
+ filter (Optional[str]): Custom result filter
+ lang (Optional[str]): Language code (default: "en")
+ country (Optional[str]): Country code (default: "us")
+ location (Optional[str]): Geo-targeting
+ timeout (Optional[int]): Request timeout in milliseconds
+ scrape_options (Optional[ScrapeOptions]): Result scraping configuration
+ **kwargs: Additional keyword arguments for future compatibility
+
+ Returns:
+ SearchResponse: Response containing:
+ * success (bool): Whether request succeeded
+ * data (List[FirecrawlDocument]): Search results
+ * warning (Optional[str]): Warning message if any
+ * error (Optional[str]): Error message if any
+
+ Raises:
+ Exception: If search fails or response cannot be parsed
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "search")
+
+ # Build search parameters
+ search_params = {}
+
+ # Add individual parameters
+ if limit is not None:
+ search_params['limit'] = limit
+ if tbs is not None:
+ search_params['tbs'] = tbs
+ if filter is not None:
+ search_params['filter'] = filter
+ if lang is not None:
+ search_params['lang'] = lang
+ if country is not None:
+ search_params['country'] = country
+ if location is not None:
+ search_params['location'] = location
+ if timeout is not None:
+ search_params['timeout'] = timeout
+ if scrape_options is not None:
+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
+
+ # Add any additional kwargs
+ search_params.update(kwargs)
+ _integration = search_params.get('integration')
+
+ # Create final params object
+ final_params = V1SearchParams(query=query, **search_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if _integration:
+ params_dict['integration'] = _integration
+
+ # Make request
+ response = requests.post(
+ f"{self.api_url}/v1/search",
+ headers={"Authorization": f"Bearer {self.api_key}"},
+ json=params_dict
+ )
+
+ if response.status_code == 200:
+ try:
+ response_json = response.json()
+ if response_json.get('success') and 'data' in response_json:
+ return V1SearchResponse(**response_json)
+ elif "error" in response_json:
+ raise Exception(f'Search failed. Error: {response_json["error"]}')
+ else:
+ raise Exception(f'Search failed. Error: {response_json}')
+ except ValueError:
+ raise Exception('Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'search')
+
+ def crawl_url(
+ self,
+ url: str,
+ *,
+ include_paths: Optional[List[str]] = None,
+ exclude_paths: Optional[List[str]] = None,
+ max_depth: Optional[int] = None,
+ max_discovery_depth: Optional[int] = None,
+ limit: Optional[int] = None,
+ allow_backward_links: Optional[bool] = None,
+ crawl_entire_domain: Optional[bool] = None,
+ allow_external_links: Optional[bool] = None,
+ ignore_sitemap: Optional[bool] = None,
+ scrape_options: Optional[V1ScrapeOptions] = None,
+ webhook: Optional[Union[str, V1WebhookConfig]] = None,
+ deduplicate_similar_urls: Optional[bool] = None,
+ ignore_query_parameters: Optional[bool] = None,
+ regex_on_full_url: Optional[bool] = None,
+ delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
+ max_concurrency: Optional[int] = None,
+ zero_data_retention: Optional[bool] = None,
+ poll_interval: Optional[int] = 2,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> V1CrawlStatusResponse:
+ """
+ Crawl a website starting from a URL.
+
+ Args:
+ url (str): Target URL to start crawling from
+ include_paths (Optional[List[str]]): Patterns of URLs to include
+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
+ max_depth (Optional[int]): Maximum crawl depth
+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
+ limit (Optional[int]): Maximum pages to crawl
+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+ crawl_entire_domain (Optional[bool]): Follow parent directory links
+ allow_external_links (Optional[bool]): Follow external domain links
+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
+ scrape_options (Optional[ScrapeOptions]): Page scraping configuration
+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs
+ ignore_query_parameters (Optional[bool]): Ignore URL parameters
+ regex_on_full_url (Optional[bool]): Apply regex to full URLs
+ delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes
+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
+ poll_interval (Optional[int]): Seconds between status checks (default: 2)
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ CrawlStatusResponse with:
+ * Crawling status and progress
+ * Crawled page contents
+ * Success/error information
+
+ Raises:
+ Exception: If crawl fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "crawl_url")
+
+ crawl_params = {}
+
+ # Add individual parameters
+ if include_paths is not None:
+ crawl_params['includePaths'] = include_paths
+ if exclude_paths is not None:
+ crawl_params['excludePaths'] = exclude_paths
+ if max_depth is not None:
+ crawl_params['maxDepth'] = max_depth
+ if max_discovery_depth is not None:
+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth
+ if limit is not None:
+ crawl_params['limit'] = limit
+ if crawl_entire_domain is not None:
+ crawl_params['crawlEntireDomain'] = crawl_entire_domain
+ elif allow_backward_links is not None:
+ crawl_params['allowBackwardLinks'] = allow_backward_links
+ if allow_external_links is not None:
+ crawl_params['allowExternalLinks'] = allow_external_links
+ if ignore_sitemap is not None:
+ crawl_params['ignoreSitemap'] = ignore_sitemap
+ if scrape_options is not None:
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
+ if webhook is not None:
+ crawl_params['webhook'] = webhook
+ if deduplicate_similar_urls is not None:
+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls
+ if ignore_query_parameters is not None:
+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters
+ if regex_on_full_url is not None:
+ crawl_params['regexOnFullURL'] = regex_on_full_url
+ if delay is not None:
+ crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
+ if max_concurrency is not None:
+ crawl_params['maxConcurrency'] = max_concurrency
+ if zero_data_retention is not None:
+ crawl_params['zeroDataRetention'] = zero_data_retention
+ # Add any additional kwargs
+ crawl_params.update(kwargs)
+ _integration = crawl_params.get('integration')
+
+ # Create final params object
+ final_params = V1CrawlParams(**crawl_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['url'] = url
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if _integration:
+ params_dict['integration'] = _integration
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = self._post_request(f'{self.api_url}/v1/crawl', params_dict, headers)
+
+ if response.status_code == 200:
+ try:
+ id = response.json().get('id')
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ return self._monitor_job_status(id, headers, poll_interval)
+ else:
+ self._handle_error(response, 'start crawl job')
+
+ def async_crawl_url(
+ self,
+ url: str,
+ *,
+ include_paths: Optional[List[str]] = None,
+ exclude_paths: Optional[List[str]] = None,
+ max_depth: Optional[int] = None,
+ max_discovery_depth: Optional[int] = None,
+ limit: Optional[int] = None,
+ allow_backward_links: Optional[bool] = None,
+ crawl_entire_domain: Optional[bool] = None,
+ allow_external_links: Optional[bool] = None,
+ ignore_sitemap: Optional[bool] = None,
+ scrape_options: Optional[V1ScrapeOptions] = None,
+ webhook: Optional[Union[str, V1WebhookConfig]] = None,
+ deduplicate_similar_urls: Optional[bool] = None,
+ ignore_query_parameters: Optional[bool] = None,
+ regex_on_full_url: Optional[bool] = None,
+ delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
+ max_concurrency: Optional[int] = None,
+ zero_data_retention: Optional[bool] = None,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> V1CrawlResponse:
+ """
+ Start an asynchronous crawl job.
+
+ Args:
+ url (str): Target URL to start crawling from
+ include_paths (Optional[List[str]]): Patterns of URLs to include
+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
+ max_depth (Optional[int]): Maximum crawl depth
+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
+ limit (Optional[int]): Maximum pages to crawl
+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+ crawl_entire_domain (Optional[bool]): Follow parent directory links
+ allow_external_links (Optional[bool]): Follow external domain links
+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
+ scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
+ webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs
+ ignore_query_parameters (Optional[bool]): Ignore URL parameters
+ regex_on_full_url (Optional[bool]): Apply regex to full URLs
+ delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes
+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1CrawlResponse with:
+ * success - Whether crawl started successfully
+ * id - Unique identifier for the crawl job
+ * url - Status check URL for the crawl
+ * error - Error message if start failed
+
+ Raises:
+ Exception: If crawl initiation fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "async_crawl_url")
+
+ crawl_params = {}
+
+ # Add individual parameters
+ if include_paths is not None:
+ crawl_params['includePaths'] = include_paths
+ if exclude_paths is not None:
+ crawl_params['excludePaths'] = exclude_paths
+ if max_depth is not None:
+ crawl_params['maxDepth'] = max_depth
+ if max_discovery_depth is not None:
+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth
+ if limit is not None:
+ crawl_params['limit'] = limit
+ if crawl_entire_domain is not None:
+ crawl_params['crawlEntireDomain'] = crawl_entire_domain
+ elif allow_backward_links is not None:
+ crawl_params['allowBackwardLinks'] = allow_backward_links
+ if allow_external_links is not None:
+ crawl_params['allowExternalLinks'] = allow_external_links
+ if ignore_sitemap is not None:
+ crawl_params['ignoreSitemap'] = ignore_sitemap
+ if scrape_options is not None:
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
+ if webhook is not None:
+ crawl_params['webhook'] = webhook
+ if deduplicate_similar_urls is not None:
+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls
+ if ignore_query_parameters is not None:
+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters
+ if regex_on_full_url is not None:
+ crawl_params['regexOnFullURL'] = regex_on_full_url
+ if delay is not None:
+ crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
+ if max_concurrency is not None:
+ crawl_params['maxConcurrency'] = max_concurrency
+ if zero_data_retention is not None:
+ crawl_params['zeroDataRetention'] = zero_data_retention
+ # Add any additional kwargs
+ crawl_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1CrawlParams(**crawl_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['url'] = url
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = self._post_request(f'{self.api_url}/v1/crawl', params_dict, headers)
+
+ if response.status_code == 200:
+ try:
+ return V1CrawlResponse(**response.json())
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'start crawl job')
+
+ def check_crawl_status(self, id: str) -> V1CrawlStatusResponse:
+ """
+ Check the status and results of a crawl job.
+
+ Args:
+ id: Unique identifier for the crawl job
+
+ Returns:
+ V1CrawlStatusResponse containing:
+
+ Status Information:
+ * status - Current state (scraping/completed/failed/cancelled)
+ * completed - Number of pages crawled
+ * total - Total pages to crawl
+ * creditsUsed - API credits consumed
+ * expiresAt - Data expiration timestamp
+
+ Results:
+ * data - List of crawled documents
+ * next - URL for next page of results (if paginated)
+ * success - Whether status check succeeded
+ * error - Error message if failed
+
+ Raises:
+ Exception: If status check fails
+ """
+ endpoint = f'/v1/crawl/{id}'
+
+ headers = self._prepare_headers()
+ response = self._get_request(f'{self.api_url}{endpoint}', headers)
+ if response.status_code == 200:
+ try:
+ status_data = response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ if status_data['status'] == 'completed':
+ if 'data' in status_data:
+ data = status_data['data']
+ while 'next' in status_data:
+ if len(status_data['data']) == 0:
+ break
+ next_url = status_data.get('next')
+ if not next_url:
+ logger.warning("Expected 'next' URL is missing.")
+ break
+ try:
+ status_response = self._get_request(next_url, headers)
+ if status_response.status_code != 200:
+ logger.error(f"Failed to fetch next page: {status_response.status_code}")
+ break
+ try:
+ next_data = status_response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ data.extend(next_data.get('data', []))
+ status_data = next_data
+ except Exception as e:
+ logger.error(f"Error during pagination request: {e}")
+ break
+ status_data['data'] = data
+
+ response = {
+ 'status': status_data.get('status'),
+ 'total': status_data.get('total'),
+ 'completed': status_data.get('completed'),
+ 'creditsUsed': status_data.get('creditsUsed'),
+ 'expiresAt': status_data.get('expiresAt'),
+ 'data': status_data.get('data')
+ }
+
+ if 'error' in status_data:
+ response['error'] = status_data['error']
+
+ if 'next' in status_data:
+ response['next'] = status_data['next']
+
+ return V1CrawlStatusResponse(
+ success=False if 'error' in status_data else True,
+ **response
+ )
+ else:
+ self._handle_error(response, 'check crawl status')
+
+ def check_crawl_errors(self, id: str) -> V1CrawlErrorsResponse:
+ """
+ Returns information about crawl errors.
+
+ Args:
+ id (str): The ID of the crawl job
+
+ Returns:
+ V1CrawlErrorsResponse containing:
+ * errors (List[Dict[str, str]]): List of errors with fields:
+ - id (str): Error ID
+ - timestamp (str): When the error occurred
+ - url (str): URL that caused the error
+ - error (str): Error message
+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt
+
+ Raises:
+ Exception: If error check fails
+ """
+ headers = self._prepare_headers()
+ response = self._get_request(f'{self.api_url}/v1/crawl/{id}/errors', headers)
+ if response.status_code == 200:
+ try:
+ return V1CrawlErrorsResponse(**response.json())
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, "check crawl errors")
+
+ def cancel_crawl(self, id: str) -> Dict[str, Any]:
+ """
+ Cancel an asynchronous crawl job.
+
+ Args:
+ id (str): The ID of the crawl job to cancel
+
+ Returns:
+ Dict[str, Any] containing:
+ * success (bool): Whether cancellation was successful
+ * error (str, optional): Error message if cancellation failed
+
+ Raises:
+ Exception: If cancellation fails
+ """
+ headers = self._prepare_headers()
+ response = self._delete_request(f'{self.api_url}/v1/crawl/{id}', headers)
+ if response.status_code == 200:
+ try:
+ return response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, "cancel crawl job")
+
+ def crawl_url_and_watch(
+ self,
+ url: str,
+ *,
+ include_paths: Optional[List[str]] = None,
+ exclude_paths: Optional[List[str]] = None,
+ max_depth: Optional[int] = None,
+ max_discovery_depth: Optional[int] = None,
+ limit: Optional[int] = None,
+ allow_backward_links: Optional[bool] = None,
+ crawl_entire_domain: Optional[bool] = None,
+ allow_external_links: Optional[bool] = None,
+ ignore_sitemap: Optional[bool] = None,
+ scrape_options: Optional[V1ScrapeOptions] = None,
+ webhook: Optional[Union[str, V1WebhookConfig]] = None,
+ deduplicate_similar_urls: Optional[bool] = None,
+ ignore_query_parameters: Optional[bool] = None,
+ regex_on_full_url: Optional[bool] = None,
+ delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
+ max_concurrency: Optional[int] = None,
+ zero_data_retention: Optional[bool] = None,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> 'V1CrawlWatcher':
+ """
+ Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket.
+
+ Args:
+ url (str): Target URL to start crawling from
+ include_paths (Optional[List[str]]): Patterns of URLs to include
+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
+ max_depth (Optional[int]): Maximum crawl depth
+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
+ limit (Optional[int]): Maximum pages to crawl
+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+ crawl_entire_domain (Optional[bool]): Follow parent directory links
+ allow_external_links (Optional[bool]): Follow external domain links
+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
+ scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
+ webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs
+ ignore_query_parameters (Optional[bool]): Ignore URL parameters
+ regex_on_full_url (Optional[bool]): Apply regex to full URLs
+ delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes
+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1CrawlWatcher: An instance to monitor the crawl job via WebSocket
+
+ Raises:
+ Exception: If crawl job fails to start
+ """
+ crawl_response = self.async_crawl_url(
+ url,
+ include_paths=include_paths,
+ exclude_paths=exclude_paths,
+ max_depth=max_depth,
+ max_discovery_depth=max_discovery_depth,
+ limit=limit,
+ allow_backward_links=allow_backward_links,
+ crawl_entire_domain=crawl_entire_domain,
+ allow_external_links=allow_external_links,
+ ignore_sitemap=ignore_sitemap,
+ scrape_options=scrape_options,
+ webhook=webhook,
+ deduplicate_similar_urls=deduplicate_similar_urls,
+ ignore_query_parameters=ignore_query_parameters,
+ regex_on_full_url=regex_on_full_url,
+ delay=delay,
+ allow_subdomains=allow_subdomains,
+ max_concurrency=max_concurrency,
+ zero_data_retention=zero_data_retention,
+ idempotency_key=idempotency_key,
+ **kwargs
+ )
+ if crawl_response.success and crawl_response.id:
+ return V1CrawlWatcher(crawl_response.id, self)
+ else:
+ raise Exception("Crawl job failed to start")
+
+ def map_url(
+ self,
+ url: str,
+ *,
+ search: Optional[str] = None,
+ ignore_sitemap: Optional[bool] = None,
+ include_subdomains: Optional[bool] = None,
+ sitemap_only: Optional[bool] = None,
+ limit: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ use_index: Optional[bool] = None,
+ **kwargs) -> V1MapResponse:
+ """
+ Map and discover links from a URL.
+
+ Args:
+ url (str): Target URL to map
+ search (Optional[str]): Filter pattern for URLs
+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
+ include_subdomains (Optional[bool]): Include subdomain links
+ sitemap_only (Optional[bool]): Only use sitemap.xml
+ limit (Optional[int]): Maximum URLs to return
+ timeout (Optional[int]): Request timeout in milliseconds
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1MapResponse: Response containing:
+ * success (bool): Whether request succeeded
+ * links (List[str]): Discovered URLs
+ * error (Optional[str]): Error message if any
+
+ Raises:
+ Exception: If mapping fails or response cannot be parsed
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "map_url")
+
+ # Build map parameters
+ map_params = {}
+
+ # Add individual parameters
+ if search is not None:
+ map_params['search'] = search
+ if ignore_sitemap is not None:
+ map_params['ignoreSitemap'] = ignore_sitemap
+ if include_subdomains is not None:
+ map_params['includeSubdomains'] = include_subdomains
+ if sitemap_only is not None:
+ map_params['sitemapOnly'] = sitemap_only
+ if limit is not None:
+ map_params['limit'] = limit
+ if timeout is not None:
+ map_params['timeout'] = timeout
+ if use_index is not None:
+ map_params['useIndex'] = use_index
+
+ # Add any additional kwargs
+ map_params.update(kwargs)
+ _integration = map_params.get('integration')
+
+ # Create final params object
+ final_params = V1MapParams(**map_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['url'] = url
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if _integration:
+ params_dict['integration'] = _integration
+
+ # Make request
+ response = requests.post(
+ f"{self.api_url}/v1/map",
+ headers={"Authorization": f"Bearer {self.api_key}"},
+ json=params_dict
+ )
+
+ if response.status_code == 200:
+ try:
+ response_json = response.json()
+ if response_json.get('success') and 'links' in response_json:
+ return V1MapResponse(**response_json)
+ elif "error" in response_json:
+ raise Exception(f'Map failed. Error: {response_json["error"]}')
+ else:
+ raise Exception(f'Map failed. Error: {response_json}')
+ except ValueError:
+ raise Exception('Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'map')
+
+ def batch_scrape_urls(
+ self,
+ urls: List[str],
+ *,
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
+ mobile: Optional[bool] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+ extract: Optional[V1JsonConfig] = None,
+ json_options: Optional[V1JsonConfig] = None,
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,
+ agent: Optional[V1AgentOptions] = None,
+ poll_interval: Optional[int] = 2,
+ max_concurrency: Optional[int] = None,
+ zero_data_retention: Optional[bool] = None,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> V1BatchScrapeStatusResponse:
+ """
+ Batch scrape multiple URLs and monitor until completion.
+
+ Args:
+ urls (List[str]): URLs to scrape
+ formats (Optional[List[Literal]]): Content formats to retrieve
+ headers (Optional[Dict[str, str]]): Custom HTTP headers
+ include_tags (Optional[List[str]]): HTML tags to include
+ exclude_tags (Optional[List[str]]): HTML tags to exclude
+ only_main_content (Optional[bool]): Extract main content only
+ wait_for (Optional[int]): Wait time in milliseconds
+ timeout (Optional[int]): Request timeout in milliseconds
+ location (Optional[LocationConfig]): Location configuration
+ mobile (Optional[bool]): Use mobile user agent
+ skip_tls_verification (Optional[bool]): Skip TLS verification
+ remove_base64_images (Optional[bool]): Remove base64 encoded images
+ block_ads (Optional[bool]): Block advertisements
+ proxy (Optional[Literal]): Proxy type to use
+ extract (Optional[JsonConfig]): Content extraction config
+ json_options (Optional[JsonConfig]): JSON extraction config
+ actions (Optional[List[Union]]): Actions to perform
+ agent (Optional[AgentOptions]): Agent configuration
+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes
+ poll_interval (Optional[int]): Seconds between status checks (default: 2)
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1BatchScrapeStatusResponse with:
+ * Scraping status and progress
+ * Scraped content for each URL
+ * Success/error information
+
+ Raises:
+ Exception: If batch scrape fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "batch_scrape_urls")
+
+ scrape_params = {}
+
+ # Add individual parameters
+ if formats is not None:
+ scrape_params['formats'] = formats
+ if headers is not None:
+ scrape_params['headers'] = headers
+ if include_tags is not None:
+ scrape_params['includeTags'] = include_tags
+ if exclude_tags is not None:
+ scrape_params['excludeTags'] = exclude_tags
+ if only_main_content is not None:
+ scrape_params['onlyMainContent'] = only_main_content
+ if wait_for is not None:
+ scrape_params['waitFor'] = wait_for
+ if timeout is not None:
+ scrape_params['timeout'] = timeout
+ if location is not None:
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
+ if mobile is not None:
+ scrape_params['mobile'] = mobile
+ if skip_tls_verification is not None:
+ scrape_params['skipTlsVerification'] = skip_tls_verification
+ if remove_base64_images is not None:
+ scrape_params['removeBase64Images'] = remove_base64_images
+ if block_ads is not None:
+ scrape_params['blockAds'] = block_ads
+ if proxy is not None:
+ scrape_params['proxy'] = proxy
+ if extract is not None:
+ extract = self._ensure_schema_dict(extract)
+ if isinstance(extract, dict) and "schema" in extract:
+ extract["schema"] = self._ensure_schema_dict(extract["schema"])
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
+ if json_options is not None:
+ json_options = self._ensure_schema_dict(json_options)
+ if isinstance(json_options, dict) and "schema" in json_options:
+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions:
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if agent is not None:
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
+ if max_concurrency is not None:
+ scrape_params['maxConcurrency'] = max_concurrency
+ if zero_data_retention is not None:
+ scrape_params['zeroDataRetention'] = zero_data_retention
+
+ # Add any additional kwargs
+ scrape_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1ScrapeParams(**scrape_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['urls'] = urls
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:
+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)
+
+ if response.status_code == 200:
+ try:
+ id = response.json().get('id')
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ return self._monitor_job_status(id, headers, poll_interval)
+ else:
+ self._handle_error(response, 'start batch scrape job')
+
+ def async_batch_scrape_urls(
+ self,
+ urls: List[str],
+ *,
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
+ mobile: Optional[bool] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+ extract: Optional[V1JsonConfig] = None,
+ json_options: Optional[V1JsonConfig] = None,
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,
+ agent: Optional[V1AgentOptions] = None,
+ max_concurrency: Optional[int] = None,
+ idempotency_key: Optional[str] = None,
+ zero_data_retention: Optional[bool] = None,
+ **kwargs
+ ) -> V1BatchScrapeResponse:
+ """
+ Initiate a batch scrape job asynchronously.
+
+ Args:
+ urls (List[str]): URLs to scrape
+ formats (Optional[List[Literal]]): Content formats to retrieve
+ headers (Optional[Dict[str, str]]): Custom HTTP headers
+ include_tags (Optional[List[str]]): HTML tags to include
+ exclude_tags (Optional[List[str]]): HTML tags to exclude
+ only_main_content (Optional[bool]): Extract main content only
+ wait_for (Optional[int]): Wait time in milliseconds
+ timeout (Optional[int]): Request timeout in milliseconds
+ location (Optional[LocationConfig]): Location configuration
+ mobile (Optional[bool]): Use mobile user agent
+ skip_tls_verification (Optional[bool]): Skip TLS verification
+ remove_base64_images (Optional[bool]): Remove base64 encoded images
+ block_ads (Optional[bool]): Block advertisements
+ proxy (Optional[Literal]): Proxy type to use
+ extract (Optional[JsonConfig]): Content extraction config
+ json_options (Optional[JsonConfig]): JSON extraction config
+ actions (Optional[List[Union]]): Actions to perform
+ agent (Optional[AgentOptions]): Agent configuration
+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes
+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1BatchScrapeResponse with:
+ * success - Whether job started successfully
+ * id - Unique identifier for the job
+ * url - Status check URL
+ * error - Error message if start failed
+
+ Raises:
+ Exception: If job initiation fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "async_batch_scrape_urls")
+
+ scrape_params = {}
+
+ # Add individual parameters
+ if formats is not None:
+ scrape_params['formats'] = formats
+ if headers is not None:
+ scrape_params['headers'] = headers
+ if include_tags is not None:
+ scrape_params['includeTags'] = include_tags
+ if exclude_tags is not None:
+ scrape_params['excludeTags'] = exclude_tags
+ if only_main_content is not None:
+ scrape_params['onlyMainContent'] = only_main_content
+ if wait_for is not None:
+ scrape_params['waitFor'] = wait_for
+ if timeout is not None:
+ scrape_params['timeout'] = timeout
+ if location is not None:
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
+ if mobile is not None:
+ scrape_params['mobile'] = mobile
+ if skip_tls_verification is not None:
+ scrape_params['skipTlsVerification'] = skip_tls_verification
+ if remove_base64_images is not None:
+ scrape_params['removeBase64Images'] = remove_base64_images
+ if block_ads is not None:
+ scrape_params['blockAds'] = block_ads
+ if proxy is not None:
+ scrape_params['proxy'] = proxy
+ if extract is not None:
+ extract = self._ensure_schema_dict(extract)
+ if isinstance(extract, dict) and "schema" in extract:
+ extract["schema"] = self._ensure_schema_dict(extract["schema"])
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
+ if json_options is not None:
+ json_options = self._ensure_schema_dict(json_options)
+ if isinstance(json_options, dict) and "schema" in json_options:
+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions:
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if agent is not None:
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
+ if max_concurrency is not None:
+ scrape_params['maxConcurrency'] = max_concurrency
+ if zero_data_retention is not None:
+ scrape_params['zeroDataRetention'] = zero_data_retention
+
+ # Add any additional kwargs
+ scrape_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1ScrapeParams(**scrape_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['urls'] = urls
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:
+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)
+
+ if response.status_code == 200:
+ try:
+ return V1BatchScrapeResponse(**response.json())
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'start batch scrape job')
+
+ def batch_scrape_urls_and_watch(
+ self,
+ urls: List[str],
+ *,
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
+ mobile: Optional[bool] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+ extract: Optional[V1JsonConfig] = None,
+ json_options: Optional[V1JsonConfig] = None,
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,
+ agent: Optional[V1AgentOptions] = None,
+ max_concurrency: Optional[int] = None,
+ zero_data_retention: Optional[bool] = None,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> 'V1CrawlWatcher':
+ """
+ Initiate a batch scrape job and return a CrawlWatcher to monitor the job via WebSocket.
+
+ Args:
+ urls (List[str]): URLs to scrape
+ formats (Optional[List[Literal]]): Content formats to retrieve
+ headers (Optional[Dict[str, str]]): Custom HTTP headers
+ include_tags (Optional[List[str]]): HTML tags to include
+ exclude_tags (Optional[List[str]]): HTML tags to exclude
+ only_main_content (Optional[bool]): Extract main content only
+ wait_for (Optional[int]): Wait time in milliseconds
+ timeout (Optional[int]): Request timeout in milliseconds
+ location (Optional[LocationConfig]): Location configuration
+ mobile (Optional[bool]): Use mobile user agent
+ skip_tls_verification (Optional[bool]): Skip TLS verification
+ remove_base64_images (Optional[bool]): Remove base64 encoded images
+ block_ads (Optional[bool]): Block advertisements
+ proxy (Optional[Literal]): Proxy type to use
+ extract (Optional[JsonConfig]): Content extraction config
+ json_options (Optional[JsonConfig]): JSON extraction config
+ actions (Optional[List[Union]]): Actions to perform
+ agent (Optional[AgentOptions]): Agent configuration
+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes
+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket
+
+ Raises:
+ Exception: If batch scrape job fails to start
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "batch_scrape_urls_and_watch")
+
+ scrape_params = {}
+
+ # Add individual parameters
+ if formats is not None:
+ scrape_params['formats'] = formats
+ if headers is not None:
+ scrape_params['headers'] = headers
+ if include_tags is not None:
+ scrape_params['includeTags'] = include_tags
+ if exclude_tags is not None:
+ scrape_params['excludeTags'] = exclude_tags
+ if only_main_content is not None:
+ scrape_params['onlyMainContent'] = only_main_content
+ if wait_for is not None:
+ scrape_params['waitFor'] = wait_for
+ if timeout is not None:
+ scrape_params['timeout'] = timeout
+ if location is not None:
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
+ if mobile is not None:
+ scrape_params['mobile'] = mobile
+ if skip_tls_verification is not None:
+ scrape_params['skipTlsVerification'] = skip_tls_verification
+ if remove_base64_images is not None:
+ scrape_params['removeBase64Images'] = remove_base64_images
+ if block_ads is not None:
+ scrape_params['blockAds'] = block_ads
+ if proxy is not None:
+ scrape_params['proxy'] = proxy
+ if extract is not None:
+ extract = self._ensure_schema_dict(extract)
+ if isinstance(extract, dict) and "schema" in extract:
+ extract["schema"] = self._ensure_schema_dict(extract["schema"])
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
+ if json_options is not None:
+ json_options = self._ensure_schema_dict(json_options)
+ if isinstance(json_options, dict) and "schema" in json_options:
+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions:
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if agent is not None:
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
+ if max_concurrency is not None:
+ scrape_params['maxConcurrency'] = max_concurrency
+ if zero_data_retention is not None:
+ scrape_params['zeroDataRetention'] = zero_data_retention
+
+ # Add any additional kwargs
+ scrape_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1ScrapeParams(**scrape_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['urls'] = urls
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:
+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)
+
+ if response.status_code == 200:
+ try:
+ crawl_response = V1BatchScrapeResponse(**response.json())
+ if crawl_response.success and crawl_response.id:
+ return V1CrawlWatcher(crawl_response.id, self)
+ else:
+ raise Exception("Batch scrape job failed to start")
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'start batch scrape job')
+
+ def check_batch_scrape_status(self, id: str) -> V1BatchScrapeStatusResponse:
+ """
+ Check the status of a batch scrape job using the Firecrawl API.
+
+ Args:
+ id (str): The ID of the batch scrape job.
+
+ Returns:
+ V1BatchScrapeStatusResponse: The status of the batch scrape job.
+
+ Raises:
+ Exception: If the status check request fails.
+ """
+ endpoint = f'/v1/batch/scrape/{id}'
+
+ headers = self._prepare_headers()
+ response = self._get_request(f'{self.api_url}{endpoint}', headers)
+ if response.status_code == 200:
+ try:
+ status_data = response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ if status_data['status'] == 'completed':
+ if 'data' in status_data:
+ data = status_data['data']
+ while 'next' in status_data:
+ if len(status_data['data']) == 0:
+ break
+ next_url = status_data.get('next')
+ if not next_url:
+ logger.warning("Expected 'next' URL is missing.")
+ break
+ try:
+ status_response = self._get_request(next_url, headers)
+ if status_response.status_code != 200:
+ logger.error(f"Failed to fetch next page: {status_response.status_code}")
+ break
+ try:
+ next_data = status_response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ data.extend(next_data.get('data', []))
+ status_data = next_data
+ except Exception as e:
+ logger.error(f"Error during pagination request: {e}")
+ break
+ status_data['data'] = data
+
+ return V1BatchScrapeStatusResponse(**{
+ 'success': False if 'error' in status_data else True,
+ 'status': status_data.get('status'),
+ 'total': status_data.get('total'),
+ 'completed': status_data.get('completed'),
+ 'creditsUsed': status_data.get('creditsUsed'),
+ 'expiresAt': status_data.get('expiresAt'),
+ 'data': status_data.get('data'),
+ 'next': status_data.get('next'),
+ 'error': status_data.get('error')
+ })
+ else:
+ self._handle_error(response, 'check batch scrape status')
+
+ def check_batch_scrape_errors(self, id: str) -> V1CrawlErrorsResponse:
+ """
+ Returns information about batch scrape errors.
+
+ Args:
+ id (str): The ID of the crawl job.
+
+ Returns:
+ V1CrawlErrorsResponse containing:
+ * errors (List[Dict[str, str]]): List of errors with fields:
+ * id (str): Error ID
+ * timestamp (str): When the error occurred
+ * url (str): URL that caused the error
+ * error (str): Error message
+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt
+
+ Raises:
+ Exception: If the error check request fails
+ """
+ headers = self._prepare_headers()
+ response = self._get_request(f'{self.api_url}/v1/batch/scrape/{id}/errors', headers)
+ if response.status_code == 200:
+ try:
+ return V1CrawlErrorsResponse(**response.json())
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, "check batch scrape errors")
+
+ def extract(
+ self,
+ urls: Optional[List[str]] = None,
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Any] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = False,
+ enable_web_search: Optional[bool] = False,
+ show_sources: Optional[bool] = False,
+ agent: Optional[Dict[str, Any]] = None,
+ **kwargs) -> V1ExtractResponse[Any]:
+ """
+ Extract structured information from URLs.
+
+ Args:
+ urls (Optional[List[str]]): URLs to extract from
+ prompt (Optional[str]): Custom extraction prompt
+ schema (Optional[Any]): JSON schema/Pydantic model
+ system_prompt (Optional[str]): System context
+ allow_external_links (Optional[bool]): Follow external links
+ enable_web_search (Optional[bool]): Enable web search
+ show_sources (Optional[bool]): Include source URLs
+ agent (Optional[Dict[str, Any]]): Agent configuration
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1ExtractResponse[Any] with:
+ * success (bool): Whether request succeeded
+ * data (Optional[Any]): Extracted data matching schema
+ * error (Optional[str]): Error message if any
+
+ Raises:
+ ValueError: If prompt/schema missing or extraction fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "extract")
+
+ headers = self._prepare_headers()
+
+ if not prompt and not schema:
+ raise ValueError("Either prompt or schema is required")
+
+ if not urls and not prompt:
+ raise ValueError("Either urls or prompt is required")
+
+ if schema:
+ schema = self._ensure_schema_dict(schema)
+
+ request_data = {
+ 'urls': urls or [],
+ 'allowExternalLinks': allow_external_links,
+ 'enableWebSearch': enable_web_search,
+ 'showSources': show_sources,
+ 'schema': schema,
+ 'origin': f'python-sdk@{get_version()}'
+ }
+
+ # Only add prompt and systemPrompt if they exist
+ if prompt:
+ request_data['prompt'] = prompt
+ if system_prompt:
+ request_data['systemPrompt'] = system_prompt
+
+ if agent:
+ request_data['agent'] = agent
+
+ # Add any additional kwargs
+ request_data.update(kwargs)
+
+ try:
+ # Send the initial extract request
+ response = self._post_request(
+ f'{self.api_url}/v1/extract',
+ request_data,
+ headers
+ )
+ if response.status_code == 200:
+ try:
+ data = response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ if data['success']:
+ job_id = data.get('id')
+ if not job_id:
+ raise Exception('Job ID not returned from extract request.')
+
+ # Poll for the extract status
+ while True:
+ status_response = self._get_request(
+ f'{self.api_url}/v1/extract/{job_id}',
+ headers
+ )
+ if status_response.status_code == 200:
+ try:
+ status_data = status_response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ if status_data['status'] == 'completed':
+ return V1ExtractResponse(**status_data)
+ elif status_data['status'] in ['failed', 'cancelled']:
+ raise Exception(f'Extract job {status_data["status"]}. Error: {status_data["error"]}')
+ else:
+ self._handle_error(status_response, "extract-status")
+
+ time.sleep(2) # Polling interval
+ else:
+ raise Exception(f'Failed to extract. Error: {data["error"]}')
+ else:
+ self._handle_error(response, "extract")
+ except Exception as e:
+ raise ValueError(str(e), 500)
+
+ return V1ExtractResponse(success=False, error="Internal server error.")
+
+ def get_extract_status(self, job_id: str) -> V1ExtractResponse[Any]:
+ """
+ Retrieve the status of an extract job.
+
+ Args:
+ job_id (str): The ID of the extract job.
+
+ Returns:
+ ExtractResponse[Any]: The status of the extract job.
+
+ Raises:
+ ValueError: If there is an error retrieving the status.
+ """
+ headers = self._prepare_headers()
+ try:
+ response = self._get_request(f'{self.api_url}/v1/extract/{job_id}', headers)
+ if response.status_code == 200:
+ try:
+ return V1ExtractResponse(**response.json())
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, "get extract status")
+ except Exception as e:
+ raise ValueError(str(e), 500)
+
+ def async_extract(
+ self,
+ urls: Optional[List[str]] = None,
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Any] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = False,
+ enable_web_search: Optional[bool] = False,
+ show_sources: Optional[bool] = False,
+ agent: Optional[Dict[str, Any]] = None) -> V1ExtractResponse[Any]:
+ """
+ Initiate an asynchronous extract job.
+
+ Args:
+ urls (List[str]): URLs to extract information from
+ prompt (Optional[str]): Custom extraction prompt
+ schema (Optional[Any]): JSON schema/Pydantic model
+ system_prompt (Optional[str]): System context
+ allow_external_links (Optional[bool]): Follow external links
+ enable_web_search (Optional[bool]): Enable web search
+ show_sources (Optional[bool]): Include source URLs
+ agent (Optional[Dict[str, Any]]): Agent configuration
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+
+ Returns:
+ ExtractResponse[Any] with:
+ * success (bool): Whether request succeeded
+ * data (Optional[Any]): Extracted data matching schema
+ * error (Optional[str]): Error message if any
+
+ Raises:
+ ValueError: If job initiation fails
+ """
+ headers = self._prepare_headers()
+
+ schema = schema
+ if schema:
+ schema = self._ensure_schema_dict(schema)
+
+ request_data = {
+ 'urls': urls,
+ 'allowExternalLinks': allow_external_links,
+ 'enableWebSearch': enable_web_search,
+ 'showSources': show_sources,
+ 'schema': schema,
+ 'origin': f'python-sdk@{version}'
+ }
+
+ if prompt:
+ request_data['prompt'] = prompt
+ if system_prompt:
+ request_data['systemPrompt'] = system_prompt
+ if agent:
+ request_data['agent'] = agent
+
+ try:
+ response = self._post_request(f'{self.api_url}/v1/extract', request_data, headers)
+ if response.status_code == 200:
+ try:
+ return V1ExtractResponse(**response.json())
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, "async extract")
+ except Exception as e:
+ raise ValueError(str(e), 500)
+
+ def generate_llms_text(
+ self,
+ url: str,
+ *,
+ max_urls: Optional[int] = None,
+ show_full_text: Optional[bool] = None,
+ cache: Optional[bool] = None,
+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextStatusResponse:
+ """
+ Generate LLMs.txt for a given URL and poll until completion.
+
+ Args:
+ url (str): Target URL to generate LLMs.txt from
+ max_urls (Optional[int]): Maximum URLs to process (default: 10)
+ show_full_text (Optional[bool]): Include full text in output (default: False)
+ cache (Optional[bool]): Whether to use cached content if available (default: True)
+ experimental_stream (Optional[bool]): Enable experimental streaming
+
+ Returns:
+ GenerateLLMsTextStatusResponse with:
+ * Generated LLMs.txt content
+ * Full version if requested
+ * Generation status
+ * Success/error information
+
+ Raises:
+ Exception: If generation fails
+ """
+ params = V1GenerateLLMsTextParams(
+ maxUrls=max_urls,
+ showFullText=show_full_text,
+ cache=cache,
+ __experimental_stream=experimental_stream
+ )
+
+ response = self.async_generate_llms_text(
+ url,
+ max_urls=max_urls,
+ show_full_text=show_full_text,
+ cache=cache,
+ experimental_stream=experimental_stream
+ )
+
+ if not response.success or not response.id:
+ return V1GenerateLLMsTextStatusResponse(
+ success=False,
+ error='Failed to start LLMs.txt generation',
+ status='failed',
+ expiresAt=''
+ )
+
+ job_id = response.id
+ while True:
+ status = self.check_generate_llms_text_status(job_id)
+
+ if status.status == 'completed':
+ return status
+ elif status.status == 'failed':
+ return status
+ elif status.status != 'processing':
+ return V1GenerateLLMsTextStatusResponse(
+ success=False,
+ error='LLMs.txt generation job terminated unexpectedly',
+ status='failed',
+ expiresAt=''
+ )
+
+ time.sleep(2) # Polling interval
+
+ def async_generate_llms_text(
+ self,
+ url: str,
+ *,
+ max_urls: Optional[int] = None,
+ show_full_text: Optional[bool] = None,
+ cache: Optional[bool] = None,
+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextResponse:
+ """
+ Initiate an asynchronous LLMs.txt generation operation.
+
+ Args:
+ url (str): The target URL to generate LLMs.txt from. Must be a valid HTTP/HTTPS URL.
+ max_urls (Optional[int]): Maximum URLs to process (default: 10)
+ show_full_text (Optional[bool]): Include full text in output (default: False)
+ cache (Optional[bool]): Whether to use cached content if available (default: True)
+ experimental_stream (Optional[bool]): Enable experimental streaming
+
+ Returns:
+ GenerateLLMsTextResponse: A response containing:
+ * success (bool): Whether the generation initiation was successful
+ * id (str): The unique identifier for the generation job
+ * error (str, optional): Error message if initiation failed
+
+ Raises:
+ Exception: If the generation job initiation fails.
+ """
+ params = V1GenerateLLMsTextParams(
+ maxUrls=max_urls,
+ showFullText=show_full_text,
+ cache=cache,
+ __experimental_stream=experimental_stream
+ )
+
+ headers = self._prepare_headers()
+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
+ json_data['origin'] = f"python-sdk@{version}"
+
+ try:
+ req = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)
+ response = req.json()
+ print("json_data", json_data)
+ print("response", response)
+ if response.get('success'):
+ try:
+ return V1GenerateLLMsTextResponse(**response)
+ except:
+ raise Exception('Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'start LLMs.txt generation')
+ except Exception as e:
+ raise ValueError(str(e))
+
+ return V1GenerateLLMsTextResponse(
+ success=False,
+ error='Internal server error'
+ )
+
+ def check_generate_llms_text_status(self, id: str) -> V1GenerateLLMsTextStatusResponse:
+ """
+ Check the status of a LLMs.txt generation operation.
+
+ Args:
+ id (str): The unique identifier of the LLMs.txt generation job to check status for.
+
+ Returns:
+ GenerateLLMsTextStatusResponse: A response containing:
+ * success (bool): Whether the generation was successful
+ * status (str): Status of generation ("processing", "completed", "failed")
+ * data (Dict[str, str], optional): Generated text with fields:
+ * llmstxt (str): Generated LLMs.txt content
+ * llmsfulltxt (str, optional): Full version if requested
+ * error (str, optional): Error message if generation failed
+ * expiresAt (str): When the generated data expires
+
+ Raises:
+ Exception: If the status check fails.
+ """
+ headers = self._prepare_headers()
+ try:
+ response = self._get_request(f'{self.api_url}/v1/llmstxt/{id}', headers)
+ if response.status_code == 200:
+ try:
+ json_data = response.json()
+ return V1GenerateLLMsTextStatusResponse(**json_data)
+ except Exception as e:
+ raise Exception(f'Failed to parse Firecrawl response as GenerateLLMsTextStatusResponse: {str(e)}')
+ elif response.status_code == 404:
+ raise Exception('LLMs.txt generation job not found')
+ else:
+ self._handle_error(response, 'check LLMs.txt generation status')
+ except Exception as e:
+ raise ValueError(str(e))
+
+ return V1GenerateLLMsTextStatusResponse(success=False, error='Internal server error', status='failed', expiresAt='')
+
+ def _prepare_headers(
+ self,
+ idempotency_key: Optional[str] = None) -> Dict[str, str]:
+ """
+ Prepare the headers for API requests.
+
+ Args:
+ idempotency_key (Optional[str]): A unique key to ensure idempotency of requests.
+
+ Returns:
+ Dict[str, str]: The headers including content type, authorization, and optionally idempotency key.
+ """
+ if idempotency_key:
+ return {
+ 'Content-Type': 'application/json',
+ 'Authorization': f'Bearer {self.api_key}',
+ 'x-idempotency-key': idempotency_key
+ }
+
+ return {
+ 'Content-Type': 'application/json',
+ 'Authorization': f'Bearer {self.api_key}',
+ }
+
+ def _post_request(
+ self,
+ url: str,
+ data: Dict[str, Any],
+ headers: Dict[str, str],
+ retries: int = 3,
+ backoff_factor: float = 0.5) -> requests.Response:
+ """
+ Make a POST request with retries.
+
+ Args:
+ url (str): The URL to send the POST request to.
+ data (Dict[str, Any]): The JSON data to include in the POST request.
+ headers (Dict[str, str]): The headers to include in the POST request.
+ retries (int): Number of retries for the request.
+ backoff_factor (float): Backoff factor for retries.
+
+ Returns:
+ requests.Response: The response from the POST request.
+
+ Raises:
+ requests.RequestException: If the request fails after the specified retries.
+ """
+ for attempt in range(retries):
+ response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] / 1000.0 + 5) if "timeout" in data and data["timeout"] is not None else None))
+ if response.status_code == 502:
+ time.sleep(backoff_factor * (2 ** attempt))
+ else:
+ return response
+ return response
+
+ def _get_request(
+ self,
+ url: str,
+ headers: Dict[str, str],
+ retries: int = 3,
+ backoff_factor: float = 0.5) -> requests.Response:
+ """
+ Make a GET request with retries.
+
+ Args:
+ url (str): The URL to send the GET request to.
+ headers (Dict[str, str]): The headers to include in the GET request.
+ retries (int): Number of retries for the request.
+ backoff_factor (float): Backoff factor for retries.
+
+ Returns:
+ requests.Response: The response from the GET request.
+
+ Raises:
+ requests.RequestException: If the request fails after the specified retries.
+ """
+ for attempt in range(retries):
+ response = requests.get(url, headers=headers)
+ if response.status_code == 502:
+ time.sleep(backoff_factor * (2 ** attempt))
+ else:
+ return response
+ return response
+
+ def _delete_request(
+ self,
+ url: str,
+ headers: Dict[str, str],
+ retries: int = 3,
+ backoff_factor: float = 0.5) -> requests.Response:
+ """
+ Make a DELETE request with retries.
+
+ Args:
+ url (str): The URL to send the DELETE request to.
+ headers (Dict[str, str]): The headers to include in the DELETE request.
+ retries (int): Number of retries for the request.
+ backoff_factor (float): Backoff factor for retries.
+
+ Returns:
+ requests.Response: The response from the DELETE request.
+
+ Raises:
+ requests.RequestException: If the request fails after the specified retries.
+ """
+ for attempt in range(retries):
+ response = requests.delete(url, headers=headers)
+ if response.status_code == 502:
+ time.sleep(backoff_factor * (2 ** attempt))
+ else:
+ return response
+ return response
+
+ def _monitor_job_status(
+ self,
+ id: str,
+ headers: Dict[str, str],
+ poll_interval: int) -> V1CrawlStatusResponse:
+ """
+ Monitor the status of a crawl job until completion.
+
+ Args:
+ id (str): The ID of the crawl job.
+ headers (Dict[str, str]): The headers to include in the status check requests.
+ poll_interval (int): Seconds between status checks.
+
+ Returns:
+ CrawlStatusResponse: The crawl results if the job is completed successfully.
+
+ Raises:
+ Exception: If the job fails or an error occurs during status checks.
+ """
+ while True:
+ api_url = f'{self.api_url}/v1/crawl/{id}'
+
+ status_response = self._get_request(api_url, headers)
+ if status_response.status_code == 200:
+ try:
+ status_data = status_response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ if status_data['status'] == 'completed':
+ if 'data' in status_data:
+ data = status_data['data']
+ while 'next' in status_data:
+ if len(status_data['data']) == 0:
+ break
+ status_response = self._get_request(status_data['next'], headers)
+ try:
+ status_data = status_response.json()
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ data.extend(status_data.get('data', []))
+ status_data['data'] = data
+ return V1CrawlStatusResponse(**status_data)
+ else:
+ raise Exception('Crawl job completed but no data was returned')
+ elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:
+ poll_interval=max(poll_interval,2)
+ time.sleep(poll_interval) # Wait for the specified interval before checking again
+ else:
+ raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}')
+ else:
+ self._handle_error(status_response, 'check crawl status')
+
+ def _handle_error(
+ self,
+ response: requests.Response,
+ action: str) -> None:
+ """
+ Handle errors from API responses.
+
+ Args:
+ response (requests.Response): The response object from the API request.
+ action (str): Description of the action that was being performed.
+
+ Raises:
+ Exception: An exception with a message containing the status code and error details from the response.
+ """
+ try:
+ response_json = response.json()
+ error_message = response_json.get('error', 'No error message provided.')
+ error_details = response_json.get('details', 'No additional error details provided.')
+ except:
+ # If we can't parse JSON, provide a helpful error message with response content
+ try:
+ response_text = response.text[:500] # Limit to first 500 chars
+ if response_text.strip():
+ error_message = f"Server returned non-JSON response: {response_text}"
+ error_details = f"Full response status: {response.status_code}"
+ else:
+ error_message = f"Server returned empty response with status {response.status_code}"
+ error_details = "No additional details available"
+ except ValueError:
+ error_message = f"Server returned unreadable response with status {response.status_code}"
+ error_details = "No additional details available"
+
+ message = self._get_error_message(response.status_code, action, error_message, error_details)
+
+ # Raise an HTTPError with the custom message and attach the response
+ raise requests.exceptions.HTTPError(message, response=response)
+
+ def _get_error_message(self, status_code: int, action: str, error_message: str, error_details: str) -> str:
+ """
+ Generate a standardized error message based on HTTP status code.
+
+ Args:
+ status_code (int): The HTTP status code from the response
+ action (str): Description of the action that was being performed
+ error_message (str): The error message from the API response
+ error_details (str): Additional error details from the API response
+
+ Returns:
+ str: A formatted error message
+ """
+ if status_code == 402:
+ return f"Payment Required: Failed to {action}. {error_message} - {error_details}"
+ elif status_code == 403:
+ return f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+ elif status_code == 408:
+ return f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
+ elif status_code == 409:
+ return f"Conflict: Failed to {action} due to a conflict. {error_message} - {error_details}"
+ elif status_code == 500:
+ return f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
+ else:
+ return f"Unexpected error during {action}: Status code {status_code}. {error_message} - {error_details}"
+
+ def deep_research(
+ self,
+ query: str,
+ *,
+ max_depth: Optional[int] = None,
+ time_limit: Optional[int] = None,
+ max_urls: Optional[int] = None,
+ analysis_prompt: Optional[str] = None,
+ system_prompt: Optional[str] = None,
+ __experimental_stream_steps: Optional[bool] = None,
+ on_activity: Optional[Callable[[Dict[str, Any]], None]] = None,
+ on_source: Optional[Callable[[Dict[str, Any]], None]] = None) -> V1DeepResearchStatusResponse:
+ """
+ Initiates a deep research operation on a given query and polls until completion.
+
+ Args:
+ query (str): Research query or topic to investigate
+ max_depth (Optional[int]): Maximum depth of research exploration
+ time_limit (Optional[int]): Time limit in seconds for research
+ max_urls (Optional[int]): Maximum number of URLs to process
+ analysis_prompt (Optional[str]): Custom prompt for analysis
+ system_prompt (Optional[str]): Custom system prompt
+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming
+ on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth}
+ on_source (Optional[Callable]): Source discovery callback receiving {url, title, description}
+
+ Returns:
+ DeepResearchStatusResponse containing:
+ * success (bool): Whether research completed successfully
+ * status (str): Current state (processing/completed/failed)
+ * error (Optional[str]): Error message if failed
+ * id (str): Unique identifier for the research job
+ * data (Any): Research findings and analysis
+ * sources (List[Dict]): List of discovered sources
+ * activities (List[Dict]): Research progress log
+ * summaries (List[str]): Generated research summaries
+
+ Raises:
+ Exception: If research fails
+ """
+ research_params = {}
+ if max_depth is not None:
+ research_params['maxDepth'] = max_depth
+ if time_limit is not None:
+ research_params['timeLimit'] = time_limit
+ if max_urls is not None:
+ research_params['maxUrls'] = max_urls
+ if analysis_prompt is not None:
+ research_params['analysisPrompt'] = analysis_prompt
+ if system_prompt is not None:
+ research_params['systemPrompt'] = system_prompt
+ if __experimental_stream_steps is not None:
+ research_params['__experimental_streamSteps'] = __experimental_stream_steps
+ research_params = V1DeepResearchParams(**research_params)
+
+ response = self.async_deep_research(
+ query,
+ max_depth=max_depth,
+ time_limit=time_limit,
+ max_urls=max_urls,
+ analysis_prompt=analysis_prompt,
+ system_prompt=system_prompt
+ )
+ if not response.get('success') or 'id' not in response:
+ return response
+
+ job_id = response['id']
+ last_activity_count = 0
+ last_source_count = 0
+
+ while True:
+ status = self.check_deep_research_status(job_id)
+
+ if on_activity and 'activities' in status:
+ new_activities = status['activities'][last_activity_count:]
+ for activity in new_activities:
+ on_activity(activity)
+ last_activity_count = len(status['activities'])
+
+ if on_source and 'sources' in status:
+ new_sources = status['sources'][last_source_count:]
+ for source in new_sources:
+ on_source(source)
+ last_source_count = len(status['sources'])
+
+ if status['status'] == 'completed':
+ return status
+ elif status['status'] == 'failed':
+ raise Exception(f'Deep research failed. Error: {status.get("error")}')
+ elif status['status'] != 'processing':
+ break
+
+ time.sleep(2) # Polling interval
+
+ return {'success': False, 'error': 'Deep research job terminated unexpectedly'}
+
+ def async_deep_research(
+ self,
+ query: str,
+ *,
+ max_depth: Optional[int] = None,
+ time_limit: Optional[int] = None,
+ max_urls: Optional[int] = None,
+ analysis_prompt: Optional[str] = None,
+ system_prompt: Optional[str] = None,
+ __experimental_stream_steps: Optional[bool] = None) -> Dict[str, Any]:
+ """
+ Initiates an asynchronous deep research operation.
+
+ Args:
+ query (str): Research query or topic to investigate
+ max_depth (Optional[int]): Maximum depth of research exploration
+ time_limit (Optional[int]): Time limit in seconds for research
+ max_urls (Optional[int]): Maximum number of URLs to process
+ analysis_prompt (Optional[str]): Custom prompt for analysis
+ system_prompt (Optional[str]): Custom system prompt
+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming
+
+ Returns:
+ Dict[str, Any]: A response containing:
+ * success (bool): Whether the research initiation was successful
+ * id (str): The unique identifier for the research job
+ * error (str, optional): Error message if initiation failed
+
+ Raises:
+ Exception: If the research initiation fails.
+ """
+ research_params = {}
+ if max_depth is not None:
+ research_params['maxDepth'] = max_depth
+ if time_limit is not None:
+ research_params['timeLimit'] = time_limit
+ if max_urls is not None:
+ research_params['maxUrls'] = max_urls
+ if analysis_prompt is not None:
+ research_params['analysisPrompt'] = analysis_prompt
+ if system_prompt is not None:
+ research_params['systemPrompt'] = system_prompt
+ if __experimental_stream_steps is not None:
+ research_params['__experimental_streamSteps'] = __experimental_stream_steps
+ research_params = V1DeepResearchParams(**research_params)
+
+ headers = self._prepare_headers()
+
+ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
+ json_data['origin'] = f"python-sdk@{version}"
+
+ # Handle json options schema if present
+ if 'jsonOptions' in json_data:
+ json_opts = json_data['jsonOptions']
+ if json_opts and 'schema' in json_opts and hasattr(json_opts['schema'], 'schema'):
+ json_data['jsonOptions']['schema'] = json_opts['schema'].schema()
+
+ try:
+ response = self._post_request(f'{self.api_url}/v1/deep-research', json_data, headers)
+ if response.status_code == 200:
+ try:
+ return response.json()
+ except:
+ raise Exception('Failed to parse Firecrawl response as JSON.')
+ else:
+ self._handle_error(response, 'start deep research')
+ except Exception as e:
+ raise ValueError(str(e))
+
+ return {'success': False, 'error': 'Internal server error'}
+
+ def check_deep_research_status(self, id: str) -> V1DeepResearchStatusResponse:
+ """
+ Check the status of a deep research operation.
+
+ Args:
+ id (str): The ID of the deep research operation.
+
+ Returns:
+ DeepResearchResponse containing:
+
+ Status:
+ * success - Whether research completed successfully
+ * status - Current state (processing/completed/failed)
+ * error - Error message if failed
+
+ Results:
+ * id - Unique identifier for the research job
+ * data - Research findings and analysis
+ * sources - List of discovered sources
+ * activities - Research progress log
+ * summaries - Generated research summaries
+
+ Raises:
+ Exception: If the status check fails.
+ """
+ headers = self._prepare_headers()
+ try:
+ response = self._get_request(f'{self.api_url}/v1/deep-research/{id}', headers)
+ if response.status_code == 200:
+ try:
+ return response.json()
+ except:
+ raise Exception('Failed to parse Firecrawl response as JSON.')
+ elif response.status_code == 404:
+ raise Exception('Deep research job not found')
+ else:
+ self._handle_error(response, 'check deep research status')
+ except Exception as e:
+ raise ValueError(str(e))
+
+ return {'success': False, 'error': 'Internal server error'}
+
+ def _validate_kwargs(self, kwargs: Dict[str, Any], method_name: str) -> None:
+ """
+ Validate additional keyword arguments before they are passed to the API.
+ This provides early validation before the Pydantic model validation.
+
+ Args:
+ kwargs (Dict[str, Any]): Additional keyword arguments to validate
+ method_name (str): Name of the method these kwargs are for
+
+ Raises:
+ ValueError: If kwargs contain invalid or unsupported parameters
+ """
+ if not kwargs:
+ return
+
+ # Known parameter mappings for each method
+ method_params = {
+ "scrape_url": {"formats", "include_tags", "exclude_tags", "only_main_content", "wait_for",
+ "timeout", "location", "mobile", "skip_tls_verification", "remove_base64_images",
+ "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "max_age", "integration"},
+ "search": {"limit", "tbs", "filter", "lang", "country", "location", "timeout", "scrape_options", "integration"},
+ "crawl_url": {"include_paths", "exclude_paths", "max_depth", "max_discovery_depth", "limit",
+ "allow_backward_links", "allow_external_links", "ignore_sitemap", "scrape_options",
+ "webhook", "deduplicate_similar_urls", "ignore_query_parameters", "regex_on_full_url", "integration"},
+ "map_url": {"search", "ignore_sitemap", "include_subdomains", "sitemap_only", "limit", "timeout", "integration"},
+ "extract": {"prompt", "schema", "system_prompt", "allow_external_links", "enable_web_search", "show_sources", "agent", "integration"},
+ "batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+ "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+ "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+ "actions", "agent", "webhook"},
+ "async_batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+ "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+ "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+ "actions", "agent", "webhook"},
+ "batch_scrape_urls_and_watch": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",
+ "wait_for", "timeout", "location", "mobile", "skip_tls_verification",
+ "remove_base64_images", "block_ads", "proxy", "extract", "json_options",
+ "actions", "agent", "webhook"}
+ }
+
+ # Get allowed parameters for this method
+ allowed_params = method_params.get(method_name, set())
+
+ # Check for unknown parameters
+ unknown_params = set(kwargs.keys()) - allowed_params
+ if unknown_params:
+ raise ValueError(f"Unsupported parameter(s) for {method_name}: {', '.join(unknown_params)}. Please refer to the API documentation for the correct parameters.")
+
+ # Additional type validation can be added here if needed
+ # For now, we rely on Pydantic models for detailed type validation
+
+ def _ensure_schema_dict(self, schema):
+ """
+ Utility to ensure a schema is a dict, not a Pydantic model class. Recursively checks dicts and lists.
+ """
+ if schema is None:
+ return schema
+ if isinstance(schema, type):
+ # Pydantic v1/v2 model class
+ if hasattr(schema, 'model_json_schema'):
+ return schema.model_json_schema()
+ elif hasattr(schema, 'schema'):
+ return schema.schema()
+ if isinstance(schema, dict):
+ return {k: self._ensure_schema_dict(v) for k, v in schema.items()}
+ if isinstance(schema, (list, tuple)):
+ return [self._ensure_schema_dict(v) for v in schema]
+ return schema
+
+class V1CrawlWatcher:
+ """
+ A class to watch and handle crawl job events via WebSocket connection.
+
+ Attributes:
+ id (str): The ID of the crawl job to watch
+ app (V1FirecrawlApp): The V1FirecrawlApp instance
+ data (List[Dict[str, Any]]): List of crawled documents/data
+ status (str): Current status of the crawl job
+ ws_url (str): WebSocket URL for the crawl job
+ event_handlers (dict): Dictionary of event type to list of handler functions
+ """
+ def __init__(self, id: str, app: V1FirecrawlApp):
+ self.id = id
+ self.app = app
+ self.data: List[Dict[str, Any]] = []
+ self.status = "scraping"
+ self.ws_url = f"{app.api_url.replace('http', 'ws')}/v1/crawl/{id}"
+ self.event_handlers = {
+ 'done': [],
+ 'error': [],
+ 'document': []
+ }
+
+ async def connect(self) -> None:
+ """
+ Establishes WebSocket connection and starts listening for messages.
+ """
+ async with websockets.connect(
+ self.ws_url,
+ max_size=None,
+ additional_headers=[("Authorization", f"Bearer {self.app.api_key}")]
+ ) as websocket:
+ await self._listen(websocket)
+
+ async def _listen(self, websocket) -> None:
+ """
+ Listens for incoming WebSocket messages and handles them.
+
+ Args:
+ websocket: The WebSocket connection object
+ """
+ async for message in websocket:
+ msg = json.loads(message)
+ await self._handle_message(msg)
+
+ def add_event_listener(self, event_type: str, handler: Callable[[Dict[str, Any]], None]) -> None:
+ """
+ Adds an event handler function for a specific event type.
+
+ Args:
+ event_type (str): Type of event to listen for ('done', 'error', or 'document')
+ handler (Callable): Function to handle the event
+ """
+ if event_type in self.event_handlers:
+ self.event_handlers[event_type].append(handler)
+
+ def dispatch_event(self, event_type: str, detail: Dict[str, Any]) -> None:
+ """
+ Dispatches an event to all registered handlers for that event type.
+
+ Args:
+ event_type (str): Type of event to dispatch
+ detail (Dict[str, Any]): Event details/data to pass to handlers
+ """
+ if event_type in self.event_handlers:
+ for handler in self.event_handlers[event_type]:
+ handler(detail)
+
+ async def _handle_message(self, msg: Dict[str, Any]) -> None:
+ """
+ Handles incoming WebSocket messages based on their type.
+
+ Args:
+ msg (Dict[str, Any]): The message to handle
+ """
+ if msg['type'] == 'done':
+ self.status = 'completed'
+ self.dispatch_event('done', {'status': self.status, 'data': self.data, 'id': self.id})
+ elif msg['type'] == 'error':
+ self.status = 'failed'
+ self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error'], 'id': self.id})
+ elif msg['type'] == 'catchup':
+ self.status = msg['data']['status']
+ self.data.extend(msg['data'].get('data', []))
+ for doc in self.data:
+ self.dispatch_event('document', {'data': doc, 'id': self.id})
+ elif msg['type'] == 'document':
+ self.data.append(msg['data'])
+ self.dispatch_event('document', {'data': msg['data'], 'id': self.id})
+
+class AsyncV1FirecrawlApp:
+ """
+ Asynchronous version of V1FirecrawlApp that implements async methods using aiohttp.
+ Provides non-blocking alternatives to all V1FirecrawlApp operations.
+ """
+
+ def __init__(self, api_key: str, api_url: str = "https://api.firecrawl.com"):
+ self.api_key = api_key
+ self.api_url = api_url
+
+ async def _async_request(
+ self,
+ method: str,
+ url: str,
+ headers: Dict[str, str],
+ data: Optional[Dict[str, Any]] = None,
+ retries: int = 3,
+ backoff_factor: float = 0.5) -> Dict[str, Any]:
+ """
+ Generic async request method with exponential backoff retry logic.
+
+ Args:
+ method (str): The HTTP method to use (e.g., "GET" or "POST").
+ url (str): The URL to send the request to.
+ headers (Dict[str, str]): Headers to include in the request.
+ data (Optional[Dict[str, Any]]): The JSON data to include in the request body (only for POST requests).
+ retries (int): Maximum number of retry attempts (default: 3).
+ backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
+ Delay will be backoff_factor * (2 ** retry_count).
+
+ Returns:
+ Dict[str, Any]: The parsed JSON response from the server.
+
+ Raises:
+ aiohttp.ClientError: If the request fails after all retries.
+ Exception: If max retries are exceeded or other errors occur.
+ """
+ async with aiohttp.ClientSession() as session:
+ for attempt in range(retries):
+ try:
+ async with session.request(
+ method=method, url=url, headers=headers, json=data
+ ) as response:
+ if response.status == 502:
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+ continue
+ if response.status >= 300:
+ await self._handle_error(response, f"make {method} request")
+ return await response.json()
+ except aiohttp.ClientError as e:
+ if attempt == retries - 1:
+ raise e
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+ raise Exception("Max retries exceeded")
+
+ async def _async_post_request(
+ self, url: str, data: Dict[str, Any], headers: Dict[str, str],
+ retries: int = 3, backoff_factor: float = 0.5) -> Dict[str, Any]:
+ """
+ Make an async POST request with exponential backoff retry logic.
+
+ Args:
+ url (str): The URL to send the POST request to.
+ data (Dict[str, Any]): The JSON data to include in the request body.
+ headers (Dict[str, str]): Headers to include in the request.
+ retries (int): Maximum number of retry attempts (default: 3).
+ backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
+ Delay will be backoff_factor * (2 ** retry_count).
+
+ Returns:
+ Dict[str, Any]: The parsed JSON response from the server.
+
+ Raises:
+ aiohttp.ClientError: If the request fails after all retries.
+ Exception: If max retries are exceeded or other errors occur.
+ """
+ return await self._async_request("POST", url, headers, data, retries, backoff_factor)
+
+ async def _async_get_request(
+ self, url: str, headers: Dict[str, str],
+ retries: int = 3, backoff_factor: float = 0.5) -> Dict[str, Any]:
+ """
+ Make an async GET request with exponential backoff retry logic.
+
+ Args:
+ url (str): The URL to send the GET request to.
+ headers (Dict[str, str]): Headers to include in the request.
+ retries (int): Maximum number of retry attempts (default: 3).
+ backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
+ Delay will be backoff_factor * (2 ** retry_count).
+
+ Returns:
+ Dict[str, Any]: The parsed JSON response from the server.
+
+ Raises:
+ aiohttp.ClientError: If the request fails after all retries.
+ Exception: If max retries are exceeded or other errors occur.
+ """
+ return await self._async_request("GET", url, headers, None, retries, backoff_factor)
+
+ async def _handle_error(self, response: aiohttp.ClientResponse, action: str) -> None:
+ """
+ Handle errors from async API responses with detailed error messages.
+
+ Args:
+ response (aiohttp.ClientResponse): The response object from the failed request
+ action (str): Description of the action that was being attempted
+
+ Raises:
+ aiohttp.ClientError: With a detailed error message based on the response status:
+ - 402: Payment Required
+ - 408: Request Timeout
+ - 409: Conflict
+ - 500: Internal Server Error
+ - Other: Unexpected error with status code
+ """
+ try:
+ error_data = await response.json()
+ error_message = error_data.get('error', 'No error message provided.')
+ error_details = error_data.get('details', 'No additional error details provided.')
+ except:
+ raise aiohttp.ClientError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status}')
+
+ message = await self._get_async_error_message(response.status, action, error_message, error_details)
+
+ raise aiohttp.ClientError(message)
+
+ async def _get_async_error_message(self, status_code: int, action: str, error_message: str, error_details: str) -> str:
+ """
+ Generate a standardized error message based on HTTP status code for async operations.
+
+ Args:
+ status_code (int): The HTTP status code from the response
+ action (str): Description of the action that was being performed
+ error_message (str): The error message from the API response
+ error_details (str): Additional error details from the API response
+
+ Returns:
+ str: A formatted error message
+ """
+ return self._get_error_message(status_code, action, error_message, error_details)
+
+ async def crawl_url_and_watch(
+ self,
+ url: str,
+ params: Optional[V1CrawlParams] = None,
+ idempotency_key: Optional[str] = None) -> 'AsyncV1CrawlWatcher':
+ """
+ Initiate an async crawl job and return an AsyncV1CrawlWatcher to monitor progress via WebSocket.
+
+ Args:
+ url (str): Target URL to start crawling from
+ params (Optional[V1CrawlParams]): See V1CrawlParams model for configuration:
+ URL Discovery:
+ * includePaths - Patterns of URLs to include
+ * excludePaths - Patterns of URLs to exclude
+ * maxDepth - Maximum crawl depth
+ * maxDiscoveryDepth - Maximum depth for finding new URLs
+ * limit - Maximum pages to crawl
+
+ Link Following:
+ * allowBackwardLinks - DEPRECATED: Use crawlEntireDomain instead
+ * crawlEntireDomain - Follow parent directory links
+ * allowExternalLinks - Follow external domain links
+ * ignoreSitemap - Skip sitemap.xml processing
+
+ Advanced:
+ * scrapeOptions - Page scraping configuration
+ * webhook - Notification webhook settings
+ * deduplicateSimilarURLs - Remove similar URLs
+ * ignoreQueryParameters - Ignore URL parameters
+ * regexOnFullURL - Apply regex to full URLs
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+
+ Returns:
+ AsyncV1CrawlWatcher: An instance to monitor the crawl job via WebSocket
+
+ Raises:
+ Exception: If crawl job fails to start
+ """
+ crawl_response = await self.async_crawl_url(url, params, idempotency_key)
+ if crawl_response.get('success') and 'id' in crawl_response:
+ return AsyncV1CrawlWatcher(crawl_response['id'], self)
+ else:
+ raise Exception("Crawl job failed to start")
+
+ async def batch_scrape_urls_and_watch(
+ self,
+ urls: List[str],
+ params: Optional[V1ScrapeParams] = None,
+ idempotency_key: Optional[str] = None) -> 'AsyncV1CrawlWatcher':
+ """
+ Initiate an async batch scrape job and return an AsyncV1CrawlWatcher to monitor progress.
+
+ Args:
+ urls (List[str]): List of URLs to scrape
+ params (Optional[V1ScrapeParams]): See V1ScrapeParams model for configuration:
+
+ Content Options:
+ * formats - Content formats to retrieve
+ * includeTags - HTML tags to include
+ * excludeTags - HTML tags to exclude
+ * onlyMainContent - Extract main content only
+
+ Request Options:
+ * headers - Custom HTTP headers
+ * timeout - Request timeout (ms)
+ * mobile - Use mobile user agent
+ * proxy - Proxy type
+
+ Extraction Options:
+ * extract - Content extraction config
+ * jsonOptions - JSON extraction config
+ * actions - Actions to perform
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+
+ Returns:
+ AsyncV1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket
+
+ Raises:
+ Exception: If batch scrape job fails to start
+ """
+ batch_response = await self.async_batch_scrape_urls(urls, params, idempotency_key)
+ if batch_response.get('success') and 'id' in batch_response:
+ return AsyncV1CrawlWatcher(batch_response['id'], self)
+ else:
+ raise Exception("Batch scrape job failed to start")
+
+ async def scrape_url(
+ self,
+ url: str,
+ *,
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
+ mobile: Optional[bool] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+ parse_pdf: Optional[bool] = None,
+ extract: Optional[V1JsonConfig] = None,
+ json_options: Optional[V1JsonConfig] = None,
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,
+ **kwargs) -> V1ScrapeResponse[Any]:
+ """
+ Scrape a single URL asynchronously.
+
+ Args:
+ url (str): Target URL to scrape
+ formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)
+ headers (Optional[Dict[str, str]]): Custom HTTP headers
+ include_tags (Optional[List[str]]): HTML tags to include
+ exclude_tags (Optional[List[str]]): HTML tags to exclude
+ only_main_content (Optional[bool]): Extract main content only
+ wait_for (Optional[int]): Wait for a specific element to appear
+ timeout (Optional[int]): Request timeout (ms)
+ location (Optional[V1LocationConfig]): Location configuration
+ mobile (Optional[bool]): Use mobile user agent
+ skip_tls_verification (Optional[bool]): Skip TLS verification
+ remove_base64_images (Optional[bool]): Remove base64 images
+ block_ads (Optional[bool]): Block ads
+ proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)
+ extract (Optional[V1JsonConfig]): Content extraction settings
+ json_options (Optional[V1JsonConfig]): JSON extraction settings
+ actions (Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]]): Actions to perform
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1ScrapeResponse with:
+ * success - Whether scrape was successful
+ * markdown - Markdown content if requested
+ * html - HTML content if requested
+ * rawHtml - Raw HTML content if requested
+ * links - Extracted links if requested
+ * screenshot - Screenshot if requested
+ * extract - Extracted data if requested
+ * json - JSON data if requested
+ * error - Error message if scrape failed
+
+ Raises:
+ Exception: If scraping fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "scrape_url")
+
+ _headers = self._prepare_headers()
+
+ # Build scrape parameters
+ scrape_params = {
+ 'url': url,
+ 'origin': f"python-sdk@{version}"
+ }
+
+ # Add optional parameters if provided and not None
+ if formats:
+ scrape_params['formats'] = formats
+ if headers:
+ scrape_params['headers'] = headers
+ if include_tags:
+ scrape_params['includeTags'] = include_tags
+ if exclude_tags:
+ scrape_params['excludeTags'] = exclude_tags
+ if only_main_content is not None:
+ scrape_params['onlyMainContent'] = only_main_content
+ if wait_for:
+ scrape_params['waitFor'] = wait_for
+ if timeout:
+ scrape_params['timeout'] = timeout
+ if location:
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
+ if mobile is not None:
+ scrape_params['mobile'] = mobile
+ if skip_tls_verification is not None:
+ scrape_params['skipTlsVerification'] = skip_tls_verification
+ if remove_base64_images is not None:
+ scrape_params['removeBase64Images'] = remove_base64_images
+ if block_ads is not None:
+ scrape_params['blockAds'] = block_ads
+ if proxy:
+ scrape_params['proxy'] = proxy
+ if parse_pdf is not None:
+ scrape_params['parsePDF'] = parse_pdf
+ if extract is not None:
+ extract = self._ensure_schema_dict(extract)
+ if isinstance(extract, dict) and "schema" in extract:
+ extract["schema"] = self._ensure_schema_dict(extract["schema"])
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
+ if json_options is not None:
+ json_options = self._ensure_schema_dict(json_options)
+ if isinstance(json_options, dict) and "schema" in json_options:
+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions:
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:
+ scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])
+ if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:
+ scrape_params['jsonOptions']['schema'] = self._ensure_schema_dict(scrape_params['jsonOptions']['schema'])
+
+ # Make async request
+ endpoint = f'/v1/scrape'
+ response = await self._async_post_request(
+ f'{self.api_url}{endpoint}',
+ scrape_params,
+ _headers
+ )
+
+ if response.get('success') and 'data' in response:
+ return V1ScrapeResponse(**response['data'])
+ elif "error" in response:
+ raise Exception(f'Failed to scrape URL. Error: {response["error"]}')
+ else:
+ # Use the response content directly if possible, otherwise a generic message
+ error_content = response.get('error', str(response))
+ raise Exception(f'Failed to scrape URL. Error: {error_content}')
+
+ async def batch_scrape_urls(
+ self,
+ urls: List[str],
+ *,
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
+ mobile: Optional[bool] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+ extract: Optional[V1JsonConfig] = None,
+ json_options: Optional[V1JsonConfig] = None,
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,
+ agent: Optional[V1AgentOptions] = None,
+ poll_interval: Optional[int] = 2,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> V1BatchScrapeStatusResponse:
+ """
+ Asynchronously scrape multiple URLs and monitor until completion.
+
+ Args:
+ urls (List[str]): URLs to scrape
+ formats (Optional[List[Literal]]): Content formats to retrieve
+ headers (Optional[Dict[str, str]]): Custom HTTP headers
+ include_tags (Optional[List[str]]): HTML tags to include
+ exclude_tags (Optional[List[str]]): HTML tags to exclude
+ only_main_content (Optional[bool]): Extract main content only
+ wait_for (Optional[int]): Wait time in milliseconds
+ timeout (Optional[int]): Request timeout in milliseconds
+ location (Optional[LocationConfig]): Location configuration
+ mobile (Optional[bool]): Use mobile user agent
+ skip_tls_verification (Optional[bool]): Skip TLS verification
+ remove_base64_images (Optional[bool]): Remove base64 encoded images
+ block_ads (Optional[bool]): Block advertisements
+ proxy (Optional[Literal]): Proxy type to use
+ extract (Optional[JsonConfig]): Content extraction config
+ json_options (Optional[JsonConfig]): JSON extraction config
+ actions (Optional[List[Union]]): Actions to perform
+ agent (Optional[AgentOptions]): Agent configuration
+ poll_interval (Optional[int]): Seconds between status checks (default: 2)
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1BatchScrapeStatusResponse with:
+ * Scraping status and progress
+ * Scraped content for each URL
+ * Success/error information
+
+ Raises:
+ Exception: If batch scrape fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "batch_scrape_urls")
+
+ scrape_params = {}
+
+ # Add individual parameters
+ if formats is not None:
+ scrape_params['formats'] = formats
+ if headers is not None:
+ scrape_params['headers'] = headers
+ if include_tags is not None:
+ scrape_params['includeTags'] = include_tags
+ if exclude_tags is not None:
+ scrape_params['excludeTags'] = exclude_tags
+ if only_main_content is not None:
+ scrape_params['onlyMainContent'] = only_main_content
+ if wait_for is not None:
+ scrape_params['waitFor'] = wait_for
+ if timeout is not None:
+ scrape_params['timeout'] = timeout
+ if location is not None:
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
+ if mobile is not None:
+ scrape_params['mobile'] = mobile
+ if skip_tls_verification is not None:
+ scrape_params['skipTlsVerification'] = skip_tls_verification
+ if remove_base64_images is not None:
+ scrape_params['removeBase64Images'] = remove_base64_images
+ if block_ads is not None:
+ scrape_params['blockAds'] = block_ads
+ if proxy is not None:
+ scrape_params['proxy'] = proxy
+ if extract is not None:
+ extract = self._ensure_schema_dict(extract)
+ if isinstance(extract, dict) and "schema" in extract:
+ extract["schema"] = self._ensure_schema_dict(extract["schema"])
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
+ if json_options is not None:
+ json_options = self._ensure_schema_dict(json_options)
+ if isinstance(json_options, dict) and "schema" in json_options:
+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions is not None:
+ scrape_params['actions'] = [action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if agent is not None:
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
+
+ # Add any additional kwargs
+ scrape_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1ScrapeParams(**scrape_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['urls'] = urls
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:
+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = await self._async_post_request(
+ f'{self.api_url}/v1/batch/scrape',
+ params_dict,
+ headers
+ )
+
+ if response.get('success'):
+ try:
+ id = response.get('id')
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ return await self._async_monitor_job_status(id, headers, poll_interval)
+ else:
+ self._handle_error(response, 'start batch scrape job')
+
+
+ async def async_batch_scrape_urls(
+ self,
+ urls: List[str],
+ *,
+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
+ mobile: Optional[bool] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,
+ extract: Optional[V1JsonConfig] = None,
+ json_options: Optional[V1JsonConfig] = None,
+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,
+ agent: Optional[V1AgentOptions] = None,
+ zero_data_retention: Optional[bool] = None,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> V1BatchScrapeResponse:
+ """
+ Initiate a batch scrape job asynchronously.
+
+ Args:
+ urls (List[str]): URLs to scrape
+ formats (Optional[List[Literal]]): Content formats to retrieve
+ headers (Optional[Dict[str, str]]): Custom HTTP headers
+ include_tags (Optional[List[str]]): HTML tags to include
+ exclude_tags (Optional[List[str]]): HTML tags to exclude
+ only_main_content (Optional[bool]): Extract main content only
+ wait_for (Optional[int]): Wait time in milliseconds
+ timeout (Optional[int]): Request timeout in milliseconds
+ location (Optional[LocationConfig]): Location configuration
+ mobile (Optional[bool]): Use mobile user agent
+ skip_tls_verification (Optional[bool]): Skip TLS verification
+ remove_base64_images (Optional[bool]): Remove base64 encoded images
+ block_ads (Optional[bool]): Block advertisements
+ proxy (Optional[Literal]): Proxy type to use
+ extract (Optional[JsonConfig]): Content extraction config
+ json_options (Optional[JsonConfig]): JSON extraction config
+ actions (Optional[List[Union]]): Actions to perform
+ agent (Optional[AgentOptions]): Agent configuration
+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1BatchScrapeResponse with:
+ * success - Whether job started successfully
+ * id - Unique identifier for the job
+ * url - Status check URL
+ * error - Error message if start failed
+
+ Raises:
+ Exception: If job initiation fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "async_batch_scrape_urls")
+
+ scrape_params = {}
+
+ # Add individual parameters
+ if formats is not None:
+ scrape_params['formats'] = formats
+ if headers is not None:
+ scrape_params['headers'] = headers
+ if include_tags is not None:
+ scrape_params['includeTags'] = include_tags
+ if exclude_tags is not None:
+ scrape_params['excludeTags'] = exclude_tags
+ if only_main_content is not None:
+ scrape_params['onlyMainContent'] = only_main_content
+ if wait_for is not None:
+ scrape_params['waitFor'] = wait_for
+ if timeout is not None:
+ scrape_params['timeout'] = timeout
+ if location is not None:
+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)
+ if mobile is not None:
+ scrape_params['mobile'] = mobile
+ if skip_tls_verification is not None:
+ scrape_params['skipTlsVerification'] = skip_tls_verification
+ if remove_base64_images is not None:
+ scrape_params['removeBase64Images'] = remove_base64_images
+ if block_ads is not None:
+ scrape_params['blockAds'] = block_ads
+ if proxy is not None:
+ scrape_params['proxy'] = proxy
+ if extract is not None:
+ extract = self._ensure_schema_dict(extract)
+ if isinstance(extract, dict) and "schema" in extract:
+ extract["schema"] = self._ensure_schema_dict(extract["schema"])
+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)
+ if json_options is not None:
+ json_options = self._ensure_schema_dict(json_options)
+ if isinstance(json_options, dict) and "schema" in json_options:
+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])
+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)
+ if actions:
+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]
+ if agent is not None:
+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)
+ if zero_data_retention is not None:
+ scrape_params['zeroDataRetention'] = zero_data_retention
+
+ # Add any additional kwargs
+ scrape_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1ScrapeParams(**scrape_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['urls'] = urls
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:
+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = await self._async_post_request(
+ f'{self.api_url}/v1/batch/scrape',
+ params_dict,
+ headers
+ )
+
+ if response.get('status_code') == 200:
+ try:
+ return V1BatchScrapeResponse(**response.json())
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ await self._handle_error(response, 'start batch scrape job')
+
+ async def crawl_url(
+ self,
+ url: str,
+ *,
+ include_paths: Optional[List[str]] = None,
+ exclude_paths: Optional[List[str]] = None,
+ max_depth: Optional[int] = None,
+ max_discovery_depth: Optional[int] = None,
+ limit: Optional[int] = None,
+ allow_backward_links: Optional[bool] = None,
+ crawl_entire_domain: Optional[bool] = None,
+ allow_external_links: Optional[bool] = None,
+ ignore_sitemap: Optional[bool] = None,
+ scrape_options: Optional[V1ScrapeOptions] = None,
+ webhook: Optional[Union[str, V1WebhookConfig]] = None,
+ deduplicate_similar_urls: Optional[bool] = None,
+ ignore_query_parameters: Optional[bool] = None,
+ regex_on_full_url: Optional[bool] = None,
+ delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
+ poll_interval: Optional[int] = 2,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> V1CrawlStatusResponse:
+ """
+ Crawl a website starting from a URL.
+
+ Args:
+ url (str): Target URL to start crawling from
+ include_paths (Optional[List[str]]): Patterns of URLs to include
+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
+ max_depth (Optional[int]): Maximum crawl depth
+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
+ limit (Optional[int]): Maximum pages to crawl
+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+ crawl_entire_domain (Optional[bool]): Follow parent directory links
+ allow_external_links (Optional[bool]): Follow external domain links
+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
+ scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
+ webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs
+ ignore_query_parameters (Optional[bool]): Ignore URL parameters
+ regex_on_full_url (Optional[bool]): Apply regex to full URLs
+ delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
+ poll_interval (Optional[int]): Seconds between status checks (default: 2)
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1CrawlStatusResponse with:
+ * Crawling status and progress
+ * Crawled page contents
+ * Success/error information
+
+ Raises:
+ Exception: If crawl fails
+ """
+ # Validate any additional kwargs
+ self._validate_kwargs(kwargs, "crawl_url")
+
+ crawl_params = {}
+
+ # Add individual parameters
+ if include_paths is not None:
+ crawl_params['includePaths'] = include_paths
+ if exclude_paths is not None:
+ crawl_params['excludePaths'] = exclude_paths
+ if max_depth is not None:
+ crawl_params['maxDepth'] = max_depth
+ if max_discovery_depth is not None:
+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth
+ if limit is not None:
+ crawl_params['limit'] = limit
+ if crawl_entire_domain is not None:
+ crawl_params['crawlEntireDomain'] = crawl_entire_domain
+ elif allow_backward_links is not None:
+ crawl_params['allowBackwardLinks'] = allow_backward_links
+ if allow_external_links is not None:
+ crawl_params['allowExternalLinks'] = allow_external_links
+ if ignore_sitemap is not None:
+ crawl_params['ignoreSitemap'] = ignore_sitemap
+ if scrape_options is not None:
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
+ if webhook is not None:
+ crawl_params['webhook'] = webhook
+ if deduplicate_similar_urls is not None:
+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls
+ if ignore_query_parameters is not None:
+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters
+ if regex_on_full_url is not None:
+ crawl_params['regexOnFullURL'] = regex_on_full_url
+ if delay is not None:
+ crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
+
+ # Add any additional kwargs
+ crawl_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1CrawlParams(**crawl_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['url'] = url
+ params_dict['origin'] = f"python-sdk@{version}"
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = await self._async_post_request(
+ f'{self.api_url}/v1/crawl', params_dict, headers)
+
+ if response.get('success'):
+ try:
+ id = response.get('id')
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ return await self._async_monitor_job_status(id, headers, poll_interval)
+ else:
+ await self._handle_error(response, 'start crawl job')
+
+
+ async def async_crawl_url(
+ self,
+ url: str,
+ *,
+ include_paths: Optional[List[str]] = None,
+ exclude_paths: Optional[List[str]] = None,
+ max_depth: Optional[int] = None,
+ max_discovery_depth: Optional[int] = None,
+ limit: Optional[int] = None,
+ allow_backward_links: Optional[bool] = None,
+ crawl_entire_domain: Optional[bool] = None,
+ allow_external_links: Optional[bool] = None,
+ ignore_sitemap: Optional[bool] = None,
+ scrape_options: Optional[V1ScrapeOptions] = None,
+ webhook: Optional[Union[str, V1WebhookConfig]] = None,
+ deduplicate_similar_urls: Optional[bool] = None,
+ ignore_query_parameters: Optional[bool] = None,
+ regex_on_full_url: Optional[bool] = None,
+ delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
+ poll_interval: Optional[int] = 2,
+ idempotency_key: Optional[str] = None,
+ **kwargs
+ ) -> V1CrawlResponse:
+ """
+ Start an asynchronous crawl job.
+
+ Args:
+ url (str): Target URL to start crawling from
+ include_paths (Optional[List[str]]): Patterns of URLs to include
+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
+ max_depth (Optional[int]): Maximum crawl depth
+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
+ limit (Optional[int]): Maximum pages to crawl
+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
+ crawl_entire_domain (Optional[bool]): Follow parent directory links
+ allow_external_links (Optional[bool]): Follow external domain links
+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
+ scrape_options (Optional[ScrapeOptions]): Page scraping configuration
+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs
+ ignore_query_parameters (Optional[bool]): Ignore URL parameters
+ regex_on_full_url (Optional[bool]): Apply regex to full URLs
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+ **kwargs: Additional parameters to pass to the API
+
+ Returns:
+ V1CrawlResponse with:
+ * success - Whether crawl started successfully
+ * id - Unique identifier for the crawl job
+ * url - Status check URL for the crawl
+ * error - Error message if start failed
+
+ Raises:
+ Exception: If crawl initiation fails
+ """
+ crawl_params = {}
+
+ # Add individual parameters
+ if include_paths is not None:
+ crawl_params['includePaths'] = include_paths
+ if exclude_paths is not None:
+ crawl_params['excludePaths'] = exclude_paths
+ if max_depth is not None:
+ crawl_params['maxDepth'] = max_depth
+ if max_discovery_depth is not None:
+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth
+ if limit is not None:
+ crawl_params['limit'] = limit
+ if crawl_entire_domain is not None:
+ crawl_params['crawlEntireDomain'] = crawl_entire_domain
+ elif allow_backward_links is not None:
+ crawl_params['allowBackwardLinks'] = allow_backward_links
+ if allow_external_links is not None:
+ crawl_params['allowExternalLinks'] = allow_external_links
+ if ignore_sitemap is not None:
+ crawl_params['ignoreSitemap'] = ignore_sitemap
+ if scrape_options is not None:
+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
+ if webhook is not None:
+ crawl_params['webhook'] = webhook
+ if deduplicate_similar_urls is not None:
+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls
+ if ignore_query_parameters is not None:
+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters
+ if regex_on_full_url is not None:
+ crawl_params['regexOnFullURL'] = regex_on_full_url
+ if delay is not None:
+ crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
+
+ # Add any additional kwargs
+ crawl_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1CrawlParams(**crawl_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['url'] = url
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ # Make request
+ headers = self._prepare_headers(idempotency_key)
+ response = await self._async_post_request(
+ f'{self.api_url}/v1/crawl',
+ params_dict,
+ headers
+ )
+
+ if response.get('success'):
+ try:
+ return V1CrawlResponse(**response)
+ except:
+ raise Exception(f'Failed to parse Firecrawl response as JSON.')
+ else:
+ await self._handle_error(response, 'start crawl job')
+
+ async def check_crawl_status(self, id: str) -> V1CrawlStatusResponse:
+ """
+ Check the status and results of an asynchronous crawl job.
+
+ Args:
+ id (str): Unique identifier for the crawl job
+
+ Returns:
+ V1CrawlStatusResponse containing:
+ Status Information:
+ * status - Current state (scraping/completed/failed/cancelled)
+ * completed - Number of pages crawled
+ * total - Total pages to crawl
+ * creditsUsed - API credits consumed
+ * expiresAt - Data expiration timestamp
+
+ Results:
+ * data - List of crawled documents
+ * next - URL for next page of results (if paginated)
+ * success - Whether status check succeeded
+ * error - Error message if failed
+
+ Raises:
+ Exception: If status check fails
+ """
+ headers = self._prepare_headers()
+ endpoint = f'/v1/crawl/{id}'
+
+ status_data = await self._async_get_request(
+ f'{self.api_url}{endpoint}',
+ headers
+ )
+
+ if status_data.get('status') == 'completed':
+ if 'data' in status_data:
+ data = status_data['data']
+ while 'next' in status_data:
+ if len(status_data['data']) == 0:
+ break
+ next_url = status_data.get('next')
+ if not next_url:
+ logger.warning("Expected 'next' URL is missing.")
+ break
+ next_data = await self._async_get_request(next_url, headers)
+ data.extend(next_data.get('data', []))
+ status_data = next_data
+ status_data['data'] = data
+ # Create V1CrawlStatusResponse object from status data
+ response = V1CrawlStatusResponse(
+ status=status_data.get('status'),
+ total=status_data.get('total'),
+ completed=status_data.get('completed'),
+ creditsUsed=status_data.get('creditsUsed'),
+ expiresAt=status_data.get('expiresAt'),
+ data=status_data.get('data'),
+ success=False if 'error' in status_data else True
+ )
+
+ if 'error' in status_data:
+ response.error = status_data.get('error')
+
+ if 'next' in status_data:
+ response.next = status_data.get('next')
+
+ return response
+
+ async def _async_monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int = 2) -> V1CrawlStatusResponse:
+ """
+ Monitor the status of an asynchronous job until completion.
+
+ Args:
+ id (str): The ID of the job to monitor
+ headers (Dict[str, str]): Headers to include in status check requests
+ poll_interval (int): Seconds between status checks (default: 2)
+
+ Returns:
+ V1CrawlStatusResponse: The job results if completed successfully
+
+ Raises:
+ Exception: If the job fails or an error occurs during status checks
+ """
+ while True:
+ status_data = await self._async_get_request(
+ f'{self.api_url}/v1/crawl/{id}',
+ headers
+ )
+
+ if status_data.get('status') == 'completed':
+ if 'data' in status_data:
+ data = status_data['data']
+ while 'next' in status_data:
+ if len(status_data['data']) == 0:
+ break
+ next_url = status_data.get('next')
+ if not next_url:
+ logger.warning("Expected 'next' URL is missing.")
+ break
+ next_data = await self._async_get_request(next_url, headers)
+ data.extend(next_data.get('data', []))
+ status_data = next_data
+ status_data['data'] = data
+ return V1CrawlStatusResponse(**status_data)
+ else:
+ raise Exception('Job completed but no data was returned')
+ elif status_data.get('status') in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:
+ await asyncio.sleep(max(poll_interval, 2))
+ else:
+ raise Exception(f'Job failed or was stopped. Status: {status_data["status"]}')
+
+ async def map_url(
+ self,
+ url: str,
+ *,
+ search: Optional[str] = None,
+ ignore_sitemap: Optional[bool] = None,
+ include_subdomains: Optional[bool] = None,
+ sitemap_only: Optional[bool] = None,
+ limit: Optional[int] = None,
+ timeout: Optional[int] = 30000,
+ params: Optional[V1MapParams] = None) -> V1MapResponse:
+ """
+ Asynchronously map and discover links from a URL.
+
+ Args:
+ url (str): Target URL to map
+ params (Optional[V1MapParams]): See V1MapParams model:
+ Discovery Options:
+ * search - Filter pattern for URLs
+ * ignoreSitemap - Skip sitemap.xml
+ * includeSubdomains - Include subdomain links
+ * sitemapOnly - Only use sitemap.xml
+
+ Limits:
+ * limit - Max URLs to return
+ * timeout - Request timeout (ms)
+
+ Returns:
+ V1MapResponse with:
+ * Discovered URLs
+ * Success/error status
+
+ Raises:
+ Exception: If mapping fails
+ """
+ map_params = {}
+ if params:
+ map_params.update(params.dict(by_alias=True, exclude_none=True))
+
+ # Add individual parameters
+ if search is not None:
+ map_params['search'] = search
+ if ignore_sitemap is not None:
+ map_params['ignoreSitemap'] = ignore_sitemap
+ if include_subdomains is not None:
+ map_params['includeSubdomains'] = include_subdomains
+ if sitemap_only is not None:
+ map_params['sitemapOnly'] = sitemap_only
+ if limit is not None:
+ map_params['limit'] = limit
+ if timeout is not None:
+ map_params['timeout'] = timeout
+
+ # Create final params object
+ final_params = V1MapParams(**map_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['url'] = url
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ # Make request
+ endpoint = f'/v1/map'
+ response = await self._async_post_request(
+ f'{self.api_url}{endpoint}',
+ params_dict,
+ headers={"Authorization": f"Bearer {self.api_key}"}
+ )
+
+ if response.get('success') and 'links' in response:
+ return V1MapResponse(**response)
+ elif 'error' in response:
+ raise Exception(f'Failed to map URL. Error: {response["error"]}')
+ else:
+ raise Exception(f'Failed to map URL. Error: {response}')
+
+ async def extract(
+ self,
+ urls: Optional[List[str]] = None,
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Any] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = False,
+ enable_web_search: Optional[bool] = False,
+ show_sources: Optional[bool] = False,
+ agent: Optional[Dict[str, Any]] = None) -> V1ExtractResponse[Any]:
+
+ """
+ Asynchronously extract structured information from URLs.
+
+ Args:
+ urls (Optional[List[str]]): URLs to extract from
+ prompt (Optional[str]): Custom extraction prompt
+ schema (Optional[Any]): JSON schema/Pydantic model
+ system_prompt (Optional[str]): System context
+ allow_external_links (Optional[bool]): Follow external links
+ enable_web_search (Optional[bool]): Enable web search
+ show_sources (Optional[bool]): Include source URLs
+ agent (Optional[Dict[str, Any]]): Agent configuration
+
+ Returns:
+ V1ExtractResponse with:
+ * Structured data matching schema
+ * Source information if requested
+ * Success/error status
+
+ Raises:
+ ValueError: If prompt/schema missing or extraction fails
+ """
+ headers = self._prepare_headers()
+
+ if not prompt and not schema:
+ raise ValueError("Either prompt or schema is required")
+
+ if not urls and not prompt:
+ raise ValueError("Either urls or prompt is required")
+
+ if schema:
+ schema = self._ensure_schema_dict(schema)
+
+ request_data = {
+ 'urls': urls or [],
+ 'allowExternalLinks': allow_external_links,
+ 'enableWebSearch': enable_web_search,
+ 'showSources': show_sources,
+ 'schema': schema,
+ 'origin': f'python-sdk@{get_version()}'
+ }
+
+ # Only add prompt and systemPrompt if they exist
+ if prompt:
+ request_data['prompt'] = prompt
+ if system_prompt:
+ request_data['systemPrompt'] = system_prompt
+
+ if agent:
+ request_data['agent'] = agent
+
+ response = await self._async_post_request(
+ f'{self.api_url}/v1/extract',
+ request_data,
+ headers
+ )
+
+ if response.get('success'):
+ job_id = response.get('id')
+ if not job_id:
+ raise Exception('Job ID not returned from extract request.')
+
+ while True:
+ status_data = await self._async_get_request(
+ f'{self.api_url}/v1/extract/{job_id}',
+ headers
+ )
+
+ if status_data['status'] == 'completed':
+ return V1ExtractResponse(**status_data)
+ elif status_data['status'] in ['failed', 'cancelled']:
+ raise Exception(f'Extract job {status_data["status"]}. Error: {status_data["error"]}')
+
+ await asyncio.sleep(2)
+ else:
+ raise Exception(f'Failed to extract. Error: {response.get("error")}')
+
+ async def check_batch_scrape_status(self, id: str) -> V1BatchScrapeStatusResponse:
+ """
+ Check the status of an asynchronous batch scrape job.
+
+ Args:
+ id (str): The ID of the batch scrape job
+
+ Returns:
+ V1BatchScrapeStatusResponse containing:
+ Status Information:
+ * status - Current state (scraping/completed/failed/cancelled)
+ * completed - Number of URLs scraped
+ * total - Total URLs to scrape
+ * creditsUsed - API credits consumed
+ * expiresAt - Data expiration timestamp
+
+ Results:
+ * data - List of scraped documents
+ * next - URL for next page of results (if paginated)
+ * success - Whether status check succeeded
+ * error - Error message if failed
+
+ Raises:
+ Exception: If status check fails
+ """
+ headers = self._prepare_headers()
+ endpoint = f'/v1/batch/scrape/{id}'
+
+ status_data = await self._async_get_request(
+ f'{self.api_url}{endpoint}',
+ headers
+ )
+
+ if status_data['status'] == 'completed':
+ if 'data' in status_data:
+ data = status_data['data']
+ while 'next' in status_data:
+ if len(status_data['data']) == 0:
+ break
+ next_url = status_data.get('next')
+ if not next_url:
+ logger.warning("Expected 'next' URL is missing.")
+ break
+ next_data = await self._async_get_request(next_url, headers)
+ data.extend(next_data.get('data', []))
+ status_data = next_data
+ status_data['data'] = data
+
+ response = V1BatchScrapeStatusResponse(
+ status=status_data.get('status'),
+ total=status_data.get('total'),
+ completed=status_data.get('completed'),
+ creditsUsed=status_data.get('creditsUsed'),
+ expiresAt=status_data.get('expiresAt'),
+ data=status_data.get('data')
+ )
+
+ if 'error' in status_data:
+ response['error'] = status_data['error']
+
+ if 'next' in status_data:
+ response['next'] = status_data['next']
+
+ return {
+ 'success': False if 'error' in status_data else True,
+ **response
+ }
+
+ async def check_batch_scrape_errors(self, id: str) -> V1CrawlErrorsResponse:
+ """
+ Get information about errors from an asynchronous batch scrape job.
+
+ Args:
+ id (str): The ID of the batch scrape job
+
+ Returns:
+ V1CrawlErrorsResponse containing:
+ errors (List[Dict[str, str]]): List of errors with fields:
+ * id (str): Error ID
+ * timestamp (str): When the error occurred
+ * url (str): URL that caused the error
+ * error (str): Error message
+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt
+
+ Raises:
+ Exception: If error check fails
+ """
+ headers = self._prepare_headers()
+ return await self._async_get_request(
+ f'{self.api_url}/v1/batch/scrape/{id}/errors',
+ headers
+ )
+
+ async def check_crawl_errors(self, id: str) -> V1CrawlErrorsResponse:
+ """
+ Get information about errors from an asynchronous crawl job.
+
+ Args:
+ id (str): The ID of the crawl job
+
+ Returns:
+ V1CrawlErrorsResponse containing:
+ * errors (List[Dict[str, str]]): List of errors with fields:
+ - id (str): Error ID
+ - timestamp (str): When the error occurred
+ - url (str): URL that caused the error
+ - error (str): Error message
+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt
+
+ Raises:
+ Exception: If error check fails
+ """
+ headers = self._prepare_headers()
+ return await self._async_get_request(
+ f'{self.api_url}/v1/crawl/{id}/errors',
+ headers
+ )
+
+ async def cancel_crawl(self, id: str) -> Dict[str, Any]:
+ """
+ Cancel an asynchronous crawl job.
+
+ Args:
+ id (str): The ID of the crawl job to cancel
+
+ Returns:
+ Dict[str, Any] containing:
+ * success (bool): Whether cancellation was successful
+ * error (str, optional): Error message if cancellation failed
+
+ Raises:
+ Exception: If cancellation fails
+ """
+ headers = self._prepare_headers()
+ async with aiohttp.ClientSession() as session:
+ async with session.delete(f'{self.api_url}/v1/crawl/{id}', headers=headers) as response:
+ return await response.json()
+
+ async def get_extract_status(self, job_id: str) -> V1ExtractResponse[Any]:
+ """
+ Check the status of an asynchronous extraction job.
+
+ Args:
+ job_id (str): The ID of the extraction job
+
+ Returns:
+ V1ExtractResponse[Any] with:
+ * success (bool): Whether request succeeded
+ * data (Optional[Any]): Extracted data matching schema
+ * error (Optional[str]): Error message if any
+ * warning (Optional[str]): Warning message if any
+ * sources (Optional[List[str]]): Source URLs if requested
+
+ Raises:
+ ValueError: If status check fails
+ """
+ headers = self._prepare_headers()
+ try:
+ return await self._async_get_request(
+ f'{self.api_url}/v1/extract/{job_id}',
+ headers
+ )
+ except Exception as e:
+ raise ValueError(str(e))
+
+ async def async_extract(
+ self,
+ urls: Optional[List[str]] = None,
+ *,
+ prompt: Optional[str] = None,
+ schema: Optional[Any] = None,
+ system_prompt: Optional[str] = None,
+ allow_external_links: Optional[bool] = False,
+ enable_web_search: Optional[bool] = False,
+ show_sources: Optional[bool] = False,
+ agent: Optional[Dict[str, Any]] = None) -> V1ExtractResponse[Any]:
+ """
+ Initiate an asynchronous extraction job without waiting for completion.
+
+ Args:
+ urls (Optional[List[str]]): URLs to extract from
+ prompt (Optional[str]): Custom extraction prompt
+ schema (Optional[Any]): JSON schema/Pydantic model
+ system_prompt (Optional[str]): System context
+ allow_external_links (Optional[bool]): Follow external links
+ enable_web_search (Optional[bool]): Enable web search
+ show_sources (Optional[bool]): Include source URLs
+ agent (Optional[Dict[str, Any]]): Agent configuration
+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests
+
+ Returns:
+ V1ExtractResponse[Any] with:
+ * success (bool): Whether request succeeded
+ * data (Optional[Any]): Extracted data matching schema
+ * error (Optional[str]): Error message if any
+
+ Raises:
+ ValueError: If job initiation fails
+ """
+ headers = self._prepare_headers()
+
+ if not prompt and not schema:
+ raise ValueError("Either prompt or schema is required")
+
+ if not urls and not prompt:
+ raise ValueError("Either urls or prompt is required")
+
+ if schema:
+ schema = self._ensure_schema_dict(schema)
+
+ request_data = V1ExtractResponse(
+ urls=urls or [],
+ allowExternalLinks=allow_external_links,
+ enableWebSearch=enable_web_search,
+ showSources=show_sources,
+ schema=schema,
+ origin=f'python-sdk@{version}'
+ )
+
+ if prompt:
+ request_data['prompt'] = prompt
+ if system_prompt:
+ request_data['systemPrompt'] = system_prompt
+ if agent:
+ request_data['agent'] = agent
+
+ try:
+ return await self._async_post_request(
+ f'{self.api_url}/v1/extract',
+ request_data,
+ headers
+ )
+ except Exception as e:
+ raise ValueError(str(e))
+
+ async def generate_llms_text(
+ self,
+ url: str,
+ *,
+ max_urls: Optional[int] = None,
+ show_full_text: Optional[bool] = None,
+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextStatusResponse:
+ """
+ Generate LLMs.txt for a given URL and monitor until completion.
+
+ Args:
+ url (str): Target URL to generate LLMs.txt from
+ max_urls (Optional[int]): Maximum URLs to process (default: 10)
+ show_full_text (Optional[bool]): Include full text in output (default: False)
+ experimental_stream (Optional[bool]): Enable experimental streaming
+
+ Returns:
+ V1GenerateLLMsTextStatusResponse containing:
+ * success (bool): Whether generation completed successfully
+ * status (str): Status of generation (processing/completed/failed)
+ * data (Dict[str, str], optional): Generated text with fields:
+ - llmstxt (str): Generated LLMs.txt content
+ - llmsfulltxt (str, optional): Full version if requested
+ * error (str, optional): Error message if generation failed
+ * expiresAt (str): When the generated data expires
+
+ Raises:
+ Exception: If generation fails
+ """
+ params = {}
+ if max_urls is not None:
+ params['maxUrls'] = max_urls
+ if show_full_text is not None:
+ params['showFullText'] = show_full_text
+ if experimental_stream is not None:
+ params['__experimental_stream'] = experimental_stream
+
+ response = await self.async_generate_llms_text(
+ url,
+ max_urls=max_urls,
+ show_full_text=show_full_text,
+ experimental_stream=experimental_stream
+ )
+ if not response.get('success') or 'id' not in response:
+ return response
+
+ job_id = response['id']
+ while True:
+ status = await self.check_generate_llms_text_status(job_id)
+
+ if status['status'] == 'completed':
+ return status
+ elif status['status'] == 'failed':
+ raise Exception(f'LLMs.txt generation failed. Error: {status.get("error")}')
+ elif status['status'] != 'processing':
+ break
+
+ await asyncio.sleep(2)
+
+ return V1GenerateLLMsTextStatusResponse(success=False, error='LLMs.txt generation job terminated unexpectedly', status='failed', expiresAt='')
+
+ async def async_generate_llms_text(
+ self,
+ url: str,
+ *,
+ max_urls: Optional[int] = None,
+ show_full_text: Optional[bool] = None,
+ cache: Optional[bool] = None,
+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextResponse:
+ """
+ Initiate an asynchronous LLMs.txt generation job without waiting for completion.
+
+ Args:
+ url (str): Target URL to generate LLMs.txt from
+ max_urls (Optional[int]): Maximum URLs to process (default: 10)
+ show_full_text (Optional[bool]): Include full text in output (default: False)
+ cache (Optional[bool]): Whether to use cached content if available (default: True)
+ experimental_stream (Optional[bool]): Enable experimental streaming
+
+ Returns:
+ V1GenerateLLMsTextResponse containing:
+ * success (bool): Whether job started successfully
+ * id (str): Unique identifier for the job
+ * error (str, optional): Error message if start failed
+
+ Raises:
+ ValueError: If job initiation fails
+ """
+ params = {}
+ if max_urls is not None:
+ params['maxUrls'] = max_urls
+ if show_full_text is not None:
+ params['showFullText'] = show_full_text
+ if experimental_stream is not None:
+ params['__experimental_stream'] = experimental_stream
+
+ params = V1GenerateLLMsTextParams(
+ maxUrls=max_urls,
+ showFullText=show_full_text,
+ cache=cache,
+ __experimental_stream=experimental_stream
+ )
+
+ headers = self._prepare_headers()
+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
+ json_data['origin'] = f"python-sdk@{version}"
+
+ try:
+ return await self._async_post_request(
+ f'{self.api_url}/v1/llmstxt',
+ json_data,
+ headers
+ )
+ except Exception as e:
+ raise ValueError(str(e))
+
+ async def check_generate_llms_text_status(self, id: str) -> V1GenerateLLMsTextStatusResponse:
+ """
+ Check the status of an asynchronous LLMs.txt generation job.
+
+ Args:
+ id (str): The ID of the generation job
+
+ Returns:
+ V1GenerateLLMsTextStatusResponse containing:
+ * success (bool): Whether generation completed successfully
+ * status (str): Status of generation (processing/completed/failed)
+ * data (Dict[str, str], optional): Generated text with fields:
+ - llmstxt (str): Generated LLMs.txt content
+ - llmsfulltxt (str, optional): Full version if requested
+ * error (str, optional): Error message if generation failed
+ * expiresAt (str): When the generated data expires
+
+ Raises:
+ ValueError: If status check fails
+ """
+ headers = self._prepare_headers()
+ try:
+ return await self._async_get_request(
+ f'{self.api_url}/v1/llmstxt/{id}',
+ headers
+ )
+ except Exception as e:
+ raise ValueError(str(e))
+
+ async def deep_research(
+ self,
+ query: str,
+ *,
+ max_depth: Optional[int] = None,
+ time_limit: Optional[int] = None,
+ max_urls: Optional[int] = None,
+ analysis_prompt: Optional[str] = None,
+ system_prompt: Optional[str] = None,
+ __experimental_stream_steps: Optional[bool] = None,
+ on_activity: Optional[Callable[[Dict[str, Any]], None]] = None,
+ on_source: Optional[Callable[[Dict[str, Any]], None]] = None) -> V1DeepResearchStatusResponse:
+ """
+ Initiates a deep research operation on a given query and polls until completion.
+
+ Args:
+ query (str): Research query or topic to investigate
+ max_depth (Optional[int]): Maximum depth of research exploration
+ time_limit (Optional[int]): Time limit in seconds for research
+ max_urls (Optional[int]): Maximum number of URLs to process
+ analysis_prompt (Optional[str]): Custom prompt for analysis
+ system_prompt (Optional[str]): Custom system prompt
+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming
+ on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth}
+ on_source (Optional[Callable]): Source discovery callback receiving {url, title, description}
+
+ Returns:
+ DeepResearchStatusResponse containing:
+ * success (bool): Whether research completed successfully
+ * status (str): Current state (processing/completed/failed)
+ * error (Optional[str]): Error message if failed
+ * id (str): Unique identifier for the research job
+ * data (Any): Research findings and analysis
+ * sources (List[Dict]): List of discovered sources
+ * activities (List[Dict]): Research progress log
+ * summaries (List[str]): Generated research summaries
+
+ Raises:
+ Exception: If research fails
+ """
+ research_params = {}
+ if max_depth is not None:
+ research_params['maxDepth'] = max_depth
+ if time_limit is not None:
+ research_params['timeLimit'] = time_limit
+ if max_urls is not None:
+ research_params['maxUrls'] = max_urls
+ if analysis_prompt is not None:
+ research_params['analysisPrompt'] = analysis_prompt
+ if system_prompt is not None:
+ research_params['systemPrompt'] = system_prompt
+ if __experimental_stream_steps is not None:
+ research_params['__experimental_streamSteps'] = __experimental_stream_steps
+ research_params = V1DeepResearchParams(**research_params)
+
+ response = await self.async_deep_research(
+ query,
+ max_depth=max_depth,
+ time_limit=time_limit,
+ max_urls=max_urls,
+ analysis_prompt=analysis_prompt,
+ system_prompt=system_prompt
+ )
+ if not response.get('success') or 'id' not in response:
+ return response
+
+ job_id = response['id']
+ last_activity_count = 0
+ last_source_count = 0
+
+ while True:
+ status = await self.check_deep_research_status(job_id)
+
+ if on_activity and 'activities' in status:
+ new_activities = status['activities'][last_activity_count:]
+ for activity in new_activities:
+ on_activity(activity)
+ last_activity_count = len(status['activities'])
+
+ if on_source and 'sources' in status:
+ new_sources = status['sources'][last_source_count:]
+ for source in new_sources:
+ on_source(source)
+ last_source_count = len(status['sources'])
+
+ if status['status'] == 'completed':
+ return status
+ elif status['status'] == 'failed':
+ raise Exception(f'Deep research failed. Error: {status.get("error")}')
+ elif status['status'] != 'processing':
+ break
+
+ await asyncio.sleep(2)
+
+ return V1DeepResearchStatusResponse(success=False, error='Deep research job terminated unexpectedly')
+
+ async def async_deep_research(
+ self,
+ query: str,
+ *,
+ max_depth: Optional[int] = None,
+ time_limit: Optional[int] = None,
+ max_urls: Optional[int] = None,
+ analysis_prompt: Optional[str] = None,
+ system_prompt: Optional[str] = None,
+ __experimental_stream_steps: Optional[bool] = None) -> Dict[str, Any]:
+ """
+ Initiates an asynchronous deep research operation.
+
+ Args:
+ query (str): Research query or topic to investigate
+ max_depth (Optional[int]): Maximum depth of research exploration
+ time_limit (Optional[int]): Time limit in seconds for research
+ max_urls (Optional[int]): Maximum number of URLs to process
+ analysis_prompt (Optional[str]): Custom prompt for analysis
+ system_prompt (Optional[str]): Custom system prompt
+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming
+
+ Returns:
+ Dict[str, Any]: A response containing:
+ * success (bool): Whether the research initiation was successful
+ * id (str): The unique identifier for the research job
+ * error (str, optional): Error message if initiation failed
+
+ Raises:
+ Exception: If the research initiation fails.
+ """
+ research_params = {}
+ if max_depth is not None:
+ research_params['maxDepth'] = max_depth
+ if time_limit is not None:
+ research_params['timeLimit'] = time_limit
+ if max_urls is not None:
+ research_params['maxUrls'] = max_urls
+ if analysis_prompt is not None:
+ research_params['analysisPrompt'] = analysis_prompt
+ if system_prompt is not None:
+ research_params['systemPrompt'] = system_prompt
+ if __experimental_stream_steps is not None:
+ research_params['__experimental_streamSteps'] = __experimental_stream_steps
+ research_params = V1DeepResearchParams(**research_params)
+
+ headers = self._prepare_headers()
+
+ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
+ json_data['origin'] = f"python-sdk@{version}"
+
+ try:
+ return await self._async_post_request(
+ f'{self.api_url}/v1/deep-research',
+ json_data,
+ headers
+ )
+ except Exception as e:
+ raise ValueError(str(e))
+
+ async def check_deep_research_status(self, id: str) -> V1DeepResearchStatusResponse:
+ """
+ Check the status of a deep research operation.
+
+ Args:
+ id (str): The ID of the deep research operation.
+
+ Returns:
+ DeepResearchResponse containing:
+
+ Status:
+ * success - Whether research completed successfully
+ * status - Current state (processing/completed/failed)
+ * error - Error message if failed
+
+ Results:
+ * id - Unique identifier for the research job
+ * data - Research findings and analysis
+ * sources - List of discovered sources
+ * activities - Research progress log
+ * summaries - Generated research summaries
+
+ Raises:
+ Exception: If the status check fails.
+ """
+ headers = self._prepare_headers()
+ try:
+ return await self._async_get_request(
+ f'{self.api_url}/v1/deep-research/{id}',
+ headers
+ )
+ except Exception as e:
+ raise ValueError(str(e))
+
+ async def search(
+ self,
+ query: str,
+ *,
+ limit: Optional[int] = None,
+ tbs: Optional[str] = None,
+ filter: Optional[str] = None,
+ lang: Optional[str] = None,
+ country: Optional[str] = None,
+ location: Optional[str] = None,
+ timeout: Optional[int] = 30000,
+ scrape_options: Optional[V1ScrapeOptions] = None,
+ params: Optional[Union[Dict[str, Any], V1SearchParams]] = None,
+ **kwargs) -> V1SearchResponse:
+ """
+ Asynchronously search for content using Firecrawl.
+
+ Args:
+ query (str): Search query string
+ limit (Optional[int]): Max results (default: 5)
+ tbs (Optional[str]): Time filter (e.g. "qdr:d")
+ filter (Optional[str]): Custom result filter
+ lang (Optional[str]): Language code (default: "en")
+ country (Optional[str]): Country code (default: "us")
+ location (Optional[str]): Geo-targeting
+ timeout (Optional[int]): Request timeout in milliseconds
+ scrape_options (Optional[ScrapeOptions]): Result scraping configuration
+ params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters
+ **kwargs: Additional keyword arguments for future compatibility
+
+ Returns:
+ SearchResponse: Response containing:
+ * success (bool): Whether request succeeded
+ * data (List[FirecrawlDocument]): Search results
+ * warning (Optional[str]): Warning message if any
+ * error (Optional[str]): Error message if any
+
+ Raises:
+ Exception: If search fails or response cannot be parsed
+ """
+ # Build search parameters
+ search_params = {}
+ if params:
+ if isinstance(params, dict):
+ search_params.update(params)
+ else:
+ search_params.update(params.dict(by_alias=True, exclude_none=True))
+
+ # Add individual parameters
+ if limit is not None:
+ search_params['limit'] = limit
+ if tbs is not None:
+ search_params['tbs'] = tbs
+ if filter is not None:
+ search_params['filter'] = filter
+ if lang is not None:
+ search_params['lang'] = lang
+ if country is not None:
+ search_params['country'] = country
+ if location is not None:
+ search_params['location'] = location
+ if timeout is not None:
+ search_params['timeout'] = timeout
+ if scrape_options is not None:
+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)
+
+ # Add any additional kwargs
+ search_params.update(kwargs)
+
+ # Create final params object
+ final_params = V1SearchParams(query=query, **search_params)
+ params_dict = final_params.dict(by_alias=True, exclude_none=True)
+ params_dict['origin'] = f"python-sdk@{version}"
+
+ return await self._async_post_request(
+ f"{self.api_url}/v1/search",
+ params_dict,
+ {"Authorization": f"Bearer {self.api_key}"}
+ )
+
+class AsyncV1CrawlWatcher(V1CrawlWatcher):
+ """
+ Async version of V1CrawlWatcher that properly handles async operations.
+ """
+ def __init__(self, id: str, app: AsyncV1FirecrawlApp):
+ super().__init__(id, app)
+
+ async def connect(self) -> None:
+ """
+ Establishes async WebSocket connection and starts listening for messages.
+ """
+ async with websockets.connect(
+ self.ws_url,
+ additional_headers=[("Authorization", f"Bearer {self.app.api_key}")]
+ ) as websocket:
+ await self._listen(websocket)
+
+ async def _listen(self, websocket) -> None:
+ """
+ Listens for incoming WebSocket messages and handles them asynchronously.
+
+ Args:
+ websocket: The WebSocket connection object
+ """
+ async for message in websocket:
+ msg = json.loads(message)
+ await self._handle_message(msg)
+
+ async def _handle_message(self, msg: Dict[str, Any]) -> None:
+ """
+ Handles incoming WebSocket messages based on their type asynchronously.
+
+ Args:
+ msg (Dict[str, Any]): The message to handle
+ """
+ if msg['type'] == 'done':
+ self.status = 'completed'
+ self.dispatch_event('done', {'status': self.status, 'data': self.data, 'id': self.id})
+ elif msg['type'] == 'error':
+ self.status = 'failed'
+ self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error'], 'id': self.id})
+ elif msg['type'] == 'catchup':
+ self.status = msg['data']['status']
+ self.data.extend(msg['data'].get('data', []))
+ for doc in self.data:
+ self.dispatch_event('document', {'data': doc, 'id': self.id})
+ elif msg['type'] == 'document':
+ self.data.append(msg['data'])
+ self.dispatch_event('document', {'data': msg['data'], 'id': self.id})
+
+ async def _handle_error(self, response: aiohttp.ClientResponse, action: str) -> None:
+ """
+ Handle errors from async API responses.
+ """
+ try:
+ error_data = await response.json()
+ error_message = error_data.get('error', 'No error message provided.')
+ error_details = error_data.get('details', 'No additional error details provided.')
+ except:
+ raise aiohttp.ClientError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status}')
+
+ # Use the app's method to get the error message
+ message = await self.app._get_async_error_message(response.status, action, error_message, error_details)
+
+ raise aiohttp.ClientError(message)
+
+ async def _get_async_error_message(self, status_code: int, action: str, error_message: str, error_details: str) -> str:
+ """
+ Generate a standardized error message based on HTTP status code for async operations.
+
+ Args:
+ status_code (int): The HTTP status code from the response
+ action (str): Description of the action that was being performed
+ error_message (str): The error message from the API response
+ error_details (str): Additional error details from the API response
+
+ Returns:
+ str: A formatted error message
+ """
+ return self._get_error_message(status_code, action, error_message, error_details)
diff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.py
new file mode 100644
index 000000000..5bedce8a8
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/__init__.py
@@ -0,0 +1,129 @@
+from typing import Optional, List, Union
+from ..types import SearchResponse, ScrapeOptions, SearchRequest, SourceOption, FormatOption
+from .methods.search import search as search_method
+from .utils.http_client import HttpClient
+
+class FirecrawlClient:
+ """
+
+ """
+
+ def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
+ self.api_key = api_key or "placeholder-key"
+ self.api_url = api_url
+ self._client = HttpClient(api_key=self.api_key, api_url=self.api_url)
+
+ # scrape
+ def scrape(
+ self
+ ):
+ pass
+
+ # batch-scrape
+ def batch_scrape(
+ self
+ ):
+ pass
+
+ # async-batch-scrape
+ def async_batch_scrape(
+ self
+ ):
+ pass
+
+ # get-batch-scrape-status
+ def get_batch_scrape_status(
+ self
+ ):
+ pass
+
+ # cancel-batch-scrape
+ def cancel_batch_scrape(
+ self
+ ):
+ pass
+ # get-batch-scrape-errors
+ def get_batch_scrape_errors(
+ self
+ ):
+ pass
+
+ # crawl
+ def crawl(
+ self
+ ):
+ pass
+
+ # async-crawl
+ def async_crawl(
+ self
+ ):
+ pass
+
+
+ # crawl-params
+ def crawl_params(
+ self
+ ):
+ pass
+
+ # get-crawl-status
+ def get_crawl_status(
+ self
+ ):
+ pass
+
+ # cancel-crawl
+ def cancel_crawl(
+ self
+ ):
+ pass
+
+ # get-active-crawls
+ def get_active_crawls(
+ self
+ ):
+ pass
+ # get-crawl-errors
+ def get_crawl_errors(
+ self
+ ):
+ pass
+
+ # map
+ def map(
+ self
+ ):
+ pass
+
+ def search(
+ self,
+ query: str,
+ sources: Optional[List[SourceOption]] = None,
+ limit: Optional[int] = 5,
+ tbs: Optional[str] = None,
+ location: Optional[str] = None,
+ ignore_invalid_urls: Optional[bool] = True,
+ timeout: Optional[int] = 60000,
+ scrape_options: Optional[ScrapeOptions] = None,
+ ) -> SearchResponse:
+ """Search for documents."""
+ request = SearchRequest(
+ query=query,
+ sources=sources,
+ limit=limit,
+ tbs=tbs,
+ location=location,
+ ignore_invalid_urls=ignore_invalid_urls,
+ timeout=timeout,
+ scrape_options=scrape_options,
+ )
+ return search_method(self._client, request)
+
+ # credit-usage
+ def credit_usage(
+ self
+ ):
+ pass
+
+__all__ = ['FirecrawlClient']
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
new file mode 100644
index 000000000..113ca961e
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -0,0 +1,121 @@
+"""
+Main Firecrawl v2 API client.
+
+This module provides the main client class that orchestrates all v2 functionality.
+"""
+
+import os
+from typing import Optional, List, Dict, Any, Callable
+from .types import (
+ ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions,
+ ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse,
+ BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document,
+ SearchRequest, SearchResponse
+)
+from .utils.http_client import HttpClient
+from .utils.error_handler import FirecrawlError
+# Import functions from modules
+from . import scrape as scrape_module
+from . import crawl as crawl_module
+from . import batch as batch_module
+from . import search as search_module
+
+
+class FirecrawlClient:
+ """
+ Main Firecrawl v2 API client.
+
+ This client provides a clean, modular interface to all Firecrawl functionality.
+ """
+
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ api_url: str = "https://api.firecrawl.dev",
+ timeout: Optional[float] = None,
+ max_retries: int = 3,
+ backoff_factor: float = 0.5
+ ):
+ """
+ Initialize the Firecrawl client.
+
+ Args:
+ api_key: Firecrawl API key (or set FIRECRAWL_API_KEY env var)
+ api_url: Base URL for the Firecrawl API
+ timeout: Request timeout in seconds
+ max_retries: Maximum number of retries for failed requests
+ backoff_factor: Exponential backoff factor for retries (e.g. 0.5 means wait 0.5s, then 1s, then 2s between retries)
+ """
+ if api_key is None:
+ api_key = os.getenv("FIRECRAWL_API_KEY")
+
+ if not api_key:
+ raise ValueError(
+ "API key is required. Set FIRECRAWL_API_KEY environment variable "
+ "or pass api_key parameter."
+ )
+
+ self.config = ClientConfig(
+ api_key=api_key,
+ api_url=api_url,
+ timeout=timeout,
+ max_retries=max_retries,
+ backoff_factor=backoff_factor
+ )
+
+ self.http_client = HttpClient(api_key, api_url)
+
+ # def scrape(
+ # self,
+ # url: str,
+ # options: Optional[ScrapeOptions] = None
+ # ) -> ScrapeResponse:
+ # """
+ # Scrape a single URL.
+
+ # Args:
+ # url: URL to scrape
+ # options: Scraping options
+
+ # Returns:
+ # ScrapeResponse containing the scraped document
+ # """
+
+ # return scrape_module.scrape(self.http_client, url, options)
+
+
+ def search(
+ self,
+ query: str,
+ *,
+ limit: Optional[int] = None,
+ tbs: Optional[str] = None,
+ location: Optional[str] = None,
+ timeout: Optional[int] = None,
+ page_options: Optional[ScrapeOptions] = None,
+ ) -> SearchResponse:
+ """
+ Search for documents.
+
+ Args:
+ query: Search query string
+ limit: Maximum number of results to return (default: 5)
+ tbs: Time-based search filter
+ location: Location string for search
+ timeout: Request timeout in milliseconds (default: 60000)
+ page_options: Options for scraping individual pages
+
+ Returns:
+ SearchResponse containing the search results
+ """
+ # Build options object from individual parameters
+ options = SearchRequest(
+ limit=limit,
+ tbs=tbs,
+ location=location,
+ timeout=timeout,
+ page_options=page_options
+ )
+
+ return search_module.search(self.http_client, query, options)
+
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.py
new file mode 100644
index 000000000..f74524aae
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py
@@ -0,0 +1,383 @@
+"""
+Batch scraping functionality for Firecrawl v2 API.
+"""
+
+import time
+from typing import Optional, List, Callable
+from .types import (
+ BatchScrapeRequest, BatchScrapeResponse, BatchScrapeStatusResponse,
+ BatchScrapeJob, BatchScrapeStatusData, ScrapeOptions, Document
+)
+from .utils.http_client import HttpClient
+from .utils.error_handler import handle_response_error
+
+
+def start_batch_scrape(
+ client: HttpClient,
+ urls: List[str],
+ options: Optional[ScrapeOptions] = None
+) -> BatchScrapeResponse:
+ """
+ Start a batch scrape job for multiple URLs.
+
+ Args:
+ client: HTTP client instance
+ urls: List of URLs to scrape
+ options: Scraping options
+
+ Returns:
+ BatchScrapeResponse containing job information
+
+ Raises:
+ FirecrawlError: If the batch scrape operation fails to start
+ """
+ # Prepare request data
+ request_data = {"urls": urls}
+
+ if options:
+ request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)
+
+ # Make the API request
+ response = client.post("/v1/batch/scrape", request_data)
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "start batch scrape")
+
+ # Parse response
+ response_data = response.json()
+
+ if response_data.get("success"):
+ job_data = response_data.get("data", {})
+ job = BatchScrapeJob(**job_data)
+
+ return BatchScrapeResponse(
+ success=True,
+ data=job,
+ warning=response_data.get("warning")
+ )
+ else:
+ return BatchScrapeResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def get_batch_scrape_status(
+ client: HttpClient,
+ job_id: str
+) -> BatchScrapeStatusResponse:
+ """
+ Get the status of a batch scrape job.
+
+ Args:
+ client: HTTP client instance
+ job_id: ID of the batch scrape job
+
+ Returns:
+ BatchScrapeStatusResponse containing job status and data
+
+ Raises:
+ FirecrawlError: If the status check fails
+ """
+ # Make the API request
+ response = client.get(f"/v1/batch/scrape/{job_id}")
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "get batch scrape status")
+
+ # Parse response
+ response_data = response.json()
+
+ if response_data.get("success"):
+ status_data = response_data.get("data", {})
+
+ # Convert documents
+ documents = []
+ if "data" in status_data:
+ for doc_data in status_data["data"]:
+ documents.append(Document(**doc_data))
+
+ batch_status = BatchScrapeStatusData(
+ status=status_data.get("status"),
+ current=status_data.get("current", 0),
+ total=status_data.get("total", 0),
+ data=documents
+ )
+
+ return BatchScrapeStatusResponse(
+ success=True,
+ data=batch_status,
+ warning=response_data.get("warning")
+ )
+ else:
+ return BatchScrapeStatusResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def cancel_batch_scrape(
+ client: HttpClient,
+ job_id: str
+) -> BatchScrapeStatusResponse:
+ """
+ Cancel a running batch scrape job.
+
+ Args:
+ client: HTTP client instance
+ job_id: ID of the batch scrape job to cancel
+
+ Returns:
+ BatchScrapeStatusResponse with updated status
+
+ Raises:
+ FirecrawlError: If the cancellation fails
+ """
+ # Make the API request
+ response = client.delete(f"/v1/batch/scrape/{job_id}")
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "cancel batch scrape")
+
+ # Parse response
+ response_data = response.json()
+
+ if response_data.get("success"):
+ status_data = response_data.get("data", {})
+
+ batch_status = BatchScrapeStatusData(
+ status=status_data.get("status", "cancelled"),
+ current=status_data.get("current", 0),
+ total=status_data.get("total", 0),
+ data=[]
+ )
+
+ return BatchScrapeStatusResponse(
+ success=True,
+ data=batch_status,
+ warning=response_data.get("warning")
+ )
+ else:
+ return BatchScrapeStatusResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def wait_for_batch_completion(
+ client: HttpClient,
+ job_id: str,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+ progress_callback: Optional[Callable[[BatchScrapeStatusData], None]] = None
+) -> BatchScrapeStatusResponse:
+ """
+ Wait for a batch scrape job to complete, polling for status updates.
+
+ Args:
+ client: HTTP client instance
+ job_id: ID of the batch scrape job
+ poll_interval: Seconds between status checks
+ timeout: Maximum seconds to wait (None for no timeout)
+ progress_callback: Optional callback for progress updates
+
+ Returns:
+ BatchScrapeStatusResponse when job completes
+
+ Raises:
+ FirecrawlError: If the job fails or timeout is reached
+ TimeoutError: If timeout is reached
+ """
+ start_time = time.time()
+
+ while True:
+ status_response = get_batch_scrape_status(client, job_id)
+
+ if not status_response.success:
+ return status_response
+
+ status_data = status_response.data
+
+ # Call progress callback if provided
+ if progress_callback and status_data:
+ progress_callback(status_data)
+
+ # Check if job is complete
+ if status_data and status_data.status in ["completed", "failed", "cancelled"]:
+ return status_response
+
+ # Check timeout
+ if timeout and (time.time() - start_time) > timeout:
+ raise TimeoutError(f"Batch scrape job {job_id} did not complete within {timeout} seconds")
+
+ # Wait before next poll
+ time.sleep(poll_interval)
+
+
+def batch_scrape_and_wait(
+ client: HttpClient,
+ urls: List[str],
+ options: Optional[ScrapeOptions] = None,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+ progress_callback: Optional[Callable[[BatchScrapeStatusData], None]] = None
+) -> BatchScrapeStatusResponse:
+ """
+ Start a batch scrape job and wait for it to complete.
+
+ Args:
+ client: HTTP client instance
+ urls: List of URLs to scrape
+ options: Scraping options
+ poll_interval: Seconds between status checks
+ timeout: Maximum seconds to wait (None for no timeout)
+ progress_callback: Optional callback for progress updates
+
+ Returns:
+ BatchScrapeStatusResponse when job completes
+
+ Raises:
+ FirecrawlError: If the batch scrape fails to start or complete
+ TimeoutError: If timeout is reached
+ """
+ # Start the batch scrape
+ batch_response = start_batch_scrape(client, urls, options)
+
+ if not batch_response.success or not batch_response.data:
+ return BatchScrapeStatusResponse(
+ success=False,
+ error=batch_response.error or "Failed to start batch scrape"
+ )
+
+ job_id = batch_response.data.id
+
+ # Wait for completion
+ return wait_for_batch_completion(
+ client, job_id, poll_interval, timeout, progress_callback
+ )
+
+
+def validate_batch_urls(urls: List[str]) -> List[str]:
+ """
+ Validate and normalize a list of URLs for batch scraping.
+
+ Args:
+ urls: List of URLs to validate
+
+ Returns:
+ Validated list of URLs
+
+ Raises:
+ ValueError: If URLs are invalid
+ """
+ if not urls:
+ raise ValueError("URLs list cannot be empty")
+
+ if len(urls) > 1000: # Assuming API limit
+ raise ValueError("Too many URLs (maximum 1000)")
+
+ validated_urls = []
+ for url in urls:
+ if not url or not isinstance(url, str):
+ raise ValueError(f"Invalid URL: {url}")
+
+ # Basic URL validation
+ if not (url.startswith("http://") or url.startswith("https://")):
+ raise ValueError(f"URL must start with http:// or https://: {url}")
+
+ validated_urls.append(url.strip())
+
+ return validated_urls
+
+
+def prepare_batch_request(urls: List[str], options: Optional[ScrapeOptions] = None) -> dict:
+ """
+ Prepare a batch scrape request payload.
+
+ Args:
+ urls: List of URLs to scrape
+ options: Scraping options
+
+ Returns:
+ Request payload dictionary
+ """
+ validated_urls = validate_batch_urls(urls)
+ request_data = {"urls": validated_urls}
+
+ if options:
+ request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)
+
+ return request_data
+
+
+def chunk_urls(urls: List[str], chunk_size: int = 100) -> List[List[str]]:
+ """
+ Split a large list of URLs into smaller chunks for batch processing.
+
+ Args:
+ urls: List of URLs to chunk
+ chunk_size: Maximum size of each chunk
+
+ Returns:
+ List of URL chunks
+ """
+ chunks = []
+ for i in range(0, len(urls), chunk_size):
+ chunks.append(urls[i:i + chunk_size])
+ return chunks
+
+
+def process_large_batch(
+ client: HttpClient,
+ urls: List[str],
+ options: Optional[ScrapeOptions] = None,
+ chunk_size: int = 100,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+ progress_callback: Optional[Callable[[int, int, List[Document]], None]] = None
+) -> List[Document]:
+ """
+ Process a large batch of URLs by splitting into smaller chunks.
+
+ Args:
+ client: HTTP client instance
+ urls: List of URLs to scrape
+ options: Scraping options
+ chunk_size: Size of each batch chunk
+ poll_interval: Seconds between status checks
+ timeout: Maximum seconds to wait per chunk
+ progress_callback: Optional callback for overall progress (completed, total, documents)
+
+ Returns:
+ List of all scraped documents
+
+ Raises:
+ FirecrawlError: If any chunk fails
+ """
+ url_chunks = chunk_urls(urls, chunk_size)
+ all_documents = []
+ completed_chunks = 0
+
+ for chunk in url_chunks:
+ # Process this chunk
+ result = batch_scrape_and_wait(
+ client, chunk, options, poll_interval, timeout
+ )
+
+ if not result.success or not result.data:
+ raise Exception(f"Failed to process chunk: {result.error}")
+
+ # Add documents from this chunk
+ if result.data.data:
+ all_documents.extend(result.data.data)
+
+ completed_chunks += 1
+
+ # Call progress callback if provided
+ if progress_callback:
+ progress_callback(completed_chunks, len(url_chunks), all_documents)
+
+ return all_documents
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
new file mode 100644
index 000000000..922e28843
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -0,0 +1,360 @@
+"""
+Crawling functionality for Firecrawl v2 API.
+"""
+
+import time
+from typing import Optional, Callable, Generator
+from .types import (
+ CrawlRequest, CrawlResponse, CrawlStatusResponse, CrawlOptions,
+ CrawlJob, CrawlStatusData, Document
+)
+from .utils.http_client import HttpClient
+from .utils.error_handler import handle_response_error
+
+
+def start_crawl(
+ client: HttpClient,
+ url: str,
+ options: Optional[CrawlOptions] = None
+) -> CrawlResponse:
+ """
+ Start a crawl job for a website.
+
+ Args:
+ client: HTTP client instance
+ url: URL to crawl
+ options: Crawling options
+
+ Returns:
+ CrawlResponse containing job information
+
+ Raises:
+ FirecrawlError: If the crawl operation fails to start
+ """
+ # Prepare request data
+ request_data = {"url": url}
+
+ if options:
+ request_data.update(options.dict(exclude_none=True, by_alias=True))
+
+ # Make the API request
+ response = client.post("/v1/crawl", request_data)
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "start crawl")
+
+ # Parse response
+ response_data = response.json()
+
+ if response_data.get("success"):
+ job_data = response_data.get("data", {})
+ job = CrawlJob(**job_data)
+
+ return CrawlResponse(
+ success=True,
+ data=job,
+ warning=response_data.get("warning")
+ )
+ else:
+ return CrawlResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def get_crawl_status(
+ client: HttpClient,
+ job_id: str
+) -> CrawlStatusResponse:
+ """
+ Get the status of a crawl job.
+
+ Args:
+ client: HTTP client instance
+ job_id: ID of the crawl job
+
+ Returns:
+ CrawlStatusResponse containing job status and data
+
+ Raises:
+ FirecrawlError: If the status check fails
+ """
+ # Make the API request
+ response = client.get(f"/v1/crawl/{job_id}")
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "get crawl status")
+
+ # Parse response
+ response_data = response.json()
+
+ if response_data.get("success"):
+ status_data = response_data.get("data", {})
+
+ # Convert documents
+ documents = []
+ if "data" in status_data:
+ for doc_data in status_data["data"]:
+ documents.append(Document(**doc_data))
+
+ # Convert partial data if present
+ partial_documents = []
+ if "partialData" in status_data:
+ for doc_data in status_data["partialData"]:
+ partial_documents.append(Document(**doc_data))
+
+ crawl_status = CrawlStatusData(
+ status=status_data.get("status"),
+ current=status_data.get("current", 0),
+ total=status_data.get("total", 0),
+ data=documents,
+ partial_data=partial_documents if partial_documents else None
+ )
+
+ return CrawlStatusResponse(
+ success=True,
+ data=crawl_status,
+ warning=response_data.get("warning")
+ )
+ else:
+ return CrawlStatusResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def cancel_crawl(
+ client: HttpClient,
+ job_id: str
+) -> CrawlStatusResponse:
+ """
+ Cancel a running crawl job.
+
+ Args:
+ client: HTTP client instance
+ job_id: ID of the crawl job to cancel
+
+ Returns:
+ CrawlStatusResponse with updated status
+
+ Raises:
+ FirecrawlError: If the cancellation fails
+ """
+ # Make the API request
+ response = client.delete(f"/v1/crawl/{job_id}")
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "cancel crawl")
+
+ # Parse response
+ response_data = response.json()
+
+ if response_data.get("success"):
+ status_data = response_data.get("data", {})
+
+ crawl_status = CrawlStatusData(
+ status=status_data.get("status", "cancelled"),
+ current=status_data.get("current", 0),
+ total=status_data.get("total", 0),
+ data=[]
+ )
+
+ return CrawlStatusResponse(
+ success=True,
+ data=crawl_status,
+ warning=response_data.get("warning")
+ )
+ else:
+ return CrawlStatusResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def wait_for_crawl_completion(
+ client: HttpClient,
+ job_id: str,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+ progress_callback: Optional[Callable[[CrawlStatusData], None]] = None
+) -> CrawlStatusResponse:
+ """
+ Wait for a crawl job to complete, polling for status updates.
+
+ Args:
+ client: HTTP client instance
+ job_id: ID of the crawl job
+ poll_interval: Seconds between status checks
+ timeout: Maximum seconds to wait (None for no timeout)
+ progress_callback: Optional callback for progress updates
+
+ Returns:
+ CrawlStatusResponse when job completes
+
+ Raises:
+ FirecrawlError: If the job fails or timeout is reached
+ TimeoutError: If timeout is reached
+ """
+ start_time = time.time()
+
+ while True:
+ status_response = get_crawl_status(client, job_id)
+
+ if not status_response.success:
+ return status_response
+
+ status_data = status_response.data
+
+ # Call progress callback if provided
+ if progress_callback and status_data:
+ progress_callback(status_data)
+
+ # Check if job is complete
+ if status_data and status_data.status in ["completed", "failed", "cancelled"]:
+ return status_response
+
+ # Check timeout
+ if timeout and (time.time() - start_time) > timeout:
+ raise TimeoutError(f"Crawl job {job_id} did not complete within {timeout} seconds")
+
+ # Wait before next poll
+ time.sleep(poll_interval)
+
+
+def crawl_and_wait(
+ client: HttpClient,
+ url: str,
+ options: Optional[CrawlOptions] = None,
+ poll_interval: int = 2,
+ timeout: Optional[int] = None,
+ progress_callback: Optional[Callable[[CrawlStatusData], None]] = None
+) -> CrawlStatusResponse:
+ """
+ Start a crawl job and wait for it to complete.
+
+ Args:
+ client: HTTP client instance
+ url: URL to crawl
+ options: Crawling options
+ poll_interval: Seconds between status checks
+ timeout: Maximum seconds to wait (None for no timeout)
+ progress_callback: Optional callback for progress updates
+
+ Returns:
+ CrawlStatusResponse when job completes
+
+ Raises:
+ FirecrawlError: If the crawl fails to start or complete
+ TimeoutError: If timeout is reached
+ """
+ # Start the crawl
+ crawl_response = start_crawl(client, url, options)
+
+ if not crawl_response.success or not crawl_response.data:
+ return CrawlStatusResponse(
+ success=False,
+ error=crawl_response.error or "Failed to start crawl"
+ )
+
+ job_id = crawl_response.data.id
+
+ # Wait for completion
+ return wait_for_crawl_completion(
+ client, job_id, poll_interval, timeout, progress_callback
+ )
+
+
+def stream_crawl_results(
+ client: HttpClient,
+ job_id: str,
+ poll_interval: int = 2
+) -> Generator[Document, None, None]:
+ """
+ Stream crawl results as they become available.
+
+ Args:
+ client: HTTP client instance
+ job_id: ID of the crawl job
+ poll_interval: Seconds between status checks
+
+ Yields:
+ Document objects as they are crawled
+
+ Raises:
+ FirecrawlError: If the job fails
+ """
+ seen_count = 0
+
+ while True:
+ status_response = get_crawl_status(client, job_id)
+
+ if not status_response.success:
+ raise Exception(f"Failed to get crawl status: {status_response.error}")
+
+ status_data = status_response.data
+ if not status_data:
+ break
+
+ # Yield new documents
+ if status_data.data:
+ for i in range(seen_count, len(status_data.data)):
+ yield status_data.data[i]
+ seen_count = len(status_data.data)
+
+ # Check if job is complete
+ if status_data.status in ["completed", "failed", "cancelled"]:
+ break
+
+ # Wait before next poll
+ time.sleep(poll_interval)
+
+
+def validate_crawl_options(options: Optional[CrawlOptions]) -> Optional[CrawlOptions]:
+ """
+ Validate and normalize crawl options.
+
+ Args:
+ options: Crawling options to validate
+
+ Returns:
+ Validated options or None
+
+ Raises:
+ ValueError: If options are invalid
+ """
+ if options is None:
+ return None
+
+ # Validate limit
+ if options.limit is not None and options.limit <= 0:
+ raise ValueError("Limit must be positive")
+
+ # Validate max_depth
+ if options.max_depth is not None and options.max_depth < 0:
+ raise ValueError("max_depth must be non-negative")
+
+ return options
+
+
+def prepare_crawl_request(url: str, options: Optional[CrawlOptions] = None) -> dict:
+ """
+ Prepare a crawl request payload.
+
+ Args:
+ url: URL to crawl
+ options: Crawling options
+
+ Returns:
+ Request payload dictionary
+ """
+ request_data = {"url": url}
+
+ if options:
+ validated_options = validate_crawl_options(options)
+ if validated_options:
+ request_data.update(validated_options.dict(exclude_none=True, by_alias=True))
+
+ return request_data
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.py
new file mode 100644
index 000000000..eb6e6eaef
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py
@@ -0,0 +1,215 @@
+"""
+Scraping functionality for Firecrawl v2 API.
+"""
+
+import json
+from typing import Optional, Dict, Any
+from .types import ScrapeRequest, ScrapeResponse, ScrapeOptions, Document
+from .utils.http_client import HttpClient
+from .utils.error_handler import handle_response_error
+
+
+def scrape(
+ client: HttpClient,
+ url: str,
+ options: Optional[ScrapeOptions] = None
+) -> ScrapeResponse:
+ """
+ Scrape a single URL and return the content.
+
+ Args:
+ client: HTTP client instance
+ url: URL to scrape
+ options: Scraping options
+
+ Returns:
+ ScrapeResponse containing the scraped document
+
+ Raises:
+ FirecrawlError: If the scraping operation fails
+ """
+ # Prepare request data
+ request_data = {"url": url}
+
+ if options:
+ request_data["options"] = options.dict(exclude_none=True, by_alias=True)
+
+ # Make the API request
+ response = client.post("/v1/scrape", request_data)
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "scrape URL")
+
+ # Parse response
+ response_data = response.json()
+
+ # Create document from response
+ if response_data.get("success"):
+ document_data = response_data.get("data", {})
+ document = Document(**document_data)
+
+ return ScrapeResponse(
+ success=True,
+ data=document,
+ warning=response_data.get("warning")
+ )
+ else:
+ return ScrapeResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def scrape_with_extract(
+ client: HttpClient,
+ url: str,
+ schema: Dict[str, Any],
+ options: Optional[ScrapeOptions] = None,
+ system_prompt: Optional[str] = None,
+ prompt: Optional[str] = None
+) -> ScrapeResponse:
+ """
+ Scrape a URL and extract structured data using a schema.
+
+ Args:
+ client: HTTP client instance
+ url: URL to scrape
+ schema: JSON schema for data extraction
+ options: Scraping options
+ system_prompt: System prompt for extraction
+ prompt: User prompt for extraction
+
+ Returns:
+ ScrapeResponse with extracted data
+
+ Raises:
+ FirecrawlError: If the operation fails
+ """
+ # Prepare request data
+ request_data = {
+ "url": url,
+ "extractorOptions": {
+ "extractionSchema": schema
+ }
+ }
+
+ if system_prompt:
+ request_data["extractorOptions"]["extractionPrompt"] = system_prompt
+
+ if prompt:
+ request_data["extractorOptions"]["userPrompt"] = prompt
+
+ if options:
+ request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)
+
+ # Make the API request
+ response = client.post("/v1/scrape", request_data)
+
+ # Handle errors
+ if not response.ok:
+ handle_response_error(response, "scrape and extract from URL")
+
+ # Parse response
+ response_data = response.json()
+
+ # Create document from response
+ if response_data.get("success"):
+ document_data = response_data.get("data", {})
+ document = Document(**document_data)
+
+ return ScrapeResponse(
+ success=True,
+ data=document,
+ warning=response_data.get("warning")
+ )
+ else:
+ return ScrapeResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def scrape_multiple(
+ client: HttpClient,
+ urls: list[str],
+ options: Optional[ScrapeOptions] = None
+) -> list[ScrapeResponse]:
+ """
+ Scrape multiple URLs concurrently.
+
+ Args:
+ client: HTTP client instance
+ urls: List of URLs to scrape
+ options: Scraping options
+
+ Returns:
+ List of ScrapeResponse objects
+
+ Note:
+ This function makes individual requests for each URL.
+ For large batches, consider using batch_scrape instead.
+ """
+ results = []
+
+ for url in urls:
+ try:
+ result = scrape(client, url, options)
+ results.append(result)
+ except Exception as e:
+ # Create error response for failed scrapes
+ error_response = ScrapeResponse(
+ success=False,
+ error=f"Failed to scrape {url}: {str(e)}"
+ )
+ results.append(error_response)
+
+ return results
+
+
+def validate_scrape_options(options: Optional[ScrapeOptions]) -> Optional[ScrapeOptions]:
+ """
+ Validate and normalize scrape options.
+
+ Args:
+ options: Scraping options to validate
+
+ Returns:
+ Validated options or None
+
+ Raises:
+ ValueError: If options are invalid
+ """
+ if options is None:
+ return None
+
+ # Validate timeout
+ if options.timeout is not None and options.timeout <= 0:
+ raise ValueError("Timeout must be positive")
+
+ # Validate wait_for
+ if options.wait_for is not None and options.wait_for < 0:
+ raise ValueError("wait_for must be non-negative")
+
+ return options
+
+
+def prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
+ """
+ Prepare a scrape request payload.
+
+ Args:
+ url: URL to scrape
+ options: Scraping options
+
+ Returns:
+ Request payload dictionary
+ """
+ request_data = {"url": url}
+
+ if options:
+ validated_options = validate_scrape_options(options)
+ if validated_options:
+ request_data["options"] = validated_options.dict(exclude_none=True, by_alias=True)
+
+ return request_data
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
new file mode 100644
index 000000000..ea56a6e19
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -0,0 +1,101 @@
+"""
+Search functionality for Firecrawl v2 API.
+"""
+
+from typing import Optional, Dict, Any, Union
+from ...types import SearchRequest, SearchResponse, SearchResult, Document
+from ..utils.http_client import HttpClient
+from ..utils.error_handler import handle_response_error
+
+
+def search(
+ client: HttpClient,
+ request: SearchRequest
+) -> SearchResponse:
+ """
+ Search for documents.
+
+ Args:
+ client: HTTP client instance
+ request: Search request
+
+ Returns:
+ SearchResponse containing the search results
+
+ Raises:
+ FirecrawlError: If the search operation fails
+ """
+ request_data = _prepare_search_request(request)
+
+ response = client.post("/v2/search", request_data)
+
+ if not response.ok:
+ handle_response_error(response, "search")
+
+ response_data = response.json()
+
+ if response_data.get("success"):
+ data = response_data.get("data", {})
+ grouped_results = {}
+
+ for source_type, source_documents in data.items():
+ if isinstance(source_documents, list):
+ results = []
+ for doc_data in source_documents:
+ if isinstance(doc_data, dict):
+ if any(key in doc_data for key in ['markdown', 'html', 'content', 'screenshot']):
+ results.append(Document(**doc_data))
+ else:
+ results.append(SearchResult(
+ url=doc_data.get('url', ''),
+ title=doc_data.get('title'),
+ description=doc_data.get('description')
+ ))
+ elif isinstance(doc_data, str):
+ results.append(SearchResult(url=doc_data))
+ grouped_results[source_type] = results
+
+ return SearchResponse(
+ success=True,
+ data=grouped_results,
+ warning=response_data.get("warning")
+ )
+ else:
+ return SearchResponse(
+ success=False,
+ error=response_data.get("error", "Unknown error occurred")
+ )
+
+
+def _validate_search_request(request: SearchRequest) -> SearchRequest:
+ """
+ Validate and normalize search request.
+
+ Args:
+ request: Search request to validate
+
+ Returns:
+ Validated request
+
+ Raises:
+ ValueError: If request is invalid
+ """
+ # Validate timeout if present
+ if request.timeout is not None and request.timeout <= 0:
+ raise ValueError("Timeout must be positive")
+
+ return request
+
+
+def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
+ """
+ Prepare a search request payload.
+
+ Args:
+ request: Search request
+
+ Returns:
+ Request payload dictionary
+ """
+ validated_request = _validate_search_request(request)
+ return validated_request.model_dump(exclude_none=True, by_alias=True)
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
new file mode 100644
index 000000000..9884a1981
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -0,0 +1,376 @@
+"""
+Type definitions for Firecrawl v2 API.
+
+This module contains clean, modern type definitions for the v2 API.
+"""
+
+from typing import Dict, List, Optional, Union, Literal, Any, TypeVar, Generic
+from pydantic import BaseModel, Field, field_validator
+from datetime import datetime
+
+T = TypeVar('T')
+
+# Base response types
+class BaseResponse(BaseModel, Generic[T]):
+ """Base response structure for all API responses."""
+ success: bool
+ data: Optional[T] = None
+ error: Optional[str] = None
+ warning: Optional[str] = None
+
+# Document and content types
+class DocumentMetadata(BaseModel):
+ """Metadata for scraped documents."""
+ title: Optional[str] = None
+ description: Optional[str] = None
+ language: Optional[str] = None
+ keywords: Optional[str] = None
+ robots: Optional[str] = None
+ og_title: Optional[str] = Field(None, alias="ogTitle")
+ og_description: Optional[str] = Field(None, alias="ogDescription")
+ og_url: Optional[str] = Field(None, alias="ogUrl")
+ og_image: Optional[str] = Field(None, alias="ogImage")
+ source_url: Optional[str] = Field(None, alias="sourceURL")
+ status_code: Optional[int] = Field(None, alias="statusCode")
+ error: Optional[str] = None
+
+class Document(BaseModel):
+ """A scraped document."""
+ url: str
+ markdown: Optional[str] = None
+ html: Optional[str] = None
+ raw_html: Optional[str] = Field(None, alias="rawHtml")
+ content: Optional[str] = None
+ metadata: Optional[DocumentMetadata] = None
+ links: Optional[List[str]] = None
+ screenshot: Optional[str] = None
+ actions: Optional[Dict[str, Any]] = None
+
+class Source(BaseModel):
+ """Configuration for a search source."""
+ type: str
+
+SourceOption = Union[str, Source]
+
+class Format(BaseModel):
+ """Configuration for a format."""
+ type: str
+
+FormatOption = Union[str, Format]
+
+# Scrape types
+class ScrapeFormats(BaseModel):
+ """Output formats for scraping."""
+ formats: Optional[List[FormatOption]] = None
+ markdown: bool = True
+ html: bool = False
+ raw_html: bool = Field(False, alias="rawHtml")
+ content: bool = False
+ links: bool = False
+ screenshot: bool = False
+ screenshot_full_page: bool = Field(False, alias="screenshot@fullPage")
+
+ @field_validator('formats')
+ @classmethod
+ def validate_formats(cls, v):
+ """Validate and normalize formats input."""
+ if v is None:
+ return v
+
+ normalized_formats = []
+ for format_item in v:
+ if isinstance(format_item, str):
+ normalized_formats.append(Format(type=format_item))
+ elif isinstance(format_item, dict):
+ normalized_formats.append(Format(**format_item))
+ elif isinstance(format_item, Format):
+ normalized_formats.append(format_item)
+ else:
+ raise ValueError(f"Invalid format format: {format_item}")
+
+ return normalized_formats
+
+class ScrapeOptions(BaseModel):
+ """Options for scraping operations."""
+ formats: Optional[ScrapeFormats] = None
+ headers: Optional[Dict[str, str]] = None
+ include_tags: Optional[List[str]] = Field(None, alias="includeTags")
+ exclude_tags: Optional[List[str]] = Field(None, alias="excludeTags")
+ only_main_content: bool = Field(True, alias="onlyMainContent")
+ timeout: Optional[int] = None
+ wait_for: Optional[int] = Field(None, alias="waitFor")
+ mobile: bool = False
+ skip_tls_verification: bool = Field(False, alias="skipTlsVerification")
+ remove_base64_images: bool = Field(True, alias="removeBase64Images")
+
+class ScrapeRequest(BaseModel):
+ """Request for scraping a single URL."""
+ url: str
+ options: Optional[ScrapeOptions] = None
+
+class ScrapeResponse(BaseResponse[Document]):
+ """Response from scraping operation."""
+ pass
+
+# Crawl types
+class CrawlOptions(BaseModel):
+ """Options for crawling operations."""
+ includes: Optional[List[str]] = None
+ excludes: Optional[List[str]] = None
+ generate_img_alt_text: bool = Field(False, alias="generateImgAltText")
+ return_only_urls: bool = Field(False, alias="returnOnlyUrls")
+ max_depth: Optional[int] = Field(None, alias="maxDepth")
+ mode: Literal["default", "fast"] = "default"
+ ignore_sitemap: bool = Field(False, alias="ignoreSitemap")
+ limit: Optional[int] = None
+ allow_backward_crawling: bool = Field(False, alias="allowBackwardCrawling")
+ allow_external_content_links: bool = Field(False, alias="allowExternalContentLinks")
+ scrape_options: Optional[ScrapeOptions] = Field(None, alias="scrapeOptions")
+
+class CrawlRequest(BaseModel):
+ """Request for crawling a website."""
+ url: str
+ options: Optional[CrawlOptions] = None
+
+class CrawlJob(BaseModel):
+ """Information about a crawl job."""
+ id: str
+ url: str
+ status: Literal["scraping", "completed", "failed", "cancelled"]
+ current: Optional[int] = None
+ total: Optional[int] = None
+ created_at: Optional[datetime] = Field(None, alias="createdAt")
+ completed_at: Optional[datetime] = Field(None, alias="completedAt")
+
+class CrawlResponse(BaseResponse[CrawlJob]):
+ """Response from starting a crawl."""
+ pass
+
+class CrawlStatusData(BaseModel):
+ """Data for crawl status response."""
+ status: Literal["scraping", "completed", "failed", "cancelled"]
+ current: int
+ total: int
+ data: List[Document]
+ partial_data: Optional[List[Document]] = Field(None, alias="partialData")
+
+class CrawlStatusResponse(BaseResponse[CrawlStatusData]):
+ """Response from checking crawl status."""
+ pass
+
+# Batch scrape types
+class BatchScrapeRequest(BaseModel):
+ """Request for batch scraping multiple URLs."""
+ urls: List[str]
+ options: Optional[ScrapeOptions] = None
+
+class BatchScrapeJob(BaseModel):
+ """Information about a batch scrape job."""
+ id: str
+ status: Literal["scraping", "completed", "failed", "cancelled"]
+ current: Optional[int] = None
+ total: Optional[int] = None
+ created_at: Optional[datetime] = Field(None, alias="createdAt")
+ completed_at: Optional[datetime] = Field(None, alias="completedAt")
+
+class BatchScrapeResponse(BaseResponse[BatchScrapeJob]):
+ """Response from starting a batch scrape."""
+ pass
+
+class BatchScrapeStatusData(BaseModel):
+ """Data for batch scrape status response."""
+ status: Literal["scraping", "completed", "failed", "cancelled"]
+ current: int
+ total: int
+ data: List[Document]
+
+class BatchScrapeStatusResponse(BaseResponse[BatchScrapeStatusData]):
+ """Response from checking batch scrape status."""
+ pass
+
+# Map types
+class MapOptions(BaseModel):
+ """Options for mapping operations."""
+ search: Optional[str] = None
+ ignore_sitemap: bool = Field(False, alias="ignoreSitemap")
+ include_subdomains: bool = Field(False, alias="includeSubdomains")
+ limit: Optional[int] = None
+
+class MapRequest(BaseModel):
+ """Request for mapping a website."""
+ url: str
+ options: Optional[MapOptions] = None
+
+class MapData(BaseModel):
+ """Data for map response."""
+ links: List[str]
+
+class MapResponse(BaseResponse[MapData]):
+ """Response from mapping operation."""
+ pass
+
+# Action types
+class WaitAction(BaseModel):
+ """Wait action to perform during scraping."""
+ type: Literal["wait"] = "wait"
+ milliseconds: Optional[int] = None
+ selector: Optional[str] = None
+
+class ScreenshotAction(BaseModel):
+ """Screenshot action to perform during scraping."""
+ type: Literal["screenshot"] = "screenshot"
+ full_page: Optional[bool] = Field(None, alias="fullPage")
+ quality: Optional[int] = None
+
+class ClickAction(BaseModel):
+ """Click action to perform during scraping."""
+ type: Literal["click"] = "click"
+ selector: str
+
+class WriteAction(BaseModel):
+ """Write action to perform during scraping."""
+ type: Literal["write"] = "write"
+ text: str
+
+class PressAction(BaseModel):
+ """Press action to perform during scraping."""
+ type: Literal["press"] = "press"
+ key: str
+
+class ScrollAction(BaseModel):
+ """Scroll action to perform during scraping."""
+ type: Literal["scroll"] = "scroll"
+ direction: Literal["up", "down"]
+ selector: Optional[str] = None
+
+class ScrapeAction(BaseModel):
+ """Scrape action to perform during scraping."""
+ type: Literal["scrape"] = "scrape"
+
+class ExecuteJavascriptAction(BaseModel):
+ """Execute javascript action to perform during scraping."""
+ type: Literal["executeJavascript"] = "executeJavascript"
+ script: str
+
+class PDFAction(BaseModel):
+ """PDF action to perform during scraping."""
+ type: Literal["pdf"] = "pdf"
+ format: Optional[Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"]] = None
+ landscape: Optional[bool] = None
+ scale: Optional[float] = None
+
+# Location types
+class Location(BaseModel):
+ """Location configuration for scraping."""
+ country: Optional[str] = None
+ languages: Optional[List[str]] = None
+
+# JSON format types
+class JsonFormat(BaseModel):
+ """Configuration for JSON extraction."""
+ prompt: Optional[str] = None
+ schema_field: Optional[Dict[str, Any]] = Field(None, alias="schema")
+ system_prompt: Optional[str] = Field(None, alias="systemPrompt")
+
+class SearchRequest(BaseModel):
+ """Request for search operations."""
+ query: str
+ sources: Optional[List[SourceOption]] = None
+ limit: Optional[int] = 5
+ tbs: Optional[str] = None
+ location: Optional[str] = None
+ ignore_invalid_urls: Optional[bool] = Field(True, alias="ignoreInvalidURLs")
+ timeout: Optional[int] = 60000
+ scrape_options: Optional[ScrapeOptions] = Field(None, alias="scrapeOptions")
+
+ @field_validator('sources')
+ @classmethod
+ def validate_sources(cls, v):
+ """Validate and normalize sources input."""
+ if v is None:
+ return v
+
+ normalized_sources = []
+ for source in v:
+ if isinstance(source, str):
+ normalized_sources.append(Source(type=source))
+ elif isinstance(source, dict):
+ normalized_sources.append(Source(**source))
+ elif isinstance(source, Source):
+ normalized_sources.append(source)
+ else:
+ raise ValueError(f"Invalid source format: {source}")
+
+ return normalized_sources
+
+class SearchResult(BaseModel):
+ """A search result with basic information."""
+ url: str
+ title: Optional[str] = None
+ description: Optional[str] = None
+
+class SearchResponse(BaseResponse[Dict[str, List[Union[SearchResult, Document]]]]):
+ """Response from search operation."""
+ pass
+
+# Error types
+class ErrorDetails(BaseModel):
+ """Detailed error information."""
+ code: Optional[str] = None
+ message: str
+ details: Optional[Dict[str, Any]] = None
+
+class ErrorResponse(BaseModel):
+ """Error response structure."""
+ success: bool = False
+ error: str
+ details: Optional[ErrorDetails] = None
+
+# Job management types
+class JobStatus(BaseModel):
+ """Generic job status information."""
+ id: str
+ status: Literal["pending", "scraping", "completed", "failed", "cancelled"]
+ current: Optional[int] = None
+ total: Optional[int] = None
+ created_at: Optional[datetime] = Field(None, alias="createdAt")
+ completed_at: Optional[datetime] = Field(None, alias="completedAt")
+ expires_at: Optional[datetime] = Field(None, alias="expiresAt")
+
+# Webhook types
+class WebhookData(BaseModel):
+ """Data sent to webhooks."""
+ job_id: str = Field(alias="jobId")
+ status: str
+ current: Optional[int] = None
+ total: Optional[int] = None
+ data: Optional[List[Document]] = None
+ error: Optional[str] = None
+
+# Configuration types
+class ClientConfig(BaseModel):
+ """Configuration for the Firecrawl client."""
+ api_key: str
+ api_url: str = "https://api.firecrawl.dev"
+ timeout: Optional[float] = None
+ max_retries: int = 3
+ backoff_factor: float = 0.5
+
+# Union types for convenience
+ScrapeResult = Union[Document, List[Document]]
+CrawlResult = Union[CrawlJob, CrawlStatusData]
+BatchResult = Union[BatchScrapeJob, BatchScrapeStatusData]
+JobResult = Union[CrawlJob, BatchScrapeJob]
+StatusResult = Union[CrawlStatusData, BatchScrapeStatusData]
+
+# Response union types
+AnyResponse = Union[
+ ScrapeResponse,
+ CrawlResponse,
+ CrawlStatusResponse,
+ BatchScrapeResponse,
+ BatchScrapeStatusResponse,
+ MapResponse,
+ SearchResponse,
+ ErrorResponse
+]
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/utils/__init__.py b/apps/python-sdk/firecrawl/v2/utils/__init__.py
new file mode 100644
index 000000000..37457ba29
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/utils/__init__.py
@@ -0,0 +1,8 @@
+"""
+Utility modules for v2 API client.
+"""
+
+from .http_client import HttpClient
+from .error_handler import FirecrawlError, handle_response_error
+
+__all__ = ['HttpClient', 'FirecrawlError', 'handle_response_error']
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/utils/error_handler.py b/apps/python-sdk/firecrawl/v2/utils/error_handler.py
new file mode 100644
index 000000000..d96bf7179
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/utils/error_handler.py
@@ -0,0 +1,107 @@
+"""
+Error handling utilities for v2 API.
+"""
+
+import requests
+from typing import Dict, Any, Optional
+
+
+class FirecrawlError(Exception):
+ """Base exception for Firecrawl API errors."""
+
+ def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[requests.Response] = None):
+ super().__init__(message)
+ self.status_code = status_code
+ self.response = response
+
+
+class BadRequestError(FirecrawlError):
+ """Raised when the request is invalid (400)."""
+ pass
+
+
+
+class UnauthorizedError(FirecrawlError):
+ """Raised when the request is unauthorized (401)."""
+ pass
+
+
+class PaymentRequiredError(FirecrawlError):
+ """Raised when payment is required (402)."""
+ pass
+
+
+class WebsiteNotSupportedError(FirecrawlError):
+ """Raised when website is not supported (403)."""
+ pass
+
+
+class RequestTimeoutError(FirecrawlError):
+ """Raised when request times out (408)."""
+ pass
+
+
+class RateLimitError(FirecrawlError):
+ """Raised when the rate limit is exceeded (429)."""
+ pass
+
+
+class InternalServerError(FirecrawlError):
+ """Raised when there's an internal server error (500)."""
+ pass
+
+
+def handle_response_error(response: requests.Response, action: str) -> None:
+ """
+ Handle API response errors and raise appropriate exceptions.
+
+ Args:
+ response: The HTTP response object
+ action: Description of the action being performed
+
+ Raises:
+ FirecrawlError: Appropriate error based on status code
+ """
+ try:
+ response_json = response.json()
+ error_message = response_json.get('error', 'No error message provided.')
+ error_details = response_json.get('details', 'No additional error details provided.')
+ except:
+ # If we can't parse JSON, provide a helpful error message
+ try:
+ response_text = response.text[:500] # Limit to first 500 chars
+ if response_text.strip():
+ error_message = f"Server returned non-JSON response: {response_text}"
+ error_details = f"Full response status: {response.status_code}"
+ else:
+ error_message = f"Server returned empty response with status {response.status_code}"
+ error_details = "No additional details available"
+ except:
+ error_message = f"Server returned unreadable response with status {response.status_code}"
+ error_details = "No additional details available"
+
+ # Create appropriate error message
+ if response.status_code == 400:
+ message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"
+ raise BadRequestError(message, response.status_code, response)
+ elif response.status_code == 401:
+ message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"
+ raise UnauthorizedError(message, response.status_code, response)
+ elif response.status_code == 402:
+ message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
+ raise PaymentRequiredError(message, response.status_code, response)
+ elif response.status_code == 403:
+ message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+ raise WebsiteNotSupportedError(message, response.status_code, response)
+ elif response.status_code == 408:
+ message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
+ raise RequestTimeoutError(message, response.status_code, response)
+ elif response.status_code == 429:
+ message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"
+ raise RateLimitError(message, response.status_code, response)
+ elif response.status_code == 500:
+ message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
+ raise InternalServerError(message, response.status_code, response)
+ else:
+ message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message} - {error_details}"
+ raise FirecrawlError(message, response.status_code, response)
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/utils/get_version.py b/apps/python-sdk/firecrawl/v2/utils/get_version.py
new file mode 100644
index 000000000..e7295ceea
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/utils/get_version.py
@@ -0,0 +1,14 @@
+import os
+import re
+
+def get_version():
+ try:
+ from pathlib import Path
+ package_path = os.path.dirname(__file__)
+ version_file = Path(os.path.join(package_path, '__init__.py')).read_text()
+ version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
+ if version_match:
+ return version_match.group(1).strip()
+ except Exception:
+ print("Failed to get version from __init__.py")
+ return None
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.py
new file mode 100644
index 000000000..787192c29
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py
@@ -0,0 +1,157 @@
+"""
+HTTP client utilities for v2 API.
+"""
+
+import time
+from typing import Dict, Any, Optional
+import requests
+from .get_version import get_version
+
+version = get_version()
+
+class HttpClient:
+ """HTTP client with retry logic and error handling."""
+
+ def __init__(self, api_key: str, api_url: str):
+ self.api_key = api_key
+ self.api_url = api_url
+
+ def _prepare_headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
+ """Prepare headers for API requests."""
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Authorization': f'Bearer {self.api_key}',
+ }
+
+ if idempotency_key:
+ headers['x-idempotency-key'] = idempotency_key
+
+ return headers
+
+ def post(
+ self,
+ endpoint: str,
+ data: Dict[str, Any],
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+ retries: int = 3,
+ backoff_factor: float = 0.5
+ ) -> requests.Response:
+ """Make a POST request with retry logic."""
+ if headers is None:
+ headers = self._prepare_headers()
+
+ data['origin'] = f'python-sdk@{version}'
+
+ url = f"{self.api_url}{endpoint}"
+
+ last_exception = None
+
+ for attempt in range(retries):
+ try:
+ response = requests.post(
+ url,
+ headers=headers,
+ json=data,
+ timeout=timeout
+ )
+
+ if response.status_code == 502:
+ if attempt < retries - 1:
+ time.sleep(backoff_factor * (2 ** attempt))
+ continue
+
+ return response
+
+ except requests.RequestException as e:
+ last_exception = e
+ if attempt == retries - 1:
+ raise e
+ time.sleep(backoff_factor * (2 ** attempt))
+
+ # This should never be reached due to the exception handling above
+ raise last_exception or Exception("Unexpected error in POST request")
+
+ def get(
+ self,
+ endpoint: str,
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+ retries: int = 3,
+ backoff_factor: float = 0.5
+ ) -> requests.Response:
+ """Make a GET request with retry logic."""
+ if headers is None:
+ headers = self._prepare_headers()
+
+ headers['origin'] = f'python-sdk@{version}'
+
+ url = f"{self.api_url}{endpoint}"
+
+ last_exception = None
+
+ for attempt in range(retries):
+ try:
+ response = requests.get(
+ url,
+ headers=headers,
+ timeout=timeout
+ )
+
+ if response.status_code == 502:
+ if attempt < retries - 1:
+ time.sleep(backoff_factor * (2 ** attempt))
+ continue
+
+ return response
+
+ except requests.RequestException as e:
+ last_exception = e
+ if attempt == retries - 1:
+ raise e
+ time.sleep(backoff_factor * (2 ** attempt))
+
+ # This should never be reached due to the exception handling above
+ raise last_exception or Exception("Unexpected error in GET request")
+
+ def delete(
+ self,
+ endpoint: str,
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+ retries: int = 3,
+ backoff_factor: float = 0.5
+ ) -> requests.Response:
+ """Make a DELETE request with retry logic."""
+ if headers is None:
+ headers = self._prepare_headers()
+
+ headers['origin'] = f'python-sdk@{version}'
+
+ url = f"{self.api_url}{endpoint}"
+
+ last_exception = None
+
+ for attempt in range(retries):
+ try:
+ response = requests.delete(
+ url,
+ headers=headers,
+ timeout=timeout
+ )
+
+ if response.status_code == 502:
+ if attempt < retries - 1:
+ time.sleep(backoff_factor * (2 ** attempt))
+ continue
+
+ return response
+
+ except requests.RequestException as e:
+ last_exception = e
+ if attempt == retries - 1:
+ raise e
+ time.sleep(backoff_factor * (2 ** attempt))
+
+ # This should never be reached due to the exception handling above
+ raise last_exception or Exception("Unexpected error in DELETE request")
\ No newline at end of file