Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk): wip - base structure and search endpoint
rafaelmmillerf22fba62959a
diff --git a/apps/python-sdk/.env.example b/apps/python-sdk/.env.examplenew file mode 100644index 000000000..7bea7e7f2--- /dev/null+++ b/apps/python-sdk/.env.example@@ -0,0 +1 @@+FIRECRAWL_API_KEY=<your api key>\ No newline at end of filediff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex 9b28e17e7..76535654e 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -1,141 +1,35 @@-from firecrawl import JsonConfig, FirecrawlApp-from pydantic import BaseModel, Field-from typing import List-import time-app = FirecrawlApp(api_url="https://api.firecrawl.dev")+#!/usr/bin/env python3+"""+Example demonstrating the v2 search functionality with individual parameters.+"""-# Scrape a website:-scrape_result = app.scrape_url('example.com', formats=["markdown", "html"])-print(scrape_result.markdown)+import os+from dotenv import load_dotenv+from firecrawl import Firecrawl+from firecrawl.v2.types import ScrapeOptions, ScrapeFormats-# # # Test batch scrape-urls = ['https://example.com', 'https://docs.firecrawl.dev']-# # Synchronous batch scrape-batch_result = app.batch_scrape_urls(urls, formats=["markdown", "html"])-print("Synchronous Batch Scrape Result:")-print(batch_result.data[0].markdown)+load_dotenv()-# # # Asynchronous batch scrape-async_batch_result = app.async_batch_scrape_urls(urls, formats=["markdown", "html"])-print("\nAsynchronous Batch Scrape Result:")-print(async_batch_result)+def main():+ api_key = os.getenv("FIRECRAWL_API_KEY")+ if not api_key:+ raise ValueError("FIRECRAWL_API_KEY is not set")-# # Crawl a website:-crawl_result = app.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])-print(crawl_result.data[0].markdown)+ firecrawl = Firecrawl(api_key=api_key)-# # Asynchronous Crawl a website:-async_result = app.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])-print(async_result)+ # search examples+ search_response = firecrawl.search(+ query="What is the capital of France?",+ sources=[+ { type: "web" },+ { type: "news" },+ { type: "images" }+ ],+ limit=10)+ print(search_response)-crawl_status = app.check_crawl_status(async_result.id)-print(crawl_status)-attempts = 15-while attempts > 0 and crawl_status.status != 'completed':- print(crawl_status)- crawl_status = app.check_crawl_status(async_result.id)- attempts -= 1- time.sleep(1)-crawl_status = app.check_crawl_status(async_result.id)-print(crawl_status)--# JSON format:-# Define schema to extract contents into using json schema-json_schema = {- "type": "object",- "properties": {- "top": {- "type": "array",- "items": {- "type": "object",- "properties": {- "title": {"type": "string"},- "points": {"type": "number"},- "by": {"type": "string"},- "commentsURL": {"type": "string"}- },- "required": ["title", "points", "by", "commentsURL"]- },- "description": "Top 5 stories on Hacker News"- }- },- "required": ["top"]-}--extract_config = JsonConfig(schema=json_schema)-llm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=["json"], json_options=extract_config)--print(llm_extraction_result.json)--# Map a website:-map_result = app.map_url('https://firecrawl.dev', search="blog")-print(map_result)--# Extract URLs:-class ExtractSchema(BaseModel):- title: str- description: str- links: List[str]--# Define the schema using Pydantic-extract_schema = ExtractSchema.schema()--# Perform the extraction-extract_result = app.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)-print(extract_result)---# Deep research example-research_result = app.deep_research(- "What are the latest developments in large language models?",- max_urls=4-)-print("Research Results:", research_result)--# Generate LLMs.txt example-llms_result = app.generate_llms_text(- "https://firecrawl.dev")-print("LLMs.txt Results:", llms_result)---# Crawl a website with WebSockets:-# inside an async function...-import nest_asyncio-nest_asyncio.apply()--# Define event handlers-def on_document(detail):- print("DOC", detail)--def on_error(detail):- print("ERR", detail['error'])--def on_done(detail):- print("DONE", detail['status'])-- # Function to start the crawl and watch process-async def start_crawl_and_watch():- # Initiate the crawl job and get the watcher- watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })-- # Add event listeners- watcher.add_event_listener("document", on_document)- watcher.add_event_listener("error", on_error)- watcher.add_event_listener("done", on_done)-- # Start the watcher- await watcher.connect()---class ExtractSchema(BaseModel):- company_mission: str- supports_sso: bool- is_open_source: bool- is_in_yc: bool--extract_config = JsonConfig(schema=ExtractSchema.model_json_schema())-data = app.scrape_url('https://docs.firecrawl.dev/', formats=['json'], json_options=extract_config)-print(data.json)\ No newline at end of file+if __name__ == "__main__":+ main()\ No newline at end of filediff --git a/apps/python-sdk/example_async.py b/apps/python-sdk/example_async.pydeleted file mode 100644index 1ad022440..000000000--- a/apps/python-sdk/example_async.py+++ /dev/null@@ -1,136 +0,0 @@-import time-import nest_asyncio-import uuid-import asyncio-from firecrawl.firecrawl import AsyncFirecrawlApp, ScrapeOptions, JsonConfig-from pydantic import BaseModel, Field-from typing import List--app = AsyncFirecrawlApp(api_url="https://api.firecrawl.dev")--async def example_scrape():- # Scrape a website:- scrape_result = await app.scrape_url('example.com', formats=["markdown", "html"])- print(scrape_result.markdown)--async def example_batch_scrape():- # Batch scrape- urls = ['https://example.com', 'https://docs.firecrawl.dev']-- # Synchronous batch scrape- batch_result = await app.batch_scrape_urls(urls, formats=["markdown", "html"])- print("Synchronous Batch Scrape Result:")- print(batch_result.data[0].markdown)-- # Asynchronous batch scrape- async_batch_result = await app.async_batch_scrape_urls(urls, formats=["markdown", "html"])- print("\nAsynchronous Batch Scrape Result:")- print(async_batch_result)--async def example_crawl():- # Crawl a website:- crawl_result = await app.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])- print(crawl_result.data[0].markdown)-- # Asynchronous Crawl a website:- async_result = await app.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])- print(async_result)-- crawl_status = await app.check_crawl_status(async_result.id)- print(crawl_status)-- attempts = 15- while attempts > 0 and crawl_status.status != 'completed':- print(crawl_status)- crawl_status = await app.check_crawl_status(async_result.id)- attempts -= 1- await asyncio.sleep(1) # Use async sleep instead of time.sleep-- crawl_status = await app.check_crawl_status(async_result.id)- print(crawl_status)--async def example_llm_extraction():- # Define schema to extract contents into using pydantic- class ArticleSchema(BaseModel):- title: str- points: int- by: str- commentsURL: str-- class TopArticlesSchema(BaseModel):- top: List[ArticleSchema] = Field(..., description="Top 5 stories")-- extract_config = JsonConfig(schema=TopArticlesSchema.model_json_schema())-- llm_extraction_result = await app.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)-- print(llm_extraction_result.extract)--async def example_map_and_extract():- # Map a website:- map_result = await app.map_url('https://firecrawl.dev', search="blog")- print(map_result)-- # Extract URLs:- class ExtractSchema(BaseModel):- title: str- description: str- links: List[str]-- # Define the schema using Pydantic- extract_schema = ExtractSchema.schema()-- # Perform the extraction- extract_result = await app.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)- print(extract_result)--async def example_deep_research():- # Deep research example- research_result = await app.deep_research(- "What are the latest developments in large language models?",- max_urls=4- )- print("Research Results:", research_result)--async def example_generate_llms_text():- # Generate LLMs.txt example- llms_result = await app.generate_llms_text(- "https://firecrawl.dev")- print("LLMs.txt Results:", llms_result)--# Define event handlers for websocket-def on_document(detail):- print("DOC", detail)--def on_error(detail):- print("ERR", detail['error'])--def on_done(detail):- print("DONE", detail['status'])--async def example_websocket_crawl():- # Initiate the crawl job and get the watcher- watcher = await app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })-- # Add event listeners- watcher.add_event_listener("document", on_document)- watcher.add_event_listener("error", on_error)- watcher.add_event_listener("done", on_done)-- # Start the watcher- await watcher.connect()--async def main():- nest_asyncio.apply()-- await example_scrape()- await example_batch_scrape()- await example_crawl()- await example_llm_extraction()- await example_map_and_extract()- await example_websocket_crawl()- await example_deep_research()- await example_generate_llms_text()--if __name__ == "__main__":- asyncio.run(main())diff --git a/apps/python-sdk/example_v1.py b/apps/python-sdk/example_v1.pynew file mode 100644index 000000000..70a98b467--- /dev/null+++ b/apps/python-sdk/example_v1.py@@ -0,0 +1,286 @@+from firecrawl import V1JsonConfig, FirecrawlApp, AsyncFirecrawlApp, V1ScrapeOptions, V1JsonConfig+from pydantic import BaseModel, Field+from typing import List+import time+import os+from dotenv import load_dotenv++# -- Async example imports+import nest_asyncio+import asyncio++# Load environment variables from .env file+load_dotenv()++# Get API key from environment+api_key = os.getenv("TEST_API_KEY")+api_url = os.getenv("TEST_API_URL", "https://api.firecrawl.dev")++app = FirecrawlApp(api_key=api_key, api_url=api_url)++# Scrape a website:+scrape_result = app.v1.scrape_url('example.com', formats=["markdown", "html"])+print(scrape_result.markdown)++# # # Test batch scrape+urls = ['https://example.com', 'https://docs.firecrawl.dev']+# # Synchronous batch scrape+batch_result = app.v1.batch_scrape_urls(urls, formats=["markdown", "html"])+print("Synchronous Batch Scrape Result:")+print(batch_result.data[0].markdown)++# # # Asynchronous batch scrape+async_batch_result = app.v1.async_batch_scrape_urls(urls, formats=["markdown", "html"])+print("\nAsynchronous Batch Scrape Result:")+print(async_batch_result)++# # Crawl a website:+crawl_result = app.v1.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])+print(crawl_result.data[0].markdown)++# # Asynchronous Crawl a website:+async_result = app.v1.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])+print(async_result)++crawl_status = app.v1.check_crawl_status(async_result.id)+print(crawl_status)++attempts = 15+while attempts > 0 and crawl_status.status != 'completed':+ print(crawl_status)+ crawl_status = app.v1.check_crawl_status(async_result.id)+ attempts -= 1+ time.sleep(1)++crawl_status = app.v1.check_crawl_status(async_result.id)+print(crawl_status)++# JSON format:+# Define schema to extract contents into using json schema+json_schema = {+ "type": "object",+ "properties": {+ "top": {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "title": {"type": "string"},+ "points": {"type": "number"},+ "by": {"type": "string"},+ "commentsURL": {"type": "string"}+ },+ "required": ["title", "points", "by", "commentsURL"]+ },+ "description": "Top 5 stories on Hacker News"+ }+ },+ "required": ["top"]+}++extract_config = V1JsonConfig(schema=json_schema)+llm_extraction_result = app.v1.scrape_url('https://news.ycombinator.com', formats=["json"], json_options=extract_config)++print(llm_extraction_result.json)++# Map a website:+map_result = app.v1.map_url('https://firecrawl.dev', search="blog")+print(map_result)++# Extract URLs:+class ExtractSchema(BaseModel):+ title: str+ description: str+ links: List[str]++# Define the schema using Pydantic+extract_schema = ExtractSchema.schema()++# Perform the extraction+extract_result = app.v1.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)+print(extract_result)+++# Deep research example+research_result = app.v1.deep_research(+ "What are the latest developments in large language models?",+ max_urls=4+)+print("Research Results:", research_result)++# Generate LLMs.txt example+llms_result = app.v1.generate_llms_text(+ "https://firecrawl.dev")+print("LLMs.txt Results:", llms_result)+++# Crawl a website with WebSockets:+# inside an async function...+import nest_asyncio+nest_asyncio.apply()++# Define event handlers+def on_document(detail):+ print("DOC", detail)++def on_error(detail):+ print("ERR", detail['error'])++def on_done(detail):+ print("DONE", detail['status'])++ # Function to start the crawl and watch process+async def start_crawl_and_watch():+ # Initiate the crawl job and get the watcher+ watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })++ # Add event listeners+ watcher.add_event_listener("document", on_document)+ watcher.add_event_listener("error", on_error)+ watcher.add_event_listener("done", on_done)++ # Start the watcher+ await watcher.connect()+++class ExtractSchema(BaseModel):+ company_mission: str+ supports_sso: bool+ is_open_source: bool+ is_in_yc: bool++extract_config = V1JsonConfig(schema=ExtractSchema.model_json_schema())+data = app.scrape_url('https://docs.firecrawl.dev/', formats=['json'], json_options=extract_config)+print(data.json)++# --- Async++async_app = AsyncFirecrawlApp(api_url="https://api.firecrawl.dev")++async def example_scrape():+ # Scrape a website:+ scrape_result = await async_app.v1.scrape_url('example.com', formats=["markdown", "html"])+ print(scrape_result.markdown)++async def example_batch_scrape():+ # Batch scrape+ urls = ['https://example.com', 'https://docs.firecrawl.dev']++ # Synchronous batch scrape+ batch_result = await async_app.v1.batch_scrape_urls(urls, formats=["markdown", "html"])+ print("Synchronous Batch Scrape Result:")+ print(batch_result.data[0].markdown)++ # Asynchronous batch scrape+ async_batch_result = await async_app.v1.async_batch_scrape_urls(urls, formats=["markdown", "html"])+ print("\nAsynchronous Batch Scrape Result:")+ print(async_batch_result)++async def example_crawl():+ # Crawl a website:+ crawl_result = await async_app.v1.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])+ print(crawl_result.data[0].markdown)++ # Asynchronous Crawl a website:+ async_result = await async_app.v1.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])+ print(async_result)++ crawl_status = await async_app.v1.check_crawl_status(async_result.id)+ print(crawl_status)++ attempts = 15+ while attempts > 0 and crawl_status.status != 'completed':+ print(crawl_status)+ crawl_status = await async_app.v1.check_crawl_status(async_result.id)+ attempts -= 1+ await asyncio.sleep(1) # Use async sleep instead of time.sleep++ crawl_status = await async_app.v1.check_crawl_status(async_result.id)+ print(crawl_status)++async def example_llm_extraction():+ # Define schema to extract contents into using pydantic+ class ArticleSchema(BaseModel):+ title: str+ points: int+ by: str+ commentsURL: str++ class TopArticlesSchema(BaseModel):+ top: List[ArticleSchema] = Field(..., description="Top 5 stories")++ extract_config = V1JsonConfig(schema=TopArticlesSchema.model_json_schema())++ llm_extraction_result = await async_app.v1.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)++ print(llm_extraction_result.extract)++async def example_map_and_extract():+ # Map a website:+ map_result = await async_app.v1.map_url('https://firecrawl.dev', search="blog")+ print(map_result)++ # Extract URLs:+ class ExtractSchema(BaseModel):+ title: str+ description: str+ links: List[str]++ # Define the schema using Pydantic+ extract_schema = ExtractSchema.schema()++ # Perform the extraction+ extract_result = await async_app.v1.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)+ print(extract_result)++async def example_deep_research():+ # Deep research example+ research_result = await async_app.v1.deep_research(+ "What are the latest developments in large language models?",+ max_urls=4+ )+ print("Research Results:", research_result)++async def example_generate_llms_text():+ # Generate LLMs.txt example+ llms_result = await async_app.v1.generate_llms_text(+ "https://firecrawl.dev")+ print("LLMs.txt Results:", llms_result)++# Define event handlers for websocket+def on_document(detail):+ print("DOC", detail)++def on_error(detail):+ print("ERR", detail['error'])++def on_done(detail):+ print("DONE", detail['status'])++async def example_websocket_crawl():+ # Initiate the crawl job and get the watcher+ watcher = await async_app.v1.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })++ # Add event listeners+ watcher.add_event_listener("document", on_document)+ watcher.add_event_listener("error", on_error)+ watcher.add_event_listener("done", on_done)++ # Start the watcher+ await watcher.connect()++async def main():+ nest_asyncio.apply()++ await example_scrape()+ await example_batch_scrape()+ await example_crawl()+ await example_llm_extraction()+ await example_map_and_extract()+ await example_websocket_crawl()+ await example_deep_research()+ await example_generate_llms_text()++if __name__ == "__main__":+ asyncio.run(main())+diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/DELETE_firecrawl.pysimilarity index 99%rename from apps/python-sdk/firecrawl/firecrawl.pyrename to apps/python-sdk/firecrawl/DELETE_firecrawl.pyindex 1207c3be7..a9c98a502 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/DELETE_firecrawl.py@@ -2345,7 +2345,7 @@ class FirecrawlApp:else:error_message = f"Server returned empty response with status {response.status_code}"error_details = "No additional details available"-+ except ValueError:+ except ValueError:error_message = f"Server returned unreadable response with status {response.status_code}"error_details = "No additional details available"diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 9119facd4..e90565779 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -1,19 +1,15 @@"""-This is the Firecrawl package.+Firecrawl Python SDK-This package provides a Python SDK for interacting with the Firecrawl API.-It includes methods to scrape URLs, perform searches, initiate and monitor crawl jobs,-and check the status of these jobs.--For more information visit https://github.com/firecrawl/"""import loggingimport os-from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa+from .client import Firecrawl, AsyncFirecrawl+from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp, V1JsonConfig-__version__ = "2.16.4"+__version__ = "3.0.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")@@ -27,17 +23,14 @@ def _configure_logger() -> None:format to the firecrawl logger."""try:- # Create the formatterformatter = logging.Formatter("[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",datefmt="%Y-%m-%d %H:%M:%S",)- # Create the console handler and set the formatterconsole_handler = logging.StreamHandler()console_handler.setFormatter(formatter)- # Add the console handler to the firecrawl loggerlogger.addHandler(console_handler)except Exception as e:logger.error("Failed to configure logging: %s", e)@@ -45,20 +38,15 @@ def _configure_logger() -> None:def setup_logging() -> None:"""Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""- # Check if the firecrawl logger already has a handlerif logger.hasHandlers():- return # To prevent duplicate logging+ return- # Check if the FIRECRAWL_LOGGING_LEVEL environment variable is setif not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):- # Attach a no-op handler to prevent warnings about no handlerslogger.addHandler(logging.NullHandler())return- # Attach the console handler to the firecrawl logger_configure_logger()- # Set the logging level based on the FIRECRAWL_LOGGING_LEVEL environment variableif env == "DEBUG":logger.setLevel(logging.DEBUG)elif env == "INFO":@@ -73,7 +61,15 @@ def setup_logging() -> None:logger.setLevel(logging.INFO)logger.warning("Unknown logging level: %s, defaulting to INFO", env)--# Initialize logging configuration when the module is importedsetup_logging()logger.debug("Debugging logger setup")++__all__ = [+ 'Firecrawl',+ 'AsyncFirecrawl',+ 'V1FirecrawlApp',+ 'AsyncV1FirecrawlApp',+ 'V1JsonConfig',+ 'V1ScrapeOptions',+ 'V1ChangeTrackingOptions',+]\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/.env.example b/apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/.env.examplesimilarity index 100%rename from apps/python-sdk/firecrawl/__tests__/e2e_withAuth/.env.examplerename to apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/.env.examplediff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/__init__.py b/apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/__init__.pysimilarity index 100%rename from apps/python-sdk/firecrawl/__tests__/e2e_withAuth/__init__.pyrename to apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/__init__.pydiff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/test.pysimilarity index 100%rename from apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.pyrename to apps/python-sdk/firecrawl/__tests__/DELETE_e2e_withAuth/test.pydiff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/.env.example b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/.env.examplesimilarity index 100%rename from apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/.env.examplerename to apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/.env.examplediff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/__init__.py b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/__init__.pysimilarity index 100%rename from apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/__init__.pyrename to apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/__init__.pydiff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/test.pysimilarity index 97%rename from apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.pyrename to apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/test.pyindex 615655651..586d700d9 100644--- a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py+++ b/apps/python-sdk/firecrawl/__tests__/DELETE_v1/e2e_withAuth/test.py@@ -44,7 +44,7 @@ def test_scrape_url_invalid_api_key():# def test_blocklisted_url():# blocklisted_url = "https://facebook.com/fake-test"-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)# with pytest.raises(Exception) as excinfo:# app.scrape_url(blocklisted_url)# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)@@ -155,7 +155,7 @@ def test_crawl_url_invalid_api_key():assert response is not None# def test_should_return_error_for_blocklisted_url():-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)# blocklisted_url = "https://twitter.com/fake-test"# with pytest.raises(Exception) as excinfo:# app.crawl_url(blocklisted_url)@@ -336,7 +336,7 @@ def test_invalid_api_key_on_map():assert response is not None# def test_blocklisted_url_on_map():-# app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)+# app = V1FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)# blocklisted_url = "https://facebook.com/fake-test"# with pytest.raises(Exception) as excinfo:# app.map_url(blocklisted_url)@@ -455,7 +455,7 @@ def test_search_with_invalid_params():# def test_scrape_url_with_parse_pdf_true():# if TEST_API_KEY:-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)# response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf', parse_pdf=True)# assert response is not None# assert 'markdown' in response@@ -463,7 +463,7 @@ def test_search_with_invalid_params():# def test_scrape_url_with_parse_pdf_false():# if TEST_API_KEY:-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)# response = app.scrape_url('https://arxiv.org/pdf/astro-ph/9301001.pdf', parse_pdf=False)# assert response is not None# assert 'markdown' in response@@ -471,9 +471,9 @@ def test_search_with_invalid_params():# def test_scrape_options_with_parse_pdf():# if TEST_API_KEY:-# from firecrawl.firecrawl import ScrapeOptions-# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)-# scrape_options = ScrapeOptions(parsePDF=False, formats=['markdown'])+# from firecrawl.firecrawl import V1ScrapeOptions+# app = V1FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# scrape_options = V1ScrapeOptions(parsePDF=False, formats=['markdown'])# response = app.search("firecrawl", limit=1, scrape_options=scrape_options)# assert response is not None# assert 'data' in responsediff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.example b/apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.examplenew file mode 100644index 000000000..4e7633804--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.example@@ -0,0 +1,2 @@+API_URL=http://localhost:3002+API_KEY=fc-YOUR_API_KEY\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.pynew file mode 100644index 000000000..d85595397--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py@@ -0,0 +1,86 @@+from firecrawl import Firecrawl+import os+import pytest+from dotenv import load_dotenv+# from pydantic import BaseModel+# from firecrawl.types import ScrapeOptions, JsonFormat, Location, WaitAction, SourceOptions, FormatOptions++load_dotenv()++if not os.getenv("API_KEY"):+ raise ValueError("API_KEY is not set")++if not os.getenv("API_URL"):+ raise ValueError("API_URL is not set")++firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))++def test_search_minimal_request():+ search_response = firecrawl.search(+ query="What is the capital of France?"+ )++ print(f"Success: {search_response.success}")+ print(f"Data: {search_response.data}")+ print(f"Warning: {search_response.warning}")+# @pytest.mark.e2e+# def test_search_images_request():+# search_response = firecrawl.search(+# query="What is the capital of France?",+# sources=[ "images" ],+# limit=5+# )++# print(search_response)++# @pytest.mark.e2e+# def test_search_all_options():+# class Schema(BaseModel):+# name: str+# description: str+# type: str+# required: bool++# search_response = firecrawl.search(+# query="What is the capital of France?",+# sources=[+# { type: "web" },+# { type: "news" },+# { type: "images" }+# ],+# limit=10,+# tbs="",+# location="",+# timeout=60000,+# ignore_invalid_urls=False,+# scrape_options=ScrapeOptions(+# formats=[+# 'markdown',+# JsonFormat(+# schema={},+# prompt=""+# )+# ],+# only_main_content=True,+# # include_tags=["<string>"], # TODO: check this+# # exclude_tags=["<string>"], # TODO: check this+# max_age=0,+# headers={},+# wait_for=0,+# mobile=False,+# skip_tls_verification=False,+# timeout=30000,+# parsers=[ 'pdf' ],+# actions=[+# WaitAction(milliseconds=2000)+# ],+# location=Location(+# country="US",+# languages=["en-US"]+# ),+# remove_base64_images=True,+# block_ads=True,+# proxy="basic",+# store_in_cache=True,+# )+# )\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pynew file mode 100644index 000000000..9e415fc15--- /dev/null+++ b/apps/python-sdk/firecrawl/client.py@@ -0,0 +1,164 @@+"""+Firecrawl Client++A Firecrawl client that enables you to scrape content from websites, crawl entire sites, search the web, and extract structured data using AI.++The client supports both v1 and v2 API versions, providing access to features like:+- Web scraping with advanced options (screenshots, markdown conversion, etc.)+- Site crawling with configurable depth and limits+- Web search with content extraction+- Structured data extraction using AI models+- Deep research capabilities++Usage:+ from firecrawl import Firecrawl+ firecrawl = Firecrawl(api_key="your-api-key")+ result = firecrawl.scrape("https://example.com")++Check example.py for other usage examples.+"""++from typing import Any, Dict, Optional, List, Union+import logging+++from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp+from .v2 import FirecrawlClient as V2FirecrawlClient++logger = logging.getLogger("firecrawl")++class V1Proxy:+ """Type-annotated proxy for v1 client methods."""+ _client: Optional[V1FirecrawlApp]++ def __init__(self, client_instance: Optional[V1FirecrawlApp]):+ self._client = client_instance++ if client_instance:+ self.scrape_url = client_instance.scrape_url+ self.crawl_url = client_instance.crawl_url+ self.batch_scrape_urls = client_instance.batch_scrape_urls+ self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls+ self.async_crawl_url = client_instance.async_crawl_url+ self.check_crawl_status = client_instance.check_crawl_status+ self.map_url = client_instance.map_url+ self.extract = client_instance.extract+ self.deep_research = client_instance.deep_research+ self.generate_llms_text = client_instance.generate_llms_text++class V2Proxy:+ """Proxy class that forwards method calls to the appropriate version client."""+ _client: Optional[V2FirecrawlClient]++ def __init__(self, client_instance: Optional[V2FirecrawlClient]):+ self._client = client_instance++ # TODO: Not implemented yet+ if client_instance:+ self.scrape = client_instance.scrape+ self.search = client_instance.search+ self.crawl = client_instance.crawl+ self.batch_scrape = client_instance.batch_scrape+ self.get_crawl_status = client_instance.get_crawl_status+ self.cancel_crawl = client_instance.cancel_crawl++ def __getattr__(self, name):+ """Forward attribute access to the underlying client."""+ return getattr(self._client, name)++class AsyncV1Proxy:+ """Type-annotated proxy for v1 client methods."""+ _client: Optional[AsyncV1FirecrawlApp]++ def __init__(self, client_instance: Optional[AsyncV1FirecrawlApp]):+ self._client = client_instance++ if client_instance:+ self.scrape_url = client_instance.scrape_url+ self.crawl_url = client_instance.crawl_url+ self.batch_scrape_urls = client_instance.batch_scrape_urls+ self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls+ self.async_crawl_url = client_instance.async_crawl_url+ self.check_crawl_status = client_instance.check_crawl_status+ self.map_url = client_instance.map_url+ self.extract = client_instance.extract+ self.deep_research = client_instance.deep_research+ self.generate_llms_text = client_instance.generate_llms_text++class AsyncV2Proxy:+ """Proxy class that forwards method calls to the appropriate version client."""+ _client: Optional[Any] = None++ def __init__(self, client_instance: Optional[Any] = None):+ self._client = client_instance+ # TODO: Async v2 client not implemented yet++ def __getattr__(self, name):+ """Forward attribute access to the underlying client."""+ if self._client:+ return getattr(self._client, name)+ raise AttributeError(f"Async v2 client not implemented yet: {name}")+++class Firecrawl:+ """+ Unified Firecrawl client with version-specific access patterns.++ This class provides a unified interface that allows users to access+ different API versions through version-specific attributes (v1, v2)+ while defaulting to v2 for direct method calls.+ """++ def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):+ """+ Initialize the unified Firecrawl client.++ Args:+ api_key: Your Firecrawl API key+ api_url: The Firecrawl API URL (defaults to production)+ """+ self.api_key = api_key+ self.api_url = api_url++ # Initialize version-specific clients+ self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None+ self._v2_client = V2FirecrawlClient(api_key=api_key, api_url=api_url) if V2FirecrawlClient else None++ # Create version-specific proxies+ self.v1 = V1Proxy(self._v1_client) if self._v1_client else None+ self.v2 = V2Proxy(self._v2_client)++ # Methods+ # TODO: Not implemented yet+ # self.scrape = self._v2_client.scrape+ # self.crawl = self._v2_client.crawl+ # self.batch_scrape = self._v2_client.batch_scrape+ # self.get_crawl_status = self._v2_client.get_crawl_status+ # self.cancel_crawl = self._v2_client.cancel_crawl+ self.search = self._v2_client.search++class AsyncFirecrawl:+ """+ Unified Firecrawl client with version-specific access patterns.++ This class provides a unified interface that allows users to access+ different API versions through version-specific attributes (v1, v2)+ while defaulting to v2 for direct method calls.+ """++ def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):+ self.api_key = api_key+ self.api_url = api_url++ # Initialize version-specific clients+ self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None+ # TODO: Async v2 client not implemented yet+ self._v2_client = None++ # Create version-specific proxies+ self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None+ self.v2 = AsyncV2Proxy(self._v2_client)++# Export Firecrawl as an alias for FirecrawlApp+FirecrawlApp = Firecrawl+AsyncFirecrawlApp = AsyncFirecrawl\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pynew file mode 100644index 000000000..66538f84d--- /dev/null+++ b/apps/python-sdk/firecrawl/types.py@@ -0,0 +1,168 @@+"""+Unified Firecrawl Types++This module provides unified access to Firecrawl types across all API versions.+Currently exports v2 types as the primary interface.+"""++from .v2.types import (+ # Base types+ BaseResponse,++ # Document types+ Document,+ DocumentMetadata,++ # Scrape types+ ScrapeFormats,+ ScrapeOptions,+ ScrapeRequest,+ ScrapeResponse,++ # Crawl types+ CrawlOptions,+ CrawlRequest,+ CrawlJob,+ CrawlResponse,+ CrawlStatusData,+ CrawlStatusResponse,++ # Batch scrape types+ BatchScrapeRequest,+ BatchScrapeJob,+ BatchScrapeResponse,+ BatchScrapeStatusData,+ BatchScrapeStatusResponse,++ # Map types+ MapOptions,+ MapRequest,+ MapData,+ MapResponse,++ # Search types+ Source,+ SourceOption,+ Format,+ FormatOption,+ SearchRequest,+ SearchResult,+ SearchResponse,++ # Action types+ WaitAction,+ ScreenshotAction,+ ClickAction,+ WriteAction,+ PressAction,+ ScrollAction,+ ScrapeAction,+ ExecuteJavascriptAction,+ PDFAction,++ # Location and format types+ Location,+ JsonFormat,++ # Error types+ ErrorDetails,+ ErrorResponse,++ # Job management types+ JobStatus,++ # Webhook types+ WebhookData,++ # Configuration types+ ClientConfig,++ # Union types+ ScrapeResult,+ CrawlResult,+ BatchResult,+ JobResult,+ StatusResult,+ AnyResponse,+)++__all__ = [+ # Base types+ 'BaseResponse',++ # Document types+ 'Document',+ 'DocumentMetadata',++ # Scrape types+ 'ScrapeFormats',+ 'ScrapeOptions',+ 'ScrapeRequest',+ 'ScrapeResponse',++ # Crawl types+ 'CrawlOptions',+ 'CrawlRequest',+ 'CrawlJob',+ 'CrawlResponse',+ 'CrawlStatusData',+ 'CrawlStatusResponse',++ # Batch scrape types+ 'BatchScrapeRequest',+ 'BatchScrapeJob',+ 'BatchScrapeResponse',+ 'BatchScrapeStatusData',+ 'BatchScrapeStatusResponse',++ # Map types+ 'MapOptions',+ 'MapRequest',+ 'MapData',+ 'MapResponse',++ # Search types+ 'Source',+ 'SourceOption',+ 'Format',+ 'FormatOption',+ 'SearchRequest',+ 'SearchResult',+ 'SearchResponse',++ # Action types+ 'WaitAction',+ 'ScreenshotAction',+ 'ClickAction',+ 'WriteAction',+ 'PressAction',+ 'ScrollAction',+ 'ScrapeAction',+ 'ExecuteJavascriptAction',+ 'PDFAction',++ # Location and format types+ 'Location',+ 'JsonFormat',++ # Error types+ 'ErrorDetails',+ 'ErrorResponse',++ # Job management types+ 'JobStatus',++ # Webhook types+ 'WebhookData',++ # Configuration types+ 'ClientConfig',++ # Union types+ 'ScrapeResult',+ 'CrawlResult',+ 'BatchResult',+ 'JobResult',+ 'StatusResult',+ 'AnyResponse',+]\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v1/__init__.py b/apps/python-sdk/firecrawl/v1/__init__.pynew file mode 100644index 000000000..6c831cc46--- /dev/null+++ b/apps/python-sdk/firecrawl/v1/__init__.py@@ -0,0 +1,14 @@+"""+Firecrawl v1 API (Legacy)++This module provides the legacy v1 API for backward compatibility.++Usage:+ from firecrawl.v1 import V1FirecrawlApp+ app = V1FirecrawlApp(api_key="your-api-key")+ result = app.scrape_url("https://example.com")+"""++from .client import V1FirecrawlApp, AsyncV1FirecrawlApp, V1JsonConfig, V1ScrapeOptions, V1ChangeTrackingOptions++__all__ = ['V1FirecrawlApp', 'AsyncV1FirecrawlApp', 'V1JsonConfig', 'V1ScrapeOptions', 'V1ChangeTrackingOptions']\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pynew file mode 100644index 000000000..a48c5f5f5--- /dev/null+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -0,0 +1,4645 @@+"""+Firecrawl v1 API Client - Legacy Implementation++This module provides the legacy v1 implementation of the Firecrawl SDK.+It contains the complete `V1FirecrawlApp` class with all v1 API methods and types+for backward compatibility. This is used by the unified client to provide+version-specific access patterns like app.v1.scrape_url().++Classes:+ - V1FirecrawlApp: Legacy v1 client for interacting with the Firecrawl API.+ - AsyncV1FirecrawlApp: Async version of the v1 client.+ - CrawlWatcher: WebSocket-based crawl monitoring for v1.+"""+import logging+import os+import time+from typing import Any, Dict, Optional, List, Union, Callable, Literal, TypeVar, Generic+import json+from datetime import datetime+import re+import requests+import pydantic+import websockets+import aiohttp+import asyncio++logger : logging.Logger = logging.getLogger("firecrawl")++def get_version():+ try:+ from pathlib import Path+ package_path = os.path.dirname(__file__)+ version_file = Path(os.path.join(package_path, '__init__.py')).read_text()+ version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)+ if version_match:+ return version_match.group(1).strip()+ except Exception:+ print("Failed to get version from __init__.py")+ return None++version = get_version()++T = TypeVar('T')++# class V1FirecrawlDocumentMetadata(pydantic.BaseModel):+# """Metadata for a Firecrawl document."""+# title: Optional[str] = None+# description: Optional[str] = None+# language: Optional[str] = None+# keywords: Optional[str] = None+# robots: Optional[str] = None+# ogTitle: Optional[str] = None+# ogDescription: Optional[str] = None+# ogUrl: Optional[str] = None+# ogImage: Optional[str] = None+# ogAudio: Optional[str] = None+# ogDeterminer: Optional[str] = None+# ogLocale: Optional[str] = None+# ogLocaleAlternate: Optional[List[str]] = None+# ogSiteName: Optional[str] = None+# ogVideo: Optional[str] = None+# dctermsCreated: Optional[str] = None+# dcDateCreated: Optional[str] = None+# dcDate: Optional[str] = None+# dctermsType: Optional[str] = None+# dcType: Optional[str] = None+# dctermsAudience: Optional[str] = None+# dctermsSubject: Optional[str] = None+# dcSubject: Optional[str] = None+# dcDescription: Optional[str] = None+# dctermsKeywords: Optional[str] = None+# modifiedTime: Optional[str] = None+# publishedTime: Optional[str] = None+# articleTag: Optional[str] = None+# articleSection: Optional[str] = None+# sourceURL: Optional[str] = None+# statusCode: Optional[int] = None+# error: Optional[str] = None++class V1AgentOptions(pydantic.BaseModel):+ """Configuration for the agent."""+ model: Literal["FIRE-1"] = "FIRE-1"+ prompt: Optional[str] = None++class V1AgentOptionsExtract(pydantic.BaseModel):+ """Configuration for the agent in extract operations."""+ model: Literal["FIRE-1"] = "FIRE-1"++class V1ActionsResult(pydantic.BaseModel):+ """Result of actions performed during scraping."""+ screenshots: List[str]+ pdfs: List[str]++class V1ChangeTrackingData(pydantic.BaseModel):+ """+ Data for the change tracking format.+ """+ previousScrapeAt: Optional[str] = None+ changeStatus: str # "new" | "same" | "changed" | "removed"+ visibility: str # "visible" | "hidden"+ diff: Optional[Dict[str, Any]] = None+ json_field: Optional[Any] = pydantic.Field(None, alias='json')++class V1FirecrawlDocument(pydantic.BaseModel, Generic[T]):+ """Document retrieved or processed by Firecrawl."""+ url: Optional[str] = None+ markdown: Optional[str] = None+ html: Optional[str] = None+ rawHtml: Optional[str] = None+ links: Optional[List[str]] = None+ extract: Optional[T] = None+ json_field: Optional[T] = pydantic.Field(None, alias='json')+ screenshot: Optional[str] = None+ metadata: Optional[Any] = None+ actions: Optional[V1ActionsResult] = None+ title: Optional[str] = None # v1 search only+ description: Optional[str] = None # v1 search only+ changeTracking: Optional[V1ChangeTrackingData] = None++class V1LocationConfig(pydantic.BaseModel):+ """Location configuration for scraping."""+ country: Optional[str] = None+ languages: Optional[List[str]] = None++class V1WebhookConfig(pydantic.BaseModel):+ """Configuration for webhooks."""+ url: str+ headers: Optional[Dict[str, str]] = None+ metadata: Optional[Dict[str, str]] = None+ events: Optional[List[Literal["completed", "failed", "page", "started"]]] = None++class V1ChangeTrackingOptions(pydantic.BaseModel):+ """Configuration for change tracking."""+ modes: Optional[List[Literal["git-diff", "json"]]] = None+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')+ prompt: Optional[str] = None+ tag: Optional[str] = None++class V1ScrapeOptions(pydantic.BaseModel):+ """Parameters for scraping operations."""+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None+ headers: Optional[Dict[str, str]] = None+ includeTags: Optional[List[str]] = None+ excludeTags: Optional[List[str]] = None+ onlyMainContent: Optional[bool] = None+ waitFor: Optional[int] = None+ timeout: Optional[int] = 30000+ location: Optional[V1LocationConfig] = None+ mobile: Optional[bool] = None+ skipTlsVerification: Optional[bool] = None+ removeBase64Images: Optional[bool] = None+ blockAds: Optional[bool] = None+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None+ changeTrackingOptions: Optional[V1ChangeTrackingOptions] = None+ maxAge: Optional[int] = None+ storeInCache: Optional[bool] = None+ parsePDF: Optional[bool] = None++class V1WaitAction(pydantic.BaseModel):+ """Wait action to perform during scraping."""+ type: Literal["wait"]+ milliseconds: Optional[int] = None+ selector: Optional[str] = None++class V1ScreenshotAction(pydantic.BaseModel):+ """Screenshot action to perform during scraping."""+ type: Literal["screenshot"]+ fullPage: Optional[bool] = None+ quality: Optional[int] = None++class V1ClickAction(pydantic.BaseModel):+ """Click action to perform during scraping."""+ type: Literal["click"]+ selector: str++class V1WriteAction(pydantic.BaseModel):+ """Write action to perform during scraping."""+ type: Literal["write"]+ text: str++class V1PressAction(pydantic.BaseModel):+ """Press action to perform during scraping."""+ type: Literal["press"]+ key: str++class V1ScrollAction(pydantic.BaseModel):+ """Scroll action to perform during scraping."""+ type: Literal["scroll"]+ direction: Literal["up", "down"]+ selector: Optional[str] = None++class V1ScrapeAction(pydantic.BaseModel):+ """Scrape action to perform during scraping."""+ type: Literal["scrape"]++class V1ExecuteJavascriptAction(pydantic.BaseModel):+ """Execute javascript action to perform during scraping."""+ type: Literal["executeJavascript"]+ script: str++class V1PDFAction(pydantic.BaseModel):+ """PDF action to perform during scraping."""+ type: Literal["pdf"]+ format: Optional[Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"]] = None+ landscape: Optional[bool] = None+ scale: Optional[float] = None++class V1ExtractAgent(pydantic.BaseModel):+ """Configuration for the agent in extract operations."""+ model: Literal["FIRE-1"] = "FIRE-1"++class V1JsonConfig(pydantic.BaseModel):+ """Configuration for extraction."""+ prompt: Optional[str] = None+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')+ systemPrompt: Optional[str] = None+ agent: Optional[V1ExtractAgent] = None++class V1ScrapeParams(V1ScrapeOptions):+ """Parameters for scraping operations."""+ extract: Optional[V1JsonConfig] = None+ jsonOptions: Optional[V1JsonConfig] = None+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None+ agent: Optional[V1AgentOptions] = None+ webhook: Optional[V1WebhookConfig] = None++class V1ScrapeResponse(V1FirecrawlDocument[T], Generic[T]):+ """Response from scraping operations."""+ success: bool = True+ warning: Optional[str] = None+ error: Optional[str] = None++class V1BatchScrapeResponse(pydantic.BaseModel):+ """Response from batch scrape operations."""+ id: Optional[str] = None+ url: Optional[str] = None+ success: bool = True+ error: Optional[str] = None+ invalidURLs: Optional[List[str]] = None++class V1BatchScrapeStatusResponse(pydantic.BaseModel):+ """Response from batch scrape status checks."""+ success: bool = True+ status: Literal["scraping", "completed", "failed", "cancelled"]+ completed: int+ total: int+ creditsUsed: int+ expiresAt: datetime+ next: Optional[str] = None+ data: List[V1FirecrawlDocument]++class V1CrawlParams(pydantic.BaseModel):+ """Parameters for crawling operations."""+ includePaths: Optional[List[str]] = None+ excludePaths: Optional[List[str]] = None+ maxDepth: Optional[int] = None+ maxDiscoveryDepth: Optional[int] = None+ limit: Optional[int] = None+ allowBackwardLinks: Optional[bool] = None+ crawlEntireDomain: Optional[bool] = None+ allowExternalLinks: Optional[bool] = None+ ignoreSitemap: Optional[bool] = None+ scrapeOptions: Optional[V1ScrapeOptions] = None+ webhook: Optional[Union[str, V1WebhookConfig]] = None+ deduplicateSimilarURLs: Optional[bool] = None+ ignoreQueryParameters: Optional[bool] = None+ regexOnFullURL: Optional[bool] = None+ delay: Optional[int] = None # Delay in seconds between scrapes+ maxConcurrency: Optional[int] = None+ allowSubdomains: Optional[bool] = None++class V1CrawlResponse(pydantic.BaseModel):+ """Response from crawling operations."""+ id: Optional[str] = None+ url: Optional[str] = None+ success: bool = True+ error: Optional[str] = None++class V1CrawlStatusResponse(pydantic.BaseModel):+ """Response from crawl status checks."""+ success: bool = True+ status: Literal["scraping", "completed", "failed", "cancelled"]+ completed: int+ total: int+ creditsUsed: int+ expiresAt: datetime+ next: Optional[str] = None+ data: List[V1FirecrawlDocument]++class V1CrawlErrorsResponse(pydantic.BaseModel):+ """Response from crawl/batch scrape error monitoring."""+ errors: List[Dict[str, str]] # {id: str, timestamp: str, url: str, error: str}+ robotsBlocked: List[str]++class V1MapParams(pydantic.BaseModel):+ """Parameters for mapping operations."""+ search: Optional[str] = None+ ignoreSitemap: Optional[bool] = None+ includeSubdomains: Optional[bool] = None+ sitemapOnly: Optional[bool] = None+ limit: Optional[int] = None+ timeout: Optional[int] = 30000+ useIndex: Optional[bool] = None++class V1MapResponse(pydantic.BaseModel):+ """Response from mapping operations."""+ success: bool = True+ links: Optional[List[str]] = None+ error: Optional[str] = None++class V1ExtractParams(pydantic.BaseModel):+ """Parameters for extracting information from URLs."""+ prompt: Optional[str] = None+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')+ systemPrompt: Optional[str] = None+ allowExternalLinks: Optional[bool] = None+ enableWebSearch: Optional[bool] = None+ includeSubdomains: Optional[bool] = None+ origin: Optional[str] = None+ showSources: Optional[bool] = None+ scrapeOptions: Optional[V1ScrapeOptions] = None++class V1ExtractResponse(pydantic.BaseModel, Generic[T]):+ """Response from extract operations."""+ id: Optional[str] = None+ status: Optional[Literal["processing", "completed", "failed"]] = None+ expiresAt: Optional[datetime] = None+ success: bool = True+ data: Optional[T] = None+ error: Optional[str] = None+ warning: Optional[str] = None+ sources: Optional[Dict[Any, Any]] = None++class V1SearchParams(pydantic.BaseModel):+ query: str+ limit: Optional[int] = 5+ tbs: Optional[str] = None+ filter: Optional[str] = None+ lang: Optional[str] = "en"+ country: Optional[str] = "us"+ location: Optional[str] = None+ origin: Optional[str] = "api"+ timeout: Optional[int] = 60000+ scrapeOptions: Optional[V1ScrapeOptions] = None++class V1SearchResponse(pydantic.BaseModel):+ """Response from search operations."""+ success: bool = True+ data: List[V1FirecrawlDocument]+ warning: Optional[str] = None+ error: Optional[str] = None++class V1GenerateLLMsTextParams(pydantic.BaseModel):+ """+ Parameters for the LLMs.txt generation operation.+ """+ maxUrls: Optional[int] = 10+ showFullText: Optional[bool] = False+ cache: Optional[bool] = True+ __experimental_stream: Optional[bool] = None++class V1DeepResearchParams(pydantic.BaseModel):+ """+ Parameters for the deep research operation.+ """+ maxDepth: Optional[int] = 7+ timeLimit: Optional[int] = 270+ maxUrls: Optional[int] = 20+ analysisPrompt: Optional[str] = None+ systemPrompt: Optional[str] = None+ __experimental_streamSteps: Optional[bool] = None++class V1DeepResearchResponse(pydantic.BaseModel):+ """+ Response from the deep research operation.+ """+ success: bool+ id: str+ error: Optional[str] = None++class V1DeepResearchStatusResponse(pydantic.BaseModel):+ """+ Status response from the deep research operation.+ """+ success: bool+ data: Optional[Dict[str, Any]] = None+ status: str+ error: Optional[str] = None+ expiresAt: str+ currentDepth: int+ maxDepth: int+ activities: List[Dict[str, Any]]+ sources: List[Dict[str, Any]]+ summaries: List[str]++class V1GenerateLLMsTextResponse(pydantic.BaseModel):+ """Response from LLMs.txt generation operations."""+ success: bool = True+ id: str+ error: Optional[str] = None++class V1GenerateLLMsTextStatusResponseData(pydantic.BaseModel):+ llmstxt: str+ llmsfulltxt: Optional[str] = None++class V1GenerateLLMsTextStatusResponse(pydantic.BaseModel):+ """Status response from LLMs.txt generation operations."""+ success: bool = True+ data: Optional[V1GenerateLLMsTextStatusResponseData] = None+ status: Literal["processing", "completed", "failed"]+ error: Optional[str] = None+ expiresAt: str++class V1SearchResponse(pydantic.BaseModel):+ """+ Response from the search operation.+ """+ success: bool+ data: List[Dict[str, Any]]+ warning: Optional[str] = None+ error: Optional[str] = None++class V1ExtractParams(pydantic.BaseModel):+ """+ Parameters for the extract operation.+ """+ prompt: Optional[str] = None+ schema_field: Optional[Any] = pydantic.Field(None, alias='schema')+ system_prompt: Optional[str] = None+ allow_external_links: Optional[bool] = False+ enable_web_search: Optional[bool] = False+ # Just for backwards compatibility+ enableWebSearch: Optional[bool] = False+ show_sources: Optional[bool] = False+ agent: Optional[Dict[str, Any]] = None++class V1FirecrawlApp:+ """+ Legacy v1 Firecrawl client for backward compatibility.++ This class provides the complete v1 API implementation including:+ - URL scraping with various formats and options+ - Website crawling with monitoring capabilities+ - Batch scraping operations+ - Search functionality+ - Data extraction with LLM integration+ - Deep research capabilities+ - LLMs.txt generation++ This is used by the unified client to provide version-specific access+ through app.v1.method_name() patterns.+ """+ def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:+ """+ Initialize the V1FirecrawlApp instance with API key, API URL.++ Args:+ api_key (Optional[str]): API key for authenticating with the Firecrawl API.+ api_url (Optional[str]): Base URL for the Firecrawl API.+ """+ self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')+ self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')++ # Only require API key when using cloud service+ if 'api.firecrawl.dev' in self.api_url and self.api_key is None:+ logger.warning("No API key provided for cloud service")+ raise ValueError('No API key provided')++ logger.debug(f"Initialized V1FirecrawlApp with API URL: {self.api_url}")++ def scrape_url(+ self,+ url: str,+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ parse_pdf: Optional[bool] = None,+ extract: Optional[V1JsonConfig] = None,+ json_options: Optional[V1JsonConfig] = None,+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,+ change_tracking_options: Optional[V1ChangeTrackingOptions] = None,+ max_age: Optional[int] = None,+ store_in_cache: Optional[bool] = None,+ zero_data_retention: Optional[bool] = None,+ **kwargs) -> V1ScrapeResponse[Any]:+ """+ Scrape and extract content from a URL.++ Args:+ url (str): Target URL to scrape+ formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait for a specific element to appear+ timeout (Optional[int]): Request timeout (ms)+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 images+ block_ads (Optional[bool]): Block ads+ proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)+ extract (Optional[JsonConfig]): Content extraction settings+ json_options (Optional[JsonConfig]): JSON extraction settings+ actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform+ change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings+ zero_data_retention (Optional[bool]): Whether to delete data after scrape is done+++ Returns:+ ScrapeResponse with:+ * Requested content formats+ * Page metadata+ * Extraction results+ * Success/error status++ Raises:+ Exception: If scraping fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "scrape_url")++ _headers = self._prepare_headers()++ # Build scrape parameters+ scrape_params = {+ 'url': url,+ 'origin': f"python-sdk@{version}"+ }++ # Add optional parameters if provided+ if formats:+ scrape_params['formats'] = formats+ if headers:+ scrape_params['headers'] = headers+ if include_tags:+ scrape_params['includeTags'] = include_tags+ if exclude_tags:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for:+ scrape_params['waitFor'] = wait_for+ if timeout:+ scrape_params['timeout'] = timeout+ if location:+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy:+ scrape_params['proxy'] = proxy+ if parse_pdf is not None:+ scrape_params['parsePDF'] = parse_pdf+ if extract is not None:+ extract = self._ensure_schema_dict(extract)+ if isinstance(extract, dict) and "schema" in extract:+ extract["schema"] = self._ensure_schema_dict(extract["schema"])+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)+ if json_options is not None:+ json_options = self._ensure_schema_dict(json_options)+ if isinstance(json_options, dict) and "schema" in json_options:+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions:+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]+ if change_tracking_options:+ scrape_params['changeTrackingOptions'] = change_tracking_options if isinstance(change_tracking_options, dict) else change_tracking_options.dict(by_alias=True, exclude_none=True)+ if max_age is not None:+ scrape_params['maxAge'] = max_age+ if store_in_cache is not None:+ scrape_params['storeInCache'] = store_in_cache+ if zero_data_retention is not None:+ scrape_params['zeroDataRetention'] = zero_data_retention++ scrape_params.update(kwargs)++ if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:+ scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])+ if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:+ scrape_params['jsonOptions']['schema'] = self._ensure_schema_dict(scrape_params['jsonOptions']['schema'])++ # Make request+ response = requests.post(+ f'{self.api_url}/v1/scrape',+ headers=_headers,+ json=scrape_params,+ timeout=(timeout / 1000.0 + 5 if timeout is not None else None)+ )++ if response.status_code == 200:+ try:+ response_json = response.json()+ if response_json.get('success') and 'data' in response_json:+ return V1ScrapeResponse(**response_json['data'])+ elif "error" in response_json:+ raise Exception(f'Failed to scrape URL. Error: {response_json["error"]}')+ else:+ raise Exception(f'Failed to scrape URL. Error: {response_json}')+ except ValueError:+ raise Exception('Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'scrape URL')++ def search(+ self,+ query: str,+ *,+ limit: Optional[int] = None,+ tbs: Optional[str] = None,+ filter: Optional[str] = None,+ lang: Optional[str] = None,+ country: Optional[str] = None,+ location: Optional[str] = None,+ timeout: Optional[int] = 30000,+ scrape_options: Optional[V1ScrapeOptions] = None,+ **kwargs) -> V1SearchResponse:+ """+ Search for content using Firecrawl.++ Args:+ query (str): Search query string+ limit (Optional[int]): Max results (default: 5)+ tbs (Optional[str]): Time filter (e.g. "qdr:d")+ filter (Optional[str]): Custom result filter+ lang (Optional[str]): Language code (default: "en")+ country (Optional[str]): Country code (default: "us")+ location (Optional[str]): Geo-targeting+ timeout (Optional[int]): Request timeout in milliseconds+ scrape_options (Optional[ScrapeOptions]): Result scraping configuration+ **kwargs: Additional keyword arguments for future compatibility++ Returns:+ SearchResponse: Response containing:+ * success (bool): Whether request succeeded+ * data (List[FirecrawlDocument]): Search results+ * warning (Optional[str]): Warning message if any+ * error (Optional[str]): Error message if any++ Raises:+ Exception: If search fails or response cannot be parsed+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "search")++ # Build search parameters+ search_params = {}++ # Add individual parameters+ if limit is not None:+ search_params['limit'] = limit+ if tbs is not None:+ search_params['tbs'] = tbs+ if filter is not None:+ search_params['filter'] = filter+ if lang is not None:+ search_params['lang'] = lang+ if country is not None:+ search_params['country'] = country+ if location is not None:+ search_params['location'] = location+ if timeout is not None:+ search_params['timeout'] = timeout+ if scrape_options is not None:+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)++ # Add any additional kwargs+ search_params.update(kwargs)+ _integration = search_params.get('integration')++ # Create final params object+ final_params = V1SearchParams(query=query, **search_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['origin'] = f"python-sdk@{version}"++ if _integration:+ params_dict['integration'] = _integration++ # Make request+ response = requests.post(+ f"{self.api_url}/v1/search",+ headers={"Authorization": f"Bearer {self.api_key}"},+ json=params_dict+ )++ if response.status_code == 200:+ try:+ response_json = response.json()+ if response_json.get('success') and 'data' in response_json:+ return V1SearchResponse(**response_json)+ elif "error" in response_json:+ raise Exception(f'Search failed. Error: {response_json["error"]}')+ else:+ raise Exception(f'Search failed. Error: {response_json}')+ except ValueError:+ raise Exception('Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'search')++ def crawl_url(+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[V1ScrapeOptions] = None,+ webhook: Optional[Union[str, V1WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,+ max_concurrency: Optional[int] = None,+ zero_data_retention: Optional[bool] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> V1CrawlStatusResponse:+ """+ Crawl a website starting from a URL.++ Args:+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[ScrapeOptions]): Page scraping configuration+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ delay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomains+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ CrawlStatusResponse with:+ * Crawling status and progress+ * Crawled page contents+ * Success/error information++ Raises:+ Exception: If crawl fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "crawl_url")++ crawl_params = {}++ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomains+ if max_concurrency is not None:+ crawl_params['maxConcurrency'] = max_concurrency+ if zero_data_retention is not None:+ crawl_params['zeroDataRetention'] = zero_data_retention+ # Add any additional kwargs+ crawl_params.update(kwargs)+ _integration = crawl_params.get('integration')++ # Create final params object+ final_params = V1CrawlParams(**crawl_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ if _integration:+ params_dict['integration'] = _integration++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = self._post_request(f'{self.api_url}/v1/crawl', params_dict, headers)++ if response.status_code == 200:+ try:+ id = response.json().get('id')+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ return self._monitor_job_status(id, headers, poll_interval)+ else:+ self._handle_error(response, 'start crawl job')++ def async_crawl_url(+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[V1ScrapeOptions] = None,+ webhook: Optional[Union[str, V1WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,+ max_concurrency: Optional[int] = None,+ zero_data_retention: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> V1CrawlResponse:+ """+ Start an asynchronous crawl job.++ Args:+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration+ webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ delay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomains+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1CrawlResponse with:+ * success - Whether crawl started successfully+ * id - Unique identifier for the crawl job+ * url - Status check URL for the crawl+ * error - Error message if start failed++ Raises:+ Exception: If crawl initiation fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "async_crawl_url")++ crawl_params = {}++ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomains+ if max_concurrency is not None:+ crawl_params['maxConcurrency'] = max_concurrency+ if zero_data_retention is not None:+ crawl_params['zeroDataRetention'] = zero_data_retention+ # Add any additional kwargs+ crawl_params.update(kwargs)++ # Create final params object+ final_params = V1CrawlParams(**crawl_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = self._post_request(f'{self.api_url}/v1/crawl', params_dict, headers)++ if response.status_code == 200:+ try:+ return V1CrawlResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'start crawl job')++ def check_crawl_status(self, id: str) -> V1CrawlStatusResponse:+ """+ Check the status and results of a crawl job.++ Args:+ id: Unique identifier for the crawl job++ Returns:+ V1CrawlStatusResponse containing:++ Status Information:+ * status - Current state (scraping/completed/failed/cancelled)+ * completed - Number of pages crawled+ * total - Total pages to crawl+ * creditsUsed - API credits consumed+ * expiresAt - Data expiration timestamp++ Results:+ * data - List of crawled documents+ * next - URL for next page of results (if paginated)+ * success - Whether status check succeeded+ * error - Error message if failed++ Raises:+ Exception: If status check fails+ """+ endpoint = f'/v1/crawl/{id}'++ headers = self._prepare_headers()+ response = self._get_request(f'{self.api_url}{endpoint}', headers)+ if response.status_code == 200:+ try:+ status_data = response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ if status_data['status'] == 'completed':+ if 'data' in status_data:+ data = status_data['data']+ while 'next' in status_data:+ if len(status_data['data']) == 0:+ break+ next_url = status_data.get('next')+ if not next_url:+ logger.warning("Expected 'next' URL is missing.")+ break+ try:+ status_response = self._get_request(next_url, headers)+ if status_response.status_code != 200:+ logger.error(f"Failed to fetch next page: {status_response.status_code}")+ break+ try:+ next_data = status_response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ data.extend(next_data.get('data', []))+ status_data = next_data+ except Exception as e:+ logger.error(f"Error during pagination request: {e}")+ break+ status_data['data'] = data++ response = {+ 'status': status_data.get('status'),+ 'total': status_data.get('total'),+ 'completed': status_data.get('completed'),+ 'creditsUsed': status_data.get('creditsUsed'),+ 'expiresAt': status_data.get('expiresAt'),+ 'data': status_data.get('data')+ }++ if 'error' in status_data:+ response['error'] = status_data['error']++ if 'next' in status_data:+ response['next'] = status_data['next']++ return V1CrawlStatusResponse(+ success=False if 'error' in status_data else True,+ **response+ )+ else:+ self._handle_error(response, 'check crawl status')++ def check_crawl_errors(self, id: str) -> V1CrawlErrorsResponse:+ """+ Returns information about crawl errors.++ Args:+ id (str): The ID of the crawl job++ Returns:+ V1CrawlErrorsResponse containing:+ * errors (List[Dict[str, str]]): List of errors with fields:+ - id (str): Error ID+ - timestamp (str): When the error occurred+ - url (str): URL that caused the error+ - error (str): Error message+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt++ Raises:+ Exception: If error check fails+ """+ headers = self._prepare_headers()+ response = self._get_request(f'{self.api_url}/v1/crawl/{id}/errors', headers)+ if response.status_code == 200:+ try:+ return V1CrawlErrorsResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, "check crawl errors")++ def cancel_crawl(self, id: str) -> Dict[str, Any]:+ """+ Cancel an asynchronous crawl job.++ Args:+ id (str): The ID of the crawl job to cancel++ Returns:+ Dict[str, Any] containing:+ * success (bool): Whether cancellation was successful+ * error (str, optional): Error message if cancellation failed++ Raises:+ Exception: If cancellation fails+ """+ headers = self._prepare_headers()+ response = self._delete_request(f'{self.api_url}/v1/crawl/{id}', headers)+ if response.status_code == 200:+ try:+ return response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, "cancel crawl job")++ def crawl_url_and_watch(+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[V1ScrapeOptions] = None,+ webhook: Optional[Union[str, V1WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,+ max_concurrency: Optional[int] = None,+ zero_data_retention: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> 'V1CrawlWatcher':+ """+ Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket.++ Args:+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration+ webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ delay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomains+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1CrawlWatcher: An instance to monitor the crawl job via WebSocket++ Raises:+ Exception: If crawl job fails to start+ """+ crawl_response = self.async_crawl_url(+ url,+ include_paths=include_paths,+ exclude_paths=exclude_paths,+ max_depth=max_depth,+ max_discovery_depth=max_discovery_depth,+ limit=limit,+ allow_backward_links=allow_backward_links,+ crawl_entire_domain=crawl_entire_domain,+ allow_external_links=allow_external_links,+ ignore_sitemap=ignore_sitemap,+ scrape_options=scrape_options,+ webhook=webhook,+ deduplicate_similar_urls=deduplicate_similar_urls,+ ignore_query_parameters=ignore_query_parameters,+ regex_on_full_url=regex_on_full_url,+ delay=delay,+ allow_subdomains=allow_subdomains,+ max_concurrency=max_concurrency,+ zero_data_retention=zero_data_retention,+ idempotency_key=idempotency_key,+ **kwargs+ )+ if crawl_response.success and crawl_response.id:+ return V1CrawlWatcher(crawl_response.id, self)+ else:+ raise Exception("Crawl job failed to start")++ def map_url(+ self,+ url: str,+ *,+ search: Optional[str] = None,+ ignore_sitemap: Optional[bool] = None,+ include_subdomains: Optional[bool] = None,+ sitemap_only: Optional[bool] = None,+ limit: Optional[int] = None,+ timeout: Optional[int] = 30000,+ use_index: Optional[bool] = None,+ **kwargs) -> V1MapResponse:+ """+ Map and discover links from a URL.++ Args:+ url (str): Target URL to map+ search (Optional[str]): Filter pattern for URLs+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ include_subdomains (Optional[bool]): Include subdomain links+ sitemap_only (Optional[bool]): Only use sitemap.xml+ limit (Optional[int]): Maximum URLs to return+ timeout (Optional[int]): Request timeout in milliseconds+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1MapResponse: Response containing:+ * success (bool): Whether request succeeded+ * links (List[str]): Discovered URLs+ * error (Optional[str]): Error message if any++ Raises:+ Exception: If mapping fails or response cannot be parsed+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "map_url")++ # Build map parameters+ map_params = {}++ # Add individual parameters+ if search is not None:+ map_params['search'] = search+ if ignore_sitemap is not None:+ map_params['ignoreSitemap'] = ignore_sitemap+ if include_subdomains is not None:+ map_params['includeSubdomains'] = include_subdomains+ if sitemap_only is not None:+ map_params['sitemapOnly'] = sitemap_only+ if limit is not None:+ map_params['limit'] = limit+ if timeout is not None:+ map_params['timeout'] = timeout+ if use_index is not None:+ map_params['useIndex'] = use_index++ # Add any additional kwargs+ map_params.update(kwargs)+ _integration = map_params.get('integration')++ # Create final params object+ final_params = V1MapParams(**map_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ if _integration:+ params_dict['integration'] = _integration++ # Make request+ response = requests.post(+ f"{self.api_url}/v1/map",+ headers={"Authorization": f"Bearer {self.api_key}"},+ json=params_dict+ )++ if response.status_code == 200:+ try:+ response_json = response.json()+ if response_json.get('success') and 'links' in response_json:+ return V1MapResponse(**response_json)+ elif "error" in response_json:+ raise Exception(f'Map failed. Error: {response_json["error"]}')+ else:+ raise Exception(f'Map failed. Error: {response_json}')+ except ValueError:+ raise Exception('Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'map')++ def batch_scrape_urls(+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ extract: Optional[V1JsonConfig] = None,+ json_options: Optional[V1JsonConfig] = None,+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,+ agent: Optional[V1AgentOptions] = None,+ poll_interval: Optional[int] = 2,+ max_concurrency: Optional[int] = None,+ zero_data_retention: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> V1BatchScrapeStatusResponse:+ """+ Batch scrape multiple URLs and monitor until completion.++ Args:+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[JsonConfig]): Content extraction config+ json_options (Optional[JsonConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1BatchScrapeStatusResponse with:+ * Scraping status and progress+ * Scraped content for each URL+ * Success/error information++ Raises:+ Exception: If batch scrape fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "batch_scrape_urls")++ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ extract = self._ensure_schema_dict(extract)+ if isinstance(extract, dict) and "schema" in extract:+ extract["schema"] = self._ensure_schema_dict(extract["schema"])+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)+ if json_options is not None:+ json_options = self._ensure_schema_dict(json_options)+ if isinstance(json_options, dict) and "schema" in json_options:+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions:+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)+ if max_concurrency is not None:+ scrape_params['maxConcurrency'] = max_concurrency+ if zero_data_retention is not None:+ scrape_params['zeroDataRetention'] = zero_data_retention++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = V1ScrapeParams(**scrape_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)++ if response.status_code == 200:+ try:+ id = response.json().get('id')+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ return self._monitor_job_status(id, headers, poll_interval)+ else:+ self._handle_error(response, 'start batch scrape job')++ def async_batch_scrape_urls(+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ extract: Optional[V1JsonConfig] = None,+ json_options: Optional[V1JsonConfig] = None,+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,+ agent: Optional[V1AgentOptions] = None,+ max_concurrency: Optional[int] = None,+ idempotency_key: Optional[str] = None,+ zero_data_retention: Optional[bool] = None,+ **kwargs+ ) -> V1BatchScrapeResponse:+ """+ Initiate a batch scrape job asynchronously.++ Args:+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[JsonConfig]): Content extraction config+ json_options (Optional[JsonConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1BatchScrapeResponse with:+ * success - Whether job started successfully+ * id - Unique identifier for the job+ * url - Status check URL+ * error - Error message if start failed++ Raises:+ Exception: If job initiation fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "async_batch_scrape_urls")++ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ extract = self._ensure_schema_dict(extract)+ if isinstance(extract, dict) and "schema" in extract:+ extract["schema"] = self._ensure_schema_dict(extract["schema"])+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)+ if json_options is not None:+ json_options = self._ensure_schema_dict(json_options)+ if isinstance(json_options, dict) and "schema" in json_options:+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions:+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)+ if max_concurrency is not None:+ scrape_params['maxConcurrency'] = max_concurrency+ if zero_data_retention is not None:+ scrape_params['zeroDataRetention'] = zero_data_retention++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = V1ScrapeParams(**scrape_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)++ if response.status_code == 200:+ try:+ return V1BatchScrapeResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'start batch scrape job')++ def batch_scrape_urls_and_watch(+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ extract: Optional[V1JsonConfig] = None,+ json_options: Optional[V1JsonConfig] = None,+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,+ agent: Optional[V1AgentOptions] = None,+ max_concurrency: Optional[int] = None,+ zero_data_retention: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> 'V1CrawlWatcher':+ """+ Initiate a batch scrape job and return a CrawlWatcher to monitor the job via WebSocket.++ Args:+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[JsonConfig]): Content extraction config+ json_options (Optional[JsonConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ max_concurrency (Optional[int]): Maximum number of concurrent scrapes+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket++ Raises:+ Exception: If batch scrape job fails to start+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "batch_scrape_urls_and_watch")++ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ extract = self._ensure_schema_dict(extract)+ if isinstance(extract, dict) and "schema" in extract:+ extract["schema"] = self._ensure_schema_dict(extract["schema"])+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)+ if json_options is not None:+ json_options = self._ensure_schema_dict(json_options)+ if isinstance(json_options, dict) and "schema" in json_options:+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions:+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)+ if max_concurrency is not None:+ scrape_params['maxConcurrency'] = max_concurrency+ if zero_data_retention is not None:+ scrape_params['zeroDataRetention'] = zero_data_retention++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = V1ScrapeParams(**scrape_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)++ if response.status_code == 200:+ try:+ crawl_response = V1BatchScrapeResponse(**response.json())+ if crawl_response.success and crawl_response.id:+ return V1CrawlWatcher(crawl_response.id, self)+ else:+ raise Exception("Batch scrape job failed to start")+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'start batch scrape job')++ def check_batch_scrape_status(self, id: str) -> V1BatchScrapeStatusResponse:+ """+ Check the status of a batch scrape job using the Firecrawl API.++ Args:+ id (str): The ID of the batch scrape job.++ Returns:+ V1BatchScrapeStatusResponse: The status of the batch scrape job.++ Raises:+ Exception: If the status check request fails.+ """+ endpoint = f'/v1/batch/scrape/{id}'++ headers = self._prepare_headers()+ response = self._get_request(f'{self.api_url}{endpoint}', headers)+ if response.status_code == 200:+ try:+ status_data = response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ if status_data['status'] == 'completed':+ if 'data' in status_data:+ data = status_data['data']+ while 'next' in status_data:+ if len(status_data['data']) == 0:+ break+ next_url = status_data.get('next')+ if not next_url:+ logger.warning("Expected 'next' URL is missing.")+ break+ try:+ status_response = self._get_request(next_url, headers)+ if status_response.status_code != 200:+ logger.error(f"Failed to fetch next page: {status_response.status_code}")+ break+ try:+ next_data = status_response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ data.extend(next_data.get('data', []))+ status_data = next_data+ except Exception as e:+ logger.error(f"Error during pagination request: {e}")+ break+ status_data['data'] = data++ return V1BatchScrapeStatusResponse(**{+ 'success': False if 'error' in status_data else True,+ 'status': status_data.get('status'),+ 'total': status_data.get('total'),+ 'completed': status_data.get('completed'),+ 'creditsUsed': status_data.get('creditsUsed'),+ 'expiresAt': status_data.get('expiresAt'),+ 'data': status_data.get('data'),+ 'next': status_data.get('next'),+ 'error': status_data.get('error')+ })+ else:+ self._handle_error(response, 'check batch scrape status')++ def check_batch_scrape_errors(self, id: str) -> V1CrawlErrorsResponse:+ """+ Returns information about batch scrape errors.++ Args:+ id (str): The ID of the crawl job.++ Returns:+ V1CrawlErrorsResponse containing:+ * errors (List[Dict[str, str]]): List of errors with fields:+ * id (str): Error ID+ * timestamp (str): When the error occurred+ * url (str): URL that caused the error+ * error (str): Error message+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt++ Raises:+ Exception: If the error check request fails+ """+ headers = self._prepare_headers()+ response = self._get_request(f'{self.api_url}/v1/batch/scrape/{id}/errors', headers)+ if response.status_code == 200:+ try:+ return V1CrawlErrorsResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, "check batch scrape errors")++ def extract(+ self,+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Any] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = False,+ enable_web_search: Optional[bool] = False,+ show_sources: Optional[bool] = False,+ agent: Optional[Dict[str, Any]] = None,+ **kwargs) -> V1ExtractResponse[Any]:+ """+ Extract structured information from URLs.++ Args:+ urls (Optional[List[str]]): URLs to extract from+ prompt (Optional[str]): Custom extraction prompt+ schema (Optional[Any]): JSON schema/Pydantic model+ system_prompt (Optional[str]): System context+ allow_external_links (Optional[bool]): Follow external links+ enable_web_search (Optional[bool]): Enable web search+ show_sources (Optional[bool]): Include source URLs+ agent (Optional[Dict[str, Any]]): Agent configuration+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1ExtractResponse[Any] with:+ * success (bool): Whether request succeeded+ * data (Optional[Any]): Extracted data matching schema+ * error (Optional[str]): Error message if any++ Raises:+ ValueError: If prompt/schema missing or extraction fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "extract")++ headers = self._prepare_headers()++ if not prompt and not schema:+ raise ValueError("Either prompt or schema is required")++ if not urls and not prompt:+ raise ValueError("Either urls or prompt is required")++ if schema:+ schema = self._ensure_schema_dict(schema)++ request_data = {+ 'urls': urls or [],+ 'allowExternalLinks': allow_external_links,+ 'enableWebSearch': enable_web_search,+ 'showSources': show_sources,+ 'schema': schema,+ 'origin': f'python-sdk@{get_version()}'+ }++ # Only add prompt and systemPrompt if they exist+ if prompt:+ request_data['prompt'] = prompt+ if system_prompt:+ request_data['systemPrompt'] = system_prompt++ if agent:+ request_data['agent'] = agent++ # Add any additional kwargs+ request_data.update(kwargs)++ try:+ # Send the initial extract request+ response = self._post_request(+ f'{self.api_url}/v1/extract',+ request_data,+ headers+ )+ if response.status_code == 200:+ try:+ data = response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ if data['success']:+ job_id = data.get('id')+ if not job_id:+ raise Exception('Job ID not returned from extract request.')++ # Poll for the extract status+ while True:+ status_response = self._get_request(+ f'{self.api_url}/v1/extract/{job_id}',+ headers+ )+ if status_response.status_code == 200:+ try:+ status_data = status_response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ if status_data['status'] == 'completed':+ return V1ExtractResponse(**status_data)+ elif status_data['status'] in ['failed', 'cancelled']:+ raise Exception(f'Extract job {status_data["status"]}. Error: {status_data["error"]}')+ else:+ self._handle_error(status_response, "extract-status")++ time.sleep(2) # Polling interval+ else:+ raise Exception(f'Failed to extract. Error: {data["error"]}')+ else:+ self._handle_error(response, "extract")+ except Exception as e:+ raise ValueError(str(e), 500)++ return V1ExtractResponse(success=False, error="Internal server error.")++ def get_extract_status(self, job_id: str) -> V1ExtractResponse[Any]:+ """+ Retrieve the status of an extract job.++ Args:+ job_id (str): The ID of the extract job.++ Returns:+ ExtractResponse[Any]: The status of the extract job.++ Raises:+ ValueError: If there is an error retrieving the status.+ """+ headers = self._prepare_headers()+ try:+ response = self._get_request(f'{self.api_url}/v1/extract/{job_id}', headers)+ if response.status_code == 200:+ try:+ return V1ExtractResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, "get extract status")+ except Exception as e:+ raise ValueError(str(e), 500)++ def async_extract(+ self,+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Any] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = False,+ enable_web_search: Optional[bool] = False,+ show_sources: Optional[bool] = False,+ agent: Optional[Dict[str, Any]] = None) -> V1ExtractResponse[Any]:+ """+ Initiate an asynchronous extract job.++ Args:+ urls (List[str]): URLs to extract information from+ prompt (Optional[str]): Custom extraction prompt+ schema (Optional[Any]): JSON schema/Pydantic model+ system_prompt (Optional[str]): System context+ allow_external_links (Optional[bool]): Follow external links+ enable_web_search (Optional[bool]): Enable web search+ show_sources (Optional[bool]): Include source URLs+ agent (Optional[Dict[str, Any]]): Agent configuration+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests++ Returns:+ ExtractResponse[Any] with:+ * success (bool): Whether request succeeded+ * data (Optional[Any]): Extracted data matching schema+ * error (Optional[str]): Error message if any++ Raises:+ ValueError: If job initiation fails+ """+ headers = self._prepare_headers()++ schema = schema+ if schema:+ schema = self._ensure_schema_dict(schema)++ request_data = {+ 'urls': urls,+ 'allowExternalLinks': allow_external_links,+ 'enableWebSearch': enable_web_search,+ 'showSources': show_sources,+ 'schema': schema,+ 'origin': f'python-sdk@{version}'+ }++ if prompt:+ request_data['prompt'] = prompt+ if system_prompt:+ request_data['systemPrompt'] = system_prompt+ if agent:+ request_data['agent'] = agent++ try:+ response = self._post_request(f'{self.api_url}/v1/extract', request_data, headers)+ if response.status_code == 200:+ try:+ return V1ExtractResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, "async extract")+ except Exception as e:+ raise ValueError(str(e), 500)++ def generate_llms_text(+ self,+ url: str,+ *,+ max_urls: Optional[int] = None,+ show_full_text: Optional[bool] = None,+ cache: Optional[bool] = None,+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextStatusResponse:+ """+ Generate LLMs.txt for a given URL and poll until completion.++ Args:+ url (str): Target URL to generate LLMs.txt from+ max_urls (Optional[int]): Maximum URLs to process (default: 10)+ show_full_text (Optional[bool]): Include full text in output (default: False)+ cache (Optional[bool]): Whether to use cached content if available (default: True)+ experimental_stream (Optional[bool]): Enable experimental streaming++ Returns:+ GenerateLLMsTextStatusResponse with:+ * Generated LLMs.txt content+ * Full version if requested+ * Generation status+ * Success/error information++ Raises:+ Exception: If generation fails+ """+ params = V1GenerateLLMsTextParams(+ maxUrls=max_urls,+ showFullText=show_full_text,+ cache=cache,+ __experimental_stream=experimental_stream+ )++ response = self.async_generate_llms_text(+ url,+ max_urls=max_urls,+ show_full_text=show_full_text,+ cache=cache,+ experimental_stream=experimental_stream+ )++ if not response.success or not response.id:+ return V1GenerateLLMsTextStatusResponse(+ success=False,+ error='Failed to start LLMs.txt generation',+ status='failed',+ expiresAt=''+ )++ job_id = response.id+ while True:+ status = self.check_generate_llms_text_status(job_id)++ if status.status == 'completed':+ return status+ elif status.status == 'failed':+ return status+ elif status.status != 'processing':+ return V1GenerateLLMsTextStatusResponse(+ success=False,+ error='LLMs.txt generation job terminated unexpectedly',+ status='failed',+ expiresAt=''+ )++ time.sleep(2) # Polling interval++ def async_generate_llms_text(+ self,+ url: str,+ *,+ max_urls: Optional[int] = None,+ show_full_text: Optional[bool] = None,+ cache: Optional[bool] = None,+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextResponse:+ """+ Initiate an asynchronous LLMs.txt generation operation.++ Args:+ url (str): The target URL to generate LLMs.txt from. Must be a valid HTTP/HTTPS URL.+ max_urls (Optional[int]): Maximum URLs to process (default: 10)+ show_full_text (Optional[bool]): Include full text in output (default: False)+ cache (Optional[bool]): Whether to use cached content if available (default: True)+ experimental_stream (Optional[bool]): Enable experimental streaming++ Returns:+ GenerateLLMsTextResponse: A response containing:+ * success (bool): Whether the generation initiation was successful+ * id (str): The unique identifier for the generation job+ * error (str, optional): Error message if initiation failed++ Raises:+ Exception: If the generation job initiation fails.+ """+ params = V1GenerateLLMsTextParams(+ maxUrls=max_urls,+ showFullText=show_full_text,+ cache=cache,+ __experimental_stream=experimental_stream+ )++ headers = self._prepare_headers()+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}+ json_data['origin'] = f"python-sdk@{version}"++ try:+ req = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)+ response = req.json()+ print("json_data", json_data)+ print("response", response)+ if response.get('success'):+ try:+ return V1GenerateLLMsTextResponse(**response)+ except:+ raise Exception('Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'start LLMs.txt generation')+ except Exception as e:+ raise ValueError(str(e))++ return V1GenerateLLMsTextResponse(+ success=False,+ error='Internal server error'+ )++ def check_generate_llms_text_status(self, id: str) -> V1GenerateLLMsTextStatusResponse:+ """+ Check the status of a LLMs.txt generation operation.++ Args:+ id (str): The unique identifier of the LLMs.txt generation job to check status for.++ Returns:+ GenerateLLMsTextStatusResponse: A response containing:+ * success (bool): Whether the generation was successful+ * status (str): Status of generation ("processing", "completed", "failed")+ * data (Dict[str, str], optional): Generated text with fields:+ * llmstxt (str): Generated LLMs.txt content+ * llmsfulltxt (str, optional): Full version if requested+ * error (str, optional): Error message if generation failed+ * expiresAt (str): When the generated data expires++ Raises:+ Exception: If the status check fails.+ """+ headers = self._prepare_headers()+ try:+ response = self._get_request(f'{self.api_url}/v1/llmstxt/{id}', headers)+ if response.status_code == 200:+ try:+ json_data = response.json()+ return V1GenerateLLMsTextStatusResponse(**json_data)+ except Exception as e:+ raise Exception(f'Failed to parse Firecrawl response as GenerateLLMsTextStatusResponse: {str(e)}')+ elif response.status_code == 404:+ raise Exception('LLMs.txt generation job not found')+ else:+ self._handle_error(response, 'check LLMs.txt generation status')+ except Exception as e:+ raise ValueError(str(e))++ return V1GenerateLLMsTextStatusResponse(success=False, error='Internal server error', status='failed', expiresAt='')++ def _prepare_headers(+ self,+ idempotency_key: Optional[str] = None) -> Dict[str, str]:+ """+ Prepare the headers for API requests.++ Args:+ idempotency_key (Optional[str]): A unique key to ensure idempotency of requests.++ Returns:+ Dict[str, str]: The headers including content type, authorization, and optionally idempotency key.+ """+ if idempotency_key:+ return {+ 'Content-Type': 'application/json',+ 'Authorization': f'Bearer {self.api_key}',+ 'x-idempotency-key': idempotency_key+ }++ return {+ 'Content-Type': 'application/json',+ 'Authorization': f'Bearer {self.api_key}',+ }++ def _post_request(+ self,+ url: str,+ data: Dict[str, Any],+ headers: Dict[str, str],+ retries: int = 3,+ backoff_factor: float = 0.5) -> requests.Response:+ """+ Make a POST request with retries.++ Args:+ url (str): The URL to send the POST request to.+ data (Dict[str, Any]): The JSON data to include in the POST request.+ headers (Dict[str, str]): The headers to include in the POST request.+ retries (int): Number of retries for the request.+ backoff_factor (float): Backoff factor for retries.++ Returns:+ requests.Response: The response from the POST request.++ Raises:+ requests.RequestException: If the request fails after the specified retries.+ """+ for attempt in range(retries):+ response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] / 1000.0 + 5) if "timeout" in data and data["timeout"] is not None else None))+ if response.status_code == 502:+ time.sleep(backoff_factor * (2 ** attempt))+ else:+ return response+ return response++ def _get_request(+ self,+ url: str,+ headers: Dict[str, str],+ retries: int = 3,+ backoff_factor: float = 0.5) -> requests.Response:+ """+ Make a GET request with retries.++ Args:+ url (str): The URL to send the GET request to.+ headers (Dict[str, str]): The headers to include in the GET request.+ retries (int): Number of retries for the request.+ backoff_factor (float): Backoff factor for retries.++ Returns:+ requests.Response: The response from the GET request.++ Raises:+ requests.RequestException: If the request fails after the specified retries.+ """+ for attempt in range(retries):+ response = requests.get(url, headers=headers)+ if response.status_code == 502:+ time.sleep(backoff_factor * (2 ** attempt))+ else:+ return response+ return response++ def _delete_request(+ self,+ url: str,+ headers: Dict[str, str],+ retries: int = 3,+ backoff_factor: float = 0.5) -> requests.Response:+ """+ Make a DELETE request with retries.++ Args:+ url (str): The URL to send the DELETE request to.+ headers (Dict[str, str]): The headers to include in the DELETE request.+ retries (int): Number of retries for the request.+ backoff_factor (float): Backoff factor for retries.++ Returns:+ requests.Response: The response from the DELETE request.++ Raises:+ requests.RequestException: If the request fails after the specified retries.+ """+ for attempt in range(retries):+ response = requests.delete(url, headers=headers)+ if response.status_code == 502:+ time.sleep(backoff_factor * (2 ** attempt))+ else:+ return response+ return response++ def _monitor_job_status(+ self,+ id: str,+ headers: Dict[str, str],+ poll_interval: int) -> V1CrawlStatusResponse:+ """+ Monitor the status of a crawl job until completion.++ Args:+ id (str): The ID of the crawl job.+ headers (Dict[str, str]): The headers to include in the status check requests.+ poll_interval (int): Seconds between status checks.++ Returns:+ CrawlStatusResponse: The crawl results if the job is completed successfully.++ Raises:+ Exception: If the job fails or an error occurs during status checks.+ """+ while True:+ api_url = f'{self.api_url}/v1/crawl/{id}'++ status_response = self._get_request(api_url, headers)+ if status_response.status_code == 200:+ try:+ status_data = status_response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ if status_data['status'] == 'completed':+ if 'data' in status_data:+ data = status_data['data']+ while 'next' in status_data:+ if len(status_data['data']) == 0:+ break+ status_response = self._get_request(status_data['next'], headers)+ try:+ status_data = status_response.json()+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ data.extend(status_data.get('data', []))+ status_data['data'] = data+ return V1CrawlStatusResponse(**status_data)+ else:+ raise Exception('Crawl job completed but no data was returned')+ elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:+ poll_interval=max(poll_interval,2)+ time.sleep(poll_interval) # Wait for the specified interval before checking again+ else:+ raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}')+ else:+ self._handle_error(status_response, 'check crawl status')++ def _handle_error(+ self,+ response: requests.Response,+ action: str) -> None:+ """+ Handle errors from API responses.++ Args:+ response (requests.Response): The response object from the API request.+ action (str): Description of the action that was being performed.++ Raises:+ Exception: An exception with a message containing the status code and error details from the response.+ """+ try:+ response_json = response.json()+ error_message = response_json.get('error', 'No error message provided.')+ error_details = response_json.get('details', 'No additional error details provided.')+ except:+ # If we can't parse JSON, provide a helpful error message with response content+ try:+ response_text = response.text[:500] # Limit to first 500 chars+ if response_text.strip():+ error_message = f"Server returned non-JSON response: {response_text}"+ error_details = f"Full response status: {response.status_code}"+ else:+ error_message = f"Server returned empty response with status {response.status_code}"+ error_details = "No additional details available"+ except ValueError:+ error_message = f"Server returned unreadable response with status {response.status_code}"+ error_details = "No additional details available"++ message = self._get_error_message(response.status_code, action, error_message, error_details)++ # Raise an HTTPError with the custom message and attach the response+ raise requests.exceptions.HTTPError(message, response=response)++ def _get_error_message(self, status_code: int, action: str, error_message: str, error_details: str) -> str:+ """+ Generate a standardized error message based on HTTP status code.++ Args:+ status_code (int): The HTTP status code from the response+ action (str): Description of the action that was being performed+ error_message (str): The error message from the API response+ error_details (str): Additional error details from the API response++ Returns:+ str: A formatted error message+ """+ if status_code == 402:+ return f"Payment Required: Failed to {action}. {error_message} - {error_details}"+ elif status_code == 403:+ return f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"+ elif status_code == 408:+ return f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"+ elif status_code == 409:+ return f"Conflict: Failed to {action} due to a conflict. {error_message} - {error_details}"+ elif status_code == 500:+ return f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"+ else:+ return f"Unexpected error during {action}: Status code {status_code}. {error_message} - {error_details}"++ def deep_research(+ self,+ query: str,+ *,+ max_depth: Optional[int] = None,+ time_limit: Optional[int] = None,+ max_urls: Optional[int] = None,+ analysis_prompt: Optional[str] = None,+ system_prompt: Optional[str] = None,+ __experimental_stream_steps: Optional[bool] = None,+ on_activity: Optional[Callable[[Dict[str, Any]], None]] = None,+ on_source: Optional[Callable[[Dict[str, Any]], None]] = None) -> V1DeepResearchStatusResponse:+ """+ Initiates a deep research operation on a given query and polls until completion.++ Args:+ query (str): Research query or topic to investigate+ max_depth (Optional[int]): Maximum depth of research exploration+ time_limit (Optional[int]): Time limit in seconds for research+ max_urls (Optional[int]): Maximum number of URLs to process+ analysis_prompt (Optional[str]): Custom prompt for analysis+ system_prompt (Optional[str]): Custom system prompt+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming+ on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth}+ on_source (Optional[Callable]): Source discovery callback receiving {url, title, description}++ Returns:+ DeepResearchStatusResponse containing:+ * success (bool): Whether research completed successfully+ * status (str): Current state (processing/completed/failed)+ * error (Optional[str]): Error message if failed+ * id (str): Unique identifier for the research job+ * data (Any): Research findings and analysis+ * sources (List[Dict]): List of discovered sources+ * activities (List[Dict]): Research progress log+ * summaries (List[str]): Generated research summaries++ Raises:+ Exception: If research fails+ """+ research_params = {}+ if max_depth is not None:+ research_params['maxDepth'] = max_depth+ if time_limit is not None:+ research_params['timeLimit'] = time_limit+ if max_urls is not None:+ research_params['maxUrls'] = max_urls+ if analysis_prompt is not None:+ research_params['analysisPrompt'] = analysis_prompt+ if system_prompt is not None:+ research_params['systemPrompt'] = system_prompt+ if __experimental_stream_steps is not None:+ research_params['__experimental_streamSteps'] = __experimental_stream_steps+ research_params = V1DeepResearchParams(**research_params)++ response = self.async_deep_research(+ query,+ max_depth=max_depth,+ time_limit=time_limit,+ max_urls=max_urls,+ analysis_prompt=analysis_prompt,+ system_prompt=system_prompt+ )+ if not response.get('success') or 'id' not in response:+ return response++ job_id = response['id']+ last_activity_count = 0+ last_source_count = 0++ while True:+ status = self.check_deep_research_status(job_id)++ if on_activity and 'activities' in status:+ new_activities = status['activities'][last_activity_count:]+ for activity in new_activities:+ on_activity(activity)+ last_activity_count = len(status['activities'])++ if on_source and 'sources' in status:+ new_sources = status['sources'][last_source_count:]+ for source in new_sources:+ on_source(source)+ last_source_count = len(status['sources'])++ if status['status'] == 'completed':+ return status+ elif status['status'] == 'failed':+ raise Exception(f'Deep research failed. Error: {status.get("error")}')+ elif status['status'] != 'processing':+ break++ time.sleep(2) # Polling interval++ return {'success': False, 'error': 'Deep research job terminated unexpectedly'}++ def async_deep_research(+ self,+ query: str,+ *,+ max_depth: Optional[int] = None,+ time_limit: Optional[int] = None,+ max_urls: Optional[int] = None,+ analysis_prompt: Optional[str] = None,+ system_prompt: Optional[str] = None,+ __experimental_stream_steps: Optional[bool] = None) -> Dict[str, Any]:+ """+ Initiates an asynchronous deep research operation.++ Args:+ query (str): Research query or topic to investigate+ max_depth (Optional[int]): Maximum depth of research exploration+ time_limit (Optional[int]): Time limit in seconds for research+ max_urls (Optional[int]): Maximum number of URLs to process+ analysis_prompt (Optional[str]): Custom prompt for analysis+ system_prompt (Optional[str]): Custom system prompt+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming++ Returns:+ Dict[str, Any]: A response containing:+ * success (bool): Whether the research initiation was successful+ * id (str): The unique identifier for the research job+ * error (str, optional): Error message if initiation failed++ Raises:+ Exception: If the research initiation fails.+ """+ research_params = {}+ if max_depth is not None:+ research_params['maxDepth'] = max_depth+ if time_limit is not None:+ research_params['timeLimit'] = time_limit+ if max_urls is not None:+ research_params['maxUrls'] = max_urls+ if analysis_prompt is not None:+ research_params['analysisPrompt'] = analysis_prompt+ if system_prompt is not None:+ research_params['systemPrompt'] = system_prompt+ if __experimental_stream_steps is not None:+ research_params['__experimental_streamSteps'] = __experimental_stream_steps+ research_params = V1DeepResearchParams(**research_params)++ headers = self._prepare_headers()++ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}+ json_data['origin'] = f"python-sdk@{version}"++ # Handle json options schema if present+ if 'jsonOptions' in json_data:+ json_opts = json_data['jsonOptions']+ if json_opts and 'schema' in json_opts and hasattr(json_opts['schema'], 'schema'):+ json_data['jsonOptions']['schema'] = json_opts['schema'].schema()++ try:+ response = self._post_request(f'{self.api_url}/v1/deep-research', json_data, headers)+ if response.status_code == 200:+ try:+ return response.json()+ except:+ raise Exception('Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'start deep research')+ except Exception as e:+ raise ValueError(str(e))++ return {'success': False, 'error': 'Internal server error'}++ def check_deep_research_status(self, id: str) -> V1DeepResearchStatusResponse:+ """+ Check the status of a deep research operation.++ Args:+ id (str): The ID of the deep research operation.++ Returns:+ DeepResearchResponse containing:++ Status:+ * success - Whether research completed successfully+ * status - Current state (processing/completed/failed)+ * error - Error message if failed++ Results:+ * id - Unique identifier for the research job+ * data - Research findings and analysis+ * sources - List of discovered sources+ * activities - Research progress log+ * summaries - Generated research summaries++ Raises:+ Exception: If the status check fails.+ """+ headers = self._prepare_headers()+ try:+ response = self._get_request(f'{self.api_url}/v1/deep-research/{id}', headers)+ if response.status_code == 200:+ try:+ return response.json()+ except:+ raise Exception('Failed to parse Firecrawl response as JSON.')+ elif response.status_code == 404:+ raise Exception('Deep research job not found')+ else:+ self._handle_error(response, 'check deep research status')+ except Exception as e:+ raise ValueError(str(e))++ return {'success': False, 'error': 'Internal server error'}++ def _validate_kwargs(self, kwargs: Dict[str, Any], method_name: str) -> None:+ """+ Validate additional keyword arguments before they are passed to the API.+ This provides early validation before the Pydantic model validation.++ Args:+ kwargs (Dict[str, Any]): Additional keyword arguments to validate+ method_name (str): Name of the method these kwargs are for++ Raises:+ ValueError: If kwargs contain invalid or unsupported parameters+ """+ if not kwargs:+ return++ # Known parameter mappings for each method+ method_params = {+ "scrape_url": {"formats", "include_tags", "exclude_tags", "only_main_content", "wait_for",+ "timeout", "location", "mobile", "skip_tls_verification", "remove_base64_images",+ "block_ads", "proxy", "extract", "json_options", "actions", "change_tracking_options", "max_age", "integration"},+ "search": {"limit", "tbs", "filter", "lang", "country", "location", "timeout", "scrape_options", "integration"},+ "crawl_url": {"include_paths", "exclude_paths", "max_depth", "max_discovery_depth", "limit",+ "allow_backward_links", "allow_external_links", "ignore_sitemap", "scrape_options",+ "webhook", "deduplicate_similar_urls", "ignore_query_parameters", "regex_on_full_url", "integration"},+ "map_url": {"search", "ignore_sitemap", "include_subdomains", "sitemap_only", "limit", "timeout", "integration"},+ "extract": {"prompt", "schema", "system_prompt", "allow_external_links", "enable_web_search", "show_sources", "agent", "integration"},+ "batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",+ "wait_for", "timeout", "location", "mobile", "skip_tls_verification",+ "remove_base64_images", "block_ads", "proxy", "extract", "json_options",+ "actions", "agent", "webhook"},+ "async_batch_scrape_urls": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",+ "wait_for", "timeout", "location", "mobile", "skip_tls_verification",+ "remove_base64_images", "block_ads", "proxy", "extract", "json_options",+ "actions", "agent", "webhook"},+ "batch_scrape_urls_and_watch": {"formats", "headers", "include_tags", "exclude_tags", "only_main_content",+ "wait_for", "timeout", "location", "mobile", "skip_tls_verification",+ "remove_base64_images", "block_ads", "proxy", "extract", "json_options",+ "actions", "agent", "webhook"}+ }++ # Get allowed parameters for this method+ allowed_params = method_params.get(method_name, set())++ # Check for unknown parameters+ unknown_params = set(kwargs.keys()) - allowed_params+ if unknown_params:+ raise ValueError(f"Unsupported parameter(s) for {method_name}: {', '.join(unknown_params)}. Please refer to the API documentation for the correct parameters.")++ # Additional type validation can be added here if needed+ # For now, we rely on Pydantic models for detailed type validation++ def _ensure_schema_dict(self, schema):+ """+ Utility to ensure a schema is a dict, not a Pydantic model class. Recursively checks dicts and lists.+ """+ if schema is None:+ return schema+ if isinstance(schema, type):+ # Pydantic v1/v2 model class+ if hasattr(schema, 'model_json_schema'):+ return schema.model_json_schema()+ elif hasattr(schema, 'schema'):+ return schema.schema()+ if isinstance(schema, dict):+ return {k: self._ensure_schema_dict(v) for k, v in schema.items()}+ if isinstance(schema, (list, tuple)):+ return [self._ensure_schema_dict(v) for v in schema]+ return schema++class V1CrawlWatcher:+ """+ A class to watch and handle crawl job events via WebSocket connection.++ Attributes:+ id (str): The ID of the crawl job to watch+ app (V1FirecrawlApp): The V1FirecrawlApp instance+ data (List[Dict[str, Any]]): List of crawled documents/data+ status (str): Current status of the crawl job+ ws_url (str): WebSocket URL for the crawl job+ event_handlers (dict): Dictionary of event type to list of handler functions+ """+ def __init__(self, id: str, app: V1FirecrawlApp):+ self.id = id+ self.app = app+ self.data: List[Dict[str, Any]] = []+ self.status = "scraping"+ self.ws_url = f"{app.api_url.replace('http', 'ws')}/v1/crawl/{id}"+ self.event_handlers = {+ 'done': [],+ 'error': [],+ 'document': []+ }++ async def connect(self) -> None:+ """+ Establishes WebSocket connection and starts listening for messages.+ """+ async with websockets.connect(+ self.ws_url,+ max_size=None,+ additional_headers=[("Authorization", f"Bearer {self.app.api_key}")]+ ) as websocket:+ await self._listen(websocket)++ async def _listen(self, websocket) -> None:+ """+ Listens for incoming WebSocket messages and handles them.++ Args:+ websocket: The WebSocket connection object+ """+ async for message in websocket:+ msg = json.loads(message)+ await self._handle_message(msg)++ def add_event_listener(self, event_type: str, handler: Callable[[Dict[str, Any]], None]) -> None:+ """+ Adds an event handler function for a specific event type.++ Args:+ event_type (str): Type of event to listen for ('done', 'error', or 'document')+ handler (Callable): Function to handle the event+ """+ if event_type in self.event_handlers:+ self.event_handlers[event_type].append(handler)++ def dispatch_event(self, event_type: str, detail: Dict[str, Any]) -> None:+ """+ Dispatches an event to all registered handlers for that event type.++ Args:+ event_type (str): Type of event to dispatch+ detail (Dict[str, Any]): Event details/data to pass to handlers+ """+ if event_type in self.event_handlers:+ for handler in self.event_handlers[event_type]:+ handler(detail)++ async def _handle_message(self, msg: Dict[str, Any]) -> None:+ """+ Handles incoming WebSocket messages based on their type.++ Args:+ msg (Dict[str, Any]): The message to handle+ """+ if msg['type'] == 'done':+ self.status = 'completed'+ self.dispatch_event('done', {'status': self.status, 'data': self.data, 'id': self.id})+ elif msg['type'] == 'error':+ self.status = 'failed'+ self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error'], 'id': self.id})+ elif msg['type'] == 'catchup':+ self.status = msg['data']['status']+ self.data.extend(msg['data'].get('data', []))+ for doc in self.data:+ self.dispatch_event('document', {'data': doc, 'id': self.id})+ elif msg['type'] == 'document':+ self.data.append(msg['data'])+ self.dispatch_event('document', {'data': msg['data'], 'id': self.id})++class AsyncV1FirecrawlApp:+ """+ Asynchronous version of V1FirecrawlApp that implements async methods using aiohttp.+ Provides non-blocking alternatives to all V1FirecrawlApp operations.+ """++ def __init__(self, api_key: str, api_url: str = "https://api.firecrawl.com"):+ self.api_key = api_key+ self.api_url = api_url++ async def _async_request(+ self,+ method: str,+ url: str,+ headers: Dict[str, str],+ data: Optional[Dict[str, Any]] = None,+ retries: int = 3,+ backoff_factor: float = 0.5) -> Dict[str, Any]:+ """+ Generic async request method with exponential backoff retry logic.++ Args:+ method (str): The HTTP method to use (e.g., "GET" or "POST").+ url (str): The URL to send the request to.+ headers (Dict[str, str]): Headers to include in the request.+ data (Optional[Dict[str, Any]]): The JSON data to include in the request body (only for POST requests).+ retries (int): Maximum number of retry attempts (default: 3).+ backoff_factor (float): Factor to calculate delay between retries (default: 0.5).+ Delay will be backoff_factor * (2 ** retry_count).++ Returns:+ Dict[str, Any]: The parsed JSON response from the server.++ Raises:+ aiohttp.ClientError: If the request fails after all retries.+ Exception: If max retries are exceeded or other errors occur.+ """+ async with aiohttp.ClientSession() as session:+ for attempt in range(retries):+ try:+ async with session.request(+ method=method, url=url, headers=headers, json=data+ ) as response:+ if response.status == 502:+ await asyncio.sleep(backoff_factor * (2 ** attempt))+ continue+ if response.status >= 300:+ await self._handle_error(response, f"make {method} request")+ return await response.json()+ except aiohttp.ClientError as e:+ if attempt == retries - 1:+ raise e+ await asyncio.sleep(backoff_factor * (2 ** attempt))+ raise Exception("Max retries exceeded")++ async def _async_post_request(+ self, url: str, data: Dict[str, Any], headers: Dict[str, str],+ retries: int = 3, backoff_factor: float = 0.5) -> Dict[str, Any]:+ """+ Make an async POST request with exponential backoff retry logic.++ Args:+ url (str): The URL to send the POST request to.+ data (Dict[str, Any]): The JSON data to include in the request body.+ headers (Dict[str, str]): Headers to include in the request.+ retries (int): Maximum number of retry attempts (default: 3).+ backoff_factor (float): Factor to calculate delay between retries (default: 0.5).+ Delay will be backoff_factor * (2 ** retry_count).++ Returns:+ Dict[str, Any]: The parsed JSON response from the server.++ Raises:+ aiohttp.ClientError: If the request fails after all retries.+ Exception: If max retries are exceeded or other errors occur.+ """+ return await self._async_request("POST", url, headers, data, retries, backoff_factor)++ async def _async_get_request(+ self, url: str, headers: Dict[str, str],+ retries: int = 3, backoff_factor: float = 0.5) -> Dict[str, Any]:+ """+ Make an async GET request with exponential backoff retry logic.++ Args:+ url (str): The URL to send the GET request to.+ headers (Dict[str, str]): Headers to include in the request.+ retries (int): Maximum number of retry attempts (default: 3).+ backoff_factor (float): Factor to calculate delay between retries (default: 0.5).+ Delay will be backoff_factor * (2 ** retry_count).++ Returns:+ Dict[str, Any]: The parsed JSON response from the server.++ Raises:+ aiohttp.ClientError: If the request fails after all retries.+ Exception: If max retries are exceeded or other errors occur.+ """+ return await self._async_request("GET", url, headers, None, retries, backoff_factor)++ async def _handle_error(self, response: aiohttp.ClientResponse, action: str) -> None:+ """+ Handle errors from async API responses with detailed error messages.++ Args:+ response (aiohttp.ClientResponse): The response object from the failed request+ action (str): Description of the action that was being attempted++ Raises:+ aiohttp.ClientError: With a detailed error message based on the response status:+ - 402: Payment Required+ - 408: Request Timeout+ - 409: Conflict+ - 500: Internal Server Error+ - Other: Unexpected error with status code+ """+ try:+ error_data = await response.json()+ error_message = error_data.get('error', 'No error message provided.')+ error_details = error_data.get('details', 'No additional error details provided.')+ except:+ raise aiohttp.ClientError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status}')++ message = await self._get_async_error_message(response.status, action, error_message, error_details)++ raise aiohttp.ClientError(message)++ async def _get_async_error_message(self, status_code: int, action: str, error_message: str, error_details: str) -> str:+ """+ Generate a standardized error message based on HTTP status code for async operations.++ Args:+ status_code (int): The HTTP status code from the response+ action (str): Description of the action that was being performed+ error_message (str): The error message from the API response+ error_details (str): Additional error details from the API response++ Returns:+ str: A formatted error message+ """+ return self._get_error_message(status_code, action, error_message, error_details)++ async def crawl_url_and_watch(+ self,+ url: str,+ params: Optional[V1CrawlParams] = None,+ idempotency_key: Optional[str] = None) -> 'AsyncV1CrawlWatcher':+ """+ Initiate an async crawl job and return an AsyncV1CrawlWatcher to monitor progress via WebSocket.++ Args:+ url (str): Target URL to start crawling from+ params (Optional[V1CrawlParams]): See V1CrawlParams model for configuration:+ URL Discovery:+ * includePaths - Patterns of URLs to include+ * excludePaths - Patterns of URLs to exclude+ * maxDepth - Maximum crawl depth+ * maxDiscoveryDepth - Maximum depth for finding new URLs+ * limit - Maximum pages to crawl++ Link Following:+ * allowBackwardLinks - DEPRECATED: Use crawlEntireDomain instead+ * crawlEntireDomain - Follow parent directory links+ * allowExternalLinks - Follow external domain links+ * ignoreSitemap - Skip sitemap.xml processing++ Advanced:+ * scrapeOptions - Page scraping configuration+ * webhook - Notification webhook settings+ * deduplicateSimilarURLs - Remove similar URLs+ * ignoreQueryParameters - Ignore URL parameters+ * regexOnFullURL - Apply regex to full URLs+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests++ Returns:+ AsyncV1CrawlWatcher: An instance to monitor the crawl job via WebSocket++ Raises:+ Exception: If crawl job fails to start+ """+ crawl_response = await self.async_crawl_url(url, params, idempotency_key)+ if crawl_response.get('success') and 'id' in crawl_response:+ return AsyncV1CrawlWatcher(crawl_response['id'], self)+ else:+ raise Exception("Crawl job failed to start")++ async def batch_scrape_urls_and_watch(+ self,+ urls: List[str],+ params: Optional[V1ScrapeParams] = None,+ idempotency_key: Optional[str] = None) -> 'AsyncV1CrawlWatcher':+ """+ Initiate an async batch scrape job and return an AsyncV1CrawlWatcher to monitor progress.++ Args:+ urls (List[str]): List of URLs to scrape+ params (Optional[V1ScrapeParams]): See V1ScrapeParams model for configuration:++ Content Options:+ * formats - Content formats to retrieve+ * includeTags - HTML tags to include+ * excludeTags - HTML tags to exclude+ * onlyMainContent - Extract main content only++ Request Options:+ * headers - Custom HTTP headers+ * timeout - Request timeout (ms)+ * mobile - Use mobile user agent+ * proxy - Proxy type++ Extraction Options:+ * extract - Content extraction config+ * jsonOptions - JSON extraction config+ * actions - Actions to perform+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests++ Returns:+ AsyncV1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket++ Raises:+ Exception: If batch scrape job fails to start+ """+ batch_response = await self.async_batch_scrape_urls(urls, params, idempotency_key)+ if batch_response.get('success') and 'id' in batch_response:+ return AsyncV1CrawlWatcher(batch_response['id'], self)+ else:+ raise Exception("Batch scrape job failed to start")++ async def scrape_url(+ self,+ url: str,+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ parse_pdf: Optional[bool] = None,+ extract: Optional[V1JsonConfig] = None,+ json_options: Optional[V1JsonConfig] = None,+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,+ **kwargs) -> V1ScrapeResponse[Any]:+ """+ Scrape a single URL asynchronously.++ Args:+ url (str): Target URL to scrape+ formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait for a specific element to appear+ timeout (Optional[int]): Request timeout (ms)+ location (Optional[V1LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 images+ block_ads (Optional[bool]): Block ads+ proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)+ extract (Optional[V1JsonConfig]): Content extraction settings+ json_options (Optional[V1JsonConfig]): JSON extraction settings+ actions (Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]]): Actions to perform+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1ScrapeResponse with:+ * success - Whether scrape was successful+ * markdown - Markdown content if requested+ * html - HTML content if requested+ * rawHtml - Raw HTML content if requested+ * links - Extracted links if requested+ * screenshot - Screenshot if requested+ * extract - Extracted data if requested+ * json - JSON data if requested+ * error - Error message if scrape failed++ Raises:+ Exception: If scraping fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "scrape_url")++ _headers = self._prepare_headers()++ # Build scrape parameters+ scrape_params = {+ 'url': url,+ 'origin': f"python-sdk@{version}"+ }++ # Add optional parameters if provided and not None+ if formats:+ scrape_params['formats'] = formats+ if headers:+ scrape_params['headers'] = headers+ if include_tags:+ scrape_params['includeTags'] = include_tags+ if exclude_tags:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for:+ scrape_params['waitFor'] = wait_for+ if timeout:+ scrape_params['timeout'] = timeout+ if location:+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy:+ scrape_params['proxy'] = proxy+ if parse_pdf is not None:+ scrape_params['parsePDF'] = parse_pdf+ if extract is not None:+ extract = self._ensure_schema_dict(extract)+ if isinstance(extract, dict) and "schema" in extract:+ extract["schema"] = self._ensure_schema_dict(extract["schema"])+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)+ if json_options is not None:+ json_options = self._ensure_schema_dict(json_options)+ if isinstance(json_options, dict) and "schema" in json_options:+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions:+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]+ if 'extract' in scrape_params and scrape_params['extract'] and 'schema' in scrape_params['extract']:+ scrape_params['extract']['schema'] = self._ensure_schema_dict(scrape_params['extract']['schema'])+ if 'jsonOptions' in scrape_params and scrape_params['jsonOptions'] and 'schema' in scrape_params['jsonOptions']:+ scrape_params['jsonOptions']['schema'] = self._ensure_schema_dict(scrape_params['jsonOptions']['schema'])++ # Make async request+ endpoint = f'/v1/scrape'+ response = await self._async_post_request(+ f'{self.api_url}{endpoint}',+ scrape_params,+ _headers+ )++ if response.get('success') and 'data' in response:+ return V1ScrapeResponse(**response['data'])+ elif "error" in response:+ raise Exception(f'Failed to scrape URL. Error: {response["error"]}')+ else:+ # Use the response content directly if possible, otherwise a generic message+ error_content = response.get('error', str(response))+ raise Exception(f'Failed to scrape URL. Error: {error_content}')++ async def batch_scrape_urls(+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ extract: Optional[V1JsonConfig] = None,+ json_options: Optional[V1JsonConfig] = None,+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,+ agent: Optional[V1AgentOptions] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> V1BatchScrapeStatusResponse:+ """+ Asynchronously scrape multiple URLs and monitor until completion.++ Args:+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[JsonConfig]): Content extraction config+ json_options (Optional[JsonConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1BatchScrapeStatusResponse with:+ * Scraping status and progress+ * Scraped content for each URL+ * Success/error information++ Raises:+ Exception: If batch scrape fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "batch_scrape_urls")++ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ extract = self._ensure_schema_dict(extract)+ if isinstance(extract, dict) and "schema" in extract:+ extract["schema"] = self._ensure_schema_dict(extract["schema"])+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)+ if json_options is not None:+ json_options = self._ensure_schema_dict(json_options)+ if isinstance(json_options, dict) and "schema" in json_options:+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions is not None:+ scrape_params['actions'] = [action.dict(by_alias=True, exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = V1ScrapeParams(**scrape_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = await self._async_post_request(+ f'{self.api_url}/v1/batch/scrape',+ params_dict,+ headers+ )++ if response.get('success'):+ try:+ id = response.get('id')+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ return await self._async_monitor_job_status(id, headers, poll_interval)+ else:+ self._handle_error(response, 'start batch scrape job')+++ async def async_batch_scrape_urls(+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ extract: Optional[V1JsonConfig] = None,+ json_options: Optional[V1JsonConfig] = None,+ actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,+ agent: Optional[V1AgentOptions] = None,+ zero_data_retention: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> V1BatchScrapeResponse:+ """+ Initiate a batch scrape job asynchronously.++ Args:+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[JsonConfig]): Content extraction config+ json_options (Optional[JsonConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ zero_data_retention (Optional[bool]): Whether to delete data after 24 hours+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1BatchScrapeResponse with:+ * success - Whether job started successfully+ * id - Unique identifier for the job+ * url - Status check URL+ * error - Error message if start failed++ Raises:+ Exception: If job initiation fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "async_batch_scrape_urls")++ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(by_alias=True, exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ extract = self._ensure_schema_dict(extract)+ if isinstance(extract, dict) and "schema" in extract:+ extract["schema"] = self._ensure_schema_dict(extract["schema"])+ scrape_params['extract'] = extract if isinstance(extract, dict) else extract.dict(by_alias=True, exclude_none=True)+ if json_options is not None:+ json_options = self._ensure_schema_dict(json_options)+ if isinstance(json_options, dict) and "schema" in json_options:+ json_options["schema"] = self._ensure_schema_dict(json_options["schema"])+ scrape_params['jsonOptions'] = json_options if isinstance(json_options, dict) else json_options.dict(by_alias=True, exclude_none=True)+ if actions:+ scrape_params['actions'] = [action if isinstance(action, dict) else action.dict(by_alias=True, exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(by_alias=True, exclude_none=True)+ if zero_data_retention is not None:+ scrape_params['zeroDataRetention'] = zero_data_retention++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = V1ScrapeParams(**scrape_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:+ params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])+ if 'jsonOptions' in params_dict and params_dict['jsonOptions'] and 'schema' in params_dict['jsonOptions']:+ params_dict['jsonOptions']['schema'] = self._ensure_schema_dict(params_dict['jsonOptions']['schema'])++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = await self._async_post_request(+ f'{self.api_url}/v1/batch/scrape',+ params_dict,+ headers+ )++ if response.get('status_code') == 200:+ try:+ return V1BatchScrapeResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ await self._handle_error(response, 'start batch scrape job')++ async def crawl_url(+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[V1ScrapeOptions] = None,+ webhook: Optional[Union[str, V1WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> V1CrawlStatusResponse:+ """+ Crawl a website starting from a URL.++ Args:+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration+ webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ delay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomains+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1CrawlStatusResponse with:+ * Crawling status and progress+ * Crawled page contents+ * Success/error information++ Raises:+ Exception: If crawl fails+ """+ # Validate any additional kwargs+ self._validate_kwargs(kwargs, "crawl_url")++ crawl_params = {}++ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomains++ # Add any additional kwargs+ crawl_params.update(kwargs)++ # Create final params object+ final_params = V1CrawlParams(**crawl_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"+ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = await self._async_post_request(+ f'{self.api_url}/v1/crawl', params_dict, headers)++ if response.get('success'):+ try:+ id = response.get('id')+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ return await self._async_monitor_job_status(id, headers, poll_interval)+ else:+ await self._handle_error(response, 'start crawl job')+++ async def async_crawl_url(+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[V1ScrapeOptions] = None,+ webhook: Optional[Union[str, V1WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> V1CrawlResponse:+ """+ Start an asynchronous crawl job.++ Args:+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[ScrapeOptions]): Page scraping configuration+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the API++ Returns:+ V1CrawlResponse with:+ * success - Whether crawl started successfully+ * id - Unique identifier for the crawl job+ * url - Status check URL for the crawl+ * error - Error message if start failed++ Raises:+ Exception: If crawl initiation fails+ """+ crawl_params = {}++ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomains++ # Add any additional kwargs+ crawl_params.update(kwargs)++ # Create final params object+ final_params = V1CrawlParams(**crawl_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = await self._async_post_request(+ f'{self.api_url}/v1/crawl',+ params_dict,+ headers+ )++ if response.get('success'):+ try:+ return V1CrawlResponse(**response)+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ await self._handle_error(response, 'start crawl job')++ async def check_crawl_status(self, id: str) -> V1CrawlStatusResponse:+ """+ Check the status and results of an asynchronous crawl job.++ Args:+ id (str): Unique identifier for the crawl job++ Returns:+ V1CrawlStatusResponse containing:+ Status Information:+ * status - Current state (scraping/completed/failed/cancelled)+ * completed - Number of pages crawled+ * total - Total pages to crawl+ * creditsUsed - API credits consumed+ * expiresAt - Data expiration timestamp++ Results:+ * data - List of crawled documents+ * next - URL for next page of results (if paginated)+ * success - Whether status check succeeded+ * error - Error message if failed++ Raises:+ Exception: If status check fails+ """+ headers = self._prepare_headers()+ endpoint = f'/v1/crawl/{id}'++ status_data = await self._async_get_request(+ f'{self.api_url}{endpoint}',+ headers+ )++ if status_data.get('status') == 'completed':+ if 'data' in status_data:+ data = status_data['data']+ while 'next' in status_data:+ if len(status_data['data']) == 0:+ break+ next_url = status_data.get('next')+ if not next_url:+ logger.warning("Expected 'next' URL is missing.")+ break+ next_data = await self._async_get_request(next_url, headers)+ data.extend(next_data.get('data', []))+ status_data = next_data+ status_data['data'] = data+ # Create V1CrawlStatusResponse object from status data+ response = V1CrawlStatusResponse(+ status=status_data.get('status'),+ total=status_data.get('total'),+ completed=status_data.get('completed'),+ creditsUsed=status_data.get('creditsUsed'),+ expiresAt=status_data.get('expiresAt'),+ data=status_data.get('data'),+ success=False if 'error' in status_data else True+ )++ if 'error' in status_data:+ response.error = status_data.get('error')++ if 'next' in status_data:+ response.next = status_data.get('next')++ return response++ async def _async_monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int = 2) -> V1CrawlStatusResponse:+ """+ Monitor the status of an asynchronous job until completion.++ Args:+ id (str): The ID of the job to monitor+ headers (Dict[str, str]): Headers to include in status check requests+ poll_interval (int): Seconds between status checks (default: 2)++ Returns:+ V1CrawlStatusResponse: The job results if completed successfully++ Raises:+ Exception: If the job fails or an error occurs during status checks+ """+ while True:+ status_data = await self._async_get_request(+ f'{self.api_url}/v1/crawl/{id}',+ headers+ )++ if status_data.get('status') == 'completed':+ if 'data' in status_data:+ data = status_data['data']+ while 'next' in status_data:+ if len(status_data['data']) == 0:+ break+ next_url = status_data.get('next')+ if not next_url:+ logger.warning("Expected 'next' URL is missing.")+ break+ next_data = await self._async_get_request(next_url, headers)+ data.extend(next_data.get('data', []))+ status_data = next_data+ status_data['data'] = data+ return V1CrawlStatusResponse(**status_data)+ else:+ raise Exception('Job completed but no data was returned')+ elif status_data.get('status') in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:+ await asyncio.sleep(max(poll_interval, 2))+ else:+ raise Exception(f'Job failed or was stopped. Status: {status_data["status"]}')++ async def map_url(+ self,+ url: str,+ *,+ search: Optional[str] = None,+ ignore_sitemap: Optional[bool] = None,+ include_subdomains: Optional[bool] = None,+ sitemap_only: Optional[bool] = None,+ limit: Optional[int] = None,+ timeout: Optional[int] = 30000,+ params: Optional[V1MapParams] = None) -> V1MapResponse:+ """+ Asynchronously map and discover links from a URL.++ Args:+ url (str): Target URL to map+ params (Optional[V1MapParams]): See V1MapParams model:+ Discovery Options:+ * search - Filter pattern for URLs+ * ignoreSitemap - Skip sitemap.xml+ * includeSubdomains - Include subdomain links+ * sitemapOnly - Only use sitemap.xml++ Limits:+ * limit - Max URLs to return+ * timeout - Request timeout (ms)++ Returns:+ V1MapResponse with:+ * Discovered URLs+ * Success/error status++ Raises:+ Exception: If mapping fails+ """+ map_params = {}+ if params:+ map_params.update(params.dict(by_alias=True, exclude_none=True))++ # Add individual parameters+ if search is not None:+ map_params['search'] = search+ if ignore_sitemap is not None:+ map_params['ignoreSitemap'] = ignore_sitemap+ if include_subdomains is not None:+ map_params['includeSubdomains'] = include_subdomains+ if sitemap_only is not None:+ map_params['sitemapOnly'] = sitemap_only+ if limit is not None:+ map_params['limit'] = limit+ if timeout is not None:+ map_params['timeout'] = timeout++ # Create final params object+ final_params = V1MapParams(**map_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ endpoint = f'/v1/map'+ response = await self._async_post_request(+ f'{self.api_url}{endpoint}',+ params_dict,+ headers={"Authorization": f"Bearer {self.api_key}"}+ )++ if response.get('success') and 'links' in response:+ return V1MapResponse(**response)+ elif 'error' in response:+ raise Exception(f'Failed to map URL. Error: {response["error"]}')+ else:+ raise Exception(f'Failed to map URL. Error: {response}')++ async def extract(+ self,+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Any] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = False,+ enable_web_search: Optional[bool] = False,+ show_sources: Optional[bool] = False,+ agent: Optional[Dict[str, Any]] = None) -> V1ExtractResponse[Any]:++ """+ Asynchronously extract structured information from URLs.++ Args:+ urls (Optional[List[str]]): URLs to extract from+ prompt (Optional[str]): Custom extraction prompt+ schema (Optional[Any]): JSON schema/Pydantic model+ system_prompt (Optional[str]): System context+ allow_external_links (Optional[bool]): Follow external links+ enable_web_search (Optional[bool]): Enable web search+ show_sources (Optional[bool]): Include source URLs+ agent (Optional[Dict[str, Any]]): Agent configuration++ Returns:+ V1ExtractResponse with:+ * Structured data matching schema+ * Source information if requested+ * Success/error status++ Raises:+ ValueError: If prompt/schema missing or extraction fails+ """+ headers = self._prepare_headers()++ if not prompt and not schema:+ raise ValueError("Either prompt or schema is required")++ if not urls and not prompt:+ raise ValueError("Either urls or prompt is required")++ if schema:+ schema = self._ensure_schema_dict(schema)++ request_data = {+ 'urls': urls or [],+ 'allowExternalLinks': allow_external_links,+ 'enableWebSearch': enable_web_search,+ 'showSources': show_sources,+ 'schema': schema,+ 'origin': f'python-sdk@{get_version()}'+ }++ # Only add prompt and systemPrompt if they exist+ if prompt:+ request_data['prompt'] = prompt+ if system_prompt:+ request_data['systemPrompt'] = system_prompt++ if agent:+ request_data['agent'] = agent++ response = await self._async_post_request(+ f'{self.api_url}/v1/extract',+ request_data,+ headers+ )++ if response.get('success'):+ job_id = response.get('id')+ if not job_id:+ raise Exception('Job ID not returned from extract request.')++ while True:+ status_data = await self._async_get_request(+ f'{self.api_url}/v1/extract/{job_id}',+ headers+ )++ if status_data['status'] == 'completed':+ return V1ExtractResponse(**status_data)+ elif status_data['status'] in ['failed', 'cancelled']:+ raise Exception(f'Extract job {status_data["status"]}. Error: {status_data["error"]}')++ await asyncio.sleep(2)+ else:+ raise Exception(f'Failed to extract. Error: {response.get("error")}')++ async def check_batch_scrape_status(self, id: str) -> V1BatchScrapeStatusResponse:+ """+ Check the status of an asynchronous batch scrape job.++ Args:+ id (str): The ID of the batch scrape job++ Returns:+ V1BatchScrapeStatusResponse containing:+ Status Information:+ * status - Current state (scraping/completed/failed/cancelled)+ * completed - Number of URLs scraped+ * total - Total URLs to scrape+ * creditsUsed - API credits consumed+ * expiresAt - Data expiration timestamp++ Results:+ * data - List of scraped documents+ * next - URL for next page of results (if paginated)+ * success - Whether status check succeeded+ * error - Error message if failed++ Raises:+ Exception: If status check fails+ """+ headers = self._prepare_headers()+ endpoint = f'/v1/batch/scrape/{id}'++ status_data = await self._async_get_request(+ f'{self.api_url}{endpoint}',+ headers+ )++ if status_data['status'] == 'completed':+ if 'data' in status_data:+ data = status_data['data']+ while 'next' in status_data:+ if len(status_data['data']) == 0:+ break+ next_url = status_data.get('next')+ if not next_url:+ logger.warning("Expected 'next' URL is missing.")+ break+ next_data = await self._async_get_request(next_url, headers)+ data.extend(next_data.get('data', []))+ status_data = next_data+ status_data['data'] = data++ response = V1BatchScrapeStatusResponse(+ status=status_data.get('status'),+ total=status_data.get('total'),+ completed=status_data.get('completed'),+ creditsUsed=status_data.get('creditsUsed'),+ expiresAt=status_data.get('expiresAt'),+ data=status_data.get('data')+ )++ if 'error' in status_data:+ response['error'] = status_data['error']++ if 'next' in status_data:+ response['next'] = status_data['next']++ return {+ 'success': False if 'error' in status_data else True,+ **response+ }++ async def check_batch_scrape_errors(self, id: str) -> V1CrawlErrorsResponse:+ """+ Get information about errors from an asynchronous batch scrape job.++ Args:+ id (str): The ID of the batch scrape job++ Returns:+ V1CrawlErrorsResponse containing:+ errors (List[Dict[str, str]]): List of errors with fields:+ * id (str): Error ID+ * timestamp (str): When the error occurred+ * url (str): URL that caused the error+ * error (str): Error message+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt++ Raises:+ Exception: If error check fails+ """+ headers = self._prepare_headers()+ return await self._async_get_request(+ f'{self.api_url}/v1/batch/scrape/{id}/errors',+ headers+ )++ async def check_crawl_errors(self, id: str) -> V1CrawlErrorsResponse:+ """+ Get information about errors from an asynchronous crawl job.++ Args:+ id (str): The ID of the crawl job++ Returns:+ V1CrawlErrorsResponse containing:+ * errors (List[Dict[str, str]]): List of errors with fields:+ - id (str): Error ID+ - timestamp (str): When the error occurred+ - url (str): URL that caused the error+ - error (str): Error message+ * robotsBlocked (List[str]): List of URLs blocked by robots.txt++ Raises:+ Exception: If error check fails+ """+ headers = self._prepare_headers()+ return await self._async_get_request(+ f'{self.api_url}/v1/crawl/{id}/errors',+ headers+ )++ async def cancel_crawl(self, id: str) -> Dict[str, Any]:+ """+ Cancel an asynchronous crawl job.++ Args:+ id (str): The ID of the crawl job to cancel++ Returns:+ Dict[str, Any] containing:+ * success (bool): Whether cancellation was successful+ * error (str, optional): Error message if cancellation failed++ Raises:+ Exception: If cancellation fails+ """+ headers = self._prepare_headers()+ async with aiohttp.ClientSession() as session:+ async with session.delete(f'{self.api_url}/v1/crawl/{id}', headers=headers) as response:+ return await response.json()++ async def get_extract_status(self, job_id: str) -> V1ExtractResponse[Any]:+ """+ Check the status of an asynchronous extraction job.++ Args:+ job_id (str): The ID of the extraction job++ Returns:+ V1ExtractResponse[Any] with:+ * success (bool): Whether request succeeded+ * data (Optional[Any]): Extracted data matching schema+ * error (Optional[str]): Error message if any+ * warning (Optional[str]): Warning message if any+ * sources (Optional[List[str]]): Source URLs if requested++ Raises:+ ValueError: If status check fails+ """+ headers = self._prepare_headers()+ try:+ return await self._async_get_request(+ f'{self.api_url}/v1/extract/{job_id}',+ headers+ )+ except Exception as e:+ raise ValueError(str(e))++ async def async_extract(+ self,+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Any] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = False,+ enable_web_search: Optional[bool] = False,+ show_sources: Optional[bool] = False,+ agent: Optional[Dict[str, Any]] = None) -> V1ExtractResponse[Any]:+ """+ Initiate an asynchronous extraction job without waiting for completion.++ Args:+ urls (Optional[List[str]]): URLs to extract from+ prompt (Optional[str]): Custom extraction prompt+ schema (Optional[Any]): JSON schema/Pydantic model+ system_prompt (Optional[str]): System context+ allow_external_links (Optional[bool]): Follow external links+ enable_web_search (Optional[bool]): Enable web search+ show_sources (Optional[bool]): Include source URLs+ agent (Optional[Dict[str, Any]]): Agent configuration+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests++ Returns:+ V1ExtractResponse[Any] with:+ * success (bool): Whether request succeeded+ * data (Optional[Any]): Extracted data matching schema+ * error (Optional[str]): Error message if any++ Raises:+ ValueError: If job initiation fails+ """+ headers = self._prepare_headers()++ if not prompt and not schema:+ raise ValueError("Either prompt or schema is required")++ if not urls and not prompt:+ raise ValueError("Either urls or prompt is required")++ if schema:+ schema = self._ensure_schema_dict(schema)++ request_data = V1ExtractResponse(+ urls=urls or [],+ allowExternalLinks=allow_external_links,+ enableWebSearch=enable_web_search,+ showSources=show_sources,+ schema=schema,+ origin=f'python-sdk@{version}'+ )++ if prompt:+ request_data['prompt'] = prompt+ if system_prompt:+ request_data['systemPrompt'] = system_prompt+ if agent:+ request_data['agent'] = agent++ try:+ return await self._async_post_request(+ f'{self.api_url}/v1/extract',+ request_data,+ headers+ )+ except Exception as e:+ raise ValueError(str(e))++ async def generate_llms_text(+ self,+ url: str,+ *,+ max_urls: Optional[int] = None,+ show_full_text: Optional[bool] = None,+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextStatusResponse:+ """+ Generate LLMs.txt for a given URL and monitor until completion.++ Args:+ url (str): Target URL to generate LLMs.txt from+ max_urls (Optional[int]): Maximum URLs to process (default: 10)+ show_full_text (Optional[bool]): Include full text in output (default: False)+ experimental_stream (Optional[bool]): Enable experimental streaming++ Returns:+ V1GenerateLLMsTextStatusResponse containing:+ * success (bool): Whether generation completed successfully+ * status (str): Status of generation (processing/completed/failed)+ * data (Dict[str, str], optional): Generated text with fields:+ - llmstxt (str): Generated LLMs.txt content+ - llmsfulltxt (str, optional): Full version if requested+ * error (str, optional): Error message if generation failed+ * expiresAt (str): When the generated data expires++ Raises:+ Exception: If generation fails+ """+ params = {}+ if max_urls is not None:+ params['maxUrls'] = max_urls+ if show_full_text is not None:+ params['showFullText'] = show_full_text+ if experimental_stream is not None:+ params['__experimental_stream'] = experimental_stream++ response = await self.async_generate_llms_text(+ url,+ max_urls=max_urls,+ show_full_text=show_full_text,+ experimental_stream=experimental_stream+ )+ if not response.get('success') or 'id' not in response:+ return response++ job_id = response['id']+ while True:+ status = await self.check_generate_llms_text_status(job_id)++ if status['status'] == 'completed':+ return status+ elif status['status'] == 'failed':+ raise Exception(f'LLMs.txt generation failed. Error: {status.get("error")}')+ elif status['status'] != 'processing':+ break++ await asyncio.sleep(2)++ return V1GenerateLLMsTextStatusResponse(success=False, error='LLMs.txt generation job terminated unexpectedly', status='failed', expiresAt='')++ async def async_generate_llms_text(+ self,+ url: str,+ *,+ max_urls: Optional[int] = None,+ show_full_text: Optional[bool] = None,+ cache: Optional[bool] = None,+ experimental_stream: Optional[bool] = None) -> V1GenerateLLMsTextResponse:+ """+ Initiate an asynchronous LLMs.txt generation job without waiting for completion.++ Args:+ url (str): Target URL to generate LLMs.txt from+ max_urls (Optional[int]): Maximum URLs to process (default: 10)+ show_full_text (Optional[bool]): Include full text in output (default: False)+ cache (Optional[bool]): Whether to use cached content if available (default: True)+ experimental_stream (Optional[bool]): Enable experimental streaming++ Returns:+ V1GenerateLLMsTextResponse containing:+ * success (bool): Whether job started successfully+ * id (str): Unique identifier for the job+ * error (str, optional): Error message if start failed++ Raises:+ ValueError: If job initiation fails+ """+ params = {}+ if max_urls is not None:+ params['maxUrls'] = max_urls+ if show_full_text is not None:+ params['showFullText'] = show_full_text+ if experimental_stream is not None:+ params['__experimental_stream'] = experimental_stream++ params = V1GenerateLLMsTextParams(+ maxUrls=max_urls,+ showFullText=show_full_text,+ cache=cache,+ __experimental_stream=experimental_stream+ )++ headers = self._prepare_headers()+ json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}+ json_data['origin'] = f"python-sdk@{version}"++ try:+ return await self._async_post_request(+ f'{self.api_url}/v1/llmstxt',+ json_data,+ headers+ )+ except Exception as e:+ raise ValueError(str(e))++ async def check_generate_llms_text_status(self, id: str) -> V1GenerateLLMsTextStatusResponse:+ """+ Check the status of an asynchronous LLMs.txt generation job.++ Args:+ id (str): The ID of the generation job++ Returns:+ V1GenerateLLMsTextStatusResponse containing:+ * success (bool): Whether generation completed successfully+ * status (str): Status of generation (processing/completed/failed)+ * data (Dict[str, str], optional): Generated text with fields:+ - llmstxt (str): Generated LLMs.txt content+ - llmsfulltxt (str, optional): Full version if requested+ * error (str, optional): Error message if generation failed+ * expiresAt (str): When the generated data expires++ Raises:+ ValueError: If status check fails+ """+ headers = self._prepare_headers()+ try:+ return await self._async_get_request(+ f'{self.api_url}/v1/llmstxt/{id}',+ headers+ )+ except Exception as e:+ raise ValueError(str(e))++ async def deep_research(+ self,+ query: str,+ *,+ max_depth: Optional[int] = None,+ time_limit: Optional[int] = None,+ max_urls: Optional[int] = None,+ analysis_prompt: Optional[str] = None,+ system_prompt: Optional[str] = None,+ __experimental_stream_steps: Optional[bool] = None,+ on_activity: Optional[Callable[[Dict[str, Any]], None]] = None,+ on_source: Optional[Callable[[Dict[str, Any]], None]] = None) -> V1DeepResearchStatusResponse:+ """+ Initiates a deep research operation on a given query and polls until completion.++ Args:+ query (str): Research query or topic to investigate+ max_depth (Optional[int]): Maximum depth of research exploration+ time_limit (Optional[int]): Time limit in seconds for research+ max_urls (Optional[int]): Maximum number of URLs to process+ analysis_prompt (Optional[str]): Custom prompt for analysis+ system_prompt (Optional[str]): Custom system prompt+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming+ on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth}+ on_source (Optional[Callable]): Source discovery callback receiving {url, title, description}++ Returns:+ DeepResearchStatusResponse containing:+ * success (bool): Whether research completed successfully+ * status (str): Current state (processing/completed/failed)+ * error (Optional[str]): Error message if failed+ * id (str): Unique identifier for the research job+ * data (Any): Research findings and analysis+ * sources (List[Dict]): List of discovered sources+ * activities (List[Dict]): Research progress log+ * summaries (List[str]): Generated research summaries++ Raises:+ Exception: If research fails+ """+ research_params = {}+ if max_depth is not None:+ research_params['maxDepth'] = max_depth+ if time_limit is not None:+ research_params['timeLimit'] = time_limit+ if max_urls is not None:+ research_params['maxUrls'] = max_urls+ if analysis_prompt is not None:+ research_params['analysisPrompt'] = analysis_prompt+ if system_prompt is not None:+ research_params['systemPrompt'] = system_prompt+ if __experimental_stream_steps is not None:+ research_params['__experimental_streamSteps'] = __experimental_stream_steps+ research_params = V1DeepResearchParams(**research_params)++ response = await self.async_deep_research(+ query,+ max_depth=max_depth,+ time_limit=time_limit,+ max_urls=max_urls,+ analysis_prompt=analysis_prompt,+ system_prompt=system_prompt+ )+ if not response.get('success') or 'id' not in response:+ return response++ job_id = response['id']+ last_activity_count = 0+ last_source_count = 0++ while True:+ status = await self.check_deep_research_status(job_id)++ if on_activity and 'activities' in status:+ new_activities = status['activities'][last_activity_count:]+ for activity in new_activities:+ on_activity(activity)+ last_activity_count = len(status['activities'])++ if on_source and 'sources' in status:+ new_sources = status['sources'][last_source_count:]+ for source in new_sources:+ on_source(source)+ last_source_count = len(status['sources'])++ if status['status'] == 'completed':+ return status+ elif status['status'] == 'failed':+ raise Exception(f'Deep research failed. Error: {status.get("error")}')+ elif status['status'] != 'processing':+ break++ await asyncio.sleep(2)++ return V1DeepResearchStatusResponse(success=False, error='Deep research job terminated unexpectedly')++ async def async_deep_research(+ self,+ query: str,+ *,+ max_depth: Optional[int] = None,+ time_limit: Optional[int] = None,+ max_urls: Optional[int] = None,+ analysis_prompt: Optional[str] = None,+ system_prompt: Optional[str] = None,+ __experimental_stream_steps: Optional[bool] = None) -> Dict[str, Any]:+ """+ Initiates an asynchronous deep research operation.++ Args:+ query (str): Research query or topic to investigate+ max_depth (Optional[int]): Maximum depth of research exploration+ time_limit (Optional[int]): Time limit in seconds for research+ max_urls (Optional[int]): Maximum number of URLs to process+ analysis_prompt (Optional[str]): Custom prompt for analysis+ system_prompt (Optional[str]): Custom system prompt+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming++ Returns:+ Dict[str, Any]: A response containing:+ * success (bool): Whether the research initiation was successful+ * id (str): The unique identifier for the research job+ * error (str, optional): Error message if initiation failed++ Raises:+ Exception: If the research initiation fails.+ """+ research_params = {}+ if max_depth is not None:+ research_params['maxDepth'] = max_depth+ if time_limit is not None:+ research_params['timeLimit'] = time_limit+ if max_urls is not None:+ research_params['maxUrls'] = max_urls+ if analysis_prompt is not None:+ research_params['analysisPrompt'] = analysis_prompt+ if system_prompt is not None:+ research_params['systemPrompt'] = system_prompt+ if __experimental_stream_steps is not None:+ research_params['__experimental_streamSteps'] = __experimental_stream_steps+ research_params = V1DeepResearchParams(**research_params)++ headers = self._prepare_headers()++ json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}+ json_data['origin'] = f"python-sdk@{version}"++ try:+ return await self._async_post_request(+ f'{self.api_url}/v1/deep-research',+ json_data,+ headers+ )+ except Exception as e:+ raise ValueError(str(e))++ async def check_deep_research_status(self, id: str) -> V1DeepResearchStatusResponse:+ """+ Check the status of a deep research operation.++ Args:+ id (str): The ID of the deep research operation.++ Returns:+ DeepResearchResponse containing:++ Status:+ * success - Whether research completed successfully+ * status - Current state (processing/completed/failed)+ * error - Error message if failed++ Results:+ * id - Unique identifier for the research job+ * data - Research findings and analysis+ * sources - List of discovered sources+ * activities - Research progress log+ * summaries - Generated research summaries++ Raises:+ Exception: If the status check fails.+ """+ headers = self._prepare_headers()+ try:+ return await self._async_get_request(+ f'{self.api_url}/v1/deep-research/{id}',+ headers+ )+ except Exception as e:+ raise ValueError(str(e))++ async def search(+ self,+ query: str,+ *,+ limit: Optional[int] = None,+ tbs: Optional[str] = None,+ filter: Optional[str] = None,+ lang: Optional[str] = None,+ country: Optional[str] = None,+ location: Optional[str] = None,+ timeout: Optional[int] = 30000,+ scrape_options: Optional[V1ScrapeOptions] = None,+ params: Optional[Union[Dict[str, Any], V1SearchParams]] = None,+ **kwargs) -> V1SearchResponse:+ """+ Asynchronously search for content using Firecrawl.++ Args:+ query (str): Search query string+ limit (Optional[int]): Max results (default: 5)+ tbs (Optional[str]): Time filter (e.g. "qdr:d")+ filter (Optional[str]): Custom result filter+ lang (Optional[str]): Language code (default: "en")+ country (Optional[str]): Country code (default: "us")+ location (Optional[str]): Geo-targeting+ timeout (Optional[int]): Request timeout in milliseconds+ scrape_options (Optional[ScrapeOptions]): Result scraping configuration+ params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters+ **kwargs: Additional keyword arguments for future compatibility++ Returns:+ SearchResponse: Response containing:+ * success (bool): Whether request succeeded+ * data (List[FirecrawlDocument]): Search results+ * warning (Optional[str]): Warning message if any+ * error (Optional[str]): Error message if any++ Raises:+ Exception: If search fails or response cannot be parsed+ """+ # Build search parameters+ search_params = {}+ if params:+ if isinstance(params, dict):+ search_params.update(params)+ else:+ search_params.update(params.dict(by_alias=True, exclude_none=True))++ # Add individual parameters+ if limit is not None:+ search_params['limit'] = limit+ if tbs is not None:+ search_params['tbs'] = tbs+ if filter is not None:+ search_params['filter'] = filter+ if lang is not None:+ search_params['lang'] = lang+ if country is not None:+ search_params['country'] = country+ if location is not None:+ search_params['location'] = location+ if timeout is not None:+ search_params['timeout'] = timeout+ if scrape_options is not None:+ search_params['scrapeOptions'] = scrape_options.dict(by_alias=True, exclude_none=True)++ # Add any additional kwargs+ search_params.update(kwargs)++ # Create final params object+ final_params = V1SearchParams(query=query, **search_params)+ params_dict = final_params.dict(by_alias=True, exclude_none=True)+ params_dict['origin'] = f"python-sdk@{version}"++ return await self._async_post_request(+ f"{self.api_url}/v1/search",+ params_dict,+ {"Authorization": f"Bearer {self.api_key}"}+ )++class AsyncV1CrawlWatcher(V1CrawlWatcher):+ """+ Async version of V1CrawlWatcher that properly handles async operations.+ """+ def __init__(self, id: str, app: AsyncV1FirecrawlApp):+ super().__init__(id, app)++ async def connect(self) -> None:+ """+ Establishes async WebSocket connection and starts listening for messages.+ """+ async with websockets.connect(+ self.ws_url,+ additional_headers=[("Authorization", f"Bearer {self.app.api_key}")]+ ) as websocket:+ await self._listen(websocket)++ async def _listen(self, websocket) -> None:+ """+ Listens for incoming WebSocket messages and handles them asynchronously.++ Args:+ websocket: The WebSocket connection object+ """+ async for message in websocket:+ msg = json.loads(message)+ await self._handle_message(msg)++ async def _handle_message(self, msg: Dict[str, Any]) -> None:+ """+ Handles incoming WebSocket messages based on their type asynchronously.++ Args:+ msg (Dict[str, Any]): The message to handle+ """+ if msg['type'] == 'done':+ self.status = 'completed'+ self.dispatch_event('done', {'status': self.status, 'data': self.data, 'id': self.id})+ elif msg['type'] == 'error':+ self.status = 'failed'+ self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error'], 'id': self.id})+ elif msg['type'] == 'catchup':+ self.status = msg['data']['status']+ self.data.extend(msg['data'].get('data', []))+ for doc in self.data:+ self.dispatch_event('document', {'data': doc, 'id': self.id})+ elif msg['type'] == 'document':+ self.data.append(msg['data'])+ self.dispatch_event('document', {'data': msg['data'], 'id': self.id})++ async def _handle_error(self, response: aiohttp.ClientResponse, action: str) -> None:+ """+ Handle errors from async API responses.+ """+ try:+ error_data = await response.json()+ error_message = error_data.get('error', 'No error message provided.')+ error_details = error_data.get('details', 'No additional error details provided.')+ except:+ raise aiohttp.ClientError(f'Failed to parse Firecrawl error response as JSON. Status code: {response.status}')++ # Use the app's method to get the error message+ message = await self.app._get_async_error_message(response.status, action, error_message, error_details)++ raise aiohttp.ClientError(message)++ async def _get_async_error_message(self, status_code: int, action: str, error_message: str, error_details: str) -> str:+ """+ Generate a standardized error message based on HTTP status code for async operations.++ Args:+ status_code (int): The HTTP status code from the response+ action (str): Description of the action that was being performed+ error_message (str): The error message from the API response+ error_details (str): Additional error details from the API response++ Returns:+ str: A formatted error message+ """+ return self._get_error_message(status_code, action, error_message, error_details)diff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.pynew file mode 100644index 000000000..5bedce8a8--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/__init__.py@@ -0,0 +1,129 @@+from typing import Optional, List, Union+from ..types import SearchResponse, ScrapeOptions, SearchRequest, SourceOption, FormatOption+from .methods.search import search as search_method+from .utils.http_client import HttpClient++class FirecrawlClient:+ """++ """++ def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):+ self.api_key = api_key or "placeholder-key"+ self.api_url = api_url+ self._client = HttpClient(api_key=self.api_key, api_url=self.api_url)++ # scrape+ def scrape(+ self+ ):+ pass++ # batch-scrape+ def batch_scrape(+ self+ ):+ pass++ # async-batch-scrape+ def async_batch_scrape(+ self+ ):+ pass++ # get-batch-scrape-status+ def get_batch_scrape_status(+ self+ ):+ pass++ # cancel-batch-scrape+ def cancel_batch_scrape(+ self+ ):+ pass+ # get-batch-scrape-errors+ def get_batch_scrape_errors(+ self+ ):+ pass++ # crawl+ def crawl(+ self+ ):+ pass++ # async-crawl+ def async_crawl(+ self+ ):+ pass+++ # crawl-params+ def crawl_params(+ self+ ):+ pass++ # get-crawl-status+ def get_crawl_status(+ self+ ):+ pass++ # cancel-crawl+ def cancel_crawl(+ self+ ):+ pass++ # get-active-crawls+ def get_active_crawls(+ self+ ):+ pass+ # get-crawl-errors+ def get_crawl_errors(+ self+ ):+ pass++ # map+ def map(+ self+ ):+ pass++ def search(+ self,+ query: str,+ sources: Optional[List[SourceOption]] = None,+ limit: Optional[int] = 5,+ tbs: Optional[str] = None,+ location: Optional[str] = None,+ ignore_invalid_urls: Optional[bool] = True,+ timeout: Optional[int] = 60000,+ scrape_options: Optional[ScrapeOptions] = None,+ ) -> SearchResponse:+ """Search for documents."""+ request = SearchRequest(+ query=query,+ sources=sources,+ limit=limit,+ tbs=tbs,+ location=location,+ ignore_invalid_urls=ignore_invalid_urls,+ timeout=timeout,+ scrape_options=scrape_options,+ )+ return search_method(self._client, request)++ # credit-usage+ def credit_usage(+ self+ ):+ pass++__all__ = ['FirecrawlClient']\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pynew file mode 100644index 000000000..113ca961e--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -0,0 +1,121 @@+"""+Main Firecrawl v2 API client.++This module provides the main client class that orchestrates all v2 functionality.+"""++import os+from typing import Optional, List, Dict, Any, Callable+from .types import (+ ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions,+ ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse,+ BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document,+ SearchRequest, SearchResponse+)+from .utils.http_client import HttpClient+from .utils.error_handler import FirecrawlError+# Import functions from modules+from . import scrape as scrape_module+from . import crawl as crawl_module+from . import batch as batch_module+from . import search as search_module+++class FirecrawlClient:+ """+ Main Firecrawl v2 API client.++ This client provides a clean, modular interface to all Firecrawl functionality.+ """++ def __init__(+ self,+ api_key: Optional[str] = None,+ api_url: str = "https://api.firecrawl.dev",+ timeout: Optional[float] = None,+ max_retries: int = 3,+ backoff_factor: float = 0.5+ ):+ """+ Initialize the Firecrawl client.++ Args:+ api_key: Firecrawl API key (or set FIRECRAWL_API_KEY env var)+ api_url: Base URL for the Firecrawl API+ timeout: Request timeout in seconds+ max_retries: Maximum number of retries for failed requests+ backoff_factor: Exponential backoff factor for retries (e.g. 0.5 means wait 0.5s, then 1s, then 2s between retries)+ """+ if api_key is None:+ api_key = os.getenv("FIRECRAWL_API_KEY")++ if not api_key:+ raise ValueError(+ "API key is required. Set FIRECRAWL_API_KEY environment variable "+ "or pass api_key parameter."+ )++ self.config = ClientConfig(+ api_key=api_key,+ api_url=api_url,+ timeout=timeout,+ max_retries=max_retries,+ backoff_factor=backoff_factor+ )++ self.http_client = HttpClient(api_key, api_url)++ # def scrape(+ # self,+ # url: str,+ # options: Optional[ScrapeOptions] = None+ # ) -> ScrapeResponse:+ # """+ # Scrape a single URL.++ # Args:+ # url: URL to scrape+ # options: Scraping options++ # Returns:+ # ScrapeResponse containing the scraped document+ # """++ # return scrape_module.scrape(self.http_client, url, options)+++ def search(+ self,+ query: str,+ *,+ limit: Optional[int] = None,+ tbs: Optional[str] = None,+ location: Optional[str] = None,+ timeout: Optional[int] = None,+ page_options: Optional[ScrapeOptions] = None,+ ) -> SearchResponse:+ """+ Search for documents.++ Args:+ query: Search query string+ limit: Maximum number of results to return (default: 5)+ tbs: Time-based search filter+ location: Location string for search+ timeout: Request timeout in milliseconds (default: 60000)+ page_options: Options for scraping individual pages++ Returns:+ SearchResponse containing the search results+ """+ # Build options object from individual parameters+ options = SearchRequest(+ limit=limit,+ tbs=tbs,+ location=location,+ timeout=timeout,+ page_options=page_options+ )++ return search_module.search(self.http_client, query, options)+\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pynew file mode 100644index 000000000..f74524aae--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -0,0 +1,383 @@+"""+Batch scraping functionality for Firecrawl v2 API.+"""++import time+from typing import Optional, List, Callable+from .types import (+ BatchScrapeRequest, BatchScrapeResponse, BatchScrapeStatusResponse,+ BatchScrapeJob, BatchScrapeStatusData, ScrapeOptions, Document+)+from .utils.http_client import HttpClient+from .utils.error_handler import handle_response_error+++def start_batch_scrape(+ client: HttpClient,+ urls: List[str],+ options: Optional[ScrapeOptions] = None+) -> BatchScrapeResponse:+ """+ Start a batch scrape job for multiple URLs.++ Args:+ client: HTTP client instance+ urls: List of URLs to scrape+ options: Scraping options++ Returns:+ BatchScrapeResponse containing job information++ Raises:+ FirecrawlError: If the batch scrape operation fails to start+ """+ # Prepare request data+ request_data = {"urls": urls}++ if options:+ request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)++ # Make the API request+ response = client.post("/v1/batch/scrape", request_data)++ # Handle errors+ if not response.ok:+ handle_response_error(response, "start batch scrape")++ # Parse response+ response_data = response.json()++ if response_data.get("success"):+ job_data = response_data.get("data", {})+ job = BatchScrapeJob(**job_data)++ return BatchScrapeResponse(+ success=True,+ data=job,+ warning=response_data.get("warning")+ )+ else:+ return BatchScrapeResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def get_batch_scrape_status(+ client: HttpClient,+ job_id: str+) -> BatchScrapeStatusResponse:+ """+ Get the status of a batch scrape job.++ Args:+ client: HTTP client instance+ job_id: ID of the batch scrape job++ Returns:+ BatchScrapeStatusResponse containing job status and data++ Raises:+ FirecrawlError: If the status check fails+ """+ # Make the API request+ response = client.get(f"/v1/batch/scrape/{job_id}")++ # Handle errors+ if not response.ok:+ handle_response_error(response, "get batch scrape status")++ # Parse response+ response_data = response.json()++ if response_data.get("success"):+ status_data = response_data.get("data", {})++ # Convert documents+ documents = []+ if "data" in status_data:+ for doc_data in status_data["data"]:+ documents.append(Document(**doc_data))++ batch_status = BatchScrapeStatusData(+ status=status_data.get("status"),+ current=status_data.get("current", 0),+ total=status_data.get("total", 0),+ data=documents+ )++ return BatchScrapeStatusResponse(+ success=True,+ data=batch_status,+ warning=response_data.get("warning")+ )+ else:+ return BatchScrapeStatusResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def cancel_batch_scrape(+ client: HttpClient,+ job_id: str+) -> BatchScrapeStatusResponse:+ """+ Cancel a running batch scrape job.++ Args:+ client: HTTP client instance+ job_id: ID of the batch scrape job to cancel++ Returns:+ BatchScrapeStatusResponse with updated status++ Raises:+ FirecrawlError: If the cancellation fails+ """+ # Make the API request+ response = client.delete(f"/v1/batch/scrape/{job_id}")++ # Handle errors+ if not response.ok:+ handle_response_error(response, "cancel batch scrape")++ # Parse response+ response_data = response.json()++ if response_data.get("success"):+ status_data = response_data.get("data", {})++ batch_status = BatchScrapeStatusData(+ status=status_data.get("status", "cancelled"),+ current=status_data.get("current", 0),+ total=status_data.get("total", 0),+ data=[]+ )++ return BatchScrapeStatusResponse(+ success=True,+ data=batch_status,+ warning=response_data.get("warning")+ )+ else:+ return BatchScrapeStatusResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def wait_for_batch_completion(+ client: HttpClient,+ job_id: str,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+ progress_callback: Optional[Callable[[BatchScrapeStatusData], None]] = None+) -> BatchScrapeStatusResponse:+ """+ Wait for a batch scrape job to complete, polling for status updates.++ Args:+ client: HTTP client instance+ job_id: ID of the batch scrape job+ poll_interval: Seconds between status checks+ timeout: Maximum seconds to wait (None for no timeout)+ progress_callback: Optional callback for progress updates++ Returns:+ BatchScrapeStatusResponse when job completes++ Raises:+ FirecrawlError: If the job fails or timeout is reached+ TimeoutError: If timeout is reached+ """+ start_time = time.time()++ while True:+ status_response = get_batch_scrape_status(client, job_id)++ if not status_response.success:+ return status_response++ status_data = status_response.data++ # Call progress callback if provided+ if progress_callback and status_data:+ progress_callback(status_data)++ # Check if job is complete+ if status_data and status_data.status in ["completed", "failed", "cancelled"]:+ return status_response++ # Check timeout+ if timeout and (time.time() - start_time) > timeout:+ raise TimeoutError(f"Batch scrape job {job_id} did not complete within {timeout} seconds")++ # Wait before next poll+ time.sleep(poll_interval)+++def batch_scrape_and_wait(+ client: HttpClient,+ urls: List[str],+ options: Optional[ScrapeOptions] = None,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+ progress_callback: Optional[Callable[[BatchScrapeStatusData], None]] = None+) -> BatchScrapeStatusResponse:+ """+ Start a batch scrape job and wait for it to complete.++ Args:+ client: HTTP client instance+ urls: List of URLs to scrape+ options: Scraping options+ poll_interval: Seconds between status checks+ timeout: Maximum seconds to wait (None for no timeout)+ progress_callback: Optional callback for progress updates++ Returns:+ BatchScrapeStatusResponse when job completes++ Raises:+ FirecrawlError: If the batch scrape fails to start or complete+ TimeoutError: If timeout is reached+ """+ # Start the batch scrape+ batch_response = start_batch_scrape(client, urls, options)++ if not batch_response.success or not batch_response.data:+ return BatchScrapeStatusResponse(+ success=False,+ error=batch_response.error or "Failed to start batch scrape"+ )++ job_id = batch_response.data.id++ # Wait for completion+ return wait_for_batch_completion(+ client, job_id, poll_interval, timeout, progress_callback+ )+++def validate_batch_urls(urls: List[str]) -> List[str]:+ """+ Validate and normalize a list of URLs for batch scraping.++ Args:+ urls: List of URLs to validate++ Returns:+ Validated list of URLs++ Raises:+ ValueError: If URLs are invalid+ """+ if not urls:+ raise ValueError("URLs list cannot be empty")++ if len(urls) > 1000: # Assuming API limit+ raise ValueError("Too many URLs (maximum 1000)")++ validated_urls = []+ for url in urls:+ if not url or not isinstance(url, str):+ raise ValueError(f"Invalid URL: {url}")++ # Basic URL validation+ if not (url.startswith("http://") or url.startswith("https://")):+ raise ValueError(f"URL must start with http:// or https://: {url}")++ validated_urls.append(url.strip())++ return validated_urls+++def prepare_batch_request(urls: List[str], options: Optional[ScrapeOptions] = None) -> dict:+ """+ Prepare a batch scrape request payload.++ Args:+ urls: List of URLs to scrape+ options: Scraping options++ Returns:+ Request payload dictionary+ """+ validated_urls = validate_batch_urls(urls)+ request_data = {"urls": validated_urls}++ if options:+ request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)++ return request_data+++def chunk_urls(urls: List[str], chunk_size: int = 100) -> List[List[str]]:+ """+ Split a large list of URLs into smaller chunks for batch processing.++ Args:+ urls: List of URLs to chunk+ chunk_size: Maximum size of each chunk++ Returns:+ List of URL chunks+ """+ chunks = []+ for i in range(0, len(urls), chunk_size):+ chunks.append(urls[i:i + chunk_size])+ return chunks+++def process_large_batch(+ client: HttpClient,+ urls: List[str],+ options: Optional[ScrapeOptions] = None,+ chunk_size: int = 100,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+ progress_callback: Optional[Callable[[int, int, List[Document]], None]] = None+) -> List[Document]:+ """+ Process a large batch of URLs by splitting into smaller chunks.++ Args:+ client: HTTP client instance+ urls: List of URLs to scrape+ options: Scraping options+ chunk_size: Size of each batch chunk+ poll_interval: Seconds between status checks+ timeout: Maximum seconds to wait per chunk+ progress_callback: Optional callback for overall progress (completed, total, documents)++ Returns:+ List of all scraped documents++ Raises:+ FirecrawlError: If any chunk fails+ """+ url_chunks = chunk_urls(urls, chunk_size)+ all_documents = []+ completed_chunks = 0++ for chunk in url_chunks:+ # Process this chunk+ result = batch_scrape_and_wait(+ client, chunk, options, poll_interval, timeout+ )++ if not result.success or not result.data:+ raise Exception(f"Failed to process chunk: {result.error}")++ # Add documents from this chunk+ if result.data.data:+ all_documents.extend(result.data.data)++ completed_chunks += 1++ # Call progress callback if provided+ if progress_callback:+ progress_callback(completed_chunks, len(url_chunks), all_documents)++ return all_documents\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pynew file mode 100644index 000000000..922e28843--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -0,0 +1,360 @@+"""+Crawling functionality for Firecrawl v2 API.+"""++import time+from typing import Optional, Callable, Generator+from .types import (+ CrawlRequest, CrawlResponse, CrawlStatusResponse, CrawlOptions,+ CrawlJob, CrawlStatusData, Document+)+from .utils.http_client import HttpClient+from .utils.error_handler import handle_response_error+++def start_crawl(+ client: HttpClient,+ url: str,+ options: Optional[CrawlOptions] = None+) -> CrawlResponse:+ """+ Start a crawl job for a website.++ Args:+ client: HTTP client instance+ url: URL to crawl+ options: Crawling options++ Returns:+ CrawlResponse containing job information++ Raises:+ FirecrawlError: If the crawl operation fails to start+ """+ # Prepare request data+ request_data = {"url": url}++ if options:+ request_data.update(options.dict(exclude_none=True, by_alias=True))++ # Make the API request+ response = client.post("/v1/crawl", request_data)++ # Handle errors+ if not response.ok:+ handle_response_error(response, "start crawl")++ # Parse response+ response_data = response.json()++ if response_data.get("success"):+ job_data = response_data.get("data", {})+ job = CrawlJob(**job_data)++ return CrawlResponse(+ success=True,+ data=job,+ warning=response_data.get("warning")+ )+ else:+ return CrawlResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def get_crawl_status(+ client: HttpClient,+ job_id: str+) -> CrawlStatusResponse:+ """+ Get the status of a crawl job.++ Args:+ client: HTTP client instance+ job_id: ID of the crawl job++ Returns:+ CrawlStatusResponse containing job status and data++ Raises:+ FirecrawlError: If the status check fails+ """+ # Make the API request+ response = client.get(f"/v1/crawl/{job_id}")++ # Handle errors+ if not response.ok:+ handle_response_error(response, "get crawl status")++ # Parse response+ response_data = response.json()++ if response_data.get("success"):+ status_data = response_data.get("data", {})++ # Convert documents+ documents = []+ if "data" in status_data:+ for doc_data in status_data["data"]:+ documents.append(Document(**doc_data))++ # Convert partial data if present+ partial_documents = []+ if "partialData" in status_data:+ for doc_data in status_data["partialData"]:+ partial_documents.append(Document(**doc_data))++ crawl_status = CrawlStatusData(+ status=status_data.get("status"),+ current=status_data.get("current", 0),+ total=status_data.get("total", 0),+ data=documents,+ partial_data=partial_documents if partial_documents else None+ )++ return CrawlStatusResponse(+ success=True,+ data=crawl_status,+ warning=response_data.get("warning")+ )+ else:+ return CrawlStatusResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def cancel_crawl(+ client: HttpClient,+ job_id: str+) -> CrawlStatusResponse:+ """+ Cancel a running crawl job.++ Args:+ client: HTTP client instance+ job_id: ID of the crawl job to cancel++ Returns:+ CrawlStatusResponse with updated status++ Raises:+ FirecrawlError: If the cancellation fails+ """+ # Make the API request+ response = client.delete(f"/v1/crawl/{job_id}")++ # Handle errors+ if not response.ok:+ handle_response_error(response, "cancel crawl")++ # Parse response+ response_data = response.json()++ if response_data.get("success"):+ status_data = response_data.get("data", {})++ crawl_status = CrawlStatusData(+ status=status_data.get("status", "cancelled"),+ current=status_data.get("current", 0),+ total=status_data.get("total", 0),+ data=[]+ )++ return CrawlStatusResponse(+ success=True,+ data=crawl_status,+ warning=response_data.get("warning")+ )+ else:+ return CrawlStatusResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def wait_for_crawl_completion(+ client: HttpClient,+ job_id: str,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+ progress_callback: Optional[Callable[[CrawlStatusData], None]] = None+) -> CrawlStatusResponse:+ """+ Wait for a crawl job to complete, polling for status updates.++ Args:+ client: HTTP client instance+ job_id: ID of the crawl job+ poll_interval: Seconds between status checks+ timeout: Maximum seconds to wait (None for no timeout)+ progress_callback: Optional callback for progress updates++ Returns:+ CrawlStatusResponse when job completes++ Raises:+ FirecrawlError: If the job fails or timeout is reached+ TimeoutError: If timeout is reached+ """+ start_time = time.time()++ while True:+ status_response = get_crawl_status(client, job_id)++ if not status_response.success:+ return status_response++ status_data = status_response.data++ # Call progress callback if provided+ if progress_callback and status_data:+ progress_callback(status_data)++ # Check if job is complete+ if status_data and status_data.status in ["completed", "failed", "cancelled"]:+ return status_response++ # Check timeout+ if timeout and (time.time() - start_time) > timeout:+ raise TimeoutError(f"Crawl job {job_id} did not complete within {timeout} seconds")++ # Wait before next poll+ time.sleep(poll_interval)+++def crawl_and_wait(+ client: HttpClient,+ url: str,+ options: Optional[CrawlOptions] = None,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+ progress_callback: Optional[Callable[[CrawlStatusData], None]] = None+) -> CrawlStatusResponse:+ """+ Start a crawl job and wait for it to complete.++ Args:+ client: HTTP client instance+ url: URL to crawl+ options: Crawling options+ poll_interval: Seconds between status checks+ timeout: Maximum seconds to wait (None for no timeout)+ progress_callback: Optional callback for progress updates++ Returns:+ CrawlStatusResponse when job completes++ Raises:+ FirecrawlError: If the crawl fails to start or complete+ TimeoutError: If timeout is reached+ """+ # Start the crawl+ crawl_response = start_crawl(client, url, options)++ if not crawl_response.success or not crawl_response.data:+ return CrawlStatusResponse(+ success=False,+ error=crawl_response.error or "Failed to start crawl"+ )++ job_id = crawl_response.data.id++ # Wait for completion+ return wait_for_crawl_completion(+ client, job_id, poll_interval, timeout, progress_callback+ )+++def stream_crawl_results(+ client: HttpClient,+ job_id: str,+ poll_interval: int = 2+) -> Generator[Document, None, None]:+ """+ Stream crawl results as they become available.++ Args:+ client: HTTP client instance+ job_id: ID of the crawl job+ poll_interval: Seconds between status checks++ Yields:+ Document objects as they are crawled++ Raises:+ FirecrawlError: If the job fails+ """+ seen_count = 0++ while True:+ status_response = get_crawl_status(client, job_id)++ if not status_response.success:+ raise Exception(f"Failed to get crawl status: {status_response.error}")++ status_data = status_response.data+ if not status_data:+ break++ # Yield new documents+ if status_data.data:+ for i in range(seen_count, len(status_data.data)):+ yield status_data.data[i]+ seen_count = len(status_data.data)++ # Check if job is complete+ if status_data.status in ["completed", "failed", "cancelled"]:+ break++ # Wait before next poll+ time.sleep(poll_interval)+++def validate_crawl_options(options: Optional[CrawlOptions]) -> Optional[CrawlOptions]:+ """+ Validate and normalize crawl options.++ Args:+ options: Crawling options to validate++ Returns:+ Validated options or None++ Raises:+ ValueError: If options are invalid+ """+ if options is None:+ return None++ # Validate limit+ if options.limit is not None and options.limit <= 0:+ raise ValueError("Limit must be positive")++ # Validate max_depth+ if options.max_depth is not None and options.max_depth < 0:+ raise ValueError("max_depth must be non-negative")++ return options+++def prepare_crawl_request(url: str, options: Optional[CrawlOptions] = None) -> dict:+ """+ Prepare a crawl request payload.++ Args:+ url: URL to crawl+ options: Crawling options++ Returns:+ Request payload dictionary+ """+ request_data = {"url": url}++ if options:+ validated_options = validate_crawl_options(options)+ if validated_options:+ request_data.update(validated_options.dict(exclude_none=True, by_alias=True))++ return request_data\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.pynew file mode 100644index 000000000..eb6e6eaef--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py@@ -0,0 +1,215 @@+"""+Scraping functionality for Firecrawl v2 API.+"""++import json+from typing import Optional, Dict, Any+from .types import ScrapeRequest, ScrapeResponse, ScrapeOptions, Document+from .utils.http_client import HttpClient+from .utils.error_handler import handle_response_error+++def scrape(+ client: HttpClient,+ url: str,+ options: Optional[ScrapeOptions] = None+) -> ScrapeResponse:+ """+ Scrape a single URL and return the content.++ Args:+ client: HTTP client instance+ url: URL to scrape+ options: Scraping options++ Returns:+ ScrapeResponse containing the scraped document++ Raises:+ FirecrawlError: If the scraping operation fails+ """+ # Prepare request data+ request_data = {"url": url}++ if options:+ request_data["options"] = options.dict(exclude_none=True, by_alias=True)++ # Make the API request+ response = client.post("/v1/scrape", request_data)++ # Handle errors+ if not response.ok:+ handle_response_error(response, "scrape URL")++ # Parse response+ response_data = response.json()++ # Create document from response+ if response_data.get("success"):+ document_data = response_data.get("data", {})+ document = Document(**document_data)++ return ScrapeResponse(+ success=True,+ data=document,+ warning=response_data.get("warning")+ )+ else:+ return ScrapeResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def scrape_with_extract(+ client: HttpClient,+ url: str,+ schema: Dict[str, Any],+ options: Optional[ScrapeOptions] = None,+ system_prompt: Optional[str] = None,+ prompt: Optional[str] = None+) -> ScrapeResponse:+ """+ Scrape a URL and extract structured data using a schema.++ Args:+ client: HTTP client instance+ url: URL to scrape+ schema: JSON schema for data extraction+ options: Scraping options+ system_prompt: System prompt for extraction+ prompt: User prompt for extraction++ Returns:+ ScrapeResponse with extracted data++ Raises:+ FirecrawlError: If the operation fails+ """+ # Prepare request data+ request_data = {+ "url": url,+ "extractorOptions": {+ "extractionSchema": schema+ }+ }++ if system_prompt:+ request_data["extractorOptions"]["extractionPrompt"] = system_prompt++ if prompt:+ request_data["extractorOptions"]["userPrompt"] = prompt++ if options:+ request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)++ # Make the API request+ response = client.post("/v1/scrape", request_data)++ # Handle errors+ if not response.ok:+ handle_response_error(response, "scrape and extract from URL")++ # Parse response+ response_data = response.json()++ # Create document from response+ if response_data.get("success"):+ document_data = response_data.get("data", {})+ document = Document(**document_data)++ return ScrapeResponse(+ success=True,+ data=document,+ warning=response_data.get("warning")+ )+ else:+ return ScrapeResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def scrape_multiple(+ client: HttpClient,+ urls: list[str],+ options: Optional[ScrapeOptions] = None+) -> list[ScrapeResponse]:+ """+ Scrape multiple URLs concurrently.++ Args:+ client: HTTP client instance+ urls: List of URLs to scrape+ options: Scraping options++ Returns:+ List of ScrapeResponse objects++ Note:+ This function makes individual requests for each URL.+ For large batches, consider using batch_scrape instead.+ """+ results = []++ for url in urls:+ try:+ result = scrape(client, url, options)+ results.append(result)+ except Exception as e:+ # Create error response for failed scrapes+ error_response = ScrapeResponse(+ success=False,+ error=f"Failed to scrape {url}: {str(e)}"+ )+ results.append(error_response)++ return results+++def validate_scrape_options(options: Optional[ScrapeOptions]) -> Optional[ScrapeOptions]:+ """+ Validate and normalize scrape options.++ Args:+ options: Scraping options to validate++ Returns:+ Validated options or None++ Raises:+ ValueError: If options are invalid+ """+ if options is None:+ return None++ # Validate timeout+ if options.timeout is not None and options.timeout <= 0:+ raise ValueError("Timeout must be positive")++ # Validate wait_for+ if options.wait_for is not None and options.wait_for < 0:+ raise ValueError("wait_for must be non-negative")++ return options+++def prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:+ """+ Prepare a scrape request payload.++ Args:+ url: URL to scrape+ options: Scraping options++ Returns:+ Request payload dictionary+ """+ request_data = {"url": url}++ if options:+ validated_options = validate_scrape_options(options)+ if validated_options:+ request_data["options"] = validated_options.dict(exclude_none=True, by_alias=True)++ return request_data\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pynew file mode 100644index 000000000..ea56a6e19--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -0,0 +1,101 @@+"""+Search functionality for Firecrawl v2 API.+"""++from typing import Optional, Dict, Any, Union+from ...types import SearchRequest, SearchResponse, SearchResult, Document+from ..utils.http_client import HttpClient+from ..utils.error_handler import handle_response_error+++def search(+ client: HttpClient,+ request: SearchRequest+) -> SearchResponse:+ """+ Search for documents.++ Args:+ client: HTTP client instance+ request: Search request++ Returns:+ SearchResponse containing the search results++ Raises:+ FirecrawlError: If the search operation fails+ """+ request_data = _prepare_search_request(request)++ response = client.post("/v2/search", request_data)++ if not response.ok:+ handle_response_error(response, "search")++ response_data = response.json()++ if response_data.get("success"):+ data = response_data.get("data", {})+ grouped_results = {}++ for source_type, source_documents in data.items():+ if isinstance(source_documents, list):+ results = []+ for doc_data in source_documents:+ if isinstance(doc_data, dict):+ if any(key in doc_data for key in ['markdown', 'html', 'content', 'screenshot']):+ results.append(Document(**doc_data))+ else:+ results.append(SearchResult(+ url=doc_data.get('url', ''),+ title=doc_data.get('title'),+ description=doc_data.get('description')+ ))+ elif isinstance(doc_data, str):+ results.append(SearchResult(url=doc_data))+ grouped_results[source_type] = results++ return SearchResponse(+ success=True,+ data=grouped_results,+ warning=response_data.get("warning")+ )+ else:+ return SearchResponse(+ success=False,+ error=response_data.get("error", "Unknown error occurred")+ )+++def _validate_search_request(request: SearchRequest) -> SearchRequest:+ """+ Validate and normalize search request.++ Args:+ request: Search request to validate++ Returns:+ Validated request++ Raises:+ ValueError: If request is invalid+ """+ # Validate timeout if present+ if request.timeout is not None and request.timeout <= 0:+ raise ValueError("Timeout must be positive")++ return request+++def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:+ """+ Prepare a search request payload.++ Args:+ request: Search request++ Returns:+ Request payload dictionary+ """+ validated_request = _validate_search_request(request)+ return validated_request.model_dump(exclude_none=True, by_alias=True)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pynew file mode 100644index 000000000..9884a1981--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -0,0 +1,376 @@+"""+Type definitions for Firecrawl v2 API.++This module contains clean, modern type definitions for the v2 API.+"""++from typing import Dict, List, Optional, Union, Literal, Any, TypeVar, Generic+from pydantic import BaseModel, Field, field_validator+from datetime import datetime++T = TypeVar('T')++# Base response types+class BaseResponse(BaseModel, Generic[T]):+ """Base response structure for all API responses."""+ success: bool+ data: Optional[T] = None+ error: Optional[str] = None+ warning: Optional[str] = None++# Document and content types+class DocumentMetadata(BaseModel):+ """Metadata for scraped documents."""+ title: Optional[str] = None+ description: Optional[str] = None+ language: Optional[str] = None+ keywords: Optional[str] = None+ robots: Optional[str] = None+ og_title: Optional[str] = Field(None, alias="ogTitle")+ og_description: Optional[str] = Field(None, alias="ogDescription")+ og_url: Optional[str] = Field(None, alias="ogUrl")+ og_image: Optional[str] = Field(None, alias="ogImage")+ source_url: Optional[str] = Field(None, alias="sourceURL")+ status_code: Optional[int] = Field(None, alias="statusCode")+ error: Optional[str] = None++class Document(BaseModel):+ """A scraped document."""+ url: str+ markdown: Optional[str] = None+ html: Optional[str] = None+ raw_html: Optional[str] = Field(None, alias="rawHtml")+ content: Optional[str] = None+ metadata: Optional[DocumentMetadata] = None+ links: Optional[List[str]] = None+ screenshot: Optional[str] = None+ actions: Optional[Dict[str, Any]] = None++class Source(BaseModel):+ """Configuration for a search source."""+ type: str++SourceOption = Union[str, Source]++class Format(BaseModel):+ """Configuration for a format."""+ type: str++FormatOption = Union[str, Format]++# Scrape types+class ScrapeFormats(BaseModel):+ """Output formats for scraping."""+ formats: Optional[List[FormatOption]] = None+ markdown: bool = True+ html: bool = False+ raw_html: bool = Field(False, alias="rawHtml")+ content: bool = False+ links: bool = False+ screenshot: bool = False+ screenshot_full_page: bool = Field(False, alias="screenshot@fullPage")++ @field_validator('formats')+ @classmethod+ def validate_formats(cls, v):+ """Validate and normalize formats input."""+ if v is None:+ return v++ normalized_formats = []+ for format_item in v:+ if isinstance(format_item, str):+ normalized_formats.append(Format(type=format_item))+ elif isinstance(format_item, dict):+ normalized_formats.append(Format(**format_item))+ elif isinstance(format_item, Format):+ normalized_formats.append(format_item)+ else:+ raise ValueError(f"Invalid format format: {format_item}")++ return normalized_formats++class ScrapeOptions(BaseModel):+ """Options for scraping operations."""+ formats: Optional[ScrapeFormats] = None+ headers: Optional[Dict[str, str]] = None+ include_tags: Optional[List[str]] = Field(None, alias="includeTags")+ exclude_tags: Optional[List[str]] = Field(None, alias="excludeTags")+ only_main_content: bool = Field(True, alias="onlyMainContent")+ timeout: Optional[int] = None+ wait_for: Optional[int] = Field(None, alias="waitFor")+ mobile: bool = False+ skip_tls_verification: bool = Field(False, alias="skipTlsVerification")+ remove_base64_images: bool = Field(True, alias="removeBase64Images")++class ScrapeRequest(BaseModel):+ """Request for scraping a single URL."""+ url: str+ options: Optional[ScrapeOptions] = None++class ScrapeResponse(BaseResponse[Document]):+ """Response from scraping operation."""+ pass++# Crawl types+class CrawlOptions(BaseModel):+ """Options for crawling operations."""+ includes: Optional[List[str]] = None+ excludes: Optional[List[str]] = None+ generate_img_alt_text: bool = Field(False, alias="generateImgAltText")+ return_only_urls: bool = Field(False, alias="returnOnlyUrls")+ max_depth: Optional[int] = Field(None, alias="maxDepth")+ mode: Literal["default", "fast"] = "default"+ ignore_sitemap: bool = Field(False, alias="ignoreSitemap")+ limit: Optional[int] = None+ allow_backward_crawling: bool = Field(False, alias="allowBackwardCrawling")+ allow_external_content_links: bool = Field(False, alias="allowExternalContentLinks")+ scrape_options: Optional[ScrapeOptions] = Field(None, alias="scrapeOptions")++class CrawlRequest(BaseModel):+ """Request for crawling a website."""+ url: str+ options: Optional[CrawlOptions] = None++class CrawlJob(BaseModel):+ """Information about a crawl job."""+ id: str+ url: str+ status: Literal["scraping", "completed", "failed", "cancelled"]+ current: Optional[int] = None+ total: Optional[int] = None+ created_at: Optional[datetime] = Field(None, alias="createdAt")+ completed_at: Optional[datetime] = Field(None, alias="completedAt")++class CrawlResponse(BaseResponse[CrawlJob]):+ """Response from starting a crawl."""+ pass++class CrawlStatusData(BaseModel):+ """Data for crawl status response."""+ status: Literal["scraping", "completed", "failed", "cancelled"]+ current: int+ total: int+ data: List[Document]+ partial_data: Optional[List[Document]] = Field(None, alias="partialData")++class CrawlStatusResponse(BaseResponse[CrawlStatusData]):+ """Response from checking crawl status."""+ pass++# Batch scrape types+class BatchScrapeRequest(BaseModel):+ """Request for batch scraping multiple URLs."""+ urls: List[str]+ options: Optional[ScrapeOptions] = None++class BatchScrapeJob(BaseModel):+ """Information about a batch scrape job."""+ id: str+ status: Literal["scraping", "completed", "failed", "cancelled"]+ current: Optional[int] = None+ total: Optional[int] = None+ created_at: Optional[datetime] = Field(None, alias="createdAt")+ completed_at: Optional[datetime] = Field(None, alias="completedAt")++class BatchScrapeResponse(BaseResponse[BatchScrapeJob]):+ """Response from starting a batch scrape."""+ pass++class BatchScrapeStatusData(BaseModel):+ """Data for batch scrape status response."""+ status: Literal["scraping", "completed", "failed", "cancelled"]+ current: int+ total: int+ data: List[Document]++class BatchScrapeStatusResponse(BaseResponse[BatchScrapeStatusData]):+ """Response from checking batch scrape status."""+ pass++# Map types+class MapOptions(BaseModel):+ """Options for mapping operations."""+ search: Optional[str] = None+ ignore_sitemap: bool = Field(False, alias="ignoreSitemap")+ include_subdomains: bool = Field(False, alias="includeSubdomains")+ limit: Optional[int] = None++class MapRequest(BaseModel):+ """Request for mapping a website."""+ url: str+ options: Optional[MapOptions] = None++class MapData(BaseModel):+ """Data for map response."""+ links: List[str]++class MapResponse(BaseResponse[MapData]):+ """Response from mapping operation."""+ pass++# Action types+class WaitAction(BaseModel):+ """Wait action to perform during scraping."""+ type: Literal["wait"] = "wait"+ milliseconds: Optional[int] = None+ selector: Optional[str] = None++class ScreenshotAction(BaseModel):+ """Screenshot action to perform during scraping."""+ type: Literal["screenshot"] = "screenshot"+ full_page: Optional[bool] = Field(None, alias="fullPage")+ quality: Optional[int] = None++class ClickAction(BaseModel):+ """Click action to perform during scraping."""+ type: Literal["click"] = "click"+ selector: str++class WriteAction(BaseModel):+ """Write action to perform during scraping."""+ type: Literal["write"] = "write"+ text: str++class PressAction(BaseModel):+ """Press action to perform during scraping."""+ type: Literal["press"] = "press"+ key: str++class ScrollAction(BaseModel):+ """Scroll action to perform during scraping."""+ type: Literal["scroll"] = "scroll"+ direction: Literal["up", "down"]+ selector: Optional[str] = None++class ScrapeAction(BaseModel):+ """Scrape action to perform during scraping."""+ type: Literal["scrape"] = "scrape"++class ExecuteJavascriptAction(BaseModel):+ """Execute javascript action to perform during scraping."""+ type: Literal["executeJavascript"] = "executeJavascript"+ script: str++class PDFAction(BaseModel):+ """PDF action to perform during scraping."""+ type: Literal["pdf"] = "pdf"+ format: Optional[Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"]] = None+ landscape: Optional[bool] = None+ scale: Optional[float] = None++# Location types+class Location(BaseModel):+ """Location configuration for scraping."""+ country: Optional[str] = None+ languages: Optional[List[str]] = None++# JSON format types+class JsonFormat(BaseModel):+ """Configuration for JSON extraction."""+ prompt: Optional[str] = None+ schema_field: Optional[Dict[str, Any]] = Field(None, alias="schema")+ system_prompt: Optional[str] = Field(None, alias="systemPrompt")++class SearchRequest(BaseModel):+ """Request for search operations."""+ query: str+ sources: Optional[List[SourceOption]] = None+ limit: Optional[int] = 5+ tbs: Optional[str] = None+ location: Optional[str] = None+ ignore_invalid_urls: Optional[bool] = Field(True, alias="ignoreInvalidURLs")+ timeout: Optional[int] = 60000+ scrape_options: Optional[ScrapeOptions] = Field(None, alias="scrapeOptions")++ @field_validator('sources')+ @classmethod+ def validate_sources(cls, v):+ """Validate and normalize sources input."""+ if v is None:+ return v++ normalized_sources = []+ for source in v:+ if isinstance(source, str):+ normalized_sources.append(Source(type=source))+ elif isinstance(source, dict):+ normalized_sources.append(Source(**source))+ elif isinstance(source, Source):+ normalized_sources.append(source)+ else:+ raise ValueError(f"Invalid source format: {source}")++ return normalized_sources++class SearchResult(BaseModel):+ """A search result with basic information."""+ url: str+ title: Optional[str] = None+ description: Optional[str] = None++class SearchResponse(BaseResponse[Dict[str, List[Union[SearchResult, Document]]]]):+ """Response from search operation."""+ pass++# Error types+class ErrorDetails(BaseModel):+ """Detailed error information."""+ code: Optional[str] = None+ message: str+ details: Optional[Dict[str, Any]] = None++class ErrorResponse(BaseModel):+ """Error response structure."""+ success: bool = False+ error: str+ details: Optional[ErrorDetails] = None++# Job management types+class JobStatus(BaseModel):+ """Generic job status information."""+ id: str+ status: Literal["pending", "scraping", "completed", "failed", "cancelled"]+ current: Optional[int] = None+ total: Optional[int] = None+ created_at: Optional[datetime] = Field(None, alias="createdAt")+ completed_at: Optional[datetime] = Field(None, alias="completedAt")+ expires_at: Optional[datetime] = Field(None, alias="expiresAt")++# Webhook types+class WebhookData(BaseModel):+ """Data sent to webhooks."""+ job_id: str = Field(alias="jobId")+ status: str+ current: Optional[int] = None+ total: Optional[int] = None+ data: Optional[List[Document]] = None+ error: Optional[str] = None++# Configuration types+class ClientConfig(BaseModel):+ """Configuration for the Firecrawl client."""+ api_key: str+ api_url: str = "https://api.firecrawl.dev"+ timeout: Optional[float] = None+ max_retries: int = 3+ backoff_factor: float = 0.5++# Union types for convenience+ScrapeResult = Union[Document, List[Document]]+CrawlResult = Union[CrawlJob, CrawlStatusData]+BatchResult = Union[BatchScrapeJob, BatchScrapeStatusData]+JobResult = Union[CrawlJob, BatchScrapeJob]+StatusResult = Union[CrawlStatusData, BatchScrapeStatusData]++# Response union types+AnyResponse = Union[+ ScrapeResponse,+ CrawlResponse,+ CrawlStatusResponse,+ BatchScrapeResponse,+ BatchScrapeStatusResponse,+ MapResponse,+ SearchResponse,+ ErrorResponse+]\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/utils/__init__.py b/apps/python-sdk/firecrawl/v2/utils/__init__.pynew file mode 100644index 000000000..37457ba29--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/utils/__init__.py@@ -0,0 +1,8 @@+"""+Utility modules for v2 API client.+"""++from .http_client import HttpClient+from .error_handler import FirecrawlError, handle_response_error++__all__ = ['HttpClient', 'FirecrawlError', 'handle_response_error']\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/utils/error_handler.py b/apps/python-sdk/firecrawl/v2/utils/error_handler.pynew file mode 100644index 000000000..d96bf7179--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/utils/error_handler.py@@ -0,0 +1,107 @@+"""+Error handling utilities for v2 API.+"""++import requests+from typing import Dict, Any, Optional+++class FirecrawlError(Exception):+ """Base exception for Firecrawl API errors."""++ def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[requests.Response] = None):+ super().__init__(message)+ self.status_code = status_code+ self.response = response+++class BadRequestError(FirecrawlError):+ """Raised when the request is invalid (400)."""+ pass++++class UnauthorizedError(FirecrawlError):+ """Raised when the request is unauthorized (401)."""+ pass+++class PaymentRequiredError(FirecrawlError):+ """Raised when payment is required (402)."""+ pass+++class WebsiteNotSupportedError(FirecrawlError):+ """Raised when website is not supported (403)."""+ pass+++class RequestTimeoutError(FirecrawlError):+ """Raised when request times out (408)."""+ pass+++class RateLimitError(FirecrawlError):+ """Raised when the rate limit is exceeded (429)."""+ pass+++class InternalServerError(FirecrawlError):+ """Raised when there's an internal server error (500)."""+ pass+++def handle_response_error(response: requests.Response, action: str) -> None:+ """+ Handle API response errors and raise appropriate exceptions.++ Args:+ response: The HTTP response object+ action: Description of the action being performed++ Raises:+ FirecrawlError: Appropriate error based on status code+ """+ try:+ response_json = response.json()+ error_message = response_json.get('error', 'No error message provided.')+ error_details = response_json.get('details', 'No additional error details provided.')+ except:+ # If we can't parse JSON, provide a helpful error message+ try:+ response_text = response.text[:500] # Limit to first 500 chars+ if response_text.strip():+ error_message = f"Server returned non-JSON response: {response_text}"+ error_details = f"Full response status: {response.status_code}"+ else:+ error_message = f"Server returned empty response with status {response.status_code}"+ error_details = "No additional details available"+ except:+ error_message = f"Server returned unreadable response with status {response.status_code}"+ error_details = "No additional details available"++ # Create appropriate error message+ if response.status_code == 400:+ message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"+ raise BadRequestError(message, response.status_code, response)+ elif response.status_code == 401:+ message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"+ raise UnauthorizedError(message, response.status_code, response)+ elif response.status_code == 402:+ message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"+ raise PaymentRequiredError(message, response.status_code, response)+ elif response.status_code == 403:+ message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"+ raise WebsiteNotSupportedError(message, response.status_code, response)+ elif response.status_code == 408:+ message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"+ raise RequestTimeoutError(message, response.status_code, response)+ elif response.status_code == 429:+ message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"+ raise RateLimitError(message, response.status_code, response)+ elif response.status_code == 500:+ message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"+ raise InternalServerError(message, response.status_code, response)+ else:+ message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message} - {error_details}"+ raise FirecrawlError(message, response.status_code, response)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/utils/get_version.py b/apps/python-sdk/firecrawl/v2/utils/get_version.pynew file mode 100644index 000000000..e7295ceea--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/utils/get_version.py@@ -0,0 +1,14 @@+import os+import re++def get_version():+ try:+ from pathlib import Path+ package_path = os.path.dirname(__file__)+ version_file = Path(os.path.join(package_path, '__init__.py')).read_text()+ version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)+ if version_match:+ return version_match.group(1).strip()+ except Exception:+ print("Failed to get version from __init__.py")+ return None\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.pynew file mode 100644index 000000000..787192c29--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py@@ -0,0 +1,157 @@+"""+HTTP client utilities for v2 API.+"""++import time+from typing import Dict, Any, Optional+import requests+from .get_version import get_version++version = get_version()++class HttpClient:+ """HTTP client with retry logic and error handling."""++ def __init__(self, api_key: str, api_url: str):+ self.api_key = api_key+ self.api_url = api_url++ def _prepare_headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:+ """Prepare headers for API requests."""+ headers = {+ 'Content-Type': 'application/json',+ 'Authorization': f'Bearer {self.api_key}',+ }++ if idempotency_key:+ headers['x-idempotency-key'] = idempotency_key++ return headers++ def post(+ self,+ endpoint: str,+ data: Dict[str, Any],+ headers: Optional[Dict[str, str]] = None,+ timeout: Optional[float] = None,+ retries: int = 3,+ backoff_factor: float = 0.5+ ) -> requests.Response:+ """Make a POST request with retry logic."""+ if headers is None:+ headers = self._prepare_headers()++ data['origin'] = f'python-sdk@{version}'++ url = f"{self.api_url}{endpoint}"++ last_exception = None++ for attempt in range(retries):+ try:+ response = requests.post(+ url,+ headers=headers,+ json=data,+ timeout=timeout+ )++ if response.status_code == 502:+ if attempt < retries - 1:+ time.sleep(backoff_factor * (2 ** attempt))+ continue++ return response++ except requests.RequestException as e:+ last_exception = e+ if attempt == retries - 1:+ raise e+ time.sleep(backoff_factor * (2 ** attempt))++ # This should never be reached due to the exception handling above+ raise last_exception or Exception("Unexpected error in POST request")++ def get(+ self,+ endpoint: str,+ headers: Optional[Dict[str, str]] = None,+ timeout: Optional[float] = None,+ retries: int = 3,+ backoff_factor: float = 0.5+ ) -> requests.Response:+ """Make a GET request with retry logic."""+ if headers is None:+ headers = self._prepare_headers()++ headers['origin'] = f'python-sdk@{version}'++ url = f"{self.api_url}{endpoint}"++ last_exception = None++ for attempt in range(retries):+ try:+ response = requests.get(+ url,+ headers=headers,+ timeout=timeout+ )++ if response.status_code == 502:+ if attempt < retries - 1:+ time.sleep(backoff_factor * (2 ** attempt))+ continue++ return response++ except requests.RequestException as e:+ last_exception = e+ if attempt == retries - 1:+ raise e+ time.sleep(backoff_factor * (2 ** attempt))++ # This should never be reached due to the exception handling above+ raise last_exception or Exception("Unexpected error in GET request")++ def delete(+ self,+ endpoint: str,+ headers: Optional[Dict[str, str]] = None,+ timeout: Optional[float] = None,+ retries: int = 3,+ backoff_factor: float = 0.5+ ) -> requests.Response:+ """Make a DELETE request with retry logic."""+ if headers is None:+ headers = self._prepare_headers()++ headers['origin'] = f'python-sdk@{version}'++ url = f"{self.api_url}{endpoint}"++ last_exception = None++ for attempt in range(retries):+ try:+ response = requests.delete(+ url,+ headers=headers,+ timeout=timeout+ )++ if response.status_code == 502:+ if attempt < retries - 1:+ time.sleep(backoff_factor * (2 ** attempt))+ continue++ return response++ except requests.RequestException as e:+ last_exception = e+ if attempt == retries - 1:+ raise e+ time.sleep(backoff_factor * (2 ** attempt))++ # This should never be reached due to the exception handling above+ raise last_exception or Exception("Unexpected error in DELETE request")\ No newline at end of file