Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge branch 'main' into sdk-improv/async

Nicolasc67425ad8b49
diff --cc apps/js-sdk/firecrawl/src/index.ts
index 27859fcb7,779c5e71b..ed090e10f
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@@ -1521,12 -1577,30 +1593,30 @@@ export default class FirecrawlApp
* @param params - Parameters for the deep research operation.
* @returns The response containing the research job ID.
*/
- async asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
+ async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
const headers = this.prepareHeaders();
- let jsonData: any = { query, ...params };
+ let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };
+
+ if (jsonData?.jsonOptions?.schema) {
+ let schema = jsonData.jsonOptions.schema;
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
-
++ // Ignore error if schema can't be parsed as Zod
+ }
+ jsonData = {
+ ...jsonData,
+ jsonOptions: {
+ ...jsonData.jsonOptions,
+ schema: schema,
+ },
+ };
+ }
+
try {
const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/deep-research`,
+ `${this.apiUrl}/v1/deep-research`,
jsonData,
headers
);
diff --cc apps/python-sdk/firecrawl/firecrawl.py
index 7dafbf489,b033a9d02..8ca94fa69
--- a/apps/python-sdk/firecrawl/firecrawl.py
+++ b/apps/python-sdk/firecrawl/firecrawl.py
@@@ -335,25 -75,51 +336,69 @@@ class DeepResearchStatusResponse(pydant
sources: List[Dict[str, Any]]
summaries: List[str]
+class GenerateLLMsTextResponse(pydantic.BaseModel):
+ """Response from LLMs.txt generation operations."""
+ success: bool = True
+ id: str
+ error: Optional[str] = None
+
+class GenerateLLMsTextStatusResponseData(pydantic.BaseModel):
+ llmstxt: str
+ llmsfulltxt: Optional[str] = None
+
+class GenerateLLMsTextStatusResponse(pydantic.BaseModel):
+ """Status response from LLMs.txt generation operations."""
+ success: bool = True
+ data: Optional[GenerateLLMsTextStatusResponseData] = None
+ status: Literal["processing", "completed", "failed"]
+ error: Optional[str] = None
+ expiresAt: str
+
+ class ChangeTrackingData(pydantic.BaseModel):
+ """
+ Data for the change tracking format.
+ """
+ previousScrapeAt: Optional[str] = None
+ changeStatus: str # "new" | "same" | "changed" | "removed"
+ visibility: str # "visible" | "hidden"
+ diff: Optional[Dict[str, Any]] = None
+ json: Optional[Any] = None
+
class FirecrawlApp:
+ class SearchResponse(pydantic.BaseModel):
+ """
+ Response from the search operation.
+ """
+ success: bool
+ data: List[Dict[str, Any]]
+ warning: Optional[str] = None
+ error: Optional[str] = None
+
+ class ExtractParams(pydantic.BaseModel):
+ """
+ Parameters for the extract operation.
+ """
+ prompt: Optional[str] = None
+ schema_: Optional[Any] = pydantic.Field(None, alias='schema')
+ system_prompt: Optional[str] = None
+ allow_external_links: Optional[bool] = False
+ enable_web_search: Optional[bool] = False
+ # Just for backwards compatibility
+ enableWebSearch: Optional[bool] = False
+ show_sources: Optional[bool] = False
+ agent: Optional[Dict[str, Any]] = None
+
+
+
+
+ class ExtractResponse(pydantic.BaseModel):
+ """
+ Response from the extract operation.
+ """
+ success: bool
+ data: Optional[Any] = None
+ error: Optional[str] = None
+
def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:
"""
Initialize the FirecrawlApp instance with API key, API URL.
@@@ -435,12 -178,19 +480,20 @@@
json['schema'] = json['schema'].schema()
scrape_params['jsonOptions'] = json
+ change_tracking = params.get("changeTrackingOptions", {})
+ if change_tracking:
+ scrape_params['changeTrackingOptions'] = change_tracking
+
# Include any other params directly at the top level of scrape_params
for key, value in params.items():
- if key not in ['jsonOptions']:
+ if key not in ['jsonOptions', 'changeTrackingOptions', 'agent']:
scrape_params[key] = value
+
+ agent = params.get('agent')
+ if agent:
+ scrape_params['agent'] = agent
+ scrape_params['origin'] = f"python-sdk@{version}"
endpoint = f'/v1/scrape'
# Make the POST request with the prepared headers and JSON data
@@@ -1769,11 -1194,7 +1825,8 @@@
time.sleep(2) # Polling interval
return {'success': False, 'error': 'Deep research job terminated unexpectedly'}
+
- def async_deep_research(
- self,
- query: str,
- params: Optional[Union[Dict[str, Any], DeepResearchParams]] = None) -> DeepResearchResponse:
+ def async_deep_research(self, query: str, params: Optional[Union[Dict[str, Any], DeepResearchParams]] = None) -> Dict[str, Any]:
"""
Initiates an asynchronous deep research operation.
@@@ -1802,9 -1217,15 +1855,16 @@@
research_params = params
headers = self._prepare_headers()
+
json_data = {'query': query, **research_params.dict(exclude_none=True)}
+ json_data['origin'] = f"python-sdk@{version}"
+ # Handle json options schema if present
+ if 'jsonOptions' in json_data:
+ json_opts = json_data['jsonOptions']
+ if json_opts and 'schema' in json_opts and hasattr(json_opts['schema'], 'schema'):
+ json_data['jsonOptions']['schema'] = json_opts['schema'].schema()
+
try:
response = self._post_request(f'{self.api_url}/v1/deep-research', json_data, headers)
if response.status_code == 200: