Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge branch 'main' into sdk-improv/async
Nicolasc67425ad8b49
diff --cc apps/js-sdk/firecrawl/src/index.tsindex 27859fcb7,779c5e71b..ed090e10f--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@@ -1521,12 -1577,30 +1593,30 @@@ export default class FirecrawlApp* @param params - Parameters for the deep research operation.* @returns The response containing the research job ID.*/- async asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {+ async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: any = { query, ...params };+ let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };++ if (jsonData?.jsonOptions?.schema) {+ let schema = jsonData.jsonOptions.schema;+ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {-++ // Ignore error if schema can't be parsed as Zod+ }+ jsonData = {+ ...jsonData,+ jsonOptions: {+ ...jsonData.jsonOptions,+ schema: schema,+ },+ };+ }+try {const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/deep-research`,+ `${this.apiUrl}/v1/deep-research`,jsonData,headers);diff --cc apps/python-sdk/firecrawl/firecrawl.pyindex 7dafbf489,b033a9d02..8ca94fa69--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@@ -335,25 -75,51 +336,69 @@@ class DeepResearchStatusResponse(pydantsources: List[Dict[str, Any]]summaries: List[str]+class GenerateLLMsTextResponse(pydantic.BaseModel):+ """Response from LLMs.txt generation operations."""+ success: bool = True+ id: str+ error: Optional[str] = None++class GenerateLLMsTextStatusResponseData(pydantic.BaseModel):+ llmstxt: str+ llmsfulltxt: Optional[str] = None++class GenerateLLMsTextStatusResponse(pydantic.BaseModel):+ """Status response from LLMs.txt generation operations."""+ success: bool = True+ data: Optional[GenerateLLMsTextStatusResponseData] = None+ status: Literal["processing", "completed", "failed"]+ error: Optional[str] = None+ expiresAt: str++ class ChangeTrackingData(pydantic.BaseModel):+ """+ Data for the change tracking format.+ """+ previousScrapeAt: Optional[str] = None+ changeStatus: str # "new" | "same" | "changed" | "removed"+ visibility: str # "visible" | "hidden"+ diff: Optional[Dict[str, Any]] = None+ json: Optional[Any] = None+class FirecrawlApp:+ class SearchResponse(pydantic.BaseModel):+ """+ Response from the search operation.+ """+ success: bool+ data: List[Dict[str, Any]]+ warning: Optional[str] = None+ error: Optional[str] = None++ class ExtractParams(pydantic.BaseModel):+ """+ Parameters for the extract operation.+ """+ prompt: Optional[str] = None+ schema_: Optional[Any] = pydantic.Field(None, alias='schema')+ system_prompt: Optional[str] = None+ allow_external_links: Optional[bool] = False+ enable_web_search: Optional[bool] = False+ # Just for backwards compatibility+ enableWebSearch: Optional[bool] = False+ show_sources: Optional[bool] = False+ agent: Optional[Dict[str, Any]] = None+++++ class ExtractResponse(pydantic.BaseModel):+ """+ Response from the extract operation.+ """+ success: bool+ data: Optional[Any] = None+ error: Optional[str] = None+def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:"""Initialize the FirecrawlApp instance with API key, API URL.@@@ -435,12 -178,19 +480,20 @@@json['schema'] = json['schema'].schema()scrape_params['jsonOptions'] = json+ change_tracking = params.get("changeTrackingOptions", {})+ if change_tracking:+ scrape_params['changeTrackingOptions'] = change_tracking+# Include any other params directly at the top level of scrape_paramsfor key, value in params.items():- if key not in ['jsonOptions']:+ if key not in ['jsonOptions', 'changeTrackingOptions', 'agent']:scrape_params[key] = value++ agent = params.get('agent')+ if agent:+ scrape_params['agent'] = agent+ scrape_params['origin'] = f"python-sdk@{version}"endpoint = f'/v1/scrape'# Make the POST request with the prepared headers and JSON data@@@ -1769,11 -1194,7 +1825,8 @@@time.sleep(2) # Polling intervalreturn {'success': False, 'error': 'Deep research job terminated unexpectedly'}+- def async_deep_research(- self,- query: str,- params: Optional[Union[Dict[str, Any], DeepResearchParams]] = None) -> DeepResearchResponse:+ def async_deep_research(self, query: str, params: Optional[Union[Dict[str, Any], DeepResearchParams]] = None) -> Dict[str, Any]:"""Initiates an asynchronous deep research operation.@@@ -1802,9 -1217,15 +1855,16 @@@research_params = paramsheaders = self._prepare_headers()+json_data = {'query': query, **research_params.dict(exclude_none=True)}+ json_data['origin'] = f"python-sdk@{version}"+ # Handle json options schema if present+ if 'jsonOptions' in json_data:+ json_opts = json_data['jsonOptions']+ if json_opts and 'schema' in json_opts and hasattr(json_opts['schema'], 'schema'):+ json_data['jsonOptions']['schema'] = json_opts['schema'].schema()+try:response = self._post_request(f'{self.api_url}/v1/deep-research', json_data, headers)if response.status_code == 200: