Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixed the sdks
Nicolas9ec08d70204d
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 6668d33b5..061f5c9c8 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -69,7 +69,7 @@ export async function extractController(});}- return res.status(202).json({+ return res.status(200).json({success: true,id: extractId,urlTrace: [],diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 474eea830..644be7f65 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -887,18 +887,32 @@ export default class FirecrawlApp {{ ...jsonData, schema: jsonSchema },headers);+if (response.status === 200) {- const responseData = response.data as ExtractResponse<T>;- if (responseData.success) {- return {- success: true,- data: responseData.data,- warning: responseData.warning,- error: responseData.error- };- } else {- throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);- }+ const jobId = response.data.id;+ let extractStatus;+ do {+ const statusResponse: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/extract/${jobId}`,+ headers+ );+ extractStatus = statusResponse.data;+ if (extractStatus.status === "completed") {+ if (extractStatus.success) {+ return {+ success: true,+ data: extractStatus.data,+ warning: extractStatus.warning,+ error: extractStatus.error+ };+ } else {+ throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);+ }+ } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {+ throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);+ }+ await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval+ } while (extractStatus.status !== "completed");} else {this.handleError(response, "extract");}diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex d32164052..e844b7330 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -542,6 +542,7 @@ class FirecrawlApp:}try:+ # Send the initial extract requestresponse = self._post_request(f'{self.api_url}/v1/extract',request_data,@@ -550,7 +551,29 @@ class FirecrawlApp:if response.status_code == 200:data = response.json()if data['success']:- return data+ job_id = data.get('id')+ if not job_id:+ raise Exception('Job ID not returned from extract request.')++ # Poll for the extract status+ while True:+ status_response = self._get_request(+ f'{self.api_url}/v1/extract/{job_id}',+ headers+ )+ if status_response.status_code == 200:+ status_data = status_response.json()+ if status_data['status'] == 'completed':+ if status_data['success']:+ return status_data+ else:+ raise Exception(f'Failed to extract. Error: {status_data["error"]}')+ elif status_data['status'] in ['failed', 'cancelled']:+ raise Exception(f'Extract job {status_data["status"]}. Error: {status_data["error"]}')+ else:+ self._handle_error(status_response, "extract-status")++ time.sleep(2) # Polling intervalelse:raise Exception(f'Failed to extract. Error: {data["error"]}')else: