Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
added check for object and trycatch as workaround for 502s
rafaelmmillerbdbc05a4c7c3
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 30277cc31..74dfcb02a 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "1.9.3",+ "version": "1.9.4","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 37fc5ef0d..a54506ad3 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -462,7 +462,7 @@ export default class FirecrawlApp {let statusData = response.dataif ("data" in statusData) {let data = statusData.data;- while ('next' in statusData) {+ while (typeof statusData === 'object' && 'next' in statusData) {statusData = (await this.getRequest(statusData.next, headers)).data;data = data.concat(statusData.data);}@@ -691,7 +691,7 @@ export default class FirecrawlApp {let statusData = response.dataif ("data" in statusData) {let data = statusData.data;- while ('next' in statusData) {+ while (typeof statusData === 'object' && 'next' in statusData) {statusData = (await this.getRequest(statusData.next, headers)).data;data = data.concat(statusData.data);}@@ -850,42 +850,46 @@ export default class FirecrawlApp {headers: AxiosRequestHeaders,checkInterval: number): Promise<CrawlStatusResponse | ErrorResponse> {- while (true) {- let statusResponse: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/crawl/${id}`,- headers- );- if (statusResponse.status === 200) {- let statusData = statusResponse.data;- if (statusData.status === "completed") {- if ("data" in statusData) {- let data = statusData.data;- while ('next' in statusData) {- statusResponse = await this.getRequest(statusData.next, headers);- statusData = statusResponse.data;- data = data.concat(statusData.data);+ try {+ while (true) {+ let statusResponse: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/crawl/${id}`,+ headers+ );+ if (statusResponse.status === 200) {+ let statusData = statusResponse.data;+ if (statusData.status === "completed") {+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ statusResponse = await this.getRequest(statusData.next, headers);+ statusData = statusResponse.data;+ data = data.concat(statusData.data);+ }+ statusData.data = data;+ return statusData;+ } else {+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);}- statusData.data = data;- return statusData;- } else {- throw new FirecrawlError("Crawl job completed but no data was returned", 500);- }- } else if (- ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)- ) {- checkInterval = Math.max(checkInterval, 2);- await new Promise((resolve) =>- setTimeout(resolve, checkInterval * 1000)- );+ } else if (+ ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)+ ) {+ checkInterval = Math.max(checkInterval, 2);+ await new Promise((resolve) =>+ setTimeout(resolve, checkInterval * 1000)+ );+ } else {+ throw new FirecrawlError(+ `Crawl job failed or was stopped. Status: ${statusData.status}`,+ 500+ );+ }} else {- throw new FirecrawlError(- `Crawl job failed or was stopped. Status: ${statusData.status}`,- 500- );+ this.handleError(statusResponse, "check crawl status");}- } else {- this.handleError(statusResponse, "check crawl status");}+ } catch (error: any) {+ throw new FirecrawlError(error, 500);}}