Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fixed empty data with next causing infinite loop
rafaelmmiller55dad5ea13da
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 6b89960ed..687325d35 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -565,6 +565,10 @@ export default class FirecrawlApp {if ("data" in statusData) {let data = statusData.data;while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ console.warn("Expected 'data' is missing.")+ break+ }statusData = (await this.getRequest(statusData.next, headers)).data;data = data.concat(statusData.data);}@@ -812,6 +816,10 @@ export default class FirecrawlApp {if ("data" in statusData) {let data = statusData.data;while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ console.warn("Expected 'data' is missing.")+ break+ }statusData = (await this.getRequest(statusData.next, headers)).data;data = data.concat(statusData.data);}@@ -995,6 +1003,10 @@ export default class FirecrawlApp {if ("data" in statusData) {let data = statusData.data;while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ console.warn("Expected 'data' is missing.")+ break+ }statusResponse = await this.getRequest(statusData.next, headers);statusData = statusResponse.data;data = data.concat(statusData.data);diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 8eb7acee9..812f7bd1f 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -250,6 +250,9 @@ class FirecrawlApp:if 'data' in status_data:data = status_data['data']while 'next' in status_data:+ if len(status_data['data']) == 0:+ logger.warning("Expected 'data' is missing.")+ breaknext_url = status_data.get('next')if not next_url:logger.warning("Expected 'next' URL is missing.")@@ -467,6 +470,9 @@ class FirecrawlApp:if 'data' in status_data:data = status_data['data']while 'next' in status_data:+ if len(status_data['data']) == 0:+ logger.warning("Expected 'data' is missing.")+ breaknext_url = status_data.get('next')if not next_url:logger.warning("Expected 'next' URL is missing.")@@ -685,6 +691,9 @@ class FirecrawlApp:if 'data' in status_data:data = status_data['data']while 'next' in status_data:+ if len(status_data['data']) == 0:+ logger.warning("Expected 'data' is missing.")+ breakstatus_response = self._get_request(status_data['next'], headers)status_data = status_response.json()data.extend(status_data.get('data', []))