Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(js-sdk/batch/scrape): add ignoreInvalidURLs option
Gergő Móricz9cc657657147
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 37fc5ef0d..020a22936 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -183,6 +183,7 @@ export interface BatchScrapeResponse {url?: string;success: true;error?: string;+ invalidURLs?: string[];}/**@@ -576,9 +577,10 @@ export default class FirecrawlApp {pollInterval: number = 2,idempotencyKey?: string,webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,): Promise<BatchScrapeStatusResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, ...params };+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params };if (jsonData?.extract?.schema) {let schema = jsonData.extract.schema;@@ -621,10 +623,12 @@ export default class FirecrawlApp {async asyncBatchScrapeUrls(urls: string[],params?: ScrapeParams,- idempotencyKey?: string+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,): Promise<BatchScrapeResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, ...(params ?? {}) };+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...(params ?? {}) };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/batch/scrape`,@@ -657,8 +661,10 @@ export default class FirecrawlApp {urls: string[],params?: ScrapeParams,idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,) {- const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);+ const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);if (crawl.success && crawl.id) {const id = crawl.id;