Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v1/batch/scrape): webhooks
Móricz Gergődf05124ef56d
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex b018dc999..9fd5cc50a 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -66,6 +66,7 @@ export async function batchScrapeController(crawl_id: id,sitemapped: true,v1: true,+ webhook: req.body.webhook,},opts: {jobId: uuidv4(),diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex b2edd6e7c..d885e128a 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -175,9 +175,21 @@ export const scrapeRequestSchema = scrapeOptions.extend({export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;+export const webhookSchema = z.preprocess(x => {+ if (typeof x === "string") {+ return { url: x };+ } else {+ return x;+ }+}, z.object({+ url: z.string().url(),+ headers: z.record(z.string(), z.string()).default({}),+}).strict(strictMessage))+export const batchScrapeRequestSchema = scrapeOptions.extend({urls: url.array(),origin: z.string().optional().default("api"),+ webhook: webhookSchema.optional(),}).strict(strictMessage).refine((obj) => {const hasExtractFormat = obj.formats?.includes("extract");@@ -220,17 +232,6 @@ const crawlerOptions = z.object({export type CrawlerOptions = z.infer<typeof crawlerOptions>;-export const webhookSchema = z.preprocess(x => {- if (typeof x === "string") {- return { url: x };- } else {- return x;- }-}, z.object({- url: z.string().url(),- headers: z.record(z.string(), z.string()).default({}),-}).strict(strictMessage))-export const crawlRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 45e19197b..180389450 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -543,16 +543,18 @@ export default class FirecrawlApp {* @param params - Additional parameters for the scrape request.* @param pollInterval - Time in seconds for job status checks.* @param idempotencyKey - Optional idempotency key for the request.+ * @param webhook - Optional webhook for the batch scrape.* @returns The response from the crawl operation.*/async batchScrapeUrls(urls: string[],params?: ScrapeParams,pollInterval: number = 2,- idempotencyKey?: string+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],): Promise<BatchScrapeStatusResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, ...(params ?? {}) };+ let jsonData: any = { urls, ...(params ?? {}), webhook };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/batch/scrape`,