Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(queue-jobs): update notification logic for concurrency limits and add parameter (jsdocs) to batchScrapeUrls
AdemÃlson F. Tonato58e587d99e8e
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 1ce2211cb..d2601d303 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,6 +1,6 @@import { getScrapeQueue } from "./queue-service";import { v4 as uuidv4 } from "uuid";-import { PlanType, WebScraperOptions } from "../types";+import { NotificationType, PlanType, WebScraperOptions } from "../types";import * as Sentry from "@sentry/node";import {cleanOldConcurrencyLimitEntries,@@ -11,6 +11,7 @@ import {} from "../lib/concurrency-limit";import { logger } from "../lib/logger";import { getConcurrencyLimitMax } from "./rate-limiter";+import { sendNotificationWithCustomDays } from './notification/email_notification';async function _addScrapeJobToConcurrencyQueue(webScraperOptions: any,@@ -80,9 +81,9 @@ async function addScrapeJobRaw(// No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2xif(concurrencyQueueJobs > maxConcurrency) {logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);- // sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 10, false).catch((error) => {- // logger.error("Error sending notification (concurrency limit reached): ", error);- // });+ sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {+ logger.error("Error sending notification (concurrency limit reached): ", error);+ });}webScraperOptions.concurrencyLimited = true;@@ -171,9 +172,9 @@ export async function addScrapeJobs(// equals 2x the max concurrencyif(addToCQ.length > maxConcurrency) {logger.info("Concurrency limited 2x (multiple) - ", "Concurrency queue jobs: ", addToCQ.length, "Max concurrency: ", maxConcurrency, "Team ID: ", jobs[0].data.team_id);- // sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 10, false).catch((error) => {- // logger.error("Error sending notification (concurrency limit reached): ", error);- // });+ sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {+ logger.error("Error sending notification (concurrency limit reached): ", error);+ });}await Promise.all(diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 11fb8d747..41d13da0d 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -922,6 +922,7 @@ export default class FirecrawlApp {* @param pollInterval - Time in seconds for job status checks.* @param idempotencyKey - Optional idempotency key for the request.* @param webhook - Optional webhook for the batch scrape.+ * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.* @returns The response from the crawl operation.*/async batchScrapeUrls(