Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(queue-jobs): add function to determine job type and update notification logic for concurrency limits
AdemÃlson F. Tonato426151c9c973
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 535f51b5f..1b4530b95 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -14,6 +14,17 @@ import { getConcurrencyLimitMax } from "./rate-limiter";import { sendNotificationWithCustomDays } from './notification/email_notification';import { shouldSendConcurrencyLimitNotification } from './notification/notification-check';+/**+ * Checks if a job is a crawl or batch scrape based on its options+ * @param options The job options containing crawlerOptions and crawl_id+ * @returns true if the job is either a crawl or batch scrape+ */+function isCrawlOrBatchScrape(options: { crawlerOptions?: any; crawl_id?: string }): boolean {+ // If crawlerOptions exists, it's a crawl+ // If crawl_id exists but no crawlerOptions, it's a batch scrape+ return !!options.crawlerOptions || !!options.crawl_id;+}+async function _addScrapeJobToConcurrencyQueue(webScraperOptions: any,options: any,@@ -83,11 +94,14 @@ async function addScrapeJobRaw(if(concurrencyQueueJobs > maxConcurrency) {logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);- const shouldSendNotification = await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id);- if (shouldSendNotification) {- sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {- logger.error("Error sending notification (concurrency limit reached): ", error);- });+ // Only send notification if it's not a crawl or batch scrape+ if (!isCrawlOrBatchScrape(webScraperOptions)) {+ const shouldSendNotification = await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id);+ if (shouldSendNotification) {+ sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {+ logger.error("Error sending notification (concurrency limit reached): ", error);+ });+ }}}@@ -178,11 +192,14 @@ export async function addScrapeJobs(if(addToCQ.length > maxConcurrency) {logger.info("Concurrency limited 2x (multiple) - ", "Concurrency queue jobs: ", addToCQ.length, "Max concurrency: ", maxConcurrency, "Team ID: ", jobs[0].data.team_id);- const shouldSendNotification = await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id);- if (shouldSendNotification) {- sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {- logger.error("Error sending notification (concurrency limit reached): ", error);- });+ // Only send notification if it's not a crawl or batch scrape+ if (!isCrawlOrBatchScrape(jobs[0].data)) {+ const shouldSendNotification = await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id);+ if (shouldSendNotification) {+ sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false).catch((error) => {+ logger.error("Error sending notification (concurrency limit reached): ", error);+ });+ }}}