Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-jobs): fix concurrency limit
Gergő Móricz2de659d81050
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 1fb470f9d..c2e3369f3 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -18,7 +18,7 @@ import {} from "../../lib/crawl-redis";import { logCrawl } from "../../services/logging/crawl_log";import { getScrapeQueue } from "../../services/queue-service";-import { addScrapeJob } from "../../services/queue-jobs";+import { addScrapeJob, addScrapeJobs } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { getJobPriority } from "../../lib/job-priority";import { callWebhook } from "../../services/webhook";@@ -139,9 +139,9 @@ export async function crawlController(name: uuid,data: {url,- mode: "single_urls",+ mode: "single_urls" as const,team_id: req.auth.team_id,- plan: req.auth.plan,+ plan: req.auth.plan!,crawlerOptions,scrapeOptions,internalOptions: sc.internalOptions,@@ -170,7 +170,7 @@ export async function crawlController(jobs.map((x) => x.opts.jobId),);logger.debug("Adding scrape jobs to BullMQ...");- await getScrapeQueue().addBulk(jobs);+ await addScrapeJobs(jobs);} else {logger.debug("Sitemap not found or ignored.", {ignoreSitemap: sc.crawlerOptions.ignoreSitemap,diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex ee9e6177e..6ce48a810 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -11,11 +11,50 @@ import {pushConcurrencyLimitedJob,} from "../lib/concurrency-limit";+async function _addScrapeJobToConcurrencyQueue(+ webScraperOptions: any,+ options: any,+ jobId: string,+ jobPriority: number,+) {+ await pushConcurrencyLimitedJob(webScraperOptions.team_id, {+ id: jobId,+ data: webScraperOptions,+ opts: {+ ...options,+ priority: jobPriority,+ jobId: jobId,+ },+ priority: jobPriority,+ });+}++async function _addScrapeJobToBullMQ(+ webScraperOptions: any,+ options: any,+ jobId: string,+ jobPriority: number,+) {+ if (+ webScraperOptions &&+ webScraperOptions.team_id &&+ webScraperOptions.plan+ ) {+ await pushConcurrencyLimitActiveJob(webScraperOptions.team_id, jobId);+ }++ await getScrapeQueue().add(jobId, webScraperOptions, {+ ...options,+ priority: jobPriority,+ jobId,+ });+}+async function addScrapeJobRaw(webScraperOptions: any,options: any,jobId: string,- jobPriority: number = 10,+ jobPriority: number,) {let concurrencyLimited = false;@@ -33,30 +72,9 @@ async function addScrapeJobRaw(}if (concurrencyLimited) {- await pushConcurrencyLimitedJob(webScraperOptions.team_id, {- id: jobId,- data: webScraperOptions,- opts: {- ...options,- priority: jobPriority,- jobId: jobId,- },- priority: jobPriority,- });+ await _addScrapeJobToConcurrencyQueue(webScraperOptions, options, jobId, jobPriority);} else {- if (- webScraperOptions &&- webScraperOptions.team_id &&- webScraperOptions.plan- ) {- await pushConcurrencyLimitActiveJob(webScraperOptions.team_id, jobId);- }-- await getScrapeQueue().add(jobId, webScraperOptions, {- ...options,- priority: jobPriority,- jobId,- });+ await _addScrapeJobToBullMQ(webScraperOptions, options, jobId, jobPriority);}}@@ -109,11 +127,87 @@ export async function addScrapeJobs(}[],) {if (jobs.length === 0) return true;- // TODO: better++ let countCanBeDirectlyAdded = Infinity;++ if (+ jobs[0].data &&+ jobs[0].data.team_id &&+ jobs[0].data.plan+ ) {+ const now = Date.now();+ const limit = await getConcurrencyLimitMax(jobs[0].data.plan);+ console.log("CC limit", limit);+ cleanOldConcurrencyLimitEntries(jobs[0].data.team_id, now);++ countCanBeDirectlyAdded = Math.max(limit - (await getConcurrencyLimitActiveJobs(jobs[0].data.team_id, now)).length, 0);+ }++ const addToBull = jobs.slice(0, countCanBeDirectlyAdded);+ const addToCQ = jobs.slice(countCanBeDirectlyAdded);++ await Promise.all(+ addToBull.map(async (job) => {+ const size = JSON.stringify(job.data).length;+ return await Sentry.startSpan(+ {+ name: "Add scrape job",+ op: "queue.publish",+ attributes: {+ "messaging.message.id": job.opts.jobId,+ "messaging.destination.name": getScrapeQueue().name,+ "messaging.message.body.size": size,+ },+ },+ async (span) => {+ await _addScrapeJobToBullMQ(+ {+ ...job.data,+ sentry: {+ trace: Sentry.spanToTraceHeader(span),+ baggage: Sentry.spanToBaggageHeader(span),+ size,+ },+ },+ job.opts,+ job.opts.jobId,+ job.opts.priority,+ );+ },+ );+ }),+ );+await Promise.all(- jobs.map((job) =>- addScrapeJob(job.data, job.opts, job.opts.jobId, job.opts.priority),- ),+ addToCQ.map(async (job) => {+ const size = JSON.stringify(job.data).length;+ return await Sentry.startSpan(+ {+ name: "Add scrape job",+ op: "queue.publish",+ attributes: {+ "messaging.message.id": job.opts.jobId,+ "messaging.destination.name": getScrapeQueue().name,+ "messaging.message.body.size": size,+ },+ },+ async (span) => {+ await _addScrapeJobToConcurrencyQueue(+ {+ ...job.data,+ sentry: {+ trace: Sentry.spanToTraceHeader(span),+ baggage: Sentry.spanToBaggageHeader(span),+ size,+ },+ },+ job.opts,+ job.opts.jobId,+ job.opts.priority,+ );+ },+ );+ }),);}