Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq,concurrency-limit): handle if there are duplicate jobs in the concurrency queue (#2343)
Gergő Móricz29a1c3689ce7
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 913e4492b..bdd7c08ba 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -301,56 +301,88 @@ export async function concurrentJobDone(job: NuQJob<any>) {await cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);}- const maxTeamConcurrency =- (- await getACUCTeam(- job.data.team_id,- false,- true,- job.data.is_extract ? RateLimiterMode.Extract : RateLimiterMode.Crawl,- )- )?.concurrency ?? 2;- const currentActiveConcurrency = (- await getConcurrencyLimitActiveJobs(job.data.team_id)- ).length;-- if (currentActiveConcurrency < maxTeamConcurrency) {- const nextJob = await getNextConcurrentJob(job.data.team_id);- if (nextJob !== null) {- await pushConcurrencyLimitActiveJob(- job.data.team_id,- nextJob.job.id,- 60 * 1000,- );-- if (nextJob.job.data.crawl_id) {- await pushCrawlConcurrencyLimitActiveJob(- nextJob.job.data.crawl_id,+ let i = 0;+ for (; i < 10; i++) {+ const maxTeamConcurrency =+ (+ await getACUCTeam(+ job.data.team_id,+ false,+ true,+ job.data.is_extract+ ? RateLimiterMode.Extract+ : RateLimiterMode.Crawl,+ )+ )?.concurrency ?? 2;+ const currentActiveConcurrency = (+ await getConcurrencyLimitActiveJobs(job.data.team_id)+ ).length;++ if (currentActiveConcurrency < maxTeamConcurrency) {+ const nextJob = await getNextConcurrentJob(job.data.team_id);+ if (nextJob !== null) {+ await pushConcurrencyLimitActiveJob(+ job.data.team_id,nextJob.job.id,60 * 1000,);- const sc = await getCrawl(nextJob.job.data.crawl_id);- if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {- await new Promise(resolve =>- setTimeout(resolve, sc.crawlerOptions.delay * 1000),+ if (nextJob.job.data.crawl_id) {+ await pushCrawlConcurrencyLimitActiveJob(+ nextJob.job.data.crawl_id,+ nextJob.job.id,+ 60 * 1000,);- }- }- abTestJob(nextJob.job.data);+ const sc = await getCrawl(nextJob.job.data.crawl_id);+ if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {+ await new Promise(resolve =>+ setTimeout(resolve, sc.crawlerOptions.delay * 1000),+ );+ }+ }- await scrapeQueue.promoteJobFromBacklogOrAdd(- nextJob.job.id,- nextJob.job.data,- {- priority: nextJob.job.priority,- listenable: nextJob.job.listenable,- ownerId: nextJob.job.data.team_id ?? undefined,- groupId: nextJob.job.data.crawl_id ?? undefined,- },- );+ abTestJob(nextJob.job.data);++ const promotedSuccessfully =+ (await scrapeQueue.promoteJobFromBacklogOrAdd(+ nextJob.job.id,+ nextJob.job.data,+ {+ priority: nextJob.job.priority,+ listenable: nextJob.job.listenable,+ ownerId: nextJob.job.data.team_id ?? undefined,+ groupId: nextJob.job.data.crawl_id ?? undefined,+ },+ )) !== null;++ if (promotedSuccessfully) {+ logger.debug("Successfully promoted concurrent queued job", {});+ break;+ } else {+ logger.warn(+ "Was unable to promote concurrent queued job as it already exists in the database",+ {+ teamId: job.data.team_id,+ jobId: nextJob.job.id,+ },+ );+ }+ } else {+ break;+ }+ } else {+ break;}}++ if (i === 10) {+ logger.warn(+ "Failed to promote a concurrent job after 10 iterations, bailing!",+ {+ teamId: job.data.team_id,+ },+ );+ }}}diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 768d78d62..8e57e1aac 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -706,6 +706,66 @@ class NuQ<JobData = any, JobReturnValue = any> {});}+ public async addJobIfNotExists(+ id: string,+ data: JobData,+ options: NuQJobOptions,+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {+ return withSpan("nuq.addJob", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.priority": options.priority ?? 0,+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,+ "nuq.listenable": options.listenable ?? false,+ });++ const start = Date.now();+ try {+ const result = this.rowToJob(+ (+ await nuqPool.query(+ `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) ON CONFLICT (id) DO NOTHING RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,+ [+ id,+ data,+ options.priority ?? 0,+ options.listenable ? this.listenChannelId : null,+ normalizeOwnerId(options.ownerId),+ options.groupId ?? null,+ ...(options.backlogged+ ? [+ options.backloggedTimesOutAt+ ? options.backloggedTimesOutAt.toISOString()+ : null,+ ]+ : []),+ ],+ )+ ).rows[0],+ );++ setSpanAttributes(span, {+ "nuq.job_created": result !== null,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqAddJob metrics", {+ module: "nuq/metrics",+ method: "nuqAddJob",+ duration,+ scrapeId: id,+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,+ });+ }+ });+ }+public async addJobs(jobs: Array<{id: string;@@ -855,7 +915,7 @@ class NuQ<JobData = any, JobReturnValue = any> {id: string,data: JobData,options: NuQJobOptions,- ): Promise<NuQJob<JobData, JobReturnValue>> {+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {return withSpan("nuq.promoteJobFromBacklogOrAdd", async span => {setSpanAttributes(span, {"nuq.queue_name": this.queueName,@@ -877,6 +937,7 @@ class NuQ<JobData = any, JobReturnValue = any> {FROM ${this.queueName}_backlog bWHERE b.id = $1LIMIT 1+ ON CONFLICT (id) DO NOTHINGRETURNING ${this.jobReturning.join(", ")}), del AS (DELETE FROM ${this.queueName}_backlog@@ -890,7 +951,7 @@ class NuQ<JobData = any, JobReturnValue = any> {);if (!result) {- return await this.addJob(id, data, {+ return await this.addJobIfNotExists(id, data, {...options,backlogged: false,});