Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(nuq,concurrency-limit): handle if there are duplicate jobs in the concurrency queue (#2343)

Gergő Móricz29a1c3689ce7
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 913e4492b..bdd7c08ba 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -301,56 +301,88 @@ export async function concurrentJobDone(job: NuQJob<any>) {
await cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);
}
- const maxTeamConcurrency =
- (
- await getACUCTeam(
- job.data.team_id,
- false,
- true,
- job.data.is_extract ? RateLimiterMode.Extract : RateLimiterMode.Crawl,
- )
- )?.concurrency ?? 2;
- const currentActiveConcurrency = (
- await getConcurrencyLimitActiveJobs(job.data.team_id)
- ).length;
-
- if (currentActiveConcurrency < maxTeamConcurrency) {
- const nextJob = await getNextConcurrentJob(job.data.team_id);
- if (nextJob !== null) {
- await pushConcurrencyLimitActiveJob(
- job.data.team_id,
- nextJob.job.id,
- 60 * 1000,
- );
-
- if (nextJob.job.data.crawl_id) {
- await pushCrawlConcurrencyLimitActiveJob(
- nextJob.job.data.crawl_id,
+ let i = 0;
+ for (; i < 10; i++) {
+ const maxTeamConcurrency =
+ (
+ await getACUCTeam(
+ job.data.team_id,
+ false,
+ true,
+ job.data.is_extract
+ ? RateLimiterMode.Extract
+ : RateLimiterMode.Crawl,
+ )
+ )?.concurrency ?? 2;
+ const currentActiveConcurrency = (
+ await getConcurrencyLimitActiveJobs(job.data.team_id)
+ ).length;
+
+ if (currentActiveConcurrency < maxTeamConcurrency) {
+ const nextJob = await getNextConcurrentJob(job.data.team_id);
+ if (nextJob !== null) {
+ await pushConcurrencyLimitActiveJob(
+ job.data.team_id,
nextJob.job.id,
60 * 1000,
);
- const sc = await getCrawl(nextJob.job.data.crawl_id);
- if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {
- await new Promise(resolve =>
- setTimeout(resolve, sc.crawlerOptions.delay * 1000),
+ if (nextJob.job.data.crawl_id) {
+ await pushCrawlConcurrencyLimitActiveJob(
+ nextJob.job.data.crawl_id,
+ nextJob.job.id,
+ 60 * 1000,
);
- }
- }
- abTestJob(nextJob.job.data);
+ const sc = await getCrawl(nextJob.job.data.crawl_id);
+ if (sc !== null && typeof sc.crawlerOptions?.delay === "number") {
+ await new Promise(resolve =>
+ setTimeout(resolve, sc.crawlerOptions.delay * 1000),
+ );
+ }
+ }
- await scrapeQueue.promoteJobFromBacklogOrAdd(
- nextJob.job.id,
- nextJob.job.data,
- {
- priority: nextJob.job.priority,
- listenable: nextJob.job.listenable,
- ownerId: nextJob.job.data.team_id ?? undefined,
- groupId: nextJob.job.data.crawl_id ?? undefined,
- },
- );
+ abTestJob(nextJob.job.data);
+
+ const promotedSuccessfully =
+ (await scrapeQueue.promoteJobFromBacklogOrAdd(
+ nextJob.job.id,
+ nextJob.job.data,
+ {
+ priority: nextJob.job.priority,
+ listenable: nextJob.job.listenable,
+ ownerId: nextJob.job.data.team_id ?? undefined,
+ groupId: nextJob.job.data.crawl_id ?? undefined,
+ },
+ )) !== null;
+
+ if (promotedSuccessfully) {
+ logger.debug("Successfully promoted concurrent queued job", {});
+ break;
+ } else {
+ logger.warn(
+ "Was unable to promote concurrent queued job as it already exists in the database",
+ {
+ teamId: job.data.team_id,
+ jobId: nextJob.job.id,
+ },
+ );
+ }
+ } else {
+ break;
+ }
+ } else {
+ break;
}
}
+
+ if (i === 10) {
+ logger.warn(
+ "Failed to promote a concurrent job after 10 iterations, bailing!",
+ {
+ teamId: job.data.team_id,
+ },
+ );
+ }
}
}
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 768d78d62..8e57e1aac 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -706,6 +706,66 @@ class NuQ<JobData = any, JobReturnValue = any> {
});
}
+ public async addJobIfNotExists(
+ id: string,
+ data: JobData,
+ options: NuQJobOptions,
+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {
+ return withSpan("nuq.addJob", async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
+ "nuq.priority": options.priority ?? 0,
+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
+ "nuq.listenable": options.listenable ?? false,
+ });
+
+ const start = Date.now();
+ try {
+ const result = this.rowToJob(
+ (
+ await nuqPool.query(
+ `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) ON CONFLICT (id) DO NOTHING RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,
+ [
+ id,
+ data,
+ options.priority ?? 0,
+ options.listenable ? this.listenChannelId : null,
+ normalizeOwnerId(options.ownerId),
+ options.groupId ?? null,
+ ...(options.backlogged
+ ? [
+ options.backloggedTimesOutAt
+ ? options.backloggedTimesOutAt.toISOString()
+ : null,
+ ]
+ : []),
+ ],
+ )
+ ).rows[0],
+ );
+
+ setSpanAttributes(span, {
+ "nuq.job_created": result !== null,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ logger.info("nuqAddJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqAddJob",
+ duration,
+ scrapeId: id,
+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
+ });
+ }
+ });
+ }
+
public async addJobs(
jobs: Array<{
id: string;
@@ -855,7 +915,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
id: string,
data: JobData,
options: NuQJobOptions,
- ): Promise<NuQJob<JobData, JobReturnValue>> {
+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {
return withSpan("nuq.promoteJobFromBacklogOrAdd", async span => {
setSpanAttributes(span, {
"nuq.queue_name": this.queueName,
@@ -877,6 +937,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
FROM ${this.queueName}_backlog b
WHERE b.id = $1
LIMIT 1
+ ON CONFLICT (id) DO NOTHING
RETURNING ${this.jobReturning.join(", ")}
), del AS (
DELETE FROM ${this.queueName}_backlog
@@ -890,7 +951,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
);
if (!result) {
- return await this.addJob(id, data, {
+ return await this.addJobIfNotExists(id, data, {
...options,
backlogged: false,
});