Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(queue-worker): PoC of concurrency limits
Gergő Móricz53fce67ca1f1
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 37e14bafe..76796742c 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -12,7 +12,7 @@ import { startWebScraperPipeline } from "../main/runWebScraper";import { callWebhook } from "./webhook";import { logJob } from "./logging/log_job";import { initSDK } from "@hyperdx/node-opentelemetry";-import { Job } from "bullmq";+import { Job, Queue } from "bullmq";import { Logger } from "../lib/logger";import { Worker } from "bullmq";import systemMonitor from "./system-monitor";@@ -99,10 +99,10 @@ process.on("SIGINT", () => {});const workerFun = async (- queueName: string,+ queue: Queue,processJobInternal: (token: string, job: Job) => Promise<any>) => {- const worker = new Worker(queueName, null, {+ const worker = new Worker(queue.name, null, {connection: redisConnection,lockDuration: 1 * 60 * 1000, // 1 minute// lockRenewTime: 15 * 1000, // 15 seconds@@ -129,6 +129,29 @@ const workerFun = async (const job = await worker.getNextJob(token);if (job) {+ const concurrencyLimiterKey = "concurrency-limiter:" + job.data?.team_id;++ if (job.data && job.data.team_id) {+ const concurrencyLimit = 100; // TODO: determine based on price id+ const now = Date.now();+ const stalledJobTimeoutMs = 2 * 60 * 1000;++ redisConnection.zremrangebyscore(concurrencyLimiterKey, -Infinity, now);+ const activeJobsOfTeam = await redisConnection.zrangebyscore(concurrencyLimiterKey, now, Infinity);+ if (activeJobsOfTeam.length >= concurrencyLimit) {+ Logger.info("Moving job " + job.id + " back the queue -- concurrency limit hit");+ // Concurrency limit hit+ await job.moveToFailed(new Error("Concurrency limit hit"), token, false);+ await queue.add(job.name, job.data, {+ ...job.opts,+ jobId: job.id,+ priority: Math.round((job.opts.priority ?? 10) * 1.25), // exponential backoff for stuck jobs+ })+ } else {+ await redisConnection.zadd(concurrencyLimiterKey, now + stalledJobTimeoutMs, job.id);+ }+ }+if (job.data && job.data.sentry && Sentry.isInitialized()) {Sentry.continueTrace({@@ -159,7 +182,15 @@ const workerFun = async (},},async () => {- const res = await processJobInternal(token, job);+ let res;+ try {+ res = await processJobInternal(token, job);+ } finally {+ if (job.id && job.data && job.data.team_id) {+ await redisConnection.zrem(concurrencyLimiterKey, job.id);+ }+ }+if (res !== null) {span.setStatus({ code: 2 }); // ERROR} else {@@ -181,7 +212,10 @@ const workerFun = async (},},() => {- processJobInternal(token, job);+ processJobInternal(token, job)+ .finally(() => {++ });});}@@ -193,7 +227,7 @@ const workerFun = async (}};-workerFun(scrapeQueueName, processJobInternal);+workerFun(getScrapeQueue(), processJobInternal);async function processJob(job: Job, token: string) {Logger.info(`🐂 Worker taking job ${job.id}`);