Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/admin): concurrency queue backfilling endpoint
Gergő Móricz823cd1226692
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.tsnew file mode 100644index 000000000..5744e5e7f--- /dev/null+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts@@ -0,0 +1,81 @@+import { logger as _logger } from "../../../lib/logger";+import { Request, Response } from "express";+import { getRedisConnection } from "../../../services/queue-service";+import { scrapeQueue } from "../../../services/worker/nuq";+import { pushConcurrencyLimitedJob } from "../../../lib/concurrency-limit";++export async function concurrencyQueueBackfillController(+ req: Request,+ res: Response,+) {+ const logger = _logger.child({+ module: "concurrencyQueueBackfillController",+ });++ logger.info("Starting concurrency queue backfill");++ const backloggedOwnerIDs = req.query.teamId+ ? [req.query.teamId as string]+ : await scrapeQueue.getBackloggedOwnerIDs(logger);++ for (const ownerId of backloggedOwnerIDs) {+ logger.info("Backfilling concurrency queue for team", { teamId: ownerId });++ const backloggedJobIDs = new Set(+ await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, logger),+ );+ const queuedJobIDs = new Set<string>();++ let cursor = "0";++ do {+ const result = await getRedisConnection().zscan(+ `concurrency-limit-queue:${ownerId}`,+ cursor,+ "COUNT",+ 20,+ );+ cursor = result[0];+ const results = result[1];++ results.forEach(x => queuedJobIDs.add(JSON.parse(x).id));+ } while (cursor !== "0");++ const jobIDsToAdd = new Set(+ [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x)),+ );++ logger.info("Team statistics", {+ teamId: ownerId,+ backloggedJobIDs: backloggedJobIDs.size,+ queuedJobIDs: queuedJobIDs.size,+ jobIDsToAdd: jobIDsToAdd.size,+ });++ const jobsToAdd = await scrapeQueue.getJobsFromBacklog(+ Array.from(jobIDsToAdd),+ logger,+ );++ for (const job of jobsToAdd) {+ await pushConcurrencyLimitedJob(+ ownerId,+ {+ id: job.id,+ data: job.data,+ priority: job.priority,+ listenable: job.listenChannelId !== undefined,+ },+ Infinity,+ );+ }++ logger.info("Finished backfilling concurrency queue for team", {+ teamId: ownerId,+ });+ }++ logger.info("Finished backfilling all teams");++ res.json({ ok: true });+}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 370e98291..c5c09ecdd 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -13,6 +13,7 @@ import {} from "../controllers/v0/admin/metrics";import { crawlCheckController } from "../controllers/v0/admin/crawl-check";import { realtimeSearchController } from "../controllers/v2/f-search";+import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";export const adminRouter = express.Router();@@ -70,3 +71,8 @@ adminRouter.post(`/admin/${process.env.BULL_AUTH_KEY}/fsearch`,wrap(realtimeSearchController),);++adminRouter.post(+ `/admin/${process.env.BULL_AUTH_KEY}/concurrency-queue-backfill`,+ wrap(concurrencyQueueBackfillController),+);diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 8e57e1aac..6f737ed5f 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -455,6 +455,30 @@ class NuQ<JobData = any, JobReturnValue = any> {}}+ public async getJobsFromBacklog(+ ids: string[],+ _logger: Logger = logger,+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {+ if (ids.length === 0) return [];++ const start = Date.now();+ try {+ return (+ await nuqPool.query(+ `SELECT ${this.jobBacklogReturning.join(", ")} FROM ${this.queueName}_backlog WHERE ${this.queueName}_backlog.id = ANY($1::uuid[]);`,+ [ids],+ )+ ).rows.map(row => this.rowToJob(row, true)!);+ } finally {+ _logger.info("nuqGetJobsFromBacklog metrics", {+ module: "nuq/metrics",+ method: "nuqGetJobsFromBacklog",+ duration: Date.now() - start,+ scrapeIds: ids.length,+ });+ }+ }+public async getJobsWithStatus(ids: string[],status: NuQJobStatus,@@ -563,6 +587,47 @@ class NuQ<JobData = any, JobReturnValue = any> {}}+ public async getBackloggedOwnerIDs(+ _logger: Logger = logger,+ ): Promise<string[]> {+ const start = Date.now();+ try {+ return (+ await nuqPool.query(+ `SELECT DISTINCT owner_id FROM ${this.queueName}_backlog;`,+ )+ ).rows.map(row => row.owner_id);+ } finally {+ _logger.info("nuqGetBackloggedOwnerIDs metrics", {+ module: "nuq/metrics",+ method: "nuqGetBackloggedOwnerIDs",+ duration: Date.now() - start,+ });+ }+ }++ public async getBackloggedJobIDsOfOnwer(+ ownerId: string,+ _logger: Logger = logger,+ ): Promise<string[]> {+ const start = Date.now();+ try {+ return (+ await nuqPool.query(+ `SELECT id FROM ${this.queueName}_backlog WHERE owner_id = $1;`,+ [ownerId],+ )+ ).rows.map(row => row.id);+ } finally {+ _logger.info("nuqGetBackloggedJobIDsOfOnwer metrics", {+ module: "nuq/metrics",+ method: "nuqGetBackloggedJobIDsOfOnwer",+ duration: Date.now() - start,+ ownerId,+ });+ }+ }+// TODO: make more generalizablepublic async getCrawlJobsForListing(groupId: string,