Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/admin): concurrency queue backfilling endpoint

Gergő Móricz823cd1226692
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
new file mode 100644
index 000000000..5744e5e7f
--- /dev/null
+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
@@ -0,0 +1,81 @@
+import { logger as _logger } from "../../../lib/logger";
+import { Request, Response } from "express";
+import { getRedisConnection } from "../../../services/queue-service";
+import { scrapeQueue } from "../../../services/worker/nuq";
+import { pushConcurrencyLimitedJob } from "../../../lib/concurrency-limit";
+
+export async function concurrencyQueueBackfillController(
+ req: Request,
+ res: Response,
+) {
+ const logger = _logger.child({
+ module: "concurrencyQueueBackfillController",
+ });
+
+ logger.info("Starting concurrency queue backfill");
+
+ const backloggedOwnerIDs = req.query.teamId
+ ? [req.query.teamId as string]
+ : await scrapeQueue.getBackloggedOwnerIDs(logger);
+
+ for (const ownerId of backloggedOwnerIDs) {
+ logger.info("Backfilling concurrency queue for team", { teamId: ownerId });
+
+ const backloggedJobIDs = new Set(
+ await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, logger),
+ );
+ const queuedJobIDs = new Set<string>();
+
+ let cursor = "0";
+
+ do {
+ const result = await getRedisConnection().zscan(
+ `concurrency-limit-queue:${ownerId}`,
+ cursor,
+ "COUNT",
+ 20,
+ );
+ cursor = result[0];
+ const results = result[1];
+
+ results.forEach(x => queuedJobIDs.add(JSON.parse(x).id));
+ } while (cursor !== "0");
+
+ const jobIDsToAdd = new Set(
+ [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x)),
+ );
+
+ logger.info("Team statistics", {
+ teamId: ownerId,
+ backloggedJobIDs: backloggedJobIDs.size,
+ queuedJobIDs: queuedJobIDs.size,
+ jobIDsToAdd: jobIDsToAdd.size,
+ });
+
+ const jobsToAdd = await scrapeQueue.getJobsFromBacklog(
+ Array.from(jobIDsToAdd),
+ logger,
+ );
+
+ for (const job of jobsToAdd) {
+ await pushConcurrencyLimitedJob(
+ ownerId,
+ {
+ id: job.id,
+ data: job.data,
+ priority: job.priority,
+ listenable: job.listenChannelId !== undefined,
+ },
+ Infinity,
+ );
+ }
+
+ logger.info("Finished backfilling concurrency queue for team", {
+ teamId: ownerId,
+ });
+ }
+
+ logger.info("Finished backfilling all teams");
+
+ res.json({ ok: true });
+}
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 370e98291..c5c09ecdd 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -13,6 +13,7 @@ import {
} from "../controllers/v0/admin/metrics";
import { crawlCheckController } from "../controllers/v0/admin/crawl-check";
import { realtimeSearchController } from "../controllers/v2/f-search";
+import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";
export const adminRouter = express.Router();
@@ -70,3 +71,8 @@ adminRouter.post(
`/admin/${process.env.BULL_AUTH_KEY}/fsearch`,
wrap(realtimeSearchController),
);
+
+adminRouter.post(
+ `/admin/${process.env.BULL_AUTH_KEY}/concurrency-queue-backfill`,
+ wrap(concurrencyQueueBackfillController),
+);
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 8e57e1aac..6f737ed5f 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -455,6 +455,30 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
+ public async getJobsFromBacklog(
+ ids: string[],
+ _logger: Logger = logger,
+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {
+ if (ids.length === 0) return [];
+
+ const start = Date.now();
+ try {
+ return (
+ await nuqPool.query(
+ `SELECT ${this.jobBacklogReturning.join(", ")} FROM ${this.queueName}_backlog WHERE ${this.queueName}_backlog.id = ANY($1::uuid[]);`,
+ [ids],
+ )
+ ).rows.map(row => this.rowToJob(row, true)!);
+ } finally {
+ _logger.info("nuqGetJobsFromBacklog metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetJobsFromBacklog",
+ duration: Date.now() - start,
+ scrapeIds: ids.length,
+ });
+ }
+ }
+
public async getJobsWithStatus(
ids: string[],
status: NuQJobStatus,
@@ -563,6 +587,47 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
+ public async getBackloggedOwnerIDs(
+ _logger: Logger = logger,
+ ): Promise<string[]> {
+ const start = Date.now();
+ try {
+ return (
+ await nuqPool.query(
+ `SELECT DISTINCT owner_id FROM ${this.queueName}_backlog;`,
+ )
+ ).rows.map(row => row.owner_id);
+ } finally {
+ _logger.info("nuqGetBackloggedOwnerIDs metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetBackloggedOwnerIDs",
+ duration: Date.now() - start,
+ });
+ }
+ }
+
+ public async getBackloggedJobIDsOfOnwer(
+ ownerId: string,
+ _logger: Logger = logger,
+ ): Promise<string[]> {
+ const start = Date.now();
+ try {
+ return (
+ await nuqPool.query(
+ `SELECT id FROM ${this.queueName}_backlog WHERE owner_id = $1;`,
+ [ownerId],
+ )
+ ).rows.map(row => row.id);
+ } finally {
+ _logger.info("nuqGetBackloggedJobIDsOfOnwer metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetBackloggedJobIDsOfOnwer",
+ duration: Date.now() - start,
+ ownerId,
+ });
+ }
+ }
+
// TODO: make more generalizable
public async getCrawlJobsForListing(
groupId: string,