Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): concurrency queue reconciler (#2929)

tomsideguide1b3494f181a2
diff --git a/apps/api/package.json b/apps/api/package.json
index 76e29e4e3..38715d488 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -27,6 +27,8 @@
"nuq-worker:production": "node dist/src/services/worker/nuq-worker.js",
"nuq-prefetch-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-prefetch-worker.js\"",
"nuq-prefetch-worker:production": "node dist/src/services/worker/nuq-prefetch-worker.js",
+ "nuq-reconciler-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-reconciler-worker.js\"",
+ "nuq-reconciler-worker:production": "node dist/src/services/worker/nuq-reconciler-worker.js",
"extract-worker": "tsc-watch --onSuccess \"node dist/src/services/extract-worker.js\"",
"extract-worker:production": "node dist/src/services/extract-worker.js",
"index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 0f1cbc167..88c5e5dcf 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -121,6 +121,7 @@ const configSchema = z.object({
NUQ_WORKER_START_PORT: z.coerce.number().default(3006),
NUQ_WORKER_COUNT: z.coerce.number().default(5),
NUQ_PREFETCH_WORKER_PORT: z.coerce.number().default(3011).catch(3011), // todo: investigate why .catch is needed
+ NUQ_RECONCILER_WORKER_PORT: z.coerce.number().default(3012).catch(3012),
EXTRACT_WORKER_PORT: z.coerce.number().default(3004),
NUQ_WAIT_MODE: z.string().optional(),
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
index 3f941aa52..ee810a7b0 100644
--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
@@ -1,14 +1,6 @@
import { logger as _logger } from "../../../lib/logger";
import { Request, Response } from "express";
-import { getRedisConnection } from "../../../services/queue-service";
-import { scrapeQueue } from "../../../services/worker/nuq";
-import {
- pushConcurrencyLimitedJobs,
- pushConcurrencyLimitActiveJob,
- getConcurrencyLimitActiveJobsCount,
-} from "../../../lib/concurrency-limit";
-import { RateLimiterMode } from "../../../types";
-import { getACUCTeam } from "../../auth";
+import { reconcileConcurrencyQueue } from "../../../lib/concurrency-queue-reconciler";
export async function concurrencyQueueBackfillController(
req: Request,
@@ -20,157 +12,16 @@ export async function concurrencyQueueBackfillController(
logger.info("Starting concurrency queue backfill");
- // Skip the precrawl team β€” it has millions of backlogged jobs that cause
- // this endpoint to time out before it can process any other teams.
- const IGNORED_TEAM_IDS = new Set([
- "86100d9a-54ec-43ab-8f90-ba5852aede34", // Precrawl team
- ]);
-
- const backloggedOwnerIDs = (
- req.query.teamId
- ? [req.query.teamId as string]
- : await scrapeQueue.getBackloggedOwnerIDs(logger)
- ).filter(id => !IGNORED_TEAM_IDS.has(id));
-
- const teamResults: {
- teamId: string;
- backloggedJobs: number;
- alreadyQueued: number;
- queued: number;
- started: number;
- }[] = [];
-
- for (const ownerId of backloggedOwnerIDs) {
- logger.info("Backfilling concurrency queue for team", { teamId: ownerId });
-
- try {
- const backloggedJobIDs = new Set(
- await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, logger),
- );
- const queuedJobIDs = new Set<string>();
-
- let cursor = "0";
-
- do {
- const result = await getRedisConnection().zscan(
- `concurrency-limit-queue:${ownerId}`,
- cursor,
- "COUNT",
- 1000,
- );
- cursor = result[0];
- const results = result[1];
-
- // zscan returns [member1, score1, member2, score2, ...]
- // Only parse members (even indices), skip scores (odd indices)
- for (let i = 0; i < results.length; i += 2) {
- queuedJobIDs.add(results[i]);
- }
- } while (cursor !== "0");
-
- const jobIDsToAdd = new Set(
- [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x)),
- );
-
- logger.info("Team statistics", {
- teamId: ownerId,
- backloggedJobIDs: backloggedJobIDs.size,
- queuedJobIDs: queuedJobIDs.size,
- jobIDsToAdd: jobIDsToAdd.size,
- });
-
- const jobsToAdd = await scrapeQueue.getJobsFromBacklog(
- Array.from(jobIDsToAdd),
- logger,
- );
-
- // Get concurrency limit (only crawl β€” extract ACUC can overflow integer for high-usage teams)
- const crawlACUC = await getACUCTeam(
- ownerId,
- false,
- true,
- RateLimiterMode.Crawl,
- );
- const maxCrawlConcurrency = crawlACUC?.concurrency ?? 2;
- const maxExtractConcurrency = crawlACUC?.concurrency ?? 2;
-
- const currentActiveConcurrency =
- await getConcurrencyLimitActiveJobsCount(ownerId);
-
- const jobsToStart: typeof jobsToAdd = [];
- const jobsToQueue: typeof jobsToAdd = [];
-
- let activeCount = currentActiveConcurrency;
- for (const job of jobsToAdd) {
- const isExtract = "is_extract" in job.data && job.data.is_extract;
- const limit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;
-
- if (activeCount < limit) {
- jobsToStart.push(job);
- activeCount++;
- } else {
- jobsToQueue.push(job);
- }
- }
-
- if (jobsToQueue.length > 0) {
- await pushConcurrencyLimitedJobs(
- ownerId,
- jobsToQueue.map(job => ({
- job: {
- id: job.id,
- data: job.data,
- priority: job.priority,
- listenable: job.listenChannelId !== undefined,
- },
- timeout: Infinity,
- })),
- );
- }
-
- // Promote jobs that can start immediately
- // These involve DB transactions per job so they remain sequential
- for (const job of jobsToStart) {
- await scrapeQueue.promoteJobFromBacklogOrAdd(job.id, job.data, {
- priority: job.priority,
- listenable: job.listenChannelId !== undefined,
- ownerId: job.data.team_id ?? undefined,
- groupId: job.data.crawl_id ?? undefined,
- });
-
- await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);
- }
-
- logger.info("Finished backfilling concurrency queue for team", {
- teamId: ownerId,
- startedCount: jobsToStart.length,
- queuedCount: jobsToQueue.length,
- });
-
- teamResults.push({
- teamId: ownerId,
- backloggedJobs: backloggedJobIDs.size,
- alreadyQueued: queuedJobIDs.size,
- queued: jobsToQueue.length,
- started: jobsToStart.length,
- });
- } catch (e) {
- logger.error("Failed to backfill team, skipping", {
- teamId: ownerId,
- error: e,
- });
- }
- }
-
- logger.info("Finished backfilling all teams", {
- teamsProcessed: teamResults.length,
+ const teamId =
+ typeof req.query.teamId === "string"
+ ? (req.query.teamId as string)
+ : undefined;
+ const summary = await reconcileConcurrencyQueue({
+ teamId,
+ logger,
});
- res.json({
- ok: true,
- teamsProcessed: teamResults.length,
- totalJobsQueued: teamResults.reduce((sum, t) => sum + t.queued, 0),
- totalJobsStarted: teamResults.reduce((sum, t) => sum + t.started, 0),
- teams: teamResults,
- });
+ logger.info("Finished backfilling all teams", summary);
+
+ res.json({ ok: true, ...summary });
}
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 71dda3794..1deef110a 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -34,6 +34,7 @@ interface Services {
worker?: ProcessResult;
nuqWorkers: ProcessResult[];
nuqPrefetchWorker?: ProcessResult;
+ nuqReconcilerWorker?: ProcessResult;
extractWorker?: ProcessResult;
indexWorker?: ProcessResult;
command?: ProcessResult;
@@ -101,6 +102,7 @@ const EXTRACT_WORKER_PORT = config.EXTRACT_WORKER_PORT;
const NUQ_WORKER_START_PORT = config.NUQ_WORKER_START_PORT;
const NUQ_WORKER_COUNT = config.NUQ_WORKER_COUNT;
const NUQ_PREFETCH_WORKER_PORT = NUQ_WORKER_START_PORT + NUQ_WORKER_COUNT;
+const NUQ_RECONCILER_WORKER_PORT = NUQ_PREFETCH_WORKER_PORT + 1;
// PostgreSQL credentials (with defaults for backward compatibility)
const POSTGRES_USER = config.POSTGRES_USER;
@@ -779,6 +781,18 @@ async function startServices(command?: string[]): Promise<Services> {
},
);
+ const nuqReconcilerWorker = execForward(
+ "nuq-reconciler",
+ process.argv[2] === "--start-docker"
+ ? "node dist/src/services/worker/nuq-reconciler-worker.js"
+ : "pnpm nuq-reconciler-worker:production",
+ {
+ NUQ_RECONCILER_WORKER_PORT: String(NUQ_RECONCILER_WORKER_PORT),
+ NUQ_REDUCE_NOISE: "true",
+ NUQ_POD_NAME: "nuq-reconciler-worker-0",
+ },
+ );
+
const indexWorker = config.USE_DB_AUTHENTICATION
? execForward(
"index-worker",
@@ -813,6 +827,7 @@ async function startServices(command?: string[]): Promise<Services> {
worker,
nuqWorkers,
nuqPrefetchWorker,
+ nuqReconcilerWorker,
indexWorker,
extractWorker,
command: commandProcess,
@@ -829,6 +844,7 @@ async function stopDevelopmentServices(services: Services) {
services.worker?.process,
...services.nuqWorkers.map(w => w.process),
services.nuqPrefetchWorker?.process,
+ services.nuqReconcilerWorker?.process,
services.indexWorker?.process,
services.extractWorker?.process,
services.command?.process,
@@ -992,6 +1008,8 @@ async function waitForTermination(services: Services): Promise<void> {
if (services.extractWorker) promises.push(services.extractWorker.promise);
if (services.nuqPrefetchWorker)
promises.push(services.nuqPrefetchWorker.promise);
+ if (services.nuqReconcilerWorker)
+ promises.push(services.nuqReconcilerWorker.promise);
promises.push(...services.nuqWorkers.map(w => w.promise));
diff --git a/apps/api/src/lib/concurrency-queue-reconciler.ts b/apps/api/src/lib/concurrency-queue-reconciler.ts
new file mode 100644
index 000000000..8c3975116
--- /dev/null
+++ b/apps/api/src/lib/concurrency-queue-reconciler.ts
@@ -0,0 +1,232 @@
+import { Logger } from "winston";
+import { getACUCTeam } from "../controllers/auth";
+import { getRedisConnection } from "../services/queue-service";
+import { scrapeQueue, type NuQJob } from "../services/worker/nuq";
+import { RateLimiterMode, type ScrapeJobData } from "../types";
+import {
+ getConcurrencyLimitActiveJobs,
+ pushConcurrencyLimitActiveJob,
+ pushConcurrencyLimitedJob,
+ pushCrawlConcurrencyLimitActiveJob,
+} from "./concurrency-limit";
+import { getCrawl } from "./crawl-redis";
+import { logger as _logger } from "./logger";
+
+interface ReconcileOptions {
+ teamId?: string;
+ logger?: Logger;
+}
+
+interface ReconcileResult {
+ teamsScanned: number;
+ teamsWithDrift: number;
+ jobsRequeued: number;
+ jobsStarted: number;
+}
+
+function isExtractJob(data: ScrapeJobData): boolean {
+ return "is_extract" in data && !!data.is_extract;
+}
+
+function getBacklogJobTimeout(jobData: ScrapeJobData): number {
+ if (jobData.crawl_id) return Infinity;
+
+ if ("scrapeOptions" in jobData && jobData.scrapeOptions?.timeout)
+ return jobData.scrapeOptions.timeout;
+
+ return 60 * 1000;
+}
+
+async function requeueJob(
+ ownerId: string,
+ job: NuQJob<ScrapeJobData>,
+): Promise<void> {
+ await pushConcurrencyLimitedJob(
+ ownerId,
+ {
+ id: job.id,
+ data: job.data,
+ priority: job.priority,
+ listenable: job.listenChannelId !== undefined,
+ },
+ getBacklogJobTimeout(job.data),
+ );
+}
+
+async function getQueuedJobIDs(teamId: string): Promise<Set<string>> {
+ const queuedJobIDs = new Set<string>();
+ let cursor = "0";
+
+ do {
+ const [nextCursor, results] = await getRedisConnection().zscan(
+ `concurrency-limit-queue:${teamId}`,
+ cursor,
+ "COUNT",
+ 100,
+ );
+ cursor = nextCursor;
+
+ // zscan returns [member1, score1, member2, score2, ...]
+ for (let i = 0; i < results.length; i += 2) {
+ queuedJobIDs.add(results[i]);
+ }
+ } while (cursor !== "0");
+
+ return queuedJobIDs;
+}
+
+async function reconcileTeam(
+ ownerId: string,
+ teamLogger: Logger,
+): Promise<{ jobsStarted: number; jobsRequeued: number } | null> {
+ const backloggedJobIDs = new Set(
+ await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, teamLogger),
+ );
+ if (backloggedJobIDs.size === 0) {
+ return null;
+ }
+
+ const queuedJobIDs = await getQueuedJobIDs(ownerId);
+ const missingJobIDs = [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x));
+
+ if (missingJobIDs.length === 0) {
+ return null;
+ }
+
+ const jobsToRecover = await scrapeQueue.getJobsFromBacklog(
+ missingJobIDs,
+ teamLogger,
+ );
+ if (jobsToRecover.length === 0) {
+ return null;
+ }
+
+ const maxCrawlConcurrency =
+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl))
+ ?.concurrency ?? 2;
+ const maxExtractConcurrency =
+ (await getACUCTeam(ownerId, false, true, RateLimiterMode.Extract))
+ ?.concurrency ?? 2;
+
+ // Split active count by type so one type's active jobs don't gate the other
+ const activeJobIds = await getConcurrencyLimitActiveJobs(ownerId);
+ const activeJobs = await scrapeQueue.getJobs(activeJobIds, teamLogger);
+ let activeCrawlCount = 0;
+ let activeExtractCount = 0;
+ for (const aj of activeJobs) {
+ if (isExtractJob(aj.data)) {
+ activeExtractCount++;
+ } else {
+ activeCrawlCount++;
+ }
+ }
+
+ const jobsToStart: typeof jobsToRecover = [];
+ const jobsToQueue: typeof jobsToRecover = [];
+
+ for (const job of jobsToRecover) {
+ const isExtract = isExtractJob(job.data);
+ const teamLimit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;
+ const activeCount = isExtract ? activeExtractCount : activeCrawlCount;
+
+ if (activeCount < teamLimit) {
+ jobsToStart.push(job);
+ if (isExtract) activeExtractCount++;
+ else activeCrawlCount++;
+ } else {
+ jobsToQueue.push(job);
+ }
+ }
+
+ let jobsStarted = 0;
+ let jobsRequeued = 0;
+
+ for (const job of jobsToQueue) {
+ await requeueJob(ownerId, job);
+ jobsRequeued++;
+ }
+
+ for (const job of jobsToStart) {
+ const promoted = await scrapeQueue.promoteJobFromBacklogOrAdd(
+ job.id,
+ job.data,
+ {
+ priority: job.priority,
+ listenable: job.listenChannelId !== undefined,
+ ownerId: job.data.team_id ?? undefined,
+ groupId: job.data.crawl_id ?? undefined,
+ },
+ );
+
+ if (promoted !== null) {
+ await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);
+
+ if (job.data.crawl_id) {
+ const sc = await getCrawl(job.data.crawl_id);
+ if (sc?.crawlerOptions?.delay || sc?.maxConcurrency) {
+ await pushCrawlConcurrencyLimitActiveJob(
+ job.data.crawl_id,
+ job.id,
+ 60 * 1000,
+ );
+ }
+ }
+
+ jobsStarted++;
+ } else {
+ teamLogger.warn("Job promotion failed, re-queuing job", {
+ jobId: job.id,
+ });
+ await requeueJob(ownerId, job);
+ jobsRequeued++;
+ }
+ }
+
+ teamLogger.info("Recovered drift in concurrency queue", {
+ missingJobs: missingJobIDs.length,
+ recoveredJobs: jobsToRecover.length,
+ requeuedJobs: jobsRequeued,
+ startedJobs: jobsStarted,
+ });
+
+ return { jobsStarted, jobsRequeued };
+}
+
+export async function reconcileConcurrencyQueue(
+ options: ReconcileOptions = {},
+): Promise<ReconcileResult> {
+ const logger = (options.logger ?? _logger).child({
+ module: "concurrencyQueueReconciler",
+ scopedTeamId: options.teamId,
+ });
+
+ const owners = options.teamId
+ ? [options.teamId]
+ : await scrapeQueue.getBackloggedOwnerIDs(logger);
+
+ const ownerIds = owners.filter((x): x is string => typeof x === "string");
+
+ const result: ReconcileResult = {
+ teamsScanned: ownerIds.length,
+ teamsWithDrift: 0,
+ jobsRequeued: 0,
+ jobsStarted: 0,
+ };
+
+ for (const ownerId of ownerIds) {
+ const teamLogger = logger.child({ teamId: ownerId });
+
+ try {
+ const teamResult = await reconcileTeam(ownerId, teamLogger);
+ if (teamResult !== null) {
+ result.teamsWithDrift++;
+ result.jobsStarted += teamResult.jobsStarted;
+ result.jobsRequeued += teamResult.jobsRequeued;
+ }
+ } catch (error) {
+ teamLogger.error("Failed to reconcile team, skipping", { error });
+ }
+ }
+
+ return result;
+}
diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.ts
new file mode 100644
index 000000000..19100c479
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts
@@ -0,0 +1,99 @@
+import "dotenv/config";
+import { config } from "../../config";
+import "../sentry";
+import { setSentryServiceTag } from "../sentry";
+import { logger as _logger } from "../../lib/logger";
+import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconciler";
+import { Counter, register } from "prom-client";
+import Express from "express";
+
+const RECONCILER_INTERVAL_MS = 60 * 1000;
+
+const reconcilerRunsTotal = new Counter({
+ name: "concurrency_queue_reconciler_runs_total",
+ help: "Total completed concurrency queue reconciler runs",
+});
+
+const reconcilerFailuresTotal = new Counter({
+ name: "concurrency_queue_reconciler_failures_total",
+ help: "Total failed concurrency queue reconciler runs",
+});
+
+const reconcilerJobsRecoveredTotal = new Counter({
+ name: "concurrency_queue_reconciler_jobs_recovered_total",
+ help: "Total drifted jobs recovered by the reconciler",
+});
+
+(async () => {
+ setSentryServiceTag("nuq-reconciler-worker");
+
+ let isShuttingDown = false;
+ let reconcilerInFlight = false;
+
+ const app = Express();
+
+ app.get("/metrics", async (_, res) => {
+ try {
+ res.contentType("text/plain").send(await register.metrics());
+ } catch (error) {
+ _logger.error("Failed to collect metrics", { error });
+ res.status(500).send("Failed to collect metrics");
+ }
+ });
+ app.get("/health", (_, res) => {
+ res.status(200).send("OK");
+ });
+
+ const server = app.listen(config.NUQ_RECONCILER_WORKER_PORT, () => {
+ _logger.info("NuQ reconciler worker started", {
+ port: config.NUQ_RECONCILER_WORKER_PORT,
+ });
+ });
+
+ async function shutdown() {
+ if (isShuttingDown) return;
+ isShuttingDown = true;
+ _logger.info("NuQ reconciler worker shutting down");
+
+ while (reconcilerInFlight) {
+ _logger.info("Waiting for in-flight reconciliation to complete...");
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+
+ server.close(() => {
+ _logger.info("NuQ reconciler worker shut down");
+ process.exit(0);
+ });
+ }
+
+ if (require.main === module) {
+ process.on("SIGINT", shutdown);
+ process.on("SIGTERM", shutdown);
+ }
+
+ while (!isShuttingDown) {
+ if (!reconcilerInFlight) {
+ reconcilerInFlight = true;
+
+ try {
+ const summary = await reconcileConcurrencyQueue({
+ logger: _logger,
+ });
+
+ reconcilerRunsTotal.inc();
+ reconcilerJobsRecoveredTotal.inc(
+ summary.jobsRequeued + summary.jobsStarted,
+ );
+
+ _logger.info("Concurrency queue reconciler run complete", summary);
+ } catch (error) {
+ reconcilerFailuresTotal.inc();
+ _logger.error("Concurrency queue reconciler run failed", { error });
+ } finally {
+ reconcilerInFlight = false;
+ }
+ }
+
+ await new Promise(resolve => setTimeout(resolve, RECONCILER_INTERVAL_MS));
+ }
+})();
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index 5e7318082..69f68eb19 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -27,9 +27,9 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
const app = Express();
- app.get("/metrics", (_, res) =>
- res.contentType("text/plain").send(nuqGetLocalMetrics()),
- );
+ app.get("/metrics", (_, res) => {
+ res.contentType("text/plain").send(nuqGetLocalMetrics());
+ });
app.get("/health", async (_, res) => {
if (await nuqHealthCheck()) {
res.status(200).send("OK");