Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(admin): avoid extract ACUC integer overflow in backfill endpoint (#2934)

firecrawl-spring[bot]b738ff61e7b5
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
index 5a67e6cba..a2f2be47b 100644
--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
@@ -35,114 +35,123 @@ export async function concurrencyQueueBackfillController(
for (const ownerId of backloggedOwnerIDs) {
logger.info("Backfilling concurrency queue for team", { teamId: ownerId });
- const backloggedJobIDs = new Set(
- await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, logger),
- );
- const queuedJobIDs = new Set<string>();
-
- let cursor = "0";
-
- do {
- const result = await getRedisConnection().zscan(
- `concurrency-limit-queue:${ownerId}`,
- cursor,
- "COUNT",
- 1000,
+ try {
+ const backloggedJobIDs = new Set(
+ await scrapeQueue.getBackloggedJobIDsOfOnwer(ownerId, logger),
+ );
+ const queuedJobIDs = new Set<string>();
+
+ let cursor = "0";
+
+ do {
+ const result = await getRedisConnection().zscan(
+ `concurrency-limit-queue:${ownerId}`,
+ cursor,
+ "COUNT",
+ 1000,
+ );
+ cursor = result[0];
+ const results = result[1];
+
+ // zscan returns [member1, score1, member2, score2, ...]
+ // Only parse members (even indices), skip scores (odd indices)
+ for (let i = 0; i < results.length; i += 2) {
+ queuedJobIDs.add(results[i]);
+ }
+ } while (cursor !== "0");
+
+ const jobIDsToAdd = new Set(
+ [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x)),
);
- cursor = result[0];
- const results = result[1];
- // zscan returns [member1, score1, member2, score2, ...]
- // Only parse members (even indices), skip scores (odd indices)
- for (let i = 0; i < results.length; i += 2) {
- queuedJobIDs.add(results[i]);
- }
- } while (cursor !== "0");
-
- const jobIDsToAdd = new Set(
- [...backloggedJobIDs].filter(x => !queuedJobIDs.has(x)),
- );
-
- logger.info("Team statistics", {
- teamId: ownerId,
- backloggedJobIDs: backloggedJobIDs.size,
- queuedJobIDs: queuedJobIDs.size,
- jobIDsToAdd: jobIDsToAdd.size,
- });
-
- const jobsToAdd = await scrapeQueue.getJobsFromBacklog(
- Array.from(jobIDsToAdd),
- logger,
- );
-
- // Get concurrency limits for both job types
- const [crawlACUC, extractACUC] = await Promise.all([
- getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl),
- getACUCTeam(ownerId, false, true, RateLimiterMode.Extract),
- ]);
- const maxCrawlConcurrency = crawlACUC?.concurrency ?? 2;
- const maxExtractConcurrency = extractACUC?.concurrency ?? 2;
-
- const currentActiveConcurrency =
- await getConcurrencyLimitActiveJobsCount(ownerId);
-
- const jobsToStart: typeof jobsToAdd = [];
- const jobsToQueue: typeof jobsToAdd = [];
-
- let activeCount = currentActiveConcurrency;
- for (const job of jobsToAdd) {
- const isExtract = "is_extract" in job.data && job.data.is_extract;
- const limit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;
-
- if (activeCount < limit) {
- jobsToStart.push(job);
- activeCount++;
- } else {
- jobsToQueue.push(job);
- }
- }
+ logger.info("Team statistics", {
+ teamId: ownerId,
+ backloggedJobIDs: backloggedJobIDs.size,
+ queuedJobIDs: queuedJobIDs.size,
+ jobIDsToAdd: jobIDsToAdd.size,
+ });
+
+ const jobsToAdd = await scrapeQueue.getJobsFromBacklog(
+ Array.from(jobIDsToAdd),
+ logger,
+ );
- if (jobsToQueue.length > 0) {
- await pushConcurrencyLimitedJobs(
+ // Get concurrency limit (only crawl — extract ACUC can overflow integer for high-usage teams)
+ const crawlACUC = await getACUCTeam(
ownerId,
- jobsToQueue.map(job => ({
- job: {
- id: job.id,
- data: job.data,
- priority: job.priority,
- listenable: job.listenChannelId !== undefined,
- },
- timeout: Infinity,
- })),
+ false,
+ true,
+ RateLimiterMode.Crawl,
);
- }
+ const maxCrawlConcurrency = crawlACUC?.concurrency ?? 2;
+ const maxExtractConcurrency = crawlACUC?.concurrency ?? 2;
+
+ const currentActiveConcurrency =
+ await getConcurrencyLimitActiveJobsCount(ownerId);
+
+ const jobsToStart: typeof jobsToAdd = [];
+ const jobsToQueue: typeof jobsToAdd = [];
+
+ let activeCount = currentActiveConcurrency;
+ for (const job of jobsToAdd) {
+ const isExtract = "is_extract" in job.data && job.data.is_extract;
+ const limit = isExtract ? maxExtractConcurrency : maxCrawlConcurrency;
+
+ if (activeCount < limit) {
+ jobsToStart.push(job);
+ activeCount++;
+ } else {
+ jobsToQueue.push(job);
+ }
+ }
+
+ if (jobsToQueue.length > 0) {
+ await pushConcurrencyLimitedJobs(
+ ownerId,
+ jobsToQueue.map(job => ({
+ job: {
+ id: job.id,
+ data: job.data,
+ priority: job.priority,
+ listenable: job.listenChannelId !== undefined,
+ },
+ timeout: Infinity,
+ })),
+ );
+ }
+
+ // Promote jobs that can start immediately
+ // These involve DB transactions per job so they remain sequential
+ for (const job of jobsToStart) {
+ await scrapeQueue.promoteJobFromBacklogOrAdd(job.id, job.data, {
+ priority: job.priority,
+ listenable: job.listenChannelId !== undefined,
+ ownerId: job.data.team_id ?? undefined,
+ groupId: job.data.crawl_id ?? undefined,
+ });
+
+ await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);
+ }
- // Promote jobs that can start immediately
- // These involve DB transactions per job so they remain sequential
- for (const job of jobsToStart) {
- await scrapeQueue.promoteJobFromBacklogOrAdd(job.id, job.data, {
- priority: job.priority,
- listenable: job.listenChannelId !== undefined,
- ownerId: job.data.team_id ?? undefined,
- groupId: job.data.crawl_id ?? undefined,
+ logger.info("Finished backfilling concurrency queue for team", {
+ teamId: ownerId,
+ startedCount: jobsToStart.length,
+ queuedCount: jobsToQueue.length,
});
- await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);
+ teamResults.push({
+ teamId: ownerId,
+ backloggedJobs: backloggedJobIDs.size,
+ alreadyQueued: queuedJobIDs.size,
+ queued: jobsToQueue.length,
+ started: jobsToStart.length,
+ });
+ } catch (e) {
+ logger.error("Failed to backfill team, skipping", {
+ teamId: ownerId,
+ error: e,
+ });
}
-
- logger.info("Finished backfilling concurrency queue for team", {
- teamId: ownerId,
- startedCount: jobsToStart.length,
- queuedCount: jobsToQueue.length,
- });
-
- teamResults.push({
- teamId: ownerId,
- backloggedJobs: backloggedJobIDs.size,
- alreadyQueued: queuedJobIDs.size,
- queued: jobsToQueue.length,
- started: jobsToStart.length,
- });
}
logger.info("Finished backfilling all teams", {