Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
perf(admin): optimize concurrency-queue-backfill endpoint (#2931)
firecrawl-spring[bot]7cdc06652913
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.tsindex ec73effb3..5a67e6cba 100644--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts@@ -5,7 +5,7 @@ import { scrapeQueue } from "../../../services/worker/nuq";import {pushConcurrencyLimitedJobs,pushConcurrencyLimitActiveJob,- getConcurrencyLimitActiveJobs,+ getConcurrencyLimitActiveJobsCount,} from "../../../lib/concurrency-limit";import { RateLimiterMode } from "../../../types";import { getACUCTeam } from "../../auth";@@ -24,6 +24,14 @@ export async function concurrencyQueueBackfillController(? [req.query.teamId as string]: await scrapeQueue.getBackloggedOwnerIDs(logger);+ const teamResults: {+ teamId: string;+ backloggedJobs: number;+ alreadyQueued: number;+ queued: number;+ started: number;+ }[] = [];+for (const ownerId of backloggedOwnerIDs) {logger.info("Backfilling concurrency queue for team", { teamId: ownerId });@@ -39,7 +47,7 @@ export async function concurrencyQueueBackfillController(`concurrency-limit-queue:${ownerId}`,cursor,"COUNT",- 20,+ 1000,);cursor = result[0];const results = result[1];@@ -68,16 +76,15 @@ export async function concurrencyQueueBackfillController();// Get concurrency limits for both job types- const maxCrawlConcurrency =- (await getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl))- ?.concurrency ?? 2;- const maxExtractConcurrency =- (await getACUCTeam(ownerId, false, true, RateLimiterMode.Extract))- ?.concurrency ?? 2;+ const [crawlACUC, extractACUC] = await Promise.all([+ getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl),+ getACUCTeam(ownerId, false, true, RateLimiterMode.Extract),+ ]);+ const maxCrawlConcurrency = crawlACUC?.concurrency ?? 2;+ const maxExtractConcurrency = extractACUC?.concurrency ?? 2;- const currentActiveConcurrency = (- await getConcurrencyLimitActiveJobs(ownerId)- ).length;+ const currentActiveConcurrency =+ await getConcurrencyLimitActiveJobsCount(ownerId);const jobsToStart: typeof jobsToAdd = [];const jobsToQueue: typeof jobsToAdd = [];@@ -110,6 +117,8 @@ export async function concurrencyQueueBackfillController();}+ // Promote jobs that can start immediately+ // These involve DB transactions per job so they remain sequentialfor (const job of jobsToStart) {await scrapeQueue.promoteJobFromBacklogOrAdd(job.id, job.data, {priority: job.priority,@@ -121,17 +130,30 @@ export async function concurrencyQueueBackfillController(await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);}- logger.info("Started jobs for team", {+ logger.info("Finished backfilling concurrency queue for team", {teamId: ownerId,startedCount: jobsToStart.length,+ queuedCount: jobsToQueue.length,});- logger.info("Finished backfilling concurrency queue for team", {+ teamResults.push({teamId: ownerId,+ backloggedJobs: backloggedJobIDs.size,+ alreadyQueued: queuedJobIDs.size,+ queued: jobsToQueue.length,+ started: jobsToStart.length,});}- logger.info("Finished backfilling all teams");+ logger.info("Finished backfilling all teams", {+ teamsProcessed: teamResults.length,+ });- res.json({ ok: true });+ res.json({+ ok: true,+ teamsProcessed: teamResults.length,+ totalJobsQueued: teamResults.reduce((sum, t) => sum + t.queued, 0),+ totalJobsStarted: teamResults.reduce((sum, t) => sum + t.started, 0),+ teams: teamResults,+ });}diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex aa69e12c9..d2698db90 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -41,10 +41,12 @@ const browserCreateRequestSchema = z.object({ttl: z.number().min(30).max(3600).default(300),activityTtl: z.number().min(10).max(3600).default(120),streamWebView: z.boolean().default(true),- profile: z.object({- name: z.string().min(1).max(128),- saveChanges: z.boolean().default(true),- }).optional(),+ profile: z+ .object({+ name: z.string().min(1).max(128),+ saveChanges: z.boolean().default(true),+ })+ .optional(),});type BrowserCreateRequest = z.infer<typeof browserCreateRequestSchema>;@@ -285,7 +287,8 @@ export async function browserCreateController(});return res.status(409).json({success: false,- error: "Another session is currently writing to this profile. Only one writer is allowed at a time. You can still access it with saveChanges: false, or try again later.",+ error:+ "Another session is currently writing to this profile. Only one writer is allowed at a time. You can still access it with saveChanges: false, or try again later.",});}