Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

perf(admin): optimize concurrency-queue-backfill endpoint (#2931)

firecrawl-spring[bot]7cdc06652913
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
index ec73effb3..5a67e6cba 100644
--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
@@ -5,7 +5,7 @@ import { scrapeQueue } from "../../../services/worker/nuq";
import {
pushConcurrencyLimitedJobs,
pushConcurrencyLimitActiveJob,
- getConcurrencyLimitActiveJobs,
+ getConcurrencyLimitActiveJobsCount,
} from "../../../lib/concurrency-limit";
import { RateLimiterMode } from "../../../types";
import { getACUCTeam } from "../../auth";
@@ -24,6 +24,14 @@ export async function concurrencyQueueBackfillController(
? [req.query.teamId as string]
: await scrapeQueue.getBackloggedOwnerIDs(logger);
+ const teamResults: {
+ teamId: string;
+ backloggedJobs: number;
+ alreadyQueued: number;
+ queued: number;
+ started: number;
+ }[] = [];
+
for (const ownerId of backloggedOwnerIDs) {
logger.info("Backfilling concurrency queue for team", { teamId: ownerId });
@@ -39,7 +47,7 @@ export async function concurrencyQueueBackfillController(
`concurrency-limit-queue:${ownerId}`,
cursor,
"COUNT",
- 20,
+ 1000,
);
cursor = result[0];
const results = result[1];
@@ -68,16 +76,15 @@ export async function concurrencyQueueBackfillController(
);
// Get concurrency limits for both job types
- const maxCrawlConcurrency =
- (await getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl))
- ?.concurrency ?? 2;
- const maxExtractConcurrency =
- (await getACUCTeam(ownerId, false, true, RateLimiterMode.Extract))
- ?.concurrency ?? 2;
+ const [crawlACUC, extractACUC] = await Promise.all([
+ getACUCTeam(ownerId, false, true, RateLimiterMode.Crawl),
+ getACUCTeam(ownerId, false, true, RateLimiterMode.Extract),
+ ]);
+ const maxCrawlConcurrency = crawlACUC?.concurrency ?? 2;
+ const maxExtractConcurrency = extractACUC?.concurrency ?? 2;
- const currentActiveConcurrency = (
- await getConcurrencyLimitActiveJobs(ownerId)
- ).length;
+ const currentActiveConcurrency =
+ await getConcurrencyLimitActiveJobsCount(ownerId);
const jobsToStart: typeof jobsToAdd = [];
const jobsToQueue: typeof jobsToAdd = [];
@@ -110,6 +117,8 @@ export async function concurrencyQueueBackfillController(
);
}
+ // Promote jobs that can start immediately
+ // These involve DB transactions per job so they remain sequential
for (const job of jobsToStart) {
await scrapeQueue.promoteJobFromBacklogOrAdd(job.id, job.data, {
priority: job.priority,
@@ -121,17 +130,30 @@ export async function concurrencyQueueBackfillController(
await pushConcurrencyLimitActiveJob(ownerId, job.id, 60 * 1000);
}
- logger.info("Started jobs for team", {
+ logger.info("Finished backfilling concurrency queue for team", {
teamId: ownerId,
startedCount: jobsToStart.length,
+ queuedCount: jobsToQueue.length,
});
- logger.info("Finished backfilling concurrency queue for team", {
+ teamResults.push({
teamId: ownerId,
+ backloggedJobs: backloggedJobIDs.size,
+ alreadyQueued: queuedJobIDs.size,
+ queued: jobsToQueue.length,
+ started: jobsToStart.length,
});
}
- logger.info("Finished backfilling all teams");
+ logger.info("Finished backfilling all teams", {
+ teamsProcessed: teamResults.length,
+ });
- res.json({ ok: true });
+ res.json({
+ ok: true,
+ teamsProcessed: teamResults.length,
+ totalJobsQueued: teamResults.reduce((sum, t) => sum + t.queued, 0),
+ totalJobsStarted: teamResults.reduce((sum, t) => sum + t.started, 0),
+ teams: teamResults,
+ });
}
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index aa69e12c9..d2698db90 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -41,10 +41,12 @@ const browserCreateRequestSchema = z.object({
ttl: z.number().min(30).max(3600).default(300),
activityTtl: z.number().min(10).max(3600).default(120),
streamWebView: z.boolean().default(true),
- profile: z.object({
- name: z.string().min(1).max(128),
- saveChanges: z.boolean().default(true),
- }).optional(),
+ profile: z
+ .object({
+ name: z.string().min(1).max(128),
+ saveChanges: z.boolean().default(true),
+ })
+ .optional(),
});
type BrowserCreateRequest = z.infer<typeof browserCreateRequestSchema>;
@@ -285,7 +287,8 @@ export async function browserCreateController(
});
return res.status(409).json({
success: false,
- error: "Another session is currently writing to this profile. Only one writer is allowed at a time. You can still access it with saveChanges: false, or try again later.",
+ error:
+ "Another session is currently writing to this profile. Only one writer is allowed at a time. You can still access it with saveChanges: false, or try again later.",
});
}