Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): ignore concurrency limit for search (FIR-2187) (#1617)
Gergő Móricz98ceda9bd534
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 8a34c4386..fe104dc1d 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -76,6 +76,7 @@ async function scrapeSearchResult(logger: Logger,costTracking: CostTracking,flags: TeamFlags,+ directToBullMQ: boolean = false,): Promise<Document> {const jobId = uuidv4();const jobPriority = await getJobPriority({@@ -102,11 +103,11 @@ async function scrapeSearchResult(internalOptions: { teamId: options.teamId, useCache: true },origin: options.origin,is_scrape: true,-},{},jobId,jobPriority,+ directToBullMQ,);const doc: Document = await waitForJob(jobId, options.timeout);@@ -229,7 +230,7 @@ export async function searchController(origin: req.body.origin,timeout: req.body.timeout,scrapeOptions: req.body.scrapeOptions,- }, logger, costTracking, req.acuc?.flags ?? null),+ }, logger, costTracking, req.acuc?.flags ?? null, (req.acuc?.price_credits ?? 0) <= 3000),);const docs = await Promise.all(scrapePromises);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 3456c82cd..3ea7de221 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -97,6 +97,7 @@ async function addScrapeJobRaw(options: any,jobId: string,jobPriority: number,+ directToBullMQ: boolean = false,) {const hasCrawlDelay = webScraperOptions.crawl_id && webScraperOptions.crawlerOptions?.delay;@@ -127,7 +128,7 @@ async function addScrapeJobRaw(const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(webScraperOptions.team_id);- if (concurrencyLimited) {+ if (concurrencyLimited && !directToBullMQ) {// Detect if they hit their concurrent limit// If above by 2x, send them an email// No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2x@@ -161,6 +162,7 @@ export async function addScrapeJob(options: any = {},jobId: string = uuidv4(),jobPriority: number = 10,+ directToBullMQ: boolean = false,) {if (Sentry.isInitialized()) {const size = JSON.stringify(webScraperOptions).length;@@ -187,6 +189,7 @@ export async function addScrapeJob(options,jobId,jobPriority,+ directToBullMQ,);},);