Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update search.ts
Nicolas7a968738762b
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 223298cd3..c40b5e7da 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -381,14 +381,14 @@ export async function searchController(if (!shouldScrape) {// No scraping - just count results for billingcredits_billed = totalResultsCount;- } else if (isAsyncScraping) {- // Async scraping - start jobs and return IDs immediately- logger.info("Starting async search scraping");+ } else {+ // Common setup for both async and sync scraping+ logger.info(`Starting ${isAsyncScraping ? 'async' : 'sync'} search scraping`);// Prepare all scrape job inputsconst scrapeInputs = prepareScrapeJobInputs(searchResponse);- // Create all scrape job promises in parallel+ // Create common optionsconst scrapeOptions = {teamId: req.auth.team_id,origin: req.body.origin,@@ -398,238 +398,202 @@ export async function searchController(const directToBullMQ = (req.acuc?.price_credits ?? 0) <= 3000;- // Start all scrape jobs in parallel- const allScrapePromises = [+ // Create all promises based on mode (async vs sync)+ const allPromises = [...scrapeInputs.web.map(input =>- startScrapeJob(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)+ isAsyncScraping+ ? startScrapeJob(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)+ : scrapeSearchResult(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)),...scrapeInputs.news.map(input =>- startScrapeJob(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)+ isAsyncScraping+ ? startScrapeJob(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)+ : scrapeSearchResult(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)),...scrapeInputs.images.map(input =>- startScrapeJob(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)+ isAsyncScraping+ ? startScrapeJob(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)+ : scrapeSearchResult(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)),];- // Wait for all jobs to be queued- const allJobIds = await Promise.all(allScrapePromises);-- // Organize job IDs by type- const scrapeIds: { web?: string[]; news?: string[]; images?: string[] } = {};- let currentIndex = 0;-- if (scrapeInputs.web.length > 0) {- scrapeIds.web = allJobIds.slice(currentIndex, currentIndex + scrapeInputs.web.length);- currentIndex += scrapeInputs.web.length;- }-- if (scrapeInputs.news.length > 0) {- scrapeIds.news = allJobIds.slice(currentIndex, currentIndex + scrapeInputs.news.length);- currentIndex += scrapeInputs.news.length;- }-- if (scrapeInputs.images.length > 0) {- scrapeIds.images = allJobIds.slice(currentIndex, currentIndex + scrapeInputs.images.length);- }+ // Execute all operations in parallel+ const results = await Promise.all(allPromises);- // Calculate credits for billing- credits_billed = allJobIds.length;-- // Bill team once for all initiated scrapes- if (!isSearchPreview) {- billTeam(- req.auth.team_id,- req.acuc?.sub_id,- credits_billed,- ).catch((error) => {- logger.error(- `Failed to bill team ${req.auth.team_id} for ${credits_billed} credits: ${error}`,- );+ if (isAsyncScraping) {+ // Async mode: organize job IDs and return immediately+ const allJobIds = results as string[];+ const scrapeIds: { web?: string[]; news?: string[]; images?: string[] } = {};+ let currentIndex = 0;++ if (scrapeInputs.web.length > 0) {+ scrapeIds.web = allJobIds.slice(currentIndex, currentIndex + scrapeInputs.web.length);+ currentIndex += scrapeInputs.web.length;+ }++ if (scrapeInputs.news.length > 0) {+ scrapeIds.news = allJobIds.slice(currentIndex, currentIndex + scrapeInputs.news.length);+ currentIndex += scrapeInputs.news.length;+ }++ if (scrapeInputs.images.length > 0) {+ scrapeIds.images = allJobIds.slice(currentIndex, currentIndex + scrapeInputs.images.length);+ }++ credits_billed = allJobIds.length;++ // Bill team for async scraping+ if (!isSearchPreview) {+ billTeam(req.auth.team_id, req.acuc?.sub_id, credits_billed).catch((error) => {+ logger.error(`Failed to bill team ${req.auth.team_id} for ${credits_billed} credits: ${error}`);+ });+ }++ const endTime = new Date().getTime();+ const timeTakenInSeconds = (endTime - startTime) / 1000;++ logger.info("Logging job (async scraping)", {+ num_docs: credits_billed,+ time_taken: timeTakenInSeconds,+ scrapeIds,});- }-- const endTime = new Date().getTime();- const timeTakenInSeconds = (endTime - startTime) / 1000;- logger.info("Logging job (async scraping)", {- num_docs: credits_billed,- time_taken: timeTakenInSeconds,- scrapeIds,- });-- logJob(- {- job_id: jobId,- success: true,- num_docs: (searchResponse.web?.length ?? 0) + (searchResponse.images?.length ?? 0) + (searchResponse.news?.length ?? 0),- docs: [searchResponse],- time_taken: timeTakenInSeconds,- team_id: req.auth.team_id,- mode: "search",- url: req.body.query,- scrapeOptions: req.body.scrapeOptions,- crawlerOptions: {- ...req.body,- query: undefined,- scrapeOptions: undefined,+ logJob(+ {+ job_id: jobId,+ success: true,+ num_docs: (searchResponse.web?.length ?? 0) + (searchResponse.images?.length ?? 0) + (searchResponse.news?.length ?? 0),+ docs: [searchResponse],+ time_taken: timeTakenInSeconds,+ team_id: req.auth.team_id,+ mode: "search",+ url: req.body.query,+ scrapeOptions: req.body.scrapeOptions,+ crawlerOptions: {+ ...req.body,+ query: undefined,+ scrapeOptions: undefined,+ },+ origin: req.body.origin,+ integration: req.body.integration,+ credits_billed,+ zeroDataRetention: false,},- origin: req.body.origin,- integration: req.body.integration,- credits_billed,- zeroDataRetention: false, // not supported- },- false,- isSearchPreview,- );+ false,+ isSearchPreview,+ );- return res.status(200).json({- success: true,- data: searchResponse,- scrapeIds,- creditsUsed: credits_billed,- });- } else {- // Sync scraping - wait for results- logger.info("Starting sync search scraping");-- // Prepare all scrape job inputs- const scrapeInputs = prepareScrapeJobInputs(searchResponse);-- // Create scrape options- const scrapeOptions = {- teamId: req.auth.team_id,- origin: req.body.origin,- timeout: req.body.timeout,- scrapeOptions: req.body.scrapeOptions,- };-- const directToBullMQ = (req.acuc?.price_credits ?? 0) <= 3000;-- // Start all scrape jobs in parallel- const allScrapePromises = [- ...scrapeInputs.web.map(input =>- scrapeSearchResult(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)- ),- ...scrapeInputs.news.map(input =>- scrapeSearchResult(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)- ),- ...scrapeInputs.images.map(input =>- scrapeSearchResult(input, scrapeOptions, logger, req.acuc?.flags ?? null, directToBullMQ, isSearchPreview)- ),- ];-- // Wait for all scraping to complete- const allDocsWithCostTracking = await Promise.all(allScrapePromises);-- // Organize results by type and merge with search response- const scrapedResponse: SearchV2Response = {};- let currentIndex = 0;-- // Process web results- if (scrapeInputs.web.length > 0) {- const webDocs = allDocsWithCostTracking.slice(currentIndex, currentIndex + scrapeInputs.web.length);- scrapedResponse.web = searchResponse.web!.map((item, index) => {- const doc = webDocs[index].document;- return {- url: doc.url || item.url,- title: doc.title || item.title,- description: doc.description || item.description,- position: item.position,- category: item.category,- markdown: doc.markdown,- html: doc.html,- rawHtml: doc.rawHtml,- links: doc.links,- screenshot: doc.screenshot,- summary: doc.summary,- metadata: doc.metadata,- };+ return res.status(200).json({+ success: true,+ data: searchResponse,+ scrapeIds,+ creditsUsed: credits_billed,});- currentIndex += scrapeInputs.web.length;- }-- // Process news results- if (scrapeInputs.news.length > 0) {- const newsDocs = allDocsWithCostTracking.slice(currentIndex, currentIndex + scrapeInputs.news.length);- const newsDocsMap = new Map(- newsDocs.map((docWithCost, index) => [- scrapeInputs.news[index].url,- docWithCost.document- ])- );+ } else {+ // Sync mode: process scraped documents+ const allDocsWithCostTracking = results as DocumentWithCostTracking[];+ const scrapedResponse: SearchV2Response = {};+ let currentIndex = 0;- scrapedResponse.news = searchResponse.news!.map(item => {- const doc = item.url ? newsDocsMap.get(item.url) : undefined;- return {- ...item,- category: item.category,- markdown: doc?.markdown,- html: doc?.html,- rawHtml: doc?.rawHtml,- summary: doc?.summary,- metadata: doc?.metadata,- };- });- currentIndex += scrapeInputs.news.length;- }-- // Process image results- if (scrapeInputs.images.length > 0) {- const imageDocs = allDocsWithCostTracking.slice(currentIndex, currentIndex + scrapeInputs.images.length);- const imageDocsMap = new Map(- imageDocs.map((docWithCost, index) => [- scrapeInputs.images[index].url,- docWithCost.document- ])- );+ // Process web results+ if (scrapeInputs.web.length > 0) {+ const webDocs = allDocsWithCostTracking.slice(currentIndex, currentIndex + scrapeInputs.web.length);+ scrapedResponse.web = searchResponse.web!.map((item, index) => {+ const doc = webDocs[index].document;+ return {+ url: doc.url || item.url,+ title: doc.title || item.title,+ description: doc.description || item.description,+ position: item.position,+ category: item.category,+ markdown: doc.markdown,+ html: doc.html,+ rawHtml: doc.rawHtml,+ links: doc.links,+ screenshot: doc.screenshot,+ summary: doc.summary,+ metadata: doc.metadata,+ };+ });+ currentIndex += scrapeInputs.web.length;+ }++ // Process news results+ if (scrapeInputs.news.length > 0) {+ const newsDocs = allDocsWithCostTracking.slice(currentIndex, currentIndex + scrapeInputs.news.length);+ const newsDocsMap = new Map(+ newsDocs.map((docWithCost, index) => [+ scrapeInputs.news[index].url,+ docWithCost.document+ ])+ );++ scrapedResponse.news = searchResponse.news!.map(item => {+ const doc = item.url ? newsDocsMap.get(item.url) : undefined;+ return {+ ...item,+ category: item.category,+ markdown: doc?.markdown,+ html: doc?.html,+ rawHtml: doc?.rawHtml,+ summary: doc?.summary,+ metadata: doc?.metadata,+ };+ });+ currentIndex += scrapeInputs.news.length;+ }++ // Process image results+ if (scrapeInputs.images.length > 0) {+ const imageDocs = allDocsWithCostTracking.slice(currentIndex, currentIndex + scrapeInputs.images.length);+ const imageDocsMap = new Map(+ imageDocs.map((docWithCost, index) => [+ scrapeInputs.images[index].url,+ docWithCost.document+ ])+ );++ scrapedResponse.images = searchResponse.images!.map(item => {+ const doc = item.url ? imageDocsMap.get(item.url) : undefined;+ return {+ ...item,+ markdown: doc?.markdown,+ html: doc?.html,+ rawHtml: doc?.rawHtml,+ summary: doc?.summary,+ metadata: doc?.metadata,+ };+ });+ }- scrapedResponse.images = searchResponse.images!.map(item => {- const doc = item.url ? imageDocsMap.get(item.url) : undefined;- return {- ...item,- markdown: doc?.markdown,- html: doc?.html,- rawHtml: doc?.rawHtml,- summary: doc?.summary,- metadata: doc?.metadata,- };+ // Calculate credits+ const creditPromises = allDocsWithCostTracking.map(async (docWithCost) => {+ return await calculateCreditsToBeBilled(+ req.body.scrapeOptions,+ { teamId: req.auth.team_id, bypassBilling: true, zeroDataRetention: false },+ docWithCost.document,+ docWithCost.costTracking,+ req.acuc?.flags ?? null,+ );});++ try {+ const individualCredits = await Promise.all(creditPromises);+ credits_billed = individualCredits.reduce((sum, credit) => sum + credit, 0);+ } catch (error) {+ logger.error("Error calculating credits for billing", { error });+ credits_billed = totalResultsCount;+ }++ // Update response with scraped data+ Object.assign(searchResponse, scrapedResponse);}-- // Calculate credits- const creditPromises = allDocsWithCostTracking.map(async (docWithCost) => {- return await calculateCreditsToBeBilled(- req.body.scrapeOptions,- { teamId: req.auth.team_id, bypassBilling: true, zeroDataRetention: false },- docWithCost.document,- docWithCost.costTracking,- req.acuc?.flags ?? null,- );- });-- try {- const individualCredits = await Promise.all(creditPromises);- credits_billed = individualCredits.reduce((sum, credit) => sum + credit, 0);- } catch (error) {- logger.error("Error calculating credits for billing", { error });- // Use the totalResultsCount which already respects the limit- credits_billed = totalResultsCount;- }-- // Update response with scraped data- Object.assign(searchResponse, scrapedResponse);}- // Bill team once for all successful results (only for sync scraping, async already billed)- if (!isSearchPreview && !isAsyncScraping) {- billTeam(- req.auth.team_id,- req.acuc?.sub_id,- credits_billed,- ).catch((error) => {- logger.error(- `Failed to bill team ${req.auth.team_id} for ${credits_billed} credits: ${error}`,- );+ // Bill team once for all successful results (only for sync scraping, async already billed above)+ if (!isSearchPreview && shouldScrape && !isAsyncScraping) {+ billTeam(req.auth.team_id, req.acuc?.sub_id, credits_billed).catch((error) => {+ logger.error(`Failed to bill team ${req.auth.team_id} for ${credits_billed} credits: ${error}`);});}@@ -667,11 +631,14 @@ export async function searchController();// For sync scraping or no scraping, don't include scrapeIds- return res.status(200).json({+ // Return response based on mode+ const response: any = {success: true,data: searchResponse,creditsUsed: credits_billed,- });+ };++ return res.status(200).json(response);} catch (error) {if (error instanceof z.ZodError) {logger.warn("Invalid request body", { error: error.errors });