Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(fix/search) Search logs fix (#1491)
Nicolase532a96b0c1d
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 1c902df49..9ac0104cc 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -95,9 +95,10 @@ async function scrapeSearchResult(mode: "single_urls" as Mode,team_id: options.teamId,scrapeOptions: options.scrapeOptions,- internalOptions: { teamId: options.teamId },+ internalOptions: { teamId: options.teamId, useCache: true },origin: options.origin,is_scrape: true,+},{},jobId,@@ -157,6 +158,13 @@ export async function searchController(method: "searchController",});+ let responseData: SearchResponse = {+ success: true,+ data: [],+ };+ const startTime = new Date().getTime();+ const costTracking = new CostTracking();+try {req.body = searchRequestSchema.parse(req.body);@@ -165,8 +173,6 @@ export async function searchController(origin: req.body.origin,});- const startTime = new Date().getTime();-let limit = req.body.limit;// Buffer results by 50% to account for filtered URLs@@ -196,90 +202,69 @@ export async function searchController(if (searchResults.length === 0) {logger.info("No search results found");- return res.status(200).json({- success: true,- data: [],- warning: "No search results found",- });- }-- if (+ responseData.warning = "No search results found";+ } else if (!req.body.scrapeOptions.formats ||req.body.scrapeOptions.formats.length === 0) {- billTeam(req.auth.team_id, req.acuc?.sub_id, searchResults.length).catch(- (error) => {- logger.error(- `Failed to bill team ${req.auth.team_id} for ${searchResults.length} credits: ${error}`,- );- },+ responseData.data = searchResults.map((r) => ({+ url: r.url,+ title: r.title,+ description: r.description,+ })) as Document[];+ } else {+ logger.info("Scraping search results");+ const scrapePromises = searchResults.map((result) =>+ scrapeSearchResult(result, {+ teamId: req.auth.team_id,+ origin: req.body.origin,+ timeout: req.body.timeout,+ scrapeOptions: req.body.scrapeOptions,+ }, logger, costTracking),);- return res.status(200).json({- success: true,- data: searchResults.map((r) => ({- url: r.url,- title: r.title,- description: r.description,- })) as Document[],++ const docs = await Promise.all(scrapePromises);+ logger.info("Scraping completed", {+ num_docs: docs.length,});- }- const costTracking = new CostTracking();+ const filteredDocs = docs.filter(+ (doc) =>+ doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),+ );- // Scrape each non-blocked result, handling timeouts individually- logger.info("Scraping search results");- const scrapePromises = searchResults.map((result) =>- scrapeSearchResult(result, {- teamId: req.auth.team_id,- origin: req.body.origin,- timeout: req.body.timeout,- scrapeOptions: req.body.scrapeOptions,- }, logger, costTracking),- );+ logger.info("Filtering completed", {+ num_docs: filteredDocs.length,+ });- const docs = await Promise.all(scrapePromises);- logger.info("Scraping completed", {- num_docs: docs.length,- });+ if (filteredDocs.length === 0) {+ responseData.data = docs;+ responseData.warning = "No content found in search results";+ } else {+ responseData.data = filteredDocs;+ }+ }- // Bill for successful scrapes only- billTeam(req.auth.team_id, req.acuc?.sub_id, docs.length).catch((error) => {+ // Bill team once for all successful results+ billTeam(req.auth.team_id, req.acuc?.sub_id, responseData.data.length).catch((error) => {logger.error(- `Failed to bill team ${req.auth.team_id} for ${docs.length} credits: ${error}`,+ `Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,);});- // Filter out empty content but keep docs with SERP results- const filteredDocs = docs.filter(- (doc) =>- doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),- );-- logger.info("Filtering completed", {- num_docs: filteredDocs.length,- });-- if (filteredDocs.length === 0) {- return res.status(200).json({- success: true,- data: docs,- warning: "No content found in search results",- });- }-const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - startTime) / 1000;logger.info("Logging job", {- num_docs: filteredDocs.length,+ num_docs: responseData.data.length,time_taken: timeTakenInSeconds,});logJob({job_id: jobId,success: true,- num_docs: filteredDocs.length,- docs: filteredDocs,+ num_docs: responseData.data.length,+ docs: responseData.data,time_taken: timeTakenInSeconds,team_id: req.auth.team_id,mode: "search",@@ -288,10 +273,8 @@ export async function searchController(cost_tracking: costTracking,});- return res.status(200).json({- success: true,- data: filteredDocs,- });+ return res.status(200).json(responseData);+} catch (error) {if (error instanceof Error &&