Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Update runWebScraper.ts"
Nicolas4c4d51e60bbd
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex d9ced895c..c67512180 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -146,25 +146,25 @@ export async function runWebScraper({EngineResultsTracker[Engine],undefined>;- // ScrapeEvents.insert(bull_job_id, {- // type: "scrape",- // url,- // method: engine,- // result: {- // success: result.state === "success",- // response_code:- // result.state === "success" ? result.result.statusCode : undefined,- // response_size:- // result.state === "success" ? result.result.html.length : undefined,- // error:- // result.state === "error"- // ? result.error- // : result.state === "timeout"- // ? "Timed out"- // : undefined,- // time_taken: result.finishedAt - result.startedAt,- // },- // });+ ScrapeEvents.insert(bull_job_id, {+ type: "scrape",+ url,+ method: engine,+ result: {+ success: result.state === "success",+ response_code:+ result.state === "success" ? result.result.statusCode : undefined,+ response_size:+ result.state === "success" ? result.result.html.length : undefined,+ error:+ result.state === "error"+ ? result.error+ : result.state === "timeout"+ ? "Timed out"+ : undefined,+ time_taken: result.finishedAt - result.startedAt,+ },+ });}if (error === undefined && response?.success) {@@ -219,7 +219,7 @@ const saveJob = async (// // I think the job won't exist here anymore// }}- // ScrapeEvents.logJobEvent(job, "completed");+ ScrapeEvents.logJobEvent(job, "completed");} catch (error) {_logger.error(`🐂 Failed to update job status`, {module: "runWebScraper",