Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update runWebScraper.ts
Nicolas40eacfacd4c5
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex c67512180..d9ced895c 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -146,25 +146,25 @@ export async function runWebScraper({EngineResultsTracker[Engine],undefined>;- ScrapeEvents.insert(bull_job_id, {- type: "scrape",- url,- method: engine,- result: {- success: result.state === "success",- response_code:- result.state === "success" ? result.result.statusCode : undefined,- response_size:- result.state === "success" ? result.result.html.length : undefined,- error:- result.state === "error"- ? result.error- : result.state === "timeout"- ? "Timed out"- : undefined,- time_taken: result.finishedAt - result.startedAt,- },- });+ // ScrapeEvents.insert(bull_job_id, {+ // type: "scrape",+ // url,+ // method: engine,+ // result: {+ // success: result.state === "success",+ // response_code:+ // result.state === "success" ? result.result.statusCode : undefined,+ // response_size:+ // result.state === "success" ? result.result.html.length : undefined,+ // error:+ // result.state === "error"+ // ? result.error+ // : result.state === "timeout"+ // ? "Timed out"+ // : undefined,+ // time_taken: result.finishedAt - result.startedAt,+ // },+ // });}if (error === undefined && response?.success) {@@ -219,7 +219,7 @@ const saveJob = async (// // I think the job won't exist here anymore// }}- ScrapeEvents.logJobEvent(job, "completed");+ // ScrapeEvents.logJobEvent(job, "completed");} catch (error) {_logger.error(`🐂 Failed to update job status`, {module: "runWebScraper",