Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
logs (#1861)
Gergő Móriczc34cef2da82d
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 254bc53a9..89763af52 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -100,6 +100,7 @@ export async function scrapeController(} catch (e) {logger.error(`Error in scrapeController`, {startTime,+ error: e,});if (zeroDataRetention) {diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 19167fd99..81c5eee20 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -81,6 +81,8 @@ export async function runWebScraper({});const tries = is_crawl ? 3 : 1;+ logger.info("runWebScraper called");+let response: ScrapeUrlResponse | undefined = undefined;let engines: EngineResultsTracker = {};let error: any = undefined;@@ -100,6 +102,7 @@ export async function runWebScraper({error = undefined;try {+ logger.info("running scrapeURL...");response = await scrapeURL(bull_job_id, url, scrapeOptions, {priority,...internalOptions,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex f6e2d3749..d0e989098 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -503,6 +503,8 @@ export async function scrapeURL(): Promise<ScrapeUrlResponse> {const meta = await buildMetaObject(id, url, options, internalOptions, costTracking);+ meta.logger.info("scrapeURL entered");+if (meta.rewrittenUrl) {meta.logger.info("Rewriting URL");}@@ -532,6 +534,8 @@ export async function scrapeURL(}}+ meta.logger.info("Pre-recording frequency");+const shouldRecordFrequency = useIndex&& meta.options.storeInCache&& !meta.internalOptions.zeroDataRetention