Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

logs (#1861)

Gergő Móriczc34cef2da82d
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 254bc53a9..89763af52 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -100,6 +100,7 @@ export async function scrapeController(
} catch (e) {
logger.error(`Error in scrapeController`, {
startTime,
+ error: e,
});
if (zeroDataRetention) {
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 19167fd99..81c5eee20 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -81,6 +81,8 @@ export async function runWebScraper({
});
const tries = is_crawl ? 3 : 1;
+ logger.info("runWebScraper called");
+
let response: ScrapeUrlResponse | undefined = undefined;
let engines: EngineResultsTracker = {};
let error: any = undefined;
@@ -100,6 +102,7 @@ export async function runWebScraper({
error = undefined;
try {
+ logger.info("running scrapeURL...");
response = await scrapeURL(bull_job_id, url, scrapeOptions, {
priority,
...internalOptions,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index f6e2d3749..d0e989098 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -503,6 +503,8 @@ export async function scrapeURL(
): Promise<ScrapeUrlResponse> {
const meta = await buildMetaObject(id, url, options, internalOptions, costTracking);
+ meta.logger.info("scrapeURL entered");
+
if (meta.rewrittenUrl) {
meta.logger.info("Rewriting URL");
}
@@ -532,6 +534,8 @@ export async function scrapeURL(
}
}
+ meta.logger.info("Pre-recording frequency");
+
const shouldRecordFrequency = useIndex
&& meta.options.storeInCache
&& !meta.internalOptions.zeroDataRetention