Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
remove logs
Gergő Móricz4659155b762c
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 049a4b57f..5444c3cfa 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -81,9 +81,9 @@ export async function scrapeController(let doc: Document;try {- doc = await waitForJob(jobId, timeout + totalWait, logger.child({ module: "queue-jobs", method: "waitForJob" }));+ doc = await waitForJob(jobId, timeout + totalWait);} catch (e) {- logger.error(`Error in scrapeController: ${e}`, {+ logger.error(`Error in scrapeController`, {startTime,});@@ -129,12 +129,8 @@ export async function scrapeController(}}- logger.info("Scrape completed");-await getScrapeQueue().remove(jobId);- logger.info("Job removed from queue");-if (!req.body.formats.includes("rawHtml")) {if (doc && doc.rawHtml) {delete doc.rawHtml;diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 9b4390f85..01ef9d311 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -87,15 +87,8 @@ export async function getJobFromGCS(jobId: string): Promise<Document[] | null> {const storage = new Storage({ credentials });const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);const blob = bucket.file(`${jobId}.json`);- const time1 = Date.now();const [content] = await blob.download();- const time2 = Date.now();const x = JSON.parse(content.toString());- logger.debug("getJobFromGCS time insights", {- downloadTook: time2 - time1,- jobId,- scrapeId: jobId,- });return x;} catch (error) {if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) {diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 48c807fbb..451d31f4a 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -121,14 +121,10 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResultselector = selector.is("location_languages", null);}- const time1 = Date.now();-const { data, error } = await selector.order("created_at", { ascending: false }).limit(5);- const time2 = Date.now();-if (error) {throw new EngineError("Failed to retrieve URL from DB index", {cause: error,@@ -162,13 +158,6 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResultthrow new EngineError("Document not found in GCS");}- const time3 = Date.now();-- meta.logger.debug("scrapeURLWithIndex time insights", {- dbTook: time2 - time1,- downloadTook: time3 - time2,- });-return {url: doc.url,html: doc.html,diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex a91756842..6d8c062fa 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -71,13 +71,8 @@ export async function getIndexFromGCS(url: string, logger?: Logger): Promise<anyconst storage = new Storage({ credentials });const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);const blob = bucket.file(`${url}`);- const time1 = Date.now();const [blobContent] = await blob.download();- const time2 = Date.now();const parsed = JSON.parse(blobContent.toString());- logger?.debug("getIndexFromGCS time insights", {- downloadTook: time2 - time1,- });return parsed;} catch (error) {if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) {diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex f74f99cc9..e92545d07 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -349,7 +349,6 @@ export async function addScrapeJobs(export function waitForJob(jobId: string,timeout: number,- logger?: Logger,): Promise<Document> {return new Promise((resolve, reject) => {const start = Date.now();@@ -360,7 +359,6 @@ export function waitForJob(} else {const state = await getScrapeQueue().getJobState(jobId);if (state === "completed") {- logger?.info("Job completed, getting document");clearInterval(int);let doc: Document;doc = (await getScrapeQueue().getJob(jobId))!.returnvalue;@@ -373,11 +371,8 @@ export function waitForJob(doc = docs[0];}- logger?.info("Document retrieved");-resolve(doc);} else if (state === "failed") {- // console.log("failed", (await getScrapeQueue().getJob(jobId)).failedReason);const job = await getScrapeQueue().getJob(jobId);if (job && job.failedReason !== "Concurrency limit hit") {clearInterval(int);