Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

remove logs

Gergő Móricz4659155b762c
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 049a4b57f..5444c3cfa 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -81,9 +81,9 @@ export async function scrapeController(
let doc: Document;
try {
- doc = await waitForJob(jobId, timeout + totalWait, logger.child({ module: "queue-jobs", method: "waitForJob" }));
+ doc = await waitForJob(jobId, timeout + totalWait);
} catch (e) {
- logger.error(`Error in scrapeController: ${e}`, {
+ logger.error(`Error in scrapeController`, {
startTime,
});
@@ -129,12 +129,8 @@ export async function scrapeController(
}
}
- logger.info("Scrape completed");
-
await getScrapeQueue().remove(jobId);
- logger.info("Job removed from queue");
-
if (!req.body.formats.includes("rawHtml")) {
if (doc && doc.rawHtml) {
delete doc.rawHtml;
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 9b4390f85..01ef9d311 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -87,15 +87,8 @@ export async function getJobFromGCS(jobId: string): Promise<Document[] | null> {
const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
const blob = bucket.file(`${jobId}.json`);
- const time1 = Date.now();
const [content] = await blob.download();
- const time2 = Date.now();
const x = JSON.parse(content.toString());
- logger.debug("getJobFromGCS time insights", {
- downloadTook: time2 - time1,
- jobId,
- scrapeId: jobId,
- });
return x;
} catch (error) {
if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 48c807fbb..451d31f4a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -121,14 +121,10 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
selector = selector.is("location_languages", null);
}
- const time1 = Date.now();
-
const { data, error } = await selector
.order("created_at", { ascending: false })
.limit(5);
- const time2 = Date.now();
-
if (error) {
throw new EngineError("Failed to retrieve URL from DB index", {
cause: error,
@@ -162,13 +158,6 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
throw new EngineError("Document not found in GCS");
}
- const time3 = Date.now();
-
- meta.logger.debug("scrapeURLWithIndex time insights", {
- dbTook: time2 - time1,
- downloadTook: time3 - time2,
- });
-
return {
url: doc.url,
html: doc.html,
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index a91756842..6d8c062fa 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -71,13 +71,8 @@ export async function getIndexFromGCS(url: string, logger?: Logger): Promise<any
const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);
const blob = bucket.file(`${url}`);
- const time1 = Date.now();
const [blobContent] = await blob.download();
- const time2 = Date.now();
const parsed = JSON.parse(blobContent.toString());
- logger?.debug("getIndexFromGCS time insights", {
- downloadTook: time2 - time1,
- });
return parsed;
} catch (error) {
if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) {
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index f74f99cc9..e92545d07 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -349,7 +349,6 @@ export async function addScrapeJobs(
export function waitForJob(
jobId: string,
timeout: number,
- logger?: Logger,
): Promise<Document> {
return new Promise((resolve, reject) => {
const start = Date.now();
@@ -360,7 +359,6 @@ export function waitForJob(
} else {
const state = await getScrapeQueue().getJobState(jobId);
if (state === "completed") {
- logger?.info("Job completed, getting document");
clearInterval(int);
let doc: Document;
doc = (await getScrapeQueue().getJob(jobId))!.returnvalue;
@@ -373,11 +371,8 @@ export function waitForJob(
doc = docs[0];
}
- logger?.info("Document retrieved");
-
resolve(doc);
} else if (state === "failed") {
- // console.log("failed", (await getScrapeQueue().getJob(jobId)).failedReason);
const job = await getScrapeQueue().getJob(jobId);
if (job && job.failedReason !== "Concurrency limit hit") {
clearInterval(int);