Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

debug(api/scrape): increased logging to diagnose scrape fluke length

Gergő Móricz0c7f864ea477
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index fb68f7ffb..d9707414e 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -1,5 +1,5 @@
import { Response } from "express";
-import { logger } from "../../lib/logger";
+import { logger as _logger } from "../../lib/logger";
import {
Document,
RequestWithAuth,
@@ -21,12 +21,18 @@ export async function scrapeController(
) {
const jobId = uuidv4();
const preNormalizedBody = { ...req.body };
+ const logger = _logger.child({
+ method: "scrapeController",
+ jobId,
+ scrapeId: jobId,
+ teamId: req.auth.team_id,
+ team_id: req.auth.team_id,
+ });
logger.debug("Scrape " + jobId + " starting", {
scrapeId: jobId,
request: req.body,
originalRequest: preNormalizedBody,
- teamId: req.auth.team_id,
account: req.account,
});
@@ -40,7 +46,6 @@ export async function scrapeController(
team_id: req.auth.team_id,
basePriority: 10,
});
- //
const isDirectToBullMQ = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
@@ -76,11 +81,9 @@ export async function scrapeController(
let doc: Document;
try {
- doc = await waitForJob(jobId, timeout + totalWait);
+ doc = await waitForJob(jobId, timeout + totalWait, logger.child({ module: "queue-jobs", method: "waitForJob" }));
} catch (e) {
logger.error(`Error in scrapeController: ${e}`, {
- jobId,
- scrapeId: jobId,
startTime,
});
@@ -91,9 +94,7 @@ export async function scrapeController(
// @Nick this is a hack pushed at 2AM pls help - mogery
const job = await supabaseGetJobById(jobId);
if (!job?.cost_tracking) {
- logger.warn("No cost tracking found for job", {
- jobId,
- });
+ logger.warn("No cost tracking found for job");
}
creditsToBeBilled = Math.ceil((job?.cost_tracking?.totalCost ?? 1) * 1800);
} else {
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 92047f1ea..3233e6c42 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -87,12 +87,21 @@ export async function getJobFromGCS(jobId: string): Promise<Document[] | null> {
const storage = new Storage({ credentials });
const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
const blob = bucket.file(`${jobId}.json`);
+ const time1 = Date.now();
const [exists] = await blob.exists();
+ const time2 = Date.now();
if (!exists) {
return null;
}
const [content] = await blob.download();
+ const time3 = Date.now();
const x = JSON.parse(content.toString());
+ logger.debug("getJobFromGCS time insights", {
+ existsTook: time2 - time1,
+ downloadTook: time3 - time2,
+ jobId,
+ scrapeId: jobId,
+ });
return x;
} catch (error) {
logger.error(`Error getting job from GCS`, {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 57edbf75c..a2306f289 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -464,15 +464,6 @@ export function buildFallbackList(meta: Meta): {
)
&& meta.options.proxy !== "stealth";
- meta.logger.warn("shouldUseIndex", {
- shouldUseIndex,
- formatsNoChangeTracking: !meta.options.formats.includes("changeTracking"),
- maxAge: meta.options.maxAge !== 0,
- headers: meta.options.headers === undefined || Object.keys(meta.options.headers).length === 0,
- actions: meta.options.actions === undefined || meta.options.actions.length === 0,
- proxy: meta.options.proxy !== "stealth",
- });
-
if (!shouldUseIndex) {
const indexIndex = _engines.indexOf("index");
if (indexIndex !== -1) {
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index c368d121a..f74f99cc9 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -19,6 +19,7 @@ import { shouldSendConcurrencyLimitNotification } from './notification/notificat
import { getACUC, getACUCTeam } from "../controllers/auth";
import { getJobFromGCS } from "../lib/gcs-jobs";
import { Document } from "../controllers/v1/types";
+import type { Logger } from "winston";
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -348,6 +349,7 @@ export async function addScrapeJobs(
export function waitForJob(
jobId: string,
timeout: number,
+ logger?: Logger,
): Promise<Document> {
return new Promise((resolve, reject) => {
const start = Date.now();
@@ -358,6 +360,7 @@ export function waitForJob(
} else {
const state = await getScrapeQueue().getJobState(jobId);
if (state === "completed") {
+ logger?.info("Job completed, getting document");
clearInterval(int);
let doc: Document;
doc = (await getScrapeQueue().getJob(jobId))!.returnvalue;
@@ -370,6 +373,8 @@ export function waitForJob(
doc = docs[0];
}
+ logger?.info("Document retrieved");
+
resolve(doc);
} else if (state === "failed") {
// console.log("failed", (await getScrapeQueue().getJob(jobId)).failedReason);