Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
debug(api/scrape): increased logging to diagnose scrape fluke length
Gergő Móricz0c7f864ea477
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex fb68f7ffb..d9707414e 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -1,5 +1,5 @@import { Response } from "express";-import { logger } from "../../lib/logger";+import { logger as _logger } from "../../lib/logger";import {Document,RequestWithAuth,@@ -21,12 +21,18 @@ export async function scrapeController() {const jobId = uuidv4();const preNormalizedBody = { ...req.body };+ const logger = _logger.child({+ method: "scrapeController",+ jobId,+ scrapeId: jobId,+ teamId: req.auth.team_id,+ team_id: req.auth.team_id,+ });logger.debug("Scrape " + jobId + " starting", {scrapeId: jobId,request: req.body,originalRequest: preNormalizedBody,- teamId: req.auth.team_id,account: req.account,});@@ -40,7 +46,6 @@ export async function scrapeController(team_id: req.auth.team_id,basePriority: 10,});- //const isDirectToBullMQ = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;@@ -76,11 +81,9 @@ export async function scrapeController(let doc: Document;try {- doc = await waitForJob(jobId, timeout + totalWait);+ doc = await waitForJob(jobId, timeout + totalWait, logger.child({ module: "queue-jobs", method: "waitForJob" }));} catch (e) {logger.error(`Error in scrapeController: ${e}`, {- jobId,- scrapeId: jobId,startTime,});@@ -91,9 +94,7 @@ export async function scrapeController(// @Nick this is a hack pushed at 2AM pls help - mogeryconst job = await supabaseGetJobById(jobId);if (!job?.cost_tracking) {- logger.warn("No cost tracking found for job", {- jobId,- });+ logger.warn("No cost tracking found for job");}creditsToBeBilled = Math.ceil((job?.cost_tracking?.totalCost ?? 1) * 1800);} else {diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 92047f1ea..3233e6c42 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -87,12 +87,21 @@ export async function getJobFromGCS(jobId: string): Promise<Document[] | null> {const storage = new Storage({ credentials });const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);const blob = bucket.file(`${jobId}.json`);+ const time1 = Date.now();const [exists] = await blob.exists();+ const time2 = Date.now();if (!exists) {return null;}const [content] = await blob.download();+ const time3 = Date.now();const x = JSON.parse(content.toString());+ logger.debug("getJobFromGCS time insights", {+ existsTook: time2 - time1,+ downloadTook: time3 - time2,+ jobId,+ scrapeId: jobId,+ });return x;} catch (error) {logger.error(`Error getting job from GCS`, {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 57edbf75c..a2306f289 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -464,15 +464,6 @@ export function buildFallbackList(meta: Meta): {)&& meta.options.proxy !== "stealth";- meta.logger.warn("shouldUseIndex", {- shouldUseIndex,- formatsNoChangeTracking: !meta.options.formats.includes("changeTracking"),- maxAge: meta.options.maxAge !== 0,- headers: meta.options.headers === undefined || Object.keys(meta.options.headers).length === 0,- actions: meta.options.actions === undefined || meta.options.actions.length === 0,- proxy: meta.options.proxy !== "stealth",- });-if (!shouldUseIndex) {const indexIndex = _engines.indexOf("index");if (indexIndex !== -1) {diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex c368d121a..f74f99cc9 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -19,6 +19,7 @@ import { shouldSendConcurrencyLimitNotification } from './notification/notificatimport { getACUC, getACUCTeam } from "../controllers/auth";import { getJobFromGCS } from "../lib/gcs-jobs";import { Document } from "../controllers/v1/types";+import type { Logger } from "winston";/*** Checks if a job is a crawl or batch scrape based on its options@@ -348,6 +349,7 @@ export async function addScrapeJobs(export function waitForJob(jobId: string,timeout: number,+ logger?: Logger,): Promise<Document> {return new Promise((resolve, reject) => {const start = Date.now();@@ -358,6 +360,7 @@ export function waitForJob(} else {const state = await getScrapeQueue().getJobState(jobId);if (state === "completed") {+ logger?.info("Job completed, getting document");clearInterval(int);let doc: Document;doc = (await getScrapeQueue().getJob(jobId))!.returnvalue;@@ -370,6 +373,8 @@ export function waitForJob(doc = docs[0];}+ logger?.info("Document retrieved");+resolve(doc);} else if (state === "failed") {// console.log("failed", (await getScrapeQueue().getJob(jobId)).failedReason);