Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
more logging + more separate connections
Gergő Móricz6ba417f79214
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 9b7e7ac4f..cc4d6bf2b 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -10,7 +10,7 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";-import { getScrapeQueue } from "../../services/queue-service";+import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -97,7 +97,7 @@ export async function scrapeController(let doc: Document;try {- doc = await waitForJob(jobId, timeout + totalWait);+ doc = await waitForJob(jobId, timeout + totalWait, logger);} catch (e) {logger.error(`Error in scrapeController`, {startTime,@@ -124,7 +124,13 @@ export async function scrapeController(}}- await getScrapeQueue().remove(jobId);+ logger.info("Done with waitForJob");++ const conn = createRedisConnection();+ await getScrapeQueue(conn).remove(jobId);+ conn.disconnect();++ logger.info("Removed job from queue");if (!req.body.formats.includes("rawHtml")) {if (doc && doc.rawHtml) {diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex d2e00074e..a6302ff5f 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -11,13 +11,14 @@ import {pushConcurrencyLimitedJob,pushCrawlConcurrencyLimitActiveJob,} from "../lib/concurrency-limit";-import { logger } from "../lib/logger";+import { logger as _logger } from "../lib/logger";import { sendNotificationWithCustomDays } from './notification/email_notification';import { shouldSendConcurrencyLimitNotification } from './notification/notification-check';import { getACUC, getACUCTeam } from "../controllers/auth";import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";import { Document } from "../controllers/v1/types";import { getCrawl } from "../lib/crawl-redis";+import { Logger } from "winston";/*** Checks if a job is a crawl or batch scrape based on its options@@ -130,7 +131,7 @@ async function addScrapeJobRaw(const shouldSendNotification = await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id);if (shouldSendNotification) {sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false, true).catch((error) => {- logger.error("Error sending notification (concurrency limit reached)", { error });+ _logger.error("Error sending notification (concurrency limit reached)", { error });});}}@@ -312,7 +313,7 @@ export async function addScrapeJobs(const shouldSendNotification = await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id);if (shouldSendNotification) {sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false, true).catch((error) => {- logger.error("Error sending notification (concurrency limit reached)", { error });+ _logger.error("Error sending notification (concurrency limit reached)", { error });});}}@@ -389,6 +390,7 @@ export async function addScrapeJobs(export function waitForJob(jobId: string,timeout: number,+ logger: Logger = _logger,): Promise<Document> {return new Promise(async (resolve, reject) => {const start = Date.now();@@ -405,10 +407,12 @@ export function waitForJob(if (state === "completed") {let doc: Document;const job = (await queue.getJob(jobId))!;+ logger.debug("Got job");doc = job.returnvalue;if (!doc) {const docs = await getJobFromGCS(jobId);+ logger.debug("Got job from GCS");if (!docs || docs.length === 0) {throw new Error("Job not found in GCS");}