Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

more logging + more separate connections

Gergő Móricz6ba417f79214
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 9b7e7ac4f..cc4d6bf2b 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -10,7 +10,7 @@ import {
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
-import { getScrapeQueue } from "../../services/queue-service";
+import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -97,7 +97,7 @@ export async function scrapeController(
let doc: Document;
try {
- doc = await waitForJob(jobId, timeout + totalWait);
+ doc = await waitForJob(jobId, timeout + totalWait, logger);
} catch (e) {
logger.error(`Error in scrapeController`, {
startTime,
@@ -124,7 +124,13 @@ export async function scrapeController(
}
}
- await getScrapeQueue().remove(jobId);
+ logger.info("Done with waitForJob");
+
+ const conn = createRedisConnection();
+ await getScrapeQueue(conn).remove(jobId);
+ conn.disconnect();
+
+ logger.info("Removed job from queue");
if (!req.body.formats.includes("rawHtml")) {
if (doc && doc.rawHtml) {
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index d2e00074e..a6302ff5f 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -11,13 +11,14 @@ import {
pushConcurrencyLimitedJob,
pushCrawlConcurrencyLimitActiveJob,
} from "../lib/concurrency-limit";
-import { logger } from "../lib/logger";
+import { logger as _logger } from "../lib/logger";
import { sendNotificationWithCustomDays } from './notification/email_notification';
import { shouldSendConcurrencyLimitNotification } from './notification/notification-check';
import { getACUC, getACUCTeam } from "../controllers/auth";
import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs";
import { Document } from "../controllers/v1/types";
import { getCrawl } from "../lib/crawl-redis";
+import { Logger } from "winston";
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -130,7 +131,7 @@ async function addScrapeJobRaw(
const shouldSendNotification = await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id);
if (shouldSendNotification) {
sendNotificationWithCustomDays(webScraperOptions.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false, true).catch((error) => {
- logger.error("Error sending notification (concurrency limit reached)", { error });
+ _logger.error("Error sending notification (concurrency limit reached)", { error });
});
}
}
@@ -312,7 +313,7 @@ export async function addScrapeJobs(
const shouldSendNotification = await shouldSendConcurrencyLimitNotification(jobs[0].data.team_id);
if (shouldSendNotification) {
sendNotificationWithCustomDays(jobs[0].data.team_id, NotificationType.CONCURRENCY_LIMIT_REACHED, 15, false, true).catch((error) => {
- logger.error("Error sending notification (concurrency limit reached)", { error });
+ _logger.error("Error sending notification (concurrency limit reached)", { error });
});
}
}
@@ -389,6 +390,7 @@ export async function addScrapeJobs(
export function waitForJob(
jobId: string,
timeout: number,
+ logger: Logger = _logger,
): Promise<Document> {
return new Promise(async (resolve, reject) => {
const start = Date.now();
@@ -405,10 +407,12 @@ export function waitForJob(
if (state === "completed") {
let doc: Document;
const job = (await queue.getJob(jobId))!;
+ logger.debug("Got job");
doc = job.returnvalue;
if (!doc) {
const docs = await getJobFromGCS(jobId);
+ logger.debug("Got job from GCS");
if (!docs || docs.length === 0) {
throw new Error("Job not found in GCS");
}