Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): errored job logging

Gergő Móricz2849ce2f1369
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index adefd143a..051217c6b 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -989,16 +989,30 @@ async function processJob(job: Job & { id: string }, token: string) {
job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",
);
}
- // if (job.data.v1) {
- // callWebhook(
- // job.data.team_id,
- // job.id as string,
- // [],
- // job.data.webhook,
- // job.data.v1,
- // "crawl.failed"
- // );
- // }
+
+ logger.debug("Logging job to DB...");
+ await logJob(
+ {
+ job_id: job.id as string,
+ success: false,
+ message:
+ typeof error === "string"
+ ? error
+ : (error.message ??
+ "Something went wrong... Contact [email protected]"),
+ num_docs: 0,
+ docs: [],
+ time_taken: 0,
+ team_id: job.data.team_id,
+ mode: job.data.mode,
+ url: job.data.url,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ origin: job.data.origin,
+ crawl_id: job.data.crawl_id,
+ },
+ true,
+ );
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
@@ -1010,30 +1024,6 @@ async function processJob(job: Job & { id: string }, token: string) {
normalizeURL(job.data.url, sc),
);
- logger.debug("Logging job to DB...");
- await logJob(
- {
- job_id: job.id as string,
- success: false,
- message:
- typeof error === "string"
- ? error
- : (error.message ??
- "Something went wrong... Contact [email protected]"),
- num_docs: 0,
- docs: [],
- time_taken: 0,
- team_id: job.data.team_id,
- mode: job.data.mode,
- url: job.data.url,
- crawlerOptions: sc.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- origin: job.data.origin,
- crawl_id: job.data.crawl_id,
- },
- true,
- );
-
await finishCrawlIfNeeded(job, sc);
// await logJob({