Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): errored job logging
Gergő Móricz2849ce2f1369
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex adefd143a..051217c6b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -989,16 +989,30 @@ async function processJob(job: Job & { id: string }, token: string) {job.data.crawlerOptions !== null ? "crawl.page" : "batch_scrape.page",);}- // if (job.data.v1) {- // callWebhook(- // job.data.team_id,- // job.id as string,- // [],- // job.data.webhook,- // job.data.v1,- // "crawl.failed"- // );- // }++ logger.debug("Logging job to DB...");+ await logJob(+ {+ job_id: job.id as string,+ success: false,+ message:+ typeof error === "string"+ ? error+ : (error.message ??+ "Something went wrong... Contact [email protected]"),+ num_docs: 0,+ docs: [],+ time_taken: 0,+ team_id: job.data.team_id,+ mode: job.data.mode,+ url: job.data.url,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ origin: job.data.origin,+ crawl_id: job.data.crawl_id,+ },+ true,+ );if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;@@ -1010,30 +1024,6 @@ async function processJob(job: Job & { id: string }, token: string) {normalizeURL(job.data.url, sc),);- logger.debug("Logging job to DB...");- await logJob(- {- job_id: job.id as string,- success: false,- message:- typeof error === "string"- ? error- : (error.message ??- "Something went wrong... Contact [email protected]"),- num_docs: 0,- docs: [],- time_taken: 0,- team_id: job.data.team_id,- mode: job.data.mode,- url: job.data.url,- crawlerOptions: sc.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- origin: job.data.origin,- crawl_id: job.data.crawl_id,- },- true,- );-await finishCrawlIfNeeded(job, sc);// await logJob({