Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v0): fix jobs
Gergő Móricz0b50349fed21
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 26ee0ee9c..aefcda134 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -10,16 +10,19 @@ import { configDotenv } from "dotenv";import { Job } from "bullmq";import { toLegacyDocument } from "../v1/types";import type { DBJob, PseudoJob } from "../v1/crawl-status";+import { getJobFromGCS } from "../../lib/gcs-jobs";configDotenv();export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {- const [bullJobs, dbJobs] = await Promise.all([+ const [bullJobs, dbJobs, gcsJobs] = await Promise.all([Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],+ process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);const bullJobMap = new Map<string, PseudoJob<any>>();const dbJobMap = new Map<string, DBJob>();+ const gcsJobMap = new Map<string, any>();for (const job of bullJobs) {bullJobMap.set(job.id, job);@@ -28,16 +31,26 @@ export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJobfor (const job of dbJobs) {dbJobMap.set(job.job_id, job);}++ for (const job of gcsJobs) {+ gcsJobMap.set(job.id, job.job);+ }const jobs: PseudoJob<any>[] = [];for (const id of ids) {const bullJob = bullJobMap.get(id);const dbJob = dbJobMap.get(id);+ const gcsJob = gcsJobMap.get(id);if (!bullJob && !dbJob) continue;- const data = dbJob?.docs ?? bullJob?.returnvalue;+ const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ if (gcsJob === null && data) {+ logger.warn("GCS Job not found", {+ jobId: id,+ });+ }const job: PseudoJob<any> = {id,