Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): return v0 fdb crawl result blobs

mogery83e1e95e5acc
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 173675c2e..98e5c62a1 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -123,13 +123,23 @@ async function getFdbCrawlStatus(crawlId: string, sc: StoredCrawl) {
)
: [];
- const data = doneJobs
+ const scrapeBlobs = await Promise.all(
+ doneJobs.map(
+ async x =>
+ [
+ x,
+ x.returnvalue ??
+ (config.GCS_BUCKET_NAME ? await getJobFromGCS(x.id) : null),
+ ] as const,
+ ),
+ );
+
+ const data = scrapeBlobs
.filter(
- x => x.failedReason !== "Concurreny limit hit" && x.returnvalue !== null,
+ ([job, scrape]) =>
+ job.failedReason !== "Concurreny limit hit" && scrape != null,
)
- .map(x =>
- Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue,
- );
+ .map(([, scrape]) => (Array.isArray(scrape) ? scrape[0] : scrape));
const firstScrapeOptions =
doneJobs.length > 0 && "scrapeOptions" in doneJobs[0].data