Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): add database fallback to crawl errors endpoint (#1909)
Gergő Móricz83c0fdb97764
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex d8d029ee6..307efe4b0 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -12,6 +12,8 @@ import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { configDotenv } from "dotenv";import { Job } from "bullmq";+import { supabase_rr_service } from "../../services/supabase";+import { logger } from "../../lib/logger";configDotenv();export async function getJob(id: string) {@@ -34,40 +36,98 @@ export async function crawlErrorsController(res: Response<CrawlErrorsResponse>,) {const sc = await getCrawl(req.params.jobId);- if (!sc) {- return res.status(404).json({ success: false, error: "Job not found" });- }++ if (sc) {+ if (sc.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }- if (sc.team_id !== req.auth.team_id) {- return res.status(403).json({ success: false, error: "Forbidden" });- }+ let jobStatuses = await Promise.all(+ (await getCrawlJobs(req.params.jobId)).map(+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ ),+ );++ const failedJobIDs: string[] = [];- let jobStatuses = await Promise.all(- (await getCrawlJobs(req.params.jobId)).map(- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,- ),- );+ for (const [id, status] of jobStatuses) {+ if (status === "failed") {+ failedJobIDs.push(id);+ }+ }- const failedJobIDs: string[] = [];+ res.status(200).json({+ errors: (await getJobs(failedJobIDs)).map((x) => ({+ id: x.id,+ timestamp:+ x.finishedOn !== undefined+ ? new Date(x.finishedOn).toISOString()+ : undefined,+ url: x.data.url,+ error: x.failedReason,+ })),+ robotsBlocked: await redisEvictConnection.smembers(+ "crawl:" + req.params.jobId + ":robots_blocked",+ ),+ });+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {+ const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("job_id", req.params.jobId)+ .limit(1)+ .throwOnError();- for (const [id, status] of jobStatuses) {- if (status === "failed") {- failedJobIDs.push(id);+ if (crawlJobError) {+ logger.error("Error getting crawl job", { error: crawlJobError });+ throw crawlJobError;+ }++ const crawlJob = crawlJobs[0];++ if (crawlJob && crawlJob.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });}- }- res.status(200).json({- errors: (await getJobs(failedJobIDs)).map((x) => ({- id: x.id,- timestamp:- x.finishedOn !== undefined- ? new Date(x.finishedOn).toISOString()- : undefined,- url: x.data.url,- error: x.failedReason,- })),- robotsBlocked: await redisEvictConnection.smembers(- "crawl:" + req.params.jobId + ":robots_blocked",- ),- });+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;++ if (+ crawlJob+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs+ ) {+ return res.status(404).json({ success: false, error: "Job expired" });+ }++ if (!crawlJobs || crawlJobs.length === 0) {+ return res.status(404).json({ success: false, error: "Job not found" });+ }++ const { data: failedJobs, error: failedJobsError } = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("crawl_id", req.params.jobId)+ .eq("team_id", req.auth.team_id)+ .eq("success", false)+ .throwOnError();++ if (failedJobsError) {+ logger.error("Error getting failed jobs", { error: failedJobsError });+ throw failedJobsError;+ }++ res.status(200).json({+ errors: (failedJobs || []).map((job) => ({+ id: job.job_id,+ timestamp: new Date(job.date_added).toISOString(),+ url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL",+ error: job.message || "Unknown error",+ })),+ robotsBlocked: await redisEvictConnection.smembers(+ "crawl:" + req.params.jobId + ":robots_blocked",+ ),+ });+ } else {+ return res.status(404).json({ success: false, error: "Job not found" });+ }}