Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): add database fallback to crawl errors endpoint (#1909)

Gergő Móricz83c0fdb97764
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index d8d029ee6..307efe4b0 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -12,6 +12,8 @@ import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { configDotenv } from "dotenv";
import { Job } from "bullmq";
+import { supabase_rr_service } from "../../services/supabase";
+import { logger } from "../../lib/logger";
configDotenv();
export async function getJob(id: string) {
@@ -34,40 +36,98 @@ export async function crawlErrorsController(
res: Response<CrawlErrorsResponse>,
) {
const sc = await getCrawl(req.params.jobId);
- if (!sc) {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
+
+ if (sc) {
+ if (sc.team_id !== req.auth.team_id) {
+ return res.status(403).json({ success: false, error: "Forbidden" });
+ }
- if (sc.team_id !== req.auth.team_id) {
- return res.status(403).json({ success: false, error: "Forbidden" });
- }
+ let jobStatuses = await Promise.all(
+ (await getCrawlJobs(req.params.jobId)).map(
+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
+ ),
+ );
+
+ const failedJobIDs: string[] = [];
- let jobStatuses = await Promise.all(
- (await getCrawlJobs(req.params.jobId)).map(
- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
- ),
- );
+ for (const [id, status] of jobStatuses) {
+ if (status === "failed") {
+ failedJobIDs.push(id);
+ }
+ }
- const failedJobIDs: string[] = [];
+ res.status(200).json({
+ errors: (await getJobs(failedJobIDs)).map((x) => ({
+ id: x.id,
+ timestamp:
+ x.finishedOn !== undefined
+ ? new Date(x.finishedOn).toISOString()
+ : undefined,
+ url: x.data.url,
+ error: x.failedReason,
+ })),
+ robotsBlocked: await redisEvictConnection.smembers(
+ "crawl:" + req.params.jobId + ":robots_blocked",
+ ),
+ });
+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("job_id", req.params.jobId)
+ .limit(1)
+ .throwOnError();
- for (const [id, status] of jobStatuses) {
- if (status === "failed") {
- failedJobIDs.push(id);
+ if (crawlJobError) {
+ logger.error("Error getting crawl job", { error: crawlJobError });
+ throw crawlJobError;
+ }
+
+ const crawlJob = crawlJobs[0];
+
+ if (crawlJob && crawlJob.team_id !== req.auth.team_id) {
+ return res.status(403).json({ success: false, error: "Forbidden" });
}
- }
- res.status(200).json({
- errors: (await getJobs(failedJobIDs)).map((x) => ({
- id: x.id,
- timestamp:
- x.finishedOn !== undefined
- ? new Date(x.finishedOn).toISOString()
- : undefined,
- url: x.data.url,
- error: x.failedReason,
- })),
- robotsBlocked: await redisEvictConnection.smembers(
- "crawl:" + req.params.jobId + ":robots_blocked",
- ),
- });
+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
+
+ if (
+ crawlJob
+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs
+ ) {
+ return res.status(404).json({ success: false, error: "Job expired" });
+ }
+
+ if (!crawlJobs || crawlJobs.length === 0) {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
+
+ const { data: failedJobs, error: failedJobsError } = await supabase_rr_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("crawl_id", req.params.jobId)
+ .eq("team_id", req.auth.team_id)
+ .eq("success", false)
+ .throwOnError();
+
+ if (failedJobsError) {
+ logger.error("Error getting failed jobs", { error: failedJobsError });
+ throw failedJobsError;
+ }
+
+ res.status(200).json({
+ errors: (failedJobs || []).map((job) => ({
+ id: job.job_id,
+ timestamp: new Date(job.date_added).toISOString(),
+ url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL",
+ error: job.message || "Unknown error",
+ })),
+ robotsBlocked: await redisEvictConnection.smembers(
+ "crawl:" + req.params.jobId + ":robots_blocked",
+ ),
+ });
+ } else {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
}