Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(crawl-status): refactor (#1971)

Gergő Móricz0dd3ac8f16f6
diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.ts
index ae2af644f..9220cf2d5 100644
--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts
@@ -103,6 +103,8 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
zeroDataRetention: scope === "Request-scoped" ? true : undefined,
}, identity);
+ await new Promise(resolve => setTimeout(resolve, 2500));
+
const postServerLogs = (await getServerLogs()).slice(preServerLogs.length);
const postWorkerLogs = (await getWorkerLogs()).slice(preWorkerLogs.length);
@@ -182,6 +184,8 @@ if (process.env.TEST_SUITE_SELF_HOSTED) {
zeroDataRetention: scope === "Request-scoped" ? true : undefined,
}, identity);
+ await new Promise(resolve => setTimeout(resolve, 2500));
+
const postServerLogs = (await getServerLogs()).slice(preServerLogs.length);
const postWorkerLogs = (await getWorkerLogs()).slice(preWorkerLogs.length);
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 8b8c84582..85a305f54 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -2,16 +2,16 @@ import { Response } from "express";
import {
CrawlStatusParams,
CrawlStatusResponse,
- ErrorResponse,
+ Document,
RequestWithAuth,
} from "./types";
import {
getCrawl,
getCrawlExpiry,
- getCrawlJobs,
- getDoneJobsOrdered,
getDoneJobsOrderedLength,
isCrawlKickoffFinished,
+ getCrawlQualifiedJobCount,
+ getDoneJobsOrderedUntil,
} from "../../lib/crawl-redis";
import { getScrapeQueue } from "../../services/queue-service";
import {
@@ -19,10 +19,9 @@ import {
supabaseGetJobsById,
} from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
-import type { Job, JobState, Queue } from "bullmq";
+import type { Job, JobState } from "bullmq";
import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
-import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";
import { getJobFromGCS } from "../../lib/gcs-jobs";
configDotenv();
@@ -141,305 +140,227 @@ export async function crawlStatusController(
typeof req.query.limit === "string"
? start + parseInt(req.query.limit, 10) - 1
: undefined;
-
+
const sc = await getCrawl(req.params.jobId);
- let status: Exclude<CrawlStatusResponse, ErrorResponse>["status"];
- let doneJobsLength: number;
- let doneJobsOrder: string[];
- let totalCount: number;
- let creditsUsed: number;
+ const djoCutoff = Date.now() - 250;
if (sc) {
if (sc.team_id !== req.auth.team_id) {
- return res.status(403).json({ success: false, error: "Forbidden" });
- }
-
- let jobIDs = await getCrawlJobs(req.params.jobId);
- let jobStatuses = await Promise.all(
- jobIDs.map(
- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
- ),
- );
-
- if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {
- for (let rangeStart = 0; ; rangeStart += 1000) {
- const rangeEnd = rangeStart + 1000;
-
- const currentJobs = await supabase_service
- .from("firecrawl_jobs")
- .select("success, job_id")
- .eq("crawl_id", req.params.jobId)
- .order("date_added", { ascending: true })
- .range(rangeStart, rangeEnd);
-
- const rangeLength = rangeEnd - rangeStart;
-
- if (currentJobs.error) {
- logger.error("Error getting current jobs", { error: currentJobs.error });
- continue;
- }
-
- const data = currentJobs.data ?? [];
-
- for (const job of data) {
- let index = jobStatuses.findIndex((x) => x[0] === job.job_id);
- if (index !== -1) {
- jobStatuses[index] = [job.job_id, job.success ? "completed" : "failed"];
- } else {
- jobStatuses.push([job.job_id, job.success ? "completed" : "failed"]);
- }
- }
-
- if (data.length < rangeLength) {
- break;
- }
- }
- }
-
- const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id));
- const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId));
-
- const validJobStatuses: [string, JobState | "unknown"][] = [];
- const validJobIDs: string[] = [];
-
- for (const [id, status] of jobStatuses) {
- if (throttledJobsSet.has(id)) {
- validJobStatuses.push([id, "prioritized"]);
- validJobIDs.push(id);
- } else if (status === "unknown" && activeJobsSet.has(id)) {
- validJobStatuses.push([id, "active"]);
- validJobIDs.push(id);
- } else if (
- status !== "failed" &&
- status !== "unknown"
- ) {
- validJobStatuses.push([id, status]);
- validJobIDs.push(id);
- }
- }
-
- status =
- sc.cancelled
- ? "cancelled"
- : validJobStatuses.every((x) => x[1] === "completed") &&
- (sc.crawlerOptions
- ? await isCrawlKickoffFinished(req.params.jobId)
- : true)
- ? "completed"
- : "scraping";
-
- // Use validJobIDs instead of jobIDs for further processing
- jobIDs = validJobIDs;
-
- doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);
- doneJobsOrder = await getDoneJobsOrdered(
- req.params.jobId,
- start,
- end ?? -1,
- );
-
- totalCount = jobIDs.length;
-
- if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {
- const x = await supabase_rr_service
- .from('firecrawl_jobs')
- .select('*', { count: 'exact', head: true })
- .eq("crawl_id", req.params.jobId)
- .eq("success", true)
-
- totalCount = x.count ?? 0;
- }
-
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- const creditsRpc = await supabase_rr_service
- .rpc("credits_billed_by_crawl_id_1", {
- i_crawl_id: req.params.jobId,
- }, { get: true });
-
- creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (
- sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
- ? 5
- : 1
- ));
- } else {
- creditsUsed = totalCount * (
- sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
- ? 5
- : 1
- )
+ return res.status(404).json({ success: false, error: "Job not found" });
}
} else if (process.env.USE_DB_AUTHENTICATION === "true") {
const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
.from("firecrawl_jobs")
.select("*")
.eq("job_id", req.params.jobId)
- .limit(1)
- .throwOnError();
+ .limit(1);
- if (crawlJobError) {
+ if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {
logger.error("Error getting crawl job", { error: crawlJobError });
- throw crawlJobError;
+ throw new Error("Error getting crawl job", { cause: crawlJobError });
}
const crawlJob = crawlJobs[0];
- if (crawlJob && crawlJob.team_id !== req.auth.team_id) {
- return res.status(403).json({ success: false, error: "Forbidden" });
+ if (crawlJob.team_id !== req.auth.team_id) {
+ return res.status(404).json({ success: false, error: "Job not found" });
}
const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
- if (
- crawlJob
- && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs
- ) {
+ if (new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs) {
return res.status(404).json({ success: false, error: "Job expired" });
}
+ } else {
+ // if SC is gone and no DB, that means the job is expired, or never existed to begin with
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
- const { data: scrapeJobCounts, error: scrapeJobError } = await supabase_rr_service
- .rpc("count_jobs_of_crawl_team", { i_crawl_id: req.params.jobId, i_team_id: req.auth.team_id }, { get: true });
+ let outputBulkA: {
+ status?: "completed" | "scraping" | "cancelled",
+ completed?: number,
+ total?: number,
+ creditsUsed?: number,
+ };
- if (scrapeJobError || !scrapeJobCounts || scrapeJobCounts.length === 0) {
- logger.error("Error getting scrape job count", { error: scrapeJobError });
- throw (scrapeJobError ?? new Error("Unknown error getting scrape job count"));
- }
+ if (sc) {
+ // Local mode (easier, low-pressure)
+ const kickoffFinished = await isCrawlKickoffFinished(req.params.jobId);
+ let total = await getCrawlQualifiedJobCount(req.params.jobId);
- const scrapeJobCount: number = scrapeJobCounts[0].count ?? 0;
+ let completed = await getDoneJobsOrderedLength(req.params.jobId, djoCutoff);
+ let creditsUsed = completed * (sc.scrapeOptions?.formats?.find(x => typeof x === "object" && x.type === "json") ? 5 : 1);
- if (!crawlJobs || crawlJobs.length === 0) {
- if (scrapeJobCount === 0) {
- return res.status(404).json({ success: false, error: "Job not found" });
- } else {
- status = "completed"; // fake completed to cut the losses
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const creditsRpc = await supabase_service
+ .rpc("credits_billed_by_crawl_id_1", {
+ i_crawl_id: req.params.jobId,
+ }, { get: true });
+
+ creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? creditsUsed;
+
+ if (total === 0 && completed === 0 && Date.now() - sc.createdAt > 1000 * 60) {
+ const x = await supabase_service
+ .rpc('crawl_status_job_count_1', {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ }, { get: true });
+
+ total = x.data?.[0]?.count ?? 0;
+ completed = total;
}
- } else {
- status = crawlJobs[0].success ? "completed" : "failed";
}
-
- doneJobsLength = scrapeJobCount!;
- doneJobsOrder = [];
-
- const step = 1000;
- let i = 0;
- while (true) {
- const rangeStart = start + (i * step);
- let rangeEnd = start + ((i + 1) * step);
- if (end !== undefined) {
- rangeEnd = Math.min(end, rangeEnd);
- }
- const currentJobs = await supabase_rr_service
- .from("firecrawl_jobs")
- .select("job_id")
- .eq("crawl_id", req.params.jobId)
- .eq("team_id", req.acuc.team_id)
- .order("date_added", { ascending: true })
- .range(rangeStart, rangeEnd)
- .throwOnError();
+ outputBulkA = {
+ status: sc.cancelled ? "cancelled" : (completed === total && kickoffFinished) ? "completed" : "scraping",
+ total,
+ completed,
+ creditsUsed,
+ };
+ } else { // DB must be specified at this point, otherwise control flow kills execution earlier
+ // DB mode (once job expires from Redis)
+ const { data: crawlJobCounts, error: crawlJobCountsError } = await supabase_service
+ .rpc('crawl_status_job_count_1', {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ }, { get: true });
+
+ if (crawlJobCountsError || !crawlJobCounts || crawlJobCounts.length === 0) {
+ logger.error("Error getting crawl job count", { error: crawlJobCountsError });
+ throw new Error("Error getting crawl job count", { cause: crawlJobCountsError });
+ }
- const rangeLength = rangeEnd - rangeStart;
+ const crawlJobCount: number = crawlJobCounts[0].count ?? 0;
- const data = currentJobs.data ?? [];
+ const creditsRpc = await supabase_service
+ .rpc("credits_billed_by_crawl_id_1", {
+ i_crawl_id: req.params.jobId,
+ }, { get: true });
- doneJobsOrder.push(...data.map(x => x.job_id));
+ const creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? crawlJobCount;
- if (data.length < rangeLength) {
- break;
+ outputBulkA = {
+ status: "completed", // Salvage expired job
+ total: crawlJobCount,
+ completed: crawlJobCount,
+ creditsUsed,
+ };
+ }
+
+ let outputBulkB: {
+ data: Document[],
+ next: string | undefined,
+ };
+
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ // new DB-based path
+ const { data, error } = await supabase_service.rpc("crawl_status_1", {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ i_start: start,
+ i_end: end ?? (start + 100),
+ }, { get: true });
+
+ if (error || !data) {
+ logger.error("Error getting crawl status from DB", { error });
+ throw new Error("Error getting crawl status from DB", { cause: error });
+ }
+
+ const scrapeIds = data?.map((x) => x.id) ?? [];
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
+ let bytes = 0;
+ const bytesLimit = 10485760; // 10 MiB in bytes
+
+ const scrapeBlobs = await Promise.all(scrapeIds.map(async (x) => [x, (await getJobFromGCS(x))?.[0]]));
+
+ for (const [id, scrape] of scrapeBlobs) {
+ if (scrape) {
+ scrapes.push(scrape);
+ bytes += JSON.stringify(scrape).length;
+ } else {
+ logger.warn(
+ "Job was considered done, but returnvalue is undefined!",
+ { jobId: id, returnvalue: scrape },
+ );
}
- if (rangeEnd === end) {
+ iteratedOver++;
+
+ if (bytes > bytesLimit) {
break;
}
+ }
- i++
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
}
- totalCount = scrapeJobCount ?? 0;
- creditsUsed = crawlJob?.credits_billed ?? totalCount;
+ outputBulkB = {
+ data: scrapes,
+ next: (outputBulkA.total ?? 0) > (start + iteratedOver)
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
} else {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
-
- let doneJobs: PseudoJob<any>[] = [];
+ // old BullMQ-based path
+ const doneJobs = await getDoneJobsOrderedUntil(req.params.jobId, djoCutoff, start, end !== undefined ? end - start : 100);
- if (end === undefined) {
- // determine 10 megabyte limit
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
let bytes = 0;
const bytesLimit = 10485760; // 10 MiB in bytes
- const factor = 100; // chunking for faster retrieval
-
- for (
- let i = 0;
- i < doneJobsOrder.length && bytes < bytesLimit;
- i += factor
- ) {
- // get current chunk and retrieve jobs
- const currentIDs = doneJobsOrder.slice(i, i + factor);
- const jobs = await getJobs(currentIDs);
-
- // iterate through jobs and add them one them one to the byte counter
- // both loops will break once we cross the byte counter
- for (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {
- const job = jobs[ii];
- const state = await job.getState();
-
- if (state === "failed" || state === "active") {
- // TODO: why is active here? race condition? shouldn't matter tho - MG
- continue;
- }
- if (job.returnvalue === undefined || job.returnvalue === null) {
+ for (const jobId of doneJobs) {
+ const job = await getScrapeQueue().getJob(jobId);
+ const state = await job?.getState();
+
+ if (state === "failed") {
+ // no iterated over, just ignore
+ continue;
+ } else {
+ if (job?.returnvalue) {
+ scrapes.push(job.returnvalue);
+ bytes += JSON.stringify(job.returnvalue).length;
+ } else {
logger.warn(
"Job was considered done, but returnvalue is undefined!",
- { jobId: job.id, state, returnvalue: job.returnvalue },
+ { scrapeId: jobId, crawlId: req.params.jobId, state, returnvalue: job?.returnvalue },
);
- continue;
}
- doneJobs.push(job);
- bytes += JSON.stringify(job.returnvalue ?? null).length;
+
+ iteratedOver++;
}
- }
- // if we ran over the bytes limit, remove the last document, except if it's the only document
- if (bytes > bytesLimit && doneJobs.length !== 1) {
- doneJobs.splice(doneJobs.length - 1, 1);
+ if (bytes > bytesLimit) {
+ break;
+ }
}
- } else {
- doneJobs = (
- await Promise.all(
- (await getJobs(doneJobsOrder)).map(async (x) =>
- (await x.getState()) === "failed" ? null : x,
- ),
- )
- ).filter((x) => x !== null) as PseudoJob<any>[];
- }
-
- const data = doneJobs.map((x) => x.returnvalue);
- const protocol = process.env.ENV === "local" ? req.protocol : "https";
- const nextURL = new URL(
- `${protocol}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}`,
- );
-
- nextURL.searchParams.set("skip", (start + data.length).toString());
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
+ }
- if (typeof req.query.limit === "string") {
- nextURL.searchParams.set("limit", req.query.limit);
+ outputBulkB = {
+ data: scrapes,
+ next: (outputBulkA.total ?? 0) > (start + iteratedOver)
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
}
- res.status(200).json({
+ return res.status(200).json({
success: true,
- status,
- completed: doneJobsLength,
- total: totalCount,
- creditsUsed,
+ status: outputBulkA.status ?? "scraping",
+ completed: outputBulkA.completed ?? 0,
+ total: outputBulkA.total ?? 0,
+ creditsUsed: outputBulkA.creditsUsed ?? 0,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
- next:
- status !== "scraping" && start + data.length === doneJobsLength // if there's not gonna be any documents after this
- ? undefined
- : nextURL.href,
- data: data,
+ next: outputBulkB.next,
+ data: outputBulkB.data,
});
}
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index d52408112..34e3fcbd3 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -2,15 +2,15 @@ import { Response } from "express";
import {
CrawlStatusParams,
CrawlStatusResponse,
- ErrorResponse,
RequestWithAuth,
+ Document,
} from "./types";
import {
getCrawl,
getCrawlExpiry,
- getCrawlJobs,
- getDoneJobsOrdered,
+ getCrawlQualifiedJobCount,
getDoneJobsOrderedLength,
+ getDoneJobsOrderedUntil,
isCrawlKickoffFinished,
} from "../../lib/crawl-redis";
import { getScrapeQueue } from "../../services/queue-service";
@@ -22,7 +22,6 @@ import { configDotenv } from "dotenv";
import type { Job, JobState, Queue } from "bullmq";
import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
-import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";
import { getJobFromGCS } from "../../lib/gcs-jobs";
configDotenv();
@@ -141,309 +140,227 @@ export async function crawlStatusController(
typeof req.query.limit === "string"
? start + parseInt(req.query.limit, 10) - 1
: undefined;
-
+
const sc = await getCrawl(req.params.jobId);
- let status: Exclude<CrawlStatusResponse, ErrorResponse>["status"];
- let doneJobsLength: number;
- let doneJobsOrder: string[];
- let totalCount: number;
- let creditsUsed: number;
+ const djoCutoff = Date.now() - 250;
if (sc) {
if (sc.team_id !== req.auth.team_id) {
- return res.status(403).json({ success: false, error: "Forbidden" });
+ return res.status(404).json({ success: false, error: "Job not found" });
}
+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("job_id", req.params.jobId)
+ .limit(1);
- let jobIDs = await getCrawlJobs(req.params.jobId);
- let jobStatuses = await Promise.all(
- jobIDs.map(
- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
- ),
- );
-
- if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {
- for (let rangeStart = 0; ; rangeStart += 1000) {
- const rangeEnd = rangeStart + 1000;
-
- const currentJobs = await supabase_service
- .from("firecrawl_jobs")
- .select("success, job_id")
- .eq("crawl_id", req.params.jobId)
- .order("date_added", { ascending: true })
- .range(rangeStart, rangeEnd);
-
- const rangeLength = rangeEnd - rangeStart;
-
- if (currentJobs.error) {
- logger.error("Error getting current jobs", { error: currentJobs.error });
- continue;
- }
-
- const data = currentJobs.data ?? [];
+ if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {
+ logger.error("Error getting crawl job", { error: crawlJobError });
+ throw new Error("Error getting crawl job", { cause: crawlJobError });
+ }
- for (const job of data) {
- let index = jobStatuses.findIndex((x) => x[0] === job.job_id);
- if (index !== -1) {
- jobStatuses[index] = [job.job_id, job.success ? "completed" : "failed"];
- } else {
- jobStatuses.push([job.job_id, job.success ? "completed" : "failed"]);
- }
- }
+ const crawlJob = crawlJobs[0];
- if (data.length < rangeLength) {
- break;
- }
- }
+ if (crawlJob.team_id !== req.auth.team_id) {
+ return res.status(404).json({ success: false, error: "Job not found" });
}
- const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id));
- const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId));
-
- const validJobStatuses: [string, JobState | "unknown"][] = [];
- const validJobIDs: string[] = [];
-
- for (const [id, status] of jobStatuses) {
- if (throttledJobsSet.has(id)) {
- validJobStatuses.push([id, "prioritized"]);
- validJobIDs.push(id);
- } else if (status === "unknown" && activeJobsSet.has(id)) {
- validJobStatuses.push([id, "active"]);
- validJobIDs.push(id);
- } else if (
- status !== "failed" &&
- status !== "unknown"
- ) {
- validJobStatuses.push([id, status]);
- validJobIDs.push(id);
- }
- }
+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
- status =
- sc.cancelled
- ? "cancelled"
- : validJobStatuses.every((x) => x[1] === "completed") &&
- (sc.crawlerOptions
- ? await isCrawlKickoffFinished(req.params.jobId)
- : true)
- ? "completed"
- : "scraping";
-
- // Use validJobIDs instead of jobIDs for further processing
- jobIDs = validJobIDs;
-
- doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);
- doneJobsOrder = await getDoneJobsOrdered(
- req.params.jobId,
- start,
- end ?? -1,
- );
-
- totalCount = jobIDs.length;
-
- if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {
- const x = await supabase_rr_service
- .from('firecrawl_jobs')
- .select('*', { count: 'exact', head: true })
- .eq("crawl_id", req.params.jobId)
- .eq("success", true)
-
- totalCount = x.count ?? 0;
+ if (new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs) {
+ return res.status(404).json({ success: false, error: "Job expired" });
}
+ } else {
+ // if SC is gone and no DB, that means the job is expired, or never existed to begin with
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
+
+ let outputBulkA: {
+ status?: "completed" | "scraping" | "cancelled",
+ completed?: number,
+ total?: number,
+ creditsUsed?: number,
+ };
+
+ if (sc) {
+ // Local mode (easier, low-pressure)
+ const kickoffFinished = await isCrawlKickoffFinished(req.params.jobId);
+ let total = await getCrawlQualifiedJobCount(req.params.jobId);
+
+ let completed = await getDoneJobsOrderedLength(req.params.jobId, djoCutoff);
+ let creditsUsed = completed * (sc.scrapeOptions?.formats?.find(x => typeof x === "object" && x.type === "json") ? 5 : 1);
if (process.env.USE_DB_AUTHENTICATION === "true") {
- const creditsRpc = await supabase_rr_service
+ const creditsRpc = await supabase_service
.rpc("credits_billed_by_crawl_id_1", {
i_crawl_id: req.params.jobId,
- });
-
- creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (
- sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
- ? 5
- : 1
- ));
- } else {
- creditsUsed = totalCount * (
- sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
- ? 5
- : 1
- )
- }
- } else if (process.env.USE_DB_AUTHENTICATION === "true") {
- // TODO: move to read replica
- const { data: scrapeJobCounts, error: scrapeJobError } = await supabase_service
- .rpc("count_jobs_of_crawl_team", { i_crawl_id: req.params.jobId, i_team_id: req.auth.team_id });
+ }, { get: true });
- if (scrapeJobError || !scrapeJobCounts || scrapeJobCounts.length === 0) {
- logger.error("Error getting scrape job count", { error: scrapeJobError });
- throw scrapeJobError;
- }
-
- const scrapeJobCount: number = scrapeJobCounts[0].count ?? 0;
+ creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? creditsUsed;
- const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", req.params.jobId)
- .limit(1)
- .throwOnError();
+ if (total === 0 && completed === 0 && Date.now() - sc.createdAt > 1000 * 60) {
+ const x = await supabase_service
+ .rpc('crawl_status_job_count_1', {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ }, { get: true });
- if (crawlJobError) {
- logger.error("Error getting crawl job", { error: crawlJobError });
- throw crawlJobError;
- }
-
- if (!crawlJobs || crawlJobs.length === 0) {
- if (scrapeJobCount === 0) {
- return res.status(404).json({ success: false, error: "Job not found" });
- } else {
- status = "completed"; // fake completed to cut the losses
+ total = x.data?.[0]?.count ?? 0;
+ completed = total;
}
- } else {
- status = crawlJobs[0].success ? "completed" : "failed";
}
- const crawlJob = crawlJobs[0];
-
- if (crawlJob && crawlJob.team_id !== req.auth.team_id) {
- return res.status(403).json({ success: false, error: "Forbidden" });
+ outputBulkA = {
+ status: sc.cancelled ? "cancelled" : (completed === total && kickoffFinished) ? "completed" : "scraping",
+ total,
+ completed,
+ creditsUsed,
+ };
+ } else { // DB must be specified at this point, otherwise control flow kills execution earlier
+ // DB mode (once job expires from Redis)
+ const { data: crawlJobCounts, error: crawlJobCountsError } = await supabase_service
+ .rpc('crawl_status_job_count_1', {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ }, { get: true });
+
+ if (crawlJobCountsError || !crawlJobCounts || crawlJobCounts.length === 0) {
+ logger.error("Error getting crawl job count", { error: crawlJobCountsError });
+ throw new Error("Error getting crawl job count", { cause: crawlJobCountsError });
}
- const teamIdsExcludedFromExpiry = [
- "8f819703-1b85-4f7f-a6eb-e03841ec6617",
- "f96ad1a4-8102-4b35-9904-36fd517d3616",
- ];
+ const crawlJobCount: number = crawlJobCounts[0].count ?? 0;
- if (
- crawlJob
- && !teamIdsExcludedFromExpiry.includes(crawlJob.team_id)
- && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000
- ) {
- return res.status(404).json({ success: false, error: "Job expired" });
- }
+ const creditsRpc = await supabase_service
+ .rpc("credits_billed_by_crawl_id_1", {
+ i_crawl_id: req.params.jobId,
+ }, { get: true });
- doneJobsLength = scrapeJobCount!;
- doneJobsOrder = [];
+ const creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? crawlJobCount;
- const step = 1000;
- let i = 0;
- while (true) {
- const rangeStart = start + (i * step);
- let rangeEnd = start + ((i + 1) * step);
- if (end !== undefined) {
- rangeEnd = Math.min(end, rangeEnd);
- }
-
- const currentJobs = await supabase_rr_service
- .from("firecrawl_jobs")
- .select("job_id")
- .eq("crawl_id", req.params.jobId)
- .eq("team_id", req.acuc.team_id)
- .order("date_added", { ascending: true })
- .range(rangeStart, rangeEnd)
- .throwOnError();
+ outputBulkA = {
+ status: "completed", // Salvage expired job
+ total: crawlJobCount,
+ completed: crawlJobCount,
+ creditsUsed,
+ };
+ }
- const rangeLength = rangeEnd - rangeStart;
+ let outputBulkB: {
+ data: Document[],
+ next: string | undefined,
+ };
+
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ // new DB-based path
+ const { data, error } = await supabase_service.rpc("crawl_status_1", {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ i_start: start,
+ i_end: end ?? (start + 100),
+ }, { get: true });
+
+ if (error || !data) {
+ logger.error("Error getting crawl status from DB", { error });
+ throw new Error("Error getting crawl status from DB", { cause: error });
+ }
- const data = currentJobs.data ?? [];
+ const scrapeIds = data?.map((x) => x.id) ?? [];
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
+ let bytes = 0;
+ const bytesLimit = 10485760; // 10 MiB in bytes
- doneJobsOrder.push(...data.map(x => x.job_id));
+ const scrapeBlobs = await Promise.all(scrapeIds.map(async (x) => [x, (await getJobFromGCS(x))?.[0]]));
- if (data.length < rangeLength) {
- break;
+ for (const [id, scrape] of scrapeBlobs) {
+ if (scrape) {
+ scrapes.push(scrape);
+ bytes += JSON.stringify(scrape).length;
+ } else {
+ logger.warn(
+ "Job was considered done, but returnvalue is undefined!",
+ { jobId: id, returnvalue: scrape },
+ );
}
- if (rangeEnd === end) {
+ iteratedOver++;
+
+ if (bytes > bytesLimit) {
break;
}
+ }
- i++
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
}
- totalCount = scrapeJobCount ?? 0;
- creditsUsed = crawlJob?.credits_billed ?? totalCount;
+ outputBulkB = {
+ data: scrapes,
+ next: (outputBulkA.total ?? 0) > (start + iteratedOver)
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
} else {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
-
- let doneJobs: PseudoJob<any>[] = [];
+ // old BullMQ-based path
+ const doneJobs = await getDoneJobsOrderedUntil(req.params.jobId, djoCutoff, start, end !== undefined ? end - start : 100);
- if (end === undefined) {
- // determine 10 megabyte limit
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
let bytes = 0;
const bytesLimit = 10485760; // 10 MiB in bytes
- const factor = 100; // chunking for faster retrieval
-
- for (
- let i = 0;
- i < doneJobsOrder.length && bytes < bytesLimit;
- i += factor
- ) {
- // get current chunk and retrieve jobs
- const currentIDs = doneJobsOrder.slice(i, i + factor);
- const jobs = await getJobs(currentIDs);
-
- // iterate through jobs and add them one them one to the byte counter
- // both loops will break once we cross the byte counter
- for (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {
- const job = jobs[ii];
- const state = await job.getState();
-
- if (state === "failed" || state === "active") {
- // TODO: why is active here? race condition? shouldn't matter tho - MG
- continue;
- }
- if (job.returnvalue === undefined || job.returnvalue === null) {
+ for (const jobId of doneJobs) {
+ const job = await getScrapeQueue().getJob(jobId);
+ const state = await job?.getState();
+
+ if (state === "failed") {
+ // no iterated over, just ignore
+ continue;
+ } else {
+ if (job?.returnvalue) {
+ scrapes.push(job.returnvalue);
+ bytes += JSON.stringify(job.returnvalue).length;
+ } else {
logger.warn(
"Job was considered done, but returnvalue is undefined!",
- { jobId: job.id, state, returnvalue: job.returnvalue },
+ { scrapeId: jobId, crawlId: req.params.jobId, state, returnvalue: job?.returnvalue },
);
- continue;
}
- doneJobs.push(job);
- bytes += JSON.stringify(job.returnvalue ?? null).length;
+
+ iteratedOver++;
}
- }
- // if we ran over the bytes limit, remove the last document, except if it's the only document
- if (bytes > bytesLimit && doneJobs.length !== 1) {
- doneJobs.splice(doneJobs.length - 1, 1);
+ if (bytes > bytesLimit) {
+ break;
+ }
}
- } else {
- doneJobs = (
- await Promise.all(
- (await getJobs(doneJobsOrder)).map(async (x) =>
- (await x.getState()) === "failed" ? null : x,
- ),
- )
- ).filter((x) => x !== null) as PseudoJob<any>[];
- }
-
- const data = doneJobs.map((x) => x.returnvalue);
- const protocol = process.env.ENV === "local" ? req.protocol : "https";
- const nextURL = new URL(
- `${protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}`,
- );
-
- nextURL.searchParams.set("skip", (start + data.length).toString());
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
+ }
- if (typeof req.query.limit === "string") {
- nextURL.searchParams.set("limit", req.query.limit);
+ outputBulkB = {
+ data: scrapes,
+ next: (outputBulkA.total ?? 0) > (start + iteratedOver)
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
}
- res.status(200).json({
+ return res.status(200).json({
success: true,
- status,
- completed: doneJobsLength,
- total: totalCount,
- creditsUsed,
+ status: outputBulkA.status ?? "scraping",
+ completed: outputBulkA.completed ?? 0,
+ total: outputBulkA.total ?? 0,
+ creditsUsed: outputBulkA.creditsUsed ?? 0,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
- next:
- status !== "scraping" && start + data.length === doneJobsLength // if there's not gonna be any documents after this
- ? undefined
- : nextURL.href,
- data: data,
+ next: outputBulkB.next,
+ data: outputBulkB.data,
});
}
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 2647c488b..77254bed0 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -67,6 +67,8 @@ export async function addCrawlJob(id: string, job_id: string, __logger: Logger =
});
await redisEvictConnection.sadd("crawl:" + id + ":jobs", job_id);
await redisEvictConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
+ await redisEvictConnection.sadd("crawl:" + id + ":jobs_qualified", job_id);
+ await redisEvictConnection.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);
}
export async function addCrawlJobs(id: string, job_ids: string[], __logger: Logger = _logger) {
@@ -80,6 +82,8 @@ export async function addCrawlJobs(id: string, job_ids: string[], __logger: Logg
});
await redisEvictConnection.sadd("crawl:" + id + ":jobs", ...job_ids);
await redisEvictConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60);
+ await redisEvictConnection.sadd("crawl:" + id + ":jobs_qualified", ...job_ids);
+ await redisEvictConnection.expire("crawl:" + id + ":jobs_qualified", 24 * 60 * 60);
}
export async function addCrawlJobDone(
@@ -101,25 +105,24 @@ export async function addCrawlJobDone(
);
if (success) {
- await redisEvictConnection.rpush("crawl:" + id + ":jobs_done_ordered", job_id);
+ await redisEvictConnection.zadd("crawl:" + id + ":jobs_donez_ordered", Date.now(), job_id);
} else {
// in case it's already been pushed, make sure it's removed
- await redisEvictConnection.lrem(
- "crawl:" + id + ":jobs_done_ordered",
- -1,
+ await redisEvictConnection.zrem(
+ "crawl:" + id + ":jobs_donez_ordered",
job_id,
);
}
await redisEvictConnection.expire(
- "crawl:" + id + ":jobs_done_ordered",
+ "crawl:" + id + ":jobs_donez_ordered",
24 * 60 * 60,
);
}
-export async function getDoneJobsOrderedLength(id: string): Promise<number> {
- await redisEvictConnection.expire("crawl:" + id + ":jobs_done_ordered", 24 * 60 * 60);
- return await redisEvictConnection.llen("crawl:" + id + ":jobs_done_ordered");
+export async function getDoneJobsOrderedLength(id: string, until: number = Infinity): Promise<number> {
+ await redisEvictConnection.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);
+ return await redisEvictConnection.zcount("crawl:" + id + ":jobs_donez_ordered", -Infinity, until);
}
export async function getDoneJobsOrdered(
@@ -127,14 +130,23 @@ export async function getDoneJobsOrdered(
start = 0,
end = -1,
): Promise<string[]> {
- await redisEvictConnection.expire("crawl:" + id + ":jobs_done_ordered", 24 * 60 * 60);
- return await redisEvictConnection.lrange(
- "crawl:" + id + ":jobs_done_ordered",
+ await redisEvictConnection.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);
+ return await redisEvictConnection.zrange(
+ "crawl:" + id + ":jobs_donez_ordered",
start,
end,
);
}
+export async function getDoneJobsOrderedUntil(id: string, until: number = Infinity, start = 0, count = -1): Promise<string[]> {
+ await redisEvictConnection.expire("crawl:" + id + ":jobs_donez_ordered", 24 * 60 * 60);
+ return await redisEvictConnection.zrangebyscore(
+ "crawl:" + id + ":jobs_donez_ordered",
+ -Infinity,
+ until,
+ );
+}
+
export async function isCrawlFinished(id: string) {
await redisEvictConnection.expire("crawl:" + id + ":kickoff:finish", 24 * 60 * 60);
return (
@@ -214,6 +226,10 @@ export async function getCrawlJobCount(id: string): Promise<number> {
return await redisEvictConnection.scard("crawl:" + id + ":jobs");
}
+export async function getCrawlQualifiedJobCount(id: string): Promise<number> {
+ return await redisEvictConnection.scard("crawl:" + id + ":jobs_qualified");
+}
+
export function normalizeURL(url: string, sc: StoredCrawl): string {
const urlO = new URL(url);
if (sc && sc.crawlerOptions && sc.crawlerOptions.ignoreQueryParameters) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 13cfa3529..32a154186 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -509,11 +509,14 @@ app.listen(workerPort, () => {
});
(async () => {
- async function failedListener(args: {
- jobId: string;
- failedReason: string;
- prev?: string | undefined;
- }) {
+ async function failedListener(args: { jobId: string; failedReason: string; prev?: string | undefined; }) {
+ const job = await getScrapeQueue().getJob(args.jobId);
+
+ if (job && job.data.crawl_id) {
+ await redisEvictConnection.srem("crawl:" + job.data.crawl_id + ":jobs_qualified", args.jobId);
+ await redisEvictConnection.expire("crawl:" + job.data.crawl_id + ":jobs_qualified", 24 * 60 * 60);
+ }
+
if (args.failedReason === "job stalled more than allowable limit") {
const set = await redisEvictConnection.set(
"stalled-job-cleaner:" + args.jobId,
@@ -526,15 +529,7 @@ app.listen(workerPort, () => {
return;
}
- const job = await getScrapeQueue().getJob(args.jobId);
-
- let logger = _logger.child({
- jobId: args.jobId,
- scrapeId: args.jobId,
- module: "queue-worker",
- method: "failedListener",
- zeroDataRetention: job?.data.zeroDataRetention,
- });
+ let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener", zeroDataRetention: job?.data.zeroDataRetention });
if (job && job.data.crawl_id) {
logger = logger.child({ crawlId: job.data.crawl_id });
logger.warn("Job stalled more than allowable limit");
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 57e3f25ea..ae6299a82 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -473,6 +473,8 @@ async function processJob(job: Job & { id: string }) {
"crawl:" + job.data.crawl_id + ":visited_unique",
normalizeURL(job.data.url, sc),
);
+
+ await redisEvictConnection.srem("crawl:" + job.data.crawl_id + ":jobs_qualified", job.id);
await finishCrawlIfNeeded(job, sc);
}