Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix crawl status

Gergő Móricz0078a8fde27d
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 8da033d48..673d9d315 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -328,65 +328,7 @@ export async function crawlStatusController(
next: string | undefined;
};
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- // new DB-based path
- const { data, error } = await supabase_service.rpc(
- "crawl_status_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- i_start: start,
- i_end: end ?? start + 100,
- },
- { get: true },
- );
-
- if (error || !data) {
- logger.error("Error getting crawl status from DB", { error });
- throw new Error("Error getting crawl status from DB", { cause: error });
- }
-
- const scrapeIds = data?.map(x => x.id) ?? [];
- let scrapes: Document[] = [];
- let iteratedOver = 0;
- let bytes = 0;
- const bytesLimit = 10485760; // 10 MiB in bytes
-
- const scrapeBlobs = await Promise.all(
- scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),
- );
-
- for (const [id, scrape] of scrapeBlobs) {
- if (scrape) {
- scrapes.push(scrape);
- bytes += JSON.stringify(scrape).length;
- } else {
- logger.warn("Job was considered done, but returnvalue is undefined!", {
- jobId: id,
- returnvalue: scrape,
- });
- }
-
- iteratedOver++;
-
- if (bytes > bytesLimit) {
- break;
- }
- }
-
- if (bytes > bytesLimit && scrapes.length !== 1) {
- scrapes.splice(scrapes.length - 1, 1);
- iteratedOver--;
- }
-
- outputBulkB = {
- data: scrapes,
- next:
- (outputBulkA.total ?? 0) > start + iteratedOver
- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
- : undefined,
- };
- } else {
+ if (sc || process.env.USE_DB_AUTHENTICATION !== "true") {
// old BullMQ-based path
const doneJobs = await getDoneJobsOrderedUntil(
req.params.jobId,
@@ -441,6 +383,64 @@ export async function crawlStatusController(
iteratedOver--;
}
+ outputBulkB = {
+ data: scrapes,
+ next:
+ (outputBulkA.total ?? 0) > start + iteratedOver
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
+ } else {
+ // new DB-based path
+ const { data, error } = await supabase_service.rpc(
+ "crawl_status_1",
+ {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ i_start: start,
+ i_end: end ?? start + 100,
+ },
+ { get: true },
+ );
+
+ if (error || !data) {
+ logger.error("Error getting crawl status from DB", { error });
+ throw new Error("Error getting crawl status from DB", { cause: error });
+ }
+
+ const scrapeIds = data?.map(x => x.id) ?? [];
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
+ let bytes = 0;
+ const bytesLimit = 10485760; // 10 MiB in bytes
+
+ const scrapeBlobs = await Promise.all(
+ scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),
+ );
+
+ for (const [id, scrape] of scrapeBlobs) {
+ if (scrape) {
+ scrapes.push(scrape);
+ bytes += JSON.stringify(scrape).length;
+ } else {
+ logger.warn("Job was considered done, but returnvalue is undefined!", {
+ jobId: id,
+ returnvalue: scrape,
+ });
+ }
+
+ iteratedOver++;
+
+ if (bytes > bytesLimit) {
+ break;
+ }
+ }
+
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
+ }
+
outputBulkB = {
data: scrapes,
next:
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 03255e85e..41f644fb7 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -346,65 +346,7 @@ export async function crawlStatusController(
next: string | undefined;
};
- if (process.env.USE_DB_AUTHENTICATION === "true" && !isPreviewTeam) {
- // new DB-based path
- const { data, error } = await supabase_service.rpc(
- "crawl_status_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- i_start: start,
- i_end: end ?? start + 100,
- },
- { get: true },
- );
-
- if (error || !data) {
- logger.error("Error getting crawl status from DB", { error });
- throw new Error("Error getting crawl status from DB", { cause: error });
- }
-
- const scrapeIds = data?.map(x => x.id) ?? [];
- let scrapes: Document[] = [];
- let iteratedOver = 0;
- let bytes = 0;
- const bytesLimit = 10485760; // 10 MiB in bytes
-
- const scrapeBlobs = await Promise.all(
- scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),
- );
-
- for (const [id, scrape] of scrapeBlobs) {
- if (scrape) {
- scrapes.push(scrape);
- bytes += JSON.stringify(scrape).length;
- } else {
- logger.warn("Job was considered done, but returnvalue is undefined!", {
- jobId: id,
- returnvalue: scrape,
- });
- }
-
- iteratedOver++;
-
- if (bytes > bytesLimit) {
- break;
- }
- }
-
- if (bytes > bytesLimit && scrapes.length !== 1) {
- scrapes.splice(scrapes.length - 1, 1);
- iteratedOver--;
- }
-
- outputBulkB = {
- data: scrapes,
- next:
- (outputBulkA.total ?? 0) > start + iteratedOver
- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
- : undefined,
- };
- } else {
+ if (sc || process.env.USE_DB_AUTHENTICATION !== "true" || isPreviewTeam) {
const doneJobs = await getDoneJobsOrderedUntil(
req.params.jobId,
djoCutoff,
@@ -458,6 +400,64 @@ export async function crawlStatusController(
iteratedOver--;
}
+ outputBulkB = {
+ data: scrapes,
+ next:
+ (outputBulkA.total ?? 0) > start + iteratedOver
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
+ } else {
+ // new DB-based path
+ const { data, error } = await supabase_service.rpc(
+ "crawl_status_1",
+ {
+ i_team_id: req.auth.team_id,
+ i_crawl_id: req.params.jobId,
+ i_start: start,
+ i_end: end ?? start + 100,
+ },
+ { get: true },
+ );
+
+ if (error || !data) {
+ logger.error("Error getting crawl status from DB", { error });
+ throw new Error("Error getting crawl status from DB", { cause: error });
+ }
+
+ const scrapeIds = data?.map(x => x.id) ?? [];
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
+ let bytes = 0;
+ const bytesLimit = 10485760; // 10 MiB in bytes
+
+ const scrapeBlobs = await Promise.all(
+ scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),
+ );
+
+ for (const [id, scrape] of scrapeBlobs) {
+ if (scrape) {
+ scrapes.push(scrape);
+ bytes += JSON.stringify(scrape).length;
+ } else {
+ logger.warn("Job was considered done, but returnvalue is undefined!", {
+ jobId: id,
+ returnvalue: scrape,
+ });
+ }
+
+ iteratedOver++;
+
+ if (bytes > bytesLimit) {
+ break;
+ }
+ }
+
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
+ }
+
outputBulkB = {
data: scrapes,
next: