Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

v2: scrape-status

Gergő Móricza4ffd546775c
diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.ts
new file mode 100644
index 000000000..8d75a8c66
--- /dev/null
+++ b/apps/api/src/controllers/v2/scrape-status.ts
@@ -0,0 +1,53 @@
+import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";
+import { getJob } from "./crawl-status";
+import { logger as _logger } from "../../lib/logger";
+
+export async function scrapeStatusController(req: any, res: any) {
+ const logger = _logger.child({
+ module: "scrape-status",
+ method: "scrapeStatusController",
+ teamId: req.auth.team_id,
+ jobId: req.params.jobId,
+ scrapeId: req.params.jobId,
+ zeroDataRetention: req.acuc?.flags?.forceZDR,
+ });
+
+ if (req.acuc?.flags?.forceZDR) {
+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on scrape status. Please contact [email protected] to unblock this feature." });
+ }
+
+ const job = await supabaseGetJobByIdOnlyData(req.params.jobId, logger);
+
+ if (!job) {
+ return res.status(404).json({
+ success: false,
+ error: "Job not found.",
+ });
+ }
+
+ if (
+ job?.team_id !== req.auth.team_id
+ ) {
+ return res.status(403).json({
+ success: false,
+ error: "You are not allowed to access this resource.",
+ });
+ }
+
+ const jobData = await getJob(req.params.jobId);
+ const data = Array.isArray(jobData?.returnvalue)
+ ? jobData?.returnvalue[0]
+ : jobData?.returnvalue;
+
+ if (!data) {
+ return res.status(404).json({
+ success: false,
+ error: "Job not found.",
+ });
+ }
+
+ return res.status(200).json({
+ success: true,
+ data,
+ });
+}
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index ead47a4b2..b819d420b 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -10,6 +10,7 @@ import { crawlStatusController } from "../controllers/v2/crawl-status";
import { mapV2Controller } from "../controllers/v2/map";
import { crawlErrorsController } from "../controllers/v2/crawl-errors";
import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";
+import { scrapeStatusController } from "../controllers/v2/scrape-status";
import {
authMiddleware,
checkCreditsMiddleware,
@@ -36,6 +37,12 @@ v2Router.post(
wrap(scrapeController),
);
+v2Router.get(
+ "/scrape/:jobId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(scrapeStatusController),
+);
+
v2Router.post(
"/batch/scrape",
authMiddleware(RateLimiterMode.Scrape),