Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
v2: scrape-status
Gergő Móricza4ffd546775c
diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.tsnew file mode 100644index 000000000..8d75a8c66--- /dev/null+++ b/apps/api/src/controllers/v2/scrape-status.ts@@ -0,0 +1,53 @@+import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";+import { getJob } from "./crawl-status";+import { logger as _logger } from "../../lib/logger";++export async function scrapeStatusController(req: any, res: any) {+ const logger = _logger.child({+ module: "scrape-status",+ method: "scrapeStatusController",+ teamId: req.auth.team_id,+ jobId: req.params.jobId,+ scrapeId: req.params.jobId,+ zeroDataRetention: req.acuc?.flags?.forceZDR,+ });++ if (req.acuc?.flags?.forceZDR) {+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on scrape status. Please contact [email protected] to unblock this feature." });+ }++ const job = await supabaseGetJobByIdOnlyData(req.params.jobId, logger);++ if (!job) {+ return res.status(404).json({+ success: false,+ error: "Job not found.",+ });+ }++ if (+ job?.team_id !== req.auth.team_id+ ) {+ return res.status(403).json({+ success: false,+ error: "You are not allowed to access this resource.",+ });+ }++ const jobData = await getJob(req.params.jobId);+ const data = Array.isArray(jobData?.returnvalue)+ ? jobData?.returnvalue[0]+ : jobData?.returnvalue;++ if (!data) {+ return res.status(404).json({+ success: false,+ error: "Job not found.",+ });+ }++ return res.status(200).json({+ success: true,+ data,+ });+}diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex ead47a4b2..b819d420b 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -10,6 +10,7 @@ import { crawlStatusController } from "../controllers/v2/crawl-status";import { mapV2Controller } from "../controllers/v2/map";import { crawlErrorsController } from "../controllers/v2/crawl-errors";import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";+import { scrapeStatusController } from "../controllers/v2/scrape-status";import {authMiddleware,checkCreditsMiddleware,@@ -36,6 +37,12 @@ v2Router.post(wrap(scrapeController),);+v2Router.get(+ "/scrape/:jobId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(scrapeStatusController),+);+v2Router.post("/batch/scrape",authMiddleware(RateLimiterMode.Scrape),