Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrape-status): adapt
Gergő Móricz29c1f126ab90
diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.tsindex 7fec74a12..5b9f9b46d 100644--- a/apps/api/src/controllers/v1/scrape-status.ts+++ b/apps/api/src/controllers/v1/scrape-status.ts@@ -3,40 +3,30 @@ import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";import { scrapeStatusRateLimiter } from "../../services/rate-limiter";export async function scrapeStatusController(req: any, res: any) {- try {- const rateLimiter = scrapeStatusRateLimiter;- const incomingIP = (req.headers["x-forwarded-for"] ||- req.socket.remoteAddress) as string;- const iptoken = incomingIP;- await rateLimiter.consume(iptoken);+ const allowedTeams = [+ "41bdbfe1-0579-4d9b-b6d5-809f16be12f5",+ "511544f2-2fce-4183-9c59-6c29b02c69b5",+ "1ec9a0b3-6e7d-49a9-ad6c-9c598ba824c8",+ ];++ if (!allowedTeams.includes(req.auth.team_id)) {+ return res.status(403).json({+ success: false,+ error: "Forbidden",+ });+ }- const job = await supabaseGetJobByIdOnlyData(req.params.jobId);- const allowedTeams = [- "41bdbfe1-0579-4d9b-b6d5-809f16be12f5",- "511544f2-2fce-4183-9c59-6c29b02c69b5",- ];+ const job = await supabaseGetJobByIdOnlyData(req.params.jobId);- if (!allowedTeams.includes(job?.team_id)) {- return res.status(403).json({- success: false,- error: "You are not allowed to access this resource.",- });- }- return res.status(200).json({- success: true,- data: job?.docs[0],+ if (!allowedTeams.includes(job?.team_id) || job?.team_id !== req.auth.team_id) {+ return res.status(403).json({+ success: false,+ error: "You are not allowed to access this resource.",});- } catch (error) {- if (error instanceof Error && error.message == "Too Many Requests") {- return res.status(429).json({- success: false,- error: "Rate limit exceeded. Please try again later.",- });- } else {- return res.status(500).json({- success: false,- error: "An unexpected error occurred.",- });- }}++ return res.status(200).json({+ success: true,+ data: job?.docs[0],+ });}diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 52f882161..656d076c9 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -192,7 +192,10 @@ v1Router.get(wrap((req: any, res): any => crawlStatusController(req, res, true)),);-v1Router.get("/scrape/:jobId", wrap(scrapeStatusController));+v1Router.get("/scrape/:jobId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(scrapeStatusController),+);v1Router.get("/concurrency-check",diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 051217c6b..450393b22 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -755,6 +755,7 @@ async function processJob(job: Job & { id: string }, token: string) {{content: doc,source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",+ id: job.id,},],},