Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrape-status): adapt

Gergő Móricz29c1f126ab90
diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.ts
index 7fec74a12..5b9f9b46d 100644
--- a/apps/api/src/controllers/v1/scrape-status.ts
+++ b/apps/api/src/controllers/v1/scrape-status.ts
@@ -3,40 +3,30 @@ import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";
import { scrapeStatusRateLimiter } from "../../services/rate-limiter";
export async function scrapeStatusController(req: any, res: any) {
- try {
- const rateLimiter = scrapeStatusRateLimiter;
- const incomingIP = (req.headers["x-forwarded-for"] ||
- req.socket.remoteAddress) as string;
- const iptoken = incomingIP;
- await rateLimiter.consume(iptoken);
+ const allowedTeams = [
+ "41bdbfe1-0579-4d9b-b6d5-809f16be12f5",
+ "511544f2-2fce-4183-9c59-6c29b02c69b5",
+ "1ec9a0b3-6e7d-49a9-ad6c-9c598ba824c8",
+ ];
+
+ if (!allowedTeams.includes(req.auth.team_id)) {
+ return res.status(403).json({
+ success: false,
+ error: "Forbidden",
+ });
+ }
- const job = await supabaseGetJobByIdOnlyData(req.params.jobId);
- const allowedTeams = [
- "41bdbfe1-0579-4d9b-b6d5-809f16be12f5",
- "511544f2-2fce-4183-9c59-6c29b02c69b5",
- ];
+ const job = await supabaseGetJobByIdOnlyData(req.params.jobId);
- if (!allowedTeams.includes(job?.team_id)) {
- return res.status(403).json({
- success: false,
- error: "You are not allowed to access this resource.",
- });
- }
- return res.status(200).json({
- success: true,
- data: job?.docs[0],
+ if (!allowedTeams.includes(job?.team_id) || job?.team_id !== req.auth.team_id) {
+ return res.status(403).json({
+ success: false,
+ error: "You are not allowed to access this resource.",
});
- } catch (error) {
- if (error instanceof Error && error.message == "Too Many Requests") {
- return res.status(429).json({
- success: false,
- error: "Rate limit exceeded. Please try again later.",
- });
- } else {
- return res.status(500).json({
- success: false,
- error: "An unexpected error occurred.",
- });
- }
}
+
+ return res.status(200).json({
+ success: true,
+ data: job?.docs[0],
+ });
}
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 52f882161..656d076c9 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -192,7 +192,10 @@ v1Router.get(
wrap((req: any, res): any => crawlStatusController(req, res, true)),
);
-v1Router.get("/scrape/:jobId", wrap(scrapeStatusController));
+v1Router.get("/scrape/:jobId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(scrapeStatusController),
+);
v1Router.get(
"/concurrency-check",
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 051217c6b..450393b22 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -755,6 +755,7 @@ async function processJob(job: Job & { id: string }, token: string) {
{
content: doc,
source: doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? "",
+ id: job.id,
},
],
},