Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(v2): pull over crawl cancel and active

Gergő Móricz5ae53bb603f0
diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts
new file mode 100644
index 000000000..00af8b31f
--- /dev/null
+++ b/apps/api/src/controllers/v2/crawl-cancel.ts
@@ -0,0 +1,53 @@
+import { Response } from "express";
+import { supabase_service } from "../../services/supabase";
+import { logger } from "../../lib/logger";
+import { getCrawl, saveCrawl } from "../../lib/crawl-redis";
+import * as Sentry from "@sentry/node";
+import { configDotenv } from "dotenv";
+import { RequestWithAuth } from "./types";
+configDotenv();
+
+export async function crawlCancelController(
+ req: RequestWithAuth<{ jobId: string }>,
+ res: Response,
+) {
+ try {
+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+
+ const sc = await getCrawl(req.params.jobId);
+ if (!sc) {
+ return res.status(404).json({ error: "Job not found" });
+ }
+
+ // check if the job belongs to the team
+ if (useDbAuthentication) {
+ const { data, error: supaError } = await supabase_service
+ .from("bulljobs_teams")
+ .select("*")
+ .eq("job_id", req.params.jobId)
+ .eq("team_id", req.auth.team_id);
+ if (supaError) {
+ return res.status(500).json({ error: supaError.message });
+ }
+
+ if (data.length === 0) {
+ return res.status(403).json({ error: "Unauthorized" });
+ }
+ }
+
+ try {
+ sc.cancelled = true;
+ await saveCrawl(req.params.jobId, sc);
+ } catch (error) {
+ logger.error(error);
+ }
+
+ res.json({
+ status: "cancelled",
+ });
+ } catch (error) {
+ Sentry.captureException(error);
+ logger.error(error);
+ return res.status(500).json({ error: error.message });
+ }
+}
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index acfdb128c..1f8dad10a 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -13,6 +13,7 @@ import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";
import { scrapeStatusController } from "../controllers/v2/scrape-status";
import { creditUsageController } from "../controllers/v2/credit-usage";
import { tokenUsageController } from "../controllers/v2/token-usage";
+import { crawlCancelController } from "../controllers/v2/crawl-cancel";
import {
authMiddleware,
checkCreditsMiddleware,
@@ -81,18 +82,36 @@ v2Router.get(
wrap(ongoingCrawlsController),
);
+v2Router.get(
+ "/crawl/active",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(ongoingCrawlsController),
+);
+
v2Router.get(
"/crawl/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
wrap(crawlStatusController),
);
+v2Router.delete(
+ "/crawl/:jobId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(crawlCancelController),
+);
+
v2Router.get(
"/batch/scrape/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
wrap((req: any, res: any) => crawlStatusController(req, res, true)),
);
+v2Router.delete(
+ "/batch/scrape/:jobId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(crawlCancelController),
+);
+
v2Router.get(
"/crawl/:jobId/errors",
authMiddleware(RateLimiterMode.CrawlStatus),