Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(v2): pull over crawl cancel and active
Gergő Móricz5ae53bb603f0
diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.tsnew file mode 100644index 000000000..00af8b31f--- /dev/null+++ b/apps/api/src/controllers/v2/crawl-cancel.ts@@ -0,0 +1,53 @@+import { Response } from "express";+import { supabase_service } from "../../services/supabase";+import { logger } from "../../lib/logger";+import { getCrawl, saveCrawl } from "../../lib/crawl-redis";+import * as Sentry from "@sentry/node";+import { configDotenv } from "dotenv";+import { RequestWithAuth } from "./types";+configDotenv();++export async function crawlCancelController(+ req: RequestWithAuth<{ jobId: string }>,+ res: Response,+) {+ try {+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";++ const sc = await getCrawl(req.params.jobId);+ if (!sc) {+ return res.status(404).json({ error: "Job not found" });+ }++ // check if the job belongs to the team+ if (useDbAuthentication) {+ const { data, error: supaError } = await supabase_service+ .from("bulljobs_teams")+ .select("*")+ .eq("job_id", req.params.jobId)+ .eq("team_id", req.auth.team_id);+ if (supaError) {+ return res.status(500).json({ error: supaError.message });+ }++ if (data.length === 0) {+ return res.status(403).json({ error: "Unauthorized" });+ }+ }++ try {+ sc.cancelled = true;+ await saveCrawl(req.params.jobId, sc);+ } catch (error) {+ logger.error(error);+ }++ res.json({+ status: "cancelled",+ });+ } catch (error) {+ Sentry.captureException(error);+ logger.error(error);+ return res.status(500).json({ error: error.message });+ }+}diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex acfdb128c..1f8dad10a 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -13,6 +13,7 @@ import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";import { scrapeStatusController } from "../controllers/v2/scrape-status";import { creditUsageController } from "../controllers/v2/credit-usage";import { tokenUsageController } from "../controllers/v2/token-usage";+import { crawlCancelController } from "../controllers/v2/crawl-cancel";import {authMiddleware,checkCreditsMiddleware,@@ -81,18 +82,36 @@ v2Router.get(wrap(ongoingCrawlsController),);+v2Router.get(+ "/crawl/active",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(ongoingCrawlsController),+);+v2Router.get("/crawl/:jobId",authMiddleware(RateLimiterMode.CrawlStatus),wrap(crawlStatusController),);+v2Router.delete(+ "/crawl/:jobId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(crawlCancelController),+);+v2Router.get("/batch/scrape/:jobId",authMiddleware(RateLimiterMode.CrawlStatus),wrap((req: any, res: any) => crawlStatusController(req, res, true)),);+v2Router.delete(+ "/batch/scrape/:jobId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(crawlCancelController),+);+v2Router.get("/crawl/:jobId/errors",authMiddleware(RateLimiterMode.CrawlStatus),