Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(v0): remove preview endpoints (#2018)

Gergő Móricza7c4b9d0ea00
diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts
deleted file mode 100644
index cc5ab4a2d..000000000
--- a/apps/api/src/controllers/v0/crawlPreview.ts
+++ /dev/null
@@ -1,177 +0,0 @@
-import { Request, Response } from "express";
-import { authenticateUser } from "../auth";
-import { RateLimiterMode } from "../../../src/types";
-import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
-import { v4 as uuidv4 } from "uuid";
-import { logger } from "../../../src/lib/logger";
-import {
- addCrawlJob,
- crawlToCrawler,
- finishCrawlKickoff,
- lockURL,
- saveCrawl,
- StoredCrawl,
-} from "../../../src/lib/crawl-redis";
-import { addScrapeJob } from "../../../src/services/queue-jobs";
-import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
-import * as Sentry from "@sentry/node";
-import { fromLegacyScrapeOptions } from "../v1/types";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
-import { fromV0ScrapeOptions } from "../v2/types";
-
-export async function crawlPreviewController(req: Request, res: Response) {
- try {
- const auth = await authenticateUser(req, res, RateLimiterMode.Preview);
-
- const incomingIP = (req.headers["x-forwarded-for"] ||
- req.socket.remoteAddress) as string;
- const iptoken = incomingIP + process.env.PREVIEW_TOKEN;
- const team_id = `preview_${iptoken}`;
-
- if (!auth.success) {
- return res.status(auth.status).json({ error: auth.error });
- }
-
- if (auth.chunk?.flags?.forceZDR) {
- return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });
- }
-
- let url = req.body.url;
- if (!url) {
- return res.status(400).json({ error: "Url is required" });
- }
- try {
- url = checkAndUpdateURL(url).url;
- } catch (e) {
- return res
- .status(e instanceof Error && e.message === "Invalid URL" ? 400 : 500)
- .json({ error: e.message ?? e });
- }
-
- if (isUrlBlocked(url, auth.chunk?.flags ?? null)) {
- return res.status(403).json({
- error: BLOCKLISTED_URL_MESSAGE,
- });
- }
-
- const crawlerOptions = req.body.crawlerOptions ?? {};
- const pageOptions = req.body.pageOptions ?? {
- onlyMainContent: false,
- includeHtml: false,
- removeTags: [],
- };
-
- // if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?
- // try {
- // const a = new WebScraperDataProvider();
- // await a.setOptions({
- // jobId: uuidv4(),
- // mode: "single_urls",
- // urls: [url],
- // crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },
- // pageOptions: pageOptions,
- // });
-
- // const docs = await a.getDocuments(false, (progress) => {
- // job.updateProgress({
- // current: progress.current,
- // total: progress.total,
- // current_step: "SCRAPING",
- // current_url: progress.currentDocumentUrl,
- // });
- // });
- // return res.json({
- // success: true,
- // documents: docs,
- // });
- // } catch (error) {
- // logger.error(error);
- // return res.status(500).json({ error: error.message });
- // }
- // }
-
- const id = uuidv4();
-
- let robots;
-
- try {
- robots = await this.getRobotsTxt();
- } catch (_) {}
-
- const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(
- pageOptions,
- undefined,
- undefined,
- team_id
- );
-
- const sc: StoredCrawl = {
- originUrl: url,
- crawlerOptions,
- scrapeOptions,
- internalOptions,
- team_id,
- robots,
- createdAt: Date.now(),
- };
-
- await saveCrawl(id, sc);
-
- const crawler = crawlToCrawler(id, sc, auth.chunk?.flags ?? null);
-
- await finishCrawlKickoff(id);
-
- const sitemap = sc.crawlerOptions?.ignoreSitemap
- ? 0
- : await crawler.tryGetSitemap(async (urls) => {
- for (const url of urls) {
- await lockURL(id, sc, url);
- const jobId = uuidv4();
- await addScrapeJob(
- {
- url,
- mode: "single_urls",
- team_id,
- crawlerOptions,
- scrapeOptions,
- internalOptions,
- origin: "website-preview",
- crawl_id: id,
- sitemapped: true,
- zeroDataRetention: false, // not supported on v0
- },
- {},
- jobId,
- );
- await addCrawlJob(id, jobId, logger);
- }
- });
-
- if (sitemap === 0) {
- await lockURL(id, sc, url);
- const jobId = uuidv4();
- await addScrapeJob(
- {
- url,
- mode: "single_urls",
- team_id,
- crawlerOptions,
- scrapeOptions,
- internalOptions,
- origin: "website-preview",
- crawl_id: id,
- zeroDataRetention: false, // not supported on v0
- },
- {},
- jobId,
- );
- await addCrawlJob(id, jobId, logger);
- }
-
- res.json({ jobId: id });
- } catch (error) {
- Sentry.captureException(error);
- logger.error(error);
- return res.status(500).json({ error: error.message });
- }
-}
diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.ts
deleted file mode 100644
index c68579eac..000000000
--- a/apps/api/src/controllers/v0/status.ts
+++ /dev/null
@@ -1,58 +0,0 @@
-import { Request, Response } from "express";
-import { logger } from "../../../src/lib/logger";
-import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";
-import { getJobs } from "./crawl-status";
-import * as Sentry from "@sentry/node";
-
-export async function crawlJobStatusPreviewController(
- req: Request,
- res: Response,
-) {
- try {
- const sc = await getCrawl(req.params.jobId);
- if (!sc) {
- return res.status(404).json({ error: "Job not found" });
- }
-
- const jobIDs = await getCrawlJobs(req.params.jobId);
-
- // let data = job.returnvalue;
- // if (process.env.USE_DB_AUTHENTICATION === "true") {
- // const supabaseData = await supabaseGetJobById(req.params.jobId);
-
- // if (supabaseData) {
- // data = supabaseData.docs;
- // }
- // }
-
- const jobs = (await getJobs(req.params.jobId, jobIDs)).sort(
- (a, b) => a.timestamp - b.timestamp,
- );
- const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
- const jobStatus = sc.cancelled
- ? "failed"
- : jobStatuses.every((x) => x === "completed")
- ? "completed"
- : jobStatuses.some((x) => x === "failed")
- ? "failed"
- : "active";
-
- const data = jobs.map((x) =>
- Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue,
- );
-
- res.json({
- status: jobStatus,
- current: jobStatuses.filter((x) => x === "completed" || x === "failed")
- .length,
- total: jobs.length,
- data: jobStatus === "completed" ? data : null,
- partial_data:
- jobStatus === "completed" ? [] : data.filter((x) => x !== null),
- });
- } catch (error) {
- Sentry.captureException(error);
- logger.error(error);
- return res.status(500).json({ error: error.message });
- }
-}
diff --git a/apps/api/src/routes/v0.ts b/apps/api/src/routes/v0.ts
index 3a7bda655..772cab581 100644
--- a/apps/api/src/routes/v0.ts
+++ b/apps/api/src/routes/v0.ts
@@ -2,8 +2,6 @@ import express from "express";
import { crawlController } from "../../src/controllers/v0/crawl";
import { crawlStatusController } from "../../src/controllers/v0/crawl-status";
import { scrapeController } from "../../src/controllers/v0/scrape";
-import { crawlPreviewController } from "../../src/controllers/v0/crawlPreview";
-import { crawlJobStatusPreviewController } from "../../src/controllers/v0/status";
import { searchController } from "../../src/controllers/v0/search";
import { crawlCancelController } from "../../src/controllers/v0/crawl-cancel";
import { keyAuthController } from "../../src/controllers/v0/keyAuth";
@@ -14,10 +12,8 @@ export const v0Router = express.Router();
v0Router.post("/v0/scrape", scrapeController);
v0Router.post("/v0/crawl", crawlController);
-v0Router.post("/v0/crawlWebsitePreview", crawlPreviewController);
v0Router.get("/v0/crawl/status/:jobId", crawlStatusController);
v0Router.delete("/v0/crawl/cancel/:jobId", crawlCancelController);
-v0Router.get("/v0/checkJobStatus/:jobId", crawlJobStatusPreviewController);
// Auth route for key based authentication
v0Router.get("/v0/keyAuth", keyAuthController);