Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(v0): remove preview endpoints (#2018)
Gergő Móricza7c4b9d0ea00
diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.tsdeleted file mode 100644index cc5ab4a2d..000000000--- a/apps/api/src/controllers/v0/crawlPreview.ts+++ /dev/null@@ -1,177 +0,0 @@-import { Request, Response } from "express";-import { authenticateUser } from "../auth";-import { RateLimiterMode } from "../../../src/types";-import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";-import { v4 as uuidv4 } from "uuid";-import { logger } from "../../../src/lib/logger";-import {- addCrawlJob,- crawlToCrawler,- finishCrawlKickoff,- lockURL,- saveCrawl,- StoredCrawl,-} from "../../../src/lib/crawl-redis";-import { addScrapeJob } from "../../../src/services/queue-jobs";-import { checkAndUpdateURL } from "../../../src/lib/validateUrl";-import * as Sentry from "@sentry/node";-import { fromLegacyScrapeOptions } from "../v1/types";-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";-import { fromV0ScrapeOptions } from "../v2/types";--export async function crawlPreviewController(req: Request, res: Response) {- try {- const auth = await authenticateUser(req, res, RateLimiterMode.Preview);-- const incomingIP = (req.headers["x-forwarded-for"] ||- req.socket.remoteAddress) as string;- const iptoken = incomingIP + process.env.PREVIEW_TOKEN;- const team_id = `preview_${iptoken}`;-- if (!auth.success) {- return res.status(auth.status).json({ error: auth.error });- }-- if (auth.chunk?.flags?.forceZDR) {- return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." });- }-- let url = req.body.url;- if (!url) {- return res.status(400).json({ error: "Url is required" });- }- try {- url = checkAndUpdateURL(url).url;- } catch (e) {- return res- .status(e instanceof Error && e.message === "Invalid URL" ? 400 : 500)- .json({ error: e.message ?? e });- }-- if (isUrlBlocked(url, auth.chunk?.flags ?? null)) {- return res.status(403).json({- error: BLOCKLISTED_URL_MESSAGE,- });- }-- const crawlerOptions = req.body.crawlerOptions ?? {};- const pageOptions = req.body.pageOptions ?? {- onlyMainContent: false,- includeHtml: false,- removeTags: [],- };-- // if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this?- // try {- // const a = new WebScraperDataProvider();- // await a.setOptions({- // jobId: uuidv4(),- // mode: "single_urls",- // urls: [url],- // crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },- // pageOptions: pageOptions,- // });-- // const docs = await a.getDocuments(false, (progress) => {- // job.updateProgress({- // current: progress.current,- // total: progress.total,- // current_step: "SCRAPING",- // current_url: progress.currentDocumentUrl,- // });- // });- // return res.json({- // success: true,- // documents: docs,- // });- // } catch (error) {- // logger.error(error);- // return res.status(500).json({ error: error.message });- // }- // }-- const id = uuidv4();-- let robots;-- try {- robots = await this.getRobotsTxt();- } catch (_) {}-- const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(- pageOptions,- undefined,- undefined,- team_id- );-- const sc: StoredCrawl = {- originUrl: url,- crawlerOptions,- scrapeOptions,- internalOptions,- team_id,- robots,- createdAt: Date.now(),- };-- await saveCrawl(id, sc);-- const crawler = crawlToCrawler(id, sc, auth.chunk?.flags ?? null);-- await finishCrawlKickoff(id);-- const sitemap = sc.crawlerOptions?.ignoreSitemap- ? 0- : await crawler.tryGetSitemap(async (urls) => {- for (const url of urls) {- await lockURL(id, sc, url);- const jobId = uuidv4();- await addScrapeJob(- {- url,- mode: "single_urls",- team_id,- crawlerOptions,- scrapeOptions,- internalOptions,- origin: "website-preview",- crawl_id: id,- sitemapped: true,- zeroDataRetention: false, // not supported on v0- },- {},- jobId,- );- await addCrawlJob(id, jobId, logger);- }- });-- if (sitemap === 0) {- await lockURL(id, sc, url);- const jobId = uuidv4();- await addScrapeJob(- {- url,- mode: "single_urls",- team_id,- crawlerOptions,- scrapeOptions,- internalOptions,- origin: "website-preview",- crawl_id: id,- zeroDataRetention: false, // not supported on v0- },- {},- jobId,- );- await addCrawlJob(id, jobId, logger);- }-- res.json({ jobId: id });- } catch (error) {- Sentry.captureException(error);- logger.error(error);- return res.status(500).json({ error: error.message });- }-}diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.tsdeleted file mode 100644index c68579eac..000000000--- a/apps/api/src/controllers/v0/status.ts+++ /dev/null@@ -1,58 +0,0 @@-import { Request, Response } from "express";-import { logger } from "../../../src/lib/logger";-import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";-import { getJobs } from "./crawl-status";-import * as Sentry from "@sentry/node";--export async function crawlJobStatusPreviewController(- req: Request,- res: Response,-) {- try {- const sc = await getCrawl(req.params.jobId);- if (!sc) {- return res.status(404).json({ error: "Job not found" });- }-- const jobIDs = await getCrawlJobs(req.params.jobId);-- // let data = job.returnvalue;- // if (process.env.USE_DB_AUTHENTICATION === "true") {- // const supabaseData = await supabaseGetJobById(req.params.jobId);-- // if (supabaseData) {- // data = supabaseData.docs;- // }- // }-- const jobs = (await getJobs(req.params.jobId, jobIDs)).sort(- (a, b) => a.timestamp - b.timestamp,- );- const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- const jobStatus = sc.cancelled- ? "failed"- : jobStatuses.every((x) => x === "completed")- ? "completed"- : jobStatuses.some((x) => x === "failed")- ? "failed"- : "active";-- const data = jobs.map((x) =>- Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue,- );-- res.json({- status: jobStatus,- current: jobStatuses.filter((x) => x === "completed" || x === "failed")- .length,- total: jobs.length,- data: jobStatus === "completed" ? data : null,- partial_data:- jobStatus === "completed" ? [] : data.filter((x) => x !== null),- });- } catch (error) {- Sentry.captureException(error);- logger.error(error);- return res.status(500).json({ error: error.message });- }-}diff --git a/apps/api/src/routes/v0.ts b/apps/api/src/routes/v0.tsindex 3a7bda655..772cab581 100644--- a/apps/api/src/routes/v0.ts+++ b/apps/api/src/routes/v0.ts@@ -2,8 +2,6 @@ import express from "express";import { crawlController } from "../../src/controllers/v0/crawl";import { crawlStatusController } from "../../src/controllers/v0/crawl-status";import { scrapeController } from "../../src/controllers/v0/scrape";-import { crawlPreviewController } from "../../src/controllers/v0/crawlPreview";-import { crawlJobStatusPreviewController } from "../../src/controllers/v0/status";import { searchController } from "../../src/controllers/v0/search";import { crawlCancelController } from "../../src/controllers/v0/crawl-cancel";import { keyAuthController } from "../../src/controllers/v0/keyAuth";@@ -14,10 +12,8 @@ export const v0Router = express.Router();v0Router.post("/v0/scrape", scrapeController);v0Router.post("/v0/crawl", crawlController);-v0Router.post("/v0/crawlWebsitePreview", crawlPreviewController);v0Router.get("/v0/crawl/status/:jobId", crawlStatusController);v0Router.delete("/v0/crawl/cancel/:jobId", crawlCancelController);-v0Router.get("/v0/checkJobStatus/:jobId", crawlJobStatusPreviewController);// Auth route for key based authenticationv0Router.get("/v0/keyAuth", keyAuthController);