Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import express from "express";
import { crawlController } from "../controllers/v1/crawl";
// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";
import { scrapeController } from "../../src/controllers/v1/scrape";
import { crawlStatusController } from "../controllers/v1/crawl-status";
import { mapController } from "../controllers/v1/map";
import { RateLimiterMode } from "../types";
import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";
import expressWs from "express-ws";
import { crawlStatusWSController } from "../controllers/v1/crawl-status-ws";
import { crawlCancelController } from "../controllers/v1/crawl-cancel";
import { scrapeStatusController } from "../controllers/v1/scrape-status";
import { concurrencyCheckController } from "../controllers/v1/concurrency-check";
import { batchScrapeController } from "../controllers/v1/batch-scrape";
import { extractController } from "../controllers/v1/extract";
import { extractStatusController } from "../controllers/v1/extract-status";
import { creditUsageController } from "../controllers/v1/credit-usage";
import { searchController } from "../controllers/v1/search";
import { crawlErrorsController } from "../controllers/v1/crawl-errors";
import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";
import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";
import { deepResearchController } from "../controllers/v1/deep-research";
import { deepResearchStatusController } from "../controllers/v1/deep-research-status";
import { tokenUsageController } from "../controllers/v1/token-usage";
import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";
import { fireclawController } from "../controllers/v1/fireclaw";
import {
authMiddleware,
checkCreditsMiddleware,
blocklistMiddleware,
scrapeBlocklistMiddleware,
restrictedOptionsCountryCheck,
idempotencyMiddleware,
requestTimingMiddleware,
validateJobIdParam,
wrap,
inferIntegrationMiddleware,
} from "./shared";
import { queueStatusController } from "../controllers/v1/queue-status";
import { creditUsageHistoricalController } from "../controllers/v1/credit-usage-historical";
import { tokenUsageHistoricalController } from "../controllers/v1/token-usage-historical";
import { deprecationMiddleware } from "../lib/deprecations";
export const v1Router = express.Router();
expressWs(express()).applyTo(v1Router);
// Add timing middleware to all v1 routes
v1Router.use(requestTimingMiddleware("v1"));
v1Router.post(
"/scrape",
authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
deprecationMiddleware("v1_scrape"),
inferIntegrationMiddleware,
restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
scrapeBlocklistMiddleware,
wrap(scrapeController),
);
v1Router.post(
"/crawl",
authMiddleware(RateLimiterMode.Crawl),
deprecationMiddleware("v1_crawl"),
inferIntegrationMiddleware,
restrictedOptionsCountryCheck,
checkCreditsMiddleware(),
scrapeBlocklistMiddleware,
idempotencyMiddleware,
wrap(crawlController),
);
v1Router.post(
"/batch/scrape",
authMiddleware(RateLimiterMode.Scrape),
deprecationMiddleware("v1_batch_scrape"),
inferIntegrationMiddleware,
restrictedOptionsCountryCheck,
checkCreditsMiddleware(),
blocklistMiddleware,
idempotencyMiddleware,
wrap(batchScrapeController),
);
v1Router.post(
"/search",
authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),
deprecationMiddleware("v1_search"),
inferIntegrationMiddleware,
restrictedOptionsCountryCheck,
checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
wrap(searchController),
);
v1Router.post(
"/map",
authMiddleware(RateLimiterMode.Map),
deprecationMiddleware("v1_map"),
inferIntegrationMiddleware,
checkCreditsMiddleware(1),
blocklistMiddleware,
wrap(mapController),
);
v1Router.get(
"/crawl/ongoing",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_crawl_ongoing"),
wrap(ongoingCrawlsController),
);
// Public facing, same as ongoing
v1Router.get(
"/crawl/active",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_crawl_active"),
wrap(ongoingCrawlsController),
);
v1Router.get(
"/crawl/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_crawl_status"),
validateJobIdParam,
wrap(crawlStatusController),
);
v1Router.get(
"/batch/scrape/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_batch_scrape_status"),
validateJobIdParam,
// Yes, it uses the same controller as the normal crawl status controller
wrap((req: any, res): any => crawlStatusController(req, res, true)),
);
v1Router.get(
"/crawl/:jobId/errors",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_crawl_errors"),
wrap(crawlErrorsController),
);
v1Router.get(
"/batch/scrape/:jobId/errors",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_batch_scrape_errors"),
wrap(crawlErrorsController),
);
v1Router.get(
"/scrape/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_scrape_status"),
wrap(scrapeStatusController),
);
v1Router.get(
"/concurrency-check",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_concurrency_check"),
wrap(concurrencyCheckController),
);
v1Router.ws("/crawl/:jobId", crawlStatusWSController);
v1Router.post(
"/extract",
authMiddleware(RateLimiterMode.Extract),
inferIntegrationMiddleware,
deprecationMiddleware("v1_extract"),
restrictedOptionsCountryCheck,
checkCreditsMiddleware(20),
wrap(extractController),
);
v1Router.get(
"/extract/:jobId",
authMiddleware(RateLimiterMode.ExtractStatus),
deprecationMiddleware("v1_extract_status"),
wrap(extractStatusController),
);
v1Router.post(
"/llmstxt",
authMiddleware(RateLimiterMode.Scrape),
deprecationMiddleware("v1_llmstxt"),
restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
blocklistMiddleware,
wrap(generateLLMsTextController),
);
v1Router.get(
"/llmstxt/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_llmstxt_status"),
wrap(generateLLMsTextStatusController),
);
v1Router.post(
"/deep-research",
authMiddleware(RateLimiterMode.Crawl),
deprecationMiddleware("v1_deep_research"),
restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
wrap(deepResearchController),
);
v1Router.get(
"/deep-research/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_deep_research_status"),
wrap(deepResearchStatusController),
);
// v1Router.post("/crawlWebsitePreview", crawlPreviewController);
v1Router.delete(
"/crawl/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_crawl_cancel"),
crawlCancelController,
);
v1Router.delete(
"/batch/scrape/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),
deprecationMiddleware("v1_batch_scrape_cancel"),
crawlCancelController,
);
// v1Router.get("/checkJobStatus/:jobId", crawlJobStatusPreviewController);
// // Auth route for key based authentication
// v1Router.get("/keyAuth", keyAuthController);
// // Search routes
// v0Router.post("/search", searchController);
// Health/Probe routes
// v1Router.get("/health/liveness", livenessController);
// v1Router.get("/health/readiness", readinessController);
v1Router.post(
"/fireclaw",
authMiddleware(RateLimiterMode.Scrape),
checkCreditsMiddleware(100),
wrap(fireclawController),
);
v1Router.get(
"/team/credit-usage",
authMiddleware(RateLimiterMode.Account),
deprecationMiddleware("v1_team_credit_usage"),
wrap(creditUsageController),
);
v1Router.get(
"/team/credit-usage/historical",
authMiddleware(RateLimiterMode.Account),
deprecationMiddleware("v1_team_credit_usage_historical"),
wrap(creditUsageHistoricalController),
);
v1Router.get(
"/team/token-usage",
authMiddleware(RateLimiterMode.Account),
deprecationMiddleware("v1_team_token_usage"),
wrap(tokenUsageController),
);
v1Router.get(
"/team/token-usage/historical",
authMiddleware(RateLimiterMode.Account),
deprecationMiddleware("v1_team_token_usage_historical"),
wrap(tokenUsageHistoricalController),
);
v1Router.get(
"/team/queue-status",
authMiddleware(RateLimiterMode.Account),
deprecationMiddleware("v1_team_queue_status"),
wrap(queueStatusController),
);