Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas5b1051b82267
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 4ff9f3f70..75d398f9d 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -1,33 +1,10 @@import express from "express";-import { crawlController } from "../controllers/v1/crawl";-// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";-import { scrapeController } from "../../src/controllers/v1/scrape";-import { crawlStatusController } from "../controllers/v1/crawl-status";-import { mapController } from "../controllers/v1/map";-import { RequestWithAuth } from "../controllers/v1/types";import { RateLimiterMode } from "../types";import expressWs from "express-ws";-import { crawlStatusWSController } from "../controllers/v1/crawl-status-ws";-import { crawlCancelController } from "../controllers/v1/crawl-cancel";-import { scrapeStatusController } from "../controllers/v1/scrape-status";-import { concurrencyCheckController } from "../controllers/v1/concurrency-check";-import { batchScrapeController } from "../controllers/v1/batch-scrape";-import { extractController } from "../controllers/v1/extract";-import { extractStatusController } from "../controllers/v1/extract-status";-import { creditUsageController } from "../controllers/v1/credit-usage";import { searchController } from "../controllers/v1/search";-import { crawlErrorsController } from "../controllers/v1/crawl-errors";-import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";-import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";-import { deepResearchController } from "../controllers/v1/deep-research";-import { deepResearchStatusController } from "../controllers/v1/deep-research-status";-import { tokenUsageController } from "../controllers/v1/token-usage";-import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";import {authMiddleware,checkCreditsMiddleware,- blocklistMiddleware,- idempotencyMiddleware,wrap,} from "./shared";diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsnew file mode 100644index 000000000..69c0c1197--- /dev/null+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -0,0 +1,110 @@+import dotenv from "dotenv";+import { SearchResult } from "../../lib/entities";+import * as Sentry from "@sentry/node";+import { logger } from "../../lib/logger";++dotenv.config();+++export async function fire_engine_search(+ q: string,+ options: {+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ numResults: number;+ page?: number;+ },+ abort?: AbortSignal,+): Promise<SearchResult[]> {+ try {+ let data = JSON.stringify({+ query: q,+ lang: options.lang,+ country: options.country,+ location: options.location,+ tbs: options.tbs,+ numResults: options.numResults,+ page: options.page ?? 1,+ });++ if (!process.env.FIRE_ENGINE_BETA_URL) {+ return [];+ }++ const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/v2/search`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "X-Disable-Cache": "true",+ },+ body: data,+ signal: abort,+ });++ if (response.ok) {+ const responseData = await response.json();+ return responseData;+ } else {+ return [];+ }+ } catch (error) {+ logger.error(error);+ Sentry.captureException(error);+ return [];+ }+}++export async function fireEngineMap(+ q: string,+ options: {+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ numResults: number;+ page?: number;+ },+ abort?: AbortSignal,+): Promise<SearchResult[]> {+ try {+ let data = JSON.stringify({+ query: q,+ lang: options.lang,+ country: options.country,+ location: options.location,+ tbs: options.tbs,+ numResults: options.numResults,+ page: options.page ?? 1,+ });++ if (!process.env.FIRE_ENGINE_BETA_URL) {+ logger.warn("(v1/map Beta) Results might differ from cloud offering currently.");+ return [];+ }++ const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/map`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "X-Disable-Cache": "true",+ },+ body: data,+ signal: abort,+ });++ if (response.ok) {+ const responseData = await response.json();+ return responseData;+ } else {+ return [];+ }+ } catch (error) {+ logger.error(error);+ Sentry.captureException(error);+ return [];+ }+}