Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge branch 'main' into nsc/v2
Gergő Móricz21cbfec39857
diff --cc apps/api/pnpm-lock.yamlindex 8a52f24fa,1c562dc4f..fd41ec068--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@@ -2111,6 -3115,16 +3118,12 @@@ packagesresolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}- resolution: {integrity: sha512-vjtV3hiLqYDNRoiAv0zC4QaGAMPomEoq83PRmYIofPswwZurCeWR5LByXm7SyoL0Zh5+2z0+HC7jG8gSZJUh0w==}- engines: {node: '>= 16'}-+ resolution: {integrity: sha512-xz7wQ8xDhdyP7tQxwdteLYeFfS68tSMNCZ/Y37WJ4bhGfKPpqEIlmIyueQHqOyoPhE6xNUqjzRr8ra0eF9VRvA==}++ resolution: {integrity: sha512-M7uio8Zt++eg3jPj+rHMfCC+IuygQHHCOU+IYsVtik6FWjuYpVt/+MRKcgsAMHh8mMFAwnB+Bs+mTrFiXjMzKg==}+resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}@@@ -3825,9 -5243,16 +5241,19 @@@resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==}engines: {node: '>=0.6'}+ resolution: {integrity: sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==}+ engines: {node: '>=6'}++ resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==}++ resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}++ resolution: {integrity: sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==}+resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}engines: {node: '>= 0.6'}@@@ -3900,9 -5333,9 +5334,12 @@@resolution: {integrity: sha512-nQFEv9gRw6SJAwWD2LrL0NmQvAcO7FBwJbwmr2ttPAacfy0xuiOjE5zt+zM4xDyuyvUaxBi/9gb2SoCyNEVJcw==}engines: {node: '>=8.6.0'}+ resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==}++ resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==}+resolution: {integrity: sha512-F3PVHdTHeLonSnrU5V6k8643LJ9QacLu3uI9M+BAFkmBmB1ELM2x7fdsziYZoSm6DmU6TKwiQCK0jf8dcNomcQ==}engines: {node: '>=18'}@@@ -4371,9 -5933,16 +5945,19 @@@resolution: {integrity: sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==}+ resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==}++ resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==}+ peerDependencies:+ react: ^16.8.0 || ^17.0.0 || ^18.0.0++ resolution: {integrity: sha512-9WXSPC5fMv61vaupRkCKCxsPxBocVnwakBEkMIHHpkTTg6icbJtg6jzgtLDm4bl3cSHAca52rYWih0k4K3PfHw==}+ peerDependencies:+ react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0+resolution: {integrity: sha512-Rb46I4cGGVBmjamjphe8L/UnvJD+uPPtTkNvX5mZgqdbavhI4EbgIWJiIHXJ8bc/i9EQGPRh4DwEURJ552Do0A==}peerDependencies:@@@ -6985,6 -10814,10 +10829,10 @@@ snapshots+ dependencies:- tslib: 2.6.3++ tslib: 2.8.1+dependencies:tslib: 2.6.3@@@ -7064,8 -10903,22 +10918,20 @@@- [email protected]: {}-+ dependencies:+ safe-buffer: 5.2.1++ [email protected]: {}++ [email protected]: {}++ dependencies:+ bindings: 1.5.0+@@@ -9036,8 -13368,17 +13381,19 @@@dependencies:side-channel: 1.1.0+ dependencies:+ decode-uri-component: 0.2.2+ filter-obj: 1.1.0+ split-on-first: 1.1.0+ strict-uri-encode: 2.0.0++ [email protected]: {}++ [email protected]: {}++ [email protected]: {}+@@@ -9121,8 -13466,8 +13481,10 @@@transitivePeerDependencies:- supports-color+ [email protected]: {}++ [email protected]: {}+dependencies:'@react-email/render': 0.0.15@@@ -9578,10 -14009,22 +14033,24 @@@+ [email protected]: {}++ dependencies:+ anymatch: 3.1.3+ chokidar: 4.0.3+ destr: 2.0.5+ h3: 1.15.4+ lru-cache: 10.4.3+ node-fetch-native: 1.6.7+ ofetch: 1.4.1+ ufo: 1.6.1+ optionalDependencies:+ idb-keyval: 6.2.2+ ioredis: 5.6.1+dependencies:async: 3.2.5@@@ -9600,11 -14043,14 +14069,19 @@@+ dependencies:+ querystringify: 2.2.0+ requires-port: 1.0.0++ dependencies:+ react: 18.3.1++ dependencies:+ react: 18.3.1+dependencies:react: 18.3.1diff --cc apps/api/src/controllers/v1/search.tsindex 94a530ab0,d4429e15b..c6c714288--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@@ -114,10 -110,10 +114,10 @@@ async function scrapeSearchResultmode: "single_urls" as Mode,team_id: options.teamId,scrapeOptions: {- ...options.scrapeOptions,- maxAge: options.scrapeOptions.maxAge === 0 ? 3 * 24 * 60 * 60 * 1000 : options.scrapeOptions.maxAge,+ ...scrapeOptions,- maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days++ maxAge: scrapeOptions.maxAge === 0 ? 3 * 24 * 60 * 60 * 1000 : scrapeOptions.maxAge,},- internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },+ internalOptions: { ...internalOptions, teamId: options.teamId, bypassBilling: true, zeroDataRetention },origin: options.origin,is_scrape: true,startTime: Date.now(),diff --cc apps/api/src/index.tsindex 1b22c71c2,3ff680558..9ebf1a04f--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@@ -26,8 -26,10 +26,11 @@@ import { v4 as uuidv4 } from "uuid"import { RateLimiterMode } from "./types";import { attachWsProxy } from "./services/agentLivecastWS";import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";+import { v2Router } from "./routes/v2";import domainFrequencyRouter from "./routes/domain-frequency";+ import { NodeSDK } from "@opentelemetry/sdk-node";+ import { LangfuseExporter } from "langfuse-vercel";+ import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter');diff --cc apps/api/src/lib/crawl-redis.tsindex 8a806d2f0,65bd96878..2647c488b--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@@ -1,8 -1,8 +1,8 @@@import { InternalOptions } from "../scraper/scrapeURL";-import { ScrapeOptions, TeamFlags } from "../controllers/v1/types";+import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";import { WebCrawler } from "../scraper/WebScraper/crawler";import { redisEvictConnection } from "../services/redis";- import { logger as _logger } from "./logger";+ import { logger as _logger, logger } from "./logger";import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";import type { Logger } from "winston";diff --cc apps/api/src/routes/v1.tsindex 583ba4a5f,a5165c927..3b3dd80c6--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@@ -15,7 -27,9 +15,8 @@@ import { batchScrapeController } from "import { extractController } from "../controllers/v1/extract";import { extractStatusController } from "../controllers/v1/extract-status";import { creditUsageController } from "../controllers/v1/credit-usage";-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";import { searchController } from "../controllers/v1/search";+ import { x402SearchController } from "../controllers/v1/x402-search";import { crawlErrorsController } from "../controllers/v1/crawl-errors";import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";@@@ -23,14 -37,154 +24,15 @@@ import { deepResearchController } fromimport { deepResearchStatusController } from "../controllers/v1/deep-research-status";import { tokenUsageController } from "../controllers/v1/token-usage";import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";-import { addDomainFrequencyJob } from "../services"+import {+ authMiddleware,+ checkCreditsMiddleware,+ blocklistMiddleware,+ idempotencyMiddleware,+ wrap,+} from "./shared";+ import { paymentMiddleware } from "x402-express";-function checkCreditsMiddleware(- _minimum?: number,-): (req: RequestWithAuth, res: Response, next: NextFunction) => void {- return (req, res, next) => {- let minimum = _minimum;- (async () => {- if (!minimum && req.body) {- minimum =- (req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1;- }- const { success, remainingCredits, chunk } = await checkTeamCredits(- req.acuc,- req.auth.team_id,- minimum ?? 1,- );- if (chunk) {- req.acuc = chunk;- }- req.account = { remainingCredits };- if (!success) {- if (!minimum && req.body && (req.body as any).limit !== undefined && remainingCredits > 0) {- logger.warn("Adjusting limit to remaining credits", {- teamId: req.auth.team_id,- remainingCredits,- request: req.body,- });- (req.body as any).limit = remainingCredits;- return next();- }-- const currencyName = req.acuc.is_extract ? "tokens" : "credits"- logger.error(- `Insufficient ${currencyName}: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`,- {- teamId: req.auth.team_id,- minimum,- remainingCredits,- request: req.body,- path: req.path- }- );- if (!res.headersSent && req.auth.team_id !== "8c528896-7882-4587-a4b6-768b721b0b53") {- return res.status(402).json({- success: false,- error:- "Insufficient " + currencyName + " to perform this request. For more " + currencyName + ", you can upgrade your plan at " + (currencyName === "credits" ? "https://firecrawl.dev/pricing or try changing the request limit to a lower value" : "https://www.firecrawl.dev/extract#pricing") + ".",- });- }- }- next();- })().catch((err) => next(err));- };-}--export function authMiddleware(- rateLimiterMode: RateLimiterMode-): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {- return (req, res, next) => {- (async () => {- let currentRateLimiterMode = rateLimiterMode;- if (currentRateLimiterMode === RateLimiterMode.Extract && isAgentExtractModelValid((req.body as any)?.agent?.model)) {- currentRateLimiterMode = RateLimiterMode.ExtractAgentPreview;- }-- // Track domain frequency regardless of caching- try {- // Use the URL from the request body if available- const urlToTrack = (req.body as any)?.url;- if (urlToTrack) {- await addDomainFrequencyJob(urlToTrack);- }- } catch (error) {- // Log error without meta.logger since it's not available in this context- logger.warn("Failed to track domain frequency", { error });- }-- // if (currentRateLimiterMode === RateLimiterMode.Scrape && isAgentExtractModelValid((req.body as any)?.agent?.model)) {- // currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;- // }-- const auth = await authenticateUser(req, res, currentRateLimiterMode);-- if (!auth.success) {- if (!res.headersSent) {- return res- .status(auth.status)- .json({ success: false, error: auth.error });- } else {- return;- }- }-- const { team_id, chunk } = auth;-- req.auth = { team_id };- req.acuc = chunk ?? undefined;- if (chunk) {- req.account = { remainingCredits: chunk.remaining_credits };- }- next();- })().catch((err) => next(err));- };-}--function idempotencyMiddleware(- req: Request,- res: Response,- next: NextFunction,-) {- (async () => {- if (req.headers["x-idempotency-key"]) {- const isIdempotencyValid = await validateIdempotencyKey(req);- if (!isIdempotencyValid) {- if (!res.headersSent) {- return res- .status(409)- .json({ success: false, error: "Idempotency key already used" });- }- }- createIdempotencyKey(req);- }- next();- })().catch((err) => next(err));-}--function blocklistMiddleware(req: RequestWithACUC<any, any, any>, res: Response, next: NextFunction) {- if (typeof req.body.url === "string" && isUrlBlocked(req.body.url, req.acuc?.flags ?? null)) {- if (!res.headersSent) {- return res.status(403).json({- success: false,- error: BLOCKLISTED_URL_MESSAGE,- });- }- }- next();-}--export function wrap(- controller: (req: Request, res: Response) => Promise<any>,-): (req: Request, res: Response, next: NextFunction) => any {- return (req, res, next) => {- controller(req, res).catch((err) => next(err));- };-}-expressWs(express());export const v1Router = express.Router();diff --cc apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex 4ea598526,2c1007d6c..e0e31c8cb--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@@ -5,7 -5,7 +5,8 @@@ import { SSLError, TimeoutError } fromimport { specialtyScrapeCheck } from "../utils/specialtyHandler";import {InsecureConnectionError,- makeSecureDispatcher,+ secureDispatcher,++ secureDispatcherSkipTlsVerification,} from "../utils/safeFetch";import { MockState, saveMock } from "../../lib/mock";import { TextDecoder } from "util";@@@ -56,7 -56,7 +57,7 @@@ export async function scrapeURLWithFetctry {const x = await Promise.race([undici.fetch(meta.rewrittenUrl ?? meta.url, {- dispatcher: await makeSecureDispatcher(meta.rewrittenUrl ?? meta.url, meta.options.skipTlsVerification),- dispatcher: secureDispatcher,++ dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,redirect: "follow",headers: meta.options.headers,signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),diff --cc apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex afaac24ae,e4e778999..0bb48c3c0--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@@ -2,7 -2,7 +2,6 @@@ import type { Socket } from "net"import type { TLSSocket } from "tls";import * as undici from "undici";import { Address6 } from "ip-address";--import { cacheableLookup } from "../../lib/cacheableLookup";import { CookieJar } from "tough-cookie";import { cookie } from "http-cookie-agent/undici";@@@ -42,14 -42,13 +41,11 @@@ function isIPv6Private(ipv6)return new Address6(ipv6).getScope() !== "Global";}-function makeSecureDispatcher() {+export function makeSecureDispatcher(- url: string,+ skipTlsVerification: boolean = false,- options?: undici.Agent.Options,+) {const agentOpts: undici.Agent.Options = {- connect: {- rejectUnauthorized: false, // bypass SSL failures -- this is fine- lookup: cacheableLookup.lookup,- },maxRedirections: 5000,- ...options,};const baseAgent = process.env.PROXY_SERVER@@@ -95,3 -86,5 +91,6 @@@return agent;}+-export const secureDispatcher = makeSecureDispatcher();++export const secureDispatcher = makeSecureDispatcher(false);++export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);diff --cc apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex ff7071bbc,05f3a319f..71a1215a3--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@@ -714,8 -786,12 +809,12 @@@ export async function performLLMExtractawait extractData({extractOptions: generationOptions,urls: [meta.rewrittenUrl ?? meta.url],- useAgent: false,+ useAgent: isAgentExtractModelValid(meta.internalOptions.v1JSONAgent?.model),scrapeId: meta.id,+ metadata: {+ teamId: meta.internalOptions.teamId,+ functionId: "performLLMExtract",+ },});if (warning) {diff --cc apps/api/src/services/queue-worker.tsindex 5c9003698,28fd8a25b..b8c11e85f--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@@ -557,5 -563,8 +590,8 @@@ app.listen(workerPort, () =>await scrapeQueueEvents.close();console.log("All jobs finished. Worker out!");+ if (langfuseOtel) {+ await langfuseOtel.shutdown();+ }process.exit(0);-})();+})();