Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge branch 'main' into nsc/v2

Gergő Móricz21cbfec39857
diff --cc apps/api/pnpm-lock.yaml
index 8a52f24fa,1c562dc4f..fd41ec068
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@@ -2111,6 -3115,16 +3118,12 @@@ packages
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
- resolution: {integrity: sha512-vjtV3hiLqYDNRoiAv0zC4QaGAMPomEoq83PRmYIofPswwZurCeWR5LByXm7SyoL0Zh5+2z0+HC7jG8gSZJUh0w==}
- engines: {node: '>= 16'}
-
+ resolution: {integrity: sha512-xz7wQ8xDhdyP7tQxwdteLYeFfS68tSMNCZ/Y37WJ4bhGfKPpqEIlmIyueQHqOyoPhE6xNUqjzRr8ra0eF9VRvA==}
+
+ resolution: {integrity: sha512-M7uio8Zt++eg3jPj+rHMfCC+IuygQHHCOU+IYsVtik6FWjuYpVt/+MRKcgsAMHh8mMFAwnB+Bs+mTrFiXjMzKg==}
+
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
@@@ -3825,9 -5243,16 +5241,19 @@@
resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==}
engines: {node: '>=0.6'}
+ resolution: {integrity: sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==}
+ engines: {node: '>=6'}
+
+ resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==}
+
+ resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}
+
+ resolution: {integrity: sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==}
+
resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
engines: {node: '>= 0.6'}
@@@ -3900,9 -5333,9 +5334,12 @@@
resolution: {integrity: sha512-nQFEv9gRw6SJAwWD2LrL0NmQvAcO7FBwJbwmr2ttPAacfy0xuiOjE5zt+zM4xDyuyvUaxBi/9gb2SoCyNEVJcw==}
engines: {node: '>=8.6.0'}
+ resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==}
+
+ resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==}
+
resolution: {integrity: sha512-F3PVHdTHeLonSnrU5V6k8643LJ9QacLu3uI9M+BAFkmBmB1ELM2x7fdsziYZoSm6DmU6TKwiQCK0jf8dcNomcQ==}
engines: {node: '>=18'}
@@@ -4371,9 -5933,16 +5945,19 @@@
resolution: {integrity: sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==}
+ resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==}
+
+ resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==}
+ peerDependencies:
+ react: ^16.8.0 || ^17.0.0 || ^18.0.0
+
+ resolution: {integrity: sha512-9WXSPC5fMv61vaupRkCKCxsPxBocVnwakBEkMIHHpkTTg6icbJtg6jzgtLDm4bl3cSHAca52rYWih0k4K3PfHw==}
+ peerDependencies:
+ react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
+
resolution: {integrity: sha512-Rb46I4cGGVBmjamjphe8L/UnvJD+uPPtTkNvX5mZgqdbavhI4EbgIWJiIHXJ8bc/i9EQGPRh4DwEURJ552Do0A==}
peerDependencies:
@@@ -6985,6 -10814,10 +10829,10 @@@ snapshots
+ dependencies:
- tslib: 2.6.3
++ tslib: 2.8.1
+
dependencies:
tslib: 2.6.3
@@@ -7064,8 -10903,22 +10918,20 @@@
-
+ dependencies:
+ safe-buffer: 5.2.1
+
+
+
+ dependencies:
+ bindings: 1.5.0
+
@@@ -9036,8 -13368,17 +13381,19 @@@
dependencies:
side-channel: 1.1.0
+ dependencies:
+ decode-uri-component: 0.2.2
+ filter-obj: 1.1.0
+ split-on-first: 1.1.0
+ strict-uri-encode: 2.0.0
+
+
+
+
@@@ -9121,8 -13466,8 +13481,10 @@@
transitivePeerDependencies:
- supports-color
+
+
dependencies:
'@react-email/render': 0.0.15
@@@ -9578,10 -14009,22 +14033,24 @@@
+
+ dependencies:
+ anymatch: 3.1.3
+ chokidar: 4.0.3
+ destr: 2.0.5
+ h3: 1.15.4
+ lru-cache: 10.4.3
+ node-fetch-native: 1.6.7
+ ofetch: 1.4.1
+ ufo: 1.6.1
+ optionalDependencies:
+ idb-keyval: 6.2.2
+ ioredis: 5.6.1
+
dependencies:
async: 3.2.5
@@@ -9600,11 -14043,14 +14069,19 @@@
+ dependencies:
+ querystringify: 2.2.0
+ requires-port: 1.0.0
+
+ dependencies:
+ react: 18.3.1
+
+ dependencies:
+ react: 18.3.1
+
dependencies:
react: 18.3.1
diff --cc apps/api/src/controllers/v1/search.ts
index 94a530ab0,d4429e15b..c6c714288
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@@ -114,10 -110,10 +114,10 @@@ async function scrapeSearchResult
mode: "single_urls" as Mode,
team_id: options.teamId,
scrapeOptions: {
- ...options.scrapeOptions,
- maxAge: options.scrapeOptions.maxAge === 0 ? 3 * 24 * 60 * 60 * 1000 : options.scrapeOptions.maxAge,
+ ...scrapeOptions,
- maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days
++ maxAge: scrapeOptions.maxAge === 0 ? 3 * 24 * 60 * 60 * 1000 : scrapeOptions.maxAge,
},
- internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },
+ internalOptions: { ...internalOptions, teamId: options.teamId, bypassBilling: true, zeroDataRetention },
origin: options.origin,
is_scrape: true,
startTime: Date.now(),
diff --cc apps/api/src/index.ts
index 1b22c71c2,3ff680558..9ebf1a04f
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@@ -26,8 -26,10 +26,11 @@@ import { v4 as uuidv4 } from "uuid"
import { RateLimiterMode } from "./types";
import { attachWsProxy } from "./services/agentLivecastWS";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
+import { v2Router } from "./routes/v2";
import domainFrequencyRouter from "./routes/domain-frequency";
+ import { NodeSDK } from "@opentelemetry/sdk-node";
+ import { LangfuseExporter } from "langfuse-vercel";
+ import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter');
diff --cc apps/api/src/lib/crawl-redis.ts
index 8a806d2f0,65bd96878..2647c488b
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@@ -1,8 -1,8 +1,8 @@@
import { InternalOptions } from "../scraper/scrapeURL";
-import { ScrapeOptions, TeamFlags } from "../controllers/v1/types";
+import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";
import { WebCrawler } from "../scraper/WebScraper/crawler";
import { redisEvictConnection } from "../services/redis";
- import { logger as _logger } from "./logger";
+ import { logger as _logger, logger } from "./logger";
import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";
import type { Logger } from "winston";
diff --cc apps/api/src/routes/v1.ts
index 583ba4a5f,a5165c927..3b3dd80c6
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@@ -15,7 -27,9 +15,8 @@@ import { batchScrapeController } from "
import { extractController } from "../controllers/v1/extract";
import { extractStatusController } from "../controllers/v1/extract-status";
import { creditUsageController } from "../controllers/v1/credit-usage";
-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
import { searchController } from "../controllers/v1/search";
+ import { x402SearchController } from "../controllers/v1/x402-search";
import { crawlErrorsController } from "../controllers/v1/crawl-errors";
import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";
import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";
@@@ -23,14 -37,154 +24,15 @@@ import { deepResearchController } from
import { deepResearchStatusController } from "../controllers/v1/deep-research-status";
import { tokenUsageController } from "../controllers/v1/token-usage";
import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";
-import { addDomainFrequencyJob } from "../services"
+import {
+ authMiddleware,
+ checkCreditsMiddleware,
+ blocklistMiddleware,
+ idempotencyMiddleware,
+ wrap,
+} from "./shared";
+ import { paymentMiddleware } from "x402-express";
-function checkCreditsMiddleware(
- _minimum?: number,
-): (req: RequestWithAuth, res: Response, next: NextFunction) => void {
- return (req, res, next) => {
- let minimum = _minimum;
- (async () => {
- if (!minimum && req.body) {
- minimum =
- (req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1;
- }
- const { success, remainingCredits, chunk } = await checkTeamCredits(
- req.acuc,
- req.auth.team_id,
- minimum ?? 1,
- );
- if (chunk) {
- req.acuc = chunk;
- }
- req.account = { remainingCredits };
- if (!success) {
- if (!minimum && req.body && (req.body as any).limit !== undefined && remainingCredits > 0) {
- logger.warn("Adjusting limit to remaining credits", {
- teamId: req.auth.team_id,
- remainingCredits,
- request: req.body,
- });
- (req.body as any).limit = remainingCredits;
- return next();
- }
-
- const currencyName = req.acuc.is_extract ? "tokens" : "credits"
- logger.error(
- `Insufficient ${currencyName}: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`,
- {
- teamId: req.auth.team_id,
- minimum,
- remainingCredits,
- request: req.body,
- path: req.path
- }
- );
- if (!res.headersSent && req.auth.team_id !== "8c528896-7882-4587-a4b6-768b721b0b53") {
- return res.status(402).json({
- success: false,
- error:
- "Insufficient " + currencyName + " to perform this request. For more " + currencyName + ", you can upgrade your plan at " + (currencyName === "credits" ? "https://firecrawl.dev/pricing or try changing the request limit to a lower value" : "https://www.firecrawl.dev/extract#pricing") + ".",
- });
- }
- }
- next();
- })().catch((err) => next(err));
- };
-}
-
-export function authMiddleware(
- rateLimiterMode: RateLimiterMode
-): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {
- return (req, res, next) => {
- (async () => {
- let currentRateLimiterMode = rateLimiterMode;
- if (currentRateLimiterMode === RateLimiterMode.Extract && isAgentExtractModelValid((req.body as any)?.agent?.model)) {
- currentRateLimiterMode = RateLimiterMode.ExtractAgentPreview;
- }
-
- // Track domain frequency regardless of caching
- try {
- // Use the URL from the request body if available
- const urlToTrack = (req.body as any)?.url;
- if (urlToTrack) {
- await addDomainFrequencyJob(urlToTrack);
- }
- } catch (error) {
- // Log error without meta.logger since it's not available in this context
- logger.warn("Failed to track domain frequency", { error });
- }
-
- // if (currentRateLimiterMode === RateLimiterMode.Scrape && isAgentExtractModelValid((req.body as any)?.agent?.model)) {
- // currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;
- // }
-
- const auth = await authenticateUser(req, res, currentRateLimiterMode);
-
- if (!auth.success) {
- if (!res.headersSent) {
- return res
- .status(auth.status)
- .json({ success: false, error: auth.error });
- } else {
- return;
- }
- }
-
- const { team_id, chunk } = auth;
-
- req.auth = { team_id };
- req.acuc = chunk ?? undefined;
- if (chunk) {
- req.account = { remainingCredits: chunk.remaining_credits };
- }
- next();
- })().catch((err) => next(err));
- };
-}
-
-function idempotencyMiddleware(
- req: Request,
- res: Response,
- next: NextFunction,
-) {
- (async () => {
- if (req.headers["x-idempotency-key"]) {
- const isIdempotencyValid = await validateIdempotencyKey(req);
- if (!isIdempotencyValid) {
- if (!res.headersSent) {
- return res
- .status(409)
- .json({ success: false, error: "Idempotency key already used" });
- }
- }
- createIdempotencyKey(req);
- }
- next();
- })().catch((err) => next(err));
-}
-
-function blocklistMiddleware(req: RequestWithACUC<any, any, any>, res: Response, next: NextFunction) {
- if (typeof req.body.url === "string" && isUrlBlocked(req.body.url, req.acuc?.flags ?? null)) {
- if (!res.headersSent) {
- return res.status(403).json({
- success: false,
- error: BLOCKLISTED_URL_MESSAGE,
- });
- }
- }
- next();
-}
-
-export function wrap(
- controller: (req: Request, res: Response) => Promise<any>,
-): (req: Request, res: Response, next: NextFunction) => any {
- return (req, res, next) => {
- controller(req, res).catch((err) => next(err));
- };
-}
-
expressWs(express());
export const v1Router = express.Router();
diff --cc apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index 4ea598526,2c1007d6c..e0e31c8cb
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@@ -5,7 -5,7 +5,8 @@@ import { SSLError, TimeoutError } from
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import {
InsecureConnectionError,
- makeSecureDispatcher,
+ secureDispatcher,
++ secureDispatcherSkipTlsVerification,
} from "../utils/safeFetch";
import { MockState, saveMock } from "../../lib/mock";
import { TextDecoder } from "util";
@@@ -56,7 -56,7 +57,7 @@@ export async function scrapeURLWithFetc
try {
const x = await Promise.race([
undici.fetch(meta.rewrittenUrl ?? meta.url, {
- dispatcher: await makeSecureDispatcher(meta.rewrittenUrl ?? meta.url, meta.options.skipTlsVerification),
- dispatcher: secureDispatcher,
++ dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,
redirect: "follow",
headers: meta.options.headers,
signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
diff --cc apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index afaac24ae,e4e778999..0bb48c3c0
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@@ -2,7 -2,7 +2,6 @@@ import type { Socket } from "net"
import type { TLSSocket } from "tls";
import * as undici from "undici";
import { Address6 } from "ip-address";
--import { cacheableLookup } from "../../lib/cacheableLookup";
import { CookieJar } from "tough-cookie";
import { cookie } from "http-cookie-agent/undici";
@@@ -42,14 -42,13 +41,11 @@@ function isIPv6Private(ipv6)
return new Address6(ipv6).getScope() !== "Global";
}
-function makeSecureDispatcher() {
+export function makeSecureDispatcher(
- url: string,
+ skipTlsVerification: boolean = false,
- options?: undici.Agent.Options,
+) {
const agentOpts: undici.Agent.Options = {
- connect: {
- rejectUnauthorized: false, // bypass SSL failures -- this is fine
- lookup: cacheableLookup.lookup,
- },
maxRedirections: 5000,
- ...options,
};
const baseAgent = process.env.PROXY_SERVER
@@@ -95,3 -86,5 +91,6 @@@
return agent;
}
+
-export const secureDispatcher = makeSecureDispatcher();
++export const secureDispatcher = makeSecureDispatcher(false);
++export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);
diff --cc apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index ff7071bbc,05f3a319f..71a1215a3
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@@ -714,8 -786,12 +809,12 @@@ export async function performLLMExtract
await extractData({
extractOptions: generationOptions,
urls: [meta.rewrittenUrl ?? meta.url],
- useAgent: false,
+ useAgent: isAgentExtractModelValid(meta.internalOptions.v1JSONAgent?.model),
scrapeId: meta.id,
+ metadata: {
+ teamId: meta.internalOptions.teamId,
+ functionId: "performLLMExtract",
+ },
});
if (warning) {
diff --cc apps/api/src/services/queue-worker.ts
index 5c9003698,28fd8a25b..b8c11e85f
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@@ -557,5 -563,8 +590,8 @@@ app.listen(workerPort, () =>
await scrapeQueueEvents.close();
console.log("All jobs finished. Worker out!");
+ if (langfuseOtel) {
+ await langfuseOtel.shutdown();
+ }
process.exit(0);
-})();
+})();