Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Refactor API endpoints and update pricing
AdemÃlson Tonatofc080ea98203
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex f560476e5..fcb564c50 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -93,5 +93,5 @@ CDP_API_KEY_ID=""CDP_API_KEY_SECRET=""# ^ KEYS taken from -> https://portal.cdp.coinbase.com/projects/api-keys-## ENDPOINT PRICING TEST-X402_ENDPOINT_PRICE_USD=$0.001\ No newline at end of file+## ENDPOINT PRICING TEST $0.01+X402_ENDPOINT_PRICE_USD=0.01\ No newline at end of filediff --git a/apps/api/requests.http b/apps/api/requests.httpindex fd53ee545..6fd622268 100644--- a/apps/api/requests.http+++ b/apps/api/requests.http@@ -132,32 +132,6 @@ content-type: application/jsonGET {{baseUrl}}/v1/crawl/active HTTP/1.1Authorization: Bearer {{$dotenv TEST_API_KEY}}---### Generate Payment Header for Protected Data-# Generate a payment header for accessing protected data-# Duration options: 1h, 1d, 1w-# @name generatePaymentHeaderProtected-POST {{baseUrl}}/v1/generate-payment-header HTTP/1.1-Authorization: Bearer {{$dotenv TEST_API_KEY}}-content-type: application/json--{- "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY",- "duration": "1h",- "resource": "protected-data"-}--### Test Protected Data with Generated Payment Header-# Use the payment header from the previous request-@paymentHeaderProtected = {{generatePaymentHeaderProtected.response.body.data.paymentHeader}}-# @name testProtectedData-GET {{baseUrl}}/v1/protected-data HTTP/1.1-Accept: application/json-Authorization: Bearer {{$dotenv TEST_API_KEY}}-X-Payment: {{paymentHeaderProtected}}--### Generate Payment Header for Search with Wallet# Generate a payment header for search with wallet# Duration options: 1h, 1d, 1w@@ -169,18 +143,18 @@ content-type: application/json{"privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY","duration": "1h",- "resource": "search-with-wallet"+ "resource": "x402/search"}### Search with Wallet@paymentHeaderSearch = {{generatePaymentHeaderSearch.response.body.data.paymentHeader}}# @name searchWithWallet-POST {{baseUrl}}/v1/search-with-wallet HTTP/1.1+POST {{baseUrl}}/v1/x402/search HTTP/1.1Authorization: Bearer {{$dotenv TEST_API_KEY}}content-type: application/jsonX-Payment: {{paymentHeaderSearch}}{- "query": "firecrawl",+ "query": "Premier League fixtures for the 2025/26 season","limit": 2}\ No newline at end of filediff --git a/apps/api/src/controllers/v1/generate-payment-header.ts b/apps/api/src/controllers/v1/generate-payment-header.tsindex 325bf20bd..14e736d29 100644--- a/apps/api/src/controllers/v1/generate-payment-header.ts+++ b/apps/api/src/controllers/v1/generate-payment-header.ts@@ -139,20 +139,13 @@ export async function generatePaymentHeaderController(req: RequestWithAuth<{}, Gconst baseUrl = `${protocol}://${host}`;switch (resource) {- case 'search-with-wallet':- return {- ...baseRequirements,- maxAmountRequired: "10000", // $0.001 in USDC base units - matches X402_SEARCH_WALLET_PRICE_USD- resource: `${baseUrl}/v1/search-with-wallet`,- description: "Search with wallet payment API",- };- case 'protected-data':+ case 'x402/search':default:return {...baseRequirements,- maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD- resource: `${baseUrl}/v1/protected-data`,- description: "Access to protected data API",+ maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD ("20000" => $0.02)+ resource: `${baseUrl}/v1/x402/search`,+ description: "x402 search payment API",};}};@@ -194,7 +187,7 @@ export async function generatePaymentHeaderController(req: RequestWithAuth<{}, Gnotes: {timeSensitive: `This payment header is time-sensitive (valid for ${duration})`,walletRequirement: "Make sure your wallet has USDC on Base Sepolia",- paymentAmount: resource === 'search-with-wallet' ? "1000 USDC (in base units = $0.001)" : "10000 USDC (in base units = $0.01)",+ paymentAmount: "10000 USDC (in base units = $0.01)", // 20000 USDC (in base units = $0.02)generatedAt: new Date().toISOString(),team_id: req.auth?.team_id || req.acuc?.team_id}diff --git a/apps/api/src/controllers/v1/protected-data.ts b/apps/api/src/controllers/v1/protected-data.tsdeleted file mode 100644index ed60698a5..000000000--- a/apps/api/src/controllers/v1/protected-data.ts+++ /dev/null@@ -1,27 +0,0 @@-import { Request, Response } from "express";-import { RequestWithAuth } from "./types";--export async function protectedDataController(- req: RequestWithAuth<{}, Response, any>,- res: Response<any>,-) {- res.json({- request: {- headers: req.headers,- body: req.body,- },- success: true,- data: {- created_at: new Date().toISOString(),- person: {- name: "John Doe",- age: 30,- email: "[email protected]",- },- company: {- name: "Example Inc.",- industry: "Technology",- },- },- });-}diff --git a/apps/api/src/controllers/v1/search-with-wallet.ts b/apps/api/src/controllers/v1/x402-search.tssimilarity index 69%rename from apps/api/src/controllers/v1/search-with-wallet.tsrename to apps/api/src/controllers/v1/x402-search.tsindex a1ff7b783..58349af64 100644--- a/apps/api/src/controllers/v1/search-with-wallet.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -10,12 +10,16 @@ import {} from "./types";import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";+import { logJob } from "../../services/logging/log_job";import { getJobPriority } from "../../lib/job-priority";import { Mode } from "../../types";import { getScrapeQueue } from "../../services/queue-service";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";+import { logger as _logger } from "../../lib/logger";+import type { Logger } from "winston";import { CostTracking } from "../../lib/extract/extraction-service";import { supabase_service } from "../../services/supabase";@@ -25,7 +29,7 @@ interface DocumentWithCostTracking {}// Used for deep research-export async function searchAndScrapeSearchResultWithWallet(+export async function searchAndScrapeX402SearchResult(query: string,options: {teamId: string;@@ -33,6 +37,7 @@ export async function searchAndScrapeSearchResultWithWallet(timeout: number;scrapeOptions: ScrapeOptions;},+ logger: Logger,flags: TeamFlags,): Promise<DocumentWithCostTracking[]> {try {@@ -43,13 +48,14 @@ export async function searchAndScrapeSearchResultWithWallet(const documentsWithCostTracking = await Promise.all(searchResults.map((result) =>- scrapeSearchResultWithWallet(+ scrapeX402SearchResult({url: result.url,title: result.title,description: result.description,},options,+ logger,flags,),),@@ -61,7 +67,7 @@ export async function searchAndScrapeSearchResultWithWallet(}}-async function scrapeSearchResultWithWallet(+async function scrapeX402SearchResult(searchResult: { url: string; title: string; description: string },options: {teamId: string;@@ -69,6 +75,7 @@ async function scrapeSearchResultWithWallet(timeout: number;scrapeOptions: ScrapeOptions;},+ logger: Logger,flags: TeamFlags,directToBullMQ: boolean = false,isSearchPreview: boolean = false,@@ -87,6 +94,13 @@ async function scrapeSearchResultWithWallet(if (isUrlBlocked(searchResult.url, flags)) {throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE);}+ logger.info("Adding scrape job [x402]", {+ scrapeId: jobId,+ url: searchResult.url,+ teamId: options.teamId,+ origin: options.origin,+ zeroDataRetention,+ });await addScrapeJob({url: searchResult.url,@@ -110,6 +124,12 @@ async function scrapeSearchResultWithWallet(const doc: Document = await waitForJob(jobId, options.timeout);+ logger.info("Scrape job [x402] completed", {+ scrapeId: jobId,+ url: searchResult.url,+ teamId: options.teamId,+ origin: options.origin,+ });await getScrapeQueue().remove(jobId);const document = {@@ -139,6 +159,11 @@ async function scrapeSearchResultWithWallet(costTracking,};} catch (error) {+ logger.error(`Error in scrapeSearchResult [x402]: ${error}`, {+ scrapeId: jobId,+ url: searchResult.url,+ teamId: options.teamId,+ });let statusCode = 0;if (error?.message?.includes("Could not scrape url")) {@@ -163,29 +188,51 @@ async function scrapeSearchResultWithWallet(}}-export async function searchWithWalletController(+export async function x402SearchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,res: Response<SearchResponse & { request?: any; }>,) {+ const jobId = uuidv4();+ let logger = _logger.child({+ jobId,+ teamId: req.auth.team_id,+ module: "x402-search",+ method: "x402SearchController",+ zeroDataRetention: req.acuc?.flags?.forceZDR,+ });+if (req.acuc?.flags?.forceZDR) {- return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search-with-wallet. Please contact [email protected] to unblock this feature." });+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on x402/search. Please contact [email protected] to unblock this feature." });}-+let responseData: SearchResponse = {success: true,data: [],};const startTime = new Date().getTime();const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;-+try {req.body = searchRequestSchema.parse(req.body);++ // IMPORTANT NOTE: Force results to be at most 10 even if a larger limit is requested+ const MAX_RESULTS = 10;+ if (req.body.limit > MAX_RESULTS) {+ req.body.limit = MAX_RESULTS;+ }++ logger = logger.child({+ query: req.body.query,+ origin: req.body.origin,+ });let limit = req.body.limit;// Buffer results by 50% to account for filtered URLsconst num_results_buffer = Math.floor(limit * 2);+ logger.info("Searching [x402] for results");+let searchResults = await search({query: req.body.query,advanced: false,@@ -203,12 +250,17 @@ export async function searchWithWalletController();}+ logger.info("Searching [x402] completed", {+ num_results: searchResults.length,+ });+// Filter blocked URLs early to avoid unnecessary billingif (searchResults.length > limit) {searchResults = searchResults.slice(0, limit);}if (searchResults.length === 0) {+ logger.info("No search [x402] results found");responseData.warning = "No search results found";} else if (!req.body.scrapeOptions.formats ||@@ -220,8 +272,9 @@ export async function searchWithWalletController(description: r.description,})) as Document[];} else {+ logger.info("Scraping search [x402] results");const scrapePromises = searchResults.map((result) =>- scrapeSearchResultWithWallet(+ scrapeX402SearchResult(result,{teamId: req.auth.team_id,@@ -229,6 +282,7 @@ export async function searchWithWalletController(timeout: req.body.timeout,scrapeOptions: req.body.scrapeOptions,},+ logger,req.acuc?.flags ?? null,(req.acuc?.price_credits ?? 0) <= 3000,isSearchPreview,@@ -236,6 +290,9 @@ export async function searchWithWalletController();const docsWithCostTracking = await Promise.all(scrapePromises);+ logger.info("Scraping [x402] completed", {+ num_docs: docsWithCostTracking.length,+ });const docs = docsWithCostTracking.map(item => item.document);const filteredDocs = docs.filter(@@ -243,6 +300,10 @@ export async function searchWithWalletController(doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),);+ logger.info("Filtering [x402] completed", {+ num_docs: filteredDocs.length,+ });+if (filteredDocs.length === 0) {responseData.data = docs;responseData.warning = "No content found in search results";@@ -251,6 +312,39 @@ export async function searchWithWalletController(}}+ const endTime = new Date().getTime();+ const timeTakenInSeconds = (endTime - startTime) / 1000;++ logger.info("Logging job", {+ num_docs: responseData.data.length,+ time_taken: timeTakenInSeconds,+ });++ logJob(+ {+ job_id: jobId,+ success: true,+ num_docs: responseData.data.length,+ docs: responseData.data,+ time_taken: timeTakenInSeconds,+ team_id: req.auth.team_id,+ mode: "x402-search",+ url: req.body.query,+ scrapeOptions: req.body.scrapeOptions,+ crawlerOptions: {+ ...req.body,+ query: undefined,+ scrapeOptions: undefined,+ },+ origin: req.body.origin,+ integration: req.body.integration,+ credits_billed: 0,+ zeroDataRetention: false, // not supported+ },+ false,+ isSearchPreview,+ );+return res.status(200).json(responseData);} catch (error) {if (@@ -263,6 +357,8 @@ export async function searchWithWalletController(});}+ Sentry.captureException(error);+ logger.error("Unhandled error occurred in search [x402]", { error });return res.status(500).json({success: false,error: error.message,diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex f5e2f5f47..f1c4142d9 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -29,7 +29,7 @@ import { extractStatusController } from "../controllers/v1/extract-status";import { creditUsageController } from "../controllers/v1/credit-usage";import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";import { searchController } from "../controllers/v1/search";-import { searchWithWalletController } from "../controllers/v1/search-with-wallet";+import { x402SearchController } from "../controllers/v1/x402-search";import { crawlErrorsController } from "../controllers/v1/crawl-errors";import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";@@ -38,7 +38,6 @@ import { deepResearchStatusController } from "../controllers/v1/deep-research-stimport { tokenUsageController } from "../controllers/v1/token-usage";import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";import { generatePaymentHeaderController } from "../controllers/v1/generate-payment-header";-import { protectedDataController } from '../controllers/v1/protected-data';import { paymentMiddleware } from "x402-express";function checkCreditsMiddleware(@@ -178,28 +177,20 @@ expressWs(express());export const v1Router = express.Router();-// First, configure the payment middleware with your routes+// Configure payment middleware to enable micropayment-protected endpoints+// This middleware handles payment verification and processing for premium API features+// x402 payments protocol - https://github.com/coinbase/x402v1Router.use(paymentMiddleware(process.env.X402_PAY_TO_ADDRESS as `0x${string}` || "0x0000000000000000000000000000000000000000",{- // Define your routes and their payment requirements- "GET /protected-data": {- price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",- network: "base-sepolia",+ "POST /x402/search": {+ price: process.env.X402_ENDPOINT_PRICE_USD as string,+ network: process.env.X402_NETWORK as "base-sepolia" | "base" | "avalanche-fuji" | "avalanche" | "iotex",config: {- description: "Access premium protected data API",+ description: "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",mimeType: "application/json",- maxTimeoutSeconds: 300,- }- },- "POST /search-with-wallet": {- price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",- network: "base-sepolia",- config: {- description: "Search with wallet payment API",- mimeType: "application/json",- maxTimeoutSeconds: 300,+ maxTimeoutSeconds: 120,}},},@@ -381,13 +372,8 @@ v1Router.post(wrap(generatePaymentHeaderController),);-v1Router.get("/protected-data",- authMiddleware(RateLimiterMode.Search),- wrap(protectedDataController),-);-v1Router.post(- "/search-with-wallet",+ "/x402/search",authMiddleware(RateLimiterMode.Search),- wrap(searchWithWalletController),+ wrap(x402SearchController),);\ No newline at end of file