Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Refactor API endpoints and update pricing

Ademílson Tonatofc080ea98203
diff --git a/apps/api/.env.example b/apps/api/.env.example
index f560476e5..fcb564c50 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -93,5 +93,5 @@ CDP_API_KEY_ID=""
CDP_API_KEY_SECRET=""
# ^ KEYS taken from -> https://portal.cdp.coinbase.com/projects/api-keys
-## ENDPOINT PRICING TEST
-X402_ENDPOINT_PRICE_USD=$0.001
\ No newline at end of file
+## ENDPOINT PRICING TEST $0.01
+X402_ENDPOINT_PRICE_USD=0.01
\ No newline at end of file
diff --git a/apps/api/requests.http b/apps/api/requests.http
index fd53ee545..6fd622268 100644
--- a/apps/api/requests.http
+++ b/apps/api/requests.http
@@ -132,32 +132,6 @@ content-type: application/json
GET {{baseUrl}}/v1/crawl/active HTTP/1.1
Authorization: Bearer {{$dotenv TEST_API_KEY}}
-
-
-### Generate Payment Header for Protected Data
-# Generate a payment header for accessing protected data
-# Duration options: 1h, 1d, 1w
-# @name generatePaymentHeaderProtected
-POST {{baseUrl}}/v1/generate-payment-header HTTP/1.1
-Authorization: Bearer {{$dotenv TEST_API_KEY}}
-content-type: application/json
-
-{
- "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY",
- "duration": "1h",
- "resource": "protected-data"
-}
-
-### Test Protected Data with Generated Payment Header
-# Use the payment header from the previous request
-@paymentHeaderProtected = {{generatePaymentHeaderProtected.response.body.data.paymentHeader}}
-# @name testProtectedData
-GET {{baseUrl}}/v1/protected-data HTTP/1.1
-Accept: application/json
-Authorization: Bearer {{$dotenv TEST_API_KEY}}
-X-Payment: {{paymentHeaderProtected}}
-
-
### Generate Payment Header for Search with Wallet
# Generate a payment header for search with wallet
# Duration options: 1h, 1d, 1w
@@ -169,18 +143,18 @@ content-type: application/json
{
"privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY",
"duration": "1h",
- "resource": "search-with-wallet"
+ "resource": "x402/search"
}
### Search with Wallet
@paymentHeaderSearch = {{generatePaymentHeaderSearch.response.body.data.paymentHeader}}
# @name searchWithWallet
-POST {{baseUrl}}/v1/search-with-wallet HTTP/1.1
+POST {{baseUrl}}/v1/x402/search HTTP/1.1
Authorization: Bearer {{$dotenv TEST_API_KEY}}
content-type: application/json
X-Payment: {{paymentHeaderSearch}}
{
- "query": "firecrawl",
+ "query": "Premier League fixtures for the 2025/26 season",
"limit": 2
}
\ No newline at end of file
diff --git a/apps/api/src/controllers/v1/generate-payment-header.ts b/apps/api/src/controllers/v1/generate-payment-header.ts
index 325bf20bd..14e736d29 100644
--- a/apps/api/src/controllers/v1/generate-payment-header.ts
+++ b/apps/api/src/controllers/v1/generate-payment-header.ts
@@ -139,20 +139,13 @@ export async function generatePaymentHeaderController(req: RequestWithAuth<{}, G
const baseUrl = `${protocol}://${host}`;
switch (resource) {
- case 'search-with-wallet':
- return {
- ...baseRequirements,
- maxAmountRequired: "10000", // $0.001 in USDC base units - matches X402_SEARCH_WALLET_PRICE_USD
- resource: `${baseUrl}/v1/search-with-wallet`,
- description: "Search with wallet payment API",
- };
- case 'protected-data':
+ case 'x402/search':
default:
return {
...baseRequirements,
- maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD
- resource: `${baseUrl}/v1/protected-data`,
- description: "Access to protected data API",
+ maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD ("20000" => $0.02)
+ resource: `${baseUrl}/v1/x402/search`,
+ description: "x402 search payment API",
};
}
};
@@ -194,7 +187,7 @@ export async function generatePaymentHeaderController(req: RequestWithAuth<{}, G
notes: {
timeSensitive: `This payment header is time-sensitive (valid for ${duration})`,
walletRequirement: "Make sure your wallet has USDC on Base Sepolia",
- paymentAmount: resource === 'search-with-wallet' ? "1000 USDC (in base units = $0.001)" : "10000 USDC (in base units = $0.01)",
+ paymentAmount: "10000 USDC (in base units = $0.01)", // 20000 USDC (in base units = $0.02)
generatedAt: new Date().toISOString(),
team_id: req.auth?.team_id || req.acuc?.team_id
}
diff --git a/apps/api/src/controllers/v1/protected-data.ts b/apps/api/src/controllers/v1/protected-data.ts
deleted file mode 100644
index ed60698a5..000000000
--- a/apps/api/src/controllers/v1/protected-data.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-import { Request, Response } from "express";
-import { RequestWithAuth } from "./types";
-
-export async function protectedDataController(
- req: RequestWithAuth<{}, Response, any>,
- res: Response<any>,
-) {
- res.json({
- request: {
- headers: req.headers,
- body: req.body,
- },
- success: true,
- data: {
- created_at: new Date().toISOString(),
- person: {
- name: "John Doe",
- age: 30,
- email: "[email protected]",
- },
- company: {
- name: "Example Inc.",
- industry: "Technology",
- },
- },
- });
-}
diff --git a/apps/api/src/controllers/v1/search-with-wallet.ts b/apps/api/src/controllers/v1/x402-search.ts
similarity index 69%
rename from apps/api/src/controllers/v1/search-with-wallet.ts
rename to apps/api/src/controllers/v1/x402-search.ts
index a1ff7b783..58349af64 100644
--- a/apps/api/src/controllers/v1/search-with-wallet.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -10,12 +10,16 @@ import {
} from "./types";
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
+import { logJob } from "../../services/logging/log_job";
import { getJobPriority } from "../../lib/job-priority";
import { Mode } from "../../types";
import { getScrapeQueue } from "../../services/queue-service";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { logger as _logger } from "../../lib/logger";
+import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
import { supabase_service } from "../../services/supabase";
@@ -25,7 +29,7 @@ interface DocumentWithCostTracking {
}
// Used for deep research
-export async function searchAndScrapeSearchResultWithWallet(
+export async function searchAndScrapeX402SearchResult(
query: string,
options: {
teamId: string;
@@ -33,6 +37,7 @@ export async function searchAndScrapeSearchResultWithWallet(
timeout: number;
scrapeOptions: ScrapeOptions;
},
+ logger: Logger,
flags: TeamFlags,
): Promise<DocumentWithCostTracking[]> {
try {
@@ -43,13 +48,14 @@ export async function searchAndScrapeSearchResultWithWallet(
const documentsWithCostTracking = await Promise.all(
searchResults.map((result) =>
- scrapeSearchResultWithWallet(
+ scrapeX402SearchResult(
{
url: result.url,
title: result.title,
description: result.description,
},
options,
+ logger,
flags,
),
),
@@ -61,7 +67,7 @@ export async function searchAndScrapeSearchResultWithWallet(
}
}
-async function scrapeSearchResultWithWallet(
+async function scrapeX402SearchResult(
searchResult: { url: string; title: string; description: string },
options: {
teamId: string;
@@ -69,6 +75,7 @@ async function scrapeSearchResultWithWallet(
timeout: number;
scrapeOptions: ScrapeOptions;
},
+ logger: Logger,
flags: TeamFlags,
directToBullMQ: boolean = false,
isSearchPreview: boolean = false,
@@ -87,6 +94,13 @@ async function scrapeSearchResultWithWallet(
if (isUrlBlocked(searchResult.url, flags)) {
throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE);
}
+ logger.info("Adding scrape job [x402]", {
+ scrapeId: jobId,
+ url: searchResult.url,
+ teamId: options.teamId,
+ origin: options.origin,
+ zeroDataRetention,
+ });
await addScrapeJob(
{
url: searchResult.url,
@@ -110,6 +124,12 @@ async function scrapeSearchResultWithWallet(
const doc: Document = await waitForJob(jobId, options.timeout);
+ logger.info("Scrape job [x402] completed", {
+ scrapeId: jobId,
+ url: searchResult.url,
+ teamId: options.teamId,
+ origin: options.origin,
+ });
await getScrapeQueue().remove(jobId);
const document = {
@@ -139,6 +159,11 @@ async function scrapeSearchResultWithWallet(
costTracking,
};
} catch (error) {
+ logger.error(`Error in scrapeSearchResult [x402]: ${error}`, {
+ scrapeId: jobId,
+ url: searchResult.url,
+ teamId: options.teamId,
+ });
let statusCode = 0;
if (error?.message?.includes("Could not scrape url")) {
@@ -163,29 +188,51 @@ async function scrapeSearchResultWithWallet(
}
}
-export async function searchWithWalletController(
+export async function x402SearchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
res: Response<SearchResponse & { request?: any; }>,
) {
+ const jobId = uuidv4();
+ let logger = _logger.child({
+ jobId,
+ teamId: req.auth.team_id,
+ module: "x402-search",
+ method: "x402SearchController",
+ zeroDataRetention: req.acuc?.flags?.forceZDR,
+ });
+
if (req.acuc?.flags?.forceZDR) {
- return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search-with-wallet. Please contact [email protected] to unblock this feature." });
+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on x402/search. Please contact [email protected] to unblock this feature." });
}
-
+
let responseData: SearchResponse = {
success: true,
data: [],
};
const startTime = new Date().getTime();
const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
-
+
try {
req.body = searchRequestSchema.parse(req.body);
+
+ // IMPORTANT NOTE: Force results to be at most 10 even if a larger limit is requested
+ const MAX_RESULTS = 10;
+ if (req.body.limit > MAX_RESULTS) {
+ req.body.limit = MAX_RESULTS;
+ }
+
+ logger = logger.child({
+ query: req.body.query,
+ origin: req.body.origin,
+ });
let limit = req.body.limit;
// Buffer results by 50% to account for filtered URLs
const num_results_buffer = Math.floor(limit * 2);
+ logger.info("Searching [x402] for results");
+
let searchResults = await search({
query: req.body.query,
advanced: false,
@@ -203,12 +250,17 @@ export async function searchWithWalletController(
);
}
+ logger.info("Searching [x402] completed", {
+ num_results: searchResults.length,
+ });
+
// Filter blocked URLs early to avoid unnecessary billing
if (searchResults.length > limit) {
searchResults = searchResults.slice(0, limit);
}
if (searchResults.length === 0) {
+ logger.info("No search [x402] results found");
responseData.warning = "No search results found";
} else if (
!req.body.scrapeOptions.formats ||
@@ -220,8 +272,9 @@ export async function searchWithWalletController(
description: r.description,
})) as Document[];
} else {
+ logger.info("Scraping search [x402] results");
const scrapePromises = searchResults.map((result) =>
- scrapeSearchResultWithWallet(
+ scrapeX402SearchResult(
result,
{
teamId: req.auth.team_id,
@@ -229,6 +282,7 @@ export async function searchWithWalletController(
timeout: req.body.timeout,
scrapeOptions: req.body.scrapeOptions,
},
+ logger,
req.acuc?.flags ?? null,
(req.acuc?.price_credits ?? 0) <= 3000,
isSearchPreview,
@@ -236,6 +290,9 @@ export async function searchWithWalletController(
);
const docsWithCostTracking = await Promise.all(scrapePromises);
+ logger.info("Scraping [x402] completed", {
+ num_docs: docsWithCostTracking.length,
+ });
const docs = docsWithCostTracking.map(item => item.document);
const filteredDocs = docs.filter(
@@ -243,6 +300,10 @@ export async function searchWithWalletController(
doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),
);
+ logger.info("Filtering [x402] completed", {
+ num_docs: filteredDocs.length,
+ });
+
if (filteredDocs.length === 0) {
responseData.data = docs;
responseData.warning = "No content found in search results";
@@ -251,6 +312,39 @@ export async function searchWithWalletController(
}
}
+ const endTime = new Date().getTime();
+ const timeTakenInSeconds = (endTime - startTime) / 1000;
+
+ logger.info("Logging job", {
+ num_docs: responseData.data.length,
+ time_taken: timeTakenInSeconds,
+ });
+
+ logJob(
+ {
+ job_id: jobId,
+ success: true,
+ num_docs: responseData.data.length,
+ docs: responseData.data,
+ time_taken: timeTakenInSeconds,
+ team_id: req.auth.team_id,
+ mode: "x402-search",
+ url: req.body.query,
+ scrapeOptions: req.body.scrapeOptions,
+ crawlerOptions: {
+ ...req.body,
+ query: undefined,
+ scrapeOptions: undefined,
+ },
+ origin: req.body.origin,
+ integration: req.body.integration,
+ credits_billed: 0,
+ zeroDataRetention: false, // not supported
+ },
+ false,
+ isSearchPreview,
+ );
+
return res.status(200).json(responseData);
} catch (error) {
if (
@@ -263,6 +357,8 @@ export async function searchWithWalletController(
});
}
+ Sentry.captureException(error);
+ logger.error("Unhandled error occurred in search [x402]", { error });
return res.status(500).json({
success: false,
error: error.message,
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index f5e2f5f47..f1c4142d9 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -29,7 +29,7 @@ import { extractStatusController } from "../controllers/v1/extract-status";
import { creditUsageController } from "../controllers/v1/credit-usage";
import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
import { searchController } from "../controllers/v1/search";
-import { searchWithWalletController } from "../controllers/v1/search-with-wallet";
+import { x402SearchController } from "../controllers/v1/x402-search";
import { crawlErrorsController } from "../controllers/v1/crawl-errors";
import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";
import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";
@@ -38,7 +38,6 @@ import { deepResearchStatusController } from "../controllers/v1/deep-research-st
import { tokenUsageController } from "../controllers/v1/token-usage";
import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";
import { generatePaymentHeaderController } from "../controllers/v1/generate-payment-header";
-import { protectedDataController } from '../controllers/v1/protected-data';
import { paymentMiddleware } from "x402-express";
function checkCreditsMiddleware(
@@ -178,28 +177,20 @@ expressWs(express());
export const v1Router = express.Router();
-// First, configure the payment middleware with your routes
+// Configure payment middleware to enable micropayment-protected endpoints
+// This middleware handles payment verification and processing for premium API features
+// x402 payments protocol - https://github.com/coinbase/x402
v1Router.use(
paymentMiddleware(
process.env.X402_PAY_TO_ADDRESS as `0x${string}` || "0x0000000000000000000000000000000000000000",
{
- // Define your routes and their payment requirements
- "GET /protected-data": {
- price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",
- network: "base-sepolia",
+ "POST /x402/search": {
+ price: process.env.X402_ENDPOINT_PRICE_USD as string,
+ network: process.env.X402_NETWORK as "base-sepolia" | "base" | "avalanche-fuji" | "avalanche" | "iotex",
config: {
- description: "Access premium protected data API",
+ description: "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",
mimeType: "application/json",
- maxTimeoutSeconds: 300,
- }
- },
- "POST /search-with-wallet": {
- price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",
- network: "base-sepolia",
- config: {
- description: "Search with wallet payment API",
- mimeType: "application/json",
- maxTimeoutSeconds: 300,
+ maxTimeoutSeconds: 120,
}
},
},
@@ -381,13 +372,8 @@ v1Router.post(
wrap(generatePaymentHeaderController),
);
-v1Router.get("/protected-data",
- authMiddleware(RateLimiterMode.Search),
- wrap(protectedDataController),
-);
-
v1Router.post(
- "/search-with-wallet",
+ "/x402/search",
authMiddleware(RateLimiterMode.Search),
- wrap(searchWithWalletController),
+ wrap(x402SearchController),
);
\ No newline at end of file