Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

draft: search with x402

Ademílson Tonatoa96ddf9e7efa
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 021456b38..f560476e5 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -78,3 +78,20 @@ RESEND_API_KEY=
# TRACE - For logging more detailed information than the DEBUG level.
# Set LOGGING_LEVEL to one of the above options to control logging output.
LOGGING_LEVEL=INFO
+
+
+
+# x402 Payment Configuration
+X402_PAY_TO_ADDRESS=
+ # ^ Address taken from -> https://portfolio.metamask.io/
+X402_NETWORK=base-sepolia
+X402_FACILITATOR_URL=https://x402.org/facilitator
+X402_ENABLED=true
+X402_VERIFICATION_TIMEOUT=30000
+
+CDP_API_KEY_ID=""
+CDP_API_KEY_SECRET=""
+ # ^ KEYS taken from -> https://portal.cdp.coinbase.com/projects/api-keys
+
+## ENDPOINT PRICING TEST
+X402_ENDPOINT_PRICE_USD=$0.001
\ No newline at end of file
diff --git a/apps/api/package.json b/apps/api/package.json
index 7be4771c4..df802d72e 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -140,10 +140,12 @@
"undici": "^7.10.0",
"unstructured-client": "^0.11.3",
"uuid": "^10.0.0",
+ "viem": "^2.32.0",
"winston": "^3.14.2",
"winston-transport": "^4.8.0",
"wordpos": "^2.1.0",
"ws": "^8.18.0",
+ "x402-express": "^0.4.1",
"xml2js": "^0.6.2",
"zod": "^3.24.2"
},
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index fdc17447a..d4f63abf3 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -272,6 +272,9 @@ importers:
uuid:
specifier: ^10.0.0
version: 10.0.0
+ viem:
+ specifier: ^2.32.0
winston:
specifier: ^3.14.2
version: 3.14.2
@@ -284,6 +287,9 @@ importers:
ws:
specifier: ^8.18.0
version: 8.18.0
+ x402-express:
+ specifier: ^0.4.1
+ version: 0.4.1([email protected])
xml2js:
specifier: ^0.6.2
version: 0.6.2
@@ -354,6 +360,9 @@ importers:
packages:
+ '@adraffy/[email protected]':
+ resolution: {integrity: sha512-/3DDPKHqqIqxUULp8yP4zODUY1i+2xvVWsv8A79xGWdCAG+8sb0hRh0Rk2QyOJUnnbyPUAZYcpBuRe3nS2OIUg==}
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-PyE7EC2fPjs9DnzRAHDrPQmcnI2m2Eojr8pfhckOejOlDEh2w7NnSJr1W3qe5hUWzKr+6d7NG1ZKR9fhmpDdEQ==}
engines: {node: '>=18'}
@@ -1063,6 +1072,18 @@ packages:
resolution: {integrity: sha512-w3btGjeTsYG32hPfbDfziYQRfqGWML3AK88ZE6sOu6qZHLbx7WWf1X/TsTogSxJ2BhnP4UoVQ4F935/G7ZWztA==}
engines: {node: '>=18.0'}
+ '@noble/[email protected]':
+ resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==}
+ engines: {node: ^14.21.3 || >=16}
+
+ '@noble/[email protected]':
+ resolution: {integrity: sha512-HxngEd2XUcg9xi20JkwlLCtYwfoFw4JGkuZpT+WlsPD4gB/cxkvTD8fSsoAnphGZhFdZYKeQIPCuFlWPm1uE0g==}
+ engines: {node: ^14.21.3 || >=16}
+
+ '@noble/[email protected]':
+ resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}
+ engines: {node: ^14.21.3 || >=16}
+
'@one-ini/[email protected]':
resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==}
@@ -1283,6 +1304,15 @@ packages:
peerDependencies:
'@redis/client': ^1.0.0
+ '@scure/[email protected]':
+ resolution: {integrity: sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==}
+
+ '@scure/[email protected]':
+ resolution: {integrity: sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==}
+
+ '@scure/[email protected]':
+ resolution: {integrity: sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==}
+
'@selderee/[email protected]':
resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==}
@@ -1885,6 +1915,17 @@ packages:
resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
+ resolution: {integrity: sha512-ZeiI6h3GnW06uYDLx0etQtX/p8E24UaHHBj57RSjK7YBFe7iuVn07EDpOeP451D06sF27VOz9JJPlIKJmXgkEg==}
+ peerDependencies:
+ typescript: '>=5.0.4'
+ zod: ^3 >=3.22.0
+ peerDependenciesMeta:
+ typescript:
+ optional: true
+ zod:
+ optional: true
+
resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
engines: {node: '>=6.5'}
@@ -2665,6 +2706,9 @@ packages:
resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
+ resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}
+
resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
engines: {node: '>=0.8.x'}
@@ -3160,6 +3204,11 @@ packages:
resolution: {integrity: sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==}
+ resolution: {integrity: sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==}
+ peerDependencies:
+ ws: '*'
+
resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==}
engines: {node: '>=8'}
@@ -3816,6 +3865,14 @@ packages:
resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}
+ resolution: {integrity: sha512-e+z5epnzV+Zuz91YYujecW8cF01mzmrUtWotJ0oEPym/G82uccs7q0WDHTYL3eiONbTUEvcZrptAKLgTBD3u2A==}
+ peerDependencies:
+ typescript: '>=5.4.0'
+ peerDependenciesMeta:
+ typescript:
+ optional: true
+
resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==}
engines: {node: '>=6'}
@@ -4711,6 +4768,14 @@ packages:
resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}
engines: {node: '>= 0.8'}
+ resolution: {integrity: sha512-pHwKXQSyEWX+8ttOQJdU5dSBfYd6L9JxARY/Sx0MBj3uF/Zaiqt6o1SbzjFjQXkNzWSgtxK7H89ZI1SMIA2iLQ==}
+ peerDependencies:
+ typescript: '>=5.0.4'
+ peerDependenciesMeta:
+ typescript:
+ optional: true
+
resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==}
engines: {node: '>=18'}
@@ -4824,6 +4889,24 @@ packages:
utf-8-validate:
optional: true
+ resolution: {integrity: sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==}
+ engines: {node: '>=10.0.0'}
+ peerDependencies:
+ bufferutil: ^4.0.1
+ utf-8-validate: '>=5.0.2'
+ peerDependenciesMeta:
+ bufferutil:
+ optional: true
+ utf-8-validate:
+ optional: true
+
+ resolution: {integrity: sha512-/DA+VYMkbpnawUIDsULyLeifp5i1vcWwJcR1+B8vv34Y7iEZrDcon50KjcbFXWCsRRAgTXOM83+VakJDPInwqw==}
+
+ resolution: {integrity: sha512-5EW6FzcZjyizk4qa+t7QzclUBq//mCWbNf75O+WgDIyGtWg2FfqWPocmncjof6cG33rrHQG/PvNw5AMvKU6lpw==}
+
resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}
engines: {node: '>=18'}
@@ -4893,6 +4976,8 @@ packages:
snapshots:
+ '@adraffy/[email protected]': {}
+
dependencies:
'@ai-sdk/provider': 1.1.3
@@ -6115,6 +6200,14 @@ snapshots:
transitivePeerDependencies:
- debug
+ '@noble/[email protected]': {}
+
+ '@noble/[email protected]':
+ dependencies:
+ '@noble/hashes': 1.8.0
+
+ '@noble/[email protected]': {}
+
'@one-ini/[email protected]': {}
@@ -6401,6 +6494,19 @@ snapshots:
dependencies:
'@redis/client': 1.5.16
+ '@scure/[email protected]': {}
+
+ '@scure/[email protected]':
+ dependencies:
+ '@noble/curves': 1.9.2
+ '@noble/hashes': 1.8.0
+ '@scure/base': 1.2.6
+
+ '@scure/[email protected]':
+ dependencies:
+ '@noble/hashes': 1.8.0
+ '@scure/base': 1.2.6
+
'@selderee/[email protected]':
dependencies:
domhandler: 5.0.3
@@ -6962,7 +7068,7 @@ snapshots:
'@supabase/node-fetch': 2.6.15
'@types/phoenix': 1.6.5
'@types/ws': 8.5.12
- ws: 8.18.0
+ ws: 8.18.2
transitivePeerDependencies:
- bufferutil
- utf-8-validate
@@ -7192,6 +7298,11 @@ snapshots:
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
dependencies:
event-target-shim: 5.0.1
@@ -7984,6 +8095,8 @@ snapshots:
+
@@ -8589,6 +8702,10 @@ snapshots:
transitivePeerDependencies:
- encoding
+ dependencies:
+ ws: 8.18.2
+
@@ -9459,6 +9576,21 @@ snapshots:
+ dependencies:
+ '@adraffy/ens-normalize': 1.11.0
+ '@noble/ciphers': 1.3.0
+ '@noble/curves': 1.9.2
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ eventemitter3: 5.0.1
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - zod
+
dependencies:
p-try: 2.2.0
@@ -9711,7 +9843,7 @@ snapshots:
chromium-bidi: 0.5.24([email protected])
debug: 4.3.5
devtools-protocol: 0.0.1299070
- ws: 8.18.0
+ ws: 8.18.2
transitivePeerDependencies:
- bufferutil
- supports-color
@@ -10382,6 +10514,23 @@ snapshots:
+ dependencies:
+ '@noble/curves': 1.9.2
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ isows: 1.0.7([email protected])
+ ws: 8.18.2
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - bufferutil
+ - utf-8-validate
+ - zod
+
dependencies:
xml-name-validator: 5.0.0
@@ -10485,6 +10634,29 @@ snapshots:
+
+ dependencies:
+ express: 4.19.2
+ x402: 0.4.1([email protected])
+ zod: 3.24.2
+ transitivePeerDependencies:
+ - bufferutil
+ - supports-color
+ - typescript
+ - utf-8-validate
+
+ dependencies:
+ zod: 3.24.2
+ transitivePeerDependencies:
+ - bufferutil
+ - typescript
+ - utf-8-validate
+
diff --git a/apps/api/requests.http b/apps/api/requests.http
index 5f55c4975..fd53ee545 100644
--- a/apps/api/requests.http
+++ b/apps/api/requests.http
@@ -131,3 +131,56 @@ content-type: application/json
# @name activeCrawls
GET {{baseUrl}}/v1/crawl/active HTTP/1.1
Authorization: Bearer {{$dotenv TEST_API_KEY}}
+
+
+
+### Generate Payment Header for Protected Data
+# Generate a payment header for accessing protected data
+# Duration options: 1h, 1d, 1w
+# @name generatePaymentHeaderProtected
+POST {{baseUrl}}/v1/generate-payment-header HTTP/1.1
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
+content-type: application/json
+
+{
+ "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY",
+ "duration": "1h",
+ "resource": "protected-data"
+}
+
+### Test Protected Data with Generated Payment Header
+# Use the payment header from the previous request
+@paymentHeaderProtected = {{generatePaymentHeaderProtected.response.body.data.paymentHeader}}
+# @name testProtectedData
+GET {{baseUrl}}/v1/protected-data HTTP/1.1
+Accept: application/json
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
+X-Payment: {{paymentHeaderProtected}}
+
+
+### Generate Payment Header for Search with Wallet
+# Generate a payment header for search with wallet
+# Duration options: 1h, 1d, 1w
+# @name generatePaymentHeaderSearch
+POST {{baseUrl}}/v1/generate-payment-header HTTP/1.1
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
+content-type: application/json
+
+{
+ "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY",
+ "duration": "1h",
+ "resource": "search-with-wallet"
+}
+
+### Search with Wallet
+@paymentHeaderSearch = {{generatePaymentHeaderSearch.response.body.data.paymentHeader}}
+# @name searchWithWallet
+POST {{baseUrl}}/v1/search-with-wallet HTTP/1.1
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
+content-type: application/json
+X-Payment: {{paymentHeaderSearch}}
+
+{
+ "query": "firecrawl",
+ "limit": 2
+}
\ No newline at end of file
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 0cc24d6f4..dcab29b28 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -91,6 +91,7 @@ const mockPreviewACUC: (team_id: string, is_extract: boolean) => AuthCreditUsage
extractStatus: 500,
extractAgentPreview: 1,
scrapeAgentPreview: 5,
+ generatePaymentHeader: 2,
},
price_credits: 99999999,
credits_used: 0,
@@ -426,6 +427,8 @@ export async function supaAuthenticateUser(
rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);
} else if (mode == RateLimiterMode.ExtractStatus) {
rateLimiter = getRateLimiter(RateLimiterMode.ExtractStatus, token);
+ } else if (mode == RateLimiterMode.GeneratePaymentHeader) {
+ rateLimiter = getRateLimiter(RateLimiterMode.GeneratePaymentHeader, token);
} else {
rateLimiter = getRateLimiter(RateLimiterMode.Preview, token);
}
diff --git a/apps/api/src/controllers/v1/generate-payment-header.ts b/apps/api/src/controllers/v1/generate-payment-header.ts
new file mode 100644
index 000000000..325bf20bd
--- /dev/null
+++ b/apps/api/src/controllers/v1/generate-payment-header.ts
@@ -0,0 +1,212 @@
+import { Request, Response } from "express";
+import crypto from 'crypto';
+import { privateKeyToAccount } from 'viem/accounts';
+import { getAddress } from 'viem';
+import {
+ GeneratePaymentHeaderResponse,
+ GeneratePaymentHeaderRequest,
+ RequestWithAuth,
+} from "./types";
+
+interface PaymentRequirements {
+ scheme: string;
+ network: string;
+ maxAmountRequired: string;
+ resource: string;
+ description: string;
+ mimeType: string;
+ payTo: string;
+ maxTimeoutSeconds: number;
+ asset: string;
+ extra: {
+ name: string;
+ version: string;
+ };
+}
+
+// Generate a random nonce (32 bytes = 64 hex chars)
+function generateNonce(): string {
+ return '0x' + crypto.randomBytes(32).toString('hex');
+}
+
+// Create the unsigned payment header
+function createUnsignedPaymentHeader(account: any, paymentRequirements: PaymentRequirements, durationSeconds: number): any {
+ const now = Math.floor(Date.now() / 1000);
+ const validAfter = (now - 60).toString(); // 60 seconds before
+ const validBefore = (now + durationSeconds).toString();
+ const nonce = generateNonce();
+
+ return {
+ x402Version: 1,
+ scheme: paymentRequirements.scheme,
+ network: paymentRequirements.network,
+ payload: {
+ signature: null,
+ authorization: {
+ from: account.address,
+ to: getAddress(paymentRequirements.payTo),
+ value: paymentRequirements.maxAmountRequired,
+ validAfter: validAfter,
+ validBefore: validBefore,
+ nonce: nonce
+ }
+ }
+ };
+}
+
+// Create EIP-712 typed data for signing
+function createTypedData(paymentRequirements: PaymentRequirements, authorization: any): any {
+ return {
+ types: {
+ TransferWithAuthorization: [
+ { name: "from", type: "address" },
+ { name: "to", type: "address" },
+ { name: "value", type: "uint256" },
+ { name: "validAfter", type: "uint256" },
+ { name: "validBefore", type: "uint256" },
+ { name: "nonce", type: "bytes32" }
+ ]
+ },
+ primaryType: "TransferWithAuthorization",
+ domain: {
+ name: paymentRequirements.extra.name,
+ version: paymentRequirements.extra.version,
+ chainId: 84532, // Base Sepolia chain ID
+ verifyingContract: getAddress(paymentRequirements.asset)
+ },
+ message: {
+ from: authorization.from,
+ to: authorization.to,
+ value: authorization.value,
+ validAfter: authorization.validAfter,
+ validBefore: authorization.validBefore,
+ nonce: authorization.nonce
+ }
+ };
+}
+
+// Base64 encode the payment header
+function encodePayment(paymentHeader: any): string {
+ const jsonString = JSON.stringify(paymentHeader);
+ return Buffer.from(jsonString).toString('base64');
+}
+
+export async function generatePaymentHeaderController(req: RequestWithAuth<{}, GeneratePaymentHeaderResponse, GeneratePaymentHeaderRequest>,
+ res: Response<GeneratePaymentHeaderResponse>) {
+ try {
+ const { privateKey, duration, resource }: GeneratePaymentHeaderRequest = req.body;
+
+ if (!privateKey) {
+ return res.status(400).json({
+ success: false,
+ error: "Private key is required"
+ });
+ }
+
+ if (!duration || !['1h', '1d', '1w'].includes(duration)) {
+ return res.status(400).json({
+ success: false,
+ error: "Duration must be one of: 1h, 1d, 1w"
+ });
+ }
+
+ // Convert duration to seconds
+ const durationMap = {
+ '1h': 3600,
+ '1d': 86400,
+ '1w': 604800
+ };
+ const durationSeconds = durationMap[duration];
+
+ // Payment requirements based on the Express server configuration
+ const getPaymentRequirements = (resource: string): PaymentRequirements => {
+ const baseRequirements = {
+ scheme: "exact",
+ network: "base-sepolia",
+ mimeType: "application/json",
+ payTo: process.env.X402_PAY_TO_ADDRESS || "0x0000000000000000000000000000000000000000",
+ maxTimeoutSeconds: durationSeconds,
+ asset: "0x036CbD53842c5426634e7929541eC2318f3dCF7e", // USDC on Base Sepolia
+ extra: {
+ name: "USDC",
+ version: "2"
+ }
+ };
+
+ // Get the base URL from the request
+ const protocol = req.protocol || 'http';
+ const host = req.get('host') || 'localhost:3002';
+ const baseUrl = `${protocol}://${host}`;
+
+ switch (resource) {
+ case 'search-with-wallet':
+ return {
+ ...baseRequirements,
+ maxAmountRequired: "10000", // $0.001 in USDC base units - matches X402_SEARCH_WALLET_PRICE_USD
+ resource: `${baseUrl}/v1/search-with-wallet`,
+ description: "Search with wallet payment API",
+ };
+ case 'protected-data':
+ default:
+ return {
+ ...baseRequirements,
+ maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD
+ resource: `${baseUrl}/v1/protected-data`,
+ description: "Access to protected data API",
+ };
+ }
+ };
+
+ const PAYMENT_REQUIREMENTS = getPaymentRequirements(resource);
+
+ // Create account from private key
+ const account = privateKeyToAccount(privateKey as `0x${string}`);
+
+ // Create unsigned payment header
+ const unsignedHeader = createUnsignedPaymentHeader(account, PAYMENT_REQUIREMENTS, durationSeconds);
+
+ // Create typed data for signing
+ const typedData = createTypedData(PAYMENT_REQUIREMENTS, unsignedHeader.payload.authorization);
+
+ // Sign the typed data
+ const signature = await account.signTypedData(typedData);
+
+ // Add signature to the header
+ const signedHeader = {
+ ...unsignedHeader,
+ payload: {
+ ...unsignedHeader.payload,
+ signature: signature
+ }
+ };
+
+ // Encode the final payment header
+ const encodedHeader = encodePayment(signedHeader);
+
+ // Return the payment header and additional info
+ res.json({
+ success: true,
+ data: {
+ paymentHeader: encodedHeader,
+ walletAddress: account.address,
+ duration: duration,
+ validUntil: new Date((Math.floor(Date.now() / 1000) + durationSeconds) * 1000).toISOString(),
+ notes: {
+ timeSensitive: `This payment header is time-sensitive (valid for ${duration})`,
+ walletRequirement: "Make sure your wallet has USDC on Base Sepolia",
+ paymentAmount: resource === 'search-with-wallet' ? "1000 USDC (in base units = $0.001)" : "10000 USDC (in base units = $0.01)",
+ generatedAt: new Date().toISOString(),
+ team_id: req.auth?.team_id || req.acuc?.team_id
+ }
+ }
+ });
+
+ } catch (error) {
+ console.error('Error generating payment header:', error);
+ res.status(500).json({
+ success: false,
+ error: "Failed to generate payment header",
+ details: error instanceof Error ? error.message : "Unknown error"
+ });
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/controllers/v1/protected-data.ts b/apps/api/src/controllers/v1/protected-data.ts
new file mode 100644
index 000000000..ed60698a5
--- /dev/null
+++ b/apps/api/src/controllers/v1/protected-data.ts
@@ -0,0 +1,27 @@
+import { Request, Response } from "express";
+import { RequestWithAuth } from "./types";
+
+export async function protectedDataController(
+ req: RequestWithAuth<{}, Response, any>,
+ res: Response<any>,
+) {
+ res.json({
+ request: {
+ headers: req.headers,
+ body: req.body,
+ },
+ success: true,
+ data: {
+ created_at: new Date().toISOString(),
+ person: {
+ name: "John Doe",
+ age: 30,
+ email: "[email protected]",
+ },
+ company: {
+ name: "Example Inc.",
+ industry: "Technology",
+ },
+ },
+ });
+}
diff --git a/apps/api/src/controllers/v1/search-with-wallet.ts b/apps/api/src/controllers/v1/search-with-wallet.ts
new file mode 100644
index 000000000..a1ff7b783
--- /dev/null
+++ b/apps/api/src/controllers/v1/search-with-wallet.ts
@@ -0,0 +1,271 @@
+import { Response } from "express";
+import {
+ Document,
+ RequestWithAuth,
+ SearchRequest,
+ SearchResponse,
+ searchRequestSchema,
+ ScrapeOptions,
+ TeamFlags,
+} from "./types";
+import { v4 as uuidv4 } from "uuid";
+import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
+import { getJobPriority } from "../../lib/job-priority";
+import { Mode } from "../../types";
+import { getScrapeQueue } from "../../services/queue-service";
+import { search } from "../../search";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { CostTracking } from "../../lib/extract/extraction-service";
+import { supabase_service } from "../../services/supabase";
+
+interface DocumentWithCostTracking {
+ document: Document;
+ costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
+}
+
+// Used for deep research
+export async function searchAndScrapeSearchResultWithWallet(
+ query: string,
+ options: {
+ teamId: string;
+ origin: string;
+ timeout: number;
+ scrapeOptions: ScrapeOptions;
+ },
+ flags: TeamFlags,
+): Promise<DocumentWithCostTracking[]> {
+ try {
+ const searchResults = await search({
+ query,
+ num_results: 5,
+ });
+
+ const documentsWithCostTracking = await Promise.all(
+ searchResults.map((result) =>
+ scrapeSearchResultWithWallet(
+ {
+ url: result.url,
+ title: result.title,
+ description: result.description,
+ },
+ options,
+ flags,
+ ),
+ ),
+ );
+
+ return documentsWithCostTracking;
+ } catch (error) {
+ return [];
+ }
+}
+
+async function scrapeSearchResultWithWallet(
+ searchResult: { url: string; title: string; description: string },
+ options: {
+ teamId: string;
+ origin: string;
+ timeout: number;
+ scrapeOptions: ScrapeOptions;
+ },
+ flags: TeamFlags,
+ directToBullMQ: boolean = false,
+ isSearchPreview: boolean = false,
+): Promise<DocumentWithCostTracking> {
+ const jobId = uuidv4();
+ const jobPriority = await getJobPriority({
+ team_id: options.teamId,
+ basePriority: 10,
+ });
+
+ const costTracking = new CostTracking();
+
+ const zeroDataRetention = flags?.forceZDR ?? false;
+
+ try {
+ if (isUrlBlocked(searchResult.url, flags)) {
+ throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE);
+ }
+ await addScrapeJob(
+ {
+ url: searchResult.url,
+ mode: "single_urls" as Mode,
+ team_id: options.teamId,
+ scrapeOptions: {
+ ...options.scrapeOptions,
+ maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days
+ },
+ internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },
+ origin: options.origin,
+ is_scrape: true,
+ startTime: Date.now(),
+ zeroDataRetention,
+ },
+ {},
+ jobId,
+ jobPriority,
+ directToBullMQ,
+ );
+
+ const doc: Document = await waitForJob(jobId, options.timeout);
+
+ await getScrapeQueue().remove(jobId);
+
+ const document = {
+ title: searchResult.title,
+ description: searchResult.description,
+ url: searchResult.url,
+ ...doc,
+ };
+
+ let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const { data: costTrackingResponse, error: costTrackingError } = await supabase_service.from("firecrawl_jobs")
+ .select("cost_tracking")
+ .eq("job_id", jobId);
+
+ if (costTrackingError) {
+ throw costTrackingError;
+ }
+
+ costTracking = costTrackingResponse?.[0]?.cost_tracking;
+ } else {
+ costTracking = new CostTracking().toJSON();
+ }
+
+ return {
+ document,
+ costTracking,
+ };
+ } catch (error) {
+
+ let statusCode = 0;
+ if (error?.message?.includes("Could not scrape url")) {
+ statusCode = 403;
+ }
+
+ const document: Document = {
+ title: searchResult.title,
+ description: searchResult.description,
+ url: searchResult.url,
+ metadata: {
+ statusCode,
+ error: error.message,
+ proxyUsed: "basic",
+ },
+ };
+
+ return {
+ document,
+ costTracking: new CostTracking().toJSON(),
+ };
+ }
+}
+
+export async function searchWithWalletController(
+ req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
+ res: Response<SearchResponse & { request?: any; }>,
+) {
+ if (req.acuc?.flags?.forceZDR) {
+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search-with-wallet. Please contact [email protected] to unblock this feature." });
+ }
+
+ let responseData: SearchResponse = {
+ success: true,
+ data: [],
+ };
+ const startTime = new Date().getTime();
+ const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+
+ try {
+ req.body = searchRequestSchema.parse(req.body);
+
+ let limit = req.body.limit;
+
+ // Buffer results by 50% to account for filtered URLs
+ const num_results_buffer = Math.floor(limit * 2);
+
+ let searchResults = await search({
+ query: req.body.query,
+ advanced: false,
+ num_results: num_results_buffer,
+ tbs: req.body.tbs,
+ filter: req.body.filter,
+ lang: req.body.lang,
+ country: req.body.country,
+ location: req.body.location,
+ });
+
+ if (req.body.ignoreInvalidURLs) {
+ searchResults = searchResults.filter(
+ (result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null),
+ );
+ }
+
+ // Filter blocked URLs early to avoid unnecessary billing
+ if (searchResults.length > limit) {
+ searchResults = searchResults.slice(0, limit);
+ }
+
+ if (searchResults.length === 0) {
+ responseData.warning = "No search results found";
+ } else if (
+ !req.body.scrapeOptions.formats ||
+ req.body.scrapeOptions.formats.length === 0
+ ) {
+ responseData.data = searchResults.map((r) => ({
+ url: r.url,
+ title: r.title,
+ description: r.description,
+ })) as Document[];
+ } else {
+ const scrapePromises = searchResults.map((result) =>
+ scrapeSearchResultWithWallet(
+ result,
+ {
+ teamId: req.auth.team_id,
+ origin: req.body.origin,
+ timeout: req.body.timeout,
+ scrapeOptions: req.body.scrapeOptions,
+ },
+ req.acuc?.flags ?? null,
+ (req.acuc?.price_credits ?? 0) <= 3000,
+ isSearchPreview,
+ ),
+ );
+
+ const docsWithCostTracking = await Promise.all(scrapePromises);
+
+ const docs = docsWithCostTracking.map(item => item.document);
+ const filteredDocs = docs.filter(
+ (doc) =>
+ doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),
+ );
+
+ if (filteredDocs.length === 0) {
+ responseData.data = docs;
+ responseData.warning = "No content found in search results";
+ } else {
+ responseData.data = filteredDocs;
+ }
+ }
+
+ return res.status(200).json(responseData);
+ } catch (error) {
+ if (
+ error instanceof Error &&
+ (error.message.startsWith("Job wait") || error.message === "timeout")
+ ) {
+ return res.status(408).json({
+ success: false,
+ error: "Request timed out",
+ });
+ }
+
+ return res.status(500).json({
+ success: false,
+ error: error.message,
+ });
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 5de6eb3c8..59268522b 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1030,6 +1030,7 @@ export type AuthCreditUsageChunk = {
extractStatus: number;
extractAgentPreview?: number;
scrapeAgentPreview?: number;
+ generatePaymentHeader?: number;
};
concurrency: number;
flags: TeamFlags;
@@ -1376,3 +1377,34 @@ export class TimeoutSignal extends Error {
super("Operation timed out");
}
}
+
+// Generate Payment Header Request
+export const generatePaymentHeaderRequestSchema = z.object({
+ privateKey: z.string(),
+ duration: z.enum(['1h', '1d', '1w']),
+ resource: z.string().optional().default('protected-data'),
+})
+
+export type GeneratePaymentHeaderRequest = z.infer<typeof generatePaymentHeaderRequestSchema>;
+export type GeneratePaymentHeaderRequestInput = z.input<typeof generatePaymentHeaderRequestSchema>;
+
+export type GeneratePaymentData = {
+ paymentHeader: string;
+ walletAddress: string;
+ duration: string;
+ validUntil: string;
+ notes: {
+ timeSensitive: string;
+ walletRequirement: string;
+ paymentAmount: string;
+ generatedAt: string;
+ [key: string]: string;
+ }
+}
+
+export type GeneratePaymentHeaderResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ data: GeneratePaymentData;
+ };
\ No newline at end of file
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index cfb98cbe7..f5e2f5f47 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -29,6 +29,7 @@ import { extractStatusController } from "../controllers/v1/extract-status";
import { creditUsageController } from "../controllers/v1/credit-usage";
import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
import { searchController } from "../controllers/v1/search";
+import { searchWithWalletController } from "../controllers/v1/search-with-wallet";
import { crawlErrorsController } from "../controllers/v1/crawl-errors";
import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";
import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";
@@ -36,6 +37,9 @@ import { deepResearchController } from "../controllers/v1/deep-research";
import { deepResearchStatusController } from "../controllers/v1/deep-research-status";
import { tokenUsageController } from "../controllers/v1/token-usage";
import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";
+import { generatePaymentHeaderController } from "../controllers/v1/generate-payment-header";
+import { protectedDataController } from '../controllers/v1/protected-data';
+import { paymentMiddleware } from "x402-express";
function checkCreditsMiddleware(
_minimum?: number,
@@ -174,6 +178,34 @@ expressWs(express());
export const v1Router = express.Router();
+// First, configure the payment middleware with your routes
+v1Router.use(
+ paymentMiddleware(
+ process.env.X402_PAY_TO_ADDRESS as `0x${string}` || "0x0000000000000000000000000000000000000000",
+ {
+ // Define your routes and their payment requirements
+ "GET /protected-data": {
+ price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",
+ network: "base-sepolia",
+ config: {
+ description: "Access premium protected data API",
+ mimeType: "application/json",
+ maxTimeoutSeconds: 300,
+ }
+ },
+ "POST /search-with-wallet": {
+ price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",
+ network: "base-sepolia",
+ config: {
+ description: "Search with wallet payment API",
+ mimeType: "application/json",
+ maxTimeoutSeconds: 300,
+ }
+ },
+ },
+ ),
+);
+
v1Router.post(
"/scrape",
authMiddleware(RateLimiterMode.Scrape),
@@ -342,3 +374,20 @@ v1Router.get(
authMiddleware(RateLimiterMode.ExtractStatus),
wrap(tokenUsageController),
);
+
+v1Router.post(
+ "/generate-payment-header",
+ authMiddleware(RateLimiterMode.GeneratePaymentHeader),
+ wrap(generatePaymentHeaderController),
+);
+
+v1Router.get("/protected-data",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(protectedDataController),
+);
+
+v1Router.post(
+ "/search-with-wallet",
+ authMiddleware(RateLimiterMode.Search),
+ wrap(searchWithWalletController),
+);
\ No newline at end of file
diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts
index bcf423ac6..cd4b22e2e 100644
--- a/apps/api/src/services/rate-limiter.ts
+++ b/apps/api/src/services/rate-limiter.ts
@@ -33,6 +33,7 @@ const fallbackRateLimits: AuthCreditUsageChunk["rate_limits"] = {
crawlStatus: 25000,
extractAgentPreview: 10,
scrapeAgentPreview: 10,
+ generatePaymentHeader: 2,
};
export function getRateLimiter(
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index c42720c1c..a23f5617b 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -162,6 +162,7 @@ export enum RateLimiterMode {
Extract = "extract",
ExtractStatus = "extractStatus",
ExtractAgentPreview = "extractAgentPreview",
+ GeneratePaymentHeader = "generatePaymentHeader",
}
export type AuthResponse =