Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
draft: search with x402
AdemÃlson Tonatoa96ddf9e7efa
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 021456b38..f560476e5 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -78,3 +78,20 @@ RESEND_API_KEY=# TRACE - For logging more detailed information than the DEBUG level.# Set LOGGING_LEVEL to one of the above options to control logging output.LOGGING_LEVEL=INFO++++# x402 Payment Configuration+X402_PAY_TO_ADDRESS=+ # ^ Address taken from -> https://portfolio.metamask.io/+X402_NETWORK=base-sepolia+X402_FACILITATOR_URL=https://x402.org/facilitator+X402_ENABLED=true+X402_VERIFICATION_TIMEOUT=30000++CDP_API_KEY_ID=""+CDP_API_KEY_SECRET=""+ # ^ KEYS taken from -> https://portal.cdp.coinbase.com/projects/api-keys++## ENDPOINT PRICING TEST+X402_ENDPOINT_PRICE_USD=$0.001\ No newline at end of filediff --git a/apps/api/package.json b/apps/api/package.jsonindex 7be4771c4..df802d72e 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -140,10 +140,12 @@"undici": "^7.10.0","unstructured-client": "^0.11.3","uuid": "^10.0.0",+ "viem": "^2.32.0","winston": "^3.14.2","winston-transport": "^4.8.0","wordpos": "^2.1.0","ws": "^8.18.0",+ "x402-express": "^0.4.1","xml2js": "^0.6.2","zod": "^3.24.2"},diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex fdc17447a..d4f63abf3 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -272,6 +272,9 @@ importers:uuid:specifier: ^10.0.0version: 10.0.0+ viem:+ specifier: ^2.32.0+ version: 2.32.0([email protected])([email protected])winston:specifier: ^3.14.2version: 3.14.2@@ -284,6 +287,9 @@ importers:ws:specifier: ^8.18.0version: 8.18.0+ x402-express:+ specifier: ^0.4.1+ version: 0.4.1([email protected])xml2js:specifier: ^0.6.2version: 0.6.2@@ -354,6 +360,9 @@ importers:packages:+ '@adraffy/[email protected]':+ resolution: {integrity: sha512-/3DDPKHqqIqxUULp8yP4zODUY1i+2xvVWsv8A79xGWdCAG+8sb0hRh0Rk2QyOJUnnbyPUAZYcpBuRe3nS2OIUg==}+'@ai-sdk/[email protected]':resolution: {integrity: sha512-PyE7EC2fPjs9DnzRAHDrPQmcnI2m2Eojr8pfhckOejOlDEh2w7NnSJr1W3qe5hUWzKr+6d7NG1ZKR9fhmpDdEQ==}engines: {node: '>=18'}@@ -1063,6 +1072,18 @@ packages:resolution: {integrity: sha512-w3btGjeTsYG32hPfbDfziYQRfqGWML3AK88ZE6sOu6qZHLbx7WWf1X/TsTogSxJ2BhnP4UoVQ4F935/G7ZWztA==}engines: {node: '>=18.0'}+ '@noble/[email protected]':+ resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==}+ engines: {node: ^14.21.3 || >=16}++ '@noble/[email protected]':+ resolution: {integrity: sha512-HxngEd2XUcg9xi20JkwlLCtYwfoFw4JGkuZpT+WlsPD4gB/cxkvTD8fSsoAnphGZhFdZYKeQIPCuFlWPm1uE0g==}+ engines: {node: ^14.21.3 || >=16}++ '@noble/[email protected]':+ resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}+ engines: {node: ^14.21.3 || >=16}+'@one-ini/[email protected]':resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==}@@ -1283,6 +1304,15 @@ packages:peerDependencies:'@redis/client': ^1.0.0+ '@scure/[email protected]':+ resolution: {integrity: sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==}++ '@scure/[email protected]':+ resolution: {integrity: sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==}++ '@scure/[email protected]':+ resolution: {integrity: sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==}+'@selderee/[email protected]':resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==}@@ -1885,6 +1915,17 @@ packages:resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-ZeiI6h3GnW06uYDLx0etQtX/p8E24UaHHBj57RSjK7YBFe7iuVn07EDpOeP451D06sF27VOz9JJPlIKJmXgkEg==}+ peerDependencies:+ typescript: '>=5.0.4'+ zod: ^3 >=3.22.0+ peerDependenciesMeta:+ typescript:+ optional: true+ zod:+ optional: true+resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}engines: {node: '>=6.5'}@@ -2665,6 +2706,9 @@ packages:resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}+ resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}+resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}engines: {node: '>=0.8.x'}@@ -3160,6 +3204,11 @@ packages:resolution: {integrity: sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==}+ resolution: {integrity: sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==}+ peerDependencies:+ ws: '*'+resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==}engines: {node: '>=8'}@@ -3816,6 +3865,14 @@ packages:resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==}+ resolution: {integrity: sha512-e+z5epnzV+Zuz91YYujecW8cF01mzmrUtWotJ0oEPym/G82uccs7q0WDHTYL3eiONbTUEvcZrptAKLgTBD3u2A==}+ peerDependencies:+ typescript: '>=5.4.0'+ peerDependenciesMeta:+ typescript:+ optional: true+resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==}engines: {node: '>=6'}@@ -4711,6 +4768,14 @@ packages:resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}engines: {node: '>= 0.8'}+ resolution: {integrity: sha512-pHwKXQSyEWX+8ttOQJdU5dSBfYd6L9JxARY/Sx0MBj3uF/Zaiqt6o1SbzjFjQXkNzWSgtxK7H89ZI1SMIA2iLQ==}+ peerDependencies:+ typescript: '>=5.0.4'+ peerDependenciesMeta:+ typescript:+ optional: true+resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==}engines: {node: '>=18'}@@ -4824,6 +4889,24 @@ packages:utf-8-validate:optional: true+ resolution: {integrity: sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==}+ engines: {node: '>=10.0.0'}+ peerDependencies:+ bufferutil: ^4.0.1+ utf-8-validate: '>=5.0.2'+ peerDependenciesMeta:+ bufferutil:+ optional: true+ utf-8-validate:+ optional: true++ resolution: {integrity: sha512-/DA+VYMkbpnawUIDsULyLeifp5i1vcWwJcR1+B8vv34Y7iEZrDcon50KjcbFXWCsRRAgTXOM83+VakJDPInwqw==}++ resolution: {integrity: sha512-5EW6FzcZjyizk4qa+t7QzclUBq//mCWbNf75O+WgDIyGtWg2FfqWPocmncjof6cG33rrHQG/PvNw5AMvKU6lpw==}+resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}engines: {node: '>=18'}@@ -4893,6 +4976,8 @@ packages:snapshots:+ '@adraffy/[email protected]': {}+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 1.1.3@@ -6115,6 +6200,14 @@ snapshots:transitivePeerDependencies:- debug+ '@noble/[email protected]': {}++ '@noble/[email protected]':+ dependencies:+ '@noble/hashes': 1.8.0++ '@noble/[email protected]': {}+'@one-ini/[email protected]': {}'@openrouter/[email protected]([email protected])':@@ -6401,6 +6494,19 @@ snapshots:dependencies:'@redis/client': 1.5.16+ '@scure/[email protected]': {}++ '@scure/[email protected]':+ dependencies:+ '@noble/curves': 1.9.2+ '@noble/hashes': 1.8.0+ '@scure/base': 1.2.6++ '@scure/[email protected]':+ dependencies:+ '@noble/hashes': 1.8.0+ '@scure/base': 1.2.6+'@selderee/[email protected]':dependencies:domhandler: 5.0.3@@ -6962,7 +7068,7 @@ snapshots:'@supabase/node-fetch': 2.6.15'@types/phoenix': 1.6.5'@types/ws': 8.5.12- ws: 8.18.0+ ws: 8.18.2transitivePeerDependencies:- bufferutil- utf-8-validate@@ -7192,6 +7298,11 @@ snapshots:+ optionalDependencies:+ typescript: 5.8.3+ zod: 3.24.2+dependencies:event-target-shim: 5.0.1@@ -7984,6 +8095,8 @@ snapshots:+ [email protected]: {}+@@ -8589,6 +8702,10 @@ snapshots:transitivePeerDependencies:- encoding+ dependencies:+ ws: 8.18.2+@@ -9459,6 +9576,21 @@ snapshots:+ dependencies:+ '@adraffy/ens-normalize': 1.11.0+ '@noble/ciphers': 1.3.0+ '@noble/curves': 1.9.2+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.0.8([email protected])([email protected])+ eventemitter3: 5.0.1+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - zod+dependencies:p-try: 2.2.0@@ -9711,7 +9843,7 @@ snapshots:chromium-bidi: 0.5.24([email protected])debug: 4.3.5devtools-protocol: 0.0.1299070- ws: 8.18.0+ ws: 8.18.2transitivePeerDependencies:- bufferutil- supports-color@@ -10382,6 +10514,23 @@ snapshots:+ dependencies:+ '@noble/curves': 1.9.2+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.0.8([email protected])([email protected])+ isows: 1.0.7([email protected])+ ox: 0.8.1([email protected])([email protected])+ ws: 8.18.2+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - bufferutil+ - utf-8-validate+ - zod+dependencies:xml-name-validator: 5.0.0@@ -10485,6 +10634,29 @@ snapshots:+ [email protected]: {}++ dependencies:+ express: 4.19.2+ viem: 2.32.0([email protected])([email protected])+ x402: 0.4.1([email protected])+ zod: 3.24.2+ transitivePeerDependencies:+ - bufferutil+ - supports-color+ - typescript+ - utf-8-validate++ dependencies:+ viem: 2.32.0([email protected])([email protected])+ zod: 3.24.2+ transitivePeerDependencies:+ - bufferutil+ - typescript+ - utf-8-validate+diff --git a/apps/api/requests.http b/apps/api/requests.httpindex 5f55c4975..fd53ee545 100644--- a/apps/api/requests.http+++ b/apps/api/requests.http@@ -131,3 +131,56 @@ content-type: application/json# @name activeCrawlsGET {{baseUrl}}/v1/crawl/active HTTP/1.1Authorization: Bearer {{$dotenv TEST_API_KEY}}++++### Generate Payment Header for Protected Data+# Generate a payment header for accessing protected data+# Duration options: 1h, 1d, 1w+# @name generatePaymentHeaderProtected+POST {{baseUrl}}/v1/generate-payment-header HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY",+ "duration": "1h",+ "resource": "protected-data"+}++### Test Protected Data with Generated Payment Header+# Use the payment header from the previous request+@paymentHeaderProtected = {{generatePaymentHeaderProtected.response.body.data.paymentHeader}}+# @name testProtectedData+GET {{baseUrl}}/v1/protected-data HTTP/1.1+Accept: application/json+Authorization: Bearer {{$dotenv TEST_API_KEY}}+X-Payment: {{paymentHeaderProtected}}+++### Generate Payment Header for Search with Wallet+# Generate a payment header for search with wallet+# Duration options: 1h, 1d, 1w+# @name generatePaymentHeaderSearch+POST {{baseUrl}}/v1/generate-payment-header HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json++{+ "privateKey": "0X-PRIVATE-KEY-PRIVATE-KEY",+ "duration": "1h",+ "resource": "search-with-wallet"+}++### Search with Wallet+@paymentHeaderSearch = {{generatePaymentHeaderSearch.response.body.data.paymentHeader}}+# @name searchWithWallet+POST {{baseUrl}}/v1/search-with-wallet HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json+X-Payment: {{paymentHeaderSearch}}++{+ "query": "firecrawl",+ "limit": 2+}\ No newline at end of filediff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 0cc24d6f4..dcab29b28 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -91,6 +91,7 @@ const mockPreviewACUC: (team_id: string, is_extract: boolean) => AuthCreditUsageextractStatus: 500,extractAgentPreview: 1,scrapeAgentPreview: 5,+ generatePaymentHeader: 2,},price_credits: 99999999,credits_used: 0,@@ -426,6 +427,8 @@ export async function supaAuthenticateUser(rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);} else if (mode == RateLimiterMode.ExtractStatus) {rateLimiter = getRateLimiter(RateLimiterMode.ExtractStatus, token);+ } else if (mode == RateLimiterMode.GeneratePaymentHeader) {+ rateLimiter = getRateLimiter(RateLimiterMode.GeneratePaymentHeader, token);} else {rateLimiter = getRateLimiter(RateLimiterMode.Preview, token);}diff --git a/apps/api/src/controllers/v1/generate-payment-header.ts b/apps/api/src/controllers/v1/generate-payment-header.tsnew file mode 100644index 000000000..325bf20bd--- /dev/null+++ b/apps/api/src/controllers/v1/generate-payment-header.ts@@ -0,0 +1,212 @@+import { Request, Response } from "express";+import crypto from 'crypto';+import { privateKeyToAccount } from 'viem/accounts';+import { getAddress } from 'viem';+import {+ GeneratePaymentHeaderResponse,+ GeneratePaymentHeaderRequest,+ RequestWithAuth,+} from "./types";++interface PaymentRequirements {+ scheme: string;+ network: string;+ maxAmountRequired: string;+ resource: string;+ description: string;+ mimeType: string;+ payTo: string;+ maxTimeoutSeconds: number;+ asset: string;+ extra: {+ name: string;+ version: string;+ };+}++// Generate a random nonce (32 bytes = 64 hex chars)+function generateNonce(): string {+ return '0x' + crypto.randomBytes(32).toString('hex');+}++// Create the unsigned payment header+function createUnsignedPaymentHeader(account: any, paymentRequirements: PaymentRequirements, durationSeconds: number): any {+ const now = Math.floor(Date.now() / 1000);+ const validAfter = (now - 60).toString(); // 60 seconds before+ const validBefore = (now + durationSeconds).toString();+ const nonce = generateNonce();++ return {+ x402Version: 1,+ scheme: paymentRequirements.scheme,+ network: paymentRequirements.network,+ payload: {+ signature: null,+ authorization: {+ from: account.address,+ to: getAddress(paymentRequirements.payTo),+ value: paymentRequirements.maxAmountRequired,+ validAfter: validAfter,+ validBefore: validBefore,+ nonce: nonce+ }+ }+ };+}++// Create EIP-712 typed data for signing+function createTypedData(paymentRequirements: PaymentRequirements, authorization: any): any {+ return {+ types: {+ TransferWithAuthorization: [+ { name: "from", type: "address" },+ { name: "to", type: "address" },+ { name: "value", type: "uint256" },+ { name: "validAfter", type: "uint256" },+ { name: "validBefore", type: "uint256" },+ { name: "nonce", type: "bytes32" }+ ]+ },+ primaryType: "TransferWithAuthorization",+ domain: {+ name: paymentRequirements.extra.name,+ version: paymentRequirements.extra.version,+ chainId: 84532, // Base Sepolia chain ID+ verifyingContract: getAddress(paymentRequirements.asset)+ },+ message: {+ from: authorization.from,+ to: authorization.to,+ value: authorization.value,+ validAfter: authorization.validAfter,+ validBefore: authorization.validBefore,+ nonce: authorization.nonce+ }+ };+}++// Base64 encode the payment header+function encodePayment(paymentHeader: any): string {+ const jsonString = JSON.stringify(paymentHeader);+ return Buffer.from(jsonString).toString('base64');+}++export async function generatePaymentHeaderController(req: RequestWithAuth<{}, GeneratePaymentHeaderResponse, GeneratePaymentHeaderRequest>,+ res: Response<GeneratePaymentHeaderResponse>) {+ try {+ const { privateKey, duration, resource }: GeneratePaymentHeaderRequest = req.body;++ if (!privateKey) {+ return res.status(400).json({+ success: false,+ error: "Private key is required"+ });+ }++ if (!duration || !['1h', '1d', '1w'].includes(duration)) {+ return res.status(400).json({+ success: false,+ error: "Duration must be one of: 1h, 1d, 1w"+ });+ }++ // Convert duration to seconds+ const durationMap = {+ '1h': 3600,+ '1d': 86400,+ '1w': 604800+ };+ const durationSeconds = durationMap[duration];++ // Payment requirements based on the Express server configuration+ const getPaymentRequirements = (resource: string): PaymentRequirements => {+ const baseRequirements = {+ scheme: "exact",+ network: "base-sepolia",+ mimeType: "application/json",+ payTo: process.env.X402_PAY_TO_ADDRESS || "0x0000000000000000000000000000000000000000",+ maxTimeoutSeconds: durationSeconds,+ asset: "0x036CbD53842c5426634e7929541eC2318f3dCF7e", // USDC on Base Sepolia+ extra: {+ name: "USDC",+ version: "2"+ }+ };++ // Get the base URL from the request+ const protocol = req.protocol || 'http';+ const host = req.get('host') || 'localhost:3002';+ const baseUrl = `${protocol}://${host}`;++ switch (resource) {+ case 'search-with-wallet':+ return {+ ...baseRequirements,+ maxAmountRequired: "10000", // $0.001 in USDC base units - matches X402_SEARCH_WALLET_PRICE_USD+ resource: `${baseUrl}/v1/search-with-wallet`,+ description: "Search with wallet payment API",+ };+ case 'protected-data':+ default:+ return {+ ...baseRequirements,+ maxAmountRequired: "10000", // $0.01 in USDC base units - matches X402_ENDPOINT_PRICE_USD+ resource: `${baseUrl}/v1/protected-data`,+ description: "Access to protected data API",+ };+ }+ };++ const PAYMENT_REQUIREMENTS = getPaymentRequirements(resource);++ // Create account from private key+ const account = privateKeyToAccount(privateKey as `0x${string}`);++ // Create unsigned payment header+ const unsignedHeader = createUnsignedPaymentHeader(account, PAYMENT_REQUIREMENTS, durationSeconds);++ // Create typed data for signing+ const typedData = createTypedData(PAYMENT_REQUIREMENTS, unsignedHeader.payload.authorization);++ // Sign the typed data+ const signature = await account.signTypedData(typedData);++ // Add signature to the header+ const signedHeader = {+ ...unsignedHeader,+ payload: {+ ...unsignedHeader.payload,+ signature: signature+ }+ };++ // Encode the final payment header+ const encodedHeader = encodePayment(signedHeader);++ // Return the payment header and additional info+ res.json({+ success: true,+ data: {+ paymentHeader: encodedHeader,+ walletAddress: account.address,+ duration: duration,+ validUntil: new Date((Math.floor(Date.now() / 1000) + durationSeconds) * 1000).toISOString(),+ notes: {+ timeSensitive: `This payment header is time-sensitive (valid for ${duration})`,+ walletRequirement: "Make sure your wallet has USDC on Base Sepolia",+ paymentAmount: resource === 'search-with-wallet' ? "1000 USDC (in base units = $0.001)" : "10000 USDC (in base units = $0.01)",+ generatedAt: new Date().toISOString(),+ team_id: req.auth?.team_id || req.acuc?.team_id+ }+ }+ });++ } catch (error) {+ console.error('Error generating payment header:', error);+ res.status(500).json({+ success: false,+ error: "Failed to generate payment header",+ details: error instanceof Error ? error.message : "Unknown error"+ });+ }+}\ No newline at end of filediff --git a/apps/api/src/controllers/v1/protected-data.ts b/apps/api/src/controllers/v1/protected-data.tsnew file mode 100644index 000000000..ed60698a5--- /dev/null+++ b/apps/api/src/controllers/v1/protected-data.ts@@ -0,0 +1,27 @@+import { Request, Response } from "express";+import { RequestWithAuth } from "./types";++export async function protectedDataController(+ req: RequestWithAuth<{}, Response, any>,+ res: Response<any>,+) {+ res.json({+ request: {+ headers: req.headers,+ body: req.body,+ },+ success: true,+ data: {+ created_at: new Date().toISOString(),+ person: {+ name: "John Doe",+ age: 30,+ email: "[email protected]",+ },+ company: {+ name: "Example Inc.",+ industry: "Technology",+ },+ },+ });+}diff --git a/apps/api/src/controllers/v1/search-with-wallet.ts b/apps/api/src/controllers/v1/search-with-wallet.tsnew file mode 100644index 000000000..a1ff7b783--- /dev/null+++ b/apps/api/src/controllers/v1/search-with-wallet.ts@@ -0,0 +1,271 @@+import { Response } from "express";+import {+ Document,+ RequestWithAuth,+ SearchRequest,+ SearchResponse,+ searchRequestSchema,+ ScrapeOptions,+ TeamFlags,+} from "./types";+import { v4 as uuidv4 } from "uuid";+import { addScrapeJob, waitForJob } from "../../services/queue-jobs";+import { getJobPriority } from "../../lib/job-priority";+import { Mode } from "../../types";+import { getScrapeQueue } from "../../services/queue-service";+import { search } from "../../search";+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";+import { CostTracking } from "../../lib/extract/extraction-service";+import { supabase_service } from "../../services/supabase";++interface DocumentWithCostTracking {+ document: Document;+ costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;+}++// Used for deep research+export async function searchAndScrapeSearchResultWithWallet(+ query: string,+ options: {+ teamId: string;+ origin: string;+ timeout: number;+ scrapeOptions: ScrapeOptions;+ },+ flags: TeamFlags,+): Promise<DocumentWithCostTracking[]> {+ try {+ const searchResults = await search({+ query,+ num_results: 5,+ });++ const documentsWithCostTracking = await Promise.all(+ searchResults.map((result) =>+ scrapeSearchResultWithWallet(+ {+ url: result.url,+ title: result.title,+ description: result.description,+ },+ options,+ flags,+ ),+ ),+ );++ return documentsWithCostTracking;+ } catch (error) {+ return [];+ }+}++async function scrapeSearchResultWithWallet(+ searchResult: { url: string; title: string; description: string },+ options: {+ teamId: string;+ origin: string;+ timeout: number;+ scrapeOptions: ScrapeOptions;+ },+ flags: TeamFlags,+ directToBullMQ: boolean = false,+ isSearchPreview: boolean = false,+): Promise<DocumentWithCostTracking> {+ const jobId = uuidv4();+ const jobPriority = await getJobPriority({+ team_id: options.teamId,+ basePriority: 10,+ });++ const costTracking = new CostTracking();++ const zeroDataRetention = flags?.forceZDR ?? false;++ try {+ if (isUrlBlocked(searchResult.url, flags)) {+ throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE);+ }+ await addScrapeJob(+ {+ url: searchResult.url,+ mode: "single_urls" as Mode,+ team_id: options.teamId,+ scrapeOptions: {+ ...options.scrapeOptions,+ maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days+ },+ internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },+ origin: options.origin,+ is_scrape: true,+ startTime: Date.now(),+ zeroDataRetention,+ },+ {},+ jobId,+ jobPriority,+ directToBullMQ,+ );++ const doc: Document = await waitForJob(jobId, options.timeout);++ await getScrapeQueue().remove(jobId);++ const document = {+ title: searchResult.title,+ description: searchResult.description,+ url: searchResult.url,+ ...doc,+ };++ let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;+ if (process.env.USE_DB_AUTHENTICATION === "true") {+ const { data: costTrackingResponse, error: costTrackingError } = await supabase_service.from("firecrawl_jobs")+ .select("cost_tracking")+ .eq("job_id", jobId);++ if (costTrackingError) {+ throw costTrackingError;+ }++ costTracking = costTrackingResponse?.[0]?.cost_tracking;+ } else {+ costTracking = new CostTracking().toJSON();+ }++ return {+ document,+ costTracking,+ };+ } catch (error) {++ let statusCode = 0;+ if (error?.message?.includes("Could not scrape url")) {+ statusCode = 403;+ }++ const document: Document = {+ title: searchResult.title,+ description: searchResult.description,+ url: searchResult.url,+ metadata: {+ statusCode,+ error: error.message,+ proxyUsed: "basic",+ },+ };++ return {+ document,+ costTracking: new CostTracking().toJSON(),+ };+ }+}++export async function searchWithWalletController(+ req: RequestWithAuth<{}, SearchResponse, SearchRequest>,+ res: Response<SearchResponse & { request?: any; }>,+) {+ if (req.acuc?.flags?.forceZDR) {+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search-with-wallet. Please contact [email protected] to unblock this feature." });+ }++ let responseData: SearchResponse = {+ success: true,+ data: [],+ };+ const startTime = new Date().getTime();+ const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;++ try {+ req.body = searchRequestSchema.parse(req.body);++ let limit = req.body.limit;++ // Buffer results by 50% to account for filtered URLs+ const num_results_buffer = Math.floor(limit * 2);++ let searchResults = await search({+ query: req.body.query,+ advanced: false,+ num_results: num_results_buffer,+ tbs: req.body.tbs,+ filter: req.body.filter,+ lang: req.body.lang,+ country: req.body.country,+ location: req.body.location,+ });++ if (req.body.ignoreInvalidURLs) {+ searchResults = searchResults.filter(+ (result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null),+ );+ }++ // Filter blocked URLs early to avoid unnecessary billing+ if (searchResults.length > limit) {+ searchResults = searchResults.slice(0, limit);+ }++ if (searchResults.length === 0) {+ responseData.warning = "No search results found";+ } else if (+ !req.body.scrapeOptions.formats ||+ req.body.scrapeOptions.formats.length === 0+ ) {+ responseData.data = searchResults.map((r) => ({+ url: r.url,+ title: r.title,+ description: r.description,+ })) as Document[];+ } else {+ const scrapePromises = searchResults.map((result) =>+ scrapeSearchResultWithWallet(+ result,+ {+ teamId: req.auth.team_id,+ origin: req.body.origin,+ timeout: req.body.timeout,+ scrapeOptions: req.body.scrapeOptions,+ },+ req.acuc?.flags ?? null,+ (req.acuc?.price_credits ?? 0) <= 3000,+ isSearchPreview,+ ),+ );++ const docsWithCostTracking = await Promise.all(scrapePromises);++ const docs = docsWithCostTracking.map(item => item.document);+ const filteredDocs = docs.filter(+ (doc) =>+ doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),+ );++ if (filteredDocs.length === 0) {+ responseData.data = docs;+ responseData.warning = "No content found in search results";+ } else {+ responseData.data = filteredDocs;+ }+ }++ return res.status(200).json(responseData);+ } catch (error) {+ if (+ error instanceof Error &&+ (error.message.startsWith("Job wait") || error.message === "timeout")+ ) {+ return res.status(408).json({+ success: false,+ error: "Request timed out",+ });+ }++ return res.status(500).json({+ success: false,+ error: error.message,+ });+ }+}\ No newline at end of filediff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 5de6eb3c8..59268522b 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1030,6 +1030,7 @@ export type AuthCreditUsageChunk = {extractStatus: number;extractAgentPreview?: number;scrapeAgentPreview?: number;+ generatePaymentHeader?: number;};concurrency: number;flags: TeamFlags;@@ -1376,3 +1377,34 @@ export class TimeoutSignal extends Error {super("Operation timed out");}}++// Generate Payment Header Request+export const generatePaymentHeaderRequestSchema = z.object({+ privateKey: z.string(),+ duration: z.enum(['1h', '1d', '1w']),+ resource: z.string().optional().default('protected-data'),+})++export type GeneratePaymentHeaderRequest = z.infer<typeof generatePaymentHeaderRequestSchema>;+export type GeneratePaymentHeaderRequestInput = z.input<typeof generatePaymentHeaderRequestSchema>;++export type GeneratePaymentData = {+ paymentHeader: string;+ walletAddress: string;+ duration: string;+ validUntil: string;+ notes: {+ timeSensitive: string;+ walletRequirement: string;+ paymentAmount: string;+ generatedAt: string;+ [key: string]: string;+ }+}++export type GeneratePaymentHeaderResponse =+ | ErrorResponse+ | {+ success: true;+ data: GeneratePaymentData;+ };\ No newline at end of filediff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex cfb98cbe7..f5e2f5f47 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -29,6 +29,7 @@ import { extractStatusController } from "../controllers/v1/extract-status";import { creditUsageController } from "../controllers/v1/credit-usage";import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";import { searchController } from "../controllers/v1/search";+import { searchWithWalletController } from "../controllers/v1/search-with-wallet";import { crawlErrorsController } from "../controllers/v1/crawl-errors";import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";@@ -36,6 +37,9 @@ import { deepResearchController } from "../controllers/v1/deep-research";import { deepResearchStatusController } from "../controllers/v1/deep-research-status";import { tokenUsageController } from "../controllers/v1/token-usage";import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";+import { generatePaymentHeaderController } from "../controllers/v1/generate-payment-header";+import { protectedDataController } from '../controllers/v1/protected-data';+import { paymentMiddleware } from "x402-express";function checkCreditsMiddleware(_minimum?: number,@@ -174,6 +178,34 @@ expressWs(express());export const v1Router = express.Router();+// First, configure the payment middleware with your routes+v1Router.use(+ paymentMiddleware(+ process.env.X402_PAY_TO_ADDRESS as `0x${string}` || "0x0000000000000000000000000000000000000000",+ {+ // Define your routes and their payment requirements+ "GET /protected-data": {+ price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",+ network: "base-sepolia",+ config: {+ description: "Access premium protected data API",+ mimeType: "application/json",+ maxTimeoutSeconds: 300,+ }+ },+ "POST /search-with-wallet": {+ price: process.env.X402_ENDPOINT_PRICE_USD || "0.01",+ network: "base-sepolia",+ config: {+ description: "Search with wallet payment API",+ mimeType: "application/json",+ maxTimeoutSeconds: 300,+ }+ },+ },+ ),+);+v1Router.post("/scrape",authMiddleware(RateLimiterMode.Scrape),@@ -342,3 +374,20 @@ v1Router.get(authMiddleware(RateLimiterMode.ExtractStatus),wrap(tokenUsageController),);++v1Router.post(+ "/generate-payment-header",+ authMiddleware(RateLimiterMode.GeneratePaymentHeader),+ wrap(generatePaymentHeaderController),+);++v1Router.get("/protected-data",+ authMiddleware(RateLimiterMode.Search),+ wrap(protectedDataController),+);++v1Router.post(+ "/search-with-wallet",+ authMiddleware(RateLimiterMode.Search),+ wrap(searchWithWalletController),+);\ No newline at end of filediff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex bcf423ac6..cd4b22e2e 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -33,6 +33,7 @@ const fallbackRateLimits: AuthCreditUsageChunk["rate_limits"] = {crawlStatus: 25000,extractAgentPreview: 10,scrapeAgentPreview: 10,+ generatePaymentHeader: 2,};export function getRateLimiter(diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex c42720c1c..a23f5617b 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -162,6 +162,7 @@ export enum RateLimiterMode {Extract = "extract",ExtractStatus = "extractStatus",ExtractAgentPreview = "extractAgentPreview",+ GeneratePaymentHeader = "generatePaymentHeader",}export type AuthResponse =