Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: (#2216)
Nicolas3aaa2f11d3b5
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 0988c9962..bfaed4071 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -125,6 +125,7 @@"rate-limiter-flexible": "2.4.2","redlock": "5.0.0-beta.2","resend": "^3.4.0",+ "response-time": "^2.3.4","robots-parser": "^3.0.1","stripe": "^16.1.0","systeminformation": "^5.27.8",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 9d29c14f7..33dfa2983 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -3,7 +3,6 @@ lockfileVersion: '9.0'settings:autoInstallPeers: trueexcludeLinksFromLockfile: false- injectWorkspacePackages: trueoverrides:brace-expansion@>=2.0.0 <=2.0.1: 2.0.2@@ -204,6 +203,9 @@ importers:resend:specifier: ^3.4.0version: 3.4.0+ response-time:+ specifier: ^2.3.4+ version: 2.3.4robots-parser:specifier: ^3.0.1version: 3.0.1@@ -1415,49 +1417,42 @@ packages:engines: {node: '>= 10'}cpu: [arm64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-yWjcPDgJ2nIL3KNvi4536dlT/CcCWO0DUyEOlBs/SacG7BeD6IjGh6yYzd3/X1Y3JItCbZoDoLUH8iB1lTXo3w==}engines: {node: '>= 10'}cpu: [arm64]os: [linux]- libc: [musl]'@napi-rs/[email protected]':resolution: {integrity: sha512-0XRhKuIU/9ZjT4WDIG/qnX7Xz7mSQHYZo9Gb3MP2gcvBgr6BA4zywQ9k3gmQaPn9ECE+CZg2V7DV7kT+x2pUMQ==}engines: {node: '>= 10'}cpu: [ppc64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-QrqDIPEUUB23GCpyQj/QFyMlr8SGxxyExeZz9OWFnHfb70kXdTLWrHS/hEI1Ru+lSbQ/6xRqeoGyQ4Aqdg+/RA==}engines: {node: '>= 10'}cpu: [riscv64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-k8RVM5aMhW86E9H0QXdquwojew4H3SwPxbRVbl49/COJQWCUjGi79X6mYruMnMPEznZinUiT1jgKbFo2A00NdA==}engines: {node: '>= 10'}cpu: [s390x]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-6rMtBgnIq2Wcl1rQdZsnM+rtCcVCbws1nF8S2NzaUsVaZv8bjrPiAa0lwg4Eqnn1d9lgwqT+cZgm5m+//K08Kw==}engines: {node: '>= 10'}cpu: [x64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-eiadGBKi7Vd0bCArBUOO/qqRYPHt/VQVvGyYvDFt6C2ZSIjlD+HuOl+2oS1sjf4CFjK4eDIog6EdXnL0NE6iyQ==}engines: {node: '>= 10'}cpu: [x64]os: [linux]- libc: [musl]'@napi-rs/[email protected]':resolution: {integrity: sha512-+VyHHlr68dvey6fXc2hehw9gHVFIW3TtGF1XkcbAu65qVXsA9D/T+uuoRVqhE+JCyFHFrO0ixRbZDRK1XJt1sA==}@@ -1527,42 +1522,36 @@ packages:engines: {node: '>= 10'}cpu: [arm64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-L/y1/26q9L/uBqiW/JdOb/Dc94egFvNALUZV2WCGKQXc6UByPBMgdiEyW2dtoYxYYYYc+AKD+jr+wQPcvX2vrQ==}engines: {node: '>= 10'}cpu: [arm64]os: [linux]- libc: [musl]'@napi-rs/[email protected]':resolution: {integrity: sha512-EPE1K/80RQvPbLRJDJs1QmCIcH+7WRi0F73+oTe1582y9RtfGRuzAkzeBuAGRXAQEjRQw/RjtNqr6UTJ+8UuWQ==}engines: {node: '>= 10'}cpu: [ppc64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-B2jhWiB1ffw1nQBqLUP1h4+J1ovAxBOoe5N2IqDMOc63fsPZKNqF1PvO/dIem8z7LL4U4bsfmhy3gBfu547oNQ==}engines: {node: '>= 10'}cpu: [s390x]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-tbZDHnb9617lTnsDMGo/eAMZxnsQFnaRe+MszRqHguKfMwkisc9CCJnks/r1o84u5fECI+J/HOrKXgczq/3Oww==}engines: {node: '>= 10'}cpu: [x64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-dV6cODlzbO8u6Anmv2N/ilQHq/AWz0xyltuXoLU3yUyXbZcnWYZuB2rL8OBGPmqNcD+x9NdScBNXh7vWN0naSQ==}engines: {node: '>= 10'}cpu: [x64]os: [linux]- libc: [musl]'@napi-rs/[email protected]':resolution: {integrity: sha512-jIa9nb2HzOrfH0F8QQ9g3WE4aMH5vSI5/1NYVNm9ysCmNjCCtMXCAhlI3WKCdm/DwHf0zLqdrrtDFXODcNaqMw==}@@ -1629,28 +1618,24 @@ packages:engines: {node: '>= 10'}cpu: [arm64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-jAasbIvjZXCgX0TCuEFQr+4D6Lla/3AAVx2LmDuMjgG4xoIXzjKWl7c4chuaD+TI+prWT0X6LJcdzFT+ROKGHQ==}engines: {node: '>= 10'}cpu: [arm64]os: [linux]- libc: [musl]'@napi-rs/[email protected]':resolution: {integrity: sha512-Plgk5rPqqK2nocBGajkMVbGm010Z7dnUgq0wtnYRZbzWWxwWcXfZMPa8EYxrK4eE8SzpI7VlZP1tdVsdjgGwMw==}engines: {node: '>= 10'}cpu: [x64]os: [linux]- libc: [glibc]'@napi-rs/[email protected]':resolution: {integrity: sha512-GW7AzGuWxtQkyHknHWYFdR0CHmW6is8rG2Rf4V6GNmMpmwtXt/ItWYWtBe4zqJWycMNazpfZKSw/BpT7/MVCXQ==}engines: {node: '>= 10'}cpu: [x64]os: [linux]- libc: [musl]'@napi-rs/[email protected]':resolution: {integrity: sha512-/nQVSTrqSsn7YdAc2R7Ips/tnw5SPUcl3D7QrXCNGPqjbatIspnaexvaOYNyKMU6xPu+pc0BTnKVmqhlJJCPLA==}@@ -2469,49 +2454,41 @@ packages:resolution: {integrity: sha512-ukHZp9Vm07AlxqdOLFf8Bj4inzpt+ISbbODvwwHxX32GfcMLWYYJGAYWc13IGhWoElvWnI7D1M9ifDGyTNRGzg==}cpu: [arm64]os: [linux]- libc: [glibc]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-atkZ1OIt6t90kjQz1iqq6cN3OpfPG5zUJlO64Vd1ieYeqHRkOFeRgnWEobTePUHi34NlYr7mNZqIaAg7gjPUFg==}cpu: [arm64]os: [linux]- libc: [musl]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-HGgV4z3JwVF4Qvg2a1GhDnqn8mKLihy5Gp4rMfqNIAlERPSyIxo8oPQIL1XQKLYyyrkEEO99uwM+4cQGwhtbpQ==}cpu: [ppc64]os: [linux]- libc: [glibc]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-+vCO7iOR1s6VGefV02R2a702IASNWhSNm/MrR8RcWjKChmU0G+d1iC0oToUrGC4ovAEfstx2/O8EkROnfcLgrA==}cpu: [riscv64]os: [linux]- libc: [glibc]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-3folNmS5gYNFy/9HYzLcdeThqAGvDJU0gQKrhHn7RPWQa58yZ0ZPpBMk6KRSSO61+wkchkL+0sdcLsoe5wZW8g==}cpu: [riscv64]os: [linux]- libc: [musl]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-Ceo4z6g8vqPUKADROFL0b7MoyXlUdOBYCxTDu/fhd/5I3Ydk2S6bxkjJdzpBdlu+h2Z+eS9lTHFvkwkaORMPzw==}cpu: [s390x]os: [linux]- libc: [glibc]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-QyFW5e43imQLxiBpCImhOiP4hY9coWGjroEm8elDqGNNaA7vXooaMQS2N3avMQawSaKhsb/3RemxaZ852XG38Q==}cpu: [x64]os: [linux]- libc: [glibc]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-JhuCqCqktqQyQVc37V+eDiP3buCIuyCLpb92tUEyAP8nY3dy2b/ojMrH1ZNnJUlfY/67AqoZPL6nQGAB2WA3Sg==}cpu: [x64]os: [linux]- libc: [musl]'@oxc-resolver/[email protected]':resolution: {integrity: sha512-sMXm5Z2rfBwkCUespZBJCPhCVbgh/fpYQ23BQs0PmnvWoXrGQHWvnvg1p/GYmleN+nwe8strBjfutirZFiC5lA==}@@ -2547,25 +2524,21 @@ packages:resolution: {integrity: sha512-N1FqdKfwhVWPpMElv8qlGqdEefTbDYaRVhdGWOjs/2f7FESa5vX0cvA7ToqzkoXyXZI5DqByWiPML33njK30Kg==}cpu: [arm64]os: [linux]- libc: [glibc]'@oxlint/[email protected]':resolution: {integrity: sha512-v/BPuiateLBb7Gz1STb69EWjkgKdlPQ1NM56z+QQur21ly2hiMkBX2n0zEhqfu9PQVRUizu6AlsYuzcPY/zsIQ==}cpu: [arm64]os: [linux]- libc: [musl]'@oxlint/[email protected]':resolution: {integrity: sha512-gUTp8KIrSYt97dn+tRRC3LKnH4xlHKCwrPwiDcGmLbCxojuN9/H5mnIhPKEfwNuZNdoKGS/ABuq3neVyvRCRtQ==}cpu: [x64]os: [linux]- libc: [glibc]'@oxlint/[email protected]':resolution: {integrity: sha512-DpN6cW2HPjYXeENG0JBbmubO8LtfKt6qJqEMBw9gUevbyBaX+k+Jn7sYgh6S23wGOkzmTNphBsf/7ulj4nIVYA==}cpu: [x64]os: [linux]- libc: [musl]'@oxlint/[email protected]':resolution: {integrity: sha512-oXxJksnUTUMgJ0NvjKS1mrCXAy1ttPgIVacRSlxQ+1XHy+aJDMM7I8fsCtoKoEcTIpPaD98eqUqlLYs0H2MGjA==}@@ -5504,6 +5477,10 @@ packages:resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}engines: {node: '>= 0.8'}+ resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==}+ engines: {node: '>= 0.8'}+resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}@@ -5983,6 +5960,10 @@ packages:resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}hasBin: true+ resolution: {integrity: sha512-fiyq1RvW5/Br6iAtT8jN1XrNY8WPu2+yEypLbaijWry8WDZmn12azG9p/+c+qpEebURLlQmqCB8BNSu7ji+xQQ==}+ engines: {node: '>= 0.8.0'}+resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==}engines: {node: '>=18'}@@ -13806,6 +13787,8 @@ snapshots:dependencies:ee-first: 1.1.1+ [email protected]: {}+dependencies:wrappy: 1.0.2@@ -14344,6 +14327,11 @@ snapshots:path-parse: 1.0.7supports-preserve-symlinks-flag: 1.0.0+ dependencies:+ depd: 2.0.0+ on-headers: 1.1.0+dependencies:onetime: 7.0.0diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 615a6537c..26dac68c1 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -19,6 +19,11 @@ export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,res: Response<ScrapeResponse>,) {+ // Get timing data from middleware (includes all middleware processing time)+ const middlewareStartTime =+ (req as any).requestTiming?.startTime || new Date().getTime();+ const controllerStartTime = new Date().getTime();+const jobId: string = uuidv4();const preNormalizedBody = { ...req.body };req.body = scrapeRequestSchema.parse(req.body);@@ -43,7 +48,10 @@ export async function scrapeController(zeroDataRetention,});+ const middlewareTime = controllerStartTime - middlewareStartTime;+logger.debug("Scrape " + jobId + " starting", {+ version: "v1",scrapeId: jobId,request: req.body,originalRequest: preNormalizedBody,@@ -89,7 +97,7 @@ export async function scrapeController(},origin,integration: req.body.integration,- startTime,+ startTime: controllerStartTime,zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,},@@ -118,7 +126,7 @@ export async function scrapeController();} catch (e) {logger.error(`Error in scrapeController`, {- startTime,+ version: "v1",error: e,});@@ -153,6 +161,18 @@ export async function scrapeController(}}+ const totalRequestTime = new Date().getTime() - middlewareStartTime;+ const controllerTime = new Date().getTime() - controllerStartTime;+ logger.info("Request metrics", {+ version: "v1",+ scrapeId: jobId,+ middlewareStartTime,+ controllerStartTime,+ middlewareTime,+ controllerTime,+ totalRequestTime,+ });+return res.status(200).json({success: true,data: doc,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex b4d3dd6e0..26917217c 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -223,6 +223,10 @@ export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,res: Response<SearchResponse>,) {+ // Get timing data from middleware (includes all middleware processing time)+ const middlewareStartTime = (req as any).requestTiming?.startTime || new Date().getTime();+ const controllerStartTime = new Date().getTime();+const jobId = uuidv4();let logger = _logger.child({jobId,@@ -245,7 +249,7 @@ export async function searchController(success: true,data: [],};- const startTime = new Date().getTime();+ const middlewareTime = controllerStartTime - middlewareStartTime;const isSearchPreview =process.env.SEARCH_PREVIEW_TOKEN !== undefined &&process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;@@ -257,6 +261,7 @@ export async function searchController(req.body = searchRequestSchema.parse(req.body);logger = logger.child({+ version: "v1",query: req.body.query,origin: req.body.origin,});@@ -411,7 +416,7 @@ export async function searchController(}const endTime = new Date().getTime();- const timeTakenInSeconds = (endTime - startTime) / 1000;+ const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;logger.info("Logging job", {num_docs: responseData.data.length,@@ -443,6 +448,20 @@ export async function searchController(isSearchPreview,);+ // Log final timing information+ const totalRequestTime = new Date().getTime() - middlewareStartTime;+ const controllerTime = new Date().getTime() - controllerStartTime;+ logger.info("Search completed successfully", {+ version: "v1",+ jobId,+ middlewareStartTime,+ controllerStartTime,+ middlewareTime,+ controllerTime,+ totalRequestTime,+ creditsUsed: credits_billed,+ });+return res.status(200).json(responseData);} catch (error) {if (error instanceof ScrapeJobTimeoutError) {@@ -454,7 +473,10 @@ export async function searchController(}Sentry.captureException(error);- logger.error("Unhandled error occurred in search", { error });+ logger.error("Unhandled error occurred in search", {+ version: "v1",+ error+ });return res.status(500).json({success: false,error: error.message,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 1eff281e4..2bdb1ccbf 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -19,6 +19,10 @@ export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,res: Response<ScrapeResponse>,) {+ // Get timing data from middleware (includes all middleware processing time)+ const middlewareStartTime = (req as any).requestTiming?.startTime || new Date().getTime();+ const controllerStartTime = new Date().getTime();+const jobId = uuidv4();const preNormalizedBody = { ...req.body };req.body = scrapeRequestSchema.parse(req.body);@@ -43,7 +47,10 @@ export async function scrapeController(zeroDataRetention,});+ const middlewareTime = controllerStartTime - middlewareStartTime;+logger.debug("Scrape " + jobId + " starting", {+ version: "v2",scrapeId: jobId,request: req.body,originalRequest: preNormalizedBody,@@ -53,7 +60,6 @@ export async function scrapeController(const origin = req.body.origin;const timeout = req.body.timeout;- const startTime = new Date().getTime();const isDirectToBullMQ =process.env.SEARCH_PREVIEW_TOKEN !== undefined &&@@ -89,7 +95,7 @@ export async function scrapeController(},origin,integration: req.body.integration,- startTime,+ startTime: controllerStartTime,zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,},@@ -115,7 +121,7 @@ export async function scrapeController();} catch (e) {logger.error(`Error in scrapeController`, {- startTime,+ version: "v2",error: e,});@@ -138,6 +144,7 @@ export async function scrapeController(}await scrapeQueue.removeJob(jobId, logger);+if (!hasFormatOfType(req.body.formats, "rawHtml")) {if (doc && doc.rawHtml) {@@ -145,6 +152,19 @@ export async function scrapeController(}}+ const totalRequestTime = new Date().getTime() - middlewareStartTime;+ const controllerTime = new Date().getTime() - controllerStartTime;+ logger.info("Request metrics", {+ version: "v2",+ scrapeId: jobId,+ middlewareStartTime,+ controllerStartTime,+ middlewareTime,+ controllerTime,+ totalRequestTime,+ });++return res.status(200).json({success: true,data: doc,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex c7bb77835..6e90d0d8f 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -200,6 +200,11 @@ export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,res: Response<SearchResponse>,) {+ // Get timing data from middleware (includes all middleware processing time)+ const middlewareStartTime =+ (req as any).requestTiming?.startTime || new Date().getTime();+ const controllerStartTime = new Date().getTime();+const jobId = uuidv4();let logger = _logger.child({jobId,@@ -217,7 +222,7 @@ export async function searchController(});}- const startTime = new Date().getTime();+ const middlewareTime = controllerStartTime - middlewareStartTime;const isSearchPreview =process.env.SEARCH_PREVIEW_TOKEN !== undefined &&process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;@@ -228,6 +233,7 @@ export async function searchController(req.body = searchRequestSchema.parse(req.body);logger = logger.child({+ version: "v2",query: req.body.query,origin: req.body.origin,});@@ -469,7 +475,7 @@ export async function searchController(credits_billed = allJobIds.length; // Just for reporting, not billingconst endTime = new Date().getTime();- const timeTakenInSeconds = (endTime - startTime) / 1000;+ const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;logger.info("Logging job (async scraping)", {num_docs: credits_billed,@@ -505,6 +511,20 @@ export async function searchController(isSearchPreview,);+ // Log final timing information for async mode+ const totalRequestTime = new Date().getTime() - middlewareStartTime;+ const controllerTime = new Date().getTime() - controllerStartTime;+ logger.info("Search completed successfully (async)", {+ version: "v2",+ jobId,+ middlewareStartTime,+ controllerStartTime,+ middlewareTime,+ controllerTime,+ totalRequestTime,+ creditsUsed: credits_billed,+ });+return res.status(200).json({success: true,data: searchResponse,@@ -612,7 +632,7 @@ export async function searchController(}const endTime = new Date().getTime();- const timeTakenInSeconds = (endTime - startTime) / 1000;+ const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;logger.info("Logging job", {num_docs: credits_billed,@@ -648,6 +668,21 @@ export async function searchController(isSearchPreview,);+ // Log final timing information+ const totalRequestTime = new Date().getTime() - middlewareStartTime;+ const controllerTime = new Date().getTime() - controllerStartTime;++ logger.info("Request metrics", {+ version: "v2",+ jobId,+ middlewareStartTime,+ controllerStartTime,+ middlewareTime,+ controllerTime,+ totalRequestTime,+ creditsUsed: credits_billed,+ });+// For sync scraping or no scraping, don't include scrapeIdsreturn res.status(200).json({success: true,@@ -673,7 +708,10 @@ export async function searchController(}Sentry.captureException(error);- logger.error("Unhandled error occurred in search", { error });+ logger.error("Unhandled error occurred in search", {+ version: "v2",+ error,+ });return res.status(500).json({success: false,error: error.message,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 9eb6822a3..7818205bb 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -40,6 +40,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { nuqShutdown } from "./services/worker/nuq";import { getErrorContactMessage } from "./lib/deployment";import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";+import responseTime from "response-time";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -97,6 +98,8 @@ app.use(bodyParser.json({ limit: "10mb" }));app.use(cors()); // Add this line to enable CORS+app.use(responseTime());+if (process.env.EXPRESS_TRUST_PROXY) {app.set("trust proxy", parseInt(process.env.EXPRESS_TRUST_PROXY, 10));}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex fd6ee7306..97e80223e 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -243,6 +243,40 @@ export function countryCheck(next();}+export function requestTimingMiddleware(version: string) {+ return (req: Request, res: Response, next: NextFunction) => {+ const startTime = new Date().getTime();++ // Attach timing data to request+ (req as any).requestTiming = {+ startTime,+ version+ };++ // Override res.json to log timing when response is sent+ const originalJson = res.json.bind(res);+ res.json = function(body: any) {+ const requestTime = new Date().getTime() - startTime;++ // Only log for successful responses to avoid duplicate error logs+ if (body?.success !== false) {+ logger.info(`${version} request completed`, {+ version,+ path: req.path,+ method: req.method,+ startTime,+ requestTime,+ statusCode: res.statusCode,+ });+ }++ return originalJson(body);+ };++ next();+ };+}+export function wrap(controller: (req: Request, res: Response) => Promise<any>,): (req: Request, res: Response, next: NextFunction) => any {diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 502787928..770624b45 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -30,6 +30,7 @@ import {blocklistMiddleware,countryCheck,idempotencyMiddleware,+ requestTimingMiddleware,wrap,} from "./shared";import { paymentMiddleware } from "x402-express";@@ -42,6 +43,9 @@ expressWs(express());export const v1Router = express.Router();+// Add timing middleware to all v1 routes+v1Router.use(requestTimingMiddleware("v1"));+// Configure payment middleware to enable micropayment-protected endpoints// This middleware handles payment verification and processing for premium API features// x402 payments protocol - https://github.com/coinbase/x402diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 78afcb14f..d0365bebf 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -24,6 +24,7 @@ import {blocklistMiddleware,countryCheck,idempotencyMiddleware,+ requestTimingMiddleware,wrap,} from "./shared";import { queueStatusController } from "../controllers/v2/queue-status";@@ -34,6 +35,9 @@ expressWs(express());export const v2Router = express.Router();+// Add timing middleware to all v2 routes+v2Router.use(requestTimingMiddleware("v2"));+v2Router.post("/search",authMiddleware(RateLimiterMode.Search),