Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Replace custom address validation functions with ipaddr.js (ENG-3404) (#2084)
tom52a8fad01607
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 6f1ddfe91..b92c3bdcf 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -114,7 +114,7 @@"gpt3-tokenizer": "^1.1.5","http-cookie-agent": "^7.0.1","ioredis": "^5.6.1",- "ip-address": "^10.0.1",+ "ipaddr.js": "^2.2.0","joplin-turndown-plugin-gfm": "^1.0.12","jsdom": "^26.0.0","json-schema-to-zod": "^2.3.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 00d04d88b..c05d3dbd8 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -185,9 +185,9 @@ importers:ioredis:specifier: ^5.6.1version: 5.6.1- ip-address:- specifier: ^10.0.1- version: 10.0.1+ ipaddr.js:+ specifier: ^2.2.0+ version: 2.2.0joplin-turndown-plugin-gfm:specifier: ^1.0.12version: 1.0.12@@ -4237,10 +4237,6 @@ packages:resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}engines: {node: '>=12.22.0'}- resolution: {integrity: sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==}- engines: {node: '>= 12'}-resolution: {integrity: sha512-c5uxc2WUTuRBVHT/6r4m7HIr/DfV0bF6DvLH3iZGSK8wp8iMwwZSgIq2do0asFf8q9ECug0SE+6+1ACMe4sorA==}engines: {node: '>= 0.10'}@@ -4249,6 +4245,10 @@ packages:resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}engines: {node: '>= 0.10'}+ resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==}+ engines: {node: '>= 10'}+resolution: {integrity: sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==}@@ -10203,17 +10203,17 @@ snapshots:dependencies:'@types/yargs-parser': 21.0.3- '@wagmi/[email protected](@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])':+ '@wagmi/[email protected](@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])':dependencies:'@base-org/account': 1.1.1([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])'@coinbase/wallet-sdk': 4.3.6([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])'@safe-global/safe-apps-sdk': 9.1.0([email protected])([email protected])([email protected])([email protected])- '@wagmi/core': 2.18.1(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/core': 2.18.1(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))'@walletconnect/ethereum-provider': 2.21.1([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])cbw-sdk: '@coinbase/[email protected]'optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -10245,11 +10245,11 @@ snapshots:- utf-8-validate- zod- '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':+ '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':dependencies:eventemitter3: 5.0.1mipd: 0.0.7([email protected])optionalDependencies:'@tanstack/query-core': 5.83.1@@ -10799,6 +10799,11 @@ snapshots:typescript: 5.8.3zod: 3.22.4+ optionalDependencies:+ typescript: 5.8.3+ zod: 3.24.2+optionalDependencies:typescript: 5.8.3@@ -12214,8 +12219,6 @@ snapshots:transitivePeerDependencies:- supports-color- [email protected]: {}-dependencies:jsbn: 1.1.0@@ -12228,6 +12231,8 @@ snapshots:+ [email protected]: {}+@@ -13214,6 +13219,21 @@ snapshots:transitivePeerDependencies:- zod+ dependencies:+ '@adraffy/ens-normalize': 1.11.0+ '@noble/ciphers': 1.3.0+ '@noble/curves': 1.9.6+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.0.8([email protected])([email protected])+ eventemitter3: 5.0.1+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - zod+dependencies:'@adraffy/ens-normalize': 1.11.0@@ -14263,6 +14283,23 @@ snapshots:- utf-8-validate- zod+ dependencies:+ '@noble/curves': 1.9.2+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.0.8([email protected])([email protected])+ ox: 0.8.6([email protected])([email protected])+ ws: 8.18.2([email protected])([email protected])+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - bufferutil+ - utf-8-validate+ - zod+dependencies:'@noble/curves': 1.9.2@@ -14287,11 +14324,11 @@ snapshots:[email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]):dependencies:'@tanstack/react-query': 5.84.1([email protected])- '@wagmi/connectors': 5.9.1(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])- '@wagmi/core': 2.18.1(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/connectors': 5.9.1(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])+ '@wagmi/core': 2.18.1(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))react: 18.3.1use-sync-external-store: 1.4.0([email protected])optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -14494,7 +14531,7 @@ snapshots:[email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:wagmi: 2.16.1(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])zod: 3.25.76transitivePeerDependencies:diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex 0bb48c3c0..b18d9af7c 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -1,9 +1,9 @@import type { Socket } from "net";import type { TLSSocket } from "tls";import * as undici from "undici";-import { Address6 } from "ip-address";import { CookieJar } from "tough-cookie";import { cookie } from "http-cookie-agent/undici";+import IPAddr from "ipaddr.js";export class InsecureConnectionError extends Error {constructor() {@@ -11,65 +11,41 @@ export class InsecureConnectionError extends Error {}}-function isIPv4Private(address: string): boolean {- const parts = address.split(".").map((x) => parseInt(x, 10));- return (- parts[0] === 0 || // Current (local, "this") network- parts[0] === 10 || // Used for local communications within a private network- (parts[0] === 100 && parts[1] >= 64 && parts[1] < 128) || // Shared address space for communications between a service provider and its subscribers when using a carrier-grade NAT- parts[0] === 127 || // Used for loopback addresses to the local host- (parts[0] === 169 && parts[1] === 254) || // Used for link-local addresses between two hosts on a single link when no IP address is otherwise specified, such as would have normally been retrieved from a DHCP server- (parts[0] === 127 && parts[1] >= 16 && parts[2] < 32) || // Used for local communications within a private network- (parts[0] === 192 && parts[1] === 0 && parts[2] === 0) || // IETF Porotocol Assignments, DS-Lite (/29)- (parts[0] === 192 && parts[1] === 0 && parts[2] === 2) || // Assigned as TEST-NET-1, documentation and examples- (parts[0] === 192 && parts[1] === 88 && parts[2] === 99) || // Reserved. Formerly used for IPv6 to IPv4 relay (included IPv6 address block 2002::/16).- (parts[0] === 192 && parts[1] === 168) || // Used for local communications within a private network- (parts[0] === 192 && parts[1] >= 18 && parts[1] < 20) || // Used for benchmark testing of inter-network communications between two separate subnets- (parts[0] === 198 && parts[1] === 51 && parts[2] === 100) || // Assigned as TEST-NET-2, documentation and examples- (parts[0] === 203 && parts[1] === 0 && parts[2] === 113) || // Assigned as TEST-NET-3, documentation and examples- (parts[0] >= 224 && parts[0] < 240) || // In use for multicast (former Class D network)- (parts[0] === 233 && parts[1] === 252 && parts[2] === 0) || // Assigned as MCAST-TEST-NET, documentation and examples (Note that this is part of the above multicast space.)- parts[0] >= 240 || // Reserved for future use (former class E network)- (parts[0] === 255 &&- parts[1] === 255 &&- parts[2] === 255 &&- parts[3] === 255)- ); // Reserved for the "limited broadcast" destination address-}+export function isIPPrivate(address: string): boolean {+ if (!IPAddr.isValid(address)) return false;-function isIPv6Private(ipv6) {- return new Address6(ipv6).getScope() !== "Global";+ const addr = IPAddr.parse(address);+ return addr.range() !== "unicast";}-export function makeSecureDispatcher(- skipTlsVerification: boolean = false,-) {+export function makeSecureDispatcher(skipTlsVerification: boolean = false) {const agentOpts: undici.Agent.Options = {maxRedirections: 5000,};const baseAgent = process.env.PROXY_SERVER? new undici.ProxyAgent({- uri: process.env.PROXY_SERVER.includes("://") ? process.env.PROXY_SERVER : ("http://" + process.env.PROXY_SERVER),- token: process.env.PROXY_USERNAME- ? `Basic ${Buffer.from(process.env.PROXY_USERNAME + ":" + (process.env.PROXY_PASSWORD ?? "")).toString("base64")}`- : undefined,- requestTls: {- rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested- },- ...agentOpts,- })+ uri: process.env.PROXY_SERVER.includes("://")+ ? process.env.PROXY_SERVER+ : "http://" + process.env.PROXY_SERVER,+ token: process.env.PROXY_USERNAME+ ? `Basic ${Buffer.from(process.env.PROXY_USERNAME + ":" + (process.env.PROXY_PASSWORD ?? "")).toString("base64")}`+ : undefined,+ requestTls: {+ rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested+ },+ ...agentOpts,+ }): new undici.Agent({- connect: {- rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested- },- ...agentOpts,- });+ connect: {+ rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested+ },+ ...agentOpts,+ });const cookieJar = new CookieJar();- const agent = baseAgent- .compose(cookie({ jar: cookieJar }));+ const agent = baseAgent.compose(cookie({ jar: cookieJar }));agent.on("connect", (_, targets) => {const client: undici.Client = targets.slice(-1)[0] as undici.Client;@@ -78,14 +54,8 @@ export function makeSecureDispatcher()!;const socket: Socket | TLSSocket = (client as any)[socketSymbol];- if (socket.remoteAddress) {- if (- socket.remoteFamily === "IPv4"- ? isIPv4Private(socket.remoteAddress!)- : isIPv6Private(socket.remoteAddress!)- ) {- socket.destroy(new InsecureConnectionError());- }+ if (socket.remoteAddress && isIPPrivate(socket.remoteAddress)) {+ socket.destroy(new InsecureConnectionError());}});@@ -93,4 +63,4 @@ export function makeSecureDispatcher(}export const secureDispatcher = makeSecureDispatcher(false);-export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);\ No newline at end of file+export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.tsindex 8a4eccd7e..005d3061b 100644--- a/apps/api/src/services/webhook.ts+++ b/apps/api/src/services/webhook.ts@@ -1,5 +1,5 @@import undici from "undici";-import { secureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";+import { isIPPrivate, secureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";import { logger as _logger, logger } from "../lib/logger";import { supabase_rr_service, supabase_service } from "./supabase";import { WebhookEventType } from "../types";@@ -150,6 +150,16 @@ export const callWebhook = async ({return null;}+ // check if the webhook URL is a private IP address *before* making the request+ // the dispatcher also performs a check once connected, however this prevents unnecessary connections+ const webhookHost = new URL(webhookUrl.url).hostname;+ if (isIPPrivate(webhookHost)) {+ logger.warn("Aborting webhook call to private IP address", {+ url: webhookUrl.url,+ });+ return null;+ }+let dataToSend: any[] = [];if (data &&