Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Replace custom address validation functions with ipaddr.js (ENG-3404) (#2084)

tom52a8fad01607
diff --git a/apps/api/package.json b/apps/api/package.json
index 6f1ddfe91..b92c3bdcf 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -114,7 +114,7 @@
"gpt3-tokenizer": "^1.1.5",
"http-cookie-agent": "^7.0.1",
"ioredis": "^5.6.1",
- "ip-address": "^10.0.1",
+ "ipaddr.js": "^2.2.0",
"joplin-turndown-plugin-gfm": "^1.0.12",
"jsdom": "^26.0.0",
"json-schema-to-zod": "^2.3.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 00d04d88b..c05d3dbd8 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -185,9 +185,9 @@ importers:
ioredis:
specifier: ^5.6.1
version: 5.6.1
- ip-address:
- specifier: ^10.0.1
- version: 10.0.1
+ ipaddr.js:
+ specifier: ^2.2.0
+ version: 2.2.0
joplin-turndown-plugin-gfm:
specifier: ^1.0.12
version: 1.0.12
@@ -4237,10 +4237,6 @@ packages:
resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}
engines: {node: '>=12.22.0'}
- resolution: {integrity: sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==}
- engines: {node: '>= 12'}
-
resolution: {integrity: sha512-c5uxc2WUTuRBVHT/6r4m7HIr/DfV0bF6DvLH3iZGSK8wp8iMwwZSgIq2do0asFf8q9ECug0SE+6+1ACMe4sorA==}
engines: {node: '>= 0.10'}
@@ -4249,6 +4245,10 @@ packages:
resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
engines: {node: '>= 0.10'}
+ resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==}
+ engines: {node: '>= 10'}
+
resolution: {integrity: sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==}
@@ -10203,17 +10203,17 @@ snapshots:
dependencies:
'@types/yargs-parser': 21.0.3
dependencies:
'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])
cbw-sdk: '@coinbase/[email protected]'
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -10245,11 +10245,11 @@ snapshots:
- utf-8-validate
- zod
dependencies:
eventemitter3: 5.0.1
mipd: 0.0.7([email protected])
optionalDependencies:
'@tanstack/query-core': 5.83.1
@@ -10799,6 +10799,11 @@ snapshots:
typescript: 5.8.3
zod: 3.22.4
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
optionalDependencies:
typescript: 5.8.3
@@ -12214,8 +12219,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
-
dependencies:
jsbn: 1.1.0
@@ -12228,6 +12231,8 @@ snapshots:
+
@@ -13214,6 +13219,21 @@ snapshots:
transitivePeerDependencies:
- zod
+ dependencies:
+ '@adraffy/ens-normalize': 1.11.0
+ '@noble/ciphers': 1.3.0
+ '@noble/curves': 1.9.6
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ eventemitter3: 5.0.1
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - zod
+
dependencies:
'@adraffy/ens-normalize': 1.11.0
@@ -14263,6 +14283,23 @@ snapshots:
- utf-8-validate
- zod
+ dependencies:
+ '@noble/curves': 1.9.2
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - bufferutil
+ - utf-8-validate
+ - zod
+
dependencies:
'@noble/curves': 1.9.2
@@ -14287,11 +14324,11 @@ snapshots:
dependencies:
'@tanstack/react-query': 5.84.1([email protected])
react: 18.3.1
use-sync-external-store: 1.4.0([email protected])
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -14494,7 +14531,7 @@ snapshots:
dependencies:
zod: 3.25.76
transitivePeerDependencies:
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index 0bb48c3c0..b18d9af7c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -1,9 +1,9 @@
import type { Socket } from "net";
import type { TLSSocket } from "tls";
import * as undici from "undici";
-import { Address6 } from "ip-address";
import { CookieJar } from "tough-cookie";
import { cookie } from "http-cookie-agent/undici";
+import IPAddr from "ipaddr.js";
export class InsecureConnectionError extends Error {
constructor() {
@@ -11,65 +11,41 @@ export class InsecureConnectionError extends Error {
}
}
-function isIPv4Private(address: string): boolean {
- const parts = address.split(".").map((x) => parseInt(x, 10));
- return (
- parts[0] === 0 || // Current (local, "this") network
- parts[0] === 10 || // Used for local communications within a private network
- (parts[0] === 100 && parts[1] >= 64 && parts[1] < 128) || // Shared address space for communications between a service provider and its subscribers when using a carrier-grade NAT
- parts[0] === 127 || // Used for loopback addresses to the local host
- (parts[0] === 169 && parts[1] === 254) || // Used for link-local addresses between two hosts on a single link when no IP address is otherwise specified, such as would have normally been retrieved from a DHCP server
- (parts[0] === 127 && parts[1] >= 16 && parts[2] < 32) || // Used for local communications within a private network
- (parts[0] === 192 && parts[1] === 0 && parts[2] === 0) || // IETF Porotocol Assignments, DS-Lite (/29)
- (parts[0] === 192 && parts[1] === 0 && parts[2] === 2) || // Assigned as TEST-NET-1, documentation and examples
- (parts[0] === 192 && parts[1] === 88 && parts[2] === 99) || // Reserved. Formerly used for IPv6 to IPv4 relay (included IPv6 address block 2002::/16).
- (parts[0] === 192 && parts[1] === 168) || // Used for local communications within a private network
- (parts[0] === 192 && parts[1] >= 18 && parts[1] < 20) || // Used for benchmark testing of inter-network communications between two separate subnets
- (parts[0] === 198 && parts[1] === 51 && parts[2] === 100) || // Assigned as TEST-NET-2, documentation and examples
- (parts[0] === 203 && parts[1] === 0 && parts[2] === 113) || // Assigned as TEST-NET-3, documentation and examples
- (parts[0] >= 224 && parts[0] < 240) || // In use for multicast (former Class D network)
- (parts[0] === 233 && parts[1] === 252 && parts[2] === 0) || // Assigned as MCAST-TEST-NET, documentation and examples (Note that this is part of the above multicast space.)
- parts[0] >= 240 || // Reserved for future use (former class E network)
- (parts[0] === 255 &&
- parts[1] === 255 &&
- parts[2] === 255 &&
- parts[3] === 255)
- ); // Reserved for the "limited broadcast" destination address
-}
+export function isIPPrivate(address: string): boolean {
+ if (!IPAddr.isValid(address)) return false;
-function isIPv6Private(ipv6) {
- return new Address6(ipv6).getScope() !== "Global";
+ const addr = IPAddr.parse(address);
+ return addr.range() !== "unicast";
}
-export function makeSecureDispatcher(
- skipTlsVerification: boolean = false,
-) {
+export function makeSecureDispatcher(skipTlsVerification: boolean = false) {
const agentOpts: undici.Agent.Options = {
maxRedirections: 5000,
};
const baseAgent = process.env.PROXY_SERVER
? new undici.ProxyAgent({
- uri: process.env.PROXY_SERVER.includes("://") ? process.env.PROXY_SERVER : ("http://" + process.env.PROXY_SERVER),
- token: process.env.PROXY_USERNAME
- ? `Basic ${Buffer.from(process.env.PROXY_USERNAME + ":" + (process.env.PROXY_PASSWORD ?? "")).toString("base64")}`
- : undefined,
- requestTls: {
- rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested
- },
- ...agentOpts,
- })
+ uri: process.env.PROXY_SERVER.includes("://")
+ ? process.env.PROXY_SERVER
+ : "http://" + process.env.PROXY_SERVER,
+ token: process.env.PROXY_USERNAME
+ ? `Basic ${Buffer.from(process.env.PROXY_USERNAME + ":" + (process.env.PROXY_PASSWORD ?? "")).toString("base64")}`
+ : undefined,
+ requestTls: {
+ rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested
+ },
+ ...agentOpts,
+ })
: new undici.Agent({
- connect: {
- rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested
- },
- ...agentOpts,
- });
+ connect: {
+ rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested
+ },
+ ...agentOpts,
+ });
const cookieJar = new CookieJar();
- const agent = baseAgent
- .compose(cookie({ jar: cookieJar }));
+ const agent = baseAgent.compose(cookie({ jar: cookieJar }));
agent.on("connect", (_, targets) => {
const client: undici.Client = targets.slice(-1)[0] as undici.Client;
@@ -78,14 +54,8 @@ export function makeSecureDispatcher(
)!;
const socket: Socket | TLSSocket = (client as any)[socketSymbol];
- if (socket.remoteAddress) {
- if (
- socket.remoteFamily === "IPv4"
- ? isIPv4Private(socket.remoteAddress!)
- : isIPv6Private(socket.remoteAddress!)
- ) {
- socket.destroy(new InsecureConnectionError());
- }
+ if (socket.remoteAddress && isIPPrivate(socket.remoteAddress)) {
+ socket.destroy(new InsecureConnectionError());
}
});
@@ -93,4 +63,4 @@ export function makeSecureDispatcher(
}
export const secureDispatcher = makeSecureDispatcher(false);
-export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);
\ No newline at end of file
+export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);
diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.ts
index 8a4eccd7e..005d3061b 100644
--- a/apps/api/src/services/webhook.ts
+++ b/apps/api/src/services/webhook.ts
@@ -1,5 +1,5 @@
import undici from "undici";
-import { secureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";
+import { isIPPrivate, secureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";
import { logger as _logger, logger } from "../lib/logger";
import { supabase_rr_service, supabase_service } from "./supabase";
import { WebhookEventType } from "../types";
@@ -150,6 +150,16 @@ export const callWebhook = async ({
return null;
}
+ // check if the webhook URL is a private IP address *before* making the request
+ // the dispatcher also performs a check once connected, however this prevents unnecessary connections
+ const webhookHost = new URL(webhookUrl.url).hostname;
+ if (isIPPrivate(webhookHost)) {
+ logger.warn("Aborting webhook call to private IP address", {
+ url: webhookUrl.url,
+ });
+ return null;
+ }
+
let dataToSend: any[] = [];
if (
data &&