Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Merge main into claude/sec-lmstudio-allowprivate-arg; match cloud metadata on address bytes
Emre Sokullubd44d4e48bcf
diff --cc lmstudio-plugin/package.jsonindex deadac8d2,1df09a80e..394081519--- a/lmstudio-plugin/package.json+++ b/lmstudio-plugin/package.json@@@ -15,7 -15,7 +15,7 @@@"scripts": {"build": "tsc","dev": "tsc --watch",- "test": "npm run build && node --test test/browserTask.test.mjs test/privateNetworkSetting.test.mjs",- "test": "npm run build && node --test test/browserTask.test.mjs test/urlGuard.test.mjs test/toolParameters.test.mjs",++ "test": "npm run build && node --test test/browserTask.test.mjs test/urlGuard.test.mjs test/toolParameters.test.mjs test/privateNetworkSetting.test.mjs","clean": "rm -rf dist"},"keywords": [diff --cc lmstudio-plugin/src/util/safeFetch.tsindex 14f693641,ae06ed83f..e92713de4--- a/lmstudio-plugin/src/util/safeFetch.ts+++ b/lmstudio-plugin/src/util/safeFetch.ts@@@ -30,12 -30,11 +30,13 @@@*/import { lookup as dnsLookup } from "node:dns/promises";+ import { isIP } from "node:net";import {assertSafeUrl,+ isCloudMetadataHost,isPrivateIpv4,isPrivateIpv6,+ PRIVATE_NETWORK_HINT,type UrlGuardOptions,} from "./urlGuard.js";@@@ -91,9 -88,17 +91,22 @@@ async function assertHostnameDoesNotReshost = host.slice(1, -1);}- // Literal IP β the sync guard already validated. Skip DNS.- if (/^\d+\.\d+\.\d+\.\d+$/.test(host)) return;- if (host.includes(":")) return; // literal IPv6+ // Literal IP β nothing to resolve, but classify it here as well+ // rather than trusting the sync guard ran with the same rules.+ const literal = isIP(host);+ if (literal === 4 || literal === 6) {- if (literal === 4 ? isPrivateIpv4(host) : isPrivateIpv6(host)) {++ if (!opts.allowPrivate && (literal === 4 ? isPrivateIpv4(host) : isPrivateIpv6(host))) {++ throw new Error(++ `Refusing private IPv${literal} ${host} β ${PRIVATE_NETWORK_HINT}`,++ );++ }++ if (isCloudMetadataHost(host)) {+ throw new Error(- `Refusing private IPv${literal} ${host} β set allowPrivate to opt in.`,++ `Refusing cloud metadata endpoint ${host} β blocked even with private network access on.`,+ );+ }+ return;+ }let addresses: Array<{ address: string; family: number }>;try {diff --cc lmstudio-plugin/src/util/urlGuard.tsindex e893cd88e,26fca7d4e..42c1e9700--- a/lmstudio-plugin/src/util/urlGuard.ts+++ b/lmstudio-plugin/src/util/urlGuard.ts@@@ -12,9 -12,10 +12,11 @@@** `assertSafeUrl(url)` (sync) rejects:* - non-http(s) protocols (file://, ftp://, gopher://, javascript:)- * - literal RFC1918 / loopback / link-local / ULA IPv6 hostnames+ * - literal RFC1918 / loopback / link-local / ULA IPv6 hostnames,+ * including IPv6 forms that embed a private IPv4 (IPv4-mapped,+ * NAT64, 6to4, Teredo)* - common private DNS spellings (localhost, *.local, *.internal, *.lan)+ * - cloud metadata endpoints, even when allowPrivate is set** The DNS-resolution-aware check* (`assertHostnameDoesNotResolveToPrivate`) lives in `safeFetch.ts`@@@ -36,20 -37,17 +38,20 @@@** To opt-out (e.g. you really do want to point this at your intranet)* call assertSafeUrl(url, { allowPrivate: true }) and pass the same- * flag through to safeFetch.+ * flag through to safeFetch. In the plugin that flag comes only from+ * the user's "Allow private network access" setting, never from a tool+ * argument: a page the model just read must not be able to switch the+ * guard off. Cloud metadata endpoints stay blocked either way.*/- const PRIVATE_IPV4_RANGES: Array<[number, number]> = [- // [first byte, second byte]; second byte 0xFF means any- [10, 0xff], // 10.0.0.0/8- [127, 0xff], // 127.0.0.0/8 (loopback)- [169, 254], // 169.254.0.0/16 (link-local + cloud metadata)- [192, 168], // 192.168.0.0/16- [100, 64], // 100.64.0.0/10 carrier-grade NAT (we treat as private)- ];+ /*+ * IP literals are classified on their bytes, never on their spelling.+ * The WHATWG URL parser rewrites `[::ffff:127.0.0.1]` to+ * `[::ffff:7f00:1]` and `http://2130706433/` to `127.0.0.1`, and an+ * IPv6 address has many equivalent texts (`::1`, `0:0:0:0:0:0:0:1`,+ * `::0001`), so any string-prefix check misses forms that still+ * connect to the same host.+ */function ipv4ToBytes(ip: string): number[] | null {const m = ip.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/);@@@ -109,64 -207,6 +211,66 @@@ function isPrivateHostname(host: stringreturn false;}+// Cloud instance-metadata services hand out the machine's cloud+// credentials. Reaching a service on the user's own LAN is a reasonable+// opt-in; giving the model those credentials is not, so these stay- // blocked even with allowPrivate. IPv6 entries are in the canonical form- // the WHATWG URL parser produces.- const CLOUD_METADATA_IPV4 = new Set([++// blocked even with allowPrivate. Like the private ranges, they are++// matched on address bytes, so every spelling of an address (and every++// IPv6 form that embeds one) is covered.++const CLOUD_METADATA_IPV4: number[][] = [+ "169.254.169.254", // AWS, GCP, Azure, OCI, DigitalOcean, OpenStack and others+ "169.254.170.2", // AWS ECS task metadata and credentials+ "169.254.170.23", // AWS EKS Pod Identity agent+ "169.254.0.23", // Tencent Cloud+ "100.100.100.200", // Alibaba Cloud- ]);- const CLOUD_METADATA_IPV6 = new Set([- "fd00:ec2::254", // AWS IMDS over IPv6- "fd00:ec2::23", // AWS EKS Pod Identity agent over IPv6- ]);++].map((ip) => ipv4ToBytes(ip)!);++const CLOUD_METADATA_IPV6: number[][] = [++ v6("fd00:ec2::254"), // AWS IMDS over IPv6++ v6("fd00:ec2::23"), // AWS EKS Pod Identity agent over IPv6++];+const CLOUD_METADATA_HOSTNAMES = new Set([+ "metadata.google.internal",+ "metadata.goog",+ "metadata",+]);+++function sameBytes(a: number[], b: number[]): boolean {++ return a.length === b.length && a.every((x, i) => x === b[i]);++}++++function isCloudMetadataIpv4Bytes(bytes: number[]): boolean {++ return CLOUD_METADATA_IPV4.some((m) => sameBytes(m, bytes));++}+++/**+ * True for a cloud metadata endpoint, given a URL hostname (with or- * without IPv6 brackets) or a resolved address. An IPv4-mapped IPv6- * address counts as its IPv4 address.++ * without IPv6 brackets) or a resolved address. IPv6 addresses that embed++ * an IPv4 address (mapped, compatible, translated, NAT64, 6to4, Teredo)++ * count as that IPv4 address, using the same embedding table as the++ * private-range check.+ */+export function isCloudMetadataHost(host: string): boolean {+ let lower = host.toLowerCase();+ if (lower.startsWith("[") && lower.endsWith("]")) lower = lower.slice(1, -1);+ if (lower.endsWith(".")) lower = lower.slice(0, -1);- if (CLOUD_METADATA_IPV4.has(lower) || CLOUD_METADATA_HOSTNAMES.has(lower)) return true;- if (!lower.includes(":")) return false;-- // Canonicalise through the URL parser so fd00:ec2:0:0:0:0:0:254 and- // ::ffff:169.254.169.254 compare equal to the forms listed above.- let canonical: string;- try {- canonical = new URL(`http://[${lower}]/`).hostname.slice(1, -1);- } catch {- return false;- }- if (CLOUD_METADATA_IPV6.has(canonical)) return true;- const mapped = canonical.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);- if (!mapped?.[1] || !mapped[2]) return false;- const hi = parseInt(mapped[1], 16);- const lo = parseInt(mapped[2], 16);- return CLOUD_METADATA_IPV4.has(`${hi >> 8}.${hi & 0xff}.${lo >> 8}.${lo & 0xff}`);++ if (CLOUD_METADATA_HOSTNAMES.has(lower)) return true;++ const v4 = ipv4ToBytes(lower);++ if (v4) return isCloudMetadataIpv4Bytes(v4);++ const bytes = ipv6ToBytes(lower);++ if (!bytes) return false;++ if (CLOUD_METADATA_IPV6.some((m) => sameBytes(m, bytes))) return true;++ return IPV4_EMBEDDING_RANGES.some(([prefix, bits, embedded]) =>++ inPrefix(bytes, prefix, bits) && embedded(bytes).some(isCloudMetadataIpv4Bytes));+}++/**+ * Appended to private-target refusals. The model reads tool errors, so+ * this tells it what to relay to the user instead of what to retry.+ */+export const PRIVATE_NETWORK_HINT =+ "private network access is off. The user can turn on \"Allow private " ++ "network access\" in this plugin's settings in LM Studio.";+export interface UrlGuardOptions {/*** Set true to allow URLs that resolve to RFC1918 / loopback / link-@@@ -195,12 -233,7 +299,17 @@@ export function assertSafeUrl(input: st`Refusing ${parsed.protocol} URL β this plugin only fetches http(s).`,);}- if (isCloudMetadataHost(parsed.hostname)) {- throw new Error(- `Refusing cloud metadata endpoint ${parsed.hostname} β blocked even with private network access on.`,- );- if (opts.allowPrivate) return parsed;++ const refuseMetadata = (): void => {++ if (isCloudMetadataHost(parsed.hostname)) {++ throw new Error(++ `Refusing cloud metadata endpoint ${parsed.hostname} β blocked even with private network access on.`,++ );++ }++ };++ if (opts.allowPrivate) {++ refuseMetadata();++ return parsed;+ }- if (opts.allowPrivate) return parsed;// hostname for IPv6 comes back wrapped in [] β strip them.let host = parsed.hostname;@@@ -220,8 -253,8 +329,10 @@@}if (isPrivateHostname(host)) {throw new Error(- `Refusing private hostname ${host} β set allowPrivate to opt in.`,+ `Refusing private hostname ${host} β ${PRIVATE_NETWORK_HINT}`,);}++ // Metadata hostnames that are not private spellings (metadata.goog).++ refuseMetadata();return parsed;}diff --cc lmstudio-plugin/test/privateNetworkSetting.test.mjsindex 93ff5686b,000000000..8a486d286mode 100644,000000..100644--- a/lmstudio-plugin/test/privateNetworkSetting.test.mjs+++ b/lmstudio-plugin/test/privateNetworkSetting.test.mjs@@@ -1,156 -1,0 +1,201 @@@+import assert from "node:assert/strict";+import { once } from "node:events";+import { createServer } from "node:http";+import { createServer as createNetServer } from "node:net";+import { after, before, test } from "node:test";++import { main, readNetworkOptions } from "../dist/index.js";+import { fetchUrl } from "../dist/tools/fetchUrl.js";+import { researchUrl } from "../dist/tools/researchUrl.js";++import { assertSafeUrl, isCloudMetadataHost } from "../dist/util/urlGuard.js";++import { safeFetch } from "../dist/util/safeFetch.js";+import { sharedBridge } from "../dist/util/bridgeClient.js";++let toolsProvider;+let configSchematicsSet;+let server;+let origin;+let requests = 0;++async function freePort() {+ const probe = createNetServer();+ probe.listen(0, "127.0.0.1");+ await once(probe, "listening");+ const address = probe.address();+ const port = typeof address === "object" && address ? address.port : 0;+ probe.close();+ await once(probe, "close");+ return port;+}++// Mirrors LM Studio's ToolsProviderController: it parses the chat's stored+// plugin config, sent over the wire as { fields: [{ key, value }] }.+function controller(fields) {+ return {+ getPluginConfig: (schematics) => schematics.parse({ fields }),+ };+}++before(async () => {+ process.env.WEBBRAIN_BRIDGE_PORT = String(await freePort());+ const context = {+ withConfigSchematics(schematics) {+ configSchematicsSet = schematics;+ return context;+ },+ withToolsProvider(provider) {+ toolsProvider = provider;+ return context;+ },+ };+ await main(context);++ server = createServer((req, res) => {+ requests += 1;+ if (req.url === "/to-metadata") {+ res.writeHead(302, { Location: "http://169.254.169.254/latest/meta-data/" });+ res.end();+ return;+ }+ res.writeHead(200, { "Content-Type": "text/html" });+ res.end(+ "<html><head><title>Router</title></head><body><main>" ++ "Local admin page. ".repeat(10) ++ "</main></body></html>",+ );+ });+ server.listen(0, "127.0.0.1");+ await once(server, "listening");+ origin = `http://127.0.0.1:${server.address().port}`;+});++after(async () => {+ server?.close();+ await sharedBridge().stop();+});++test("private network access is a user plugin setting that defaults to off", () => {+ assert.ok(configSchematicsSet, "main() registers the plugin config schematics");+ assert.equal(configSchematicsSet.parse({ fields: [] }).get("allowPrivateNetwork"), false);+ assert.deepEqual(readNetworkOptions(controller([])), { allowPrivate: false });+ assert.deepEqual(+ readNetworkOptions(controller([{ key: "allowPrivateNetwork", value: true }])),+ { allowPrivate: true },+ );+ assert.deepEqual(+ readNetworkOptions({+ getPluginConfig() {+ throw new Error("config unavailable");+ },+ }),+ { allowPrivate: false },+ );+});++test("the HTTP tools do not offer allowPrivate to the model", async () => {+ const tools = await toolsProvider(controller([{ key: "allowPrivateNetwork", value: true }]));+ for (const name of ["fetch_url", "research_url"]) {+ const registered = tools.find((registeredTool) => registeredTool.name === name);+ assert.ok(registered, name);+ assert.equal("allowPrivate" in registered.parametersSchema.shape, false, name);+ }+});++test("a model-sent allowPrivate cannot reach a loopback service", async () => {+ requests = 0;+ // fetchUrl/researchUrl take the model's arguments as-is; a stray flag there is ignored.+ const fetched = await fetchUrl({ url: `${origin}/`, allowPrivate: true });+ assert.equal(fetched.success, false);+ assert.match(fetched.error, /private network access is off/);++ const researched = await researchUrl({ url: `${origin}/`, allowPrivate: true });+ assert.equal(researched.success, false);+ assert.match(researched.error, /private network access is off/);++ assert.equal(requests, 0);+});++test("the user's setting opens loopback targets to both HTTP tools", async () => {+ const network = readNetworkOptions(controller([{ key: "allowPrivateNetwork", value: true }]));++ const fetched = await fetchUrl({ url: `${origin}/` }, network);+ assert.equal(fetched.success, true);+ assert.equal(fetched.title, "Router");++ const researched = await researchUrl({ url: `${origin}/` }, network);+ assert.equal(researched.success, true);+ assert.match(researched.text, /Local admin page/);+});++test("cloud metadata endpoints stay blocked with private network access on", async () => {+ requests = 0;+ const network = { allowPrivate: true };+ for (const url of [+ "http://169.254.169.254/latest/meta-data/",+ "http://2852039166/latest/meta-data/",+ "http://[::ffff:169.254.169.254]/latest/meta-data/",+ "http://[fd00:ec2::254]/latest/meta-data/",+ "http://169.254.170.2/v2/credentials/",+ "http://100.100.100.200/latest/meta-data/",+ "http://metadata.google.internal/computeMetadata/v1/",+ ]) {+ const result = await fetchUrl({ url }, network);+ assert.equal(result.success, false, url);+ assert.match(result.error, /cloud metadata endpoint/, url);+ }++ const researched = await researchUrl(+ { url: "http://169.254.169.254/latest/meta-data/" },+ network,+ );+ assert.equal(researched.success, false);+ assert.match(researched.error, /cloud metadata endpoint/);++ const redirected = await fetchUrl({ url: `${origin}/to-metadata` }, network);+ assert.equal(redirected.success, false);+ assert.match(redirected.error, /cloud metadata endpoint/);+ assert.equal(requests, 1);+});++++test("cloud metadata endpoints are matched on address bytes, in every IPv6 form that embeds them", () => {++ const blocked = [++ "169.254.169.254",++ "[::ffff:a9fe:a9fe]", // IPv4-mapped (WHATWG form)++ "[::ffff:169.254.169.254]", // IPv4-mapped, dotted++ "[0:0:0:0:0:ffff:a9fe:a9fe]", // IPv4-mapped, uncompressed++ "[::a9fe:a9fe]", // IPv4-compatible++ "[::ffff:0:a9fe:a9fe]", // IPv4-translated++ "[64:ff9b::a9fe:a9fe]", // NAT64++ "[64:ff9b::169.254.169.254]", // NAT64, dotted++ "[2002:a9fe:a9fe::1]", // 6to4++ "[2001:0:a9fe:a9fe::f7f7:f7f7]", // Teredo server++ "[::ffff:6464:64c8]", // mapped Alibaba 100.100.100.200++ "[fd00:ec2:0:0:0:0:0:254]", // AWS IPv6, uncompressed++ "[FD00:EC2::23]", // AWS EKS IPv6, upper-case++ "metadata.google.internal.", // trailing dot++ ];++ for (const host of blocked) {++ assert.equal(isCloudMetadataHost(host), true, host);++ assert.throws(() => assertSafeUrl(`http://${host}/`, { allowPrivate: true }), /cloud metadata/, host);++ }++ for (const host of ["8.8.8.8", "[::ffff:808:808]", "[64:ff9b::808:808]", "[2002:808:808::1]",++ "169.254.169.253", "[fd00:ec2::255]", "192.168.1.1", "[::1]"]) {++ assert.equal(isCloudMetadataHost(host), false, host);++ assert.doesNotThrow(() => assertSafeUrl(`http://${host}/`, { allowPrivate: true }), host);++ }++});++++test("with the setting on, safeFetch's own literal-IP check lets loopback through but not metadata", async () => {++ requests = 0;++ const loopback = origin.replace("localhost", "127.0.0.1");++ const res = await safeFetch(`${loopback}/`, { allowPrivate: true });++ assert.equal(res.status, 200);++ await res.body?.cancel();++ assert.equal(requests, 1);++ await assert.rejects(safeFetch(`${loopback}/`), /private network access is off/);++ assert.equal(requests, 1, "still blocked with the setting off");++ await assert.rejects(++ safeFetch("http://[64:ff9b::a9fe:a9fe]/latest/meta-data/", { allowPrivate: true }),++ /cloud metadata/,++ );++});