Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Safe public-URL fetch — SSRF-guarded text/binary fetch for "go grab this
* remote thing the user pointed us at" flows (RSS feeds, remote images, …).
*
* Centralizes the guard so new callers reuse ONE implementation instead of
* copying the host-parsing logic a fourth time (it already lives, subtly
* differently, in feeds.js `fetchFeedXml` and catalogIngestSources.js
* `assertIngestUrlSafe`). The hard part — classifying a host literal as
* loopback/link-local/cloud-metadata across IPv4 / IPv6 / IPv4-mapped forms —
* is reused from `catalogValidation.isBlockedIngestHost`; this module adds the
* DNS-resolve check and the redirect-revalidating fetch wrappers on top.
*
* Posture (matches the catalog ingest gate): block non-http(s) schemes,
* loopback, link-local, and the cloud-metadata endpoint; ALLOW other
* private/LAN hosts (a single-user tool legitimately reaches Tailscale peers /
* home wikis). A redirect to a blocked target fails CLOSED (returns null) — the
* landed hop is revalidated exactly like the first.
*
* Strict posture (`blockPrivate: true`): additionally reject ALL private/LAN
* addresses (RFC1918, CGNAT-adjacent, IPv6 ULA fc00::/7, …) — both host literals
* and DNS-resolved addresses. RSS feeds use this so an arbitrary user-added feed
* URL can't be pointed at the home network, preserving feeds.js's historical
* block-all-private posture while gaining the connect-time IP pin.
*/
import dns from 'dns/promises';
import { Agent, Dispatcher1Wrapper } from 'undici';
import { ServerError } from './errorHandler.js';
import { fetchWithTimeout } from './fetchWithTimeout.js';
import { isSafeIngestUrl, isBlockedIngestHost } from './catalogValidation.js';
const DEFAULT_TIMEOUT_MS = 15000;
const DEFAULT_MAX_BYTES = 12 * 1024 * 1024; // 12 MB — generous for a single image/feed
/**
* Classify an IP literal as private/loopback/link-local (IPv4 or IPv6) for the
* strict `blockPrivate` posture. `isBlockedIngestHost` only covers
* loopback/link-local/metadata (the default posture ALLOWS LAN); this widens the
* net to the full RFC1918 / CGNAT-adjacent / IPv6-ULA ranges so a feed URL can't
* reach the home network. An unparseable / empty value fails closed (private).
*/
export function isPrivateAddress(ip) {
if (!ip) return true;
const lower = ip.toLowerCase();
if (lower.includes(':')) {
if (lower === '::1' || lower === '::') return true;
// Embedded IPv4 — IPv4-mapped (::ffff:…) OR the deprecated IPv4-compatible
// (::…) form, either dotted (::ffff:1.2.3.4) or the two-hextet form URL
// parsing normalizes to (::ffff:c0a8:101 AND the ffff-less ::c0a8:101 that
// `new URL('http://[::192.168.1.1]')` yields). Decode and re-check as IPv4 so
// a private v4 embedded in an IPv6 literal can't slip past. Mirrors the
// embedded-v4 handling in catalogValidation.isBlockedIngestHost.
const embedded = /^::(?:ffff:)?(.+)$/i.exec(lower);
if (embedded) {
const tail = embedded[1];
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(tail)) return isPrivateAddress(tail);
const parts = tail.split(':');
if (parts.length === 2 && parts.every((p) => /^[0-9a-f]{1,4}$/.test(p))) {
const hi = parseInt(parts[0], 16);
const lo = parseInt(parts[1], 16);
return isPrivateAddress(`${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`);
}
}
const firstGroup = lower.split(':')[0];
if (firstGroup) {
const firstWord = parseInt(firstGroup, 16);
if (Number.isFinite(firstWord)) {
if ((firstWord & 0xfe00) === 0xfc00) return true; // ULA fc00::/7
if ((firstWord & 0xffc0) === 0xfe80) return true; // link-local fe80::/10
}
}
return false;
}
const parts = lower.split('.').map(Number);
if (parts.length === 4 && parts.every((p) => !isNaN(p) && p >= 0 && p <= 255)) {
if (parts[0] === 10) return true;
if (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) return true;
if (parts[0] === 192 && parts[1] === 168) return true;
if (parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) return true; // CGNAT 100.64/10 (Tailscale et al.)
if (parts[0] === 169 && parts[1] === 254) return true;
if (parts[0] === 127) return true;
if (parts[0] === 0) return true;
}
return false;
}
/**
* Core SSRF resolve: scheme + blocked-host-literal (sync, via isSafeIngestUrl),
* AND a single DNS resolve so a hostname whose A record points at a blocked
* address (cloud metadata / loopback / link-local) is rejected too. Returns the
* vetted IP so the caller can PIN it at connect time — closing the DNS-rebinding
* TOCTOU where a re-resolution at connect could land on a private address the
* validation lookup never saw. Never throws.
*
* `address` is null only for an IP-literal target (the connect goes straight to
* the literal — there's no name to re-resolve, so no TOCTOU and nothing to pin).
* A hostname that can't be resolved fails CLOSED ({ safe: false }) rather than
* falling through to an unpinned fetch: an unpinned connect would re-resolve at
* connect time, exactly the rebinding window this guard exists to close (an
* attacker can fail the validation lookup, then answer a private IP at connect).
*
* @returns {Promise<{ safe: boolean, address?: string|null, family?: number }>}
*/
async function resolvePublicHttpUrl(target, { blockPrivate = false } = {}) {
if (!isSafeIngestUrl(target)) return { safe: false };
const { hostname } = new URL(target);
// Host literals were already classified by isSafeIngestUrl and need no pinning
// (net connects straight to the literal — there's no name to rebind).
const isIpLiteral = /^[\d.]+$/.test(hostname) || hostname.includes(':');
if (isIpLiteral) {
// Strict posture must still reject a private LITERAL (e.g. http://192.168.1.5)
// that isSafeIngestUrl deliberately allows in the default LAN-friendly gate.
const literal = hostname.replace(/^\[|\]$/g, '');
if (blockPrivate && isPrivateAddress(literal)) return { safe: false };
return { safe: true, address: null };
}
const resolved = await dns.lookup(hostname).catch(() => null);
if (!resolved?.address) return { safe: false }; // can't vet → can't safely pin → fail closed
if (isBlockedIngestHost(resolved.address)) return { safe: false };
if (blockPrivate && isPrivateAddress(resolved.address)) return { safe: false };
return { safe: true, address: resolved.address, family: resolved.family };
}
/**
* Boolean SSRF gate. Returns false on any failure — never throws — so redirect
* revalidation can fail closed. `blockPrivate` opts into the strict posture.
*/
export async function isPublicHttpUrlSafe(target, { blockPrivate = false } = {}) {
return (await resolvePublicHttpUrl(target, { blockPrivate })).safe;
}
/**
* net.lookup-compatible function that ALWAYS yields the SSRF-vetted IP, ignoring
* the hostname undici would otherwise re-resolve. This is what closes the
* rebinding TOCTOU: undici's connector calls this instead of DNS, so the socket
* goes to exactly the address the guard approved, while Host header + TLS SNI
* stay the original hostname (undici's connector derives `servername` from the
* requested host, not from the resolved address), so cert validation is
* unaffected. undici asks with `all: true`, so honor both the array and
* single-address callback shapes.
*
* undici 8 kept `connect.lookup` on the built-in connector — the v8 port did not
* change how the pin itself is expressed, only how the dispatcher is handed to
* fetch (see buildPinnedDispatcher).
*/
export function buildPinnedLookup(address, family) {
const fam = family || (address.includes(':') ? 6 : 4);
return (_hostname, options, callback) => {
const cb = typeof options === 'function' ? options : callback;
const entry = { address, family: fam };
cb(null, options?.all ? [entry] : entry.address, entry.family);
};
}
/**
* One-shot dispatcher that pins every connection to the vetted IP. Created
* per-request and left for GC after the body is consumed (single-user,
* low-volume tool — a pooled keep-alive agent would only hold a stale socket).
*
* The `Dispatcher1Wrapper` is REQUIRED, not decorative. We hand this dispatcher
* to the runtime's global `fetch` (via fetchWithTimeout), and Node's *bundled*
* undici drives a dispatcher with whichever request-handler contract that Node
* version speaks. undici 8 dropped v7's implicit wrapping of the legacy
* (onConnect/onHeaders/onComplete) handler shape, so a bare undici-8 `Agent`
* handed to the built-in fetch of any Node that still bundles undici 7 (22, 24,
* 25 — including the 22.22.2 minimum CI job) throws `InvalidArgumentError: invalid
* onRequestStart method` and the request never leaves. Since fetchGuarded
* swallows network errors to null, that failure would be SILENT: every guarded
* fetch would return null rather than fall back to an unpinned connect, but the
* feature would be dead. `Dispatcher1Wrapper` is undici 8's documented adapter
* for exactly this (the same one `setGlobalDispatcher` mirrors for Node's
* built-in fetch) and is accepted by both the legacy and the v8 handler
* contracts, so one build works across Node 22→26.
*/
export function buildPinnedDispatcher(address, family) {
return new Dispatcher1Wrapper(new Agent({ connect: { lookup: buildPinnedLookup(address, family) } }));
}
// Fetch the URL with its connection pinned to the SSRF-vetted address (when we
// have one), falling back to the runtime's own resolution otherwise.
function pinnedFetch(url, resolved, options, timeoutMs) {
const fetchOptions = resolved.address
? { ...options, dispatcher: buildPinnedDispatcher(resolved.address, resolved.family) }
: options;
return fetchWithTimeout(url, fetchOptions, timeoutMs).catch(() => null);
}
function throwUnsafeUrl() {
throw new ServerError('refusing to fetch a non-http(s) or loopback/link-local URL', {
status: 400,
code: 'UNSAFE_URL',
});
}
/**
* Throwing variant for the FIRST hop (the user-supplied / stored URL) so a bad
* target surfaces a clean 400 at the route instead of a silent null.
*/
export async function assertPublicHttpUrl(target, { blockPrivate = false } = {}) {
if (!await isPublicHttpUrlSafe(target, { blockPrivate })) throwUnsafeUrl();
}
// Fetch with the first-hop gate + manual redirect revalidation (fails closed).
// Every hop is resolved AND pinned, so the connect can't re-resolve to a blocked
// address after the check passes. Returns the Response, or null on a network
// error / blocked redirect / missing Location. The caller decides what to do
// with a non-ok status. First-hop unsafe THROWS a clean 400 by default (routes
// want that); `throwOnUnsafe: false` makes it fail closed to null instead — for
// callers (RSS feeds) whose own contract is a friendly "couldn't fetch" result
// rather than a bubbled error.
async function fetchGuarded(url, { timeoutMs = DEFAULT_TIMEOUT_MS, headers, blockPrivate = false, throwOnUnsafe = true } = {}) {
const first = await resolvePublicHttpUrl(url, { blockPrivate });
if (!first.safe) {
if (throwOnUnsafe) throwUnsafeUrl();
return null;
}
const res = await pinnedFetch(url, first, { redirect: 'manual', headers }, timeoutMs);
if (res && res.status >= 300 && res.status < 400) {
const location = res.headers.get('location');
if (!location) return null;
// A malformed Location must fail closed to null (not throw) so the
// throwOnUnsafe:false contract holds for the redirect hop too — same
// guarded-parse pattern as isSafeIngestUrl.
let redirectUrl;
try { redirectUrl = new URL(location, url).href; } catch { return null; }
const hop = await resolvePublicHttpUrl(redirectUrl, { blockPrivate });
if (!hop.safe) return null;
return pinnedFetch(redirectUrl, hop, { redirect: 'error', headers }, timeoutMs);
}
return res;
}
/**
* Resolve a short URL without reading its body. Every hop is DNS-vetted and
* pinned, with a bounded redirect count and a shared request-time budget.
* `allowUrl(URL)` can restrict every hop to a service's own hosts. Returns the
* final 2xx URL, or null on an unsafe/disallowed hop or resolution failure.
*/
export async function resolvePublicUrl(url, {
timeoutMs = DEFAULT_TIMEOUT_MS, maxRedirects = 3, headers,
blockPrivate = false, allowUrl,
} = {}) {
if (!Number.isInteger(maxRedirects) || maxRedirects < 0 || maxRedirects > 10) return null;
const deadline = Date.now() + timeoutMs;
let current = url;
for (let redirects = 0; redirects <= maxRedirects; redirects++) {
if (!URL.canParse(current)) return null;
const target = new URL(current);
if (allowUrl && !allowUrl(target)) return null;
const resolved = await resolvePublicHttpUrl(target.href, { blockPrivate });
const remainingMs = deadline - Date.now();
if (!resolved.safe || remainingMs <= 0) return null;
const res = await pinnedFetch(target.href, resolved, { redirect: 'manual', headers }, remainingMs);
if (!res) return null;
// Headers are all we need; do not download a Pinterest page (or retain a
// redirect response's socket) just to learn its destination.
await res.body?.cancel().catch(() => {});
if (res.ok) return target.href;
if (res.status < 300 || res.status >= 400 || redirects === maxRedirects) return null;
const location = res.headers.get('location');
if (!location || !URL.canParse(location, target.href)) return null;
current = new URL(location, target.href).href;
}
return null;
}
/**
* Read a response body into a Buffer bounded by `maxBytes`, or null when the
* body exceeds the cap. The cap is enforced first via Content-Length (cheap
* early-out) and then bounds PEAK MEMORY by streaming the body and aborting the
* moment accumulated bytes exceed it — so a server that omits or lies about
* Content-Length can't stream gigabytes into memory before a post-read check
* fires. Falls back to a buffered read only when the response exposes no
* stream (e.g. a test double). Shared by fetchPublicBinary and the opt-in
* fetchPublicText cap.
*/
export async function readBodyCapped(res, maxBytes) {
const declared = Number(res.headers.get('content-length'));
if (maxBytes && Number.isFinite(declared) && declared > maxBytes) return null;
if (!res.body || typeof res.body.getReader !== 'function') {
// No readable stream (some runtimes / test doubles) — fall back to a
// buffered read, still capped post-read.
const buffer = Buffer.from(await res.arrayBuffer());
if (maxBytes && buffer.byteLength > maxBytes) return null;
return buffer;
}
const reader = res.body.getReader();
const chunks = [];
let total = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
total += value.byteLength;
if (maxBytes && total > maxBytes) {
await reader.cancel();
return null;
}
chunks.push(Buffer.from(value));
}
} finally {
try { reader.releaseLock(); } catch { /* already released by cancel() */ }
}
return Buffer.concat(chunks);
}
/**
* Fetch with every hop vetted and DNS-pinned, returning the raw Response of ANY
* status (3xx without a followable Location excluded) so callers that classify
* on status (privacy scan) keep it. Returns null on a network error or a
* blocked/invalid redirect. The caller owns reading the body.
*/
export function fetchPublicResponse(url, { timeoutMs, headers, blockPrivate = false, throwOnUnsafe = true } = {}) {
return fetchGuarded(url, { timeoutMs, headers, blockPrivate, throwOnUnsafe });
}
/**
* Fetch a URL's body as text. Returns the string on a 2xx, or null on any
* failure (network error, non-ok status, blocked redirect). `blockPrivate` opts
* into the strict block-all-private posture and `throwOnUnsafe: false` returns
* null (instead of throwing a 400) when the first hop is unsafe — both used by
* RSS feed fetches. `maxBytes` (opt-in — no default, so existing callers keep
* the plain `res.text()` behavior) streams the body through the shared cap and
* returns null when it's exceeded, bounding peak memory exactly like
* fetchPublicBinary.
*/
export async function fetchPublicText(url, { timeoutMs, headers, maxBytes, blockPrivate = false, throwOnUnsafe = true } = {}) {
const res = await fetchGuarded(url, { timeoutMs, headers, blockPrivate, throwOnUnsafe });
if (!res?.ok) return null;
if (!maxBytes) return res.text();
const buffer = await readBodyCapped(res, maxBytes);
return buffer === null ? null : buffer.toString('utf-8');
}
/**
* Fetch a URL's body as a Buffer. Returns `{ buffer, contentType }` on a 2xx
* within the size cap, or null on any failure (see readBodyCapped for the
* streaming peak-memory bound).
*/
export async function fetchPublicBinary(url, { timeoutMs, headers, maxBytes = DEFAULT_MAX_BYTES, blockPrivate = false, throwOnUnsafe = true } = {}) {
const res = await fetchGuarded(url, { timeoutMs, headers, blockPrivate, throwOnUnsafe });
if (!res?.ok) return null;
const contentType = res.headers.get('content-type') || '';
const buffer = await readBodyCapped(res, maxBytes);
return buffer === null ? null : { buffer, contentType };
}