Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Tests for the SSRF-guarded public-URL fetch helpers. The DNS resolver and the
* underlying fetchWithTimeout are mocked so the guard + redirect-revalidation
* logic is exercised without a network.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import http from 'node:http';
const lookupMock = vi.fn();
vi.mock('dns/promises', () => ({
default: { lookup: (...a) => lookupMock(...a) },
lookup: (...a) => lookupMock(...a),
}));
const fetchMock = vi.fn();
vi.mock('./fetchWithTimeout.js', () => ({
fetchWithTimeout: (...a) => fetchMock(...a),
}));
const {
isPublicHttpUrlSafe, assertPublicHttpUrl, fetchPublicText, fetchPublicBinary, buildPinnedLookup,
buildPinnedDispatcher, resolvePublicUrl,
} = await import('./safeUrlFetch.js');
const res = ({ ok = true, status = 200, headers = {}, text = '', body = new ArrayBuffer(0) } = {}) => ({
ok,
status,
headers: new Map(Object.entries(headers)), // Headers-like: supports .get()
text: async () => text,
arrayBuffer: async () => body,
});
beforeEach(() => {
lookupMock.mockReset();
fetchMock.mockReset();
lookupMock.mockResolvedValue({ address: '93.184.216.34' }); // public
});
describe('isPublicHttpUrlSafe', () => {
it('accepts a public https URL', async () => {
expect(await isPublicHttpUrlSafe('https://www.pinterest.com/x.rss')).toBe(true);
});
it('rejects non-http(s) schemes', async () => {
expect(await isPublicHttpUrlSafe('file:///etc/passwd')).toBe(false);
expect(await isPublicHttpUrlSafe('ftp://host/x')).toBe(false);
});
it('rejects loopback / metadata host literals without resolving', async () => {
expect(await isPublicHttpUrlSafe('http://127.0.0.1/x')).toBe(false);
expect(await isPublicHttpUrlSafe('http://169.254.169.254/latest/meta-data')).toBe(false);
expect(lookupMock).not.toHaveBeenCalled();
});
it('rejects a hostname that RESOLVES to a blocked address', async () => {
lookupMock.mockResolvedValue({ address: '127.0.0.1' });
expect(await isPublicHttpUrlSafe('https://evil.example.com/x')).toBe(false);
});
});
describe('isPublicHttpUrlSafe — strict blockPrivate posture', () => {
it('allows a private/LAN host by default but rejects it under blockPrivate', async () => {
lookupMock.mockResolvedValue({ address: '192.168.1.50', family: 4 });
expect(await isPublicHttpUrlSafe('https://nas.local/feed')).toBe(true);
expect(await isPublicHttpUrlSafe('https://nas.local/feed', { blockPrivate: true })).toBe(false);
});
it('rejects a private IPv4 LITERAL under blockPrivate (no resolve)', async () => {
expect(await isPublicHttpUrlSafe('http://192.168.1.5/x')).toBe(true); // LAN allowed by default
expect(await isPublicHttpUrlSafe('http://192.168.1.5/x', { blockPrivate: true })).toBe(false);
expect(await isPublicHttpUrlSafe('http://10.1.2.3/x', { blockPrivate: true })).toBe(false);
expect(lookupMock).not.toHaveBeenCalled();
});
it('rejects an IPv6 ULA literal under blockPrivate', async () => {
expect(await isPublicHttpUrlSafe('http://[fd12:3456:789a::1]/x', { blockPrivate: true })).toBe(false);
});
it('rejects a CGNAT 100.64/10 literal (Tailscale) under blockPrivate', async () => {
expect(await isPublicHttpUrlSafe('http://100.100.100.200/x', { blockPrivate: true })).toBe(false);
expect(await isPublicHttpUrlSafe('http://100.64.0.1/x', { blockPrivate: true })).toBe(false);
// 100.63.x / 100.128.x are public — must NOT be blocked (literal, no resolve).
expect(await isPublicHttpUrlSafe('http://100.63.0.1/x', { blockPrivate: true })).toBe(true);
expect(await isPublicHttpUrlSafe('http://100.128.0.1/x', { blockPrivate: true })).toBe(true);
});
it('rejects a CGNAT/private address a hostname RESOLVES to under blockPrivate', async () => {
lookupMock.mockResolvedValue({ address: '100.100.42.7', family: 4 });
expect(await isPublicHttpUrlSafe('https://tailnet.example.com/x', { blockPrivate: true })).toBe(false);
});
it('rejects the deprecated IPv4-compatible IPv6 form of a private v4 under blockPrivate', async () => {
// new URL('http://[::192.168.1.1]') normalizes to hostname [::c0a8:101] —
// the ffff-less compatible form. It must still decode to 192.168.1.1.
expect(await isPublicHttpUrlSafe('http://[::192.168.1.1]/x', { blockPrivate: true })).toBe(false);
expect(await isPublicHttpUrlSafe('http://[::10.0.0.1]/x', { blockPrivate: true })).toBe(false);
});
it('rejects a hostname that RESOLVES to a private address under blockPrivate', async () => {
lookupMock.mockResolvedValue({ address: '10.0.0.42', family: 4 });
expect(await isPublicHttpUrlSafe('https://home.example.com/x', { blockPrivate: true })).toBe(false);
});
it('still allows a genuinely public host under blockPrivate', async () => {
lookupMock.mockResolvedValue({ address: '93.184.216.34', family: 4 });
expect(await isPublicHttpUrlSafe('https://example.com/x', { blockPrivate: true })).toBe(true);
});
});
describe('fetchPublicText — strict blockPrivate + throwOnUnsafe:false (RSS feed path)', () => {
it('returns null (no fetch, no throw) for a host that resolves private', async () => {
lookupMock.mockResolvedValue({ address: '10.0.0.5', family: 4 });
fetchMock.mockResolvedValue(res({ text: 'secret' }));
// The feeds path opts out of the 400 throw so its caller can show a friendly
// "couldn't fetch" message; the request must still never leave the box.
expect(await fetchPublicText('https://home.example.com/feed', { blockPrivate: true, throwOnUnsafe: false })).toBeNull();
expect(fetchMock).not.toHaveBeenCalled();
});
it('still THROWS a 400 for an unsafe first hop by default (route contract)', async () => {
lookupMock.mockResolvedValue({ address: '10.0.0.5', family: 4 });
await expect(fetchPublicText('https://home.example.com/feed', { blockPrivate: true }))
.rejects.toMatchObject({ status: 400, code: 'UNSAFE_URL' });
});
});
describe('assertPublicHttpUrl', () => {
it('throws a 400 UNSAFE_URL for a blocked target', async () => {
await expect(assertPublicHttpUrl('http://localhost/x')).rejects.toMatchObject({ status: 400, code: 'UNSAFE_URL' });
});
it('resolves for a safe target', async () => {
await expect(assertPublicHttpUrl('https://example.com/x')).resolves.toBeUndefined();
});
});
describe('fetchPublicText', () => {
it('returns the body on a 2xx', async () => {
fetchMock.mockResolvedValue(res({ text: '<rss/>' }));
expect(await fetchPublicText('https://example.com/feed.rss')).toBe('<rss/>');
});
it('returns null on a non-ok status', async () => {
fetchMock.mockResolvedValue(res({ ok: false, status: 404 }));
expect(await fetchPublicText('https://example.com/feed.rss')).toBeNull();
});
it('follows a redirect only after revalidating the target', async () => {
fetchMock
.mockResolvedValueOnce(res({ status: 301, headers: { location: 'https://cdn.example.com/feed.rss' } }))
.mockResolvedValueOnce(res({ text: 'ok' }));
expect(await fetchPublicText('https://example.com/feed.rss')).toBe('ok');
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it('drops a redirect to a blocked host (fails closed)', async () => {
fetchMock.mockResolvedValueOnce(res({ status: 302, headers: { location: 'http://169.254.169.254/x' } }));
expect(await fetchPublicText('https://example.com/feed.rss')).toBeNull();
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('returns null (no throw) on a malformed redirect Location', async () => {
fetchMock.mockResolvedValueOnce(res({ status: 301, headers: { location: 'http://[bad' } }));
expect(await fetchPublicText('https://example.com/feed.rss')).toBeNull();
expect(fetchMock).toHaveBeenCalledTimes(1);
});
});
describe('connect-time IP pinning (DNS-rebinding TOCTOU, #1859)', () => {
it('pins the vetted address, ignoring the hostname undici would re-resolve', () => {
const lookup = buildPinnedLookup('93.184.216.34', 4);
// undici asks with { all: true } → array of { address, family } entries.
let all;
lookup('evil.example.com', { all: true }, (_e, addrs) => { all = addrs; });
expect(all).toEqual([{ address: '93.184.216.34', family: 4 }]);
// net.lookup single-address shape (no `all`).
let single;
lookup('evil.example.com', {}, (_e, addr, fam) => { single = [addr, fam]; });
expect(single).toEqual(['93.184.216.34', 4]);
});
it('derives the family from the address when undici omits it', () => {
const lookup = buildPinnedLookup('2606:2800:220:1:248:1893:25c8:1946');
let single;
lookup('h', {}, (_e, addr, fam) => { single = [addr, fam]; });
expect(single).toEqual(['2606:2800:220:1:248:1893:25c8:1946', 6]);
});
it('hands the fetch a dispatcher pinned to the validation-time address', async () => {
// Validation resolves the host to a PUBLIC ip; the connection is pinned to
// that exact address so a connect-time rebind to a private ip is impossible.
lookupMock.mockResolvedValue({ address: '93.184.216.34', family: 4 });
fetchMock.mockResolvedValue(res({ text: 'ok' }));
await fetchPublicText('https://rebind.example.com/x');
expect(fetchMock.mock.calls[0][1].dispatcher).toBeDefined();
});
it('fails closed when the validation lookup fails (no unpinned connect-time re-resolve)', async () => {
lookupMock.mockRejectedValue(new Error('ENOTFOUND'));
// A hostname we can't vet must not fall through to an unpinned fetch — the
// first hop throws like any other unsafe URL and never reaches the network.
await expect(fetchPublicText('https://unresolvable.example.com/x'))
.rejects.toMatchObject({ status: 400, code: 'UNSAFE_URL' });
expect(fetchMock).not.toHaveBeenCalled();
expect(await isPublicHttpUrlSafe('https://unresolvable.example.com/x')).toBe(false);
});
// End-to-end proof the mechanism actually defeats rebinding: a hostname that
// never resolves in real DNS still reaches the server, so undici MUST have
// used our pinned address rather than re-resolving at connect time.
// Uses the PRODUCTION dispatcher factory (not a hand-rolled `new Agent(...)`)
// against the runtime's global fetch — the same pairing fetchWithTimeout makes
// — so this also proves the undici-8 dispatcher is accepted by the built-in
// fetch of whichever Node runs the suite (see buildPinnedDispatcher's note on
// the v7/v8 request-handler contract).
it('undici connects to the pinned address for a non-resolving hostname', async () => {
const server = http.createServer((req, res2) => res2.end(`host:${req.headers.host}`));
await new Promise((r) => server.listen(0, '127.0.0.1', r));
const { port } = server.address();
const dispatcher = buildPinnedDispatcher('127.0.0.1', 4);
const out = await fetch(`http://rebind.invalid:${port}/`, { dispatcher });
expect(await out.text()).toBe(`host:rebind.invalid:${port}`); // Host preserved, IP pinned
await dispatcher.close();
await new Promise((r) => server.close(r));
});
});
// Streaming response double — getReader() yields the given Uint8Array chunks.
const streamRes = (chunks, { headers = {}, ok = true, status = 200 } = {}) => {
let i = 0;
const cancel = vi.fn(async () => {});
return {
ok,
status,
headers: new Map(Object.entries(headers)),
arrayBuffer: async () => new ArrayBuffer(0),
body: {
getReader: () => ({
read: async () => (i < chunks.length ? { done: false, value: chunks[i++] } : { done: true, value: undefined }),
cancel,
releaseLock: () => {},
}),
_cancel: cancel,
},
};
};
describe('fetchPublicText — opt-in maxBytes cap', () => {
const enc = (s) => new TextEncoder().encode(s);
it('returns the streamed text when within the cap', async () => {
fetchMock.mockResolvedValue(streamRes([enc('hel'), enc('lo')]));
expect(await fetchPublicText('https://example.com/page', { maxBytes: 1024 })).toBe('hello');
});
it('returns null when the declared Content-Length exceeds the cap', async () => {
fetchMock.mockResolvedValue(res({ text: 'x', headers: { 'content-length': String(99 * 1024 * 1024) } }));
expect(await fetchPublicText('https://example.com/page', { maxBytes: 1024 })).toBeNull();
});
it('aborts a no-Content-Length body that exceeds the cap (bounds peak memory)', async () => {
const r = streamRes([new Uint8Array(600), new Uint8Array(600)]); // 1200 > cap
fetchMock.mockResolvedValue(r);
expect(await fetchPublicText('https://example.com/page', { maxBytes: 1024 })).toBeNull();
expect(r.body._cancel).toHaveBeenCalled();
});
it('keeps the uncapped res.text() path when maxBytes is not passed', async () => {
// A response with NO readable stream still works without maxBytes —
// proving existing callers see no behavior change.
fetchMock.mockResolvedValue(res({ text: 'plain body' }));
expect(await fetchPublicText('https://example.com/page')).toBe('plain body');
});
});
describe('fetchPublicBinary', () => {
it('returns the buffer + content-type from a non-streaming response (fallback)', async () => {
const bytes = new Uint8Array([1, 2, 3]).buffer;
fetchMock.mockResolvedValue(res({ body: bytes, headers: { 'content-type': 'image/jpeg' } }));
const out = await fetchPublicBinary('https://i.pinimg.com/736x/x.jpg');
expect(out.contentType).toBe('image/jpeg');
expect(out.buffer).toBeInstanceOf(Buffer);
expect(out.buffer.length).toBe(3);
});
it('rejects a body over the declared Content-Length cap', async () => {
fetchMock.mockResolvedValue(res({ headers: { 'content-length': String(99 * 1024 * 1024) } }));
expect(await fetchPublicBinary('https://i.pinimg.com/x.jpg', { maxBytes: 1024 })).toBeNull();
});
it('streams a chunked body and concatenates within the cap', async () => {
const r = streamRes([new Uint8Array([1, 2]), new Uint8Array([3, 4])], { headers: { 'content-type': 'image/png' } });
fetchMock.mockResolvedValue(r);
const out = await fetchPublicBinary('https://i.pinimg.com/x.png', { maxBytes: 1024 });
expect(out.buffer.length).toBe(4);
expect(out.contentType).toBe('image/png');
});
it('aborts a no-Content-Length body that exceeds the cap (bounds peak memory)', async () => {
const r = streamRes([new Uint8Array(600), new Uint8Array(600)]); // 1200 > cap, no content-length
fetchMock.mockResolvedValue(r);
expect(await fetchPublicBinary('https://evil.example.com/x', { maxBytes: 1024 })).toBeNull();
expect(r.body._cancel).toHaveBeenCalled(); // stream was cancelled rather than fully read
});
});
describe('resolvePublicUrl', () => {
it('resolves a two-redirect share chain, pins each hop and cancels every body', async () => {
const cancel = vi.fn().mockResolvedValue();
fetchMock.mockResolvedValueOnce({ ...res({ status: 302, ok: false, headers: { location: 'https://api.pinterest.com/url_shortener/example/redirect/' } }), body: { cancel } })
.mockResolvedValueOnce({ ...res({ status: 302, ok: false, headers: { location: 'https://www.pinterest.com/example/board/?invite_code=example' } }), body: { cancel } })
.mockResolvedValueOnce({ ...res(), body: { cancel } });
expect(await resolvePublicUrl('https://pin.it/example', { timeoutMs: 8000, blockPrivate: true }))
.toBe('https://www.pinterest.com/example/board/?invite_code=example');
expect(lookupMock.mock.calls.map(([host]) => host)).toEqual(['pin.it', 'api.pinterest.com', 'www.pinterest.com']);
expect(fetchMock).toHaveBeenCalledTimes(3);
for (const [, options, timeout] of fetchMock.mock.calls) {
expect(options).toMatchObject({ redirect: 'manual', dispatcher: expect.anything() });
expect(timeout).toBeLessThanOrEqual(8000);
}
expect(cancel).toHaveBeenCalledTimes(3);
});
it('stops before another fetch when the shared timeout budget expires', async () => {
const now = vi.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValueOnce(100).mockReturnValue(900);
fetchMock.mockResolvedValue(res({ status: 302, ok: false, headers: { location: '/next' } }));
try {
expect(await resolvePublicUrl('https://pin.it/example', { timeoutMs: 800 })).toBeNull();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock.mock.calls[0][2]).toBe(700);
} finally {
now.mockRestore();
}
});
it('refuses a redirect outside the caller allow-list before resolving or fetching it', async () => {
fetchMock.mockResolvedValue(res({ status: 302, ok: false, headers: { location: 'https://example.com/board' } }));
expect(await resolvePublicUrl('https://pin.it/example', { allowUrl: (url) => url.hostname === 'pin.it' })).toBeNull();
expect(lookupMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('refuses a redirect whose DNS resolves private under strict policy', async () => {
lookupMock.mockResolvedValueOnce({ address: '93.184.216.34' }).mockResolvedValueOnce({ address: '192.168.1.1' });
fetchMock.mockResolvedValue(res({ status: 302, ok: false, headers: { location: 'https://www.pinterest.com/example/board/' } }));
expect(await resolvePublicUrl('https://pin.it/example', { blockPrivate: true })).toBeNull();
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('bounds redirect loops and fails closed on missing/malformed locations and network errors', async () => {
fetchMock.mockResolvedValue(res({ status: 302, ok: false, headers: { location: '/loop' } }));
expect(await resolvePublicUrl('https://pin.it/example', { maxRedirects: 2 })).toBeNull();
expect(fetchMock).toHaveBeenCalledTimes(3);
fetchMock.mockResolvedValue(res({ status: 302, ok: false }));
expect(await resolvePublicUrl('https://pin.it/example')).toBeNull();
fetchMock.mockResolvedValue(res({ status: 302, ok: false, headers: { location: 'http://[' } }));
expect(await resolvePublicUrl('https://pin.it/example')).toBeNull();
fetchMock.mockRejectedValue(new Error('network failure'));
expect(await resolvePublicUrl('https://pin.it/example')).toBeNull();
});
});