Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Browser Service - manages the portos-browser CDP instance
* Communicates with the portos-browser process (port 5557 health, port 5556 CDP)
* Stores config in data/browser-config.json
*/
import { readdir, stat, unlink } from 'fs/promises';
import { join, basename, dirname, resolve, extname } from 'path';
import { EventEmitter } from 'events';
import { ensureDir, safeJSONParse, PATHS, readJSONFileStrict, unreadableStoreError, atomicWrite, sleep } from '../lib/fileUtils.js';
import { PORTS } from '../lib/ports.js';
import { normalizeBrowserConfig } from '../lib/browserConfig.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { readResponseJson } from '../lib/readResponseJson.js';
import { execPm2, listProcessesStrict } from './pm2.js';
const PM2_SETTLE_MS = 1500;
const HEALTH_TIMEOUT_MS = 3000;
const NAVIGATE_TIMEOUT_MS = 10000;
const LOGS_TIMEOUT_MS = 5000;
const CDP_DEFAULT_TIMEOUT_MS = 10000;
const CDP_EVALUATE_TIMEOUT_MS = 60000;
// Auth/login redirect detection across providers (Microsoft, Okta, Google, generic)
const AUTH_PATTERNS = ['login.microsoftonline.com', 'okta.com', 'login.live.com', 'accounts.google.com', 'Sign in'];
const CONFIG_FILE = join(PATHS.data, 'browser-config.json');
const ECOSYSTEM_FILE = join(PATHS.root, 'ecosystem.config.cjs');
export const browserEvents = new EventEmitter();
const DEFAULT_PROFILE_DIR = PATHS.browserProfile;
const DEFAULT_DOWNLOAD_DIR = PATHS.browserDownloads;
const DEFAULT_CONFIG = {
cdpPort: PORTS.CDP,
cdpHost: process.env.CDP_HOST || '127.0.0.1',
healthPort: PORTS.CDP_HEALTH,
autoConnect: true,
// Default HEADED — the managed CDP browser is meant to be visible (see
// browser/server.js, which already launches headed unless `headless === true`).
// Keeping this fallback headed matches that and the shipped seed.
headless: false,
userDataDir: DEFAULT_PROFILE_DIR,
downloadDir: DEFAULT_DOWNLOAD_DIR
};
let cachedConfig = null;
let cachedConfigMtimeMs = null;
// ---------- Config persistence ----------
async function getConfigMtimeMs() {
const info = await stat(CONFIG_FILE).catch(() => null);
return info?.isFile() ? info.mtimeMs : null;
}
export async function loadConfig() {
const mtimeMs = await getConfigMtimeMs();
if (cachedConfig && cachedConfigMtimeMs === mtimeMs) return cachedConfig;
const { ok, value: parsed } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false });
const config = normalizeBrowserConfig(parsed ? { ...DEFAULT_CONFIG, ...parsed } : { ...DEFAULT_CONFIG });
// Display readers serve defaults for an unreadable file, but never cache them:
// a repaired file must be picked up even when its mtime is unchanged.
if (!ok) return config;
cachedConfig = config;
cachedConfigMtimeMs = mtimeMs;
return cachedConfig;
}
// Every write path refuses while the stored bytes are unreadable, so defaults
// served to display readers are never persisted over the real config.
async function assertConfigWritable() {
const { ok } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false });
if (!ok) throw unreadableStoreError(CONFIG_FILE);
}
export async function saveConfig(config) {
await ensureDir(PATHS.data);
await assertConfigWritable();
cachedConfig = normalizeBrowserConfig({ ...DEFAULT_CONFIG, ...config });
await atomicWrite(CONFIG_FILE, cachedConfig);
cachedConfigMtimeMs = await getConfigMtimeMs();
browserEvents.emit('config:changed', cachedConfig);
return cachedConfig;
}
export async function getConfig() {
return loadConfig();
}
export async function updateConfig(updates) {
const current = await loadConfig();
return saveConfig({ ...current, ...updates });
}
// ---------- Status / Health ----------
export async function getHealthStatus({ strict = false } = {}) {
const config = await loadConfig();
// Bind-all addresses are not connectable; use loopback instead
const connectHost = config.cdpHost === '0.0.0.0' ? '127.0.0.1'
: config.cdpHost === '::' ? '[::1]'
: config.cdpHost;
const healthUrl = `http://${connectHost}:${config.healthPort}/health`;
const response = await fetchWithTimeout(healthUrl, {}, HEALTH_TIMEOUT_MS).catch(() => null);
if (!response || !response.ok) {
// 503 while the supervisor is relaunching still carries the reason. The
// Browser page already renders `error` when Chrome is down.
const detail = response ? await readResponseJson(response) : null;
const relaunching = detail?.status === 'relaunching';
const lastError = typeof detail?.lastError === 'string' && detail.lastError ? detail.lastError : null;
return {
connected: false,
processRunning: false,
cdpPort: detail?.cdpPort || config.cdpPort,
cdpHost: detail?.cdpHost || config.cdpHost,
healthPort: config.healthPort,
cdpEndpoint: detail?.cdpEndpoint || `ws://${config.cdpHost}:${config.cdpPort}`,
headless: detail?.headless ?? config.headless,
status: typeof detail?.status === 'string' ? detail.status : undefined,
...(relaunching && Number.isInteger(detail.attempt) && detail.attempt > 0 ? { attempt: detail.attempt } : {}),
error: relaunching && lastError
? lastError
: (response ? `Health check returned ${response.status}` : 'Health check unreachable')
};
}
const data = await readResponseJson(response);
if (strict && typeof data?.status !== 'string') throw new Error('Browser health unavailable');
return {
connected: data.status === 'healthy',
processRunning: true,
cdpPort: data.cdpPort || config.cdpPort,
cdpHost: data.cdpHost || config.cdpHost,
healthPort: config.healthPort,
cdpEndpoint: data.cdpEndpoint || `ws://${config.cdpHost}:${config.cdpPort}`,
headless: data.headless ?? config.headless,
status: data.status
};
}
// ---------- PM2 process management ----------
async function pm2Action(action, args) {
console.log(`🌐 Browser PM2 ${action}: portos-browser`);
// execPm2 runs `node pm2/bin/pm2` directly. Resolving bare `pm2` through a
// shell picks up pm2.cmd on Windows, and that cmd.exe hop opens a console
// window out of PortOS's console-less PM2 fork — see docs/WINDOWS_CONSOLE.md.
await execPm2([action, ...args]);
console.log(`✅ Browser PM2 ${action} complete`);
// Give PM2 a moment to settle
await sleep(PM2_SETTLE_MS);
const status = await getHealthStatus();
browserEvents.emit('status:changed', status);
return status;
}
export async function launchBrowser() {
// Use ecosystem file so PM2 has the full process config even after pm2 flush/delete
return pm2Action('start', [ECOSYSTEM_FILE, '--only', 'portos-browser']);
}
export async function stopBrowser() {
return pm2Action('stop', ['portos-browser']);
}
export async function restartBrowser() {
return pm2Action('restart', ['portos-browser']);
}
// ---------- PM2 status (process-level) ----------
export async function getProcessStatus() {
// `listProcessesStrict()` returns `null` when the PM2 read itself FAILED (vs
// `[]` for a successful read with no processes) — the absent-vs-empty
// contract from issue #968, now the one non-test owner of raw `pm2 jlist`
// execution/parsing alongside `autofixer/shared.js` (#8164).
const processes = await listProcessesStrict();
if (processes === null) {
// A failed read must never report `exists: false` — that claims the
// browser process is confidently absent when its status is actually
// unavailable (a transient PM2 blip should not read as "not installed").
return { exists: null, status: 'unavailable', pm2_id: null };
}
const browserProc = processes.find(p => p.name === 'portos-browser');
if (!browserProc) {
return { exists: false, status: 'not_found', pm2_id: null };
}
return {
exists: true,
status: browserProc.status || 'unknown',
pm2_id: browserProc.pm_id,
pid: browserProc.pid,
memory: browserProc.memory || 0,
cpu: browserProc.cpu || 0,
uptime: browserProc.uptime || null,
restarts: browserProc.restarts || 0,
unstableRestarts: browserProc.unstableRestarts || 0
};
}
// ---------- Logs ----------
export async function getRecentLogs(lines = 50) {
const { stdout, stderr } = await execPm2(['logs', 'portos-browser', '--nostream', '--lines', String(lines)], {
timeout: LOGS_TIMEOUT_MS
}).catch(() => ({ stdout: '', stderr: '' }));
return { stdout: stdout || '', stderr: stderr || '' };
}
// ---------- CDP shared helpers ----------
// Bind-all addresses (0.0.0.0, ::) are not connectable — fall back to IPv4 loopback
async function getCdpConnectHost() {
const config = await loadConfig();
const host = (config.cdpHost === '0.0.0.0' || config.cdpHost === '::') ? '127.0.0.1' : config.cdpHost;
return { host, port: config.cdpPort };
}
export async function cdpRequest(path, options = {}) {
const { host, port } = await getCdpConnectHost();
const url = `http://${host}:${port}${path}`;
const { timeout, ...rest } = options;
return fetchWithTimeout(url, rest, timeout || CDP_DEFAULT_TIMEOUT_MS);
}
// Returns raw CDP page objects (includes webSocketDebuggerUrl, unlike getOpenPages)
export async function listCdpPages({ strict = false } = {}) {
const response = await cdpRequest('/json/list', { timeout: HEALTH_TIMEOUT_MS }).catch(() => null);
if (!response || !response.ok) {
if (strict) throw new Error('Browser pages unavailable');
return [];
}
const pages = await readResponseJson(response, { fallback: null, emptyValue: null });
if (!Array.isArray(pages)) {
if (strict) throw new Error('Browser pages unavailable');
return [];
}
return pages;
}
export async function findOrOpenPage(targetUrl) {
const pages = await listCdpPages();
const existing = pages.find(p => p.url?.includes(new URL(targetUrl).hostname));
if (existing) return existing;
const response = await cdpRequest(`/json/new?${encodeURIComponent(targetUrl)}`, { method: 'PUT' });
if (!response.ok) return null;
// Preserve the null-on-failure contract: a malformed body stays null, not {}.
const page = await readResponseJson(response, { fallback: null, emptyValue: null });
if (page) browserEvents.emit('pages:changed');
return page;
}
export function isAuthPage(page) {
const url = page?.url || '';
const title = page?.title || '';
return AUTH_PATTERNS.some(p => url.includes(p) || title.includes(p));
}
export async function evaluateOnPage(page, expression, { timeout = CDP_EVALUATE_TIMEOUT_MS } = {}) {
const wsUrl = page?.webSocketDebuggerUrl;
if (!wsUrl) return null;
const { default: WebSocket } = await import('ws');
return new Promise((resolve) => {
const ws = new WebSocket(wsUrl);
const timer = setTimeout(() => { ws.close(); resolve(null); }, timeout);
ws.on('open', () => {
ws.send(JSON.stringify({
id: 1,
method: 'Runtime.evaluate',
params: { expression, returnByValue: true, awaitPromise: true }
}));
});
ws.on('message', (data) => {
const msg = safeJSONParse(data.toString(), null, { context: 'cdp-ws' });
if (!msg || msg.id !== 1) return;
clearTimeout(timer);
ws.close();
if (msg.error || msg.result?.exceptionDetails) return resolve(null);
resolve(msg.result?.result?.value ?? null);
});
ws.on('error', () => { clearTimeout(timer); ws.close(); resolve(null); });
});
}
// ---------- CDP navigation ----------
export async function navigateToUrl(url) {
const response = await cdpRequest(`/json/new?${encodeURIComponent(url)}`, {
method: 'PUT',
timeout: NAVIGATE_TIMEOUT_MS
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(`CDP navigate failed (${response.status})`);
}
// A successful CDP /json/new always returns a target with an id; a malformed
// 200 body must fail like a !ok navigate rather than return a truthy tab with
// undefined id/url that a caller mistakes for a successful navigation.
const page = await readResponseJson(response, { fallback: null, emptyValue: null });
if (!page?.id) {
throw new Error('CDP navigate returned a malformed response');
}
browserEvents.emit('pages:changed');
console.log('🌐 Opened tab in CDP browser');
return { id: page.id, title: page.title || '(loading)', url: page.url, type: page.type };
}
// ---------- CDP pinned navigation (SSRF: verify Chrome's ACTUAL connect IP) ----------
// `serviceWorkerResponseSource` values that mean the service worker answered
// from a local store instead of dialing out: its own Cache Storage, the HTTP
// cache, or the worker's synthesized fallback response. `'network'` — the SW
// passed the request THROUGH — is deliberately absent: that hop did make a
// connection, it just isn't annotated with the peer address.
const NO_NETWORK_SW_SOURCES = new Set(['cache-storage', 'http-cache', 'fallback-code']);
// Hostname of a URL, or null when it isn't parseable. Used wherever the only
// address signal available is the URL itself (WebSockets and connectionless
// hops), never as a substitute for a verified peer IP.
const hostOf = (raw) => {
if (typeof raw !== 'string') return null;
try { return new URL(raw).hostname; } catch { return null; }
};
// One main-frame hop from a CDP `Network.Response` (a `responseReceived`
// payload or the `redirectResponse` riding on the next request).
const toHop = (r, requestId) => ({
requestId,
url: r.url || null,
remoteIPAddress: r.remoteIPAddress || '',
status: r.status ?? null,
fromServiceWorker: r.fromServiceWorker === true,
fromDiskCache: r.fromDiskCache === true,
fromPrefetchCache: r.fromPrefetchCache === true,
serviceWorkerResponseSource: r.serviceWorkerResponseSource || null,
});
// True when Chrome explicitly reported that this hop was served WITHOUT opening
// a connection (disk cache, prefetch cache, or a service worker answering from
// its own store). Such a hop has no peer IP to pin because there is no peer —
// distinct from an empty `remoteIPAddress` with no explanation, which stays a
// refusal. Exported for testing.
export function hopMadeNoConnection(hop) {
if (!hop) return false;
return hop.fromDiskCache === true
|| hop.fromPrefetchCache === true
|| NO_NETWORK_SW_SOURCES.has(hop.serviceWorkerResponseSource);
}
// True when a `Network.loadingFailed` describes a load Chrome ABANDONED rather
// than one that failed against a peer. Chrome 150 flags a superseded top-level
// navigation with `canceled: true` and `net::ERR_ABORTED`; either signal alone
// is accepted so a Chrome that reports only one of them still classifies. Every
// other error kind is a real connection outcome we could not pin. Exported for
// testing.
export function navigationWasCanceled(params) {
return params?.canceled === true || params?.errorText === 'net::ERR_ABORTED';
}
// Human-readable cause for an empty-IP refusal, so the next occurrence is
// diagnosable from the thrown message without a manual CDP replay.
const describeHopDelivery = (hop) => `no remoteIPAddress; fromServiceWorker=${hop.fromServiceWorker === true}`
+ `, fromDiskCache=${hop.fromDiskCache === true}`
+ `, fromPrefetchCache=${hop.fromPrefetchCache === true}`
+ `, serviceWorkerResponseSource=${hop.serviceWorkerResponseSource || 'none'}`;
/**
* Extract EVERY main-frame document network hop (each top-level navigation's
* initial request + every HTTP redirect + final response) from a captured stream
* of CDP Network events, each annotated with the *actual* IP Chrome connected to
* (`remoteIPAddress`).
*
* CDP marks a main-document load with `requestId === loaderId`; redirects reuse
* that requestId and arrive as a `redirectResponse` on the following
* `Network.requestWillBeSent`, and the final hop lands as a
* `Network.responseReceived` with the same requestId. A page can start MORE than
* one top-level navigation (a client-side `location.replace` / meta-refresh
* during the settle window is a fresh `requestId === loaderId`), so we track a
* SET of main requestIds — not just the first — and collect the hops of each.
* Walking these reconstructs every address Chrome dialed for the top document
* over the whole capture window — the thing a pre-navigation `dns.lookup` can't
* know, because Chrome resolves DNS itself (the rebinding TOCTOU).
*
* Pure + exported so the SSRF-pin decision is unit-testable without a live
* browser. Returns `{ hops: [{ requestId, url, remoteIPAddress, status,
* fromServiceWorker, fromDiskCache, fromPrefetchCache,
* serviceWorkerResponseSource }], finalUrl, mainRequestIds: string[],
* pendingMainRequestIds: string[], pendingMainRequestUrls: string[] }` (the
* URLs are positional to the pending ids, so a refusal can name the destination
* without re-parsing the stream). The delivery flags ride along so the gate
* can tell "Chrome dialed somewhere we can't see" apart from "Chrome made no
* connection at all" (see `hopMadeNoConnection`), and `requestId` keeps the hops
* of ONE top-level navigation distinguishable from another's.
* `pendingMainRequestIds` are top-level navigations that STARTED but produced no
* `responseReceived` in the captured window — i.e. a navigation still in flight
* whose final connection IP was never observed. The caller must fail closed on a
* non-empty pending set: Chrome could complete that navigation (to a private /
* metadata target) right after we stop capturing, leaving it unpinned. A
* navigation Chrome reported as CANCELED (`Network.loadingFailed` with
* `canceled`/`net::ERR_ABORTED` — what a client-side route change that
* supersedes an in-flight one produces) is NOT pending: it was abandoned before
* any answer, so nothing commits and no connection outcome is hidden. Any other
* failure kind stays pending and keeps failing closed — see
* `navigationWasCanceled`.
*/
export function pickMainFrameHops(messages, topFrameId = null) {
// CDP sets `requestId === loaderId` on the main resource of EVERY frame — the
// top document AND cross-origin sub-frames (iframes). When the caller passes
// the top frame's `frameId` (from the `Page.navigate` response) we classify a
// top-level navigation by `frameId === topFrameId` instead, so a slow/cached
// iframe document can't land in the pending/empty-IP gate and over-refuse a safe
// public page. Defensive fallback: if NO document request in the stream carries
// `topFrameId` (a frameId-format surprise), retain the `requestId === loaderId`
// classification so the gate can never regress to refuse-all. Sub-frame /
// sub-resource IPs are still fully verified by `collectConnectedIps`.
const useFrameId = topFrameId != null && messages.some((m) => (
m?.method === 'Network.requestWillBeSent' && m.params?.type === 'Document' && m.params?.frameId === topFrameId
));
const isMainDocRequest = (p) => p.type === 'Document' && p.requestId
&& (useFrameId ? p.frameId === topFrameId : p.requestId === p.loaderId);
const mainRequestIds = new Set();
const respondedIds = new Set();
const failedIds = new Set();
// Latest requested URL per main-frame navigation (a redirect overwrites it),
// so a pending navigation can be NAMED in the refusal without a second parse.
const mainRequestUrls = new Map();
const hops = [];
let finalUrl = null;
for (const msg of messages) {
const p = msg?.params;
if (!p) continue;
if (msg.method === 'Network.requestWillBeSent') {
if (isMainDocRequest(p)) {
mainRequestIds.add(p.requestId);
}
if (mainRequestIds.has(p.requestId)) {
if (p.request?.url) mainRequestUrls.set(p.requestId, p.request.url);
if (p.redirectResponse) hops.push(toHop(p.redirectResponse, p.requestId));
}
} else if (msg.method === 'Network.responseReceived') {
if (mainRequestIds.has(p.requestId) && p.response) {
respondedIds.add(p.requestId);
hops.push(toHop(p.response, p.requestId));
finalUrl = p.response.url || finalUrl;
}
} else if (msg.method === 'Network.loadingFailed' && mainRequestIds.has(p.requestId)) {
// ONLY a navigation Chrome CANCELED clears the pending gate. A cancel is
// "this load was abandoned before any answer" — nothing committed and no
// connection outcome is being hidden. Every OTHER failure kind
// (ERR_CONNECTION_RESET, ERR_CONNECTION_REFUSED, …) means Chrome did
// reach out and got something back, and `loadingFailed` carries no
// `remoteIPAddress`, so we cannot verify WHERE — a private endpoint that
// accepts and resets would otherwise pass unpinned. Those stay pending
// and the gate keeps failing closed on them.
// A failure arriving AFTER the response is only a truncated body, and
// that hop is already pinned, so `respondedIds` still wins either way.
if (navigationWasCanceled(p)) failedIds.add(p.requestId);
}
}
const pendingMainRequestIds = [...mainRequestIds]
.filter((id) => !respondedIds.has(id) && !failedIds.has(id));
return {
hops,
finalUrl,
mainRequestIds: [...mainRequestIds],
pendingMainRequestIds,
pendingMainRequestUrls: pendingMainRequestIds.map((id) => mainRequestUrls.get(id) || '(url unknown)'),
};
}
// Session-scoped CDP commands the pinned tab issues after `Network.enable` and
// BEFORE `Page.navigate`, so that the main document arrives with a peer address
// the SSRF pin can actually verify. Both remove a delivery path that hands
// Chrome a response it cannot annotate with a `remoteIPAddress`:
// - `setBypassServiceWorker`: a SW-mediated document carries no peer address
// even when the worker passed the fetch straight through to the network,
// which made every PWA unreadable behind the fail-closed empty-IP gate.
// - `setCacheDisabled`: a document served from the HTTP/prefetch cache made no
// connection at all, so nothing about the *current* resolution of its host
// was verified — a body cached earlier from a since-blocked address would be
// ingested unpinned. Forcing a real fetch is what the pin is for.
// Both are scoped to this fresh, disposable tab, so the user's normal browsing
// is unaffected; cookies are untouched, so a signed-in session still applies.
// Ids are offset well past the navigate (2) / read (3) request ids.
const PIN_SETUP_COMMANDS = [
{ method: 'Network.setBypassServiceWorker', params: { bypass: true } },
{ method: 'Network.setCacheDisabled', params: { cacheDisabled: true } },
];
const PIN_SETUP_ID_BASE = 10;
// Close a CDP tab by target id (best-effort). Used by `navigateToUrlPinned` to
// fail closed after an SSRF-pin refusal (a tab that connected to a disallowed
// address is torn down rather than left open for the DOM reader) AND to tear
// down a tab it read from itself (`closeAfterRead`). Read callers should NOT
// call this for a tab whose read `navigateToUrlPinned` performed — that tab is
// already closed. It stays exported for callers that own a tab outright: a
// handoff, or `POST /api/browser/navigate`, whose tab IS the deliverable.
export async function closeCdpPage(id) {
if (!id) return;
await cdpRequest(`/json/close/${id}`, { timeout: HEALTH_TIMEOUT_MS }).catch(() => {});
browserEvents.emit('pages:changed');
}
// Every address Chrome ACTUALLY dialed across the capture window — the main
// document, its redirects, AND every sub-resource / XHR / fetch the page issued
// (any request type), each with a non-empty `remoteIPAddress`. Empty IPs
// (data:/blob:/cache/service-worker responses — no network connection) are
// omitted so legit pages aren't false-refused. Pure; exported for testing.
export function collectConnectedIps(messages) {
const out = [];
for (const msg of messages) {
const p = msg?.params;
if (!p) continue;
if (msg.method === 'Network.requestWillBeSent' && p.redirectResponse?.remoteIPAddress) {
out.push({ url: p.redirectResponse.url || null, remoteIPAddress: p.redirectResponse.remoteIPAddress });
} else if (msg.method === 'Network.responseReceived' && p.response?.remoteIPAddress) {
out.push({ url: p.response.url || null, remoteIPAddress: p.response.remoteIPAddress });
}
}
return out;
}
// Hosts of every WebSocket the page opened during the capture window. CDP routes
// WS through dedicated `Network.webSocket*` events whose payloads carry NO
// `remoteIPAddress`, so we can only gate the WS by its URL host, not by Chrome's
// actual connect IP (the resolver-level pin HTTP gets). Pure; exported for tests.
export function collectWebSocketHosts(messages) {
const out = [];
for (const msg of messages) {
if (msg?.method !== 'Network.webSocketCreated') continue;
const raw = msg.params?.url;
if (typeof raw !== 'string') continue;
out.push({ url: raw, host: hostOf(raw) });
}
return out;
}
// Pure gate over a captured CDP message stream: returns a refusal reason string,
// or null when the navigation is safe to read. Exported for unit testing.
export function ssrfPinRefusalReason(messages, verifyRemoteIp, _url, topFrameId = null) {
const { hops, pendingMainRequestUrls } = pickMainFrameHops(messages, topFrameId);
if (!hops.length) return 'no main-frame document response was observed';
// The main document must have a verifiable (present) connection IP — an empty
// one can't be checked, so fail closed. The ONE exception is a hop Chrome
// explicitly flagged as served without a connection (disk/prefetch cache, or a
// service worker answering from its own store): there is no peer to pin, and
// refusing it false-positives every PWA. Two conditions bound that exception so
// it can never stand in for a network check:
// 1. Another hop of the SAME top-level navigation (same requestId — i.e. an
// earlier link in this redirect chain) must have verified a real IP. A
// verified IP from a DIFFERENT navigation proves nothing about this one:
// a page that loads clean from public and then client-side-navigates to a
// cached `http://127.0.0.1/…` document would otherwise be admitted.
// 2. The hop's own URL host must still pass `verifyRemoteIp` — that catches a
// cached document whose URL is a blocked literal (loopback/metadata),
// which is the only address signal a connectionless hop carries. (Same
// host-level fallback the WebSocket gate uses.)
const verifiedRequestIds = new Set(hops.filter((h) => h.remoteIPAddress).map((h) => h.requestId));
for (const hop of hops) {
if (hop.remoteIPAddress) continue;
const noConnection = hopMadeNoConnection(hop);
const host = hostOf(hop.url);
if (noConnection && verifiedRequestIds.has(hop.requestId) && host && verifyRemoteIp(host)) continue;
const detail = noConnection
? `${describeHopDelivery(hop)}; no verified network address for this navigation`
: describeHopDelivery(hop);
return `Chrome connected to an unverifiable address for a main-frame document (${detail})`;
}
// A top-level navigation that STARTED, produced no response, and was not
// reported as failed was never pinned. Counts retain diagnostic value without
// copying browsing history or URL credentials into errors and their logs.
if (pendingMainRequestUrls.length) {
return `a top-level navigation was still in flight (unpinned): ${pendingMainRequestUrls.length} pending`;
}
// EVERY connection Chrome made must dial an allowed address — not just the
// main document. A rebinding page can load public, then `fetch()` its
// now-private-resolving hostname; the browser treats it as same-origin, JS
// injects that private response into the DOM we ingest, and the sub-resource's
// real `remoteIPAddress` surfaces it here. (`isBlockedIngestHost` blocks only
// loopback/link-local/metadata, not RFC1918 LAN, so LAN pages aren't refused.)
for (const conn of collectConnectedIps(messages)) {
if (!verifyRemoteIp(conn.remoteIPAddress)) {
return 'Chrome connected to a disallowed address';
}
}
// WebSockets: CDP exposes no remoteIPAddress for WS, so we can only refuse a WS
// opened to a blocked HOST (a direct ws://127.0.0.1 / ws://localhost / metadata
// literal). A WS to a hostname that DNS-rebinds to a private IP is a residual
// this interception approach can't see — closing it fully needs Chrome-launch
// `--host-resolver-rules` (a per-navigation relaunch of the shared browser,
// out of scope here). Bounded by the single-user trust model; cloud metadata,
// the primary target, is HTTP-only and fully covered above.
for (const ws of collectWebSocketHosts(messages)) {
if (!ws.host || !verifyRemoteIp(ws.host)) {
return 'page opened a WebSocket to a disallowed host';
}
}
return null;
}
/**
* Navigate to a URL in a fresh tab and verify — against Chrome's OWN reported
* connection IP — that EVERY main-frame hop connected to an allowed address:
* the initial `Page.navigate`, every HTTP redirect, AND any client-side
* top-level navigation (meta-refresh / `location.replace`) that fires during the
* post-load `settleMs` window. This closes the DNS-rebinding TOCTOU a
* pre-navigation `dns.lookup` cannot — Chrome resolves DNS itself, so we open a
* BLANK tab, subscribe to CDP Network events, THEN drive `Page.navigate`, keep
* the subscription open across the settle window, and check the
* `remoteIPAddress` Chrome actually dialed for each hop. The tab also bypasses
* the site's service worker and disables its cache (`PIN_SETUP_COMMANDS`), so
* the main document is fetched by Chrome itself and therefore carries a
* pinnable peer IP instead of arriving from a store we can't verify.
*
* `verifyRemoteIp(ip)` returns false to refuse. On ANY refusal (unverifiable /
* empty IP, a navigation that never yields a document response, or a top-level
* nav still in flight) we fail closed: close the tab and throw, so a
* rebind-to-private answer never reaches the caller.
*
* When `evaluateExpression` is given, the DOM read runs on the SAME CDP session
* (a `Runtime.evaluate` after settle), and the pin is RE-checked over events
* captured up to and during that read — so there is no gap between "stop
* monitoring" and "read the DOM" for a late client-side navigation to slip
* through. The evaluated value is returned as `evalResult` (or null). Without
* `evaluateExpression` the caller gets a page handle to read separately.
*
* TAB OWNERSHIP follows the read, and is NOT the caller's to remember:
* `closeAfterRead` defaults to true exactly when `evaluateExpression` was
* supplied, because a tab we already read from is scratch — leaving it open
* litters the user's browser one tab per navigation (which is precisely how the
* catalog URL ingest and the Stacker News identity check each leaked). So the
* teardown happens HERE, on both the refusal path and the success path. Only a
* caller that omits `evaluateExpression` — one for whom the TAB is the
* deliverable (a browser handoff, `POST /api/browser/navigate`) — gets a tab
* back to close on its own schedule. Pass `closeAfterRead: false` to read AND
* keep the tab; passing it explicitly decides teardown either way.
*
* Because of that, `id` / `webSocketDebuggerUrl` in the return value name a
* CLOSED tab whenever this function performed the read. They are still returned
* (for logging and for the `closeAfterRead: false` case) but must NOT be used to
* re-derive a handoff URL, re-evaluate on the session, or close the tab again.
*
* The caller does NOT sleep — the settle wait happens HERE, so the DOM has had
* `settleMs` to render (and every navigation in that window is pinned) by the
* time this resolves. Returns `{ id, url, title, webSocketDebuggerUrl, evalResult }`.
*/
export async function navigateToUrlPinned(url, {
verifyRemoteIp,
settleMs = 0,
navigateTimeoutMs = NAVIGATE_TIMEOUT_MS,
evaluateExpression = null,
evaluateTimeoutMs = CDP_EVALUATE_TIMEOUT_MS,
closeAfterRead = Boolean(evaluateExpression),
} = {}) {
if (typeof verifyRemoteIp !== 'function') {
throw new Error('navigateToUrlPinned requires a verifyRemoteIp(ip) predicate');
}
// Open a BLANK tab first so Network listeners attach BEFORE the target URL is
// fetched — `/json/new?<url>` would navigate immediately and we'd miss the
// document request (and its remoteIPAddress).
const response = await cdpRequest('/json/new?about:blank', { method: 'PUT', timeout: navigateTimeoutMs });
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(`CDP open-blank failed (${response.status})`);
}
browserEvents.emit('pages:changed');
const target = await readResponseJson(response, { fallback: null, emptyValue: null });
if (!target?.id || !target?.webSocketDebuggerUrl) {
throw new Error('CDP open-blank returned a malformed response');
}
const READ_ID = 3;
const { default: WebSocket } = await import('ws');
const messages = [];
const result = await new Promise((resolve) => {
const ws = new WebSocket(target.webSocketDebuggerUrl);
let settled = false;
let phase = 'nav';
// The top frame's id, reported by the `Page.navigate` response (id 2). Threaded
// into the pin so ONLY the top document gates the pending/empty-IP checks —
// slow/cached iframes (which also carry `requestId === loaderId`) don't.
let topFrameId = null;
let overallTimer;
let settleTimer = null;
let evalTimer = null;
const finish = (value) => {
if (settled) return;
settled = true;
clearTimeout(overallTimer);
clearTimeout(settleTimer);
clearTimeout(evalTimer);
try { ws.close(); } catch {}
resolve(value);
};
// Overall cap: fail closed if the FIRST document response never arrives.
overallTimer = setTimeout(
() => finish({ ok: false, reason: 'navigation timed out before the document response' }),
navigateTimeoutMs,
);
// Settle expired: verify everything captured so far, then either read the DOM
// on THIS same session (so no monitoring gap) or resolve for a separate read.
const onSettleEnd = () => {
const bad = ssrfPinRefusalReason(messages, verifyRemoteIp, url, topFrameId);
if (bad) return finish({ ok: false, reason: bad });
if (!evaluateExpression) {
const { finalUrl } = pickMainFrameHops(messages, topFrameId);
return finish({ ok: true, finalUrl, evalResult: null });
}
phase = 'read';
evalTimer = setTimeout(() => finish({ ok: false, reason: 'DOM read timed out' }), evaluateTimeoutMs);
ws.send(JSON.stringify({
id: READ_ID,
method: 'Runtime.evaluate',
params: { expression: evaluateExpression, returnByValue: true, awaitPromise: true },
}));
};
ws.on('open', () => {
ws.send(JSON.stringify({ id: 1, method: 'Network.enable' }));
PIN_SETUP_COMMANDS.forEach((cmd, i) => ws.send(JSON.stringify({ id: PIN_SETUP_ID_BASE + i, ...cmd })));
ws.send(JSON.stringify({ id: 2, method: 'Page.navigate', params: { url } }));
});
ws.on('message', (data) => {
const msg = safeJSONParse(data.toString(), null, { context: 'cdp-pin' });
if (!msg) return;
// A Page.navigate Chrome rejects outright (bad scheme, etc.) errors on id 2;
// otherwise its result carries the top frame's `frameId` we pin against.
if (msg.id === 2) {
if (msg.error) return finish({ ok: false, reason: 'CDP rejected navigation' });
if (msg.result?.frameId) topFrameId = msg.result.frameId;
return;
}
const setupIndex = msg.id - PIN_SETUP_ID_BASE;
if (setupIndex >= 0 && setupIndex < PIN_SETUP_COMMANDS.length) {
// Non-fatal: a Chrome that rejects one of these just keeps its service
// worker / cache in front of the document, and the empty-IP gate still
// fails closed on the unpinnable response — never under-verifies.
if (msg.error) console.warn(`⚠️ CDP ${PIN_SETUP_COMMANDS[setupIndex].method} rejected`);
return;
}
if (msg.id === READ_ID) {
// DOM read returned. RE-verify over ALL events captured up to now: any
// top-level navigation that committed during the read is in `messages`,
// so a late rebind that changed the page under us fails closed here.
const bad = ssrfPinRefusalReason(messages, verifyRemoteIp, url, topFrameId);
if (bad) return finish({ ok: false, reason: bad });
const evalResult = (msg.error || msg.result?.exceptionDetails) ? null : (msg.result?.result?.value ?? null);
const { finalUrl } = pickMainFrameHops(messages, topFrameId);
return finish({ ok: true, finalUrl, evalResult });
}
if (!msg.method) return;
messages.push(msg);
// First main-frame document RESPONSE (type 'Document', emitted before any
// sub-frame doc) starts the settle window; keep capturing across it so a
// client-side navigation during settle is pinned too.
if (phase === 'nav' && msg.method === 'Network.responseReceived' && msg.params?.type === 'Document' && !settleTimer) {
clearTimeout(overallTimer);
settleTimer = setTimeout(onSettleEnd, settleMs);
}
});
ws.on('error', () => finish({ ok: false, reason: 'CDP websocket error during navigation' }));
});
if (!result.ok) {
await closeCdpPage(target.id);
throw new Error(`refusing to ingest: ${result.reason}`);
}
// The read (if any) already happened on this session, so the tab has served
// its purpose — tear it down here rather than trusting every call site to.
if (closeAfterRead) await closeCdpPage(target.id);
else browserEvents.emit('pages:changed');
return {
id: target.id,
url: result.finalUrl || url,
title: '',
webSocketDebuggerUrl: target.webSocketDebuggerUrl,
evalResult: result.evalResult,
};
}
// ---------- CDP page listing (UI-shaped subset) ----------
export async function getOpenPages(options) {
const pages = await listCdpPages(options);
return pages.map(p => ({
id: p.id,
title: p.title || '(untitled)',
url: p.url,
type: p.type
}));
}
// ---------- CDP version info ----------
export async function getCdpVersion() {
const response = await cdpRequest('/json/version', { timeout: HEALTH_TIMEOUT_MS }).catch(() => null);
if (!response || !response.ok) return null;
// Preserve the null-on-failure contract: the /version route 503s when this is
// falsy, so a malformed body must stay null, not become a truthy {}.
return readResponseJson(response, { fallback: null, emptyValue: null });
}
// ---------- Downloads ----------
export async function getDownloads({ strict = false } = {}) {
const config = await loadConfig();
const downloadDir = config.downloadDir || DEFAULT_DOWNLOAD_DIR;
const entries = await readdir(downloadDir).catch(err => {
if (strict && err.code !== 'ENOENT') throw err;
return [];
});
// Filter out hidden files and .crdownload (partial Chrome downloads)
const files = [];
for (const name of entries) {
if (name.startsWith('.') || name.endsWith('.crdownload')) continue;
const filePath = join(downloadDir, name);
const info = await stat(filePath).catch(err => {
if (strict && err.code !== 'ENOENT') throw err;
return null;
});
if (info?.isFile()) {
files.push({
name,
size: info.size,
modified: info.mtime.toISOString()
});
}
}
// Most recent first
files.sort((a, b) => b.modified.localeCompare(a.modified));
return { downloadDir, files };
}
const DOWNLOAD_MIME_TYPES = {
'.txt': 'text/plain', '.md': 'text/markdown', '.json': 'application/json',
'.csv': 'text/csv', '.xml': 'application/xml', '.pdf': 'application/pdf',
'.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
'.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp',
'.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.ogg': 'audio/ogg', '.m4a': 'audio/mp4',
'.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime',
'.zip': 'application/zip', '.tar': 'application/x-tar', '.gz': 'application/gzip',
'.7z': 'application/x-7z-compressed'
};
export async function resolveDownload(name) {
const config = await loadConfig();
const downloadDir = resolve(config.downloadDir || DEFAULT_DOWNLOAD_DIR);
const safeName = basename(name || '');
if (!safeName || safeName.startsWith('.') || safeName.endsWith('.crdownload')) return null;
const absPath = resolve(downloadDir, safeName);
if (dirname(absPath) !== downloadDir) return null;
const info = await stat(absPath).catch(() => null);
if (!info?.isFile()) return null;
const ext = extname(safeName).toLowerCase();
return {
absPath,
name: safeName,
ext,
mime: DOWNLOAD_MIME_TYPES[ext] || 'application/octet-stream'
};
}
export async function deleteDownload(name) {
const file = await resolveDownload(name);
if (!file) return false;
await unlink(file.absPath);
browserEvents.emit('downloads:changed');
return true;
}
// ---------- Full combined status ----------
export async function getFullStatus({ strict = false } = {}) {
if (strict) return readObservedStatus();
const [health, process, pages, version, config, downloads] = await Promise.all([
getHealthStatus(),
getProcessStatus(),
getOpenPages().catch(() => []),
getCdpVersion().catch(() => null),
getConfig(),
getDownloads().catch(() => ({ downloadDir: DEFAULT_DOWNLOAD_DIR, files: [] }))
]);
return {
...health,
process,
pages,
pageCount: pages.length,
version,
config,
downloads
};
}
// Observation never opens a CDP target or starts a process. Failed reads are
// distinct from confirmed empty/stopped state, so the shared cache can retain
// its last successful sample rather than publishing a false empty list.
async function readObservedStatus() {
const [health, process, config, downloads] = await Promise.all([
getHealthStatus({ strict: true }), getProcessStatus(), getConfig(), getDownloads({ strict: true })
]);
if (process.status === 'unavailable') throw new Error('Browser process status unavailable');
if (process.status === 'online' && health.error === 'Health check unreachable') {
throw new Error('Browser health unavailable');
}
const pages = health.connected ? await getOpenPages({ strict: true }) : [];
const version = health.connected ? await getCdpVersion() : null;
if (health.connected && !version) throw new Error('Browser version unavailable');
return { ...health, process, pages, pageCount: pages.length, version, config, downloads };
}