Skip to content

Open-source design agent: on-brand stills, decks, diagrams, charts and motion graphics from a chat thread, built on Deep Agents.

13.6 KB
Raw Blame
#!/usr/bin/env node
// fetch-asset.mjs — download public web assets (images by default, any file with --kind any) into the
// workspace, so a composition references a local file and renders offline. Only a compact JSON line
// per asset reaches the model; the bytes stay a sandbox file.
//
// URLs are untrusted input (often pulled from a page or a message), so the fetch is SSRF-hardened:
//   - scheme allowlist: http and https only;
//   - every address is checked: literal IPs before connecting, and hostnames inside the connection's
//     own DNS lookup, so the address that is checked is the address that is dialled (no rebinding
//     window between a check and the connect). Private, loopback, link-local (including the
//     169.254.169.254 cloud metadata service), CGNAT, multicast, reserved and documentation ranges
//     are refused, and so are IPv6 forms that embed one (mapped, compatible, NAT64, 6to4, Teredo);
//   - redirects are followed by hand, at most --max-redirects, and each hop is checked again;
//   - a hard size cap (Content-Length up front, then streamed), a total timeout, a content-type check,
//     and for images a real decode with sharp;
//   - writes stay under the workspace (--out is confined to --root).
// It connects directly; it does not use an HTTP proxy from the environment.
//
// Usage:
//   node /workspace/tools/fetch-asset.mjs <url> [<url> ...] --out <dir> [--kind image|any] [--max-mb 25]
//        [--name <stem>] [--timeout-ms 30000] [--max-redirects 4]
// Output: `__FETCH_OK__` then one JSON line per saved asset {url, path, contentType, bytes, sha256,
// format, width, height}; `__FETCH_ERR__ <url> <reason>` per failure.
// Exit: 0 every URL saved · 1 any URL refused or failed · 2 bad arguments or the toolchain is missing.

import crypto from "node:crypto";
import dns from "node:dns";
import { mkdir, writeFile } from "node:fs/promises";
import http from "node:http";
import https from "node:https";
import net from "node:net";
import path from "node:path";
import { EXIT, ToolingError, WORKSPACE, isMain, renderRequire, runCli } from "./toolchain.mjs";

const SCHEMES = new Set(["http:", "https:"]);

// --- address policy -------------------------------------------------------------------------------
const BLOCKED_V4 = [
  ["0.0.0.0", 8], // "this network"
  ["10.0.0.0", 8], // private
  ["100.64.0.0", 10], // CGNAT
  ["127.0.0.0", 8], // loopback
  ["169.254.0.0", 16], // link-local, including the 169.254.169.254 metadata service
  ["172.16.0.0", 12], // private
  ["192.0.0.0", 24], // IETF protocol assignments
  ["192.0.2.0", 24], // documentation
  ["192.88.99.0", 24], // 6to4 relay anycast
  ["192.168.0.0", 16], // private
  ["198.18.0.0", 15], // benchmarking
  ["198.51.100.0", 24], // documentation
  ["203.0.113.0", 24], // documentation
  ["224.0.0.0", 4], // multicast
  ["240.0.0.0", 4], // reserved, including broadcast
];

const v4Int = (ip) => ip.split(".").reduce((acc, part) => ((acc << 8) | Number(part)) >>> 0, 0);
function inV4Range(ip, base, bits) {
  const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;
  return (v4Int(ip) & mask) === (v4Int(base) & mask);
}

/** 16 bytes for an IPv6 address (zone id dropped, dotted IPv4 tail accepted), or null. */
function ipv6Bytes(input) {
  let ip = input.split("%")[0];
  let tail = null;
  const dotted = /(\d+\.\d+\.\d+\.\d+)$/.exec(ip);
  if (dotted) {
    if (!net.isIPv4(dotted[1])) return null;
    tail = dotted[1].split(".").map(Number);
    ip = `${ip.slice(0, -dotted[1].length)}0:0`;
  }
  const halves = ip.split("::");
  if (halves.length > 2) return null;
  const head = halves[0] ? halves[0].split(":") : [];
  const rest = halves.length === 2 && halves[1] ? halves[1].split(":") : [];
  const fill = halves.length === 2 ? 8 - head.length - rest.length : 0;
  if (fill < 0) return null;
  const groups = [...head, ...Array(fill).fill("0"), ...rest];
  if (groups.length !== 8 || groups.some((g) => !/^[0-9a-f]{1,4}$/i.test(g))) return null;
  const bytes = groups.flatMap((g) => {
    const n = parseInt(g, 16);
    return [n >> 8, n & 255];
  });
  if (tail) bytes.splice(12, 4, ...tail);
  return bytes;
}

const hasPrefix = (bytes, prefix, bits) => {
  for (let bit = 0; bit < bits; bit++) {
    const byte = bit >> 3;
    const mask = 0x80 >> (bit & 7);
    if ((bytes[byte] & mask) !== (prefix[byte] & mask)) return false;
  }
  return true;
};
const v4Of = (bytes, at) => bytes.slice(at, at + 4).join(".");
const BLOCKED_V6 = [
  [[0xfc], 7], // unique local
  [[0xfe, 0x80], 10], // link-local
  [[0xfe, 0xc0], 10], // site-local
  [[0xff], 8], // multicast
  [[0x20, 0x01, 0x0d, 0xb8], 32], // documentation
  [[0x20, 0x01, 0x00, 0x00], 32], // Teredo (embeds an obfuscated IPv4)
  [[0x01, 0x00, 0, 0, 0, 0, 0, 0], 64], // discard-only
  [[0x00, 0x64, 0xff, 0x9b, 0x00, 0x01], 48], // NAT64 local use
];

/** True for any address a fetch must not reach. Anything unparseable is refused. */
export function isBlockedIP(input) {
  const ip = String(input).replace(/^\[|\]$/g, "");
  if (net.isIPv4(ip)) return BLOCKED_V4.some(([base, bits]) => inV4Range(ip, base, bits));
  if (!net.isIPv6(ip)) return true;
  const bytes = ipv6Bytes(ip);
  if (!bytes) return true;
  // ::/96 (compatible, which includes :: and ::1) and ::ffff:0:0/96 (mapped) carry an IPv4 address.
  if (bytes.slice(0, 10).every((b) => b === 0) && ((bytes[10] === 0 && bytes[11] === 0) || (bytes[10] === 0xff && bytes[11] === 0xff))) {
    return isBlockedIP(v4Of(bytes, 12));
  }
  if (hasPrefix(bytes, [0x00, 0x64, 0xff, 0x9b], 96)) return isBlockedIP(v4Of(bytes, 12)); // NAT64
  if (hasPrefix(bytes, [0x20, 0x02], 16)) return isBlockedIP(v4Of(bytes, 2)); // 6to4
  return BLOCKED_V6.some(([prefix, bits]) => hasPrefix(bytes, [...prefix, ...Array(16).fill(0)], bits));
}

/**
 * The lookup every connection uses: resolve, refuse if ANY address is blocked, and hand the socket
 * exactly the addresses that were checked.
 */
export function guardedLookup(hostname, options, callback) {
  const opts = typeof options === "function" ? {} : options || {};
  const done = typeof options === "function" ? options : callback;
  dns.lookup(hostname, { all: true, family: opts.family || 0 }, (error, addresses) => {
    if (error) return done(error);
    if (!addresses.length) return done(new Error(`host does not resolve: ${hostname}`));
    const bad = addresses.find(({ address }) => isBlockedIP(address));
    if (bad) return done(Object.assign(new Error(`blocked address ${bad.address} for host ${hostname}`), { code: "EBLOCKED" }));
    if (opts.all) return done(null, addresses);
    return done(null, addresses[0].address, addresses[0].family);
  });
}

/** Parse and check one hop before any connection: scheme, credentials, literal addresses. */
export function checkUrl(raw) {
  let url;
  try {
    url = new URL(raw);
  } catch {
    throw new Error(`bad URL: ${String(raw).slice(0, 200)}`);
  }
  if (!SCHEMES.has(url.protocol)) throw new Error(`only http(s) URLs are fetched, not ${url.protocol}`);
  if (url.username || url.password) throw new Error("URLs with credentials are refused");
  const host = url.hostname.replace(/^\[|\]$/g, "");
  if (!host) throw new Error("URL has no host");
  if (host.toLowerCase() === "localhost" || host.toLowerCase().endsWith(".localhost")) throw new Error(`blocked host: ${host}`);
  if (net.isIP(host) && isBlockedIP(host)) throw new Error(`blocked address: ${host}`);
  return url;
}

// --- output ---------------------------------------------------------------------------------------
export function confineDir(root, dir) {
  const base = path.resolve(root);
  const abs = path.resolve(base, dir);
  if (abs !== base && !abs.startsWith(base + path.sep)) throw new ToolingError(`--out must be under ${base} (got ${abs})`);
  return abs;
}

const EXT_BY_MIME = {
  "image/png": "png", "image/jpeg": "jpg", "image/webp": "webp", "image/gif": "gif",
  "image/svg+xml": "svg", "image/avif": "avif", "application/pdf": "pdf",
};
export function safeName(url, base, contentType, sharpFormat) {
  let stem = base;
  if (!stem) {
    try {
      stem = path.basename(new URL(url).pathname) || "asset";
    } catch {
      stem = "asset";
    }
    stem = stem.replace(/\.[a-z0-9]{1,5}$/i, "");
  }
  stem = (stem || "asset").replace(/[^a-zA-Z0-9._-]/g, "-").replace(/-+/g, "-").replace(/^[.-]+/, "").slice(0, 60) || "asset";
  const hash = crypto.createHash("sha256").update(url).digest("hex").slice(0, 8);
  const ext = EXT_BY_MIME[(contentType || "").split(";")[0].trim().toLowerCase()] || (sharpFormat === "jpeg" ? "jpg" : sharpFormat) || "bin";
  return `${stem}-${hash}.${ext}`;
}

// --- fetch ----------------------------------------------------------------------------------------
function get(url, opt, signal) {
  return new Promise((resolve, reject) => {
    const lib = url.protocol === "https:" ? https : http;
    const req = lib.request(url, {
      method: "GET",
      agent: false,
      lookup: guardedLookup,
      signal,
      headers: { "user-agent": "design-agent/fetch-asset", accept: opt.kind === "image" ? "image/*" : "*/*" },
    });
    req.on("response", resolve);
    req.on("error", reject);
    req.end();
  });
}

export async function fetchOne(startUrl, opt) {
  const maxBytes = Math.floor(opt.maxMb * 1024 * 1024);
  const signal = AbortSignal.timeout(opt.timeoutMs);
  let current = startUrl;
  for (let hop = 0; hop <= opt.maxRedirects; hop++) {
    const url = checkUrl(current);
    const res = await get(url, opt, signal);
    if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
      res.resume();
      current = new URL(res.headers.location, url).href; // the next pass checks the new hop
      continue;
    }
    if (res.statusCode < 200 || res.statusCode >= 300) {
      res.resume();
      throw new Error(`HTTP ${res.statusCode}`);
    }
    const contentType = String(res.headers["content-type"] || "");
    if (opt.kind === "image" && !contentType.toLowerCase().startsWith("image/")) {
      res.resume();
      throw new Error(`not an image (content-type: ${contentType || "none"})`);
    }
    if (Number(res.headers["content-length"]) > maxBytes) {
      res.resume();
      throw new Error(`Content-Length exceeds --max-mb ${opt.maxMb}`);
    }
    const chunks = [];
    let size = 0;
    for await (const chunk of res) {
      size += chunk.length;
      if (size > maxBytes) {
        res.destroy();
        throw new Error(`exceeds --max-mb ${opt.maxMb}`);
      }
      chunks.push(chunk);
    }
    const buf = Buffer.concat(chunks);
    return { buf, contentType, finalUrl: url.href, sha256: crypto.createHash("sha256").update(buf).digest("hex") };
  }
  throw new Error(`too many redirects (> ${opt.maxRedirects})`);
}

function parseArgs(argv) {
  const urls = [];
  const opt = { out: null, kind: "image", maxMb: 25, name: null, timeoutMs: 30000, maxRedirects: 4, root: WORKSPACE };
  const value = (i) => {
    if (argv[i + 1] === undefined || argv[i + 1].startsWith("--")) throw new ToolingError(`${argv[i]} needs a value`);
    return argv[i + 1];
  };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--out") opt.out = value(i++);
    else if (a === "--kind") opt.kind = value(i++);
    else if (a === "--max-mb") opt.maxMb = Number(value(i++));
    else if (a === "--name") opt.name = value(i++);
    else if (a === "--timeout-ms") opt.timeoutMs = Number(value(i++));
    else if (a === "--max-redirects") opt.maxRedirects = Number(value(i++));
    else if (a === "--root") opt.root = value(i++);
    else if (a.startsWith("--")) throw new ToolingError(`unknown flag: ${a}`);
    else urls.push(a);
  }
  if (!urls.length) throw new ToolingError("usage: fetch-asset.mjs <url> [<url> ...] --out <dir> [--kind image|any]");
  if (!opt.out) throw new ToolingError("--out <dir> is required");
  if (!["image", "any"].includes(opt.kind)) throw new ToolingError("--kind must be image or any");
  if (!(opt.maxMb > 0)) throw new ToolingError("--max-mb must be > 0");
  if (!(opt.timeoutMs > 0)) throw new ToolingError("--timeout-ms must be > 0");
  if (!(Number.isInteger(opt.maxRedirects) && opt.maxRedirects >= 0 && opt.maxRedirects <= 10)) throw new ToolingError("--max-redirects must be 0-10");
  return { urls, opt };
}

if (isMain(import.meta.url)) {
  await runCli("fetch-asset", async () => {
    const { urls, opt } = parseArgs(process.argv.slice(2));
    const outDir = confineDir(opt.root, opt.out);
    const saved = [];
    let failures = 0;
    let sharp = null;
    for (const url of urls) {
      try {
        const { buf, contentType, finalUrl, sha256 } = await fetchOne(url, opt);
        let meta = { format: null, width: null, height: null };
        if (opt.kind === "image") {
          sharp ??= renderRequire("sharp"); // a missing toolchain is a ToolingError: exit 2
          const m = await sharp(buf).metadata().catch(() => null);
          if (!m?.width || !m?.height) throw new Error("not a decodable image");
          meta = { format: m.format, width: m.width, height: m.height };
        }
        await mkdir(outDir, { recursive: true });
        const file = path.join(outDir, safeName(url, opt.name && urls.length === 1 ? opt.name : null, contentType, meta.format));
        await writeFile(file, buf);
        saved.push({ url, finalUrl, path: file, contentType, bytes: buf.length, sha256, ...meta });
      } catch (error) {
        if (error instanceof ToolingError) throw error;
        failures++;
        console.log(`__FETCH_ERR__ ${url} ${error?.name === "TimeoutError" ? `timed out after ${opt.timeoutMs}ms` : error?.message || error}`);
      }
    }
    if (saved.length) {
      console.log("__FETCH_OK__");
      for (const asset of saved) console.log(JSON.stringify(asset));
    }
    return failures ? EXIT.FINDINGS : EXIT.PASS;
  });
}