Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

10.9 KB
Raw Blame
/**
 * Public-web backends for the WebFetch and WebSearch catalog tools.
 *
 * WebFetch resolves each hop's hostname once, rejects the hop if any resolved
 * address is private, and pins the socket to the validated address through a
 * custom `lookup`. The connection therefore never performs a second DNS query,
 * which closes the rebinding window between validation and connect. Bodies are
 * streamed with a byte cap and truncated with a visible marker.
 */
import * as NodeDnsPromises from "node:dns/promises";
import * as NodeHttp from "node:http";
import * as NodeHttps from "node:https";
import * as NodeNet from "node:net";

export const AKERU_WEB_FETCH_MAX_BYTES = 2 * 1024 * 1024;

export const AKERU_WEB_FETCH_TIMEOUT_MS = 15_000;

export const AKERU_WEB_FETCH_MAX_REDIRECTS = 5;

export const AKERU_WEB_FETCH_TRUNCATION_MARKER = "\n\n[WebFetch truncated the response at";

// Separate lists because a BlockList also matches IPv4 addresses against
// IPv4-mapped IPv6 rules, which would block every public IPv4 address.
const PRIVATE_IPV4 = (() => {
  const list = new NodeNet.BlockList();

  for (const [network, prefix] of [
    ["0.0.0.0", 8],
    ["10.0.0.0", 8],
    ["100.64.0.0", 10],
    ["127.0.0.0", 8],
    ["169.254.0.0", 16],
    ["172.16.0.0", 12],
    ["192.0.0.0", 24],
    ["192.168.0.0", 16],
    ["198.18.0.0", 15],
    ["224.0.0.0", 3],
  ] as const) {
    list.addSubnet(network, prefix, "ipv4");
  }

  return list;
})();

const PRIVATE_IPV6 = (() => {
  const list = new NodeNet.BlockList();

  for (const [network, prefix] of [
    ["::", 128],
    ["::1", 128],
    // IPv4-mapped and IPv4-compatible forms can smuggle any IPv4 address.
    ["::ffff:0:0", 96],
    ["::", 96],
    // NAT64 and 6to4 prefixes also embed an IPv4 address the network can reach.
    ["64:ff9b::", 96],
    ["64:ff9b:1::", 48],
    ["2002::", 16],
    ["fc00::", 7],
    ["fe80::", 10],
    // Deprecated site-local space is still routed on some internal networks.
    ["fec0::", 10],
    ["ff00::", 8],
  ] as const) {
    list.addSubnet(network, prefix, "ipv6");
  }

  return list;
})();

/** True for loopback, private, link-local, CGNAT, multicast, and mapped addresses. */
export function isAkeruPrivateAddress(address: string): boolean {
  const family = NodeNet.isIP(address);

  if (family === 4) return PRIVATE_IPV4.check(address, "ipv4");

  if (family === 6) return PRIVATE_IPV6.check(address, "ipv6");

  return false;
}

/**
 * Shape check for a WebFetch URL. It rejects non-HTTP(S) schemes, embedded
 * credentials, local hostnames, and private literal addresses. DNS validation
 * happens only in the fetch backend, against the address it connects to.
 */
export function parseAkeruPublicUrl(raw: string): URL {
  let url: URL;

  try {
    url = new URL(raw);
  } catch {
    throw new Error("WebFetch URL must be valid HTTP or HTTPS.");
  }

  if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) {
    throw new Error("WebFetch only accepts public HTTP(S) URLs without credentials.");
  }

  const host = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");

  if (
    host === "localhost" ||
    host.endsWith(".localhost") ||
    host.endsWith(".local") ||
    host.endsWith(".internal") ||
    isAkeruPrivateAddress(host)
  ) {
    throw new Error("WebFetch rejects private, loopback, and local addresses.");
  }

  return url;
}

export type AkeruWebFetchLookup = (
  hostname: string,
) => Promise<ReadonlyArray<{ readonly address: string; readonly family: number }>>;

export interface AkeruWebFetchOptions {
  readonly lookup?: AkeruWebFetchLookup;
  /** Tests allow loopback here. Production rejects every private address. */
  readonly allowAddress?: (address: string) => boolean;
  readonly maxBytes?: number;
  readonly timeoutMs?: number;
  readonly maxRedirects?: number;
}

export interface AkeruWebFetchResult {
  readonly url: string;
  readonly status: number;
  readonly contentType: string | null;
  readonly text: string;
  readonly truncated: boolean;
}

interface PinnedAddress {
  readonly address: string;
  readonly family: 4 | 6;
}

async function resolvePinnedAddress(
  url: URL,
  lookup: AkeruWebFetchLookup,
  allowAddress: (address: string) => boolean,
): Promise<PinnedAddress> {
  const host = url.hostname.replace(/^\[|\]$/g, "");
  const literal = NodeNet.isIP(host);
  const records = literal !== 0 ? [{ address: host, family: literal }] : await lookup(host);

  if (records.length === 0) throw new Error(`WebFetch could not resolve ${host}.`);

  if (records.some((record) => NodeNet.isIP(record.address) === 0)) {
    throw new Error(`WebFetch could not resolve ${host} to a valid address.`);
  }

  if (records.some((record) => !allowAddress(record.address))) {
    throw new Error("WebFetch rejects private, loopback, and local addresses.");
  }

  const first = records[0]!;

  return { address: first.address, family: NodeNet.isIP(first.address) === 6 ? 6 : 4 };
}

function pinnedLookup(pinned: PinnedAddress): NodeHttp.RequestOptions["lookup"] {
  // SAFETY: This lookup implements both Node DNS overloads, returning a list only when all is requested.
  return ((
    _hostname: string,
    options: { readonly all?: boolean },
    callback: (...args: unknown[]) => void,
  ) => {
    if (options?.all) {
      callback(null, [{ address: pinned.address, family: pinned.family }]);
    } else {
      callback(null, pinned.address, pinned.family);
    }
  }) as NodeHttp.RequestOptions["lookup"];
}

interface HopResponse {
  readonly status: number;
  readonly location: string | undefined;
  readonly contentType: string | null;
  readonly body: Buffer;
  readonly truncated: boolean;
}

function requestHop(
  url: URL,
  pinned: PinnedAddress,
  maxBytes: number,
  timeoutMs: number,
): Promise<HopResponse> {
  const transport = url.protocol === "https:" ? NodeHttps : NodeHttp;

  return new Promise((resolve, reject) => {
    const request = transport.request(
      url,
      {
        method: "GET",
        // A dedicated agent never picks up an environment proxy, which would
        // resolve the hostname itself and bypass the pinned address.
        agent: false,
        lookup: pinnedLookup(pinned),
        timeout: timeoutMs,
        headers: {
          "accept-encoding": "identity",
          "user-agent": "akeru-bot WebFetch",
        },
      },
      (response) => {
        const status = response.statusCode ?? 0;
        const location = response.headers.location;
        const contentType = response.headers["content-type"] ?? null;

        if (status >= 300 && status < 400) {
          response.resume();
          resolve({ status, location, contentType, body: Buffer.alloc(0), truncated: false });

          return;
        }

        const chunks: Buffer[] = [];
        let received = 0;
        let truncated = false;
        response.on("data", (chunk: Buffer) => {
          if (truncated) return;
          const remaining = maxBytes - received;

          if (chunk.length > remaining) {
            chunks.push(chunk.subarray(0, remaining));
            received = maxBytes;
            truncated = true;
            resolve({ status, location, contentType, body: Buffer.concat(chunks), truncated });
            response.destroy();
            request.destroy();

            return;
          }

          chunks.push(chunk);
          received += chunk.length;
        });
        response.on("end", () =>
          resolve({ status, location, contentType, body: Buffer.concat(chunks), truncated }),
        );
        response.on("error", (error) => {
          if (!truncated) reject(error);
        });
        response.on("close", () => {
          if (!truncated && !response.complete) {
            reject(new Error("WebFetch lost the connection before the response finished."));
          }
        });
      },
    );

    // `timeout` only covers idle sockets. The deadline also bounds a server
    // that trickles bytes to hold the request open.
    const deadline = setTimeout(() => request.destroy(new Error("WebFetch timed out.")), timeoutMs);
    request.on("close", () => clearTimeout(deadline));
    request.on("timeout", () => request.destroy(new Error("WebFetch timed out.")));
    request.on("error", reject);
    request.end();
  });
}

// A stalled resolver must not hold the tool call open past the fetch deadline.
function lookupWithDeadline(lookup: AkeruWebFetchLookup, timeoutMs: number): AkeruWebFetchLookup {
  return (hostname) =>
    new Promise((resolve, reject) => {
      const deadline = setTimeout(
        () => reject(new Error(`WebFetch timed out resolving ${hostname}.`)),
        timeoutMs,
      );

      lookup(hostname).then(
        (records) => {
          clearTimeout(deadline);
          resolve(records);
        },
        (cause: unknown) => {
          clearTimeout(deadline);
          reject(cause);
        },
      );
    });
}

export function createAkeruWebFetch(options: AkeruWebFetchOptions = {}) {
  const timeoutMs = options.timeoutMs ?? AKERU_WEB_FETCH_TIMEOUT_MS;

  const lookup = lookupWithDeadline(
    options.lookup ?? ((hostname) => NodeDnsPromises.lookup(hostname, { all: true })),
    timeoutMs,
  );

  const allowAddress = options.allowAddress ?? ((address) => !isAkeruPrivateAddress(address));
  const maxBytes = options.maxBytes ?? AKERU_WEB_FETCH_MAX_BYTES;
  const maxRedirects = options.maxRedirects ?? AKERU_WEB_FETCH_MAX_REDIRECTS;

  return async (input: { readonly url: string }): Promise<AkeruWebFetchResult> => {
    let url = parseAkeruPublicUrl(input.url);

    for (let redirects = 0; ; redirects += 1) {
      const pinned = await resolvePinnedAddress(url, lookup, allowAddress);
      const hop = await requestHop(url, pinned, maxBytes, timeoutMs);

      if (hop.status >= 300 && hop.status < 400) {
        if (!hop.location) throw new Error("WebFetch received a redirect without a location.");

        if (redirects >= maxRedirects) throw new Error("WebFetch followed too many redirects.");
        url = parseAkeruPublicUrl(new URL(hop.location, url).toString());
        continue;
      }

      if (hop.status < 200 || hop.status >= 300) {
        throw new Error(`WebFetch failed with HTTP ${hop.status}.`);
      }

      const text = new TextDecoder().decode(hop.body);

      return {
        url: url.toString(),
        status: hop.status,
        contentType: hop.contentType,
        text: hop.truncated
          ? `${text}${AKERU_WEB_FETCH_TRUNCATION_MARKER} ${maxBytes} bytes.]`
          : text,
        truncated: hop.truncated,
      };
    }
  };
}

/**
 * WebSearch has no search index of its own. Mastra-driven providers do not
 * expose a native search call Akeru can invoke, so the tool reports that plainly
 * instead of inventing results.
 */
export async function akeruWebSearchUnavailable(input: {
  readonly query: string;
  readonly domains?: readonly string[];
}) {
  return {
    status: "unavailable" as const,
    query: input.query,
    results: [],
    reason: "Web search is not available for this bot's provider in Akeru yet.",
    suggestion: "Use WebFetch with a URL you already know.",
  };
}