Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
12.2 KB
Raw Blame
/**
 * Seam 2 of the knowledge-intake-and-consolidation wave (R1, t_1dcbf352):
 * the keyed external-fetch helper shared by the Brave provider, the Tavily
 * provider, and the full-page extract step.
 *
 * Responsibilities kept deliberately small (KISS):
 *   - env gate: a `null` API key makes every call a typed `disabled` error,
 *     so a keyless caller behaves byte-identically to doing nothing;
 *   - auth: `Authorization: Bearer <key>` by default, or a named header for
 *     APIs that carry the key elsewhere (Brave's `X-Subscription-Token`);
 *   - typed errors: network, auth, http, and payload failures are distinct
 *     {@link ExternalFetchError} kinds so a caller can carry them in a report
 *     instead of inventing content;
 *   - a shared response cache keyed by the NORMALIZED request (method, url,
 *     sorted query, body) - never by the key, so a credential can never leak
 *     into a cache key.
 *
 * The transport is an injected seam so the whole module is unit-testable with
 * no network. The real transport ({@link createFetchTransport}) is a thin
 * `fetch` wrapper built only by callers that actually reach the network.
 *
 * Key hygiene: the helper never logs, and every composed error message is run
 * through the shared redactor with the key as a literal, so even an upstream
 * error string that happens to embed the key is scrubbed before it surfaces.
 */

import { canonicalJson } from "../../integrity/digest.ts";
import { redactRawOutput } from "../../redactor.ts";
import { assertHttpEgressEndpoint } from "../../search/embeddings/http-util.ts";

/** Distinct failure kinds a keyed fetch can produce. */
export type ExternalFetchErrorKind =
  | "disabled"
  | "refused"
  | "network"
  | "auth"
  | "http"
  | "payload";

/** A typed failure of a keyed external fetch. Carries the HTTP status when one exists. */
export class ExternalFetchError extends Error {
  readonly kind: ExternalFetchErrorKind;
  readonly status: number | null;

  constructor(kind: ExternalFetchErrorKind, message: string, status: number | null = null) {
    super(message);
    this.name = "ExternalFetchError";
    this.kind = kind;
    this.status = status;
  }
}

export type ExternalFetchMethod = "GET" | "POST";

/** Auth scheme applied to the request. Bearer is the default and primary. */
export type ExternalFetchAuth =
  | { readonly scheme: "bearer" }
  | { readonly scheme: "header"; readonly header: string };

export interface ExternalFetchRequest {
  readonly url: string;
  /** Defaults to GET. */
  readonly method?: ExternalFetchMethod;
  /** Query parameters, appended in a deterministic (sorted) order. */
  readonly query?: Readonly<Record<string, string>>;
  /** JSON request body for POST calls. */
  readonly body?: unknown;
  /** Auth scheme; defaults to Bearer. */
  readonly auth?: ExternalFetchAuth;
  /** Response shape to parse; defaults to json. */
  readonly accept?: "json" | "text";
}

/** Minimal response shape the transport must return. */
export interface ExternalFetchResponse {
  readonly ok: boolean;
  readonly status: number;
  json(): Promise<unknown>;
  text(): Promise<string>;
}

/** Transport seam - the single call the helper makes. Injected in tests. */
export interface ExternalFetchTransport {
  (input: {
    readonly url: string;
    readonly method: ExternalFetchMethod;
    readonly headers: Readonly<Record<string, string>>;
    readonly body: string | null;
  }): Promise<ExternalFetchResponse>;
}

/** A response cache keyed by the normalized request. Keys never carry the API key. */
export interface ResponseCache {
  get(key: string): unknown | undefined;
  set(key: string, value: unknown): void;
}

/** An in-memory {@link ResponseCache}. Suitable for one process run. */
export function createMemoryResponseCache(): ResponseCache {
  const store = new Map<string, unknown>();
  return {
    get: (key) => store.get(key),
    set: (key, value) => {
      store.set(key, value);
    },
  };
}

export interface KeyedFetchConfig {
  /** API key; `null` disables the helper - every call is a typed `disabled` error. */
  readonly apiKey: string | null;
  readonly transport: ExternalFetchTransport;
  /** Optional shared response cache keyed by the normalized request. */
  readonly cache?: ResponseCache;
}

/** Query string in deterministic (sorted-key) order. */
function sortedQuery(query: Readonly<Record<string, string>> | undefined): string {
  if (query === undefined) return "";
  return Object.keys(query)
    .toSorted()
    .map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(query[key]!)}`)
    .join("&");
}

/**
 * Deterministic cache key for a request. Built ONLY from the accept type,
 * method, url, sorted query, and body - never from the API key or auth headers
 * - so no credential can leak into a cache key or the paths derived from one.
 * The accept type is part of the key so a shared cache can never serve a JSON
 * value where text was expected (or vice versa).
 *
 * The body is serialized by the shared {@link canonicalJson}. This module
 * carried its own sorted-key stringifier with the same key ordering and one
 * difference: an object entry whose value is `undefined` rendered as
 * `"key":null` instead of being omitted. Omitting it is what the body
 * actually sent does - `JSON.stringify` drops those entries - so the shared
 * encoding keys two requests that go out as identical bytes to one entry
 * rather than two, and no value the module persists changes, because the key
 * is only ever an in-memory cache key.
 */
export function normalizeRequestKey(req: ExternalFetchRequest): string {
  const accept = req.accept ?? "json";
  const method = req.method ?? "GET";
  const query = sortedQuery(req.query);
  const body = req.body === undefined ? "" : canonicalJson(req.body);
  return `${accept} ${method} ${req.url}?${query}#${body}`;
}

function buildUrl(req: ExternalFetchRequest): string {
  const query = sortedQuery(req.query);
  if (query.length === 0) return req.url;
  return req.url.includes("?") ? `${req.url}&${query}` : `${req.url}?${query}`;
}

function buildHeaders(apiKey: string, req: ExternalFetchRequest): Record<string, string> {
  const headers: Record<string, string> = {};
  const auth = req.auth ?? { scheme: "bearer" };
  if (auth.scheme === "bearer") {
    headers["Authorization"] = `Bearer ${apiKey}`;
  } else {
    headers[auth.header] = apiKey;
  }
  if ((req.method ?? "GET") === "POST") headers["content-type"] = "application/json";
  headers["accept"] = req.accept === "text" ? "text/plain, text/html" : "application/json";
  return headers;
}

function messageOf(err: unknown): string {
  return err instanceof Error ? err.message : String(err);
}

/**
 * The URL's host when it is an IPv4 or IPv6 literal in a private,
 * link-local, carrier-grade-NAT or unspecified range, else null. Loopback
 * is left to {@link assertHttpEgressEndpoint}, which admits it as the
 * local-server case.
 */
export function privateIpLiteral(rawUrl: string): string | null {
  let host: string;
  try {
    host = new URL(rawUrl).hostname.toLowerCase();
  } catch {
    return null;
  }
  const v6 = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : null;
  if (v6 !== null) {
    // IPv4-mapped: the URL parser normalises `::ffff:10.0.0.1` to the
    // hex form `::ffff:a00:1`, so both spellings are read back to dotted.
    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(v6);
    if (mapped) return isPrivateV4(mapped[1]!) ? host : null;
    const mappedHex = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(v6);
    if (mappedHex) {
      const hi = Number.parseInt(mappedHex[1]!, 16);
      const lo = Number.parseInt(mappedHex[2]!, 16);
      const dotted = `${hi >> 8}.${hi & 0xff}.${lo >> 8}.${lo & 0xff}`;
      return isPrivateV4(dotted) ? host : null;
    }
    if (v6 === "::") return host;
    // fc00::/7 unique-local, fe80::/10 link-local.
    if (/^f[cd][0-9a-f]{0,2}:/.test(v6) || /^fe[89ab][0-9a-f]?:/.test(v6)) return host;
    return null;
  }
  return /^\d+\.\d+\.\d+\.\d+$/.test(host) && isPrivateV4(host) ? host : null;
}

function isPrivateV4(dotted: string): boolean {
  const [a, b] = dotted.split(".").map(Number) as [number, number];
  return (
    a === 0 ||
    a === 10 ||
    (a === 100 && b >= 64 && b <= 127) ||
    (a === 169 && b === 254) ||
    (a === 172 && b >= 16 && b <= 31) ||
    (a === 192 && b === 168)
  );
}

/**
 * Perform a keyed external fetch through the injected transport. Enforces the
 * env gate, applies auth, consults and populates the cache, and maps every
 * failure to a typed {@link ExternalFetchError}. The API key is redacted from
 * any error message it composes.
 */
export async function keyedFetch(
  config: KeyedFetchConfig,
  req: ExternalFetchRequest,
): Promise<unknown> {
  const key = config.apiKey;
  if (key === null || key.trim().length === 0) {
    throw new ExternalFetchError("disabled", "external fetch is disabled: no API key configured");
  }

  // The key rides every request as a bearer credential, so the URL the
  // request names is validated BEFORE the key is attached to it: https
  // only, with plain http allowed for loopback hosts (the local-server
  // case). The two named providers use constant https endpoints today;
  // this wall is for the day a caller does not - a URL harvested from
  // content must not become a cleartext or internal-network bearer drop.
  try {
    assertHttpEgressEndpoint(req.url, "external fetch url");
  } catch (err) {
    throw new ExternalFetchError("refused", messageOf(err));
  }
  // The scheme rule admits `https://169.254.169.254/` and every other
  // private address, and `extractPage` takes a URL harvested from content
  // (audit L2). An IP literal in a private, link-local, carrier-grade NAT
  // or unspecified range is refused before the key is attached: a bearer
  // credential has no business on the metadata service or the LAN. A
  // name that RESOLVES to such an address is not caught here (no DNS is
  // done); the constant provider endpoints are public names.
  const privateHost = privateIpLiteral(req.url);
  if (privateHost !== null) {
    throw new ExternalFetchError(
      "refused",
      `external fetch url names a private or link-local address (${privateHost}); ` +
        `a keyed request is never sent there`,
    );
  }

  const cacheKey = normalizeRequestKey(req);
  if (config.cache !== undefined) {
    const hit = config.cache.get(cacheKey);
    if (hit !== undefined) return hit;
  }

  const method = req.method ?? "GET";
  const headers = buildHeaders(key, req);
  const body = method === "POST" && req.body !== undefined ? JSON.stringify(req.body) : null;

  const redact = (message: string): string => redactRawOutput(message, { literals: [key] });

  let res: ExternalFetchResponse;
  try {
    res = await config.transport({ url: buildUrl(req), method, headers, body });
  } catch (err) {
    throw new ExternalFetchError("network", redact(`network failure: ${messageOf(err)}`));
  }

  if (!res.ok) {
    if (res.status === 401 || res.status === 403) {
      throw new ExternalFetchError(
        "auth",
        `authentication failed with HTTP ${res.status}`,
        res.status,
      );
    }
    throw new ExternalFetchError("http", `request failed with HTTP ${res.status}`, res.status);
  }

  let value: unknown;
  try {
    value = req.accept === "text" ? await res.text() : await res.json();
  } catch (err) {
    throw new ExternalFetchError(
      "payload",
      redact(`response payload was not valid: ${messageOf(err)}`),
    );
  }

  if (config.cache !== undefined) config.cache.set(cacheKey, value);
  return value;
}

/**
 * The real `fetch`-backed transport. Built by callers that reach the
 * network; its tests drive it against a loopback server only.
 *
 * `redirect: "error"`, like the three search-provider transports: the
 * request carries the key, and the fetch default would follow a 3xx to
 * any host it names, key and all.
 */
export function createFetchTransport(): ExternalFetchTransport {
  return async (input) => {
    const res = await fetch(input.url, {
      method: input.method,
      headers: input.headers,
      redirect: "error",
      ...(input.body !== null ? { body: input.body } : {}),
    });
    return {
      ok: res.ok,
      status: res.status,
      json: () => res.json() as Promise<unknown>,
      text: () => res.text(),
    };
  };
}