Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
16.1 KB
Raw Blame
/**
 * Parameterized research pipeline (Knowledge Provenance suite).
 *
 * Turns N sources plus an agent-run synthesis into one dated, cited report
 * page in the vault. Each finding cites the source(s) that flagged it, so the
 * report is auditable back to its inputs and becomes a first-class recall
 * input itself.
 *
 * Provider-agnostic: the agent pulls the sources and writes the findings; OSB
 * runs no model. OSB owns the deterministic half - validating that every
 * finding cites at least one of the consulted sources (no uncited claims),
 * stamping provenance, and writing the report page idempotently (one
 * date+title maps to one report, rewritten in place).
 *
 * The citation constraint is the point: a finding with no source, or a finding
 * citing a source that was not consulted, is rejected rather than written as
 * an unprovenanced claim.
 */

import { relative } from "node:path";

import type { FrontmatterMap } from "../../types.ts";
import { canonicalNotePath } from "../../path-safety.ts";
import { slugify } from "../../vault.ts";
import { isoDate, isoSecond } from "../time.ts";
import { reportPagePath } from "../paths.ts";
import {
  RESEARCH_REPORT_SHAPE,
  RESEARCH_REPORT_SURFACE,
  assertResponseShape,
} from "../response-shape.ts";
import { isSecretReferenceValue, resolveNamedSecret } from "../../secret-resolver.ts";
import { renderProvenanceSection, type Provenance } from "../provenance/provenance.ts";
import { CONTENT_CLASS, CONTENT_CLASS_FIELD } from "../provenance/content-class.ts";
import {
  CAPTURE_SCOPE,
  captureScopesFrontmatter,
  resolveCaptureScope,
  type CaptureScope,
  type CaptureScopeResolution,
} from "../provenance/capture-scope.ts";
import { SourceTrustError } from "../intake/source-trust.ts";
import {
  ExternalFetchError,
  createFetchTransport,
  createMemoryResponseCache,
  type ExternalFetchTransport,
  type KeyedFetchConfig,
  type ResponseCache,
} from "./external-fetch.ts";
import { BRAVE_API_KEY_ENV, BRAVE_PROVIDER_NAME, createBraveProvider } from "./providers/brave.ts";
import {
  TAVILY_API_KEY_ENV,
  TAVILY_PROVIDER_NAME,
  createTavilyProvider,
} from "./providers/tavily.ts";
import type { ProviderSearchResult, ResearchProvider } from "./providers/provider.ts";
import { assertLaneWrite, lanePageExists, writeLanePage } from "../../store/lane-pages.ts";

export { BRAVE_API_KEY_ENV, BRAVE_PROVIDER_NAME, TAVILY_API_KEY_ENV, TAVILY_PROVIDER_NAME };
export type { ProviderSearchResult, ResearchProvider };

/** Frontmatter `kind:` marker of a research report page. */
export const BRAIN_REPORT_KIND = "brain-report";

/** One finding plus the sources that flagged it. */
export interface ResearchFinding {
  readonly statement: string;
  /** Source identifiers (a subset of the consulted sources) that flagged this. */
  readonly sources: readonly string[];
}

export interface ResearchReportInput {
  readonly title: string;
  readonly findings: readonly ResearchFinding[];
  /** Every source consulted for the report. */
  readonly sources: readonly string[];
}

export interface ResearchReportOptions {
  readonly agent: string;
  readonly now: Date;
  /**
   * May the caller read this vault-relative path? Supplied by a surface
   * that answers at a reach narrower than the vault (the MCP tool at remote
   * reach); a local caller passes nothing. A source backed by a file the
   * caller cannot read is reported `url-only`, the same answer as an absent
   * file, in the result and on the page.
   */
  readonly readable?: (rel: string) => boolean;
}

export interface ResearchReportResult {
  /** Vault-relative path of the report page. */
  readonly reportPath: string;
  readonly created: boolean;
  readonly findingCount: number;
  /**
   * The current capture scope of every consulted source, parallel to
   * `input.sources`. A report stores no excerpt, so each member is
   * `full-local` (a vault file) or `url-only` (no local bytes).
   */
  readonly captureScopes: ReadonlyArray<CaptureScope>;
}

/** A research report failed validation; nothing was written. */
export class ResearchValidationError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "ResearchValidationError";
  }
}

/**
 * Ingress of the agent-authored report payload. The payload is checked
 * against the frozen research-report shape BEFORE the citation contract is
 * evaluated, so a finding whose `sources` is not a list of strings is refused
 * by path instead of being read as one. A violation throws
 * `ResponseShapeError` and no report page is written - not even the findings
 * that preceded the malformed one.
 */
export function parseResearchReportInput(payload: unknown): ResearchReportInput {
  assertResponseShape(RESEARCH_REPORT_SURFACE, RESEARCH_REPORT_SHAPE, payload);
  const rec = payload as Record<string, unknown>;
  const findings = rec["findings"] as ReadonlyArray<Record<string, unknown>>;
  return {
    title: rec["title"] as string,
    sources: Object.freeze([...(rec["sources"] as ReadonlyArray<string>)]),
    findings: Object.freeze(
      findings.map((finding) =>
        Object.freeze({
          statement: finding["statement"] as string,
          sources: Object.freeze([...(finding["sources"] as ReadonlyArray<string>)]),
        }),
      ),
    ),
  };
}

/** Line breaks a title may not hold: it is rendered as the page's one heading line. */
const TITLE_LINE_BREAK_RE = /[\r\n]/;

/** Wrap a bare source identifier in a wikilink; leave an existing one as-is. */
function asWikilink(source: string): string {
  const trimmed = source.trim();
  return trimmed.startsWith("[[") ? trimmed : `[[${canonicalNotePath(trimmed)}]]`;
}

function validate(input: ResearchReportInput): void {
  if (input.title.trim().length === 0) {
    throw new ResearchValidationError("report title must not be empty");
  }
  if (TITLE_LINE_BREAK_RE.test(input.title)) {
    throw new ResearchValidationError("report title spans more than one line; a title is one line");
  }
  if (input.sources.length === 0) {
    throw new ResearchValidationError("a report must consult at least one source");
  }
  if (input.findings.length === 0) {
    throw new ResearchValidationError("a report must contain at least one finding");
  }
  const consulted = new Set(input.sources.map((s) => s.trim()));
  for (const [i, finding] of input.findings.entries()) {
    if (finding.statement.trim().length === 0) {
      throw new ResearchValidationError(`finding[${i}] statement must not be empty`);
    }
    if (finding.sources.length === 0) {
      throw new ResearchValidationError(
        `finding[${i}] must cite at least one source (no uncited claims)`,
      );
    }
    for (const src of finding.sources) {
      if (!consulted.has(src.trim())) {
        throw new ResearchValidationError(
          `finding[${i}] cites a source not in the consulted set: ${JSON.stringify(src)}`,
        );
      }
    }
  }
}

/**
 * The capture scope of one consulted source as the caller may know it. A
 * `full-local` source whose backing file the caller cannot read answers
 * `url-only`, so the scope never tells a narrower caller that a hidden page
 * exists. The backing file is the identity's own path, or its `.md` note
 * when the identity names no extension (see `resolveCaptureScope`).
 *
 * A source whose `stat` the filesystem refuses is inside this vault but
 * unreadable. The scope is an annotation, so it never aborts the report, and
 * it answers `url-only`: the vault can show none of the file's bytes, and the
 * answer is the same at every reach, so it names nothing about the path.
 * The `capture-scope` hygiene detector deliberately treats the same refused
 * stat as backing, unless the caller may not read the file: this stamp is a
 * snapshot of what the report could show, while the detector avoids warning
 * about a file that may well exist.
 */
function captureScopeAtReach(
  vault: string,
  source: string,
  readable: ((rel: string) => boolean) | undefined,
): CaptureScope {
  let resolved: CaptureScopeResolution;
  try {
    resolved = resolveCaptureScope(vault, source);
  } catch (err) {
    if (!(err instanceof SourceTrustError)) throw err;
    return CAPTURE_SCOPE.urlOnly;
  }
  const { scope, backing } = resolved;
  if (backing === null || readable === undefined) return scope;
  return readable(backing) ? scope : CAPTURE_SCOPE.urlOnly;
}

/**
 * Write a dated, cited research report. Validates the citation contract first
 * (throwing {@link ResearchValidationError} with no write on failure), then
 * writes the report page idempotently on the date+title path.
 */
export function writeResearchReport(
  vault: string,
  input: ResearchReportInput,
  opts: ResearchReportOptions,
): ResearchReportResult {
  // Vault-identity write guard (context-integrity-gates, Unit J).
  assertLaneWrite(vault);
  validate(input);

  const date = isoDate(opts.now);
  const stamp = isoSecond(opts.now);
  const absPath = reportPagePath(vault, date, slugify(input.title));

  const findingLines = input.findings.map((f) => {
    const cites = f.sources.map(asWikilink).join(", ");
    return `- ${f.statement.trim()} (cites: ${cites})`;
  });
  const provenance: Provenance = {
    level: "stated",
    sources: input.sources.map(asWikilink),
    premises: [],
  };

  // Classified from the identity alone (no byte read), in the consulted
  // order, so `capture_scopes[i]` always describes `sources[i]`.
  const captureScopes = Object.freeze(
    input.sources.map((source) => captureScopeAtReach(vault, source, opts.readable)),
  );

  const body = [
    `# ${input.title.trim()}`,
    ["## Findings", "", ...findingLines].join("\n"),
    renderProvenanceSection(provenance),
  ].join("\n\n");

  const meta: FrontmatterMap = {
    kind: BRAIN_REPORT_KIND,
    title: input.title.trim(),
    report_date: date,
    provenance: provenance.level,
    source_count: input.sources.length,
    // Written only when some source is not a local file, so a report built
    // from vault files alone stays byte-identical to before.
    ...captureScopesFrontmatter(captureScopes),
    // Server-derived content class (F5): a research report is prompt-
    // derived material by definition - the pipeline synthesised it from
    // provider runs - so the writer that knows the material stamps it
    // here, at creation. The shared exports leave such pages out by
    // default and name the omission.
    [CONTENT_CLASS_FIELD]: CONTENT_CLASS.promptDerived,
    created_at: stamp,
    updated_at: stamp,
    tags: ["brain", "brain/report"],
  };

  // Idempotent on date+title: a re-run rewrites the same report page in
  // place (a file on an unswitched machine, a lane entry on a store-only
  // one). `lanePageExists` drives the `created` flag in both modes; the
  // overwrite then surfaces a real write error directly instead of a catch
  // masking it as a re-run.
  const created = !lanePageExists(vault, absPath);
  writeLanePage(vault, absPath, meta, body, { overwrite: true });

  return {
    reportPath: canonicalNotePath(relative(vault, absPath)),
    created,
    findingCount: input.findings.length,
    captureScopes,
  };
}

// ----- Provider pool wiring (R1, t_1dcbf352) --------------------------------
//
// The pool is additive: it holds a provider only when that provider's key env
// is set. A keyless deployment builds an EMPTY pool and the rest of the
// research surface (writeResearchReport) is untouched, so a vault that never
// configures a key behaves byte-identically to before this feature.

/** Resolved provider keys, read from the environment (or an injected map). */
export interface ResearchPoolEnv {
  readonly braveApiKey: string | null;
  readonly tavilyApiKey: string | null;
}

/** Read the provider key envs. An injected map keeps this deterministic in tests.
 * A key env value written as a `$secret:NAME` reference resolves through the
 * custody store of `secretsVault` (store first, env fallback); without a vault
 * the values are taken as-is, exactly as before the resolver routing existed.
 * An unresolvable reference surfaces the named resolver error instead of
 * quietly yielding an empty pool.
 */
export function resolveResearchPoolEnv(
  env: Readonly<Record<string, string | undefined>> = process.env,
  secretsVault?: string,
): ResearchPoolEnv {
  return {
    braveApiKey: researchKey(env[BRAVE_API_KEY_ENV], secretsVault),
    tavilyApiKey: researchKey(env[TAVILY_API_KEY_ENV], secretsVault),
  };
}

/** One provider key: trimmed and non-empty, with references resolved. */
function researchKey(raw: string | undefined, secretsVault: string | undefined): string | null {
  const trimmed = raw?.trim();
  if (trimmed === undefined || trimmed.length === 0) return null;
  if (secretsVault !== undefined && isSecretReferenceValue(trimmed)) {
    return resolveNamedSecret(secretsVault, trimmed);
  }
  return trimmed;
}

export interface BuildResearchPoolOptions {
  /** Transport seam; defaults to the real fetch transport for production use. */
  readonly transport?: ExternalFetchTransport;
  /** Shared response cache keyed by normalized request; one is created if absent. */
  readonly cache?: ResponseCache;
}

/** The set of enabled providers, plus an explicit empty-state signal. */
export interface ResearchPool {
  readonly providers: readonly ResearchProvider[];
  /** Names of enabled providers, in wiring order. */
  readonly enabledNames: readonly string[];
  /** True when no provider key was configured. */
  isEmpty(): boolean;
}

/** One provider result with its origin provider name. */
export interface PooledResult extends ProviderSearchResult {
  readonly provider: string;
}

/** A typed provider failure carried in a pool report (never invented content). */
export interface ProviderError {
  readonly provider: string;
  readonly kind: ExternalFetchError["kind"];
  readonly message: string;
}

/** The outcome of running the pool for one query: results plus typed errors. */
export interface ResearchPoolReport {
  readonly results: readonly PooledResult[];
  readonly errors: readonly ProviderError[];
}

/**
 * Build the provider pool from resolved keys. Each provider joins only when its
 * key is present; with no keys the pool is empty and {@link ResearchPool.isEmpty}
 * returns true.
 */
export function buildResearchPool(
  env: ResearchPoolEnv,
  opts: BuildResearchPoolOptions = {},
): ResearchPool {
  const transport = opts.transport ?? createFetchTransport();
  const cache = opts.cache ?? createMemoryResponseCache();
  const providers: ResearchProvider[] = [];

  if (env.braveApiKey !== null) {
    const config: KeyedFetchConfig = { apiKey: env.braveApiKey, transport, cache };
    providers.push(createBraveProvider(config));
  }
  if (env.tavilyApiKey !== null) {
    const config: KeyedFetchConfig = { apiKey: env.tavilyApiKey, transport, cache };
    providers.push(createTavilyProvider(config));
  }

  const enabledNames = providers.map((provider) => provider.name);
  return {
    providers: Object.freeze(providers),
    enabledNames: Object.freeze(enabledNames),
    isEmpty: () => providers.length === 0,
  };
}

/**
 * Run every pooled provider for a query, aggregating results and typed errors.
 * A provider that throws an {@link ExternalFetchError} contributes an error
 * entry, not content; any other throw is rethrown (it is a real defect).
 */
export async function runResearchPool(
  pool: ResearchPool,
  query: string,
): Promise<ResearchPoolReport> {
  const settled = await Promise.all(
    pool.providers.map(
      async (provider): Promise<{ results: PooledResult[]; error: ProviderError | null }> => {
        try {
          const results = await provider.search(query);
          return {
            results: results.map((result) => ({ ...result, provider: provider.name })),
            error: null,
          };
        } catch (err) {
          if (err instanceof ExternalFetchError) {
            return {
              results: [],
              error: { provider: provider.name, kind: err.kind, message: err.message },
            };
          }
          throw err;
        }
      },
    ),
  );

  const results: PooledResult[] = [];
  const errors: ProviderError[] = [];
  for (const outcome of settled) {
    results.push(...outcome.results);
    if (outcome.error !== null) errors.push(outcome.error);
  }
  return { results: Object.freeze(results), errors: Object.freeze(errors) };
}