Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
13.2 KB
Raw Blame
/**
 * Subject diarization (t_28ba3fc4): a deterministic, read-only profile
 * assembler for one registry entity.
 *
 * Given an entity, it collects the subject's document set from the
 * entity registry and the ingested source pages, then computes a
 * stated-vs-evidenced section: STATED claims come from the existing
 * atomic-fact machinery run over the entity's own registered body;
 * EVIDENCED signals are the frequency and recency with which the
 * subject actually appears across ingested sources (behavioral signals,
 * pure counts, no language analysis of the claim text). Each line
 * carries the shared evidence-identity type from deep-synthesis
 * (t_40fa4e8d) and is gated on it, so an identity-less line is a
 * visible loss rather than a silent drop.
 *
 * The module never calls a model. It emits a structured profile note
 * skeleton plus exactly one needs-llm-step envelope describing the
 * prose the caller must generate to finish the profile; the summary
 * prose stays with the calling agent, never inside core.
 */

import { posix, relative } from "node:path";

import { sha256Hex } from "../integrity/digest.ts";
import { decomposeAtomicFacts, type AtomicEntityLike } from "./atomic-facts.ts";
import { hasEvidenceIdentity, type EvidenceIdentity } from "./deep-synthesis.ts";
import { resolveEntityLexicon } from "./entities/lexicon.ts";
import { getEntity } from "./entities/registry.ts";
import type { EntityRef } from "./entities/types.ts";
import { getIngestedSource, listIngestedSources } from "./ingest/sources-registry.ts";
import { buildNeedsLlmStep, type NeedsLlmStep } from "./llm-step.ts";
import {
  buildLinkCandidateManifest,
  linkCandidateSchemaHint,
  type LinkCandidateManifest,
} from "./notes/link-candidates.ts";
import { ownerScopeView } from "./owner-scope-view.ts";

/** Vault-relative directory a fleshed profile note lands in. */
export const PROFILE_DIR_REL = "Brain/profiles";

/** Evidence-identity kind labels this module attributes. */
const EVIDENCE_KIND_ENTITY = "entity";
const EVIDENCE_KIND_CLAIM = "claim";
const EVIDENCE_KIND_SOURCE_PAGE = "source_page";

/** The needs-llm-step step name for the deferred profile prose. */
const PROFILE_PROSE_STEP = "profile-prose";

/** Marker the skeleton carries where the deferred prose must land. */
const PROSE_MARKER = "<!-- o2b:needs-llm-step profile-prose -->";

/** A diarization request failed because the subject does not exist. */
export class DiarizationError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "DiarizationError";
  }
}

/**
 * One line of the stated-vs-evidenced section:
 *   - `stated_corroborated` - a stated claim the subject is also evidenced for.
 *   - `stated_unevidenced`  - a stated claim with zero evidence frequency.
 *   - `evidenced_unstated`  - the subject appears in a source but states nothing.
 */
export type DiarizationGapKind =
  | "stated_corroborated"
  | "stated_unevidenced"
  | "evidenced_unstated";

export interface DiarizationGapLine {
  readonly kind: DiarizationGapKind;
  /** The stated claim text, or a structural descriptor of the source. */
  readonly statement: string;
  /** Shared S1 identity of the artifact grounding this line. */
  readonly evidence: EvidenceIdentity;
  /** Number of ingested source pages the subject appears in. */
  readonly evidenceFrequency: number;
  /** Most recent source mention timestamp, or null when unevidenced. */
  readonly lastEvidencedAt: string | null;
}

/**
 * The single deferred generation step: the shared envelope spine plus the
 * one field a profile needs and the spine will not carry - the wikilink
 * targets the prose may cite. It stays plain data otherwise: diarization
 * is read-only and opens no durable session.
 */
export interface DiarizationLlmStep extends NeedsLlmStep {
  readonly link_candidates: LinkCandidateManifest;
}

export interface DiarizationReport {
  readonly entityId: string;
  readonly entityName: string;
  readonly category: string;
  readonly generatedAt: string;
  /** Identities of every artifact considered: the entity plus sources. */
  readonly documentSet: ReadonlyArray<EvidenceIdentity>;
  readonly statedVsEvidenced: ReadonlyArray<DiarizationGapLine>;
  /** Lines dropped for lacking an evidence identity (visible loss). */
  readonly excludedLineCount: number;
  readonly skeleton: string;
  readonly llmStep: DiarizationLlmStep;
}

export interface DiarizationOptions {
  readonly now: Date;
  /**
   * Enforced owner scope for this caller, or null when none applies. It
   * filters the envelope's link candidates: the basenames of the
   * preference, retired, inbox and entity trees are exactly the ids the
   * ownership rule governs, so an unscoped manifest would offer another
   * owner's private artifact as a citation target. Required rather than
   * optional - a surface that forgot it would leak silently, which is
   * the one failure mode this option exists for.
   */
  readonly ownerScope: string | null;
  /**
   * May the caller read the page at this vault-relative path at its
   * reach? ANDed into the link-candidate filter beside the owner scope, so
   * a page withheld from the caller is not offered as a citation target
   * nor counted in the manifest's total, exactly as an absent page. A
   * withheld subject page answers as an unknown entity, and a withheld
   * source page is not evidence: it is neither counted nor named. The
   * MCP handler binds it to `readableAtContextReach(ctx)`; absent (the
   * CLI) filters nothing beyond the owner scope.
   */
  readonly readable?: (rel: string) => boolean;
}

function toPosixRel(vault: string, abs: string): string {
  return relative(vault, abs).split(/[\\/]/).join(posix.sep);
}

/** One source page and the anchoring metadata diarization needs. */
interface EvidencedSource {
  readonly identity: EvidenceIdentity;
  readonly at: string | null;
}

/**
 * Assemble a subject profile for one registry entity. Deterministic and
 * read-only. Throws {@link DiarizationError} when the entity is unknown.
 */
export function diarize(
  vault: string,
  ref: EntityRef,
  opts: DiarizationOptions,
): DiarizationReport {
  const entity = getEntity(vault, ref);
  if (entity === null) {
    throw new DiarizationError(`unknown entity: ${ref.query}`);
  }

  const entityLike: AtomicEntityLike = {
    id: entity.id,
    name: entity.name,
    aliases: entity.aliases,
    status: entity.status,
  };
  const entityRel = toPosixRel(vault, entity.path);
  // A subject page the caller may not read at its reach is an unknown
  // entity, exactly as an absent one.
  if (opts.readable !== undefined && !opts.readable(entityRel)) {
    throw new DiarizationError(`unknown entity: ${ref.query}`);
  }
  const entityIdentity: EvidenceIdentity = Object.freeze({
    path: entityRel,
    kind: EVIDENCE_KIND_ENTITY,
    contentHash: sha256Hex(entity.body),
  });

  // The same store-learned lexicon the creation gate resolves (fail-open
  // null), threaded into the shared anchoring kernel so the stated-claim
  // and evidence verdicts skip exactly the junk forms creation refuses
  // (recall-quality M6). No artifact, and every verdict is byte-identical
  // to the pre-lexicon behavior.
  const lexicon = resolveEntityLexicon(vault);

  // STATED claims: atomic assertions decomposed from the subject's own
  // registered body that anchor the subject. No claim-text matching
  // against evidence - that would be language analysis.
  const statedClaims: string[] = [];
  for (const assertion of decomposeAtomicFacts(entity.body, {
    entities: [entityLike],
    lexicon,
  })) {
    if (assertion.entities.includes(entity.id)) statedClaims.push(assertion.text);
  }

  // EVIDENCED signals: ingested source pages the subject appears in,
  // detected through the same anchoring machinery. Frequency and recency
  // are the behavioral signals; both are pure counts over structure.
  const evidencedSources: EvidencedSource[] = [];
  for (const listed of listIngestedSources(vault)) {
    // A source page the caller may not read at its reach is neither
    // evidence, counted nor named, exactly as an absent one.
    if (opts.readable !== undefined && !opts.readable(listed.path)) continue;
    const detail = getIngestedSource(vault, listed.path);
    if (detail === null) continue;
    const anchored = decomposeAtomicFacts(detail.body, { entities: [entityLike], lexicon }).some(
      (a) => a.entities.includes(entity.id),
    );
    if (!anchored) continue;
    evidencedSources.push({
      identity: Object.freeze({
        path: listed.path,
        kind: EVIDENCE_KIND_SOURCE_PAGE,
        contentHash: sha256Hex(detail.body),
      }),
      at: detail.updatedAt ?? detail.createdAt,
    });
  }

  const evidenceFrequency = evidencedSources.length;
  const lastEvidencedAt = evidencedSources.reduce<string | null>((latest, source) => {
    if (source.at === null) return latest;
    if (latest === null || source.at > latest) return source.at;
    return latest;
  }, null);

  // Build the stated-vs-evidenced lines, gating each on evidence
  // identity so an identity-less line is reported as a visible loss.
  const lines: DiarizationGapLine[] = [];
  let excludedLineCount = 0;
  const push = (line: DiarizationGapLine): void => {
    // Defensive gate: every line constructed below carries a non-empty path,
    // kind, and content hash, so this check is currently unreachable. It is
    // kept deliberately so that if the evidence construction ever changes, an
    // identity-less line is reported as a visible loss rather than emitted.
    if (hasEvidenceIdentity(line.evidence)) lines.push(line);
    else excludedLineCount += 1;
  };

  for (const statement of statedClaims) {
    const evidence: EvidenceIdentity = Object.freeze({
      path: entityRel,
      kind: EVIDENCE_KIND_CLAIM,
      contentHash: sha256Hex(statement),
    });
    push({
      kind: evidenceFrequency > 0 ? "stated_corroborated" : "stated_unevidenced",
      statement,
      evidence,
      evidenceFrequency,
      lastEvidencedAt,
    });
  }

  // When the subject states nothing but the sources evidence it, the gap
  // is the evidence itself: one line per corroborating source page.
  if (statedClaims.length === 0) {
    for (const source of evidencedSources) {
      push({
        kind: "evidenced_unstated",
        statement: `subject appears in ${source.identity.path}`,
        evidence: source.identity,
        evidenceFrequency,
        lastEvidencedAt,
      });
    }
  }

  const documentSet: EvidenceIdentity[] = [
    entityIdentity,
    ...evidencedSources.map((s) => s.identity),
  ];

  const targetPath = posix.join(PROFILE_DIR_REL, `${entity.id}.md`);
  const generatedAt = opts.now.toISOString();
  const skeleton = renderSkeleton({
    entity: { id: entity.id, name: entity.name, category: entity.category },
    generatedAt,
    lines,
    documentSet,
  });
  // The subject and its aliases are the query: a profile cites the notes
  // that are about its subject, and those are the ones an over-bound
  // vault has to keep when the manifest truncates.
  const ownerView = ownerScopeView(vault, opts.ownerScope);
  const linkCandidates = buildLinkCandidateManifest(vault, {
    query: [entity.name, ...entity.aliases].join(" "),
    visible: (rel) => ownerView.visible(rel) && (opts.readable?.(rel) ?? true),
  });
  const llmStep: DiarizationLlmStep = buildNeedsLlmStep({
    step: PROFILE_PROSE_STEP,
    prompt:
      `Write the summary prose for the profile of ${entity.name} (${entity.id}). ` +
      "Ground every statement in the stated-vs-evidenced section and the document set below; " +
      `replace the ${PROSE_MARKER} marker with the prose and submit the full note.`,
    schema_hints: [
      "frontmatter: preserve the skeleton block verbatim",
      "body: replace only the prose marker; keep the structured sections intact",
      linkCandidateSchemaHint(linkCandidates),
    ],
    target_path: targetPath,
    link_candidates: linkCandidates,
  });

  return Object.freeze({
    entityId: entity.id,
    entityName: entity.name,
    category: entity.category,
    generatedAt,
    documentSet: Object.freeze(documentSet),
    statedVsEvidenced: Object.freeze(lines),
    excludedLineCount,
    skeleton,
    llmStep,
  });
}

function renderSkeleton(params: {
  readonly entity: { id: string; name: string; category: string };
  readonly generatedAt: string;
  readonly lines: ReadonlyArray<DiarizationGapLine>;
  readonly documentSet: ReadonlyArray<EvidenceIdentity>;
}): string {
  const { entity, generatedAt, lines, documentSet } = params;
  const out: string[] = [
    "---",
    "kind: brain-profile",
    `entity_id: ${entity.id}`,
    `category: ${entity.category}`,
    `generated_at: ${generatedAt}`,
    "---",
    "",
    `# Profile: ${entity.name}`,
    "",
    "## Summary",
    "",
    PROSE_MARKER,
    "",
    "## Stated vs evidenced",
    "",
  ];
  if (lines.length === 0) {
    out.push("(no stated claims and no evidenced signals)");
  } else {
    for (const line of lines) {
      const recency = line.lastEvidencedAt ?? "none";
      out.push(
        `- [${line.kind}] ${line.statement} ` +
          `(evidence: ${line.evidence.path}, frequency: ${line.evidenceFrequency}, ` +
          `last_evidenced: ${recency})`,
      );
    }
  }
  out.push("", "## Document set", "");
  for (const doc of documentSet) out.push(`- ${doc.kind}: ${doc.path}`);
  out.push("");
  return out.join("\n");
}