Self-hosted MCP server + dashboard giving Claude Code, Cursor & Hermes agents shared long-term memory over a git-backed markdown vault. Agent-native, Obsidian-compatible second brain — markdown RAG, no vector DB.
TypeScript agent-memoryai-agentsanthropicclaude-codecursorgitknowledge-baseknowledge-graphllmmarkdownmcpmcp-servermodel-context-protocolnote-takingobsidianpkmragsecond-brainself-hostedzettelkasten
feat(validity): query-time expiry overlay + explainable rejection in search
Ronalds Palaciseaa9a32f750a
diff --git a/app/api/search/route.ts b/app/api/search/route.tsindex bb0ff83..716aba9 100644--- a/app/api/search/route.ts+++ b/app/api/search/route.ts@@ -5,7 +5,12 @@ export const dynamic = "force-dynamic";export async function GET(req: Request) {const params = new URL(req.url).searchParams;const q = params.get("q") ?? "";- // Humans browsing the dashboard should be able to find archived notes; they still rank- // far below live ones. Agents (MCP) get them excluded by default.- return Response.json({ results: searchNotes(q, { includeArchive: params.get("archive") !== "false" }) });+ // Humans browsing the dashboard should find archived AND retired notes (they still rank far+ // below live ones). Agents (MCP) get both excluded by default. Insulation point: map the new+ // { hits, excluded } shape back to a `results` array so the dashboard's contract is unchanged.+ const r = searchNotes(q, {+ includeArchive: params.get("archive") !== "false",+ includeInvalid: params.get("invalid") !== "false",+ });+ return Response.json({ results: r.hits, excluded: r.excluded });}diff --git a/lib/mcp/tools.ts b/lib/mcp/tools.tsindex ccabb00..1385734 100644--- a/lib/mcp/tools.ts+++ b/lib/mcp/tools.ts@@ -91,25 +91,27 @@ export const TOOLS: Tool[] = [{name: "brain_search",description:- "Keyword search across the vault. Returns ranked hits with `path`, `title`, `folder`, `status`, `snippet`, and `authority`.\n\n" +- "IMPORTANT — ranking is by keyword relevance, NOT by truth. A superseded document repeats the query words just as often as the live one, so it can outrank it. Every hit carries an `authority` field: `authoritative` (source of truth — prefer it), `current`, `provisional` (draft/proposed — never quote as settled), `superseded`, `archived`. Rank order is a suggestion; `authority` is the signal.\n\n" +- "Before quoting a price, guarantee, contract term, or any other single-valued fact, open the `authoritative` note and read it. If two notes disagree on such a fact, that is a defect in the vault — report it rather than averaging them.\n\n" +- "Archived notes are excluded unless `includeArchive` is true.",+ "Keyword search across the vault. Returns `{ hits, excluded }`. Each hit has `path`, `title`, `folder`, `status`, `snippet`, and `authority`.\n\n" ++ "IMPORTANT — ranking is by keyword relevance, NOT by truth. A superseded document repeats the query words just as often as the live one, so it can outrank it. Every hit carries an `authority`: `authoritative` (source of truth — prefer it), `current`, `provisional` (draft/proposed — never quote as settled), `superseded`, `archived`. Rank order is a suggestion; `authority` is the signal.\n\n" ++ "`excluded` is the explainable-rejection list: notes that matched the query but were withheld because they are archived, superseded, or **expired** (past their `valid_until`), each with a `reason` (e.g. \"superseded by price-live\", \"expired 2026-06-01\"). **When you deliberately ignore a stale note, cite its `excluded` entry** — say what you skipped and why, instead of quoting it. If you need a withheld note, re-search with `includeInvalid: true` (superseded/expired) or `includeArchive: true`.\n\n" ++ "Before quoting a price, guarantee, contract term, or any other single-valued fact, open the `authoritative` note. If two live notes disagree on such a fact, that is a defect in the vault — report it rather than averaging them.",inputSchema: {type: "object",properties: {query: s("search query — a few keywords beat a full sentence"),- limit: { type: "number", description: "max results (default 20)" },+ limit: { type: "number", description: "max hits (default 20)" },folder: s("restrict to one top-level folder, e.g. 'decisions'"),- includeArchive: { type: "boolean", description: "include historical notes in archive folders (default false)" },+ includeArchive: { type: "boolean", description: "return archived notes as hits (default false → they go to `excluded`)" },+ includeInvalid: { type: "boolean", description: "return superseded/expired notes as hits, still demoted (default false → `excluded`)" },},required: ["query"],},- handler: ({ query, limit, folder, includeArchive }) =>+ handler: ({ query, limit, folder, includeArchive, includeInvalid }) =>searchNotes(String(query ?? ""), {limit: typeof limit === "number" ? limit : 20,folder: folder ? String(folder) : undefined,includeArchive: includeArchive === true,+ includeInvalid: includeInvalid === true,}),},{diff --git a/lib/vault/store.ts b/lib/vault/store.tsindex 4973ded..e374667 100644--- a/lib/vault/store.ts+++ b/lib/vault/store.ts@@ -6,7 +6,7 @@ import { VAULT_IGNORE } from "@/lib/config";import { activeVaultDir } from "@/lib/repos";import { scanVault } from "./scan";import { parseNote, stemOf } from "./parse";-import { authorityOf, authorityRules, isArchivedPath, weightOf, type Authority } from "./authority";+import { authorityOf, authorityRules, isArchivedPath, overlayValidity, weightOf, type Authority } from "./authority";import type { Graph, GraphEdge, GraphNode, Note, NoteMeta, TreeNode } from "./types";interface IndexDoc {@@ -20,6 +20,10 @@ interface IndexDoc {/** stored, not indexed — used for filtering + authority-weighted ranking */status: string;authority: Authority;+ /** epoch ms deadline (or null). Read at query time so expiry is live without re-indexing. */+ validUntil: number | null;+ /** stem of the superseding note (or null). Marks this doc retired at query time. */+ supersededBy: string | null;mtimeMs: number;}@@ -44,7 +48,7 @@ let watchedDir = "";function newIndex(): MiniSearch<IndexDoc> {return new MiniSearch<IndexDoc>({fields: ["title", "aliases", "tags", "body", "folder", "type"],- storeFields: ["title", "folder", "type", "status", "authority", "mtimeMs"],+ storeFields: ["title", "folder", "type", "status", "authority", "validUntil", "supersededBy", "mtimeMs"],searchOptions: {boost: { title: 3, aliases: 3, tags: 2 },prefix: true,@@ -70,6 +74,8 @@ function toDoc(meta: NoteMeta, body: string): IndexDoc {type: meta.type ?? "",status: meta.status ?? "",authority: authorityOf(meta),+ validUntil: meta.validUntil ?? null,+ supersededBy: meta.supersededBy ?? null,mtimeMs: meta.mtimeMs,};}@@ -300,6 +306,24 @@ export interface SearchHit {authority: Authority;score: number;snippet?: string;+ /** Present only when a retired note is returned anyway (includeInvalid) — why it's stale. */+ reason?: string;+}++/** A note that matched the query but was withheld — the "explainable rejection". */+export interface SearchExclusion {+ path: string;+ title: string;+ folder: string;+ authority: Authority;+ /** Why it was withheld, e.g. "superseded by price-live" or "expired 2026-06-01". */+ reason: string;+}++export interface SearchResult {+ hits: SearchHit[];+ /** Matches held back as archived/superseded/expired, so an agent can say what it ignored and why. */+ excluded: SearchExclusion[];}export interface SearchOpts {@@ -308,6 +332,8 @@ export interface SearchOpts {folder?: string;/** Include notes in archive folders (still ranked far below live ones). Default false. */includeArchive?: boolean;+ /** Include superseded/expired notes as hits (still demoted). Default false — they go to `excluded`. */+ includeInvalid?: boolean;/** Attach ~200 chars of matching context per hit. Default true. */snippets?: boolean;}@@ -333,55 +359,89 @@ function snippetFor(dir: string, rel: string, terms: string[]): string | undefin}}+interface StoredDoc {+ id: string;+ title: string;+ folder: string;+ type?: string;+ status?: string;+ authority: Authority;+ validUntil?: number | null;+ supersededBy?: string | null;+ score: number;+}+/**- * Keyword search, then reweighted by authority.+ * Keyword search, reweighted by authority, then partitioned by validity.*- * The ranking is deliberately two-stage: MiniSearch says how well a note *matches*,- * authorityOf() says how far it can be *trusted*. Without the second stage a superseded- * price list outranks the locked one, because it repeats the query words just as often.+ * Three stages: MiniSearch says how well a note *matches*; authority says how far it can be+ * *trusted*; and the validity overlay (superseded / expired) decides whether it's still true at+ * all. A retired note is withheld from `hits` and reported in `excluded` with a reason — so an+ * agent can state what it ignored and why, instead of quietly quoting a dead fact.*/-export function searchNotes(q: string, opts: SearchOpts = {}): SearchHit[] {+export function searchNotes(q: string, opts: SearchOpts = {}): SearchResult {const s = ensure();- if (!q.trim()) return [];- const { limit = 20, folder, includeArchive = false, snippets = true } = opts;+ if (!q.trim()) return { hits: [], excluded: [] };+ const { limit = 20, folder, includeArchive = false, includeInvalid = false, snippets = true } = opts;+ const now = Date.now();+ const EXCLUDE_CAP = 10;const raw = s.search.search(q, {- boostDocument: (_id, _term, stored) => weightOf((stored?.authority as Authority) ?? "current"),+ // Effective weight: overlay validity onto the stored authority so an expired/superseded note+ // is demoted at query time — no re-index needed (the index-time authority is frozen).+ boostDocument: (_id, _term, stored) => {+ const d = stored as { authority?: Authority; validUntil?: number | null; supersededBy?: string | null } | undefined;+ const eff = overlayValidity((d?.authority as Authority) ?? "current", d?.validUntil, d?.supersededBy, now);+ return weightOf(eff.authority);+ },+ // Folder is the only hard filter here; archived/superseded/expired flow through so they can be+ // partitioned into `excluded` with a reason rather than silently dropped.filter: (r) => {- const stored = r as unknown as { folder: string; authority: Authority };- if (!includeArchive && stored.authority === "archived") return false;- if (folder && stored.folder !== folder) return false;- return true;+ const stored = r as unknown as { folder: string };+ return !folder || stored.folder === folder;},});- const terms = q- .toLowerCase()- .split(/\s+/)- .map((t) => t.replace(/[^a-z0-9-]/g, ""))- .filter((t) => t.length >= 3);-- return raw.slice(0, limit).map((r) => {- const hit = r as unknown as {- id: string;- title: string;- folder: string;- type?: string;- status?: string;- authority: Authority;- score: number;- };- return {- path: hit.id,- title: hit.title,- folder: hit.folder,- type: hit.type || undefined,- status: hit.status || undefined,- authority: hit.authority,- score: hit.score,- snippet: snippets ? snippetFor(s.dir, hit.id, terms) : undefined,- };- });+ const hits: SearchHit[] = [];+ const excluded: SearchExclusion[] = [];++ for (const r of raw) {+ if (hits.length >= limit && excluded.length >= EXCLUDE_CAP) break;+ const d = r as unknown as StoredDoc;+ const eff = overlayValidity(d.authority, d.validUntil, d.supersededBy, now);+ const isArchived = eff.authority === "archived";+ const withhold = isArchived ? !includeArchive : eff.retired && !includeInvalid;++ if (withhold) {+ if (excluded.length < EXCLUDE_CAP) {+ excluded.push({ path: d.id, title: d.title, folder: d.folder, authority: eff.authority, reason: eff.reason ?? "retired" });+ }+ continue;+ }+ if (hits.length < limit) {+ hits.push({+ path: d.id,+ title: d.title,+ folder: d.folder,+ type: d.type || undefined,+ status: d.status || undefined,+ authority: eff.authority,+ score: d.score,+ reason: eff.retired ? eff.reason : undefined,+ });+ }+ }++ if (snippets && hits.length > 0) {+ const terms = q+ .toLowerCase()+ .split(/\s+/)+ .map((t) => t.replace(/[^a-z0-9-]/g, ""))+ .filter((t) => t.length >= 3);+ for (const h of hits) h.snippet = snippetFor(s.dir, h.path, terms);+ }++ return { hits, excluded };}export function getBacklinks(relPath: string): NoteMeta[] {@@ -405,6 +465,9 @@ export function vaultConventions() {const types = new Set<string>();const malformed: Array<{ path: string; error: string }> = [];let archived = 0;+ let superseded = 0;+ let expired = 0;+ const now = Date.now();for (const n of s.notes.values()) {folders.add(n.folder);@@ -412,11 +475,15 @@ export function vaultConventions() {if (n.status) statuses.set(n.status, (statuses.get(n.status) ?? 0) + 1);if (isArchivedPath(n.path)) archived++;if (n.frontmatterError) malformed.push({ path: n.path, error: n.frontmatterError });+ if (n.supersededBy) superseded++;+ else if (n.validUntil != null && n.validUntil < now) expired++;}return {noteCount: s.notes.size,archivedCount: archived,+ supersededCount: superseded,+ expiredCount: expired,folders: [...folders].sort(),types: [...types].sort(),statusesInUse: [...statuses.entries()]