Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Open Knowledge Format (OKF) export / import round-trip
* (Brain Portability & Interop suite, Unit C).
*
* OKF is a portable, producer-agnostic *directory* bundle for knowledge
* interchange between wiki / second-brain systems. Where the bank bundle
* ({@link ./bundle.ts}) is a single JSON envelope tuned for OSB-to-OSB
* migration, an OKF bundle is a tree of plain markdown a foreign tool can
* read without knowing OSB internals:
*
* <bundle>/
* okf.json machine-readable manifest (schema, producer,
* per-page derived fields + preserved foreign
* provenance)
* index.md human-readable index, grouped by page class
* concepts/<slug>.md regular knowledge pages (the default class)
* queries/<slug>.md saved-question / research-report pages
* references/<slug>.md cited external-source pages
* log.md date-grouped change log (from Brain/log/)
*
* Page class is derived *structurally* from the frontmatter `kind:`
* field — never from natural-language heuristics — so the split is
* deterministic and language-agnostic. Each bundle page file carries the
* page's frontmatter and body **verbatim** plus the OKF v0.2 derived
* keys (`type`, `status`, `sources`, `generated` — recomputed on every
* export; `type` is the spec's one always-required per-concept key) and
* the `okf_raw` marker declaring the raw native keys riding beside
* them; the manifest carries the *derived* standard fields (kind,
* citations, status, sources, freshness) computed from current page
* state.
*
* Re-export honesty: when OSB imported a page from a *foreign* producer,
* the page keeps that producer's raw type value (`okf_type:`) and any
* producer-specific frontmatter (`x-*` keys) on disk. A later
* {@link buildOkfBundle} surfaces the raw foreign type as
* {@link OkfManifestPage.foreign_type} and the producer-specific keys as
* {@link OkfManifestPage.producer_meta} while still deriving the standard
* fields from the page's current state — foreign provenance is preserved,
* never overwritten or guessed.
*
* Import trust gradient: by default imported pages are *staged* under
* `OKF Review/` with `okf_review: pending` so nothing in the live vault is
* touched until an operator promotes them. `--trusted` writes each page
* directly to its recorded vault-relative path (a true round-trip).
*
* The bundle's self-declared `producer` is a LABEL, never a credential:
* any bundle can write `"producer": "open-second-brain"` into its
* manifest. It decides only provenance stamping. Every security decision
* - whether machinery frontmatter survives, whether the Brain page lanes
* are writable - keys on the operator's explicit `--trusted` instead.
*/
import { existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
import { dirname, join, posix, relative, resolve } from "node:path";
import { atomicWriteFileSync } from "../../fs-atomic.ts";
import {
EXCLUDED_DIRS,
extractWikilinks,
formatFrontmatter,
parseFrontmatterText,
} from "../../vault.ts";
import type { FrontmatterMap, FrontmatterValue } from "../../types.ts";
import {
extractFrontmatterRelations,
normalizeRelationTarget,
} from "../../graph/frontmatter-relations.ts";
import { listLogShardFiles } from "../log-jsonl.ts";
import {
BRAIN_ROOT_REL,
BRAIN_SOURCES_REL,
BRAIN_REPORTS_REL,
ensureInsideVault,
isInBrainLane,
isUnderBrainRoot,
} from "../paths.ts";
import { isoSecond } from "../time.ts";
import { vaultDisplayName } from "../templates.ts";
import { ORIGIN_CHANNEL_FIELD, isOriginChannelStamp } from "../../origin-channel.ts";
import { contentClassCreationStamp } from "../provenance/content-class.ts";
import { KNOWLEDGE_PACK_FIELD, KNOWLEDGE_PACK_SHA_FIELD } from "./pack-stamp.ts";
import { OPEN_SECOND_BRAIN_VERSION } from "../../version.ts";
import {
assertLaneWrite,
laneEntryClockMs,
laneEntryInPrivate,
lanePageExists,
lanePagesInStore,
listLaneTree,
readLaneFile,
writeLanePage,
} from "../../store/lane-pages.ts";
import { assertNoteVaultForWrite } from "../../vault-adapter/index.ts";
import { groupLogDays, WHOLE_LOG } from "../log-days.ts";
import { readStoreLogEventsReport } from "../../store/read/log-events.ts";
import { protectionStore } from "../../store/read/protection-store.ts";
import { renderLogDayHeader, renderLogEntryBlock } from "../log.ts";
import { pageVisibility, REMOTE_DENY_VISIBILITY_TOKEN } from "../../graph/visibility.ts";
export const OKF_SCHEMA_VERSION = "1";
export const OKF_PRODUCER = "open-second-brain";
/**
* The OKF spec version this export emits, declared as
* `okf_version: "0.2"` in the bundle-root `index.md` frontmatter (a MAY
* per the spec, and the only frontmatter an index file may carry). This
* is the INTERCHANGE version, not the manifest dialect: Open Second
* Brain's own `okf.json` keeps its `schema: "1"` field beside it.
*/
export const OKF_VERSION = "0.2";
/** The OKF v0.2 `generated.by` actor string for pages this system exports. */
const OKF_GENERATED_BY = `${OKF_PRODUCER}/${OPEN_SECOND_BRAIN_VERSION}`;
/**
* Bundle-relative filename of the machine-readable manifest. Named
* because three sites spelled it as a literal and a fourth - the export
* verb, which re-serialises this one file after redacting the manifest
* tree - would have made four.
*/
export const OKF_MANIFEST_FILENAME = "okf.json";
/**
* Bundle-relative filename of the human-readable index. Named for the
* same reason as {@link OKF_MANIFEST_FILENAME}: the export verb
* re-renders this one file from the REDACTED manifest after the guard
* runs, and a reserved name spelled as a literal in a second module is
* a name that can drift away from the writer that reserves it.
*/
export const OKF_INDEX_FILENAME = "index.md";
/** Vault-relative root the default (untrusted) import stages pages under. */
export const OKF_REVIEW_REL = "OKF Review";
/** OKF page classes and the bundle subdirectory each maps to. */
export type OkfPageClass = "concept" | "query" | "reference";
const CLASS_DIR: Readonly<Record<OkfPageClass, string>> = {
concept: "concepts",
query: "queries",
reference: "references",
};
/** `kind:` values that route a page to `queries/`. */
const QUERY_KINDS: ReadonlySet<string> = new Set(["query", "brain-report", "report"]);
/** `kind:` values that route a page to `references/`. */
const REFERENCE_KINDS: ReadonlySet<string> = new Set(["reference", "brain-source", "source"]);
/** Structural default `kind` for a page with no frontmatter `kind:`. */
const DEFAULT_KIND = "note";
/** Frontmatter key prefix marking producer-specific (foreign) fields. */
const PRODUCER_META_PREFIX = "x-";
/** Frontmatter key carrying a foreign producer's raw type value. */
const FOREIGN_TYPE_KEY = "okf_type";
/** Frontmatter key carrying the producer that authored an imported page. */
const PRODUCER_KEY = "okf_producer";
/** Frontmatter key stamping the import timestamp. */
const IMPORTED_AT_KEY = "okf_imported_at";
/** Frontmatter key marking a staged (un-promoted) review candidate. */
const REVIEW_KEY = "okf_review";
/**
* Frontmatter keys carrying a page's NATIVE `status` / `sources` /
* `generated` / `type` values beside the derived v0.2 block. The v0.2
* status enum is closed (`draft | stable | deprecated`) and the derived
* sources are re-translated from citations, so exporting the native
* values in place would silently rewrite them: a quarantined page would
* export as `stable` and un-quarantine on import, workflow statuses like
* `done` or `active` would be lost, and user-written source lists would
* be replaced. The same holds for `type`, the spec's one always-required
* per-concept key: the export emits it DERIVED (the preserved foreign
* type, else the structural kind), so a native `type` the operator wrote
* rides the raw namespace instead of being shadowed by the projection.
* The raw keys ride the same `okf_` producer namespace as
* {@link FOREIGN_TYPE_KEY} / {@link PRODUCER_KEY}; an own-producer
* import restores them and drops the derived block (see
* {@link restoreRawOkfKeys}) - trusted round-trip and review staging
* alike - while foreign bundles import as-is and shed the keys.
*/
const RAW_STATUS_KEY = "okf_status_raw";
const RAW_SOURCES_KEY = "okf_sources_raw";
const RAW_GENERATED_KEY = "okf_generated_raw";
const RAW_TYPE_KEY = "okf_type_raw";
/**
* The raw/native key pairs the producer namespace carries beside the
* derived v0.2 block, in one list: `raw` is the exported carrier,
* `native` the frontmatter field it preserves. The export's carry and
* strip, the foreign shed and the import restore all walk THIS, so the
* set cannot drift between the sites that must agree on it.
*/
const RAW_OKF_KEYS: ReadonlyArray<{ readonly raw: string; readonly native: string }> =
Object.freeze([
{ raw: RAW_STATUS_KEY, native: "status" },
{ raw: RAW_SOURCES_KEY, native: "sources" },
{ raw: RAW_GENERATED_KEY, native: "generated" },
{ raw: RAW_TYPE_KEY, native: "type" },
]);
/**
* The per-page marker that says "this bundle page was written by a
* raw-carrying exporter", stamped on EVERY page of a fresh export
* beside the derived block - raw keys or not. It is what lets the
* import distinguish a page the current exporter wrote with no native
* status / sources / generated (absent raw keys mean "the field never
* had a value": strip the derived block, the page lands plain) from a
* LEGACY bundle's page (no marker: the frontmatter already is the
* native state, leave it as-is). Without the marker, a fresh-export
* plain page took the legacy early-return and landed WITH the derived
* block - `status: stable` materializing where none existed - and the
* next export carried the materialized status as a raw native,
* permanent.
*
* The marker IS its coverage: the exporter stamps the list of native
* names it manages, and the restore drops a derived value only for a
* pair the marker declares (see {@link rawCarryDeclaredNatives}). A
* bare `true` is what the pre-list exporters stamped, so it reads as
* the legacy shape and covers only the old status/sources/generated
* set - never `type`, whose pair did not exist yet when those exports
* carried a native type verbatim beside the marker.
*/
const RAW_CARRY_KEY = "okf_raw";
/**
* The native fields a bare {@link RAW_CARRY_KEY} `true` covers: the
* three the exporters that stamped the bare shape managed. The `type`
* pair did not exist yet in those exports - a native `type` rode the
* page verbatim beside the marker - so `true` must never name it, or
* the restore would read a legacy page's absent `okf_type_raw` as "the
* field never had a value" and delete the real type.
*/
const RAW_CARRY_TRUE_NATIVES: ReadonlySet<string> = new Set(["status", "sources", "generated"]);
const FRESHNESS_FIELDS = ["updated_at", "updated"] as const;
// ----- Export ---------------------------------------------------------------
/**
* The OKF v0.2 concept lifecycle enum - the spec's CLOSED status
* vocabulary (`draft | stable | deprecated`; an absent status means
* stable). Open Second Brain's own page statuses land on it through the
* conservative map in {@link okfStatusOf}.
*/
export type OkfStatus = "draft" | "stable" | "deprecated";
/** One OKF v0.2 per-page source (a frontmatter `sources:` entry). */
export interface OkfSource {
/**
* REQUIRED per the spec: a path or URL identifying the source. A
* citation that resolves to neither is dropped rather than guessed.
*/
readonly resource: string;
/**
* Optional stable key. Carried when the citation resolves to a bundle
* page, whose id is that stable key.
*/
readonly id?: string;
}
/** A page collected for OKF export, with derived + preserved metadata. */
export interface OkfPage {
/** Stable id (the file basename stem). */
readonly id: string;
/** Original vault-relative POSIX path. */
readonly path: string;
/** Derived OKF class (`concept` | `query` | `reference`). */
readonly cls: OkfPageClass;
/** Path within the bundle (`concepts/<slug>.md`, …). */
readonly bundle_path: string;
/** Derived standard kind (current frontmatter `kind:` else `note`). */
readonly kind: string;
/** Sorted-unique body wikilink + typed-relation targets. */
readonly citations: ReadonlyArray<string>;
/** Mapped OKF v0.2 lifecycle (`draft` | `stable` | `deprecated`). */
readonly status: OkfStatus;
/** Frontmatter `aliases`, else empty. */
readonly aliases: ReadonlyArray<string>;
/** Frontmatter `updated_at`/`updated`, else file mtime (ISO second). */
readonly freshness: string | null;
/** Preserved raw foreign type (`okf_type:`), else null. */
readonly foreign_type: string | null;
/** Preserved producer-specific frontmatter (`x-*` keys). */
readonly producer_meta: Readonly<Record<string, string>>;
/** Verbatim frontmatter of the page. */
readonly frontmatter: FrontmatterMap;
/** Verbatim markdown body of the page. */
readonly body: string;
}
export interface OkfManifestPage {
readonly id: string;
readonly path: string;
readonly class: OkfPageClass;
readonly bundle_path: string;
readonly kind: string;
readonly citations: ReadonlyArray<string>;
/** Mapped OKF v0.2 lifecycle (`draft` | `stable` | `deprecated`). */
readonly status: OkfStatus;
/** OKF v0.2 per-page sources translated from `citations`. */
readonly sources: ReadonlyArray<OkfSource>;
readonly aliases: ReadonlyArray<string>;
readonly freshness: string | null;
readonly foreign_type: string | null;
readonly producer_meta: Readonly<Record<string, string>>;
}
export interface OkfManifest {
readonly schema: string;
readonly producer: string;
readonly generated_at: string;
readonly vault_basename: string;
readonly pages: ReadonlyArray<OkfManifestPage>;
/**
* Count of DAYS folded into `log.md`, across every markdown shard of
* `Brain/log/` - `<date>.md` and `<date>.<deviceId>.md` alike. Two
* shards of one day are one day here, not two.
*/
readonly log_days: number;
/**
* Log events the export left out of `log.md` for being private, counted
* before any content-class filtering. Zero on the legacy layout and for
* a machine that holds no `private` partition.
*/
readonly log_private_omitted: number;
}
/** A single file in the in-memory bundle write-plan. */
export interface OkfBundleFile {
/** Bundle-relative POSIX path. */
readonly path: string;
readonly contents: string;
}
export interface OkfBundle {
readonly manifest: OkfManifest;
/** Every file to write, including `okf.json`, sorted by path. */
readonly files: ReadonlyArray<OkfBundleFile>;
}
function stringField(meta: FrontmatterMap, key: string): string | null {
const value: FrontmatterValue | undefined = meta[key];
return typeof value === "string" && value.length > 0 ? value : null;
}
function aliasField(meta: FrontmatterMap): ReadonlyArray<string> {
const value: FrontmatterValue | undefined = meta["aliases"];
if (!Array.isArray(value)) return [];
return value.filter((item): item is string => typeof item === "string");
}
function classifyKind(kind: string): OkfPageClass {
if (REFERENCE_KINDS.has(kind)) return "reference";
if (QUERY_KINDS.has(kind)) return "query";
return "concept";
}
function collectCitations(meta: FrontmatterMap, body: string): string[] {
const targets: string[] = [];
for (const raw of extractWikilinks(body)) {
const norm = normalizeRelationTarget(raw);
if (norm !== null) targets.push(norm);
}
for (const edge of extractFrontmatterRelations(meta)) targets.push(edge.target);
return [...new Set(targets.filter((v) => v.length > 0))].toSorted();
}
/**
* Frontmatter `status:` values that mean "no longer current" and map to
* the OKF `deprecated` value. Deliberately NARROW: the OKF enum has no
* room for nuance, and a conservative producer prefers understating.
*/
const OKF_DEPRECATED_STATUSES: ReadonlySet<string> = new Set([
"deprecated",
"retired",
"superseded",
"archived",
"terminal",
]);
/**
* Map a page onto the OKF v0.2 concept lifecycle, which admits ONLY
* `draft | stable | deprecated`.
*
* PINNED SOURCE FIELD: the page's frontmatter `status:` value - the
* operator-declared page state - and nothing else. The system-managed
* `_lifecycle` axis is not consulted: its `verified`/`disputed` values
* have no OKF counterpart, and a conservative map must not guess one.
* Completion states (`done` / `closed` / `resolved`) map to `stable` -
* completion is not deprecation, never the reverse - and a status the
* vocabulary does not name (including no status at all) lands on
* `stable`, which is also the OKF default for an absent status.
*/
function okfStatusOf(meta: FrontmatterMap): OkfStatus {
const raw = stringField(meta, "status");
if (raw === null) return "stable";
const status = raw.trim().toLowerCase();
if (status === "draft") return "draft";
if (OKF_DEPRECATED_STATUSES.has(status)) return "deprecated";
return "stable";
}
function freshnessOf(vault: string, meta: FrontmatterMap, absPath: string): string | null {
for (const field of FRESHNESS_FIELDS) {
const stamp = stringField(meta, field);
if (stamp !== null) return stamp;
}
// A file's mtime, or a lane record's clock on a store-only machine.
const ms = laneEntryClockMs(vault, absPath);
return ms === null ? null : isoSecond(new Date(ms));
}
function producerMeta(meta: FrontmatterMap): Record<string, string> {
const out: Record<string, string> = {};
for (const [key, value] of Object.entries(meta)) {
if (key.startsWith(PRODUCER_META_PREFIX)) out[key] = String(value);
}
return out;
}
/** POSIX vault-relative path for an absolute file under the vault. */
function vaultRel(vault: string, abs: string): string {
return relative(vault, abs).split(/[\\/]/).join(posix.sep);
}
/** Basename stem of a POSIX-or-native path (drops one trailing extension). */
function stem(p: string): string {
const base = p.split(/[\\/]/).pop() ?? p;
const dot = base.lastIndexOf(".");
return dot > 0 ? base.slice(0, dot) : base;
}
/**
* Walk the vault and collect every exportable page. Three sources feed
* the bundle: ordinary user pages (everything outside `Brain/`), the
* ingested source pages under `Brain/sources/` (references), and the
* research-report pages under `Brain/reports/` (queries). All other
* Brain machinery (preferences, inbox, entities, log, …) is excluded —
* the log feeds `log.md` separately, the rest is not knowledge content.
*/
export function collectOkfPages(vault: string): ReadonlyArray<OkfPage> {
const seenBundlePaths = new Set<string>();
const out: OkfPage[] = [];
const consider = (abs: string, forceClass?: OkfPageClass): void => {
let raw: string;
try {
raw = readLaneFile(vault, abs);
} catch {
return;
}
const [parsed, body] = parseFrontmatterText(raw);
const meta = privateStamped(vault, abs, parsed);
const kind = stringField(meta, "kind") ?? DEFAULT_KIND;
const cls = forceClass ?? classifyKind(kind);
const path = vaultRel(vault, abs);
const id = stem(path);
// De-duplicate within the bundle's class subdir; suffix on collision.
let slug = id;
let bundlePath = posix.join(CLASS_DIR[cls], `${slug}.md`);
for (let n = 2; seenBundlePaths.has(bundlePath); n++) {
slug = `${id}-${n}`;
bundlePath = posix.join(CLASS_DIR[cls], `${slug}.md`);
}
seenBundlePaths.add(bundlePath);
out.push({
id,
path,
cls,
bundle_path: bundlePath,
kind,
citations: collectCitations(meta, body),
status: okfStatusOf(meta),
aliases: aliasField(meta),
freshness: freshnessOf(vault, meta, abs),
foreign_type: stringField(meta, FOREIGN_TYPE_KEY),
producer_meta: producerMeta(meta),
frontmatter: meta,
body,
});
};
// 1. User pages (outside Brain). listing is sorted by path below.
for (const abs of listMarkdown(vault, [...EXCLUDED_DIRS, BRAIN_ROOT_REL])) consider(abs);
// 2. References: Brain/sources/*.md (lane records once switched).
for (const abs of laneMarkdown(vault, join(vault, BRAIN_SOURCES_REL))) consider(abs, "reference");
// 3. Queries: Brain/reports/*.md (lane records once switched).
for (const abs of laneMarkdown(vault, join(vault, BRAIN_REPORTS_REL))) consider(abs, "query");
out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
return out;
}
/**
* A page read from a lane entry of the `private` partition carries the
* remote-deny token in the bundle (review finding F333). Such an entry can
* be private by partition alone, with no `visibility:` of its own, and
* would otherwise land in another vault as an ordinary shared page. The
* token is added to the tokens the page already declares, never in place
* of them.
*/
function privateStamped(vault: string, abs: string, meta: FrontmatterMap): FrontmatterMap {
if (!laneEntryInPrivate(vault, abs)) return meta;
const tokens = pageVisibility(meta);
if (tokens.includes(REMOTE_DENY_VISIBILITY_TOKEN)) return meta;
return { ...meta, visibility: [...tokens, REMOTE_DENY_VISIBILITY_TOKEN] };
}
/** The `.md` pages under a `Brain/` lane folder, in both modes. */
function laneMarkdown(vault: string, root: string): string[] {
return lanePagesInStore(vault) ? listLaneTree(vault, root) : listMarkdown(root, []);
}
/** Recursively list `.md` files under `root`, skipping `skipDirs` names. */
function listMarkdown(root: string, skipDirs: ReadonlyArray<string>): string[] {
const skip = new Set(skipDirs);
const found: string[] = [];
const walk = (dir: string): void => {
let entries;
try {
entries = readdirSync(dir, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
if (entry.isDirectory()) {
if (skip.has(entry.name) || entry.name.startsWith(".")) continue;
walk(join(dir, entry.name));
continue;
}
if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
found.push(join(dir, entry.name));
}
}
};
walk(root);
found.sort();
return found;
}
/**
* Read and date-group the markdown side of `Brain/log/` into one change
* log.
*
* Days come from {@link listLogShardFiles}, the recogniser in the module
* that owns the log layout, rather than from a pattern of this module's
* own. The local pattern here matched `<date>.md` only, while every
* default install has written the per-device `<date>.<deviceId>.md` since
* sharding landed - so a vault with a full change log exported an empty
* `log.md` and a manifest reading `0 log day(s)`, with nothing to
* distinguish that from a vault that had never logged anything.
*
* Shards of one day fold into ONE dated section, ordered by shard id
* (`listLogShardFiles` sorts by name), so the bundle is byte-identical
* whatever order Syncthing delivered the shards in.
*
* ## The content-class filter
*
* With a {@link OkfLogFilter} the log is NOT whole: the shards hold
* agent-written material, and a default export that carried it would
* undo, page by page, the omission the class policy just performed. Two
* kinds of entry leave, and each drop is counted into the returned
* `droppedEntries` so the calling verb can name the count:
*
* 1. an entry whose `origin_channel` bullet carries the channel
* {@link contentClassCreationStamp} derives the reserved class from -
* an entry the MCP transport appended is agent-dictated material. A
* log entry carries no `content_class` of its own, so the server-
* derived channel it is stamped with is the one provable marker it
* can carry; a channel this build does not know marks nothing.
* 2. an entry whose `target` bullet names a page in the filter's omitted
* set - the entry would disclose the very name the split withheld.
*
* Kept entries are preserved BYTE-EXACTLY: the shard is split at the
* event-block boundaries the log parser itself reads (`## ` headings),
* and only the dropped spans are removed - no reformatting, no
* re-serialisation of what stays. A shard whose every entry is dropped
* contributes nothing, and a day left with no contributing shard stops
* counting as a log day. Without a filter - and under
* `includePromptDerived` - each shard rides whole, exactly as before.
*/
export interface OkfLogFilter {
/**
* The operator override: when true the log rides whole, the same
* decision `--include-prompt-derived` makes for pages.
*/
readonly includePromptDerived: boolean;
/** The vault-relative paths the page split left out of the bundle. */
readonly omittedPaths: ReadonlySet<string>;
}
export interface OkfLog {
/** The date-grouped change log, ready for the `log.md` bundle file. */
readonly contents: string;
/** Count of DAYS with at least one kept entry, across every shard. */
readonly days: number;
/** Entries the content-class filter dropped; zero without a filter. */
readonly droppedEntries: number;
/**
* Log events the store held in its `private` partition, left out of
* `log.md` and counted before any content-class filtering; zero on the
* legacy layout.
*/
readonly privateEntries: number;
}
/**
* One top-level log bullet, in the shape the log parser itself reads
* (`- key: value`). Spelled locally because the parser does not export
* it and this module reads no other log syntax.
*/
const LOG_BULLET_RE = /^-\s+([A-Za-z_][A-Za-z0-9_-]*)\s*:\s*(.*)$/;
export function buildLog(vault: string, filter?: OkfLogFilter): OkfLog {
const bodiesByDate = new Map<string, string[]>();
let droppedEntries = 0;
// One read of the store log, counted by partition: a private event never
// reaches `log.md` (review finding F333), and the export refuses when a
// log shard cannot be read, so the bundle is never silently short of
// events and the private count is never a lower bound.
const report = readStoreLogEventsReport(WHOLE_LOG, { vault });
// A store-only machine whose store cannot be opened reads no events and
// no unreadable shard: refuse as a protection reader does, switched or
// not, rather than export an empty log that claims nothing private was
// left out.
if (report !== null) protectionStore(undefined, { openErrorFails: true, vault });
if (report !== null && report.unreadable.length > 0) {
throw new Error(
`okf export: cannot read log shard(s) ${report.unreadable.join(", ")}; the bundle would be short of their events`,
);
}
// Counted before the content-class filter: the partition rule and the
// class policy are independent, so the two counts never overlap.
const privateEntries = report?.events.filter((e) => e.partition === "private").length ?? 0;
// A store-only machine keeps no day files: the days of the shared events
// are rendered the way the day file renders it (header, then one block
// per entry), so the bundle's `log.md` reads the same.
const storedDays =
report === null
? null
: groupLogDays(report.events.filter((e) => e.partition === "shared").map((e) => e.entry));
for (const day of storedDays ?? []) {
const body = [renderLogDayHeader(day.date), ...day.entries.map(renderLogEntryBlock)]
.join("\n")
.trimEnd();
const text =
filter === undefined || filter.includePromptDerived
? body
: filterLogShard(body, filter, (dropped) => {
droppedEntries += dropped;
});
if (text.length > 0) bodiesByDate.set(day.date, [text]);
}
for (const shard of storedDays === null ? listLogShardFiles(vault) : []) {
// The bundle is markdown; the JSONL sidecar of a day carries the same
// entries in machine form and would duplicate every line.
if (shard.ext !== "md") continue;
let body: string;
try {
body = readFileSync(shard.path, "utf8").trimEnd();
} catch (err) {
// Never `continue`: a shard the export could not read is a day of
// the change log missing from the bundle, and a bundle that is
// silently short a day is exactly what this export must not produce.
throw new Error(
`okf export: cannot read log shard ${shard.name}: ${err instanceof Error ? err.message : String(err)}`,
);
}
if (body.length === 0) continue;
const text =
filter === undefined || filter.includePromptDerived
? body
: filterLogShard(body, filter, (dropped) => {
droppedEntries += dropped;
});
if (text.length === 0) continue;
const bodies = bodiesByDate.get(shard.date);
if (bodies === undefined) bodiesByDate.set(shard.date, [text]);
else bodies.push(text);
}
const sections = [...bodiesByDate.keys()]
.toSorted()
.map((date) => `## ${date}\n\n${bodiesByDate.get(date)!.join("\n\n")}`);
return {
contents: sections.join("\n\n"),
days: bodiesByDate.size,
droppedEntries,
privateEntries,
};
}
/**
* Drop the filtered entries from one shard's text, keeping every other
* byte where it was. The shard is split at the `## ` event-heading
* boundaries the log parser reads; the text before the first heading is
* the shard's own frontmatter and title and always stays, and each
* entry's span runs to the next heading, so removing an entry removes
* exactly its lines and nothing else. Splitting on `\n` and rejoining on
* `\n` keeps a hand-edited CRLF shard's kept bytes untouched.
*/
function filterLogShard(
body: string,
filter: OkfLogFilter,
recordDrop: (dropped: number) => void,
): string {
const lines = body.split("\n");
let firstBlock = -1;
for (let i = 0; i < lines.length; i++) {
if (lines[i]!.startsWith("## ")) {
firstBlock = i;
break;
}
}
// No event headings: no entry can be provably marked, so the shard
// stays whole rather than being guessed about.
if (firstBlock < 0) return body;
const starts: number[] = [];
for (let i = firstBlock; i < lines.length; i++) {
if (lines[i]!.startsWith("## ")) starts.push(i);
}
const prologue = lines.slice(0, firstBlock).join("\n");
const kept: string[] = [];
for (let n = 0; n < starts.length; n++) {
const end = n + 1 < starts.length ? starts[n + 1]! : lines.length;
if (!logEntryIsFilteredOut(lines.slice(starts[n]!, end), filter)) {
kept.push(lines.slice(starts[n]!, end).join("\n"));
} else {
recordDrop(1);
}
}
// Every entry dropped: the shard has nothing left to say, and a bare
// frontmatter-plus-title shell would count as a log day that logs
// nothing.
if (kept.length === 0) return "";
return `${prologue}${kept.join("")}`.trimEnd();
}
/**
* Whether one event block leaves under the filter. The block's top-level
* bullets are read the way the log parser reads them (`- key: value`,
* later occurrences of a key winning), and the decision is fail-open: an
* entry that carries neither a known channel nor an omitted target stays.
*/
function logEntryIsFilteredOut(blockLines: ReadonlyArray<string>, filter: OkfLogFilter): boolean {
const bullets = new Map<string, string>();
for (const line of blockLines) {
const bullet = LOG_BULLET_RE.exec(line);
if (bullet !== null) bullets.set(bullet[1]!, bullet[2]!);
}
// The channel the creation stamp derives the reserved class from: an
// entry appended over the MCP transport is agent-dictated material,
// exactly as a page created there is. A bullet naming a channel this
// build does not know marks nothing.
const channel = bullets.get(ORIGIN_CHANNEL_FIELD);
if (
typeof channel === "string" &&
isOriginChannelStamp(channel) &&
contentClassCreationStamp(channel) !== null
) {
return true;
}
// A write entry whose target is the page the split left out would name
// the omission right back into the bundle.
const target = bullets.get("target");
return typeof target === "string" && filter.omittedPaths.has(target);
}
/**
* Render the bundle-root `index.md` from a manifest. Exported because
* the export verb re-renders the index from the REDACTED manifest after
* the guard runs - an index naming a value the shipped `okf.json` no
* longer carries (a redacted foreign type, an omitted page) would make
* the bundle disagree with itself in the file a human reads first.
*/
export function renderIndex(manifest: OkfManifest): string {
const lines: string[] = [
"---",
// OKF v0.2's version declaration. Per the spec it is a MAY in the
// bundle-root index, and the ONLY frontmatter an index file carries.
`okf_version: "${OKF_VERSION}"`,
"---",
"",
`# ${manifest.vault_basename} — Open Knowledge Format bundle`,
"",
`> Portable knowledge bundle produced by \`${manifest.producer}\`.`,
`> Schema ${manifest.schema}. ${manifest.pages.length} page(s),`,
`> ${manifest.log_days} log day(s). See \`okf.json\` for machine-readable`,
`> metadata and \`log.md\` for the change log.`,
"",
];
const order: ReadonlyArray<[OkfPageClass, string]> = [
["concept", "Concepts"],
["query", "Queries"],
["reference", "References"],
];
for (const [cls, label] of order) {
const subset = manifest.pages.filter((p) => p.class === cls);
if (subset.length === 0) continue;
lines.push(`## ${label}`, "");
for (const p of subset) {
const foreign = p.foreign_type ? ` (foreign type: ${p.foreign_type})` : "";
lines.push(`- [${p.id}](${p.bundle_path}) — \`${p.path}\`${foreign}`);
}
lines.push("");
}
return lines.join("\n");
}
/**
* Serialise a manifest to the exact bytes `okf.json` carries. Exported
* because the export verb redacts the manifest as a TREE - a pass over
* serialised JSON can eat a closing quote - and then has to re-render
* this one file from the redacted result.
*/
export function renderOkfManifest(manifest: OkfManifest): string {
return JSON.stringify(manifest, null, 2) + "\n";
}
/** A citation target that names an absolute resource, not a vault page. */
const URL_TARGET_RE = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//;
/**
* Build the resolver that turns citation targets into OKF v0.2
* `sources:` entries. It keys on the page walk's own bundle_path
* assignment, so a resolved source names exactly the file the bundle
* wrote for the cited page. Resolution order:
*
* 1. a URL target is carried as-is;
* 2. a path-shaped target must name a collected page (with or without
* the `.md` suffix) - otherwise it is dropped, never guessed. The
* entry carries the page's id only when that stem is unique; a
* colliding basename has no stable id to name;
* 3. a bare basename resolves only when EXACTLY ONE page carries that
* stem: the colliding-basename case (two `Dup.md` pages) is dropped
* rather than guessed.
*
* Anything else is dangling and dropped - consumers must tolerate
* broken links, and a wrong source is worse than no source.
*/
function sourceResolver(pages: ReadonlyArray<OkfPage>): (citation: string) => OkfSource | null {
const byPath = new Map<string, OkfPage>();
const byStem = new Map<string, OkfPage[]>();
for (const p of pages) {
byPath.set(p.path, p);
const stems = byStem.get(p.id);
if (stems === undefined) byStem.set(p.id, [p]);
else stems.push(p);
}
return (citation: string): OkfSource | null => {
if (URL_TARGET_RE.test(citation)) return { resource: citation };
if (citation.includes("/")) {
const direct = byPath.get(citation) ?? byPath.get(`${citation}.md`);
if (direct === undefined) return null;
// The path resolves the page, but its stem names an id only when it
// is unique: two colliding basenames are both id "Dup", and the id
// field is a stable key. The entry carries the bundle path alone -
// same refuse-to-guess rule as the basename branch.
const stems = byStem.get(direct.id);
if (stems !== undefined && stems.length > 1) return { resource: direct.bundle_path };
return { resource: direct.bundle_path, id: direct.id };
}
// The stem map keys on stems, but a citation may keep its extension
// (`[[Dup.md]]` names the same page `[[Dup]]` does), so a direct miss
// falls back to stem(citation). Exact-unique either way: the
// colliding-basename case still drops rather than guess.
const stems = byStem.get(citation) ?? byStem.get(stem(citation));
if (stems === undefined || stems.length !== 1) return null;
return { resource: stems[0]!.bundle_path, id: stems[0]!.id };
};
}
/** Translate a page's citations into de-duplicated source entries. */
function translateSources(
page: OkfPage,
resolveSource: (citation: string) => OkfSource | null,
): OkfSource[] {
const out: OkfSource[] = [];
const seen = new Set<string>();
for (const citation of page.citations) {
const source = resolveSource(citation);
if (source === null || seen.has(source.resource)) continue;
seen.add(source.resource);
out.push(source);
}
return out;
}
/**
* Render one scalar inside a YAML flow mapping (`{resource: …, id: …}`).
* EVERY value is emitted as a JSON string literal: JSON string syntax is
* valid YAML double-quoted style, so the scalar always parses back as a
* string — a bare-looking id like `2024` would be an integer to a strict
* YAML reader, bare `true` / `null` / `~` would be a boolean or null, and
* a hand-rolled escape covers nothing like the full metacharacter set
* (control characters included) JSON.stringify handles. The flow shapes
* around the scalars are kept.
*/
function okfFlowScalar(value: string): string {
return JSON.stringify(value);
}
/**
* Serialise one bundle page file: the page's own frontmatter and body
* verbatim, plus the OKF v0.2 per-page keys (`type`, `status`,
* `sources`, `generated`) as DERIVED fields. The derived keys are
* recomputed from current page state on every export and never carried
* through from a previous bundle, so re-exporting an imported page
* re-derives them instead of stacking a foreign bundle's values a second
* time. Because the derived block is a projection - a closed status
* enum, sources re-translated from citations, a type taken from the
* preserved foreign type or the structural kind - the page's NATIVE
* values ride beside it under the producer namespace (`okf_status_raw`,
* `okf_sources_raw`, `okf_generated_raw`, `okf_type_raw`), so a trusted
* own-producer import can restore what the projection cannot name. The
* derived `type` is the spec's one always-required per-concept key, so
* every page carries it even when its frontmatter never did.
*
* The derived block is rendered by hand in YAML flow style because the
* shared frontmatter writer emits scalars and inline arrays only - and
* the flow shapes chosen (one line per source entry, one line for
* `generated`) keep the page parseable by consumers with full YAML
* parsers while never producing a line this system's own scanner would
* misread as a top-level key.
*/
function renderOkfPageFile(
page: OkfPage,
sources: ReadonlyArray<OkfSource>,
generatedAt: string,
): string {
const carried: FrontmatterMap = {};
for (const [key, value] of Object.entries(page.frontmatter)) {
// Both sides of every pair are skipped: the native field is re-derived
// below, and the raw carrier is re-stamped from the current state - the
// pair list is the one source of what must not ride through verbatim.
if (RAW_OKF_KEYS.some(({ raw, native }) => raw === key || native === key)) continue;
if (key === RAW_CARRY_KEY) continue;
carried[key] = value;
}
// The native values ride the producer namespace beside the derived
// block: the derived status maps onto a closed enum and the derived
// sources are re-translated from citations, so exporting them in place
// would rewrite what the operator wrote. Recomputed from current page
// state on every export and never carried through from a previous
// bundle - the same honesty rule as the derived block. A value is
// carried on PRESENCE alone (the rule sources and generated follow):
// a non-string status - a number from a hand-built frontmatter map, a
// list - is exactly the value the closed enum cannot name, so reading
// it through the string-only field would drop the one raw key that
// preserves it.
for (const { raw, native } of RAW_OKF_KEYS) {
if (page.frontmatter[native] !== undefined) carried[raw] = page.frontmatter[native];
}
// The raw-carrying declaration, stamped fresh on every page of every
// export: this is what tells the own-producer import that absent raw
// keys mean "the field never had a value" rather than "a legacy
// bundle's page - leave the frontmatter alone". It names exactly the
// natives this exporter manages - the derived `type` included - so
// the restore drops the whole derived block from a page that never
// carried one natively, and a bare `true` stays what it always was:
// a LEGACY page's marker, covering the pre-type-pair set only.
carried[RAW_CARRY_KEY] = RAW_OKF_KEYS.map(({ native }) => native);
// formatFrontmatter with an empty body is exactly "---\n<keys>---\n";
// drop the closing fence, append the derived block, re-close.
let out = formatFrontmatter(carried, "").slice(0, -"---\n".length);
// `type` is the spec's only always-required per-concept key (non-empty),
// so the derived block opens with it: the preserved foreign type when
// the page was imported from a foreign producer, else the page's own
// structural kind - the honest fields the page walk collected. A native
// `type` is NOT shadowed by this projection: it rode out under
// okf_type_raw above, and the own-producer restore puts it back.
out += `type: ${okfFlowScalar(page.foreign_type ?? page.kind)}\n`;
out += `status: ${page.status}\n`;
if (sources.length > 0) {
out += "sources:\n";
for (const source of sources) {
out += ` - {resource: ${okfFlowScalar(source.resource)}`;
if (source.id !== undefined) out += `, id: ${okfFlowScalar(source.id)}`;
out += "}\n";
}
}
out += `generated: {by: ${okfFlowScalar(OKF_GENERATED_BY)}, at: "${generatedAt}"}\n`;
out += "---\n";
if (page.body.length > 0) out += `\n${page.body}\n`;
return out;
}
/**
* Build an OKF bundle write-plan from the vault. Pure and read-only:
* returns the manifest plus every file to write. Content is
* deterministic across runs (pages and log days are sorted); the only
* value that varies is the single shared generated instant, which the
* manifest's `generated_at` and every page's `generated.at` carry.
*/
export function buildOkfBundle(vault: string): OkfBundle {
return assembleOkfBundle(vault, collectOkfPages(vault), buildLog(vault));
}
/**
* Build an OKF bundle write-plan over a caller-chosen SUBSET of pages and
* no change log. The knowledge-pack export is the caller: a pack carries
* the pages an operator selected, never the vault's whole history, so the
* date-grouped `log.md` a full bundle folds in is left out (the manifest
* says `log_days: 0`). Pure; the pages are written as given.
*/
export function buildOkfSubsetBundle(vault: string, pages: ReadonlyArray<OkfPage>): OkfBundle {
return assembleOkfBundle(vault, pages, null);
}
/**
* Build a full OKF bundle write-plan - change log included - over
* caller-collected pages, composing from the array as given instead of
* walking the vault again. {@link buildOkfSubsetBundle} is the existing
* page-input entry point, but a subset bundle carries no `log.md` by
* contract, so a caller that has ALREADY walked the vault (the shared
* export verbs, which split the collected pages through the egress
* content-class policy before composition) had no way to keep the change
* log without a second walk. Composing from the caller's page array
* closes that double-walk window: between the caller's walk and a
* re-walk here, a page can change or appear, and the bundle would mix
* the two reads - or, with the class policy applied to the first walk
* only, re-admit a page the policy had just left out. The pages are
* written as given.
*
* The `log` argument is the change log to fold in, already read (and
* already filtered through the caller's content-class policy) by
* {@link buildLog}. Omitted, the log is read here unfiltered - the
* whole-log behavior the callers that have no split in hand rely on - so
* a verb that needs the filtered log passes its {@link buildLog} result
* through rather than re-reading the vault a second time.
*/
export function buildOkfBundleFromPages(
vault: string,
pages: ReadonlyArray<OkfPage>,
log: Pick<OkfLog, "contents" | "days" | "privateEntries"> = buildLog(vault),
): OkfBundle {
return assembleOkfBundle(vault, pages, log);
}
function assembleOkfBundle(
vault: string,
pages: ReadonlyArray<OkfPage>,
log: Pick<OkfLog, "contents" | "days" | "privateEntries"> | null,
): OkfBundle {
// ONE instant for the whole bundle: the manifest's `generated_at` and
// every page's `generated.at` share it, so whole-bundle byte equality
// modulo that single value stays the determinism contract.
const generatedAt = isoSecond();
const resolveSource = sourceResolver(pages);
const derived = pages.map((p) => ({ page: p, sources: translateSources(p, resolveSource) }));
const manifest: OkfManifest = {
schema: OKF_SCHEMA_VERSION,
producer: OKF_PRODUCER,
generated_at: generatedAt,
vault_basename: vaultDisplayName(vault),
pages: derived.map(({ page, sources }) => ({
id: page.id,
path: page.path,
class: page.cls,
bundle_path: page.bundle_path,
kind: page.kind,
citations: page.citations,
status: page.status,
sources,
aliases: page.aliases,
freshness: page.freshness,
foreign_type: page.foreign_type,
producer_meta: page.producer_meta,
})),
log_days: log?.days ?? 0,
log_private_omitted: log?.privateEntries ?? 0,
};
const files: OkfBundleFile[] = [
{ path: OKF_MANIFEST_FILENAME, contents: renderOkfManifest(manifest) },
{ path: OKF_INDEX_FILENAME, contents: renderIndex(manifest) },
];
if (log !== null) files.push({ path: "log.md", contents: log.contents });
for (const { page, sources } of derived) {
files.push({ path: page.bundle_path, contents: renderOkfPageFile(page, sources, generatedAt) });
}
files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
return { manifest, files };
}
/**
* Write a bundle to `dir`. Refuses to clobber a non-empty existing
* directory unless `force` is set. Creates parent subdirectories as
* needed; each file is written atomically.
*/
export function writeOkfBundle(
dir: string,
bundle: OkfBundle,
opts: { force?: boolean } = {},
): void {
if (existsSync(dir)) {
let existing: string[] = [];
try {
existing = readdirSync(dir);
} catch {
/* treat unreadable as empty */
}
if (existing.length > 0 && !opts.force) {
throw new OkfError(`${dir} is not empty; pass --force to overwrite`);
}
}
for (const file of bundle.files) {
const abs = ensureInsideVault(resolve(dir, file.path), dir);
mkdirSync(dirname(abs), { recursive: true });
atomicWriteFileSync(abs, file.contents);
}
}
// ----- Import ---------------------------------------------------------------
/** Raised when an OKF bundle is structurally invalid or unsupported. */
export class OkfError extends Error {
constructor(message: string) {
super(message);
this.name = "OkfError";
}
}
/** A page parsed back out of a bundle on disk. */
export interface ParsedOkfPage {
readonly entry: OkfManifestPage;
readonly frontmatter: FrontmatterMap;
readonly body: string;
}
export interface ParsedOkfBundle {
readonly manifest: OkfManifest;
/** True when the bundle was produced by some tool other than OSB. */
readonly foreign: boolean;
readonly pages: ReadonlyArray<ParsedOkfPage>;
}
interface ManifestInput {
readonly schema?: unknown;
readonly producer?: unknown;
readonly generated_at?: unknown;
readonly vault_basename?: unknown;
readonly log_days?: unknown;
readonly log_private_omitted?: unknown;
readonly pages?: unknown;
}
/** Parse a manifest `sources` entry; entries without a resource are dropped. */
function asOkfSource(value: unknown): OkfSource | null {
if (value === null || typeof value !== "object") return null;
const resource = (value as Record<string, unknown>)["resource"];
if (typeof resource !== "string" || resource.length === 0) return null;
const id = (value as Record<string, unknown>)["id"];
return typeof id === "string" && id.length > 0 ? { resource, id } : { resource };
}
function asManifestPage(value: unknown): OkfManifestPage | null {
if (value === null || typeof value !== "object") return null;
const v = value as Record<string, unknown>;
if (typeof v["bundle_path"] !== "string" || typeof v["path"] !== "string") return null;
const cls = v["class"];
const klass: OkfPageClass =
cls === "query" || cls === "reference" || cls === "concept" ? cls : "concept";
const status = v["status"];
return {
id: typeof v["id"] === "string" ? v["id"] : stem(v["path"]),
path: v["path"],
class: klass,
bundle_path: v["bundle_path"],
kind: typeof v["kind"] === "string" ? v["kind"] : DEFAULT_KIND,
citations: Array.isArray(v["citations"]) ? (v["citations"] as string[]) : [],
status: status === "draft" || status === "deprecated" ? status : "stable",
sources: Array.isArray(v["sources"])
? v["sources"].map(asOkfSource).filter((s): s is OkfSource => s !== null)
: [],
aliases: Array.isArray(v["aliases"]) ? (v["aliases"] as string[]) : [],
freshness: typeof v["freshness"] === "string" ? v["freshness"] : null,
foreign_type: typeof v["foreign_type"] === "string" ? v["foreign_type"] : null,
producer_meta:
v["producer_meta"] && typeof v["producer_meta"] === "object"
? (v["producer_meta"] as Record<string, string>)
: {},
};
}
/**
* Read the bundle-root index's `okf_version` declaration, or null when
* the index is missing, unreadable, or declares no version. Consulted
* only when the manifest does not carry the dialect schema, so the
* schema-"1" path (sealed knowledge-pack halves included) never grows a
* dependency on an index file being present.
*/
function indexOkfVersion(dir: string): string | null {
let raw: string;
try {
raw = readFileSync(join(dir, "index.md"), "utf8");
} catch {
return null;
}
const [meta] = parseFrontmatterText(raw);
const value = meta["okf_version"];
return typeof value === "string" && value.length > 0 ? value : null;
}
/**
* The bundle-directory names that map onto page classes, derived from
* {@link CLASS_DIR} so the export's lanes and the import's reading of
* them cannot drift.
*/
const SPEC_DIR_CLASS: Readonly<Record<string, OkfPageClass>> = Object.freeze(
Object.fromEntries(Object.entries(CLASS_DIR).map(([cls, dir]) => [dir, cls as OkfPageClass])),
);
/**
* Reserved bundle-root files per the OKF v0.2 spec: the version-
* declaring index and the change log. The reservation is a ROOT one -
* the spec reserves the two names at the top of the bundle tree, so a
* foreign bundle's nested `concepts/index.md` or a per-project
* `notes/log.md` is a real page. The walk matches these against the
* page's full bundle-relative path, lowercased - never the basename,
* which would skip real pages at any depth.
*/
const SPEC_RESERVED_PATHS: ReadonlySet<string> = new Set([OKF_INDEX_FILENAME, "log.md"]);
/**
* The producer label a manifest-less spec bundle reports: there is no
* manifest to declare one, so the parsed manifest names the walk
* honestly instead of guessing a producer from the tree. Named because
* the bundle is FOREIGN by construction - a walk with no manifest can
* never be this system's own dialect - and the old inline comparison
* against {@link OKF_PRODUCER} restated that fact as a tautology.
*/
const SPEC_BUNDLE_PRODUCER = "unknown";
/**
* Read a manifest-less, spec-shaped OKF v0.2 bundle: the spec defines a
* bundle as a plain directory tree of markdown and requires NO machine
* manifest, so the page list is built by walking the tree. Reserved
* bundle-root files (index.md, log.md) are skipped, the class comes from
* the page's top-level directory when it names a class lane (concepts /
* queries / references) and defaults to `concept` everywhere else, and
* each walked page's own `type` frontmatter is recorded as its foreign
* type - the producer's raw type value, which the import's provenance
* stamp preserves. There is no manifest to declare a producer, so the
* parsed manifest reports {@link SPEC_BUNDLE_PRODUCER} and the bundle is
* FOREIGN by construction: every page imports through the foreign path
* (machinery stripped, raw keys shed, provenance stamped, review-staged
* by default) - the walk never restores machinery keys on a spec
* bundle's word.
*/
function readSpecOkfBundle(dir: string): ParsedOkfBundle {
const producer = SPEC_BUNDLE_PRODUCER;
const entries: OkfManifestPage[] = [];
const pages: ParsedOkfPage[] = [];
for (const abs of listMarkdown(dir, [])) {
const rel = relative(dir, abs).split(/[\\/]/).join(posix.sep);
if (SPEC_RESERVED_PATHS.has(rel.toLowerCase())) continue;
// Same wall the manifest path applies: a walked file that is not
// (symlink-)inside the bundle is skipped, never read.
try {
ensureInsideVault(abs, dir);
} catch {
continue;
}
let raw: string;
try {
raw = readFileSync(abs, "utf8");
} catch {
continue;
}
const [frontmatter, body] = parseFrontmatterText(raw);
const topDir = rel.includes("/") ? rel.slice(0, rel.indexOf("/")) : "";
const entry: OkfManifestPage = {
id: stem(rel),
path: rel,
class: SPEC_DIR_CLASS[topDir] ?? "concept",
bundle_path: rel,
kind: stringField(frontmatter, "kind") ?? DEFAULT_KIND,
citations: [],
status: "stable",
sources: [],
aliases: [],
freshness: null,
foreign_type: stringField(frontmatter, "type"),
producer_meta: {},
};
entries.push(entry);
pages.push({ entry, frontmatter, body });
}
return {
manifest: {
schema: "",
producer,
generated_at: "",
vault_basename: "",
pages: entries,
log_days: 0,
log_private_omitted: 0,
},
// No manifest, no own-producer claim to honour: the walk's output is
// foreign by construction - stating it directly instead of resting
// on a literal-vs-constant comparison that could only ever be true.
foreign: true,
pages,
};
}
/**
* Read an OKF bundle directory: parse `okf.json`, validate the schema,
* then parse each manifest-listed page file. Three bundle generations
* are accepted: a manifest carrying the Open Second Brain dialect
* `schema: "1"`, a bundle whose manifest declares NO schema while its
* bundle-root `index.md` declares `okf_version: "0.2"` (the OKF v0.2
* marker this export writes beside the dialect schema), and - the spec
* shape - a bundle with NO manifest at all whose bundle-root index
* declares the same version, which is read by walking the tree (see
* {@link readSpecOkfBundle}). A PRESENT non-dialect schema is refused
* whatever the index says: this export's own index always declares the
* version, so treating the declaration as sufficient on its own would
* admit every future dialect bump under v1 assumptions. Everything else
* is refused by name. A page whose file is missing or unreadable is
* skipped (the manifest is the source of truth for what *should* be
* there; a partial bundle still imports what it has). An unsupported
* schema or an absent manifest without a v0.2 index declaration throws
* {@link OkfError}.
*/
export function readOkfBundle(dir: string): ParsedOkfBundle {
const manifestPath = join(dir, OKF_MANIFEST_FILENAME);
if (!existsSync(manifestPath)) {
// No manifest at all. The v0.2 spec requires none, so the spec shape
// - a v0.2 declaration on the bundle-root index - is accepted and
// read as a tree; everything else is refused exactly as before.
if (indexOkfVersion(dir) === OKF_VERSION) return readSpecOkfBundle(dir);
throw new OkfError(`not an OKF bundle: ${manifestPath} is missing`);
}
let parsed: ManifestInput;
try {
parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as ManifestInput;
} catch (exc) {
throw new OkfError(`okf.json is not valid JSON: ${(exc as Error).message ?? exc}`);
}
if (parsed === null || typeof parsed !== "object") {
throw new OkfError("okf.json must be a JSON object");
}
// A null schema declares nothing, exactly like an absent or empty one:
// a writer that nulls the field is not naming a schema, and falling
// through to the by-name refusal would report "expected 1, got null"
// even beside a v0.2 index declaration.
const schemaAbsent =
parsed.schema === undefined || parsed.schema === null || parsed.schema === "";
if (parsed.schema === OKF_SCHEMA_VERSION) {
// The dialect path. Sealed knowledge-pack halves exported before
// v0.2 emission take it, so it never depends on an index file.
} else if (schemaAbsent && indexOkfVersion(dir) === OKF_VERSION) {
// The version-declaration path: no manifest schema, and the bundle
// root says OKF v0.2. Version-level only - the gate never reads the
// producer.
} else if (schemaAbsent) {
throw new OkfError(
`unsupported OKF schema: no schema ${OKF_SCHEMA_VERSION} in ${OKF_MANIFEST_FILENAME} ` +
`and no okf_version ${OKF_VERSION} declaration in index.md`,
);
} else {
throw new OkfError(
`unsupported OKF schema: expected ${OKF_SCHEMA_VERSION}, got ${String(parsed.schema)}`,
);
}
const producer = typeof parsed.producer === "string" ? parsed.producer : "unknown";
const rawPages = Array.isArray(parsed.pages) ? parsed.pages : [];
const entries = rawPages.map(asManifestPage).filter((p): p is OkfManifestPage => p !== null);
const pages: ParsedOkfPage[] = [];
for (const entry of entries) {
// Validate the bundle-relative path stays inside the bundle dir
// before reading — a hostile manifest cannot make us read outside.
let abs: string;
try {
abs = ensureInsideVault(join(dir, entry.bundle_path), dir);
} catch {
continue;
}
let raw: string;
try {
raw = readFileSync(abs, "utf8");
} catch {
continue;
}
const [frontmatter, body] = parseFrontmatterText(raw);
pages.push({ entry, frontmatter, body });
}
return {
manifest: {
// Preserve the ON-DISK schema value rather than stamping this
// module's constant: a reader inspecting the parsed manifest must
// see which declaration the bundle actually made (a v0.2-accepted
// bundle may name a non-dialect schema, or none at all).
schema: typeof parsed.schema === "string" ? parsed.schema : "",
producer,
// Preserve the bundle's own metadata instead of synthesizing
// placeholders: readOkfBundle is a reader, and a caller that
// inspects the parsed manifest should see what is on disk.
generated_at: typeof parsed.generated_at === "string" ? parsed.generated_at : "",
vault_basename: typeof parsed.vault_basename === "string" ? parsed.vault_basename : "",
pages: entries,
log_days:
typeof parsed.log_days === "number" && Number.isFinite(parsed.log_days)
? parsed.log_days
: 0,
log_private_omitted:
typeof parsed.log_private_omitted === "number" &&
Number.isFinite(parsed.log_private_omitted)
? parsed.log_private_omitted
: 0,
},
foreign: producer !== OKF_PRODUCER,
pages,
};
}
export interface OkfImportOptions {
/**
* When true, write each page directly to its recorded vault-relative
* path (a true round-trip). When false (default), stage every page
* under `OKF Review/` with `okf_review: pending` so the live vault is
* untouched until an operator promotes the candidates.
*
* This flag is the operator's vouch, and the only one the import
* honours: without it machinery frontmatter (`_status`, `owner`,
* `origin_channel`, ...) is stripped and the Brain page lanes are not
* writable, whatever `producer` the bundle's manifest declares.
*/
readonly trusted?: boolean;
/** Wall clock for the import stamp. Tests pin this. */
readonly now?: Date;
/**
* Server-side provenance written onto every landed page AFTER the
* machinery strip, so the bundle can neither supply nor forge it. The
* knowledge-pack installer passes its `knowledge_pack` stamp here.
*/
readonly provenance?: Readonly<Record<string, string>>;
}
export interface OkfImportResult {
/** Vault-relative paths written. */
readonly written: ReadonlyArray<string>;
/** Paths skipped because the target already existed (review mode). */
readonly skipped: ReadonlyArray<string>;
readonly errors: ReadonlyArray<{ path: string; message: string }>;
/** True when the bundle producer was foreign (provenance was stamped). */
readonly foreign: boolean;
/** `"trusted"` (direct write) or `"review"` (staged). */
readonly mode: "trusted" | "review";
}
/**
* Frontmatter keys an imported page may not carry across the trust
* boundary. These are the machinery fields this system's own writers
* stamp server-side - the `_status`/`_confidence` lifecycle, the
* `_revision`/`_content_hash` write accounting, the `origin_channel`
* transport stamp, the `owner` scope claim, and the `okf_review` staging
* marker - so a bundle supplying them verbatim would mint pages that
* read as system-authored the moment an operator promotes them out of
* the review lane.
*
* Stripped on every import the operator did not vouch for with
* `--trusted`, whatever the manifest's `producer` says: the producer
* field is bundle-supplied and a forged `open-second-brain` must not
* carry `_status: confirmed` through. A trusted import of a bundle that
* declares a foreign producer strips them too - nothing a foreign tool
* wrote is this system's machinery. The one lossless path is a trusted
* import of a bundle that declares this producer: the operator's own
* round-trip, vouched for by the operator, not by the bundle.
*
* `visibility` is deliberately NOT in this set. It is the page's
* read-scope, and every token it can carry only NARROWS reach (a page
* with no `visibility:` is reachable by every consumer; see
* `graph/visibility.ts`). Stripping it could only widen what a page
* exposes - a private page in the operator's own bundle would land
* public in the review lane - while preserving it can at worst hide a
* staged page from a remote read, which the review lane survives.
*/
const FOREIGN_MACHINERY_KEYS: ReadonlySet<string> = new Set([
"_status",
"_confidence",
"_confidence_value",
"_revision",
"_content_hash",
"_force_confirmed_via",
"origin_channel",
"owner",
"okf_review",
// The knowledge-pack provenance stamp: written by the pack installer
// after this strip, so a bundle cannot enrol a page in a pack's uninstall.
KNOWLEDGE_PACK_FIELD,
// The staged page's install-time fingerprint, same rule as the stamp.
KNOWLEDGE_PACK_SHA_FIELD,
]);
/**
* A copy of `meta` without this system's machinery keys (see
* {@link FOREIGN_MACHINERY_KEYS}). The knowledge-pack export runs every
* selected page through it: a pack is handed to someone else, and this
* vault's lifecycle, write-accounting, transport and owner stamps are not
* knowledge the recipient can use - the recipient's import strips them
* anyway, so leaving them in would only widen what the file leaks.
*/
export function stripOkfMachinery(meta: FrontmatterMap): FrontmatterMap {
const out: FrontmatterMap = { ...meta };
for (const key of FOREIGN_MACHINERY_KEYS) delete out[key];
return out;
}
/**
* The Brain directories an OKF bundle carries pages for: exactly the two
* lanes {@link buildOkfBundle} exports (`sources/` as references,
* `reports/` as queries).
*/
const OKF_BRAIN_PAGE_LANES: ReadonlyArray<string> = Object.freeze([
BRAIN_SOURCES_REL,
BRAIN_REPORTS_REL,
]);
/**
* The refusal reason for a recorded page path, or `null` when the path may
* proceed into its lane. Two walls:
*
* 1. A recorded path is a vault-relative POSIX path: absolute, empty,
* backslash, `.`, and `..` segments are refused before any join.
* Refusing `..` is what keeps the review lane unescapable -
* `posix.join("OKF Review", "../Brain/standing-rules.md")` normalizes
* to `Brain/standing-rules.md`, which would let a hostile manifest
* plant machinery files straight into the live vault. Backslash is
* refused as a character because it is a separator on the Windows
* filesystems this project ships on, where a posix-string check would
* not see the escape.
* 2. In trusted mode `Brain/` is machinery, not pages: only the two page
* lanes the exporter emits (`sources/`, `reports/`) are writable, and
* only because the operator passed `--trusted` - the bundle's
* `producer` claim plays no part. The Brain segment is compared
* case-insensitively (`brain/` opens `Brain/` on macOS and Windows).
* Review mode needs no such wall - wall 1 already guarantees staging
* stays under `OKF Review/`, where a Brain-shaped subpath is inert.
*/
function okfTargetRefusal(entryPath: string, trusted: boolean): string | null {
if (
entryPath.length === 0 ||
entryPath.startsWith("/") ||
entryPath.includes("\\") ||
entryPath.split("/").some((s) => s.length === 0 || s === "." || s === "..")
) {
return `recorded path must be a relative POSIX path with no empty, '.', '..', or backslash segments: ${entryPath}`;
}
if (trusted && isUnderBrainRoot(entryPath) && !isInBrainLane(entryPath, OKF_BRAIN_PAGE_LANES)) {
return (
`trusted import refuses Brain machinery paths except the ` +
`${BRAIN_SOURCES_REL}/ and ${BRAIN_REPORTS_REL}/ page lanes: ${entryPath}`
);
}
return null;
}
/**
* Read the native-key coverage a page's {@link RAW_CARRY_KEY} marker
* declares. A bare `true` - the shape every exporter stamped before the
* marker could carry a list - covers exactly the old three-key set
* ({@link RAW_CARRY_TRUE_NATIVES}); a list of native names covers what
* it names, so an exporter managing a different set declares it instead
* of the import guessing. An unrecognized shape still says legacy
* raw-carrying. An ABSENT marker (a page whose raw keys survived while
* the marker was lost) declares nothing extra: the present raw keys
* restore their own pairs, and nothing else is dropped.
*/
function rawCarryDeclaredNatives(marker: FrontmatterValue | undefined): ReadonlySet<string> {
if (marker === undefined) return new Set();
if (Array.isArray(marker)) {
return new Set(marker.filter((item): item is string => typeof item === "string"));
}
return RAW_CARRY_TRUE_NATIVES;
}
/**
* Restore the native frontmatter an own-producer export carried under
* the producer namespace (the `okf_*_raw` keys), dropping the derived
* v0.2 block the export wrote beside it. The operator's round-trip is
* "restore the recorded state": a page quarantined before export lands
* quarantined again instead of silently promoted to the enum's
* `stable`, a completion status comes back itself, user-written sources
* survive, and a page that never carried a status (or sources) again
* does not. A raw key restores its native verbatim; a MISSING raw key
* drops its derived value only when the page's marker declared that
* pair (see {@link rawCarryDeclaredNatives}) - a fresh-export plain
* page comes back without the derived status it never had, while a
* legacy marker-bearing page keeps the native `type` its export rode
* verbatim beside a marker that predates the type pair.
*
* The restore runs only on a page that declares raw-carrying: the
* {@link RAW_CARRY_KEY} marker, or - saying the same thing when the
* marker went missing - any present raw key, which no legacy bundle
* ever wrote. A page with NEITHER is a legacy bundle's (exported
* before the raw-carrying declaration existed - sealed knowledge-pack
* halves accepted on purpose included), its natives are the
* frontmatter already in place, and reading their absence as "never
* had a value" deleted them. The restore is skipped for such a page,
* so a legacy quarantine survives its own round-trip; a page that kept
* its raw keys but lost the marker is restored anyway, instead of
* stranding the keys beside the derived block they were written to
* replace.
*/
function restoreRawOkfKeys(meta: FrontmatterMap): void {
const marked =
meta[RAW_CARRY_KEY] !== undefined || RAW_OKF_KEYS.some(({ raw }) => meta[raw] !== undefined);
if (!marked) return;
const declared = rawCarryDeclaredNatives(meta[RAW_CARRY_KEY]);
delete meta[RAW_CARRY_KEY];
for (const { raw, native } of RAW_OKF_KEYS) {
// Presence, not shape: the raw value round-tripped through the page
// file is whatever the export carried, and the restore replaces the
// derived value with it verbatim - a numeric or whitespace status
// included.
if (meta[raw] !== undefined) meta[native] = meta[raw];
else if (declared.has(native)) delete meta[native];
delete meta[raw];
}
}
/**
* Compose the frontmatter a page lands with. Standard fields are taken
* from the bundle file verbatim. This system's machinery keys (they are
* server-stamped, not caller-supplied - see
* {@link FOREIGN_MACHINERY_KEYS}) are stripped unless the operator
* vouched for an own-producer bundle with `--trusted`. The native values
* the export carried under the producer namespace are restored on every
* OWN-producer path - the trusted round-trip and review staging alike,
* where the staged page's `okf_review: pending` already keeps it out of
* the live vault (see {@link restoreRawOkfKeys}). A bundle that declares
* a foreign producer imports verbatim instead, shedding the raw keys so
* no foreign page lands speaking this system's machinery namespace, and
* gets provenance stamped (producer + raw foreign type) without
* clobbering any such key the page already carries - preserving, never
* overwriting, producer-specific metadata. Review-staged pages also get
* `okf_review: pending`.
*/
function importFrontmatter(
page: ParsedOkfPage,
producer: string,
foreign: boolean,
trusted: boolean,
nowIso: string,
): FrontmatterMap {
const meta: FrontmatterMap = { ...page.frontmatter };
if (!trusted || foreign) {
for (const key of FOREIGN_MACHINERY_KEYS) delete meta[key];
}
if (!foreign) {
// The raw keys are this system's namespace, so the restore keys on
// the producer alone - not on the operator's vouch. A forged
// own-producer claim without --trusted still lands in the review
// lane, where restoring the recorded status promotes nothing.
restoreRawOkfKeys(meta);
} else {
// A foreign bundle's pages shed the raw keys and the raw-carry
// marker: honoured on no path, kept on none either - a foreign
// bundle must not plant stray restore machinery into the vault.
for (const { raw } of RAW_OKF_KEYS) delete meta[raw];
delete meta[RAW_CARRY_KEY];
}
if (foreign) {
if (meta[PRODUCER_KEY] === undefined) meta[PRODUCER_KEY] = producer;
// Preserve the raw foreign type: prefer the bundle's recorded
// foreign_type, else the page's declared kind at import time.
if (meta[FOREIGN_TYPE_KEY] === undefined) {
meta[FOREIGN_TYPE_KEY] = page.entry.foreign_type ?? page.entry.kind;
}
meta[IMPORTED_AT_KEY] = nowIso;
}
if (!trusted) meta[REVIEW_KEY] = "pending";
return meta;
}
/**
* Import a parsed bundle into the vault. See {@link OkfImportOptions} for
* the trust gradient. Each target path is funnelled through
* {@link ensureInsideVault} so a hostile recorded path cannot escape the
* vault root. In review mode an existing target is left untouched
* (reported under `skipped`); in trusted mode it is overwritten (the
* round-trip contract is "restore the recorded state").
*/
export function importOkfBundle(
vault: string,
bundle: ParsedOkfBundle,
opts: OkfImportOptions = {},
): OkfImportResult {
// Lane write guard: the vault identity guard on an unswitched machine.
// Every page this import writes lands in the user's vault (OKF Review/
// or a trusted user path), so a store-only machine runs the vault
// adapter's guard once up front too: a frozen or swapped vault is
// refused before the walk, as it is unswitched.
assertLaneWrite(vault);
if (lanePagesInStore(vault)) assertNoteVaultForWrite(vault, "okf import");
const trusted = opts.trusted === true;
const nowIso = isoSecond(opts.now ?? new Date());
const written: string[] = [];
const skipped: string[] = [];
const errors: { path: string; message: string }[] = [];
for (const page of bundle.pages) {
// The lane check runs BEFORE any join: a recorded path that is not a
// clean vault-relative POSIX path (or, in trusted mode, reaches into
// Brain machinery) is reported as an error and never touches disk.
const refusal = okfTargetRefusal(page.entry.path, trusted);
if (refusal !== null) {
errors.push({ path: page.entry.path, message: refusal });
continue;
}
const targetRel = trusted ? page.entry.path : posix.join(OKF_REVIEW_REL, page.entry.path);
let abs: string;
try {
abs = ensureInsideVault(join(vault, targetRel), vault);
} catch (exc) {
errors.push({ path: targetRel, message: (exc as Error).message ?? String(exc) });
continue;
}
if (!trusted && lanePageExists(vault, abs)) {
skipped.push(targetRel);
continue;
}
const meta: FrontmatterMap = {
...importFrontmatter(page, bundle.manifest.producer, bundle.foreign, trusted, nowIso),
...opts.provenance,
};
try {
// overwrite: trusted closes the TOCTOU window in review mode: if a
// file appears between the existence check above and this write,
// the exclusive create throws EEXIST instead of clobbering it. A
// `Brain/` page is a lane record once switched; a user page is a
// file, behind the vault adapter's guard.
writeLanePage(vault, abs, meta, page.body, { overwrite: trusted });
written.push(targetRel);
} catch (exc) {
if (!trusted && (exc as NodeJS.ErrnoException).code === "EEXIST") {
skipped.push(targetRel);
continue;
}
errors.push({ path: targetRel, message: (exc as Error).message ?? String(exc) });
}
}
return {
written,
skipped,
errors,
foreign: bundle.foreign,
mode: trusted ? "trusted" : "review",
};
}