Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Signal (`sig-*.md`) parser and writer.
*
* Signals are the immutable raw input to the dream algorithm. Each one
* is a single Markdown file under `Brain/inbox/` (or `processed/` once
* folded into a preference). The frontmatter shape is fully specified
* in design doc §5.2 and mirrored on the `BrainSignal` interface in
* `./types.ts`.
*
* This module exposes a parser + writer pair:
*
* - `parseSignal(path)` validates required fields, returns a frozen
* `BrainSignal` (mutation throws). Missing required fields surface
* as `Error('signal missing field: <name>')` so the caller knows
* exactly which field is wrong.
*
* - `writeSignal(vault, input, options)` allocates a free slug under
* `Brain/inbox/`, writes the file atomically through `fs-atomic`,
* and returns the resulting path + id. Slug collisions are resolved
* by `allocateAndCreate` (suffixes `-2`, `-3`, …). On a store-only
* machine (task 2.10) no file is written: the signal is its `signal`
* record, its id allocated against the store's signal fold, and the
* returned path names the page it would have been.
*
* The body shape ("## Raw" section with the optional `raw` text) is
* deterministic so two writes with identical input produce byte-for-byte
* identical files — a property exercised by the roundtrip test.
*/
import { dualWrite } from "../store/dual-write.ts";
import {
isPrivateFrontmatter,
signalRecord,
type SignalRawShape,
} from "../store/dual-write-records.ts";
import {
callerInput,
operationPrivacy,
SIGNAL_OPAQUE_FIELDS,
type OperationPrivacy,
} from "../store/operation-privacy.ts";
import { withOperationPrivacy } from "../store/operation-scope.ts";
import { withLocalLock } from "../store/local-lock.ts";
import { storesStateDir } from "../store/store-paths.ts";
import { storeWrite } from "../store/store-write.ts";
import { FileAlreadyExistsError } from "../fs-atomic.ts";
import { processDirsEnv } from "../platform-dirs.ts";
import { signalsStoreOnly, storeSignal } from "./signal-state.ts";
import { memoryPageNamed } from "./memory-pages.ts";
import { existsSync } from "node:fs";
import { join, posix, relative, resolve } from "node:path";
import type { WriteSubject } from "./write-disposition.ts";
import type { FrontmatterMap } from "../types.ts";
import { sanitiseTextField } from "../redactor.ts";
import {
computePayloadHash,
IdempotencyPayloadMismatchError,
lookupKey,
rememberKey,
} from "./idempotency-ledger.ts";
import { sanitisePrinciple } from "./text/sanitize-principle.ts";
import { ROUTE_STAGE, timeStageSync } from "../route-scope.ts";
import {
ORIGIN_CHANNEL_FIELD,
isOriginChannelStamp,
originChannelStamp,
type OriginChannelStamp,
} from "../origin-channel.ts";
import { EXPIRATION_DATE_FIELD, normalizeExpirationDate } from "./expiration.ts";
import {
formatFrontmatter,
parseFrontmatter,
parseFrontmatterText,
slugify,
writeFrontmatterAtomic,
} from "../vault.ts";
import { requireObsidianTagValue, scopeTagValues } from "./tag-syntax.ts";
import { REVIEW_LANE } from "./write-gate.ts";
import { resolveWriteDisposition } from "./write-disposition.ts";
import { compress, expand, CODEC_VERSION } from "./portability/codec.ts";
import { allocateAndCreate, brainDirsForWrite, validateIsoDate, BRAIN_INBOX_REL } from "./paths.ts";
import { BRAIN_PENDING_REL } from "./path-constants.ts";
import { allocateRecordName, deviceTaggedSlug } from "./record-name.ts";
import {
isKnownSchemaToken,
validateSchemaToken,
type BrainSchemaVocabulary,
} from "./schema-vocab.ts";
import {
BRAIN_SIGNAL_SIGN,
BRAIN_SIGNAL_SOURCE_TYPE,
BRAIN_SIGNAL_SOURCE_TYPES,
isBrainSignalSourceType,
type BrainSignal,
type BrainSignalSign,
type BrainSignalSourceType,
} from "./types.ts";
/**
* The legal `source_type` values, rendered for a refusal message. Derived
* from the vocabulary rather than restated, so a new member cannot leave a
* refusal naming a set the guard no longer enforces.
*/
function renderSourceTypes(): string {
return BRAIN_SIGNAL_SOURCE_TYPES.map((v) => `'${v}'`).join(", ");
}
/**
* Read the `origin_channel` frontmatter key back, refusing a value this
* build cannot interpret. `undefined` means the key is absent, which is
* what every signal written before Unit C carries.
*/
function readOriginChannel(meta: FrontmatterMap, path: string): OriginChannelStamp | undefined {
const value = meta[ORIGIN_CHANNEL_FIELD];
if (value === undefined) return undefined;
if (typeof value !== "string") {
throw new Error(`signal field '${ORIGIN_CHANNEL_FIELD}' must be a string (${path})`);
}
const trimmed = value.trim();
if (trimmed === "") return undefined;
if (!isOriginChannelStamp(trimmed)) {
throw new Error(
`signal field '${ORIGIN_CHANNEL_FIELD}' is not a channel this build reads: ` +
`${JSON.stringify(trimmed)} (${path})`,
);
}
return trimmed;
}
/** Filename prefix without the trailing dash, e.g. `sig-2026-05-14`. */
function signalPrefix(date: string): string {
return `sig-${validateIsoDate(date)}`;
}
/**
* Input contract for {@link writeSignal}. Mirrors the design-doc §5.2
* required/optional split. `slug` is the slug stem only (no `sig-` or
* date prefix); the writer composes the final filename.
*
* `created_at` is included so callers can write deterministic
* fixtures; production callers pass `new Date().toISOString()`.
*/
export interface WriteSignalInput {
readonly topic: string;
readonly signal: BrainSignalSign;
readonly agent: string;
readonly principle: string;
readonly created_at: string;
/** Calendar date (UTC) used in the filename and the `created_at`. */
readonly date: string;
/** Slug stem; collision is resolved by `allocateAndCreate` suffixes. */
readonly slug: string;
readonly scope?: string;
readonly source?: ReadonlyArray<string>;
/** Free-form body that follows the frontmatter under "## Raw". */
readonly raw?: string;
/** Optional extra tags merged after the canonical set. */
readonly extraTags?: ReadonlyArray<string>;
/**
* Capture-extension fields (§9 / §16). When `source_type` is omitted
* on the input side, the writer treats the signal as `live` and
* emits NO `source_type` / `brain/source/*` key — the absence of the
* field carries the meaning.
*/
readonly source_type?: BrainSignalSourceType;
readonly schema_type?: string;
/** Normalised payload hash for idempotency (§9 / §16). */
readonly dedup_hash?: string;
/** Session coordinates `<path>#<turn-id>` (§16). */
readonly session_ref?: string;
/**
* Cross-agent shared namespace (t_936a1a61): basename of the origin
* vault, stamped on MIRRORED records only. Absent on primary writes
* so existing signals stay byte-identical.
*/
readonly origin_vault?: string;
/**
* Bi-temporal event-time slots (A2 / t_7526e8d3). Additive and
* optional: absent → the written file is byte-identical to the
* historical path. When supplied they flow into the SAME frontmatter
* keys the read-side (`readBiTemporal`) already parses, so no reader
* change is needed. Backfill / batch-import stamps these with the
* record's ORIGINAL event-time (`valid_from` = when it happened;
* `recorded_at` = the record time recency reconciliation trusts) so
* an imported old session stays historically faithful instead of
* being mis-stamped with the import wall-clock.
*/
readonly valid_from?: string;
readonly recorded_at?: string;
/**
* Conversation-chronology authoring instant (S1 / t_347e8224). The
* transcript turn instant a session-imported signal was authored at,
* carried from `resolveEventInstant`. Additive and optional: absent →
* the written file is byte-identical to the historical path. Stamped
* alongside `valid_from` / `recorded_at` when the import preserves the
* turn's original event-time, so the search layer can expose it and
* break exact hybrid-score ties toward more recent statements.
*/
readonly authored_at?: string;
/**
* Caller-settable expiration (C5 / t_a82b674e). ISO date
* (`YYYY-MM-DD`) or full timestamp. Additive and optional: absent →
* the write is byte-identical to the historical path. When present it
* is validated and stamped into frontmatter; the default read/list
* path drops a signal past this date unless the caller opts into
* `showExpired`. Orthogonal to dream retirement — the file is never
* deleted or moved on expiry.
*/
readonly expiration_date?: string;
/**
* Vault portability suite (v0.22.0). Opt-in: when true and `raw` is
* present, the body is stored through the deterministic codec and a
* `_raw_codec` marker is stamped so `parseSignal` expands it on read.
* Default (absent/false) writes the raw body verbatim - byte-identical.
*/
readonly rawCodec?: boolean;
/**
* Client-supplied idempotency key (C1 / t_213f356b). Additive and
* optional: absent → the write is byte-identical to the historical
* path. When present, the writer hashes the SEMANTIC payload (topic,
* sign, principle, scope, source, raw, capture fields — never the
* timestamp/slug) and consults the idempotency ledger before writing.
* A repeat with the same key + same payload is a deduped no-op; the
* same key + a different payload throws
* {@link IdempotencyPayloadMismatchError} rather than silently
* appending a second, conflicting signal.
*/
readonly idempotency_key?: string;
}
export interface WriteSignalOptions {
/** Maximum distinct names to try; defaults to the allocator's own cap. */
readonly maxSlugAttempts?: number;
/**
* Vault-configured fallback scope (`feedback.default_scope`). Applied
* only when the input carries no explicit non-empty `scope`; an
* explicit per-call scope always wins. Absent: the historical
* scope-less behaviour is byte-identical.
*/
readonly defaultScope?: string;
/**
* Absolute target directory for the signal file (A3 / t_e540b093).
* Absent: writes to `Brain/inbox/` exactly as before - the default path
* stays byte-for-byte identical. Supplied (e.g. `Brain/pending/`) it
* stages the IDENTICAL frontmatter document elsewhere; the caller is
* responsible for passing a directory that resolves inside the vault.
*/
readonly targetDir?: string;
/**
* The credential-minted subject of the caller (write-side-trust, Task
* 7), when the transport resolved one. It is the subject the review
* disposition consults the permissions document under, so a document
* rule that denies the token's agent refuses this write; absent
* (stdio, the CLI) the resolver falls back to the config identity.
* Never consulted when `targetDir` is set: that caller IS the staging
* path, and must not re-enter the gate it feeds.
*/
readonly subject?: WriteSubject;
}
/**
* Resolve the effective signal scope from an explicit per-call value and
* a configured fallback default. Precedence: a non-empty (after trim)
* explicit scope wins; otherwise a non-empty default is used; otherwise
* `undefined` (the signal stays scope-less). The returned value is
* trimmed but otherwise unsanitised — `writeSignal` runs it through the
* same field sanitiser as any explicit scope.
*
* Exported so the CLI / MCP surfaces can compute one effective scope and
* reuse it for a force-confirmed preference write, keeping the signal and
* its preference on the same scope.
*/
export function resolveEffectiveScope(
explicit: string | undefined,
fallback: string | undefined,
): string | undefined {
if (explicit !== undefined && explicit.trim() !== "") return explicit.trim();
if (fallback !== undefined && fallback.trim() !== "") return fallback.trim();
return undefined;
}
export interface WriteSignalResult {
readonly path: string;
readonly id: string;
/**
* True when the write-approval review gate resolved ON for the signals
* lane and the signal was staged into `Brain/pending/` instead of
* `Brain/inbox/` (write-side trust, Task 6). The document is
* byte-for-byte identical either way - staging is purely a change of
* directory - and apply moves it verbatim. False on every direct
* write, and false on a deduped no-op (a dedup staged nothing).
*/
readonly staged: boolean;
/**
* Set to `true` when an `idempotency_key` matched a prior write with an
* identical payload, so this call was a deduped no-op (no new file). The
* returned `path`/`id` point at the ORIGINAL signal. Absent on a normal
* (first) write, so existing callers are unaffected.
*/
readonly deduped?: boolean;
}
export interface ParseSignalOptions {
readonly schemaVocabulary?: BrainSchemaVocabulary;
}
const REQUIRED_INPUT_FIELDS: ReadonlyArray<keyof WriteSignalInput> = [
"topic",
"signal",
"agent",
"principle",
"created_at",
"date",
"slug",
];
/**
* Write a signal atomically. Funnels through `allocateAndCreate` so a
* second call with the same slug receives a `-2` suffix automatically -
* including when the second call is a concurrent process that lost the
* first name between the probe and the create (GitHub #161).
*
* The body is the canonical "## Raw" section. When `raw` is omitted, the
* section is rendered with an `_(not provided)_` placeholder so every
* signal has the same heading structure — consistent with the Pay
* Memory receipt style.
*
* ## `origin_channel` is a SIBLING of `source_type`, not a rename
*
* Every signal written from here carries `origin_channel`: the
* server-derived channel of the process that wrote it
* ({@link originChannelStamp}). It sits beside `source_type` and neither
* field may be folded into the other, because they are different axes
* that only look alike:
*
* | `source_type` | what it says | axis |
* |----------------|-----------------------------------------------|------------|
* | `live` | written by live `brain_feedback` | origin |
* | `inline` | captured from an `@osb` marker in a vault file| origin |
* | `session` | replayed from a session JSONL | origin |
* | `extracted` | mined by a regex over structure | extraction |
* | `auto_extract` | mined by a model reading prose | extraction |
*
* Three of the five name where the content came from and two name how it
* was derived, so `source_type` is a MIXED axis and cannot answer "which
* transport wrote this". `origin_channel` answers only that:
*
* | `origin_channel` | who wrote the record |
* |------------------|-----------------------------------------------|
* | `mcp-tool` | an agent called a tool over the MCP transport |
* | `cli` | an operator or script ran `o2b` |
* | `import` | a bulk replay of records authored elsewhere |
*
* Neither determines the other. A `live` signal is `mcp-tool` when a
* tool call wrote it and `cli` when `o2b brain feedback` did, and
* `extracted` says nothing about the transport at all. Renaming
* `source_type` to the channel spelling was considered and refused: it
* would silently reinterpret every signal already on disk and break the
* fixtures and read-side parsers that carry its meaning.
*/
export function writeSignal(
vault: string | null,
input: WriteSignalInput,
options: WriteSignalOptions = {},
): WriteSignalResult {
// Resolve the effective scope FIRST: an explicit non-empty per-call
// scope wins; otherwise the configured `defaultScope` fallback applies;
// otherwise the field is dropped so the signal stays scope-less and
// byte-identical to the historical no-default path.
const { scope: explicitScope, ...restInput } = input;
const effectiveScope = resolveEffectiveScope(explicitScope, options.defaultScope);
const resolvedInput: WriteSignalInput =
effectiveScope === undefined ? restInput : { ...restInput, scope: effectiveScope };
// Sanitise free-form fields BEFORE the required-field check so an
// input that is purely C0-control characters lands in the
// "missing-field" branch rather than smuggling itself into YAML.
// `topic`, `slug`, `agent`, `date`, `signal` are constrained by
// their own validators (slug rules / ISO date / enum) and are
// already rejected if they carry junk — don't double-process them.
const sanitised = timeStageSync(ROUTE_STAGE.validate, () =>
validateSignalInput(sanitiseSignalInput(resolvedInput)),
);
// One privacy verdict for every store record this write produces (the
// review gate's decision row, the signal and its idempotency receipt),
// judged lazily so dual writes off judge nothing: the frontmatter
// reserving the page as private, or any content field (`principle`,
// `source`, `raw`, ...) naming a private or unresolvable page or carrying
// a `<private>` region, or any field of the caller's input (its
// idempotency key included) doing so. Judged from the frontmatter the
// signal will carry; its final id (a collision suffix) is opaque, so the
// first candidate's id judges the same. Every record written from here
// on, by this call or any writer it calls, carries it
// (`store/operation-scope.ts`).
const candidate = renderSignalDocument(
sanitised,
`${signalPrefix(sanitised.date)}-${sanitised.slug}`,
).metadata;
const privacy = operationPrivacy(vault, {
target: () => isPrivateFrontmatter(candidate),
content: [
{
value: { frontmatter: candidate, raw: sanitised.raw ?? null },
opaque: SIGNAL_OPAQUE_FIELDS,
},
callerInput(input, SIGNAL_OPAQUE_FIELDS),
],
});
return withOperationPrivacy(privacy, () =>
writeJudgedSignal(vault, input, options, resolvedInput, sanitised, privacy),
);
}
function writeJudgedSignal(
vault: string | null,
input: WriteSignalInput,
options: WriteSignalOptions,
resolvedInput: WriteSignalInput,
sanitised: WriteSignalInput,
privacy: OperationPrivacy,
): WriteSignalResult {
// Idempotency consult (C1): when a client key is supplied, hash the
// semantic payload and check the ledger BEFORE allocating a slug or
// touching disk. A prior write with the same key short-circuits here —
// matching payload dedupes (return the original coordinates), a
// differing payload throws rather than writing a conflicting signal.
const idKey = sanitised.idempotency_key?.trim();
let contentHash: string | undefined;
if (idKey) {
contentHash = computePayloadHash(signalPayloadFields(sanitised));
const existing = timeStageSync(ROUTE_STAGE.idempotencyLookup, () => lookupKey(vault, idKey));
if (existing) {
if (existing.contentHash === contentHash) {
const ref = (existing.ref ?? {}) as { id?: string; path?: string };
return {
path: ref.path ? (vault === null ? ref.path : join(vault, ref.path)) : "",
id: ref.id ?? "",
staged: false,
deduped: true,
};
}
throw new IdempotencyPayloadMismatchError(idKey, existing.contentHash, contentHash);
}
}
// With no vault the signal is a store record named against the fold
// (design D14); the store write checks the store identity.
if (vault === null) {
return writeVaultlessSignal(options, resolvedInput, sanitised, privacy, idKey, contentHash);
}
// Write-intent variant (Unit J): a signal write is the path that
// materializes a mis-resolved root, so it asserts the vault identity
// before allocating a filename under it.
const dirs = brainDirsForWrite(vault);
// Review disposition (write-side trust, Tasks 6, 9 and 12). Resolved
// ONLY when the caller named no targetDir: an explicit target is the
// staging path itself (`stagePendingSignal`), and it must never
// re-enter the gate it feeds. With no permissions document the signals
// lane's toggle decides, exactly as before; with one, the document is
// the ONLY gate - a deny throws WriteRefusedError here (before any
// byte exists, one ledger row behind it), an ask stages into
// `Brain/pending/` with byte-identical bytes, and an allow publishes.
// The target is the first candidate's inbox path: exact enough for the
// document's target entries, and the write intent the caller asked
// for even where the allocator's collision suffix moves it. The
// subject is the transport-minted one when a caller threaded it; the
// fallback is the write's own agent field, which every surface that
// gets here has already resolved against its config (a token caller's
// foreign claim was refused before this write was attempted) - letting
// the resolver re-resolve instead would answer for an identity no
// config on this machine declared. Every ungated caller (MCP and CLI
// feedback, inline scan, session import, session lifecycle, session
// checkpoint) inherits the gate here at the chokepoint with no change
// of its own.
const prefix = signalPrefix(sanitised.date);
let gateStaged = false;
if (options.targetDir === undefined) {
const disposition = resolveWriteDisposition(
vault,
REVIEW_LANE.signals,
options.subject ?? { agent: resolvedInput.agent, via: "config" },
{ target: `${BRAIN_INBOX_REL}/${prefix}-${sanitised.slug}.md` },
);
gateStaged = disposition.verdict === "stage";
}
// `staged` says where the signal lands: the review gate's staging, or an
// explicit target that is the pending directory (`stagePendingSignal`).
const landedStaged =
gateStaged ||
(options.targetDir !== undefined && resolve(options.targetDir) === resolve(dirs.pending));
const allocationOptions = {
vault,
targetDir: options.targetDir ?? (gateStaged ? dirs.pending : dirs.inbox),
prefix,
slug: sanitised.slug,
maxAttempts: options.maxSlugAttempts,
};
// Store-only (task 2.10): the signal is a `signal` record and no file is
// written. Its id is allocated against the signal fold (and against any
// legacy page still on disk), under a device-local lock so two writers on
// this machine cannot take the same id.
if (signalsStoreOnly(vault)) {
// The name carries this device's tag: the fold this machine checks is
// only its own view, so another device may mint the same name before
// their shards meet (`deviceMintTag`).
const storeAllocation = { ...allocationOptions, slug: deviceTaggedSlug(sanitised.slug) };
const { allocation: allocated, value: id } = timeStageSync(ROUTE_STAGE.documentWrite, () =>
withLocalLock(signalAllocationLock(), () =>
allocateAndCreate(storeAllocation, (allocation) => {
const signalId = `${prefix}-${allocation.slug}`;
if (storeSignal(signalId) !== undefined) {
throw new FileAlreadyExistsError(allocation.path, { kind: "signal" });
}
const { metadata } = renderSignalDocument(sanitised, signalId);
storeWrite(
() =>
signalRecord({
vault,
path: allocation.path,
id: signalId,
staged: landedStaged,
frontmatter: metadata,
raw: sanitised.raw,
private: privacy(),
}),
{ vault },
);
return signalId;
}),
),
);
rememberSignalKey(vault, idKey, contentHash, sanitised, id, allocated.path, privacy);
return { path: allocated.path, id, staged: gateStaged };
}
// Allocation and creation are one step (#161): a signal write is the
// most contended name in the vault - inline scan, pending drain and
// session import each write many signals per run on the same date and
// often the same topic stem - so losing the name to a concurrent
// writer has to cost this call the next candidate, not the event.
const { allocation: allocated, value: id } = timeStageSync(ROUTE_STAGE.documentWrite, () =>
allocateAndCreate(allocationOptions, (allocation) => {
// The on-disk id always equals the filename basename, including any
// `-2`/`-3` collision suffix — which is why the document is rendered
// per attempt rather than once: a retry that reused the first
// attempt's bytes would ship a file whose `id` names a different
// file. We surface it as the canonical id and duplicate it inside
// the frontmatter so a manual `mv` keeps the link.
const signalId = `${prefix}-${allocation.slug}`;
const { metadata, body } = renderSignalDocument(sanitised, signalId);
writeFrontmatterAtomic(allocation.path, metadata, body, {
overwrite: false,
existsErrorKind: "signal",
vaultForRelativePath: vault,
});
return signalId;
}),
);
const frontmatter = renderSignalDocument(sanitised, id).metadata;
rememberSignalKey(vault, idKey, contentHash, sanitised, id, allocated.path, privacy);
// Brain store dual write (task 1.11): off by default, never fails this write.
dualWrite(
() =>
signalRecord({
vault,
path: allocated.path,
id,
staged: landedStaged,
frontmatter,
raw: sanitised.raw,
private: privacy(),
}),
{ vault },
);
return { path: allocated.path, id, staged: gateStaged };
}
/**
* Record the idempotency key AFTER the signal lands so a future retry
* dedupes. `ref` stores the vault-relative path + id so the dedupe branch
* can return the original coordinates without re-deriving them.
*/
function rememberSignalKey(
vault: string | null,
idKey: string | undefined,
contentHash: string | undefined,
sanitised: WriteSignalInput,
id: string,
path: string,
privacy: OperationPrivacy,
): void {
if (!idKey || contentHash === undefined) return;
timeStageSync(ROUTE_STAGE.idempotencyRemember, () =>
rememberKey(vault, {
key: idKey,
contentHash,
createdAt: sanitised.created_at,
ref: { id, path: vault === null ? path : relative(vault, path) },
private: privacy,
}),
);
}
/**
* The no-vault write: a `signal` record whose id is allocated against the
* signal fold under the device-local allocation lock. The record carries
* the vault-relative path the signal would have had (the inbox, or
* `Brain/pending/` when the review gate stages it), so it reads the same
* as one written on a vault, and the result's `path` is that
* vault-relative name (the convention of `memory-pages.ts` with no vault).
*/
function writeVaultlessSignal(
options: WriteSignalOptions,
resolvedInput: WriteSignalInput,
sanitised: WriteSignalInput,
privacy: OperationPrivacy,
idKey: string | undefined,
contentHash: string | undefined,
): WriteSignalResult {
if (options.targetDir !== undefined) {
throw new Error("writeSignal: an explicit targetDir needs a vault");
}
const prefix = signalPrefix(sanitised.date);
const disposition = resolveWriteDisposition(
null,
REVIEW_LANE.signals,
options.subject ?? { agent: resolvedInput.agent, via: "config" },
{ target: `${BRAIN_INBOX_REL}/${prefix}-${sanitised.slug}.md` },
);
const staged = disposition.verdict === "stage";
const { id, path } = timeStageSync(ROUTE_STAGE.documentWrite, () =>
withLocalLock(signalAllocationLock(), () => {
const { name } = allocateRecordName(
prefix,
deviceTaggedSlug(sanitised.slug),
(candidate) => storeSignal(candidate) !== undefined,
options.maxSlugAttempts,
);
const relPath = posix.join(staged ? BRAIN_PENDING_REL : BRAIN_INBOX_REL, `${name}.md`);
const { metadata } = renderSignalDocument(sanitised, name);
storeWrite(
() =>
signalRecord({
vault: null,
path: relPath,
id: name,
staged,
frontmatter: metadata,
raw: sanitised.raw,
private: privacy(),
}),
{ vault: null },
);
return { id: name, path: relPath };
}),
);
rememberSignalKey(null, idKey, contentHash, sanitised, id, path, privacy);
return { path, id, staged };
}
/**
* Whether a signal write's record still exists, in the coordinates a
* deduped idempotency result carries (the receipt `ref`'s path and id).
* A vault signal is its page; a store-only signal is its record in the
* fold. A signal that stayed alive but MOVED does not answer at its
* original path any more - the dream pass moves inbox pages into
* `Brain/inbox/processed/`, and applying a staged signal moves it out
* of `Brain/pending/` into `Brain/inbox/` - so the id is also looked up
* in those three folders and found there counts as present. False for a record
* `deleteBySource` forgot (archived is not looked up: archived is
* gone), for a receipt with no `ref` (empty id or path), and for
* anything unresolvable - callers use it as positive evidence, so every
* doubt answers "gone".
*/
export function signalRecordExists(
vault: string | null,
path: string,
id: string,
configPath?: string,
): boolean {
if (path === "" || id === "") return false;
if (vault === null || signalsStoreOnly(vault, configPath))
return storeSignal(id, configPath) !== undefined;
if (existsSync(path)) return true;
return (
memoryPageNamed(vault, "inbox", `${id}.md`, { configPath }) !== null ||
memoryPageNamed(vault, "processed", `${id}.md`, { configPath }) !== null ||
memoryPageNamed(vault, "pending", `${id}.md`, { configPath }) !== null
);
}
/** Device-local lock that serialises store-only signal id allocation on this machine. */
function signalAllocationLock(): string {
return join(storesStateDir(processDirsEnv()), "signal-ids");
}
/**
* Render one signal's frontmatter and body for a given id.
*
* Split out of {@link writeSignal} because the id is only known once a
* name has been successfully claimed: the fused allocator may hand this
* function a different id on a second attempt, and the document must
* follow the name rather than the other way round.
*/
function renderSignalDocument(
sanitised: WriteSignalInput,
id: string,
): { readonly metadata: FrontmatterMap; readonly body: string } {
const tags = composeSignalTags(sanitised);
const metadata: FrontmatterMap = {
kind: "brain-signal",
id,
created_at: sanitised.created_at,
tags: [...tags],
topic: sanitised.topic.trim(),
signal: sanitised.signal,
agent: sanitised.agent.trim(),
principle: sanitised.principle.trim(),
};
// Optional fields: keep the file lean. The parser tolerates absence and
// returns `undefined` on the interface — no `_(not provided)_` for
// metadata.
if (sanitised.scope && sanitised.scope.trim()) {
metadata["scope"] = sanitised.scope.trim();
}
if (sanitised.source && sanitised.source.length > 0) {
metadata["source"] = [...sanitised.source];
}
// Capture-extension fields. `source_type: 'live'` is the implicit
// default — we skip writing it so files written by older OSB
// versions and freshly-written live signals stay byte-stable. Only
// `inline` / `session` get surfaced explicitly.
if (
sanitised.source_type !== undefined &&
sanitised.source_type !== BRAIN_SIGNAL_SOURCE_TYPE.live
) {
metadata["source_type"] = sanitised.source_type;
}
// Server-derived origin channel (Unit C), the sibling of `source_type`
// the table on `writeSignal` maps. Emitted ALWAYS, including the
// `unset` literal: unlike `source_type`, absence here already means
// something else - a signal written before the stamp existed - and
// overloading it with "no entry point claimed the process" would make
// the two indistinguishable. It is read off the module, never off the
// input: no caller may name its own channel.
metadata[ORIGIN_CHANNEL_FIELD] = originChannelStamp();
if (sanitised.schema_type?.trim()) {
metadata["schema_type"] = validateSchemaToken(sanitised.schema_type, "schema_type");
}
if (sanitised.dedup_hash && sanitised.dedup_hash.trim()) {
metadata["dedup_hash"] = sanitised.dedup_hash.trim();
}
if (sanitised.session_ref && sanitised.session_ref.trim()) {
metadata["session_ref"] = sanitised.session_ref.trim();
}
if (sanitised.origin_vault && sanitised.origin_vault.trim()) {
metadata["origin_vault"] = sanitised.origin_vault.trim();
}
// Bi-temporal event-time slots (A2): emitted only when supplied, so
// legacy / live writes stay byte-identical. The read-side already
// parses these keys via readBiTemporal.
if (sanitised.valid_from && sanitised.valid_from.trim()) {
metadata["valid_from"] = sanitised.valid_from.trim();
}
if (sanitised.recorded_at && sanitised.recorded_at.trim()) {
metadata["recorded_at"] = sanitised.recorded_at.trim();
}
// Conversation-chronology authoring instant (S1). Emitted only when
// supplied, so legacy / live / non-turn writes stay byte-identical.
if (sanitised.authored_at && sanitised.authored_at.trim()) {
metadata["authored_at"] = sanitised.authored_at.trim();
}
// Caller-settable expiration (C5): validated + emitted only when
// supplied so legacy / live writes stay byte-identical. Reader-side
// support is via parseSignal's optional read + the expiration filter.
if (sanitised.expiration_date && sanitised.expiration_date.trim()) {
metadata[EXPIRATION_DATE_FIELD] = normalizeExpirationDate(sanitised.expiration_date);
}
// Opt-in codec (v0.22.0): store the raw body compressed and stamp a
// `_raw_codec` marker so the reader expands it. Default path is verbatim.
let bodyInput = sanitised;
if (sanitised.rawCodec === true && sanitised.raw) {
// Normalise trailing whitespace first (matching renderSignalBody on the
// verbatim path) so the codec and verbatim paths agree byte-for-byte on
// read; otherwise a trailing whitespace run would survive inside a marker.
const normalised = sanitised.raw.replace(/\s+$/u, "");
bodyInput = { ...sanitised, raw: compress(normalised) };
metadata["_raw_codec"] = CODEC_VERSION;
}
return { metadata, body: renderSignalBody(bodyInput) };
}
/**
* The semantic identity of a signal for idempotency hashing: the fields
* that make two signals "the same content". Timestamps, the calendar
* date, the allocated slug, and portability markers are intentionally
* excluded so a retry with a later wall-clock still dedupes.
*
* `origin_channel` is excluded for the stronger version of that reason:
* it is provenance, not content, and including it would make the same
* signal retried from a different channel collide as an
* {@link IdempotencyPayloadMismatchError} instead of deduping.
*/
function signalPayloadFields(input: WriteSignalInput): Record<string, unknown> {
return {
topic: input.topic.trim(),
signal: input.signal,
agent: input.agent.trim(),
principle: input.principle.trim(),
scope: input.scope?.trim(),
source: input.source ? [...input.source] : undefined,
raw: input.raw,
source_type: input.source_type,
schema_type: input.schema_type?.trim(),
session_ref: input.session_ref?.trim(),
};
}
// ----- Sanitisation --------------------------------------------------------
/**
* Refuse a sanitised signal input that lacks a required field or carries
* an out-of-enum `signal` / `source_type`; returns the input unchanged.
*/
function validateSignalInput(sanitised: WriteSignalInput): WriteSignalInput {
for (const field of REQUIRED_INPUT_FIELDS) {
const value = sanitised[field];
if (value === undefined || value === null || String(value).trim() === "") {
throw new Error(`signal missing field: ${String(field)}`);
}
}
if (
sanitised.signal !== BRAIN_SIGNAL_SIGN.positive &&
sanitised.signal !== BRAIN_SIGNAL_SIGN.negative
) {
throw new Error(
`signal field 'signal' must be 'positive' or 'negative'; got ${JSON.stringify(sanitised.signal)}`,
);
}
if (sanitised.source_type !== undefined && !isBrainSignalSourceType(sanitised.source_type)) {
throw new Error(
`signal field 'source_type' must be one of ${renderSourceTypes()}; got ${JSON.stringify(sanitised.source_type)}`,
);
}
return sanitised;
}
/**
* Hard caps for free-form fields. `principle` is rendered as a
* single-line YAML scalar in frontmatter; `note`/`raw` shapes can
* carry paragraphs. `scope` is a short slug-adjacent tag.
*/
const PRINCIPLE_MAX_LEN = 512;
/** Hard cap on a signal `scope` slug; shared with config validation so a
* configured `feedback.default_scope` cannot diverge from the write rule. */
export const SCOPE_MAX_LEN = 128;
const RAW_MAX_LEN = 4096;
const SOURCE_ITEM_MAX_LEN = 512;
function sanitiseSignalInput(input: WriteSignalInput): WriteSignalInput {
// Principle-specific repair first (leaked tool-call fragments,
// escape-amplified quote chains), then the generic field sanitiser.
const principle = sanitiseTextField(sanitisePrinciple(input.principle), {
maxLen: PRINCIPLE_MAX_LEN,
singleLine: true,
});
const scope = input.scope
? sanitiseTextField(input.scope, {
maxLen: SCOPE_MAX_LEN,
singleLine: true,
})
: input.scope;
const raw = input.raw ? sanitiseTextField(input.raw, { maxLen: RAW_MAX_LEN }) : input.raw;
const source = input.source
? input.source.map((s) =>
sanitiseTextField(s, { maxLen: SOURCE_ITEM_MAX_LEN, singleLine: true }),
)
: input.source;
return {
...input,
principle,
...(scope !== undefined ? { scope } : {}),
...(raw !== undefined ? { raw } : {}),
...(source !== undefined ? { source } : {}),
};
}
/**
* Read a signal file from disk and validate its frontmatter against the
* `BrainSignal` contract. Returns a frozen object — the parsed shape is
* immutable at runtime to match the `readonly` declarations on the type.
*
* Throws `Error('signal missing field: <name>')` for any absent required
* field. `signal` value mismatch (anything outside positive/negative)
* surfaces as a separate, distinguishable error.
*/
export function parseSignal(path: string, options: ParseSignalOptions = {}): BrainSignal {
const [meta, body] = parseFrontmatter(path);
return signalFromPage(meta, body, path, options);
}
/**
* A signal of the store's signal fold (task 2.10) parsed by the same rules
* as its page: the fold's frontmatter and the raw text the `signal` record
* carries, as the page would hold them. `path` names the page the record
* says it would live at.
*/
export function parseStoreSignal(
path: string,
frontmatter: Readonly<Record<string, unknown>>,
raw: string | null,
options: ParseSignalOptions & {
/** What `raw` holds (`signalRawShape` of the record); default the bare text. */
readonly rawShape?: SignalRawShape;
} = {},
): BrainSignal {
// Two raw shapes reach here, named by the record (FORMAT.md, `signal`).
// The bare raw text `writeSignal` captured, or a page the store rebuilt
// already gives, uncompressed, so the codec marker the page would carry
// does not apply to it. A record the migration imported carries the
// page body itself (`## Raw` heading included, still compressed when the
// page had a `_raw_codec`): that body is the page's, read as the page is,
// with its codec marker kept.
const pageBody = raw !== null && options.rawShape === "page";
const { _raw_codec: _codec, ...bare } = frontmatter;
const meta = pageBody ? frontmatter : bare;
let bodyText = "";
if (raw !== null) bodyText = pageBody ? raw : `## Raw\n\n${raw}\n`;
const text = formatFrontmatter(meta as FrontmatterMap, bodyText);
const [parsed, body] = parseFrontmatterText(text);
return signalFromPage(parsed, body, path, options);
}
/**
* {@link parseSignal} over a page already read: its frontmatter and body,
* or the page the store's fold rebuilt (`memory-pages.ts`).
*/
export function parseSignalPage(
meta: FrontmatterMap,
body: string,
path: string,
options: ParseSignalOptions = {},
): BrainSignal {
return signalFromPage(meta, body, path, options);
}
function signalFromPage(
meta: FrontmatterMap,
body: string,
path: string,
options: ParseSignalOptions,
): BrainSignal {
requireField(meta, "kind");
if (meta["kind"] !== "brain-signal") {
throw new Error(
`signal kind must be 'brain-signal'; got ${JSON.stringify(meta["kind"])} (${path})`,
);
}
const id = requireString(meta, "id");
const created_at = requireString(meta, "created_at");
const tags = requireStringArray(meta, "tags");
const topic = requireString(meta, "topic");
const signalValue = requireString(meta, "signal");
if (signalValue !== BRAIN_SIGNAL_SIGN.positive && signalValue !== BRAIN_SIGNAL_SIGN.negative) {
throw new Error(
`signal field 'signal' must be 'positive' or 'negative'; got ${JSON.stringify(signalValue)} (${path})`,
);
}
const agent = requireString(meta, "agent");
const principle = requireString(meta, "principle");
// Optional fields. `scope` is a plain scalar; `source` is an inline
// array of wikilink strings. `raw` is extracted from the body if the
// canonical heading is present.
const scope = readOptionalTrimmedString(meta, "scope", path);
let source: ReadonlyArray<string> | undefined;
if (meta["source"] !== undefined) {
if (!Array.isArray(meta["source"])) {
throw new Error(`signal field 'source' must be an array (${path})`);
}
for (const item of meta["source"]) {
if (typeof item !== "string") {
throw new Error(`signal field 'source' must be an array of strings (${path})`);
}
}
source = [...(meta["source"] as ReadonlyArray<string>)];
}
const rawSection = extractRawSection(body);
// Expand a codec-compressed body iff the signal carries a marker that
// matches the codec version we support (v0.22.0). Signals without the
// marker - i.e. every default-config signal - take the verbatim path
// unchanged. A marker with an unknown version fails fast rather than
// silently misdecoding a future or hand-authored payload.
const rawCodec = meta["_raw_codec"];
if (rawCodec !== undefined && rawCodec !== CODEC_VERSION) {
throw new Error(`signal field '_raw_codec' must be ${JSON.stringify(CODEC_VERSION)} (${path})`);
}
const raw =
rawSection !== undefined && rawCodec === CODEC_VERSION ? expand(rawSection) : rawSection;
// Read the marker back beside the expanded `raw` (additive optional):
// `raw` alone cannot tell a byte-level rewriter that the section on
// disk is the compressed form, and a rewriter that edits the file's
// bytes while its anchor reads come from the expanded text would write
// a line the next parse never reads. A mismatched marker already threw
// above, so true here means exactly "expanded on read, compressed on
// disk".
const storedCodec = rawCodec === CODEC_VERSION && rawSection !== undefined;
// Capture-extension optional fields. Absence stays as `undefined`
// on the returned object — never coerced to a default, so callers
// can distinguish files written by older OSB versions from
// explicit values.
let source_type: BrainSignalSourceType | undefined;
if (meta["source_type"] !== undefined) {
const v = meta["source_type"];
if (typeof v !== "string") {
throw new Error(`signal field 'source_type' must be a string (${path})`);
}
const trimmed = v.trim();
if (trimmed) {
if (!isBrainSignalSourceType(trimmed)) {
throw new Error(
`signal field 'source_type' must be one of ${renderSourceTypes()}; got ${JSON.stringify(trimmed)} (${path})`,
);
}
source_type = trimmed;
}
}
let schema_type: string | undefined;
if (meta["schema_type"] !== undefined) {
schema_type = validateSchemaToken(meta["schema_type"], "schema_type");
const vocab = options.schemaVocabulary;
if (vocab !== undefined && !isKnownSchemaToken(vocab, "signal_types", schema_type)) {
throw new Error(
`schema_type ${JSON.stringify(schema_type)} is not declared in signal_types (${path})`,
);
}
}
const dedup_hash = readOptionalTrimmedString(meta, "dedup_hash", path);
const session_ref = readOptionalTrimmedString(meta, "session_ref", path);
// The write side has emitted `origin_vault` since the shared-namespace
// mirror shipped and nothing read it back, so a mirrored signal arrived in
// the shared vault indistinguishable from one written there
// (a-label-is-not-a-boundary, U5). Absent on every non-mirrored signal,
// which is every signal a single-vault install writes.
const origin_vault = readOptionalTrimmedString(meta, "origin_vault", path);
// Read back rather than left write-only. `origin_vault` shipped with no
// reader at all and had to be given one a release later; a provenance
// field nothing can query is a field nothing can act on. Absent on every
// signal written before Unit C, and those are never rewritten - the
// no-backfill refusal in `src/core/origin-channel.ts`. A value outside
// the vocabulary is refused rather than coerced, for the same reason
// `source_type` refuses one: a channel this build cannot read is not a
// channel it may silently rename.
const origin_channel = readOriginChannel(meta, path);
const result: BrainSignal = {
kind: "brain-signal",
id,
created_at,
tags,
topic,
signal: signalValue as BrainSignalSign,
agent,
principle,
...(scope !== undefined ? { scope } : {}),
...(source !== undefined ? { source } : {}),
...(raw !== undefined ? { raw } : {}),
...(storedCodec ? { rawCodec: true } : {}),
...(source_type !== undefined ? { source_type } : {}),
...(schema_type !== undefined ? { schema_type } : {}),
...(dedup_hash !== undefined ? { dedup_hash } : {}),
...(session_ref !== undefined ? { session_ref } : {}),
...(origin_vault !== undefined ? { origin_vault } : {}),
...(origin_channel !== undefined ? { origin_channel } : {}),
...readBiTemporal(meta, path),
...(readOptionalTrimmedString(meta, "expiration_date", path) !== undefined
? { expiration_date: readOptionalTrimmedString(meta, "expiration_date", path) }
: {}),
};
return Object.freeze(result);
}
/**
* Read the additive bi-temporal slots (`valid_from`, `valid_until`,
* `recorded_at`) from a signal frontmatter map. Returns only the
* slots the file actually carries; absent on legacy files.
*
* Slot values are validated as non-empty strings; an empty / whitespace
* value is rejected (the field is present in the file but carries no
* useful timestamp - that's a corruption worth surfacing).
*/
function readBiTemporal(
meta: ReturnType<typeof parseFrontmatter>[0],
path: string,
): {
readonly valid_from?: string;
readonly valid_until?: string;
readonly recorded_at?: string;
} {
const valid_from = readOptionalTrimmedString(meta, "valid_from", path);
const valid_until = readOptionalTrimmedString(meta, "valid_until", path);
const recorded_at = readOptionalTrimmedString(meta, "recorded_at", path);
return {
...(valid_from !== undefined ? { valid_from } : {}),
...(valid_until !== undefined ? { valid_until } : {}),
...(recorded_at !== undefined ? { recorded_at } : {}),
};
}
/**
* Read an optional string frontmatter field, trimmed. Returns `undefined`
* when the key is absent or trims to empty; throws if present but not a
* string (a wrong-typed value is a corruption worth surfacing, not
* something to silently drop). Shared by every optional-string signal
* field (`scope`, `dedup_hash`, `session_ref`) and the bi-temporal slots.
*/
function readOptionalTrimmedString(
meta: ReturnType<typeof parseFrontmatter>[0],
key: string,
path: string,
): string | undefined {
const v = meta[key];
if (v === undefined) return undefined;
if (typeof v !== "string") {
throw new Error(`signal field '${key}' must be a string (${path})`);
}
const trimmed = v.trim();
return trimmed.length > 0 ? trimmed : undefined;
}
// ----- Helpers --------------------------------------------------------------
/**
* Build the canonical tag set: `brain`, `brain/signal`, plus per-topic
* and (optionally) per-scope tags. Any `extraTags` supplied by the
* caller are appended after dedup. We preserve insertion order so the
* file is byte-stable across writes with identical input.
*
* Every composed value obeys the ONE shared tag rule (`src/core/tags.ts`,
* via `tag-syntax.ts`): the topic segment is slugified - the same
* treatment the filename slug gets - and refused only if even the slug
* fails the rule; scope segments and `extraTags` entries are
* caller-controlled input and are refused unrewritten, with the field
* named (`TagSyntaxError`).
*/
function composeSignalTags(input: WriteSignalInput): string[] {
const out: string[] = [];
const seen = new Set<string>();
const push = (t: string): void => {
if (!seen.has(t)) {
seen.add(t);
out.push(t);
}
};
push("brain");
push("brain/signal");
push(`brain/topic/${requireObsidianTagValue(slugify(input.topic.trim()), "topic")}`);
for (const scope of scopeTagValues(input.scope ?? "")) {
push(`brain/scope/${scope}`);
}
// Non-default source_type gets its own tag so Obsidian users can
// filter `tag:brain/source/inline` etc. `live` is the implicit
// default and stays tag-less.
if (input.source_type !== undefined && input.source_type !== BRAIN_SIGNAL_SOURCE_TYPE.live) {
push(`brain/source/${input.source_type}`);
}
for (const t of input.extraTags ?? []) {
if (t.trim()) push(requireObsidianTagValue(t.trim(), "extraTags"));
}
return out;
}
/**
* Render the body. The "## Raw" section is emitted **only** when the
* caller actually provided a verbatim quote. A signal recorded without
* `raw` (the common case for tests and corner runs) used to ship a
* `_(not provided)_` placeholder; v0.10.1 drops that placeholder so
* the file is honest about its contents — no body at all when there is
* none. Parsers stay tolerant of both shapes (old placeholder and new
* absent section).
*/
function renderSignalBody(input: WriteSignalInput): string {
if (!input.raw || !input.raw.trim()) return "";
// Normalise line endings and trailing whitespace so two callers
// passing semantically-equal text produce byte-identical output.
const body = input.raw.replace(/\r\n?/g, "\n").replace(/\s+$/g, "");
return ["## Raw", "", body].join("\n");
}
function extractRawSection(body: string): string | undefined {
// The signal body is exactly one "## Raw" section. Capture everything
// after it until EOF (no further headings in this schema). Returning
// `undefined` for the placeholder keeps the type shape clean.
//
// No `m` flag: with multiline mode `$` matches end-of-line, which
// truncates the lazy `[\s\S]*?` capture at the first newline instead
// of running to EOF. We anchor the heading at start-of-string with an
// explicit `(?:^|\n)` and let `$` mean end-of-string only.
const match = /(?:^|\n)##\s+Raw\s*\n+([\s\S]*?)\s*$/.exec(body);
if (!match) return undefined;
const raw = match[1]?.trim();
if (!raw || raw === "_(not provided)_") return undefined;
return raw;
}
function requireField(meta: Record<string, unknown>, field: string): void {
if (!(field in meta) || meta[field] === undefined || meta[field] === null) {
throw new Error(`signal missing field: ${field}`);
}
if (typeof meta[field] === "string" && (meta[field] as string).trim() === "") {
throw new Error(`signal missing field: ${field}`);
}
}
function requireString(meta: Record<string, unknown>, field: string): string {
requireField(meta, field);
const v = meta[field];
if (typeof v !== "string") {
throw new Error(`signal field '${field}' must be a string`);
}
return v;
}
function requireStringArray(meta: Record<string, unknown>, field: string): ReadonlyArray<string> {
requireField(meta, field);
const v = meta[field];
if (!Array.isArray(v)) {
throw new Error(`signal field '${field}' must be an array`);
}
for (const item of v) {
if (typeof item !== "string") {
throw new Error(`signal field '${field}' must be an array of strings`);
}
}
return [...(v as ReadonlyArray<string>)];
}