Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { readFileSync, existsSync } from 'fs';
import { join } from 'path';
/**
* Storage tier configuration loaded from gbrain.yml.
*
* The canonical key names are `db_tracked` and `db_only` (engine-agnostic).
* The deprecated keys `git_tracked` and `supabase_only` are still read for
* backward compatibility but emit a once-per-process deprecation warning.
* Sunset: future release will reject the deprecated names.
*/
export interface StorageConfig {
db_tracked: string[];
db_only: string[];
/**
* #6696 C7: opt-in policy, present only when declared. Drifted pages under these prefixes are reconciled with
* the database copy by the trusted local maintenance worker on the canonical owner host (repair/keep-db-policy.ts).
*/
keep_db?: string[];
/**
* #8007/#8015: the directories of a deprecated `supabase_only` list that sat next to canonical keys and was ignored
* (canonical wins), minus those `db_only` already covers. Whether a page under one is database-only is ambiguous, so
* a caller deciding whether its missing file deletes it holds it instead (`dbOnlyVerdict`).
*/
ignored_db_only?: string[];
}
export type StorageTier = 'db_tracked' | 'db_only' | 'unspecified';
/** Recognized YAML keys (canonical and deprecated). */
const STORAGE_KEYS = new Set([
'db_tracked', 'db_only', 'keep_db',
'git_tracked', 'supabase_only', // deprecated aliases
]);
const CANONICAL_STORAGE_KEYS = ['db_tracked', 'db_only', 'keep_db'];
/**
* Parse the gbrain.yml shape: a top-level `storage:` section with up to five
* array-valued nested keys (canonical `db_tracked` / `db_only`, the opt-in
* `keep_db` policy, and the deprecated aliases `git_tracked` /
* `supabase_only`). Any other nested key is collected into `unknown`, which
* `loadStorageConfig` warns about once per key and process.
*
* Intentionally narrow. Does NOT handle the full YAML spec — only the file
* shape gbrain controls. Trades expressiveness for zero-dep parsing and
* predictable behavior. Returns null if the file has no `storage:` section
* (so callers can distinguish "no config" from "empty config").
*
* Replaces gray-matter, which silently returned `{data: {}}` on
* delimiter-less YAML and broke the entire feature on every install.
* The defect that prompted this rewrite: storage-config.ts:24 in the
* pre-v0.22.3 implementation.
*
* Returns the raw key map. The caller (loadStorageConfig) is responsible
* for normalizing deprecated keys → canonical, emitting deprecation
* warnings, and merging if both old and new keys appear.
*/
type RawStorage = {
db_tracked?: string[];
db_only?: string[];
keep_db?: string[];
git_tracked?: string[];
supabase_only?: string[];
};
/**
* The values of a one-line flow list (`[]`, `[dir/, 'other/']`), or null when `text` is not one the parser resolves: not a
* flow list, a nested list, or an unquoted value holding whitespace (a missing comma: `[a/ b/]`, #8015).
*/
function parseFlowList(text: string): string[] | null {
const flow = /^\[(.*)\]$/.exec(text);
if (!flow || /[[\]]/.test(flow[1]!)) return null;
const tokens = [...flow[1]!.matchAll(/\s*("[^"]*"|'[^']*'|[^,]*)\s*(?:,|$)/g)].map((m) => m[1]!.trim());
if (tokens.some((token) => !/^["']/.test(token) && /\s/.test(token))) return null;
return tokens.map((token) => token.replace(/^["']|["']$/g, '')).filter(Boolean);
}
function parseStorageYaml(content: string, unknown: string[] = []): RawStorage | null {
const lines = content.split('\n').map((line) => line.replace(/\r$/, ''));
let inStorage = false;
let currentList: keyof RawStorage | null = null;
const raw: RawStorage = {};
let sawStorage = false;
for (const line of lines) {
// Strip comments. Conservative: drop trailing `# ...` and full-line `#`.
const noComment = line.replace(/\s+#.*$/, '').replace(/^#.*$/, '');
if (noComment.trim() === '') continue;
// Top-level key (no leading whitespace).
if (!noComment.startsWith(' ') && !noComment.startsWith('\t')) {
const colon = noComment.indexOf(':');
if (colon === -1) continue;
const key = noComment.slice(0, colon).trim();
if (key === 'storage') {
inStorage = true;
sawStorage = true;
currentList = null;
continue;
}
inStorage = false;
currentList = null;
continue;
}
if (!inStorage) continue;
const indented = noComment.replace(/^\s+/, '');
if (indented.startsWith('-')) {
if (!currentList) continue;
const value = indented.slice(1).trim().replace(/^["']|["']$/g, '');
if (value) {
if (!raw[currentList]) raw[currentList] = [];
raw[currentList]!.push(value);
}
continue;
}
const colon = indented.indexOf(':');
if (colon === -1) continue;
const key = indented.slice(0, colon).trim();
if (STORAGE_KEYS.has(key)) {
currentList = key as keyof RawStorage;
const values = parseFlowList(indented.slice(colon + 1).trim());
if (values) raw[currentList] = [...(raw[currentList] ?? []), ...values];
continue;
}
if (!unknown.includes(key)) unknown.push(key);
currentList = null;
}
if (!sawStorage) return null;
return raw;
}
/**
* Normalize raw parsed keys into canonical StorageConfig shape.
*
* Resolution order (per plan eng-review pass 2 finding #2):
* 1. If canonical keys present, use them.
* 2. Else if deprecated keys present, map to canonical AND emit a
* once-per-process deprecation warning asking for a rename by hand.
* 3. If both are present, canonical wins. Deprecated keys are ignored
* with a stronger warning (the user is mid-migration).
*
* Validation (validateStorageConfig) always runs against the canonical
* shape, so error messages reference `db_only` / `db_tracked` regardless
* of which keys the user wrote.
*/
let _deprecationWarned = false;
function normalizeStorageConfig(raw: RawStorage): StorageConfig {
const hasCanonical = Boolean(raw.db_tracked || raw.db_only);
const hasDeprecated = Boolean(raw.git_tracked || raw.supabase_only);
if (hasDeprecated && !_deprecationWarned) {
_deprecationWarned = true;
const which = [
raw.git_tracked ? '`git_tracked`' : null,
raw.supabase_only ? '`supabase_only`' : null,
].filter(Boolean).join(' and ');
if (hasCanonical) {
console.warn(
`Warning: ${which} in gbrain.yml is deprecated and ignored ` +
`(canonical keys db_tracked/db_only are present). ` +
`Move their directories under db_tracked / db_only by hand and remove them (see docs/storage-tiering.md).`,
);
} else {
console.warn(
`Warning: ${which} in gbrain.yml is deprecated. ` +
`Rename them to db_tracked / db_only by hand — see docs/storage-tiering.md.`,
);
}
}
const keep = raw.keep_db ? { keep_db: raw.keep_db } : {};
if (hasCanonical) {
return {
db_tracked: raw.db_tracked ?? [],
db_only: raw.db_only ?? [],
...keep,
...(raw.supabase_only?.length ? { ignored_db_only: raw.supabase_only } : {}),
};
}
return {
db_tracked: raw.git_tracked ?? [],
db_only: raw.supabase_only ?? [],
...keep,
};
}
/**
* Load gbrain.yml configuration from the brain repository root.
*
* Returns null when:
* - repoPath is null/undefined
* - gbrain.yml doesn't exist at the repo root
* - gbrain.yml exists but has no `storage:` section (with sanity warning)
*
* Throws when:
* - gbrain.yml exists but is unreadable (permission denied, etc.) — D36 lock:
* fail loud rather than silently disable the feature.
*
* Logs a console.warn (once per process) when:
* - File parses but `storage:` section is empty or missing — Issue #1 lock:
* surface "your config didn't take" rather than silently no-op.
*/
let _missingStorageWarned = false;
const _unknownKeysWarned = new Set<string>();
export function loadStorageConfig(repoPath?: string | null): StorageConfig | null {
if (!repoPath) return null;
const yamlPath = join(repoPath, 'gbrain.yml');
if (!existsSync(yamlPath)) return null;
// Read failure is a real error (not a "feature not configured" signal).
// Throwing here lets the caller decide whether to crash or fall back.
const content = readFileSync(yamlPath, 'utf-8');
return storageConfigFromYaml(content, yamlPath);
}
export class StorageConfigError extends Error {
constructor(message: string) {
super(message);
this.name = 'StorageConfigError';
}
}
/**
* Validate storage configuration for conflicts and issues.
* Returns warning strings; callers decide how to surface them.
*
* Always runs against the canonical (db_tracked / db_only) shape — error
* messages reference canonical names regardless of which keys the user
* wrote in gbrain.yml.
*
* Pure: does not mutate. For the auto-normalize behavior (D7), see
* `normalizeAndValidateStorageConfig` below.
*/
export function validateStorageConfig(config: StorageConfig): string[] {
const warnings: string[] = [];
const trackedSet = new Set(config.db_tracked);
for (const path of config.db_only) {
if (trackedSet.has(path)) {
warnings.push(`Directory "${path}" appears in both db_tracked and db_only`);
}
}
const allPaths = [...config.db_tracked, ...config.db_only];
for (const path of allPaths) {
if (!path.endsWith('/')) {
warnings.push(`Directory path "${path}" should end with "/" for consistency`);
}
}
return warnings;
}
/**
* Auto-normalize and strict-validate per D7+D8.
*
* 1. Cosmetic fixups are applied silently with a one-time info message
* naming what changed:
* - missing trailing `/` is added
* The message helps the user learn the canonical form without nagging.
* 2. Semantic problems THROW (don't return warnings):
* - same directory in both tiers (ambiguous routing)
*
* Caller passes a fresh raw config; this returns the normalized shape that
* the rest of the code (matcher, sync, etc.) sees.
*/
let _normalizationInfoEmitted = false;
export function normalizeAndValidateStorageConfig(input: StorageConfig): StorageConfig {
const normalize = (paths: string[]): { normalized: string[]; changed: string[] } => {
const normalized: string[] = [];
const changed: string[] = [];
for (const p of paths) {
if (p.endsWith('/')) {
normalized.push(p);
} else {
normalized.push(p + '/');
changed.push(`"${p}" → "${p}/"`);
}
}
return { normalized, changed };
};
const tracked = normalize(input.db_tracked);
const dbonly = normalize(input.db_only);
const keep = input.keep_db ? normalize(input.keep_db) : null;
const allChanged = [...tracked.changed, ...dbonly.changed, ...(keep?.changed ?? [])];
if (allChanged.length > 0 && !_normalizationInfoEmitted) {
_normalizationInfoEmitted = true;
console.warn(
`Note: normalized ${allChanged.length} storage path(s) in gbrain.yml — ` +
`${allChanged.join(', ')}. Add trailing "/" to suppress this note.`,
);
}
// Semantic check: overlap between tiers throws. Ambiguous routing.
const trackedSet = new Set(tracked.normalized);
for (const path of dbonly.normalized) {
if (trackedSet.has(path)) {
throw new StorageConfigError(
`gbrain.yml: directory "${path}" appears in both db_tracked and db_only — ` +
`pick one tier. Edit gbrain.yml to remove the overlap.`,
);
}
}
return { db_tracked: tracked.normalized, db_only: dbonly.normalized, ...(keep ? { keep_db: keep.normalized } : {}),
...ignoredDbOnly(input.ignored_db_only, dbonly.normalized) };
}
/**
* Path-segment match: a slug belongs to a tier directory iff the directory
* is a complete path-segment ancestor of the slug. `media/x/` matches
* `media/x/foo` but NOT `media/xerox/foo` — eliminates the prefix-collision
* class of bug (Issue #5 of the eng review, D6 lock).
*
* Strict: requires the configured directory to end with `/`. The validator
* (per D7+D8) auto-normalizes input so the matcher only ever sees canonical
* trailing-`/` directories.
*/
function matchesTierDir(slug: string, dir: string): boolean {
if (!dir.endsWith('/')) return false; // not normalized — matcher refuses
// slug must equal dir's bare prefix OR start with the trailing-slash form.
// Example: dir = 'media/x/' matches 'media/x/anything' but not 'media/x'
// or 'media/xerox'. (A slug that exactly equals 'media/x' is a directory-
// level entry the brain doesn't write.)
return slug.startsWith(dir);
}
export function isDbTracked(slug: string, config: StorageConfig): boolean {
return config.db_tracked.some((dir) => matchesTierDir(slug, dir));
}
export function isDbOnly(slug: string, config: StorageConfig): boolean {
return config.db_only.some((dir) => matchesTierDir(slug, dir));
}
/**
* Declared db_only match for a page slug. Slugs are always lowercase, so the
* declared directory is lowercased before matching (same reasoning as
* `effectiveDbOnlyDirs`, issue #3766). Only declared dirs count: the
* derive-phase defaults may be file-backed on some brains.
*/
function ignoredDbOnly(ignored: string[] | undefined, dbOnly: string[]): { ignored_db_only?: string[] } {
const uncovered = (ignored ?? []).map((dir) => (dir.endsWith('/') ? dir : dir + '/'))
.filter((dir) => !dbOnly.some((declared) => dir.toLowerCase() === declared.toLowerCase() || matchesTierDir(dir.toLowerCase(), declared.toLowerCase())));
return uncovered.length ? { ignored_db_only: uncovered } : {};
}
/**
* #8007/#8015: whether a page slug is database-only under this config, for a delete decision: true under a declared
* db_only dir, `unknown` under an ignored deprecated `supabase_only` dir that db_only does not cover, false otherwise.
*/
export function dbOnlyVerdict(slug: string, config: StorageConfig | null): boolean | 'unknown' {
if (isDeclaredDbOnlySlug(slug, config)) return true;
return config?.ignored_db_only?.some((dir) => matchesTierDir(slug, dir.toLowerCase())) ? 'unknown' : false;
}
export function isDeclaredDbOnlySlug(slug: string, config: StorageConfig | null): boolean {
return config?.db_only.some((dir) => matchesTierDir(slug, dir.toLowerCase())) ?? false;
}
/** #6696 C7: whether a page slug sits under a declared `keep_db` prefix (case-insensitive, like `isDeclaredDbOnlySlug`). */
export function isKeepDbSlug(slug: string, keepDb: readonly string[]): boolean {
return keepDb.some((dir) => matchesTierDir(slug, dir.toLowerCase()));
}
/**
* True when gbrain.yml has a `db_only` (or deprecated `supabase_only`) key
* line but no db_only directory resolved from it. `loadStorageConfig` warns
* and resolves nothing for syntax the narrow parser does not handle (e.g. a
* flow list split over several lines, or a key outside the storage section),
* so a caller that must not guess a tier refuses instead. Only YAML key lines
* count, not comments or prose that mention the word; an explicit empty
* `db_only: []` declares no directory and does not count.
*/
export function hasUnresolvedDbOnlyDeclaration(repoPath: string, config: StorageConfig | null): boolean {
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- repoPath is the registered local source root and the file name is a constant; this only reads its gbrain.yml
const yamlPath = join(repoPath, 'gbrain.yml');
return unresolvedDbOnlyIn(existsSync(yamlPath) ? readFileSync(yamlPath, 'utf-8') : '', config);
}
/**
* `hasUnresolvedDbOnlyDeclaration` over gbrain.yml's content. #8015: a db_only or supabase_only key whose inline value
* the parser refuses counts even when another key resolved, so a partly read declaration never answers "not db_only".
*/
export function unresolvedDbOnlyIn(content: string, config: StorageConfig | null): boolean {
const keys = content.split('\n').flatMap((line) => {
const trimmed = line.trim();
const key = trimmed.startsWith('#') ? null : /^(db_only|supabase_only)\s*:(.*)$/.exec(trimmed.replace(/\s+#.*$/, ''));
return key ? [key[2]!.trim()] : [];
});
if (keys.some((value) => value !== '' && parseFlowList(value) === null)) return true;
if (config && config.db_only.length > 0) return false;
return keys.some((value) => value.replace(/\s+/g, '') !== '[]');
}
/** gbrain.yml content as `loadStorageConfig` reads it (`label` names the file in warnings); throws on overlapping tiers. */
export function storageConfigFromYaml(content: string, yamlPath: string): StorageConfig | null {
let raw: RawStorage | null;
const unknown: string[] = [];
try {
raw = parseStorageYaml(content, unknown);
} catch (error) {
console.warn(
`Warning: Failed to parse gbrain.yml: ${error instanceof Error ? error.message : String(error)}`,
);
return null;
}
// No storage section at all → null (with sanity warning).
if (raw === null) {
if (!_missingStorageWarned) {
_missingStorageWarned = true;
console.warn(
`Warning: ${yamlPath} exists but has no storage configuration. ` +
`Add a "storage:" section with db_tracked / db_only arrays, ` +
`or remove gbrain.yml to suppress this warning.`,
);
}
return null;
}
for (const key of unknown.filter(key => !_unknownKeysWarned.has(key))) {
_unknownKeysWarned.add(key);
console.warn(`Warning: ${yamlPath}: unknown storage key '${key}' is ignored; storage takes ${CANONICAL_STORAGE_KEYS.join(', ')} (see docs/storage-tiering.md).`);
}
const merged = normalizeStorageConfig(raw);
// Empty storage section → return as-is but warn.
if (merged.db_tracked.length === 0 && merged.db_only.length === 0 && !merged.keep_db?.length) {
if (!_missingStorageWarned) {
_missingStorageWarned = true;
console.warn(
`Warning: ${yamlPath} exists but has no storage configuration. ` +
`Add a "storage:" section with db_tracked / db_only arrays, ` +
`or remove gbrain.yml to suppress this warning.`,
);
}
return merged;
}
// Normalize cosmetic issues + throw on semantic overlap (D7).
// Throws StorageConfigError on overlap — propagates to the caller.
return normalizeAndValidateStorageConfig(merged);
}
/**
* Derive-phase output prefixes the engine itself writes as DB-only machine
* output (issue #2784, reported by @alexputici). These are re-derivable by
* design and rarely file-backed, so the `undeclared_db_only_pages` doctor
* check treats them as implicitly declared db_only. They are deliberately
* NOT merged into `loadStorageConfig` — doing so would auto-gitignore these
* dirs via `manageGitignore` and silently kill ingestion for brains that DO
* file-back them (the exact #2788 silent-death class).
*/
export const DERIVE_PHASE_DB_ONLY_DEFAULTS: readonly string[] = [
'life/events/',
'atoms/',
// synthesize_concepts writes through importFromContent without creating a
// backing markdown file. Keep this implicit (rather than merging it into
// storage.db_only) so installations that do file-back concepts are never
// auto-gitignored by manageGitignore.
'concepts/',
// Extract receipts (extract/receipt-writer.ts): derived bookkeeping, never content; the writer refuses a slug that holds content or a file.
'extracts/',
'dream-cycle-summaries/',
// The drift phase's nightly report (cycle/drift.ts) publishes without a
// file; only that slug shape, so the rest of a user's reports/ keeps its files.
'reports/drift-',
];
/**
* Declared db_only dirs plus the derive-phase defaults, deduped.
*
* `declared` is lowercased before the union (issue #3766): its only consumer
* (`checkUndeclaredDbOnlyPages`) matches these prefixes against page `slug`
* values via a plain `.startsWith()`, and slugs are ALWAYS lowercased at
* creation time (`pathToSlug`/`slugifyCodePath` in sync.ts) regardless of the
* host filesystem's case sensitivity. Without this, a `storage.db_only`
* entry typed with any uppercase (e.g. `Notes/` in gbrain.yml) silently never
* matches a single slug and every page under it gets falsely flagged as
* undeclared. This is deliberately NOT done in `normalizeAndValidateStorageConfig`
* / `loadStorageConfig` — `manageGitignore` (sync.ts) reads that raw,
* case-preserved config to write `.gitignore` entries that must match the
* REAL on-disk directory name (case-sensitive on Linux); lowercasing there
* would break gitignore management instead of fixing this doctor check.
*/
export function effectiveDbOnlyDirs(declared: string[]): string[] {
return [...new Set([...declared.map((d) => d.toLowerCase()), ...DERIVE_PHASE_DB_ONLY_DEFAULTS])];
}
/**
* Collector-output vs db_only collision detection (issue #2788, reported by
* @alexputici). A collector output path collides when it equals a db_only
* dir or sits anywhere inside one — such dirs are auto-gitignored by sync,
* so both the git-walking sync AND `gbrain import` (which honors .gitignore)
* silently skip every file the collector writes.
*/
export function findDbOnlyCollisions(
outputs: Array<{ id: string; output_path: string }>,
dbOnlyDirs: string[],
): Array<{ id: string; output_path: string; db_only_dir: string }> {
const hits: Array<{ id: string; output_path: string; db_only_dir: string }> = [];
for (const o of outputs) {
const out = o.output_path.endsWith('/') ? o.output_path : o.output_path + '/';
for (const rawDir of dbOnlyDirs) {
const dir = rawDir.endsWith('/') ? rawDir : rawDir + '/';
if (out.startsWith(dir)) {
hits.push({ id: o.id, output_path: o.output_path, db_only_dir: rawDir });
break;
}
}
}
return hits;
}
export function getStorageTier(slug: string, config: StorageConfig): StorageTier {
if (isDbTracked(slug, config)) return 'db_tracked';
if (isDbOnly(slug, config)) return 'db_only';
return 'unspecified';
}
// ── Deprecated aliases — to be removed in a future release ────────
// Kept so existing callers (storage.ts, export.ts) compile during the
// step-by-step refactor. Will be deleted once those call sites migrate
// to the canonical names.
export const isGitTracked = isDbTracked;
export const isSupabaseOnly = isDbOnly;
/** Reset once-per-process warning flags. Test-only. */
export function __resetMissingStorageWarning(): void {
_missingStorageWarned = false;
_unknownKeysWarned.clear();
_deprecationWarned = false;
_normalizationInfoEmitted = false;
}