A self-organizing Obsidian vault that gives AI coding agents persistent memory.
TypeScript ai-agentsclaude-codecodex-cligemini-cliknowledge-managementobsidianobsidian-vaultpersistent-memorysecond-braintemplate
#!/usr/bin/env node
/**
* qmd-bootstrap.ts — idempotent QMD setup for this vault.
*
* Run once on a fresh clone (new machine, wiped cache, rebuilt index) to:
*
* 1. Register this vault as a QMD collection under the named index declared
* in vault-manifest.json (`qmd_index`).
* 2. Attach the vault's context string (`qmd_context`) so QMD's snippets
* and rerank step know what this collection is.
* 3. Sync `.obsidian/app.json` userIgnoreFilters into the QMD YAML config
* so both engines hide the same files from search.
* 4. Walk the vault and build the sparse index.
* 5. Generate vector embeddings.
*
* Safe to re-run. Every step reports current state rather than failing on
* "already exists" — the context string is re-attached so updates to
* vault-manifest.json propagate. The SQLite store itself lives in
* ~/.cache/qmd/<index>.sqlite (derived data, not version-controlled), which
* is why this script is the portable instruction set for regenerating it.
*
* Cross-platform: every spawn routes through `buildQmdCommand`, which resolves
* @tobilu/qmd's real JS entry and runs it with the current Node binary. No
* platform conditionals — the same command path executes on Windows, macOS,
* and Linux.
*
* Usage:
* node --experimental-strip-types .scripts/qmd-bootstrap.ts
*/
import { spawnSync, type SpawnSyncOptionsWithStringEncoding, type SpawnSyncReturns } from "node:child_process";
import { readFileSync } from "node:fs";
import { warn } from "../.claude/scripts/lib/hook-io.ts";
import { isMainModule } from "../.claude/scripts/lib/main-guard.ts";
import {
buildCollectionAddArgs,
isContextRemoveBenign,
isUnknownSubcommandFailure,
legacyCollectionCandidate,
makeCollectionAddBenignMatcher,
} from "../.claude/scripts/lib/qmd-bootstrap.ts";
import {
buildQmdCommand,
qmdVersionAtLeast,
resolveQmdEntry,
} from "../.claude/scripts/lib/qmd.ts";
import {
qmdConfigPath,
readObsidianIgnore,
translateToGlob,
writeQmdIgnore,
} from "../.claude/scripts/lib/qmd-ignore.ts";
import { writeQmdEmbedModel } from "../.claude/scripts/lib/qmd-models.ts";
import {
deriveQmdIndex,
isValidQmdIndex,
resolveQmdIndex,
} from "../.claude/scripts/lib/session-start.ts";
type ManifestSubset = {
readonly qmd_index?: string;
readonly qmd_context?: string;
readonly qmd_min_version?: string;
readonly template?: string;
};
function readManifestRaw(): string | null {
try {
return readFileSync("vault-manifest.json", { encoding: "utf-8" });
} catch {
return null; /* handled by caller */
}
}
function parseManifest(raw: string | null): ManifestSubset | null {
if (raw === null) return null;
try {
const parsed = JSON.parse(raw) as unknown;
if (parsed !== null && typeof parsed === "object") {
return parsed as ManifestSubset;
}
} catch {
/* handled by caller */
}
return null;
}
type SpawnOutcome = {
readonly status: number | null;
readonly signal: NodeJS.Signals | null;
readonly stdout: string;
readonly stderr: string;
};
/**
* Run qmd and capture both streams so callers can classify failures.
* Captured output is echoed to the user afterward so the visible log matches
* `stdio: "inherit"` ordering. `spawn` is a seam for tests, which check the
* options without starting qmd.
*/
export function spawnQmd(
entry: string | null,
subcommandArgs: readonly string[],
spawn: (
cmd: string,
args: string[],
options: SpawnSyncOptionsWithStringEncoding,
) => Pick<SpawnSyncReturns<string>, "status" | "signal" | "stdout" | "stderr"> = spawnSync,
): SpawnOutcome {
const { cmd, args, shell } = buildQmdCommand(entry, subcommandArgs);
// windowsHide: session-start starts this bootstrap detached, so it has no
// console, and on Windows a console program started from it without the
// flag opens a console window of its own (#381).
const r = spawn(cmd, args as string[], { shell, encoding: "utf-8", windowsHide: true });
return {
status: r.status,
signal: r.signal,
stdout: r.stdout ?? "",
stderr: r.stderr ?? "",
};
}
function echo(outcome: SpawnOutcome): void {
if (outcome.stdout) process.stdout.write(outcome.stdout);
if (outcome.stderr) process.stderr.write(outcome.stderr);
}
/** Verify qmd is installed; returns the raw `--version` output for the
* min-version gate in main(). */
function ensureQmd(entry: string | null): string {
const probe = spawnQmd(entry, ["--version"]);
if (probe.status !== 0) {
process.stderr.write(
"qmd not found. Install it first: npm i -g @tobilu/qmd\n",
);
process.exit(1);
}
return probe.stdout;
}
/**
* Quiet existence probe — no echo, because "not found" here is an expected
* state during migration checks, not a diagnostic the user needs to see.
*/
function collectionExists(
entry: string | null,
index: string,
name: string,
): boolean {
return (
spawnQmd(entry, ["--index", index, "collection", "show", name]).status === 0
);
}
/**
* Run a qmd subcommand and treat any non-zero exit as fatal. Echoes captured
* stdout/stderr before exiting so the user sees qmd's own diagnostic.
*/
function run(
entry: string | null,
args: readonly string[],
description: string,
): void {
process.stdout.write(`→ ${description}\n`);
const outcome = spawnQmd(entry, args);
echo(outcome);
if (outcome.status !== 0) {
process.stderr.write(
`qmd exited with code ${outcome.status ?? "?"} during: ${description}\n`,
);
process.exit(outcome.status ?? 1);
}
}
/**
* Run a qmd subcommand that is *expected* to fail idempotently (e.g. removing
* a context entry that may not exist). Callers pass a predicate that inspects
* the captured stderr/stdout and returns true when the failure matches the
* known-benign case; any other failure is surfaced as a prominent warning so
* it isn't silently masked.
*
* This replaces a plain "ignore all failures" helper that would swallow real
* problems (invalid pattern, permissions, qmd install drift) alongside the
* benign ones.
*/
function runIdempotent(
entry: string | null,
args: readonly string[],
description: string,
isBenignFailure: (outcome: SpawnOutcome) => boolean,
): void {
process.stdout.write(`→ ${description}\n`);
const outcome = spawnQmd(entry, args);
echo(outcome);
if (outcome.status !== 0 && !isBenignFailure(outcome)) {
process.stderr.write(
`\n⚠ qmd exited with code ${outcome.status ?? "?"} during: ${description}\n` +
` This wasn't the expected idempotent failure. Continuing, but review the output above.\n\n`,
);
}
}
function main(): void {
const manifestRaw = readManifestRaw();
const manifest = parseManifest(manifestRaw);
if (!manifest) {
process.stderr.write(
"vault-manifest.json missing or unreadable. Run from the vault root.\n",
);
process.exit(1);
}
// A non-empty `qmd_index` is an explicit pin and is taken literally — a
// typo there must fail loudly rather than silently fall through to a
// derived name, or the vault quietly indexes somewhere the user didn't ask
// for. Only an absent/empty field opts into derivation (#137).
const pinned = manifest.qmd_index;
if (pinned !== undefined && pinned !== "" && !isValidQmdIndex(pinned)) {
process.stderr.write(
`vault-manifest.json \`qmd_index\` value ${JSON.stringify(pinned)} is not a valid index name.\n` +
"Allowed: alphanumerics, dot, dash, underscore; must start with an alphanumeric.\n" +
"(The value is used both in CLI argv and a filesystem path, so path separators and whitespace aren't accepted.)\n",
);
process.exit(1);
}
// Route through the SAME resolver the read surfaces use rather than
// reimplementing the precedence here. This script WRITES the store; a
// private fallback would populate one index while SessionStart, the
// refresh worker, and the MCP wrapper read another — which surfaces as
// "0 documents", not as an error.
// `process.cwd()` is the vault root: `readManifest()` above already
// resolved `vault-manifest.json` relative to it, so a wrong cwd has
// already failed.
const index = resolveQmdIndex(manifestRaw, process.cwd());
if (index === null) {
process.stderr.write(
"Could not determine a qmd index name for this vault.\n" +
"The vault folder name yields no usable slug and the manifest has no " +
"`template` to fall back on.\n" +
'Set one explicitly: "qmd_index": "my-vault" in vault-manifest.json.\n',
);
process.exit(1);
}
// Reaching the `template` fallback means this vault shares a store with
// every other install that did the same — the exact collision #137 fixes.
// Keyed on derivation FAILING, not on the resolved name matching
// `template`: a vault whose folder legitimately slugifies to the template
// name is correctly isolated and must not be warned at.
if (!isValidQmdIndex(pinned) && deriveQmdIndex(process.cwd()) === null) {
warn(
`Vault folder name yields no usable index slug — falling back to the shared '${index}' index. ` +
'Set "qmd_index" in vault-manifest.json to keep this vault\'s search store isolated.',
);
}
// Resolve once up front so every downstream spawn reuses the same entry.
const entry = resolveQmdEntry();
const versionOut = ensureQmd(entry);
// Min-version gate (#100): a silently-old qmd is the failure the version
// declaration exists to catch — fail loud here, where the user is already
// taking explicit setup action. Fails OPEN on unparseable versions (an
// unknown version must never brick the bootstrap); vaults without the
// manifest field skip the gate entirely.
const minVersion = manifest.qmd_min_version;
if (
typeof minVersion === "string" &&
!qmdVersionAtLeast(versionOut, minVersion)
) {
process.stderr.write(
`Installed qmd (${versionOut.trim()}) is below this vault's declared minimum (${minVersion}).\n` +
"Update it: npm i -g @tobilu/qmd\n",
);
process.exit(1);
}
// Collection name = qmd_index, the per-vault identity (#105). `template`
// is the shared package identity — the same string for every install —
// so deriving the collection name from it breaks idempotency on any
// vault where the two differ (and mismatches the on-disk collection,
// which qmd creates under the vault's own name). qmd_index was validated
// above, so no second validation branch is needed.
const collectionName = index;
const contextPath = `qmd://${collectionName}/`;
const contextText =
manifest.qmd_context ??
"Obsidian vault template with persistent AI agent memory.";
process.stdout.write(`→ Bootstrapping QMD index '${index}'\n`);
// Migration (#105): bootstraps before the fix registered the collection
// under the shared `template` name. Field states to handle:
// 1. No legacy collection (fresh install, or the old bootstrap failed
// before registering) → nothing to do.
// 2. Legacy exists, correctly-named one doesn't → RENAME in place.
// Rename keeps the indexed data, so migrated users pay no re-index.
// 3. Both exist (user already re-registered by hand) → REMOVE the
// legacy one; two collections over the same corpus double every
// search result.
// `collection rename` may not exist on older qmd installs — fall back to
// remove, and the `collection add` below registers the correct name fresh.
const legacyName = legacyCollectionCandidate(manifest.template, index);
if (legacyName !== null && collectionExists(entry, index, legacyName)) {
if (collectionExists(entry, index, collectionName)) {
run(
entry,
["--index", index, "collection", "remove", legacyName],
`Removing legacy collection '${legacyName}' (superseded by '${collectionName}')`,
);
} else {
process.stdout.write(
`→ Renaming legacy collection '${legacyName}' → '${collectionName}' (#105 migration)\n`,
);
const renamed = spawnQmd(entry, [
"--index",
index,
"collection",
"rename",
legacyName,
collectionName,
]);
echo(renamed);
// qmd exits 0 even for unknown subcommands, so rename success is
// verified by probing the target collection — never by exit status.
if (!collectionExists(entry, index, collectionName)) {
if (isUnknownSubcommandFailure(renamed)) {
// Older qmd without `collection rename`: remove the legacy
// entry; the registration below re-adds under the correct
// name and the update/embed steps rebuild its index rows.
run(
entry,
["--index", index, "collection", "remove", legacyName],
`Rename unavailable — removing legacy collection '${legacyName}' instead`,
);
} else {
// Unexpected failure (locked store, CLI drift): do NOT
// destroy the legacy collection — leave it for inspection.
// The registration below surfaces the conflict loudly.
warn(
`Could not rename legacy collection '${legacyName}' — leaving it in place. ` +
`Migrate manually: qmd --index ${index} collection rename ${legacyName} ${collectionName}`,
);
}
}
}
// The legacy context row is keyed by the legacy path — clear it so a
// stale summary doesn't shadow the one attached below.
runIdempotent(
entry,
["--index", index, "context", "rm", `qmd://${legacyName}/`],
"Clearing legacy context (if any)",
isContextRemoveBenign,
);
}
// Re-runs are idempotent (matcher recognises the by-name "already exists"
// case); a path-collision warning is intentionally NOT swallowed.
runIdempotent(
entry,
buildCollectionAddArgs(index, collectionName),
`Registering collection '${collectionName}' (mask **/*.md)`,
makeCollectionAddBenignMatcher(collectionName),
);
// Round-trip the registration: if a future qmd CLI change silently breaks
// our argv shape, this is where we find out — `collection show` exits 1
// with "Collection not found: <name>" rather than running past the issue.
run(
entry,
["--index", index, "collection", "show", collectionName],
`Verifying collection '${collectionName}' is registered`,
);
// Re-attach the context string so edits to vault-manifest.json propagate.
// On first run, `context rm` fails because the row doesn't exist — that's
// the expected benign case. Any other failure (including a genuine
// "Collection not found") is intentionally NOT swallowed.
runIdempotent(
entry,
["--index", index, "context", "rm", contextPath],
"Clearing previous context (if any)",
isContextRemoveBenign,
);
run(
entry,
["--index", index, "context", "add", contextPath, contextText],
"Attaching vault context from manifest",
);
// Propagate Obsidian's userIgnoreFilters into QMD's YAML so both engines
// honor the same hidden-file list. Runs after `collection add` because
// that call overwrites the collection entry without preserving `ignore`.
// readObsidianIgnore returns null when app.json is unreadable/unparseable —
// we skip propagation in that case so a user typo doesn't strip the
// existing QMD ignore block. A warn() has already been emitted.
const obsidianIgnore = readObsidianIgnore();
if (obsidianIgnore !== null) {
const qmdIgnore: string[] = [];
for (const p of obsidianIgnore) {
const glob = translateToGlob(p);
if (glob === null) {
warn(
`Skipping regex pattern ${JSON.stringify(p)} — QMD ignore field accepts globs only.`,
);
continue;
}
qmdIgnore.push(glob);
}
// Print the step header only after the write succeeds, so logs never
// show "→ Syncing..." as a completed step when writeQmdIgnore actually
// skipped (missing config or unknown collection — both already warn()).
const wrote = writeQmdIgnore(
qmdConfigPath(index),
collectionName,
qmdIgnore,
);
if (wrote) {
process.stdout.write(
"→ Syncing ignore patterns from .obsidian/app.json\n",
);
if (qmdIgnore.length > 0) {
process.stdout.write(
` ${qmdIgnore.length} ignore pattern(s) synced from .obsidian/app.json\n`,
);
}
}
}
// Set the embedding model before indexing. qmd materialises its own default
// into the config the first time it touches an index, so this has to run
// after the collection exists and before anything is embedded.
//
// Changing it invalidates every existing vector — the two models produce
// different dimensions, and qmd refuses to mix them: `embed` errors out and
// a query throws outright rather than degrading. So a change here forces a
// full re-embed in the same run, and the two are never left apart.
const embedderChanged = writeQmdEmbedModel(qmdConfigPath(index));
if (embedderChanged) {
process.stdout.write("→ Setting the embedding model\n");
}
run(entry, ["--index", index, "update"], "Indexing vault files");
run(
entry,
["--index", index, "embed", ...(embedderChanged ? ["-f"] : [])],
embedderChanged
? "Generating embeddings (full rebuild — the model changed)"
: "Generating embeddings",
);
process.stdout.write(
`\n✓ QMD index '${index}' ready. Test with:\n qmd --index ${index} query "<topic>"\n`,
);
}
if (isMainModule(import.meta.url)) main();