Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Batch-plan step for large-folder ingest (A3, t_9eeb8ca2).
*
* OSB's ingest pipeline ({@link ingestSource}) handles ONE source at a time and
* has no planner to shard a large folder into bounded parallel work. This module
* supplies that planner: {@link planBatches} discovers the ingestible files under
* a source directory, consults A1's content-hash manifest to skip everything that
* is `unchanged` since the last ingest, and splits the `new`/`modified` remainder
* into size+count-bounded batches. The caller (an agent or the CLI) dispatches
* each batch as a parallel subagent - the kernel runs NO ingestion and spawns NO
* subagents itself. Planning stays deterministic and model-free.
*
* Determinism: files are discovered by a stable recursive walk, sorted by their
* canonical vault-relative path, then packed greedily - each batch is filled up
* to the byte cap and the file-count cap, whichever binds first. The same
* directory (same bytes on disk) always yields the same batch list.
*
* Ingestibility mirrors the pipeline's "text-bearing sources only" contract (no
* OCR, binary, or media): a small, extensible set of text/document extensions.
* The match is language-agnostic - it is over file extension and bytes, never
* over natural-language content. Hidden files and dot-directories (`.git`,
* `.open-second-brain`, ...) are skipped so machine artifacts never masquerade
* as ingestible sources.
*
* Discovery is git-aware (t_4b2bd8f7) through the shared
* {@link ../../fs/git-discovery.ts} module the hygiene repo scan also composes:
* a source directory's own `.gitignore`, its nested `.gitignore` files and its
* `.git/info/exclude` all narrow the walk, and a submodule or nested checkout
* is not descended into - not as a `readdir` child, and not as a directory a
* `--src-subpath` starts below either (that is refused outright, since those
* files belong to another repository). An ignore file that exists but cannot be
* honoured never passes silently: it comes back on
* {@link BatchPlan.ignoreWarnings}. Layer precedence, lowest first: `.git/info/exclude`,
* the source dir's `.gitignore`, each nested `.gitignore` by depth, then the
* operator's `--exclude` - so an operator pattern always has the last word.
* Discovery stays in-process: shelling out to `git ls-files` would make the
* reproducible {@link BatchPlan.planId} depend on an external binary and on
* user-global git configuration.
*
* Path space: layer base directories and the paths matched against them are
* both VAULT-relative here, which is the base-dir contract git-discovery
* documents. The hygiene scan chooses repo-root-relative instead; the shared
* module assumes neither.
*/
import { existsSync, readdirSync, statSync } from "node:fs";
import { isAbsolute, join, posix, relative } from "node:path";
import {
buildRepositoryBaseScope,
extendWithDirectoryIgnore,
IGNORE_WARNING_NO_LINE,
isNestedRepositoryBoundary,
} from "../../fs/git-discovery.ts";
import {
parseIgnorePatterns,
type IgnoreLayer,
type IgnoreScope,
type IgnoreWarning,
} from "../../fs/ignore.ts";
import { canonicalNotePath, ensureInsideVault } from "../../path-safety.ts";
import { computeExtractionContractFingerprint } from "./contract.ts";
import { computePlanId, readCheckpoint } from "./checkpoint.ts";
import { classifyPaths, readManifest } from "./content-manifest.ts";
import {
extractableAllowlist,
partitionExtractable,
SKIPPED_PAGE_REASON,
SKIPPED_PAGE_REASONS,
type SkippedPage,
type SkippedPageReason,
} from "./extractable-gate.ts";
import {
DEFAULT_INGESTIBLE_EXTENSIONS,
SOURCE_FORMAT,
SOURCE_FORMAT_BY_EXTENSION,
sourceFormatOf,
sourceFormatSpec,
type SourceFormat,
} from "./source-formats.ts";
/**
* Default set of ingestible extensions, lowercase with the dot: every
* extension the source format registry gives an extractor, the six text
* extensions first. Callers may override via
* {@link BatchPlanOptions.extensions}. Re-exported under the planner's
* historical name so nothing that iterates it reorders.
*/
export { DEFAULT_INGESTIBLE_EXTENSIONS };
/**
* The closed vocabulary of why-a-file-is-planned tokens (the wire `status`).
* Bytes-changed and contract-changed are deliberately DISTINCT tokens
* (t_586d5d8b): a source reprocessing because the extraction contract moved,
* not because its bytes moved, reports that fact instead of masquerading as an
* edit.
*/
export const PLANNED_FILE_STATUS = Object.freeze({
/** On disk, absent from the manifest. */
NEW: "new",
/** On disk, manifest bytes differ. */
BYTES_CHANGED: "modified",
/** On disk, manifest bytes identical, but the extraction contract changed. */
CONTRACT_CHANGED: "contract-changed",
} as const);
export type PlannedFileStatus = (typeof PLANNED_FILE_STATUS)[keyof typeof PLANNED_FILE_STATUS];
export interface BatchPlanOptions {
/** Hard upper bound on the summed bytes of one batch (must be > 0). */
readonly maxBatchBytes: number;
/** Hard upper bound on the file count of one batch (must be a positive int). */
readonly maxBatchFiles: number;
/**
* Override the ingestible extension set (lowercase, dot-prefixed). Absent →
* {@link DEFAULT_INGESTIBLE_EXTENSIONS}.
*/
readonly extensions?: readonly string[];
/**
* Resume an interrupted plan (t_ba1fa5f6): when true, items already recorded
* in this plan's checkpoint are excluded from the batches, on top of the
* manifest-`unchanged` skip. The plan id is derived from the full discovered
* set so it is stable across resumes. Absent → a fresh, checkpoint-blind plan.
*/
readonly resume?: boolean;
/**
* Scope discovery to a subtree of `sourceDir` (t_e82101a5), e.g. `pkg/a` in a
* monorepo. Resolved relative to the source dir; a value escaping it throws.
* Narrows the walk, never widens it: a subpath under a directory the
* repository ignores plans nothing and reports why on
* {@link BatchPlan.ignoreWarnings} (override it with an `exclude`
* re-include), and a subpath under a submodule or nested checkout throws.
* Absent → the whole source dir is walked (byte-identical to before).
*/
readonly srcSubpath?: string;
/**
* Gitignore-style patterns (t_e82101a5), matched by the shared
* {@link parseIgnorePatterns} engine relative to `sourceDir`. Discovered
* paths matching any pattern are dropped. Layered ABOVE every ignore file the
* repository declares, so an operator pattern - including a `!` re-include -
* overrides the repository. Absent/empty → no exclusion.
*/
readonly exclude?: readonly string[];
/**
* May the caller read this vault-relative file? Supplied by a surface that
* answers at a reach narrower than the vault; a local caller passes
* nothing. An ingestible file it refuses is treated as absent: never
* discovered, so it is in no batch, no total, no skip list and not in the
* plan id.
*/
readonly include?: (rel: string) => boolean;
}
/** One file selected for (re-)ingest, with the reason it was selected. */
export interface PlannedFile {
/** Canonical vault-relative POSIX path (the id ingest addresses). */
readonly path: string;
/** File size in bytes (drives the byte-cap packing). */
readonly bytes: number;
/** Why the file is being ingested: see {@link PLANNED_FILE_STATUS}. */
readonly status: PlannedFileStatus;
/**
* The file's registered format when it is not `text` (HTML, CSV, TSV, or a
* named format a caller's `extensions` override admitted). Absent for text
* and for an unregistered extension, so a Markdown plan is unchanged.
*/
readonly format?: SourceFormat;
}
/** One bounded batch the caller dispatches as a single parallel unit. */
export interface IngestBatch {
/** Zero-based position of the batch within the plan. */
readonly index: number;
/** Files in this batch, in sorted-by-path order. */
readonly files: readonly PlannedFile[];
/** Summed bytes of the batch's files. */
readonly totalBytes: number;
}
/**
* Files the discovery walk dropped because their extension is not ingestible,
* aggregated per extension (P4). Hidden entries and ignore-rule matches are
* excluded by declaration, not unclassifiable, and never counted; a file with
* no extension counts under the empty string. A `total` of 0 means every
* walked file was ingestible (the aggregate is absent on the wire).
*/
export interface UnclassifiableFiles {
/** How many files the walk dropped as non-ingestible. */
readonly total: number;
/** Per-extension counts, sorted by extension. "" keys extensionless files. */
readonly byExtension: Readonly<Record<string, number>>;
}
export interface BatchPlan {
/** Canonical vault-relative POSIX path of the planned source directory. */
readonly sourceDir: string;
readonly maxBatchBytes: number;
readonly maxBatchFiles: number;
/** The bounded batches, in packing order. Empty when nothing needs ingest. */
readonly batches: readonly IngestBatch[];
/** Canonical paths classified `unchanged` and therefore skipped, sorted. */
readonly skipped: readonly string[];
/**
* Canonical paths whose manifest entry is marked `retired_into` a
* canonical page (the resurrect guard, storage lifecycle review H3) and
* whose bytes are unchanged, sorted. They are never queued - re-ingesting
* one would quietly undo the retirement - while a real byte change
* re-admits the source as `modified` and drops it from this list. Unlike
* {@link BatchPlan.skipped} the skip survives a contract change: a
* retirement is not a content change, so the marked entry keeps its skip
* under the new contract too.
*/
readonly skippedRetired: readonly string[];
/**
* Files skipped before extraction: first every file whose format the
* source format registry names with no extractor (`format-not-extractable`,
* `detail` the format), sorted by path; then every page whose
* `schema_type` is not in the schema `extractable` allowlist (t_ed856388).
* Empty when neither applies, keeping the plan byte-identical to before.
*/
readonly skippedNonExtractable: readonly SkippedPage[];
/**
* {@link BatchPlan.skippedNonExtractable} counted per typed reason token
* (P4). Only non-zero reasons appear, in membership order; empty when
* nothing was skipped (absent on the wire, byte-identical-when-absent).
*/
readonly skipReasonCounts: Readonly<Partial<Record<SkippedPageReason, number>>>;
/**
* Files dropped during discovery because no ingestible extension matched
* (P4), aggregated per extension. Exclusion by declaration (hidden entries,
* ignore-rule matches) is not counted, and the counting never alters the
* discovered set - hence not {@link BatchPlan.planId} either.
*/
readonly unclassifiable: UnclassifiableFiles;
/** Total number of files across all batches (`new` + `modified`). */
readonly totalFiles: number;
/** Total bytes across all batches. */
readonly totalBytes: number;
/**
* Whether the manifest's recorded extraction contract
* ({@link ./contract.ts}) differs from the live one - including a v1
* manifest, which recorded none. Every `unchanged` source reprocesses once
* under the new contract (see {@link PLANNED_FILE_STATUS.CONTRACT_CHANGED});
* an unchanged contract plans exactly as before.
*/
readonly contractChanged: boolean;
/**
* How many of the planned files reprocess because the contract changed
* rather than their bytes. Zero unless {@link BatchPlan.contractChanged}.
*/
readonly contractChangedFiles: number;
/**
* Stable id for this plan (t_ba1fa5f6), derived from the source dir and the
* full discovered path set. The key an interrupted run resumes against.
*/
readonly planId: string;
/**
* Count of new/modified items excluded because this plan's checkpoint already
* recorded them completed. Zero on a fresh plan or when `resume` is off.
*/
readonly resumedCompleted: number;
/**
* Everything the walk could not apply as declared (t_4b2bd8f7), sorted by
* source then line: a malformed pattern in the repository's OWN ignore files,
* an ignore file that exists but could not be read (unreadable, a symlink,
* oversized - `line` 0), and a `--src-subpath` that pointed inside a subtree
* the repository ignores (`source` `--src-subpath`, `line` 0). None of these
* fails the plan - the repository is an input, not operator intent - but none
* of them passes silently either. A malformed operator `--exclude` still
* throws, and so does a subpath crossing a repository boundary. Empty on a
* tree whose ignore files all compile, and on a tree that declares none.
*/
readonly ignoreWarnings: readonly IgnoreWarning[];
}
/**
* Plan the ingest of a source directory into bounded parallel batches.
*
* Discovers the ingestible files under `sourceDir` (a vault-relative path),
* skips those the content-hash manifest classifies `unchanged` (and those it
* classifies `retired` - the H3 resurrect guard, reported on
* {@link BatchPlan.skippedRetired} and never queued), and packs the
* `new`/`modified` remainder greedily into batches that respect BOTH caps. A
* single file larger than the byte cap cannot be split, so it forms its own
* singleton batch (documented, honest - never silently dropped or truncated).
*
* Throws when a cap is not a positive number, or when `sourceDir` does not
* resolve to an existing directory inside the vault (no misleading empty-plan
* fallback for a typo'd or escaping path).
*/
export function planBatches(vault: string, sourceDir: string, opts: BatchPlanOptions): BatchPlan {
if (!Number.isFinite(opts.maxBatchBytes) || opts.maxBatchBytes <= 0) {
throw new Error(`planBatches: maxBatchBytes must be > 0, got ${opts.maxBatchBytes}`);
}
if (!Number.isInteger(opts.maxBatchFiles) || opts.maxBatchFiles <= 0) {
throw new Error(
`planBatches: maxBatchFiles must be a positive integer, got ${opts.maxBatchFiles}`,
);
}
const dirRel = canonicalNotePath(sourceDir);
// Refuse an absolute argument BEFORE the existence check: `join` treats
// an absolute second operand as relative text, so `join(vault, "/abs")`
// nests the path inside the vault and the directory the operator can
// list comes out "not an existing directory" - a falsehood, and one that
// never states the actual contract, that the argument is vault-relative.
if (isAbsolute(sourceDir)) {
throw new Error(
`planBatches: source dir must be a vault-relative path inside the vault ` +
`(got "${sourceDir}"); rerun with the directory path relative to the vault root`,
);
}
const dirAbs = ensureInsideVault(join(vault, dirRel), vault);
if (!existsSync(dirAbs) || !statSync(dirAbs).isDirectory()) {
throw new Error(`planBatches: source dir is not an existing directory: ${sourceDir}`);
}
// Optional subtree scoping (t_e82101a5): resolve `srcSubpath` under the
// source dir. ensureInsideVault throws a typed error if it escapes.
const srcSubpath =
opts.srcSubpath !== undefined && opts.srcSubpath.length > 0 ? opts.srcSubpath : null;
let walkRoot = dirAbs;
if (srcSubpath !== null) {
walkRoot = ensureInsideVault(join(dirAbs, srcSubpath), dirAbs);
if (!existsSync(walkRoot) || !statSync(walkRoot).isDirectory()) {
throw new Error(`planBatches: src subpath is not an existing directory: ${srcSubpath}`);
}
}
const extensions = normalizeExtensions(opts.extensions ?? DEFAULT_INGESTIBLE_EXTENSIONS);
// Optional exclusion (t_e82101a5): reuse the shared ignore engine so there is
// no second matcher. Patterns are relative to the source dir. A malformed
// pattern is a user error, surfaced loudly rather than silently ignored.
const excludeLayer = buildExcludeLayer(opts.exclude ?? [], dirRel);
// What the repository declares about itself (t_4b2bd8f7), beneath the
// operator's layer. The source dir is the repository root for this purpose; a
// subpath-scoped walk starts from the declarations above it AND obeys them, so
// naming a subpath can neither re-enter an ignored subtree nor cross into
// another repository (both would make the same tree answer two ways).
const ignoreWarnings: IgnoreWarning[] = [];
const base = buildRepositoryBaseScope(dirAbs, dirRel);
ignoreWarnings.push(...base.warnings);
let walkScope = base.scope;
let prunedWarning: IgnoreWarning | null = null;
if (srcSubpath !== null) {
const descent = descendToSubpathRoot({
scope: base.scope,
excludeLayer,
sourceDirAbs: dirAbs,
sourceDirRel: dirRel,
subpath: srcSubpath,
walkRoot,
warnings: ignoreWarnings,
});
walkScope = descent.scope;
prunedWarning = descent.prunedWarning;
if (prunedWarning !== null) ignoreWarnings.push(prunedWarning);
}
// Discover ingestible files as canonical vault-relative paths, sorted. A walk
// root inside an ignored subtree discovers nothing and says why on the plan.
const discovered: string[] = [];
const unclassifiable = new Map<string, number>();
const formatSkips: SkippedPage[] = [];
if (prunedWarning === null) {
collectIngestible(walkRoot, walkScope, {
vault,
extensions,
excludeLayer,
out: discovered,
warnings: ignoreWarnings,
unclassifiable,
formatSkips,
...(opts.include !== undefined ? { include: opts.include } : {}),
});
}
const discoveredRel = discovered
.map((abs) => canonicalNotePath(toPosixRel(vault, abs)))
.toSorted();
// Honor the schema `extractable` allowlist (t_ed856388): pages whose
// schema_type is not extractable are skipped BEFORE extraction, reported with
// a reason, and logged. An empty allowlist gates nothing, so the discovered
// set (hence planId and the whole plan) stays byte-identical to before.
const partition = partitionExtractable(vault, discoveredRel, extractableAllowlist(vault));
const relPaths = partition.extractable;
// A file meets the format gate before any schema gate, so format skips
// come first, each group in path order.
const skippedNonExtractable: SkippedPage[] = [
...formatSkips.toSorted((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)),
...partition.skipped,
];
// The plan id keys on the FULL discovered set plus the LIVE extraction
// contract (t_586d5d8b), so it is identical before and after an interruption
// and a contract change gets a fresh id - the old checkpoint then orphans
// instead of resume-skipping work extracted under the old contract.
const contractFingerprint = computeExtractionContractFingerprint(vault);
const planId = computePlanId(dirRel, relPaths, contractFingerprint);
// On resume, drop items this plan's checkpoint already recorded completed
// BEFORE classification, so those items are not re-hashed at all - the
// resume fast-path that keeps a large-vault resume near-free. The content
// manifest stays authoritative for anything the checkpoint does not cover.
const completed =
opts.resume === true
? new Set(readCheckpoint(vault, planId)?.completed ?? [])
: new Set<string>();
const toClassify = relPaths.filter((p) => !completed.has(p));
const resumedCompleted = relPaths.length - toClassify.length;
// Consult A1's manifest, then compare its recorded extraction contract
// against the live one IMMEDIATELY after the read (t_586d5d8b): a manifest
// written under an older contract - or a v1 one, which recorded none - must
// not keep serving `unchanged` skips shaped by that contract.
const manifest = readManifest(vault);
const classification = classifyPaths(vault, toClassify, manifest);
const contractChanged =
manifest.contract === null || manifest.contract.fingerprint !== contractFingerprint;
// Build the planned-file list, tagging each with why it was selected, in
// sorted-by-path order so packing (and the whole plan) is deterministic.
// Under a changed contract the byte-identical sources reprocess too, with
// the distinct contract-changed token - "bytes changed" stays "modified".
const status = new Map<string, PlannedFileStatus>();
for (const p of classification.new) status.set(p, PLANNED_FILE_STATUS.NEW);
for (const p of classification.modified) status.set(p, PLANNED_FILE_STATUS.BYTES_CHANGED);
if (contractChanged) {
for (const p of classification.unchanged) status.set(p, PLANNED_FILE_STATUS.CONTRACT_CHANGED);
}
// The resurrect guard (H3): a retired entry never enters the status map -
// not as new (its entry is recorded), not as contract-changed (a
// retirement is not a content change, so the marker outlives a contract
// move) - and therefore never lands in a batch.
const skippedRetired = [...classification.retired].toSorted();
const planned: PlannedFile[] = [...status.keys()].toSorted().map((path) => {
const format = sourceFormatOf(path);
return {
path,
bytes: statSync(join(vault, path)).size,
status: status.get(path)!,
...(format !== null && format !== SOURCE_FORMAT.text ? { format } : {}),
};
});
const batches = packBatches(planned, opts.maxBatchBytes, opts.maxBatchFiles);
return {
sourceDir: dirRel,
maxBatchBytes: opts.maxBatchBytes,
maxBatchFiles: opts.maxBatchFiles,
batches,
// Nothing skips under a changed contract: every unchanged source is in the
// reprocessing list above, so the skip list is empty by construction. The
// retired skip is independent of that rule - see {@link BatchPlan.skippedRetired}.
skipped: contractChanged ? [] : [...classification.unchanged].toSorted(),
skippedRetired,
skippedNonExtractable,
skipReasonCounts: countSkipReasons(skippedNonExtractable),
unclassifiable: aggregateUnclassifiable(unclassifiable),
totalFiles: planned.length,
totalBytes: planned.reduce((sum, f) => sum + f.bytes, 0),
contractChanged,
contractChangedFiles: contractChanged ? classification.unchanged.length : 0,
planId,
resumedCompleted,
// Sorted so the plan is fully deterministic: the walk visits directories in
// readdir order, which is not guaranteed stable across platforms.
ignoreWarnings: ignoreWarnings.toSorted(compareIgnoreWarnings),
};
}
/** Order warnings by their ignore file, then by line within it. */
function compareIgnoreWarnings(a: IgnoreWarning, b: IgnoreWarning): number {
if (a.source !== b.source) return a.source < b.source ? -1 : 1;
return a.line - b.line;
}
/**
* Count the plan's skipped pages per typed reason token, in membership order
* so the aggregate (and the wire record built from it) is deterministic.
*/
function countSkipReasons(
skipped: readonly SkippedPage[],
): Partial<Record<SkippedPageReason, number>> {
const counts: Partial<Record<SkippedPageReason, number>> = {};
for (const reason of SKIPPED_PAGE_REASONS) {
const n = skipped.filter((s) => s.reason === reason).length;
if (n > 0) counts[reason] = n;
}
return counts;
}
/**
* Aggregate the walk's per-extension drop counts into the plan's shape,
* sorted by extension so the record (and the wire object built from it) is
* deterministic.
*/
function aggregateUnclassifiable(counts: ReadonlyMap<string, number>): UnclassifiableFiles {
const entries = [...counts.entries()].toSorted(([a], [b]) => (a < b ? -1 : 1));
return {
total: entries.reduce((sum, [, n]) => sum + n, 0),
byExtension: Object.fromEntries(entries),
};
}
/**
* Greedily pack sorted files into batches. A file joins the current batch unless
* doing so would breach the byte cap or the count cap, in which case the current
* batch is flushed and the file opens a new one. The `current.length > 0` guard
* ensures a lone file that itself exceeds the byte cap is never deferred forever
* - it becomes its own (oversize, unsplittable) singleton batch.
*/
function packBatches(
files: readonly PlannedFile[],
maxBatchBytes: number,
maxBatchFiles: number,
): IngestBatch[] {
const batches: IngestBatch[] = [];
let current: PlannedFile[] = [];
let currentBytes = 0;
const flush = (): void => {
if (current.length === 0) return;
batches.push({ index: batches.length, files: current, totalBytes: currentBytes });
current = [];
currentBytes = 0;
};
for (const file of files) {
const wouldExceedBytes = current.length > 0 && currentBytes + file.bytes > maxBatchBytes;
const wouldExceedCount = current.length >= maxBatchFiles;
if (wouldExceedBytes || wouldExceedCount) flush();
current.push(file);
currentBytes += file.bytes;
}
flush();
return batches;
}
/**
* Compile the `--exclude` patterns into one layer, relative to the source dir.
* A malformed pattern is a user error and throws (never a silent skip) - unlike
* a malformed pattern in the repository's own ignore files, which the operator
* did not write and which is reported on {@link BatchPlan.ignoreWarnings}. An
* empty pattern list yields `null`, so the no-flag path stacks no layer at all
* and stays byte-identical.
*/
function buildExcludeLayer(patterns: readonly string[], baseDir: string): IgnoreLayer | null {
if (patterns.length === 0) return null;
const { layer, warnings } = parseIgnorePatterns(patterns, baseDir, "--exclude");
if (warnings.length > 0) {
const w = warnings[0]!;
throw new Error(`planBatches: malformed --exclude pattern "${w.pattern}": ${w.reason}`);
}
return layer;
}
/** Everything {@link descendToSubpathRoot} needs to reach a subpath walk root. */
interface SubpathDescentInput {
/** Scope governing the source dir, before any intermediate directory is added. */
readonly scope: IgnoreScope;
/** Operator `--exclude`, stacked above every repository layer. Null when absent. */
readonly excludeLayer: IgnoreLayer | null;
readonly sourceDirAbs: string;
readonly sourceDirRel: string;
/** The operator's `--src-subpath` value, recorded on a prune warning. */
readonly subpath: string;
readonly walkRoot: string;
/** Accumulates malformed patterns found on the way down. */
readonly warnings: IgnoreWarning[];
}
/** The scope governing the walk root, or the reason it must not be walked. */
interface SubpathDescent {
readonly scope: IgnoreScope;
/**
* Null when every directory on the way down is walkable. Non-null when one of
* them is ignored: the requested subtree sits inside an ignored one, is NOT
* walked, and this warning carries the explanation onto the plan.
*/
readonly prunedWarning: IgnoreWarning | null;
}
/** Provenance recorded on a warning caused by the `--src-subpath` flag itself. */
const SRC_SUBPATH_WARNING_SOURCE = "--src-subpath";
/**
* Walk from the source dir (exclusive) down to the subpath walk root
* (inclusive), applying to every directory on the way the same two rules
* {@link collectIngestible} applies to a directory it meets as a `readdir`
* entry - a nested repository is a boundary, an ignored directory prunes its
* subtree - and layering each directory's own `.gitignore` for what lies below
* it. Without this a `--src-subpath` would re-enter a subtree the repository
* declared off-limits, or a foreign repository, purely because the walk never
* passed through the parent that would have stopped it.
*
* Throws when a directory on the way down is a submodule or nested checkout.
*/
function descendToSubpathRoot(input: SubpathDescentInput): SubpathDescent {
let current = input.scope;
let abs = input.sourceDirAbs;
let rel = input.sourceDirRel;
for (const segment of toPosixRel(input.sourceDirAbs, input.walkRoot).split(posix.sep)) {
if (segment.length === 0) continue;
abs = join(abs, segment);
rel = posix.join(rel, segment);
// A boundary, not an ignore rule: those files are another repository's, so
// no operator flag can re-include them and the plan refuses outright rather
// than attributing a foreign repository's files to this tree.
if (isNestedRepositoryBoundary(abs)) {
throw new Error(
`planBatches: src subpath "${input.subpath}" crosses a nested repository ` +
`boundary at "${rel}": those files belong to that repository - plan against ` +
"it directly",
);
}
// Ignored-ness is decided by the layers governing the PARENT plus the
// operator's `--exclude`, exactly as it is for a readdir child, so an
// `--exclude` re-include remains the one way to override the repository.
const effective = input.excludeLayer === null ? current : current.extend(input.excludeLayer);
if (effective.isIgnored(rel, true)) {
return { scope: current, prunedWarning: subpathIgnoredWarning(input.subpath, rel) };
}
const extended = extendWithDirectoryIgnore(current, abs, rel);
input.warnings.push(...extended.warnings);
current = extended.scope;
}
return { scope: current, prunedWarning: null };
}
/**
* The structured warning a subpath pointing into an ignored subtree produces.
* The plan comes back empty on purpose - the same tree must answer the same way
* however it is addressed - and this says so instead of leaving the operator
* with a silent empty plan.
*/
function subpathIgnoredWarning(subpath: string, ignoredDir: string): IgnoreWarning {
return {
source: SRC_SUBPATH_WARNING_SOURCE,
line: IGNORE_WARNING_NO_LINE,
pattern: subpath,
reason:
`"${ignoredDir}" is ignored by the repository, so nothing under the requested ` +
'subpath was planned; override it with an --exclude "!" re-include',
};
}
/** Everything the recursive walk needs beyond the current directory and scope. */
interface WalkContext {
readonly vault: string;
readonly extensions: ReadonlySet<string>;
/** Operator `--exclude`, stacked above every repository layer. Null when absent. */
readonly excludeLayer: IgnoreLayer | null;
/** Accumulates discovered absolute file paths. */
readonly out: string[];
/** Accumulates malformed patterns found in the repository's own ignore files. */
readonly warnings: IgnoreWarning[];
/** Counts files dropped for a non-ingestible extension, per extension. */
readonly unclassifiable: Map<string, number>;
/** Accumulates files of a registered format that has no extractor. */
readonly formatSkips: SkippedPage[];
/** See {@link BatchPlanOptions.include}. Absent: every file is included. */
readonly include?: (rel: string) => boolean;
}
/**
* Recursively collect ingestible regular-file absolute paths, skipping hidden
* entries and anything the composed ignore scope matches. An ignored directory
* is pruned so its subtree is never walked.
*
* `scope` already carries the layers governing `dir`; a child directory's own
* `.gitignore` is stacked before descending into it, so a deeper file scopes
* only what it governs and a nearer `!` re-include wins. The operator's
* `--exclude` is applied on top for every match, keeping it above the whole
* repository-declared stack no matter how deep the walk goes.
*
* A regular file the scope lets through whose extension is not ingestible is
* either named on {@link WalkContext.formatSkips} - its format is one the
* source format registry names with no extractor (PDF, Office, images) - or
* counted on {@link WalkContext.unclassifiable} (P4). Both run before the
* reach predicate: visibility is a Markdown rule, and asking it about a binary
* would read the binary whole.
*/
function collectIngestible(dir: string, scope: IgnoreScope, ctx: WalkContext): void {
const effective = ctx.excludeLayer === null ? scope : scope.extend(ctx.excludeLayer);
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (entry.name.startsWith(".")) continue; // hidden file or dot-directory
const abs = join(dir, entry.name);
const rel = toPosixRel(ctx.vault, abs);
if (entry.isDirectory()) {
// A submodule or nested checkout belongs to another repository's index.
// This is a boundary, not an ignore rule, so `--exclude` cannot re-include
// it - an operator wanting those files plans against that repository.
if (isNestedRepositoryBoundary(abs)) continue;
if (effective.isIgnored(rel, true)) continue;
const child = extendWithDirectoryIgnore(scope, abs, rel);
ctx.warnings.push(...child.warnings);
collectIngestible(abs, child.scope, ctx);
} else if (entry.isFile()) {
// An ignore-rule match is exclusion by DECLARATION - checked before the
// extension test so an ignored binary never inflates the unclassifiable
// count. The discovered set is untouched either way.
if (effective.isIgnored(rel, false)) continue;
const ext = extname(entry.name).toLowerCase();
if (!ctx.extensions.has(ext)) {
const format = SOURCE_FORMAT_BY_EXTENSION.get(ext);
if (format !== undefined && sourceFormatSpec(format).extractor === null) {
ctx.formatSkips.push({
path: canonicalNotePath(rel),
reason: SKIPPED_PAGE_REASON.formatNotExtractable,
detail: format,
});
} else {
ctx.unclassifiable.set(ext, (ctx.unclassifiable.get(ext) ?? 0) + 1);
}
continue;
}
// After the extension test, so the predicate reads only files the plan
// could carry; a refused one is not this caller's to discover.
if (ctx.include !== undefined && !ctx.include(rel)) continue;
ctx.out.push(abs);
}
}
}
/** Lowercase, dot-prefixed extension of a filename, or "" when it has none. */
function extname(name: string): string {
const dot = name.lastIndexOf(".");
return dot <= 0 ? "" : name.slice(dot).toLowerCase();
}
/** Normalize an extension list to a lowercase, dot-prefixed lookup set. */
function normalizeExtensions(exts: readonly string[]): ReadonlySet<string> {
const set = new Set<string>();
for (const raw of exts) {
const trimmed = raw.trim().toLowerCase();
if (trimmed.length === 0) continue;
set.add(trimmed.startsWith(".") ? trimmed : `.${trimmed}`);
}
return set;
}
/** Vault-relative POSIX path for an absolute path inside the vault. */
function toPosixRel(vault: string, abs: string): string {
return relative(vault, abs).split(/[\\/]/).join(posix.sep);
}