Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.
TypeScript
fix(kb): stop orphan sweep from purging an empty but present scan root
jakeaturnerae1eae78e7bd
diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.tsindex c6e1be91..7995c41c 100644--- a/admin/app/services/rag_service.ts+++ b/admin/app/services/rag_service.ts@@ -22,11 +22,11 @@ import { OllamaService } from './ollama_service.js'import { SERVICE_NAMES } from '../../constants/service_names.js'import { removeStopwords } from 'stopword'import { randomUUID } from 'node:crypto'-import { join, resolve, sep } from 'node:path'+import { join, relative, resolve, sep } from 'node:path'import KVStore from '#models/kv_store'import KbIngestState from '#models/kb_ingest_state'import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js'-import { decideOrphans, filterOrphanCandidates } from '../utils/kb_orphan_decision.js'+import { decideOrphans } from '../utils/kb_orphan_decision.js'import { decideContentReindex, type ReindexOutcome } from '../utils/content_reindex_decision.js'import KbRatioRegistry from '#models/kb_ratio_registry'import { decideWarnings } from '../utils/kb_warning_decision.js'@@ -2097,9 +2097,11 @@ export class RagService {* install legitimately has no kb_uploads until the first upload. That makes* an absent root indistinguishable from a present-but-empty one by looking* at the file list alone — and the orphan sweep in scanAndSyncStorage()- * cannot afford to confuse the two. "This root scanned clean and had no- * files" means everything indexed under it is an orphan; "this root wasn't- * there" means we know nothing about it and must not touch it.+ * cannot afford to confuse the two. "This root wasn't there" means we know+ * nothing about it and must not touch it. A root that was there but empty+ * is not much better evidence: boot creates the zim directory, so an+ * unmounted volume shows up as an empty one (#1378). decideOrphans()+ * therefore also refuses to purge under a walked root with no files.** That distinction is the whole reason this variant exists. If the zim root* is missing, renamed, or not yet mounted (see #1050 — relocating the data@@ -2388,24 +2390,29 @@ export class RagService {// leftover from ZimService.delete() (which never touched Qdrant) or// from reconcileReplacedContentFile's qdrant_not_running no-op. Running// this in sync (rather than only in the delete path) also self-heals- // installs already in this state. decideOrphans no-ops when- // embeddableFiles came back empty, so a filesystem hiccup can't be- // misread as "every file was deleted."+ // installs already in this state.//- // Allowlisted (via filterOrphanCandidates) to `scannedRoots` — the roots- // the scan above actually walked, not the roots it meant to walk. Two- // separate things are excluded by that one rule. Nomad's own bundled- // docs (README.md + docs/) are embedded by discoverNomadDocs() from- // outside these roots, so they're left alone without being named here.- // And a root that wasn't present at scan time contributes no candidates- // at all, because a missing root is skipped rather than fatal: without- // this, a relocated or unmounted zim directory (#1050) would leave the- // scan non-empty via kb_uploads, sail past decideOrphans' empty-scan- // guard, and purge every ZIM in the index in a single batch.- const orphanCandidates = filterOrphanCandidates([...sourcesInQdrant], scannedRoots)- const orphans = decideOrphans(orphanCandidates, embeddableFiles)+ // Confined to `scannedRoots`, the roots the scan above actually walked,+ // not the roots it meant to walk. Nomad's own bundled docs (README.md ++ // docs/) live outside these roots, so they're left alone without being+ // named here, and a root missing at scan time contributes nothing+ // (#1050). Within each walked root, decideOrphans also withholds the+ // purge when the root holds no embeddable files (an unmounted volume+ // leaves an empty mountpoint behind, #1378) or when it would remove most+ // of the root at once. Withheld roots are reported back to the operator+ // rather than silently skipped.+ const { orphans, withheld } = decideOrphans(+ [...sourcesInQdrant],+ embeddableFiles,+ scannedRoots+ )+ for (const w of withheld) {+ logger.warn(+ `[RAG] Withheld purge of ${w.count} indexed source(s) under ${w.root} (${w.reason}); the directory may be unmounted or pointing at the wrong location`+ )+ }let orphansPurged = 0- if (orphans && orphans.length > 0) {+ if (orphans.length > 0) {logger.info(`[RAG] Found ${orphans.length} orphaned source(s) with no corresponding file on disk`)@@ -2478,10 +2485,20 @@ export class RagService {`[RAG] Scan results (policy=${policy}): ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${createdPending} waiting on user, ${skipped} skipped`)+ const withheldNote = withheld+ .map(+ (w) =>+ `; left ${w.count} indexed source${w.count !== 1 ? 's' : ''} under ${relative(process.cwd(), w.root)} untouched because ${+ w.reason === 'empty_root'+ ? 'that folder has no files (is the drive mounted?)'+ : 'removing them would clear most of that folder (is it pointing at the right drive?)'+ }`+ )+ .join('')const orphanNote =- orphansPurged > 0+ (orphansPurged > 0? `; purged ${orphansPurged} orphaned source${orphansPurged !== 1 ? 's' : ''}`- : ''+ : '') + withheldNoteif (filesToEmbed.length === 0) {return {diff --git a/admin/app/utils/kb_orphan_decision.ts b/admin/app/utils/kb_orphan_decision.tsindex 20655b9f..e03b5ed6 100644--- a/admin/app/utils/kb_orphan_decision.ts+++ b/admin/app/utils/kb_orphan_decision.ts@@ -1,5 +1,43 @@import { sep } from 'node:path'+/**+ * Past this share of a root's indexed sources, a single sync refuses to purge+ * that root at all. Losing most of a root at once is far likelier to mean the+ * root points somewhere wrong (a different disk, a stale copy, a half-finished+ * data-path move per #1050) than that the user deleted most of their library+ * by hand. Real deletions through the UI purge their own vectors immediately+ * (ZimService.delete, deleteFileBySource), so the sweep only ever mops up+ * stragglers and has no business removing the bulk of a root.+ */+export const ORPHAN_PURGE_MAX_FRACTION = 0.5++/**+ * The mass-removal guard only engages once this many sources would go. Below+ * it, the fraction is too noisy to mean anything (1 of 2 is 50%) and the cost+ * of a wrong purge is a few minutes of re-embedding, not hours.+ */+export const ORPHAN_PURGE_MIN_GUARDED = 5++export type WithheldOrphans = {+ /** Scan root whose orphans were left in place. */+ root: string+ /** How many indexed sources under it would have been purged. */+ count: number+ /**+ * `empty_root`: the root was walked but held no embeddable files.+ * `mass_removal`: the purge would remove more than ORPHAN_PURGE_MAX_FRACTION+ * of the root's indexed sources.+ */+ reason: 'empty_root' | 'mass_removal'+}++export type OrphanDecision = {+ /** Sources safe to purge. */+ orphans: string[]+ /** Roots whose missing sources were held back, for the operator to see. */+ withheld: WithheldOrphans[]+}+/*** Decision for the reverse sweep in `RagService.scanAndSyncStorage`.*@@ -12,26 +50,64 @@ import { sep } from 'node:path'* `reconcileReplacedContentFile`'s `qdrant_not_running` no-op therefore never* got reaped.*- * Guarded so a transient failure can't be misread as "every file was- * deleted": if the disk scan came back empty, we return `null` (do nothing)- * rather than treating every indexed source as an orphan. An empty- * `embeddableFiles` list is indistinguishable from a filesystem hiccup, and- * the blast radius of wrongly deleting a healthy knowledge base outweighs the- * cost of skipping a sweep for one cycle.+ * Decided per scanned root, because the roots fail independently. Two guards+ * apply to each, and either one withholds the whole root for this cycle:+ *+ * 1. A root with zero embeddable files keeps its sources (#1378). Walking a+ * directory successfully does not prove it holds the content: boot runs+ * ensureDirectoryExists() on the zim root, so a separate volume that+ * failed to mount leaves a real, empty mountpoint behind (often holding a+ * freshly regenerated kiwix-library.xml, which is why this counts+ * embeddable files rather than directory entries). An empty root is+ * indistinguishable from an unmounted one, and a single upload in+ * kb_uploads is enough to make the overall scan non-empty.+ *+ * 2. A purge that would remove more than ORPHAN_PURGE_MAX_FRACTION of a+ * root's indexed sources (and at least ORPHAN_PURGE_MIN_GUARDED of them) is+ * withheld. This catches the root pointing at the wrong place while still+ * holding a few files, which guard 1 can't see.+ *+ * Both guards trade a missed cleanup for safety: re-embedding a wrongly purged+ * library takes hours to days, while a stale source costs nothing until the+ * next sync or an explicit reset.*/export function decideOrphans(sourcesInQdrant: string[],- embeddableFiles: string[]-): string[] | null {- if (embeddableFiles.length === 0) return null-+ embeddableFiles: string[],+ scannedRoots: string[]+): OrphanDecision {const onDisk = new Set(embeddableFiles)- return sourcesInQdrant.filter((source) => !onDisk.has(source))+ const orphans: string[] = []+ const withheld: WithheldOrphans[] = []++ for (const root of scannedRoots) {+ const indexed = filterOrphanCandidates(sourcesInQdrant, [root])+ const missing = indexed.filter((source) => !onDisk.has(source))+ if (missing.length === 0) continue++ if (filterOrphanCandidates(embeddableFiles, [root]).length === 0) {+ withheld.push({ root, count: missing.length, reason: 'empty_root' })+ continue+ }++ if (+ missing.length >= ORPHAN_PURGE_MIN_GUARDED &&+ missing.length / indexed.length > ORPHAN_PURGE_MAX_FRACTION+ ) {+ withheld.push({ root, count: missing.length, reason: 'mass_removal' })+ continue+ }++ orphans.push(...missing)+ }++ return { orphans, withheld }}/**- * Narrows Qdrant sources down to the ones decideOrphans() can actually make an- * informed call about: sources under a root the disk scan genuinely walked.+ * Narrows paths down to the ones under a root the disk scan genuinely walked.+ * decideOrphans() applies it per root, both to Qdrant sources and to the files+ * found on disk.** `scannedRoots` is deliberately the roots that were *walked*, not the roots* that were *configured*. Two different failure modes collapse into that one@@ -45,9 +121,9 @@ export function decideOrphans(* 2. A configured root that wasn't there at scan time. `_discoverKbFiles()`* skips a missing root rather than failing, so a relocated or not-yet-* mounted zim directory (#1050) still leaves the scan non-empty via- * kb_uploads. decideOrphans' empty-scan guard doesn't fire, and without- * this filter every ZIM in the index would be purged in one batch. A root- * we couldn't read tells us nothing about what belongs under it.+ * kb_uploads. A root we couldn't read tells us nothing about what belongs+ * under it. (A root that exists but is empty is the same problem one step+ * removed; decideOrphans' per-root guard handles that, see #1378.)** Passing an empty `scannedRoots` therefore yields no candidates, which is the* correct reading of "we couldn't see any of the storage."diff --git a/admin/tests/unit/kb_orphan_decision.spec.ts b/admin/tests/unit/kb_orphan_decision.spec.tsindex 2301c46a..bb379d9e 100644--- a/admin/tests/unit/kb_orphan_decision.spec.ts+++ b/admin/tests/unit/kb_orphan_decision.spec.ts@@ -20,28 +20,39 @@ const upload = (name: string) => join(KB_UPLOADS_ROOT, name)const zim = (name: string) => join(ZIM_ROOT, name)test('no sources in Qdrant → no orphans', () => {- assert.deepEqual(decideOrphans([], [zim('a.zim')]), [])+ assert.deepEqual(decideOrphans([], [zim('a.zim')], SCAN_ROOTS), { orphans: [], withheld: [] })})test('every Qdrant source still has a file on disk → no orphans', () => {- assert.deepEqual(decideOrphans([zim('a.zim'), zim('b.zim')], [zim('a.zim'), zim('b.zim')]), [])+ assert.deepEqual(+ decideOrphans([zim('a.zim'), zim('b.zim')], [zim('a.zim'), zim('b.zim')], SCAN_ROOTS),+ { orphans: [], withheld: [] }+ )})test('a source with no matching file on disk is an orphan', () => {- assert.deepEqual(decideOrphans([zim('a.zim'), zim('gone.zim')], [zim('a.zim')]), [- zim('gone.zim'),- ])+ assert.deepEqual(decideOrphans([zim('a.zim'), zim('gone.zim')], [zim('a.zim')], SCAN_ROOTS), {+ orphans: [zim('gone.zim')],+ withheld: [],+ })})-test('every Qdrant source is orphaned when none remain on disk (but disk scan was non-empty)', () => {- assert.deepEqual(decideOrphans([zim('gone1.zim'), zim('gone2.zim')], [zim('unrelated.zim')]), [- zim('gone1.zim'),- zim('gone2.zim'),- ])+test('sources outside every scanned root are never orphans (e.g. bundled docs)', () => {+ const readme = join('/data', 'README.md')+ assert.deepEqual(decideOrphans([readme, zim('a.zim')], [zim('a.zim')], SCAN_ROOTS), {+ orphans: [],+ withheld: [],+ })})-test('empty disk scan is treated as a transient failure, not "everything was deleted"', () => {- assert.equal(decideOrphans([zim('a.zim'), zim('b.zim')], []), null)+test('empty disk scan withholds every root instead of purging everything', () => {+ assert.deepEqual(decideOrphans([upload('a.pdf'), zim('a.zim')], [], SCAN_ROOTS), {+ orphans: [],+ withheld: [+ { root: KB_UPLOADS_ROOT, count: 1, reason: 'empty_root' },+ { root: ZIM_ROOT, count: 1, reason: 'empty_root' },+ ],+ })})test('filterOrphanCandidates keeps sources under the kb_uploads or zim scan roots', () => {@@ -80,7 +91,10 @@ test('a root that was not scanned contributes no candidates, even though the scaassert.deepEqual(candidates, [upload('a.pdf')])// End to end: the ZIMs survive despite having no backing file in the scan.- assert.deepEqual(decideOrphans(candidates, [upload('a.pdf')]), [])+ assert.deepEqual(decideOrphans(sourcesInQdrant, [upload('a.pdf')], [KB_UPLOADS_ROOT]), {+ orphans: [],+ withheld: [],+ })})test('no roots scanned at all yields no candidates', () => {@@ -89,12 +103,67 @@ test('no roots scanned at all yields no candidates', () => {assert.deepEqual(filterOrphanCandidates([zim('a.zim')], []), [])})-test('a scanned-but-empty root still yields orphans for what it contains', () => {- // The counterpart to the case above, and the reason the distinction matters:- // zim scanned clean and legitimately holds no files, so its indexed sources- // really are orphaned and should be purged. A missing root and an empty one- // must not behave the same way.- const candidates = filterOrphanCandidates([zim('gone.zim')], SCAN_ROOTS)- assert.deepEqual(candidates, [zim('gone.zim')])- assert.deepEqual(decideOrphans(candidates, [upload('a.pdf')]), [zim('gone.zim')])+test('a walked root with no embeddable files keeps its sources (#1378)', () => {+ // The unmounted-volume shape: boot's ensureDirectoryExists() recreates the+ // zim mountpoint as an empty directory, so the scan walks it without error+ // and reports it as scanned. kb_uploads keeps the overall scan non-empty.+ // Treating "walked and empty" as "everything under it was deleted" would+ // purge every ZIM in the index.+ const sourcesInQdrant = [upload('a.pdf'), zim('wikipedia_en_all_maxi.zim'), zim('gutenberg.zim')]+ assert.deepEqual(decideOrphans(sourcesInQdrant, [upload('a.pdf')], SCAN_ROOTS), {+ orphans: [],+ withheld: [{ root: ZIM_ROOT, count: 2, reason: 'empty_root' }],+ })+})++test('a root holding only non-embeddable files still counts as empty', () => {+ // Kiwix regenerates kiwix-library.xml in the empty mountpoint, so the+ // directory has entries. Only embeddable files are evidence the content is+ // there, and _discoverKbFilesWithRoots() drops the XML before this point, so+ // the zim root arrives with nothing under it.+ const embeddable = [upload('a.pdf')]+ assert.deepEqual(decideOrphans([zim('a.zim')], embeddable, SCAN_ROOTS).withheld, [+ { root: ZIM_ROOT, count: 1, reason: 'empty_root' },+ ])+})++test('an empty kb_uploads root keeps its sources while zim is swept normally', () => {+ const zims = ['a', 'b', 'c'].map((n) => zim(`${n}.zim`))+ assert.deepEqual(+ decideOrphans([upload('a.pdf'), ...zims, zim('gone.zim')], zims, SCAN_ROOTS),+ {+ orphans: [zim('gone.zim')],+ withheld: [{ root: KB_UPLOADS_ROOT, count: 1, reason: 'empty_root' }],+ }+ )+})++test('removing most of a root in one sync is withheld as a likely wrong mount', () => {+ // The root is present and non-empty, but it holds 2 of 10 indexed ZIMs:+ // far likelier a different disk or stale copy than 8 hand-deletions.+ const indexed = Array.from({ length: 10 }, (_, i) => zim(`z${i}.zim`))+ const onDisk = indexed.slice(0, 2)+ assert.deepEqual(decideOrphans(indexed, onDisk, SCAN_ROOTS), {+ orphans: [],+ withheld: [{ root: ZIM_ROOT, count: 8, reason: 'mass_removal' }],+ })+})++test('removing up to half a root is still purged', () => {+ const indexed = Array.from({ length: 10 }, (_, i) => zim(`z${i}.zim`))+ const onDisk = indexed.slice(0, 5)+ assert.deepEqual(decideOrphans(indexed, onDisk, SCAN_ROOTS), {+ orphans: indexed.slice(5),+ withheld: [],+ })+})++test('the mass-removal guard does not engage below the minimum count', () => {+ // 4 of 5 missing is 80%, but only 4 sources: too few for the ratio to mean+ // anything, and cheap to re-embed if wrong.+ const indexed = Array.from({ length: 5 }, (_, i) => zim(`z${i}.zim`))+ assert.deepEqual(decideOrphans(indexed, indexed.slice(0, 1), SCAN_ROOTS), {+ orphans: indexed.slice(1),+ withheld: [],+ })})