Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
110 KB
Raw Blame
import { QdrantClient } from '@qdrant/js-client-rest'
import { DockerService } from './docker_service.js'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import { TokenChunker } from '@chonkiejs/core'
import sharp from 'sharp'
import {
  deleteFileIfExists,
  determineFileType,
  getFile,
  getFileStatsIfExists,
  listDirectoryContentsRecursive,
  ZIM_STORAGE_PATH,
} from '../utils/fs.js'
import { PDFParse } from 'pdf-parse'
import { createWorker } from 'tesseract.js'
import { fromBuffer } from 'pdf2pic'
import JSZip from 'jszip'
import * as cheerio from 'cheerio'
import mammoth from 'mammoth'
import { OllamaService } from './ollama_service.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { removeStopwords } from 'stopword'
import { randomUUID } from 'node:crypto'
import { join, relative, resolve, sep } from 'node:path'
import KVStore from '#models/kv_store'
import KbIngestState from '#models/kb_ingest_state'
import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js'
import { decideOrphans } from '../utils/kb_orphan_decision.js'
import { decideContentReindex, type ReindexOutcome } from '../utils/content_reindex_decision.js'
import KbRatioRegistry from '#models/kb_ratio_registry'
import { decideWarnings } from '../utils/kb_warning_decision.js'
import type { FileWarning, FileWarningsResult, RetrievalFloorStats, RetrievalStages, StoredFileInfo } from '../../types/rag.js'
import { applyRelevanceFloor } from '../utils/misc.js'
import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js'

/**
 * Qdrant filter that hides the developer eval corpus from every read path that
 * enumerates the *user's* knowledge base. The fixtures share this collection
 * (NOMAD collections are a payload tag), so without this they would show up as
 * broken files in the KB UI and skew the ingest-health warnings.
 */
const EXCLUDE_EVAL_FILTER = {
  must_not: [{ key: 'collection', match: { value: KB_EVAL_COLLECTION } }],
}
import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
import { ZIMExtractionService } from './zim_extraction_service.js'
import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
import { hasMoreArticleBatches } from '../utils/zim_batch_decision.js'
import { EMBEDDING_MODEL_NAME } from '../../constants/ollama.js'
import {
  ProcessAndEmbedFileResponse,
  ProcessZIMFileResponse,
  RAGResult,
  RerankedRAGResult,
} from '../../types/rag.js'

export type EmbedSingleFileFailureCode =
  | 'not_found'
  | 'inflight'
  | 'delete_failed'
  | 'dispatch_failed'

export type EmbedSingleFileResult =
  | { success: true; message: string }
  | { success: false; code: EmbedSingleFileFailureCode; message: string }

@inject()
export class RagService {
  private qdrant: QdrantClient | null = null
  private qdrantInitPromise: Promise<void> | null = null
  private embeddingModelVerified = false
  private resolvedEmbeddingModel: string | null = null
  // Collections already verified this session (created + payload indexes in place).
  // Skips the getCollections/createPayloadIndex round-trips that otherwise run on
  // every embed call — ~45% of per-document Qdrant time on large ingestions (#1129)
  private ensuredCollections = new Set<string>()
  public static UPLOADS_STORAGE_PATH = 'storage/kb_uploads'
  public static CONTENT_COLLECTION_NAME = 'nomad_knowledge_base'
  public static EMBEDDING_DIMENSION = 768 // Nomic Embed Text v1.5 dimension is 768
  // Upper bound on distinct sources returned by Qdrant's facet API. Real
  // NOMADs cap out at a few hundred ZIM files + uploaded PDFs; 10k leaves
  // generous headroom without paying the cost of an unbounded request.
  public static FACET_SOURCE_LIMIT = 10_000
  public static MODEL_CONTEXT_LENGTH = 2048 // nomic-embed-text has 2K token context
  public static MAX_SAFE_TOKENS = 1600 // Leave buffer for prefix and tokenization variance
  public static TARGET_TOKENS_PER_CHUNK = 1500 // Target 1500 tokens per chunk for embedding
  // Overlap between adjacent chunks. Was an inline literal at the chunker call
  // site; named here because it shapes what lands in the vector store and so
  // must be part of the eval corpus fingerprint.
  public static CHUNK_OVERLAP_TOKENS = 150
  public static PREFIX_TOKEN_BUDGET = 10 // Reserve ~10 tokens for prefixes
  public static CHAR_TO_TOKEN_RATIO = 2 // Conservative chars-per-token estimate; technical docs
  // (numbers, symbols, abbreviations) tokenize denser
  // than plain prose (~3), so 2 avoids context overflows
  // Nomic Embed Text v1.5 uses task-specific prefixes for optimal performance
  public static SEARCH_DOCUMENT_PREFIX = 'search_document: '
  public static SEARCH_QUERY_PREFIX = 'search_query: '
  public static EMBEDDING_BATCH_SIZE = 8 // Conservative batch size for low-end hardware

  constructor(
    private dockerService: DockerService,
    private ollamaService: OllamaService
  ) {}

  private async _initializeQdrantClient() {
    if (!this.qdrantInitPromise) {
      this.qdrantInitPromise = (async () => {
        const qdrantUrl = await this.dockerService.getServiceURL(SERVICE_NAMES.QDRANT)
        if (!qdrantUrl) {
          throw new Error(
            'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.'
          )
        }
        this.qdrant = new QdrantClient({ url: qdrantUrl })
      })().catch((err) => {
        this.qdrantInitPromise = null
        this.qdrant = null
        throw err
      })
    }
    return this.qdrantInitPromise
  }

  public async checkQdrantHealth(): Promise<{ online: boolean; message?: string }> {
    try {
      await this._ensureDependencies()
      await this.qdrant!.getCollections()
      return { online: true }
    } catch {
      this.qdrant = null
      this.qdrantInitPromise = null
      // Qdrant may have restarted (or been recreated) — re-verify collections on reconnect
      this.ensuredCollections.clear()
      return {
        online: false,
        message:
          'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.',
      }
    }
  }

  private async _ensureDependencies() {
    if (!this.qdrant) {
      await this._initializeQdrantClient()
    }
  }

  private async _ensureCollection(
    collectionName: string,
    dimensions: number = RagService.EMBEDDING_DIMENSION
  ) {
    try {
      await this._ensureDependencies()

      if (this.ensuredCollections.has(collectionName)) {
        return
      }

      const collections = await this.qdrant!.getCollections()
      const collectionExists = collections.collections.some((col) => col.name === collectionName)

      if (!collectionExists) {
        await this.qdrant!.createCollection(collectionName, {
          vectors: {
            size: dimensions,
            distance: 'Cosine',
          },
        })
      }

      // Create payload indexes for faster filtering (idempotent — Qdrant ignores duplicates)
      await this.qdrant!.createPayloadIndex(collectionName, {
        field_name: 'source',
        field_schema: 'keyword',
      })
      await this.qdrant!.createPayloadIndex(collectionName, {
        field_name: 'content_type',
        field_schema: 'keyword',
      })
      await this.qdrant!.createPayloadIndex(collectionName, {
        field_name: 'collection',
        field_schema: 'keyword',
      })
      await this.qdrant!.createPayloadIndex(collectionName, {
        field_name: 'active',
        field_schema: 'bool',
      })

      // Backfill: stamp `active: true` on any point that predates this field.
      // `is_empty` (not a match-all filter) means this only ever touches points
      // that have never been explicitly set, so re-running this on every boot
      // can never clobber a user's explicit toggle-off. This is data hygiene
      // only — search correctness never depends on it having run (see the
      // must_not filter in searchSimilarDocuments).
      await this.qdrant!.setPayload(collectionName, {
        payload: { active: true },
        filter: { must: [{ is_empty: { key: 'active' } }] },
      })

      // Only memoize after every step succeeded, so a partial failure is retried
      this.ensuredCollections.add(collectionName)
    } catch (error) {
      logger.error('Error ensuring Qdrant collection:', error)
      throw error
    }
  }

  /**
   * Sanitizes text to ensure it's safe for JSON encoding and Qdrant storage.
   * Removes problematic characters that can cause "unexpected end of hex escape" errors:
   * - Null bytes (\x00)
   * - Invalid Unicode sequences
   * - Control characters (except newlines, tabs, and carriage returns)
   */
  private sanitizeText(text: string): string {
    return (
      text
        // Null bytes
        .replace(/\x00/g, '')
        // Problematic control characters (keep \n, \r, \t)
        .replace(/[\x01-\x08\x0B-\x0C\x0E-\x1F\x7F]/g, '')
        // Invalid Unicode surrogates
        .replace(/[\uD800-\uDFFF]/g, '')
        // Trim extra whitespace
        .trim()
    )
  }

  /**
   * Estimates token count for text. This is a conservative approximation:
   * - English text: ~1 token per 3 characters
   * - Adds buffer for special characters and tokenization variance
   *
   * Note: This is approximate and realistic english
   * tokenization is ~4 chars/token, but we use 3 here to be safe.
   * Actual tokenization may differ, but being
   * conservative prevents context length errors.
   */
  private estimateTokenCount(text: string): number {
    // This accounts for special characters, numbers, and punctuation
    return Math.ceil(text.length / RagService.CHAR_TO_TOKEN_RATIO)
  }

  /**
   * Truncates text to fit within token limit, preserving word boundaries.
   * Ensures the text + prefix won't exceed the model's context window.
   */
  private truncateToTokenLimit(text: string, maxTokens: number): string {
    const estimatedTokens = this.estimateTokenCount(text)

    if (estimatedTokens <= maxTokens) {
      return text
    }

    // Calculate how many characters we can keep using our ratio
    const maxChars = Math.floor(maxTokens * RagService.CHAR_TO_TOKEN_RATIO)

    // Truncate at word boundary
    let truncated = text.substring(0, maxChars)
    const lastSpace = truncated.lastIndexOf(' ')

    if (lastSpace > maxChars * 0.8) {
      // If we found a space in the last 20%, use it
      truncated = truncated.substring(0, lastSpace)
    }

    logger.warn(
      `[RAG] Truncated text from ${text.length} to ${truncated.length} chars (est. ${estimatedTokens} → ${this.estimateTokenCount(truncated)} tokens)`
    )

    return truncated
  }

  /**
   * Preprocesses a query to improve retrieval by expanding it with context.
   * This helps match documents even when using different terminology.
   * TODO: We could probably move this to a separate QueryPreprocessor class if it grows more complex, but for now it's manageable here.
   */
  private static QUERY_EXPANSION_DICTIONARY: Record<string, string> = {
    bob: 'bug out bag',
    bov: 'bug out vehicle',
    bol: 'bug out location',
    edc: 'every day carry',
    mre: 'meal ready to eat',
    shtf: 'shit hits the fan',
    teotwawki: 'the end of the world as we know it',
    opsec: 'operational security',
    ifak: 'individual first aid kit',
    ghb: 'get home bag',
    ghi: 'get home in',
    wrol: 'without rule of law',
    emp: 'electromagnetic pulse',
    ham: 'ham amateur radio',
    nbr: 'nuclear biological radiological',
    cbrn: 'chemical biological radiological nuclear',
    sar: 'search and rescue',
    comms: 'communications radio',
    fifo: 'first in first out',
    mylar: 'mylar bag food storage',
    paracord: 'paracord 550 cord',
    ferro: 'ferro rod fire starter',
    bivvy: 'bivvy bivy emergency shelter',
    bdu: 'battle dress uniform',
    gmrs: 'general mobile radio service',
    frs: 'family radio service',
    nbc: 'nuclear biological chemical',
  }

  private preprocessQuery(query: string): string {
    let expanded = query.trim()

    // Expand known domain abbreviations/acronyms
    const words = expanded.toLowerCase().split(/\s+/)
    const expansions: string[] = []

    for (const word of words) {
      const cleaned = word.replace(/[^\w]/g, '')
      if (RagService.QUERY_EXPANSION_DICTIONARY[cleaned]) {
        expansions.push(RagService.QUERY_EXPANSION_DICTIONARY[cleaned])
      }
    }

    if (expansions.length > 0) {
      expanded = `${expanded} ${expansions.join(' ')}`
      logger.debug(`[RAG] Query expanded with domain terms: "${expanded}"`)
    }

    logger.debug(`[RAG] Original query: "${query}"`)
    logger.debug(`[RAG] Preprocessed query: "${expanded}"`)
    return expanded
  }

  /**
   * Extract keywords from query for hybrid search
   */
  private extractKeywords(query: string): string[] {
    const split = query.split(' ')
    const noStopWords = removeStopwords(split)

    // Future: This is basic normalization, could be improved with stemming/lemmatization later
    const keywords = noStopWords
      .map((word) => word.replace(/[^\w]/g, '').toLowerCase())
      .filter((word) => word.length > 2)

    return [...new Set(keywords)]
  }

  public async embedAndStoreText(
    text: string,
    metadata: Record<string, any> = {},
    onProgress?: (percent: number) => Promise<void>
  ): Promise<{ chunks: number } | null> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      if (!this.embeddingModelVerified) {
        const allModels = await this.ollamaService.getModels(true)
        const embeddingModel =
          allModels.find((model) => model.name === EMBEDDING_MODEL_NAME) ??
          allModels.find((model) => model.name.toLowerCase().includes('nomic-embed-text'))

        if (!embeddingModel) {
          try {
            const downloadResult = await this.ollamaService.downloadModel(EMBEDDING_MODEL_NAME)
            if (!downloadResult.success) {
              throw new Error(downloadResult.message || 'Unknown error during model download')
            }
          } catch (modelError) {
            logger.error(
              `[RAG] Embedding model ${EMBEDDING_MODEL_NAME} not found locally and failed to download:`,
              modelError
            )
            this.embeddingModelVerified = false
            return null
          }
        }
        this.resolvedEmbeddingModel = embeddingModel?.name ?? EMBEDDING_MODEL_NAME
        this.embeddingModelVerified = true
      }

      // TokenChunker uses character-based tokenization (1 char = 1 token)
      // We need to convert our embedding model's token counts to character counts
      // since nomic-embed-text tokenizer uses ~3 chars per token
      const targetCharsPerChunk = Math.floor(RagService.TARGET_TOKENS_PER_CHUNK * RagService.CHAR_TO_TOKEN_RATIO)
      const overlapChars = Math.floor(RagService.CHUNK_OVERLAP_TOKENS * RagService.CHAR_TO_TOKEN_RATIO)

      const chunker = await TokenChunker.create({
        chunkSize: targetCharsPerChunk,
        chunkOverlap: overlapChars,
      })

      const chunkResults = await chunker.chunk(text)

      if (!chunkResults || chunkResults.length === 0) {
        throw new Error('No text chunks generated for embedding.')
      }

      // Extract text from chunk results
      const chunks = chunkResults.map((chunk) => chunk.text)

      // Prepare all chunk texts with prefix and truncation
      const prefixedChunks: string[] = []
      for (let i = 0; i < chunks.length; i++) {
        let chunkText = chunks[i]

        // Final safety check: ensure chunk + prefix fits
        const prefixText = RagService.SEARCH_DOCUMENT_PREFIX
        const withPrefix = prefixText + chunkText
        const estimatedTokens = this.estimateTokenCount(withPrefix)

        if (estimatedTokens > RagService.MAX_SAFE_TOKENS) {
          const prefixTokens = this.estimateTokenCount(prefixText)
          const maxTokensForText = RagService.MAX_SAFE_TOKENS - prefixTokens
          logger.warn(
            `[RAG] Chunk ${i} estimated at ${estimatedTokens} tokens (${chunkText.length} chars), truncating to ${maxTokensForText} tokens`
          )
          chunkText = this.truncateToTokenLimit(chunkText, maxTokensForText)
        }

        prefixedChunks.push(RagService.SEARCH_DOCUMENT_PREFIX + chunkText)
      }

      // Batch embed chunks for performance
      const embeddings: number[][] = []
      const batchSize = RagService.EMBEDDING_BATCH_SIZE
      const totalBatches = Math.ceil(prefixedChunks.length / batchSize)

      for (let batchIdx = 0; batchIdx < totalBatches; batchIdx++) {
        const batchStart = batchIdx * batchSize
        const batch = prefixedChunks.slice(batchStart, batchStart + batchSize)

        logger.debug(
          `[RAG] Embedding batch ${batchIdx + 1}/${totalBatches} (${batch.length} chunks)`
        )

        const response = await this.ollamaService.embed(
          this.resolvedEmbeddingModel ?? EMBEDDING_MODEL_NAME,
          batch
        )

        embeddings.push(...response.embeddings)

        if (onProgress) {
          const progress = ((batchStart + batch.length) / prefixedChunks.length) * 100
          await onProgress(progress)
        }
      }

      const timestamp = Date.now()

      // Sanitize source metadata once up front — every chunk in this call shares
      // the same source.
      const sanitizedSource =
        typeof metadata.source === 'string' ? this.sanitizeText(metadata.source) : 'unknown'

      // Preserve an existing file's active/inactive toggle across re-ingestion
      // (retries, force re-embeds, replaced-content reindexing, etc.) instead of
      // resetting it to active on every write. A genuinely new file (no row yet)
      // still defaults to active.
      const active = await this._readActiveFlag(sanitizedSource)

      const points = chunks.map((chunkText, index) => {
        // Sanitize text to prevent JSON encoding errors
        const sanitizedText = this.sanitizeText(chunkText)

        // Extract keywords from content
        const contentKeywords = this.extractKeywords(sanitizedText)

        // For ZIM content, also extract keywords from structural metadata
        let structuralKeywords: string[] = []
        if (metadata.full_title) {
          structuralKeywords = this.extractKeywords(metadata.full_title as string)
        } else if (metadata.article_title) {
          structuralKeywords = this.extractKeywords(metadata.article_title as string)
        }

        // Combine and dedup keywords
        const allKeywords = [...new Set([...structuralKeywords, ...contentKeywords])]

        logger.debug(`[RAG] Extracted keywords for chunk ${index}: [${allKeywords.join(', ')}]`)
        if (structuralKeywords.length > 0) {
          logger.debug(
            `[RAG]   - Structural: [${structuralKeywords.join(', ')}], Content: [${contentKeywords.join(', ')}]`
          )
        }

        return {
          id: randomUUID(), // qdrant requires either uuid or unsigned int
          vector: embeddings[index],
          payload: {
            ...metadata,
            text: sanitizedText,
            chunk_index: index,
            total_chunks: chunks.length,
            keywords: allKeywords.join(' '), // store as space-separated string for text search
            char_count: sanitizedText.length,
            created_at: timestamp,
            source: sanitizedSource,
            active,
          },
        }
      })

      await this.qdrant!.upsert(RagService.CONTENT_COLLECTION_NAME, { points })

      // A toggle can land between the read above and the upsert, and its
      // setPayload then misses these points. The toggles write the row before
      // Qdrant, so re-reading after the upsert closes the gap: either this read
      // sees the new value, or the toggle's setPayload runs after the upsert and
      // covers these points itself.
      const currentActive = await this._readActiveFlag(sanitizedSource)
      if (currentActive !== active) {
        await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, {
          payload: { active: currentActive },
          points: points.map((p) => p.id),
        })
      }

      logger.debug(`[RAG] Successfully embedded and stored ${chunks.length} chunks`)
      logger.debug(`[RAG] First chunk preview: "${chunks[0].substring(0, 100)}..."`)

      return { chunks: chunks.length }
    } catch (error) {
      console.error(error)
      logger.error('[RAG] Error embedding text:', error)
      return null
    }
  }

  private async preprocessImage(filebuffer: Buffer): Promise<Buffer> {
    return await sharp(filebuffer)
      .grayscale()
      .normalize()
      .sharpen()
      .resize({ width: 2000, fit: 'inside' })
      .toBuffer()
  }

  private async convertPDFtoImages(filebuffer: Buffer): Promise<Buffer[]> {
    const converted = await fromBuffer(filebuffer, {
      quality: 50,
      density: 200,
      format: 'png',
    }).bulk(-1, {
      responseType: 'buffer',
    })
    return converted.filter((res) => res.buffer).map((res) => res.buffer!)
  }

  private async extractPDFText(filebuffer: Buffer): Promise<string> {
    const parser = new PDFParse({ data: filebuffer })
    const data = await parser.getText()
    await parser.destroy()
    return data.text
  }

  private async extractTXTText(filebuffer: Buffer): Promise<string> {
    return filebuffer.toString('utf-8')
  }

  private async extractImageText(filebuffer: Buffer): Promise<string> {
    const worker = await createWorker('eng')
    const result = await worker.recognize(filebuffer)
    await worker.terminate()
    return result.data.text
  }

  private async processImageFile(fileBuffer: Buffer): Promise<string> {
    const preprocessedBuffer = await this.preprocessImage(fileBuffer)
    return await this.extractImageText(preprocessedBuffer)
  }

  /**
   * Will process the PDF and attempt to extract text.
   * If the extracted text is minimal, it will fallback to OCR on each page.
   */
  private async processPDFFile(fileBuffer: Buffer): Promise<string> {
    let extractedText = await this.extractPDFText(fileBuffer)

    // Check if there was no extracted text or it was very minimal
    if (!extractedText || extractedText.trim().length < 100) {
      logger.debug('[RAG] PDF text extraction minimal, attempting OCR on pages')
      // Convert PDF pages to images for OCR if text extraction was poor
      const imageBuffers = await this.convertPDFtoImages(fileBuffer)
      extractedText = ''

      for (const imgBuffer of imageBuffers) {
        const preprocessedImg = await this.preprocessImage(imgBuffer)
        const pageText = await this.extractImageText(preprocessedImg)
        extractedText += pageText + '\n'
      }
    }

    return extractedText
  }

  /**
   * Process a ZIM file: extract content with metadata and embed each chunk.
   * Returns early with complete result since ZIM processing is self-contained.
   * Supports batch processing to prevent lock timeouts on large ZIM files.
   */
  private async processZIMFile(
    filepath: string,
    deleteAfterEmbedding: boolean,
    batchOffset?: number,
    onProgress?: (percent: number) => Promise<void>,
    collection?: string
  ): Promise<ProcessZIMFileResponse> {
    const zimExtractionService = new ZIMExtractionService()

    // Process in batches to avoid lock timeout
    const startOffset = batchOffset || 0

    logger.info(
      `[RAG] Extracting ZIM content (batch: offset=${startOffset}, size=${ZIM_BATCH_SIZE})`
    )

    const {
      chunks: zimChunks,
      totalArticles,
      articlesProcessed,
    } = await zimExtractionService.extractZIMContent(filepath, {
      startOffset,
      batchSize: ZIM_BATCH_SIZE,
    })

    logger.info(
      `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata (file totalArticles=${totalArticles})`
    )

    // Process each chunk individually with its metadata
    let totalChunks = 0
    for (let i = 0; i < zimChunks.length; i++) {
      const zimChunk = zimChunks[i]
      const result = await this.embedAndStoreText(zimChunk.text, {
        source: filepath,
        content_type: 'zim_article',
        // Without this the ZIM path writes points with no `collection` at all, so
        // getKnowledgeCollections() (which facets on it) never sees them.
        ...(collection ? { collection } : {}),

        // Article-level context
        article_title: zimChunk.articleTitle,
        article_path: zimChunk.articlePath,

        // Section-level context
        section_title: zimChunk.sectionTitle,
        full_title: zimChunk.fullTitle,
        hierarchy: zimChunk.hierarchy,
        section_level: zimChunk.sectionLevel,

        // Use the same document ID for all chunks from the same article for grouping in search results
        document_id: zimChunk.documentId,

        // Archive metadata
        archive_title: zimChunk.archiveMetadata.title,
        archive_creator: zimChunk.archiveMetadata.creator,
        archive_publisher: zimChunk.archiveMetadata.publisher,
        archive_date: zimChunk.archiveMetadata.date,
        archive_language: zimChunk.archiveMetadata.language,
        archive_description: zimChunk.archiveMetadata.description,

        // Extraction metadata - not overly relevant for search, but could be useful for debugging and future features...
        extraction_strategy: zimChunk.strategy,
      })

      if (result) {
        totalChunks += result.chunks
      }

      if (onProgress) {
        await onProgress(((i + 1) / zimChunks.length) * 100)
      }
    }

    // Gate the continuation on articles the extractor CONSUMED, not on articles that
    // produced chunks. Articles whose text is empty after cleaning (redirect stubs,
    // category pages, media/PDF wrappers) contribute no documentIds, so a chunk-derived
    // count reads a normal sparse batch as end-of-archive and silently abandons the rest
    // of the file. See `zim_batch_decision.ts` for the full rationale.
    const articlesWithContent = new Set(zimChunks.map((c) => c.documentId)).size
    const hasMoreBatches = hasMoreArticleBatches({
      articlesProcessed,
      batchSize: ZIM_BATCH_SIZE,
    })

    logger.info(
      `[RAG] Successfully embedded ${totalChunks} total chunks from ${articlesWithContent}/${articlesProcessed} articles (hasMore: ${hasMoreBatches})`
    )

    // Only delete the file when:
    // 1. deleteAfterEmbedding is true (caller wants deletion)
    // 2. No more batches remain (this is the final batch)
    // This prevents race conditions where early batches complete after later ones
    const shouldDelete = deleteAfterEmbedding && !hasMoreBatches
    if (shouldDelete) {
      logger.info(`[RAG] Final batch complete, deleting ZIM file: ${filepath}`)
      await deleteFileIfExists(filepath)
    } else if (!hasMoreBatches) {
      logger.info(`[RAG] Final batch complete, but file deletion was not requested`)
    }

    return {
      success: true,
      message: hasMoreBatches
        ? 'ZIM batch processed successfully. More batches remain.'
        : 'ZIM file processed and embedded successfully with enhanced metadata.',
      chunks: totalChunks,
      hasMoreBatches,
      // MUST be the consumed count: EmbedFileJob advances the next batch offset by this
      // value. Reporting the content-bearing count instead re-reads the overlap on every
      // sparse batch, and a batch that yields no text at all would advance the offset by
      // zero and re-run the same window forever.
      articlesProcessed,
      totalArticles,
    }
  }

  private async processTextFile(fileBuffer: Buffer): Promise<string> {
    return await this.extractTXTText(fileBuffer)
  }

  /**
   * Extract text content from a DOCX file using mammoth. DOCX is a ZIP-based
   * XML format, so raw-text extraction (extractTXTText) would return garbage —
   * this parses the document XML properly and returns clean plain text.
   */
  private async processDocxFile(fileBuffer: Buffer): Promise<string> {
    const { value: text } = await mammoth.extractRawText({ buffer: fileBuffer })
    return text
  }

  /**
   * Extract text content from an EPUB file.
   * EPUBs are ZIP archives containing XHTML content files.
   * Reads the OPF manifest to determine reading order, then extracts
   * text from each content document in sequence.
   */
  private async processEPUBFile(fileBuffer: Buffer): Promise<string> {
    const zip = await JSZip.loadAsync(fileBuffer)

    // Read container.xml to find the OPF file path
    const containerXml = await zip.file('META-INF/container.xml')?.async('text')
    if (!containerXml) {
      throw new Error('Invalid EPUB: missing META-INF/container.xml')
    }

    // Parse container.xml to get the OPF rootfile path
    const $container = cheerio.load(containerXml, { xml: true })
    const opfPath = $container('rootfile').attr('full-path')
    if (!opfPath) {
      throw new Error('Invalid EPUB: no rootfile found in container.xml')
    }

    // Determine the base directory of the OPF file for resolving relative paths
    const opfDir = opfPath.includes('/') ? opfPath.substring(0, opfPath.lastIndexOf('/') + 1) : ''

    // Read and parse the OPF file
    const opfContent = await zip.file(opfPath)?.async('text')
    if (!opfContent) {
      throw new Error(`Invalid EPUB: OPF file not found at ${opfPath}`)
    }

    const $opf = cheerio.load(opfContent, { xml: true })

    // Build a map of manifest items (id -> href)
    const manifestItems = new Map<string, string>()
    $opf('manifest item').each((_, el) => {
      const id = $opf(el).attr('id')
      const href = $opf(el).attr('href')
      const mediaType = $opf(el).attr('media-type') || ''
      // Only include XHTML/HTML content documents
      if (id && href && (mediaType.includes('html') || mediaType.includes('xml'))) {
        manifestItems.set(id, href)
      }
    })

    // Get the reading order from the spine
    const spineOrder: string[] = []
    $opf('spine itemref').each((_, el) => {
      const idref = $opf(el).attr('idref')
      if (idref && manifestItems.has(idref)) {
        spineOrder.push(manifestItems.get(idref)!)
      }
    })

    // If no spine found, fall back to all manifest items
    const contentFiles = spineOrder.length > 0 ? spineOrder : Array.from(manifestItems.values())

    // Extract text from each content file in order
    const textParts: string[] = []
    for (const href of contentFiles) {
      const fullPath = opfDir + href
      const content = await zip.file(fullPath)?.async('text')
      if (content) {
        const $ = cheerio.load(content)
        // Remove script and style elements
        $('script, style').remove()
        const text = $('body').text().trim()
        if (text) {
          textParts.push(text)
        }
      }
    }

    const fullText = textParts.join('\n\n')
    logger.debug(
      `[RAG] EPUB extracted ${textParts.length} chapters, ${fullText.length} characters total`
    )
    return fullText
  }

  private async embedTextAndCleanup(
    extractedText: string,
    filepath: string,
    deleteAfterEmbedding: boolean = false,
    onProgress?: (percent: number) => Promise<void>,
    collection?: string
  ): Promise<{ success: boolean; message: string; chunks?: number }> {
    if (!extractedText || extractedText.trim().length === 0) {
      return {
        success: false,
        message: 'Process completed succesfully, but no text was found to embed.',
      }
    }

    const embedResult = await this.embedAndStoreText(
      extractedText,
      {
        source: filepath,
        ...(collection ? { collection } : {}),
      },
      onProgress
    )

    if (!embedResult) {
      return { success: false, message: 'Failed to embed and store the extracted text.' }
    }

    if (deleteAfterEmbedding) {
      logger.info(`[RAG] Embedding complete, deleting uploaded file: ${filepath}`)
      await deleteFileIfExists(filepath)
    }

    return {
      success: true,
      message: 'File processed and embedded successfully.',
      chunks: embedResult.chunks,
    }
  }

  /**
   * Main pipeline to process and embed an uploaded file into the RAG knowledge base.
   * This includes text extraction, chunking, embedding, and storing in Qdrant.
   *
   * Orchestrates file type detection and delegates to specialized processors.
   * For ZIM files, supports batch processing via batchOffset parameter.
   */
  public async processAndEmbedFile(
    filepath: string,
    deleteAfterEmbedding: boolean = false,
    batchOffset?: number,
    onProgress?: (percent: number) => Promise<void>,
    collection?: string
  ): Promise<ProcessAndEmbedFileResponse> {
    try {
      const fileType = determineFileType(filepath)
      logger.debug(`[RAG] Processing file: ${filepath} (detected type: ${fileType})`)

      if (fileType === 'unknown') {
        return { success: false, message: 'Unsupported file type.' }
      }

      // Read file buffer (not needed for ZIM as it reads directly)
      const fileBuffer = fileType !== 'zim' ? await getFile(filepath, 'buffer') : null
      if (fileType !== 'zim' && !fileBuffer) {
        return { success: false, message: 'Failed to read the uploaded file.' }
      }

      // Process based on file type
      // ZIM files are handled specially since they have their own embedding workflow
      if (fileType === 'zim') {
        return await this.processZIMFile(filepath, deleteAfterEmbedding, batchOffset, onProgress, collection)
      }

      // Extract text based on file type
      // Report ~10% when extraction begins; actual embedding progress follows via callback
      if (onProgress) await onProgress(10)
      let extractedText: string
      switch (fileType) {
        case 'image':
          extractedText = await this.processImageFile(fileBuffer!)
          break
        case 'pdf':
          extractedText = await this.processPDFFile(fileBuffer!)
          break
        case 'docx':
          extractedText = await this.processDocxFile(fileBuffer!)
          break
        case 'epub':
          extractedText = await this.processEPUBFile(fileBuffer!)
          break
        case 'text':
        default:
          extractedText = await this.processTextFile(fileBuffer!)
          break
      }

      // Extraction done — scale remaining embedding progress from 15% to 100%
      if (onProgress) await onProgress(15)
      const scaledProgress = onProgress ? (p: number) => onProgress(15 + p * 0.85) : undefined

      // Embed extracted text and cleanup
      return await this.embedTextAndCleanup(
        extractedText,
        filepath,
        deleteAfterEmbedding,
        scaledProgress,
        collection
      )
    } catch (error) {
      logger.error('[RAG] Error processing and embedding file:', error)
      return { success: false, message: 'Error processing and embedding file.' }
    }
  }

  /**
   * Search for documents similar to the query text in the Qdrant knowledge base.
   * Uses a hybrid approach combining semantic similarity and keyword matching.
   * Implements adaptive thresholds and result reranking for optimal retrieval.
   * @param query - The search query text
   * @param limit - Maximum number of results to return (default: 5)
   * @param scoreThreshold - Minimum similarity score threshold (default: 0.3, much lower than before)
   * @param minFinalScore - Post-rerank relevance floor; below it a chunk is dropped
   * @returns Array of relevant text chunks with their scores
   */
  public async searchSimilarDocuments(
    query: string,
    limit: number = 5,
    scoreThreshold: number = 0.3, // Lower default threshold - was 0.7, now 0.3
    collection?: string,
    /**
     * Optional sink for the intermediate ranked lists. When supplied, the raw
     * dense order, the post-rerank order, and the post-diversity order are all
     * written here. Purely observational — nothing about the returned result
     * changes — and it is what lets the eval harness answer "is the reranking
     * heuristic actually helping?" without a second implementation of search.
     */
    stagesOut?: RetrievalStages,
    /**
     * Relevance floor on the post-rerank score. Defaults to 0 so the parameter
     * is purely additive for callers that predate it; the chat pipeline and the
     * eval harness both pass a real value.
     */
    minFinalScore: number = 0,
    /** Optional sink for the floor's counts; see RetrievalFloorStats. */
    floorOut?: RetrievalFloorStats
  ): Promise<Array<{ text: string; score: number; metadata?: Record<string, any> }>> {
    try {
      logger.debug(`[RAG] Starting similarity search for query: "${query}"`)

      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      // Check if collection has any points
      const collectionInfo = await this.qdrant!.getCollection(RagService.CONTENT_COLLECTION_NAME)
      const pointCount = collectionInfo.points_count || 0
      logger.debug(`[RAG] Knowledge base contains ${pointCount} document chunks`)

      if (pointCount === 0) {
        logger.debug('[RAG] Knowledge base is empty. Could not perform search.')
        return []
      }

      if (!this.embeddingModelVerified) {
        const allModels = await this.ollamaService.getModels(true)
        const embeddingModel =
          allModels.find((model) => model.name === EMBEDDING_MODEL_NAME) ??
          allModels.find((model) => model.name.toLowerCase().includes('nomic-embed-text'))

        if (!embeddingModel) {
          logger.warn(`[RAG] ${EMBEDDING_MODEL_NAME} not found. Cannot perform similarity search.`)
          this.embeddingModelVerified = false
          return []
        }
        this.resolvedEmbeddingModel = embeddingModel.name
        this.embeddingModelVerified = true
      }

      // Preprocess query for better matching
      const processedQuery = this.preprocessQuery(query)
      const keywords = this.extractKeywords(processedQuery)
      logger.debug(`[RAG] Extracted keywords: [${keywords.join(', ')}]`)

      // Generate embedding for the query with search_query prefix
      // Ensure query doesn't exceed token limit
      const prefixTokens = this.estimateTokenCount(RagService.SEARCH_QUERY_PREFIX)
      const maxQueryTokens = RagService.MAX_SAFE_TOKENS - prefixTokens
      const truncatedQuery = this.truncateToTokenLimit(processedQuery, maxQueryTokens)

      const prefixedQuery = RagService.SEARCH_QUERY_PREFIX + truncatedQuery
      logger.debug(`[RAG] Generating embedding with prefix: "${RagService.SEARCH_QUERY_PREFIX}"`)

      // Validate final token count
      const queryTokenCount = this.estimateTokenCount(prefixedQuery)
      if (queryTokenCount > RagService.MAX_SAFE_TOKENS) {
        logger.error(
          `[RAG] Query too long even after truncation: ${queryTokenCount} tokens (max: ${RagService.MAX_SAFE_TOKENS})`
        )
        return []
      }

      const response = await this.ollamaService.embed(
        this.resolvedEmbeddingModel ?? EMBEDDING_MODEL_NAME,
        [prefixedQuery]
      )

      // Perform semantic search with a higher limit to enable reranking
      const searchLimit = limit * 3 // Get more results for reranking
      logger.debug(
        `[RAG] Searching for top ${searchLimit} semantic matches (threshold: ${scoreThreshold})`
      )

      const searchResults = await this.qdrant!.search(RagService.CONTENT_COLLECTION_NAME, {
        vector: response.embeddings[0],
        limit: searchLimit,
        score_threshold: scoreThreshold,
        with_payload: true,
        filter: {
          // Denylist, not allowlist: a point with no `active` field at all
          // (not yet backfilled) must still be found by default. Search
          // correctness must never depend on the backfill's timing.
          must_not: [{ key: 'active', match: { value: false } }],
          ...(collection ? { must: [{ key: 'collection', match: { value: collection } }] } : {}),
        },
      })

      logger.debug(`[RAG] Found ${searchResults.length} results above threshold ${scoreThreshold}`)

      // Map results with metadata for reranking
      const resultsWithMetadata: RAGResult[] = searchResults.map((result) => ({
        text: (result.payload?.text as string) || '',
        score: result.score,
        keywords: (result.payload?.keywords as string) || '',
        chunk_index: (result.payload?.chunk_index as number) || 0,
        created_at: (result.payload?.created_at as number) || 0,
        // Enhanced ZIM metadata (likely be undefined for non-ZIM content)
        article_title: result.payload?.article_title as string | undefined,
        section_title: result.payload?.section_title as string | undefined,
        full_title: result.payload?.full_title as string | undefined,
        hierarchy: result.payload?.hierarchy as string | undefined,
        document_id: result.payload?.document_id as string | undefined,
        content_type: result.payload?.content_type as string | undefined,
        source: result.payload?.source as string | undefined,
        // Citation metadata (#1179) -- date/title of the archive a chunk was
        // extracted from, when known. Undefined for non-ZIM content, which
        // carries no equivalent embedded metadata.
        archive_title: result.payload?.archive_title as string | undefined,
        archive_date: result.payload?.archive_date as string | undefined,
      }))

      const rerankedResults = this.rerankResults(resultsWithMetadata, keywords, query)

      logger.debug(`[RAG] Top 3 results after reranking:`)
      rerankedResults.slice(0, 3).forEach((result, idx) => {
        logger.debug(
          `[RAG]   ${idx + 1}. Score: ${result.finalScore.toFixed(4)} (semantic: ${result.score.toFixed(4)}) - "${result.text.substring(0, 100)}..."`
        )
      })

      // Relevance floor, applied here — after reranking, before diversity.
      //
      // The ordering is deliberate. The floor is a judgement about relevance,
      // and the reranked score is where that judgement is best informed. The
      // diversity penalty that follows is about redundancy, not relevance: it
      // multiplies by 0.85^n, so flooring afterwards would drop the fourth chunk
      // of the one document that actually answers the question and blame a knob
      // labelled "relevance" for it.
      const { survivors, belowFloor } = applyRelevanceFloor(rerankedResults, minFinalScore)
      if (floorOut) {
        floorOut.candidates = rerankedResults.length
        floorOut.belowFloor = belowFloor
      }
      if (belowFloor > 0) {
        logger.debug(
          `[RAG] Relevance floor ${minFinalScore}: dropped ${belowFloor} of ${rerankedResults.length} chunk(s)`
        )
      }
      if (survivors.length === 0 && rerankedResults.length > 0) {
        // Nothing cleared the bar. Returning empty is the point: the pipeline
        // then injects no context block at all, rather than handing the model
        // passages it has to be talked out of using.
        logger.debug(
          `[RAG] Nothing cleared the relevance floor (best was ${rerankedResults[0].finalScore.toFixed(4)}) — injecting no context`
        )
      }

      // Apply source diversity penalty to avoid all results from the same document
      const diverseResults = this.applySourceDiversity(survivors)

      // Record the three ranked lists for the eval harness's stage ablation.
      // Sliced to `limit` so each stage is compared on the window that would
      // actually have been injected, not on the wider rerank candidate pool.
      //
      // `reranked` is recorded pre-floor on purpose: the ablation asks whether
      // each heuristic *orders* better, and a filter applied to two of the three
      // lists would answer a different question. The floor's own effect is
      // measured by sweeping --min-final-score against the headline metrics.
      if (stagesOut) {
        const summarize = (rows: Array<{ text: string; score: number; source?: string }>) =>
          rows.slice(0, limit).map((r) => ({ source: r.source, score: r.score }))
        stagesOut.dense = summarize(resultsWithMetadata)
        stagesOut.reranked = rerankedResults
          .slice(0, limit)
          .map((r) => ({ source: r.source, score: r.finalScore }))
        stagesOut.diversified = diverseResults
          .slice(0, limit)
          .map((r) => ({ source: r.source, score: r.finalScore }))
        stagesOut.candidates = rerankedResults.map((r) => ({
          source: r.source,
          score: r.finalScore,
          semanticScore: r.score,
        }))
      }

      // Return top N results with enhanced metadata
      return diverseResults.slice(0, limit).map((result) => ({
        text: result.text,
        score: result.finalScore,
        metadata: {
          chunk_index: result.chunk_index,
          created_at: result.created_at,
          semantic_score: result.score,
          // The originating file/ZIM path. Stored on every point but previously
          // dropped here, which made it impossible to map a retrieved chunk back
          // to its document — needed for citations and for recall@k scoring.
          source: result.source,
          // Enhanced ZIM metadata (likely be undefined for non-ZIM content)
          article_title: result.article_title,
          section_title: result.section_title,
          full_title: result.full_title,
          hierarchy: result.hierarchy,
          document_id: result.document_id,
          content_type: result.content_type,
          // Citation metadata (#1179)
          archive_title: result.archive_title,
          archive_date: result.archive_date,
        },
      }))
    } catch (error) {
      logger.error('[RAG] Error searching similar documents:', error)
      return []
    }
  }

  /**
   * Rerank search results using hybrid scoring that combines:
   * 1. Semantic similarity score (primary signal)
   * 2. Keyword overlap bonus (conservative, quality-gated)
   * 3. Direct term matches (conservative)
   *
   * Tries to boost only already-relevant results, not promote
   * low-quality results just because they have keyword matches.
   *
   * Future: this is a decent feature-based approach, but we could
   * switch to a python-based reranker in the future if the benefits
   * outweigh the overhead.
   */
  private rerankResults(
    results: Array<RAGResult>,
    queryKeywords: string[],
    originalQuery: string
  ): Array<RerankedRAGResult> {
    return results
      .map((result) => {
        let finalScore = result.score

        // Quality gate: Only apply boosts if semantic score is reasonable
        // Try to prevent promoting irrelevant results that just happen to have keyword matches
        const MIN_SEMANTIC_THRESHOLD = 0.35

        if (result.score < MIN_SEMANTIC_THRESHOLD) {
          // For low-scoring results, use semantic score as-is
          // This prevents false positives from keyword gaming
          logger.debug(
            `[RAG] Skipping boost for low semantic score: ${result.score.toFixed(3)} (threshold: ${MIN_SEMANTIC_THRESHOLD})`
          )
          return {
            ...result,
            finalScore,
          }
        }

        // Boost score based on keyword overlap (diminishing returns - overlap goes down, so does boost)
        const docKeywords = result.keywords
          .toLowerCase()
          .split(' ')
          .filter((k) => k.length > 0)
        const matchingKeywords = queryKeywords.filter(
          (kw) =>
            docKeywords.includes(kw.toLowerCase()) ||
            result.text.toLowerCase().includes(kw.toLowerCase())
        )
        const keywordOverlap = matchingKeywords.length / Math.max(queryKeywords.length, 1)

        // Use square root for diminishing returns: 100% overlap = sqrt(1.0) = 1.0, 25% = 0.5
        // Then scale conservatively (max 10% boost instead of 20%)
        const keywordBoost = Math.sqrt(keywordOverlap) * 0.1 * result.score

        if (keywordOverlap > 0) {
          logger.debug(
            `[RAG] Keyword overlap: ${matchingKeywords.length}/${queryKeywords.length} - Boost: ${keywordBoost.toFixed(3)}`
          )
        }

        // Boost if original query terms appear in text (case-insensitive)
        // Scale boost proportionally to base score to avoid over-promoting weak matches
        const queryTerms = originalQuery
          .toLowerCase()
          .split(/\s+/)
          .filter((t) => t.length > 3)
        const directMatches = queryTerms.filter((term) =>
          result.text.toLowerCase().includes(term)
        ).length

        if (queryTerms.length > 0) {
          const directMatchRatio = directMatches / queryTerms.length
          // Conservative boost: max 7.5% of the base score
          const directMatchBoost = Math.sqrt(directMatchRatio) * 0.075 * result.score

          if (directMatches > 0) {
            logger.debug(
              `[RAG] Direct term matches: ${directMatches}/${queryTerms.length} - Boost: ${directMatchBoost.toFixed(3)}`
            )
            finalScore += directMatchBoost
          }
        }

        // Boost when query keywords match the chunk's section/article heading. ZIM
        // content carries this structural metadata (already fetched, no extra cost),
        // and a query term appearing in a heading is a strong relevance signal that
        // body-text matching alone misses. Same conservative, score-scaled, diminishing
        // -returns shape as the boosts above, so it can't promote a weak match.
        const headingText = [result.full_title, result.section_title, result.article_title]
          .filter(Boolean)
          .join(' ')
          .toLowerCase()
        if (headingText) {
          const headingHits = queryKeywords.filter((kw) =>
            headingText.includes(kw.toLowerCase())
          ).length
          if (headingHits > 0) {
            const headingRatio = headingHits / Math.max(queryKeywords.length, 1)
            const headingBoost = Math.sqrt(headingRatio) * 0.1 * result.score
            logger.debug(
              `[RAG] Heading match: ${headingHits}/${queryKeywords.length} - Boost: ${headingBoost.toFixed(3)}`
            )
            finalScore += headingBoost
          }
        }

        finalScore = Math.min(1.0, finalScore + keywordBoost)

        return {
          ...result,
          finalScore,
        }
      })
      .sort((a, b) => b.finalScore - a.finalScore)
  }

  /**
   * Applies a diversity penalty so results from the same source are down-weighted.
   * Uses greedy selection: for each result, apply 0.85^n penalty where n is the
   * number of results already selected from the same source.
   */
  private applySourceDiversity(results: Array<RerankedRAGResult>) {
    const sourceCounts = new Map<string, number>()
    const DIVERSITY_PENALTY = 0.85

    return results
      .map((result) => {
        const sourceKey = result.document_id || result.source || 'unknown'
        const count = sourceCounts.get(sourceKey) || 0
        const penalty = Math.pow(DIVERSITY_PENALTY, count)
        const diverseScore = result.finalScore * penalty

        sourceCounts.set(sourceKey, count + 1)

        if (count > 0) {
          logger.debug(
            `[RAG] Source diversity penalty for "${sourceKey}": ${result.finalScore.toFixed(4)} → ${diverseScore.toFixed(4)} (seen ${count}x)`
          )
        }

        return { ...result, finalScore: diverseScore }
      })
      .sort((a, b) => b.finalScore - a.finalScore)
  }

  /**
   * Retrieve all unique source files that have been stored in the knowledge base.
   * @returns Array of unique full source paths
   */
  public async hasDocuments(): Promise<boolean> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )
      const collectionInfo = await this.qdrant!.getCollection(RagService.CONTENT_COLLECTION_NAME)
      return (collectionInfo.points_count ?? 0) > 0
    } catch {
      return false
    }
  }

  public async getStoredFiles(): Promise<StoredFileInfo[]> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      // Use Qdrant's facet API to enumerate distinct `source` values in one
      // call. The previous scroll-loop walked every point in the collection
      // (3M+ on a fully-ingested NOMAD) just to learn the ~40 unique sources,
      // which made this endpoint take 50+ seconds. Facet returns the unique
      // values directly. `exact: true` so the count we'll reuse for warnings
      // matches what would be reported by an exhaustive walk.
      const sources = new Set<string>()
      const facetResult = await this.qdrant!.facet(RagService.CONTENT_COLLECTION_NAME, {
        key: 'source',
        limit: RagService.FACET_SOURCE_LIMIT,
        exact: true,
        // Keep the developer eval corpus out of the user's Stored Files list.
        // It shares this Qdrant collection but is not the user's content, and
        // its fixture paths would render as broken/missing files in the KB UI.
        filter: EXCLUDE_EVAL_FILTER,
      })
      for (const hit of facetResult.hits) {
        if (typeof hit.value === 'string') sources.add(hit.value)
      }

      // Union the Qdrant-derived list with the disk-backed file paths the
      // state machine has tracked. Without this, files known to the scanner
      // but with zero embedded chunks (video-only ZIMs, failed-before-first-
      // chunk ingestions, browse_only opt-outs) never get a row in Stored
      // Files — which means warnings keyed off those files (#895 zero_chunks
      // in particular) have no row to attach to. The state machine is the
      // authoritative "what's on disk?" view; Qdrant is "what made it into
      // the vector store?". Both are needed to render the KB UI honestly.
      const stateByPath = new Map<
        string,
        {
          state: KbIngestStateValue
          chunks_embedded: number
          collection: string | null
          active: boolean
        }
      >()
      try {
        const stateRows = await KbIngestState.query().select(
          'file_path',
          'state',
          'chunks_embedded',
          'collection',
          'active'
        )
        for (const row of stateRows) {
          sources.add(row.file_path)
          stateByPath.set(row.file_path, {
            state: row.state,
            chunks_embedded: row.chunks_embedded,
            collection: row.collection,
            // MySQL hands back tinyint(1) as 0/1, not a boolean. Coerce here so
            // every consumer sees a real boolean -- the raw 1 reached the DOM as
            // aria-checked="1", which is not a valid ARIA value and leaves screen
            // readers with no idea whether the toggle is on.
            active: Boolean(row.active),
          })
        }
      } catch (error) {
        // Non-fatal: if the state machine query fails for any reason we'd
        // rather return the Qdrant-derived list than 500 the whole panel.
        logger.warn(
          { err: error },
          '[RagService.getStoredFiles] state-machine union skipped; returning Qdrant-only list'
        )
      }

      const uploadsAbsPath = resolve(join(process.cwd(), RagService.UPLOADS_STORAGE_PATH))
      return await Promise.all(
        Array.from(sources).map(async (source) => {
          const row = stateByPath.get(source)
          const fileName = source.split(/[/\\]/).at(-1) ?? source
          const isUserUpload = resolve(source).startsWith(uploadsAbsPath + sep)
          const stats = await getFileStatsIfExists(source)
          return {
            source,
            state: row?.state ?? null,
            chunksEmbedded: row?.chunks_embedded ?? 0,
            fileName,
            size: stats?.size ?? null,
            uploadedAt: stats?.modifiedTime.toISOString() ?? null,
            isUserUpload,
            collection: row?.collection ?? null,
            active: row ? Boolean(row.active) : true,
          }
        })
      )
    } catch (error) {
      logger.error('Error retrieving stored files:', error)
      return []
    }
  }

  /**
   * Enumerate distinct `collection` values currently in the knowledge base,
   * for populating a subject-picker in the upload/chat UI. Mirrors the
   * `source` facet pattern used elsewhere in this file (see getStoredFiles).
   */
  public async getKnowledgeCollections(): Promise<string[]> {
    await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
    const facetResult = await this.qdrant!.facet(RagService.CONTENT_COLLECTION_NAME, {
      key: 'collection',
      limit: RagService.FACET_SOURCE_LIMIT,
      exact: true,
    })
    const collections = new Set<string>()
    for (const hit of facetResult.hits) {
      // The reserved eval tag is internal; never offer it in a subject picker.
      if (typeof hit.value === 'string' && hit.value !== KB_EVAL_COLLECTION) {
        collections.add(hit.value)
      }
    }
    return Array.from(collections).sort()
  }

  /**
   * Reassign a stored file's collection after the fact. Updates the `collection`
   * payload field on every existing Qdrant point for this source in place (no
   * re-chunking or re-embedding needed), then mirrors the change onto the
   * KbIngestState row so getStoredFiles() reflects it immediately.
   */
  public async updateFileCollection(
    source: string,
    collection: string | null
  ): Promise<{ success: boolean; message: string }> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, {
        payload: { collection },
        filter: { must: [{ key: 'source', match: { value: source } }] },
      })

      // The setPayload above only reaches points that already exist, so for a file
      // that has not been indexed yet it matches nothing. The row is therefore the
      // only durable record of the choice until EmbedFileJob picks it up -- create
      // it when absent rather than reporting success and storing the value nowhere.
      const row = await KbIngestState.getOrCreate(source)
      row.collection = collection
      await row.save()

      return {
        success: true,
        message: collection ? `Moved to "${collection}".` : 'Moved to Uncategorized.',
      }
    } catch (error) {
      logger.error('[RAG] Error updating file collection:', error)
      return { success: false, message: 'Error updating file collection.' }
    }
  }

  /**
   * Toggle a file's active (searchable) state. Writes the KbIngestState row,
   * then updates the `active` payload field on every existing Qdrant point for
   * this source in place — no deletion or re-embedding, so this is instant in
   * either direction. Vectors stay in Qdrant permanently either way; only
   * `searchSimilarDocuments()`'s query-time filter is affected. See #1119.
   *
   * The row goes first so an in-flight embedAndStoreText() call always ends up
   * with the right value: its post-upsert re-read either sees this write, or
   * the setPayload below runs after its upsert and covers the new points.
   */
  public async setFileActive(
    source: string,
    active: boolean
  ): Promise<{ success: boolean; message: string }> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      // A source can have chunks in Qdrant but no state row: a ZIM mid-ingestion
      // (markIndexed only runs after the final batch), a pre-RFC install, or a
      // lost row. getStoredFiles() reports such a file as active, so skipping the
      // write here left the switch stuck on while every point went inactive, and
      // the user had no way to turn it back on. Create the row the same way the
      // scanner backfills it: `indexed` when chunks exist, so the file doesn't
      // regress to pending_decision and get re-dispatched.
      let row = await KbIngestState.query().where('file_path', source).first()
      const previousActive = row ? Boolean(row.active) : true
      if (!row) {
        const { count } = await this.qdrant!.count(RagService.CONTENT_COLLECTION_NAME, {
          filter: { must: [{ key: 'source', match: { value: source } }] },
          exact: false,
        })
        row = await KbIngestState.firstOrCreate(
          { file_path: source },
          {
            file_path: source,
            state: count > 0 ? 'indexed' : 'pending_decision',
            chunks_embedded: 0,
            collection: null,
            active,
          }
        )
      }
      row.active = active
      await row.save()

      try {
        await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, {
          payload: { active },
          filter: { must: [{ key: 'source', match: { value: source } }] },
        })
      } catch (error) {
        // Put the row back so the panel keeps showing what retrieval does.
        row.active = previousActive
        await row.save()
        throw error
      }

      return { success: true, message: active ? 'File is now active.' : 'File is now inactive.' }
    } catch (error) {
      logger.error('[RAG] Error updating file active state:', error)
      return { success: false, message: 'Error updating file active state.' }
    }
  }

  /**
   * A source's active flag as a real boolean. MySQL returns tinyint(1) as 0/1,
   * and a raw 0 stamped into a Qdrant payload slips past the
   * `must_not: active == false` search filter. A source with no row is active.
   */
  private async _readActiveFlag(source: string): Promise<boolean> {
    const row = await KbIngestState.query().where('file_path', source).first()
    return row ? Boolean(row.active) : true
  }

  /**
   * Toggle every file tagged with a KB collection (the personal-upload
   * `collection` field -- not the unrelated curated ZIM-pack "collections"
   * feature) active/inactive in one bulk operation, mirroring the
   * collection-scoped bulk writes in renameKnowledgeCollection/
   * deleteKnowledgeCollection rather than looping setFileActive() per file.
   * `collection: null` targets the "Uncategorized" bucket.
   *
   * Returns the count of files whose active value actually changed (not the
   * collection's total membership), so the caller can report accurate scope
   * for what is otherwise an opaque bulk action -- e.g. "Turned off 9 files",
   * not "Turned off 12" when 3 were already inactive.
   */
  public async setKnowledgeCollectionActive(
    collection: string | null,
    active: boolean
  ): Promise<{ success: boolean; message: string; affectedCount: number }> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      const collectionQuery = () =>
        collection === null
          ? KbIngestState.query().whereNull('collection')
          : KbIngestState.query().where('collection', collection)

      const collectionFilterClause =
        collection === null
          ? { is_empty: { key: 'collection' } }
          : { key: 'collection', match: { value: collection } }

      // Rows before Qdrant, for the same reason as setFileActive(). Only rows
      // that change are touched, so a failed setPayload can restore exactly
      // those without flipping files that already held the target value.
      const changedPaths = (
        await collectionQuery().where('active', !active).select('file_path')
      ).map((r) => r.file_path)
      const affectedCount = changedPaths.length
      await collectionQuery().update({ active })

      try {
        await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, {
          payload: { active },
          filter: { must: [collectionFilterClause] },
        })
      } catch (error) {
        if (changedPaths.length > 0) {
          await KbIngestState.query().whereIn('file_path', changedPaths).update({ active: !active })
        }
        throw error
      }

      const label = collection ?? 'Uncategorized'
      const verb = active ? 'Turned on' : 'Turned off'
      return {
        success: true,
        message: `${verb} ${affectedCount} file${affectedCount === 1 ? '' : 's'} in "${label}".`,
        affectedCount,
      }
    } catch (error) {
      logger.error('[RAG] Error updating KB collection active state:', error)
      return {
        success: false,
        message: 'Error updating collection active state.',
        affectedCount: 0,
      }
    }
  }

  /**
   * Rename a knowledge-base collection everywhere it's referenced: updates every
   * Qdrant point tagged with the old name in place, and mirrors the change onto
   * any matching KbIngestState rows so getStoredFiles() reflects it immediately.
   */
  public async renameKnowledgeCollection(
    oldName: string,
    newName: string
  ): Promise<{ success: boolean; message: string }> {
    try {
      if (!oldName || !newName || oldName === newName) {
        return { success: false, message: 'Invalid collection names.' }
      }
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, {
        payload: { collection: newName },
        filter: { must: [{ key: 'collection', match: { value: oldName } }] },
      })

      await KbIngestState.query().where('collection', oldName).update({ collection: newName })

      return { success: true, message: `Renamed "${oldName}" to "${newName}".` }
    } catch (error) {
      logger.error('[RAG] Error renaming knowledge collection:', error)
      return { success: false, message: 'Error renaming collection.' }
    }
  }

  /**
   * Remove a collection by reassigning every file tagged with it back to
   * Uncategorized (collection: null). Non-destructive — no files or embeddings
   * are deleted, only the grouping label is cleared so items can be
   * recategorized later.
   */
  public async deleteKnowledgeCollection(
    name: string
  ): Promise<{ success: boolean; message: string }> {
    try {
      if (!name) {
        return { success: false, message: 'Invalid collection name.' }
      }
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, {
        payload: { collection: null },
        filter: { must: [{ key: 'collection', match: { value: name } }] },
      })

      await KbIngestState.query().where('collection', name).update({ collection: null })

      return { success: true, message: `"${name}" removed. Files moved to Uncategorized.` }
    } catch (error) {
      logger.error('[RAG] Error deleting knowledge collection:', error)
      return { success: false, message: 'Error deleting collection.' }
    }
  }

  /**
   * Count the points carrying a given `collection` tag.
   *
   * Distinct from the facet-based counts above, which deliberately exclude
   * internal collections. This answers "how many chunks are in exactly this
   * collection", which is what the eval harness needs to confirm an ingest
   * landed and what a future per-collection UI would want.
   */
  public async countChunksInCollection(collection: string): Promise<number> {
    await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
    const result = await this.qdrant!.count(RagService.CONTENT_COLLECTION_NAME, {
      filter: { must: [{ key: 'collection', match: { value: collection } }] },
      exact: true,
    })
    return result.count
  }

  /**
   * Hard-delete every point carrying a given `collection` tag.
   *
   * Note this is NOT what `deleteKnowledgeCollection` does — that one clears the
   * tag and leaves the user's documents in place, because deleting a user's
   * content because they renamed a folder would be indefensible. This one
   * genuinely removes the points, and exists for the eval corpus, which is
   * disposable by construction. Returns the number of points removed.
   */
  public async deleteCollectionPoints(collection: string): Promise<number> {
    await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
    const before = await this.countChunksInCollection(collection)
    if (before === 0) return 0
    await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, {
      wait: true,
      filter: { must: [{ key: 'collection', match: { value: collection } }] },
    })
    return before
  }

  /**
   * Resolve a stored-file `source` to an absolute disk path, but only if the
   * path lives under the uploads directory. Mirrors the docs_service traversal
   * guard: resolve, then require the resolved path to be strictly inside the
   * base + path separator (so siblings of `kb_uploads` can't slip through).
   * Returns null for anything else — ZIMs, admin docs, README, or paths
   * outside the app entirely. The viewer/download endpoints lean on this so
   * they don't need to repeat the check.
   */
  private resolveUploadPath(source: string): string | null {
    const uploadsAbsPath = resolve(join(process.cwd(), RagService.UPLOADS_STORAGE_PATH))
    const resolved = resolve(source)
    if (!resolved.startsWith(uploadsAbsPath + sep)) return null
    return resolved
  }

  private static readonly VIEWABLE_TEXT_EXTENSIONS: ReadonlySet<string> = new Set([
    'md',
    'txt',
    'csv',
    'json',
    'yaml',
    'yml',
    'toml',
    'xml',
    'html',
  ])

  /**
   * Read the text content of a user-uploaded file for in-browser viewing.
   * Returns null when the source is outside uploads, missing, or not a
   * recognized text extension. The extension allowlist is intentionally narrow
   * — PDFs/EPUBs/ZIMs round-trip through Download, not the viewer.
   */
  public async readFileContent(
    source: string
  ): Promise<{ content: string; extension: string; fileName: string } | null> {
    const resolved = this.resolveUploadPath(source)
    if (!resolved) return null

    const extension = resolved.split('.').at(-1)?.toLowerCase() ?? ''
    if (!RagService.VIEWABLE_TEXT_EXTENSIONS.has(extension)) return null

    const stats = await getFileStatsIfExists(resolved)
    if (!stats) return null

    try {
      const { readFile } = await import('node:fs/promises')
      const content = await readFile(resolved, 'utf-8')
      const fileName = resolved.split(/[/\\]/).at(-1) ?? resolved
      return { content, extension, fileName }
    } catch (error) {
      logger.warn({ err: error, source }, '[RagService.readFileContent] read failed')
      return null
    }
  }

  /**
   * Resolve a download-target path for a stored upload. Returns null if the
   * source isn't an upload or the file is missing on disk.
   */
  public async resolveDownloadPath(source: string): Promise<string | null> {
    const resolved = this.resolveUploadPath(source)
    if (!resolved) return null
    const stats = await getFileStatsIfExists(resolved)
    return stats ? resolved : null
  }

  /**
   * Compute whether the first-chat JIT prompt should fire and surface the file
   * count the banner uses in its copy ("Index your N existing files?"). The
   * banner appears when the user hasn't yet picked a global ingest policy
   * (`rag.defaultIngestPolicy` unset) and the scanner has actually seen at
   * least one embeddable file — i.e., the prompt is actionable, not theoretical
   * on a freshly-installed empty NOMAD.
   *
   * Once the user picks a policy (Always or Manual) via the banner buttons or
   * the KB modal toggle, `shouldPrompt` flips to false for good.
   */
  public async getPolicyPromptState(): Promise<{
    shouldPrompt: boolean
    hasContent: boolean
    totalFiles: number
  }> {
    const policy = await KVStore.getValue('rag.defaultIngestPolicy')
    const countRow = await KbIngestState.query().count('* as total').first()
    const totalFiles = Number((countRow as any)?.$extras?.total ?? 0)
    return {
      shouldPrompt: policy === null && totalFiles > 0,
      hasContent: totalFiles > 0,
      totalFiles,
    }
  }

  /**
   * Compute conditional warnings (RFC #883 §6) for every source the scanner
   * sees on disk. Returns `{ ok, warnings }` — `ok: false` distinguishes a
   * computation failure (Qdrant unreachable, DB outage, FS error) from the
   * healthy-but-empty case, which is critical because the whole point of this
   * surface is to expose silent failures; reporting "everything healthy" when
   * we couldn't actually check would reintroduce the bug we set out to fix.
   *
   * Per-source chunk counts come from a single Qdrant scroll over the
   * collection's points; expected-chunk estimates come from the ratio
   * registry. Files in the scanner's directories that have no qdrant points
   * at all show up with `chunksInQdrant: 0` so Warning A can fire.
   */
  public async computeFileWarnings(): Promise<FileWarningsResult> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      // Per-source chunk count via Qdrant's facet API. Was a full scroll of
      // every point in the collection, which on a fully-ingested NOMAD takes
      // ~50s for a 3M-point KB just to count ~40 sources. Facet returns the
      // distinct values + counts in a single call. `exact: true` because the
      // counts feed Warning A's zero_chunks decision — approximate counts
      // could mis-fire warnings near thresholds.
      const chunksBySource = new Map<string, number>()
      const facetResult = await this.qdrant!.facet(RagService.CONTENT_COLLECTION_NAME, {
        key: 'source',
        limit: RagService.FACET_SOURCE_LIMIT,
        exact: true,
        // Same exclusion as getStoredFiles: eval fixtures are not user files,
        // and counting them here would raise bogus zero_chunks warnings.
        filter: EXCLUDE_EVAL_FILTER,
      })
      for (const hit of facetResult.hits) {
        if (typeof hit.value === 'string') chunksBySource.set(hit.value, hit.count)
      }

      // Scan the filesystem the same way scanAndSyncStorage does so Warning A
      // can fire on files with zero qdrant points (the headline "video-only
      // ZIM" case).
      const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH)
      const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH)
      const allSources = new Set<string>(chunksBySource.keys())
      const sizeByPath = new Map<string, number>()

      for (const dir of [KB_UPLOADS_PATH, ZIM_PATH]) {
        try {
          const entries = await listDirectoryContentsRecursive(dir)
          for (const entry of entries) {
            if (entry.type !== 'file') continue
            allSources.add(entry.key)
            const stat = await getFileStatsIfExists(entry.key)
            if (stat) sizeByPath.set(entry.key, Number(stat.size))
          }
        } catch (error: any) {
          if (error?.code !== 'ENOENT') throw error
        }
      }

      const out: Record<string, FileWarning[]> = {}
      for (const source of allSources) {
        const fileSizeBytes = sizeByPath.get(source) ?? 0
        const chunksInQdrant = chunksBySource.get(source) ?? 0
        const fileName = source.split(/[/\\]/).pop() ?? source
        // ignoreCatchAll: the partial_stall warning must only fire when the
        // registry has a *specific* expectation for this file. The empty-pattern
        // fallback (100 chunks/MB) over-predicts wildly for atypical ZIMs that
        // are mostly PDFs/images/link-outs (e.g. military-medicine), producing
        // false "ingestion stalled" warnings. Suppress Warning B in that case. (#913)
        const expectedChunks =
          fileSizeBytes > 0
            ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes, {
                ignoreCatchAll: true,
              })
            : null

        const warnings = decideWarnings({ fileSizeBytes, chunksInQdrant, expectedChunks })
        if (warnings.length > 0) out[source] = warnings
      }

      return { ok: true, warnings: out }
    } catch (error) {
      logger.error('[RAG] Error computing file warnings:', error)
      return { ok: false, warnings: {} }
    }
  }

  /**
   * Delete all Qdrant points associated with a given source path and remove
   * the corresponding file from disk if it lives under the uploads directory.
   * @param source - Full source path as stored in Qdrant payloads
   */
  public async deleteFileBySource(source: string): Promise<{ success: boolean; message: string }> {
    try {
      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, {
        filter: {
          must: [{ key: 'source', match: { value: source } }],
        },
      })

      logger.info(`[RAG] Deleted all points for source: ${source}`)

      /** Delete the physical file only if it lives inside the uploads directory.
       * resolve() normalises path traversal sequences (e.g. "/../..") before the
       * check to prevent path traversal vulns
       * The trailing sep is to ensure a prefix like "kb_uploads_{something_incorrect}" can't slip through.
       */
      const uploadsAbsPath = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH)
      const resolvedSource = resolve(source)
      if (resolvedSource.startsWith(uploadsAbsPath + sep)) {
        await deleteFileIfExists(resolvedSource)
        logger.info(`[RAG] Deleted uploaded file from disk: ${resolvedSource}`)
      } else {
        logger.warn(
          `[RAG] File was removed from knowledge base but doesn't live in Nomad's uploads directory, so it can't be safely removed. Skipping deletion of physical file...`
        )
      }

      // Drop the ingest state row last so the file disappears entirely. Without
      // this, the next scanAndSyncStorage would see `indexed + no chunks` for a
      // path that no longer exists in storage and try to re-embed nothing.
      await KbIngestState.remove(source)

      return { success: true, message: 'File removed from knowledge base.' }
    } catch (error) {
      logger.error('[RAG] Error deleting file from knowledge base:', error)
      return { success: false, message: 'Error deleting file from knowledge base.' }
    }
  }

  /**
   * Reconcile the knowledge base after a curated content file (a ZIM) is
   * replaced on disk by a newer downloaded version. Called from
   * `RunDownloadJob.onComplete` once the new file is written and the old file
   * has been deleted from disk.
   *
   * The decision (see `decideContentReindex` for the exhaustive, tested
   * contract) mirrors the REPLACED file's prior indexed state instead of the
   * global `rag.defaultIngestPolicy`. On a content update the user has already
   * chosen whether this content belongs in the AI knowledge base, so we honor
   * that choice in both directions:
   *
   *   1. (caller already deleted the outdated file from disk)
   *   2. Qdrant not installed          → no-op (no knowledge base exists)
   *   3. Old file WAS indexed + Qdrant
   *      running                       → delete ONLY the old file's points
   *                                       (filter `source == oldFilePath`), drop
   *                                       its ingest-state row, and queue the new
   *                                       file for embedding — BYPASSING the
   *                                       Always/Manual policy on purpose.
   *   4. Old file NOT indexed          → no-op (respect the prior un-indexed /
   *                                       browse-only choice; do NOT auto-embed
   *                                       even under an Always policy)
   *   5. Old indexed but Qdrant NOT
   *      currently running             → no-op. We can't remove the stale points,
   *                                       and a queued embed job could be cleared
   *                                       before Qdrant returns. Acting half-way is
   *                                       wasteful, so we defer entirely. Accepted
   *                                       tradeoff: the old file's points linger in
   *                                       Qdrant until a future re-index; they are
   *                                       NOT auto-reaped here.
   *
   * Point deletion is exact: ZIM chunks are stored with `source` equal to the
   * full file path (see embedAndStoreText callers), so filtering on the old path
   * can only ever remove the replaced file's own points.
   *
   * Returns the decision outcome for logging/tests; never throws on a Qdrant
   * hiccup mid-reindex (logged and surfaced as `qdrant_not_running` semantics by
   * the caller's try/catch).
   */
  public async reconcileReplacedContentFile(params: {
    oldFilePath: string
    newFilePath: string
    fileName: string
  }): Promise<ReindexOutcome> {
    const { oldFilePath, newFilePath, fileName } = params

    const isReplacement = !!oldFilePath && oldFilePath !== newFilePath

    // Step 2: is the knowledge base even installed? Short-circuits before any
    // KB-state lookup, per the spec ordering.
    const qdrantInstalled = isReplacement
      ? !!(await this.dockerService.getServiceURL(SERVICE_NAMES.QDRANT))
      : false

    // Steps 3/4: was the OUTDATED file actually indexed? The state row is the
    // authoritative signal (RFC #883) — chunk presence alone can't distinguish
    // a fully-indexed file from a stalled ingestion.
    let oldFileWasIndexed = false
    if (isReplacement && qdrantInstalled) {
      const oldState = await KbIngestState.query().where('file_path', oldFilePath).first()
      oldFileWasIndexed = oldState?.state === 'indexed'
    }

    // Step 5: only check liveness once we know we'd otherwise act. The health
    // check is a real network round-trip, so we avoid it on the common no-op
    // paths above.
    let qdrantRunning = false
    if (isReplacement && qdrantInstalled && oldFileWasIndexed) {
      qdrantRunning = (await this.checkQdrantHealth()).online
    }

    const outcome = decideContentReindex({
      isReplacement,
      qdrantInstalled,
      oldFileWasIndexed,
      qdrantRunning,
    })

    if (outcome === 'reindex') {
      // Order matters: remove the stale points and state row BEFORE queueing the
      // new embed so a fresh index can't be conflated with the old one. Each step
      // targets only `oldFilePath` / `newFilePath` — never another resource.
      await this.purgeIndexedSource(oldFilePath)
      const { EmbedFileJob } = await import('#jobs/embed_file_job')
      await EmbedFileJob.dispatch({ fileName, filePath: newFilePath })
    }

    return outcome
  }

  /**
   * Root paths for Nomad's own bundled docs (README.md + docs/), embedded by
   * discoverNomadDocs(). They live outside kb_uploads/zim storage, so
   * _discoverKbFiles()'s scan never sees them — the orphan sweep in
   * scanAndSyncStorage() uses this to exclude them rather than hardcoding
   * the same two paths a second time.
   */
  private _nomadDocsRoots(): { readmePath: string; docsDir: string } {
    return {
      readmePath: join(process.cwd(), 'README.md'),
      docsDir: join(process.cwd(), 'docs'),
    }
  }

  public async discoverNomadDocs(force?: boolean): Promise<{ success: boolean; message: string }> {
    try {
      const { readmePath: README_PATH, docsDir: DOCS_DIR } = this._nomadDocsRoots()

      const alreadyEmbeddedRaw = await KVStore.getValue('rag.docsEmbedded')
      if (alreadyEmbeddedRaw && !force) {
        logger.info('[RAG] Nomad docs have already been discovered and queued. Skipping.')
        return {
          success: true,
          message: 'Nomad docs have already been discovered and queued. Skipping.',
        }
      }

      const filesToEmbed: Array<{ path: string; source: string }> = []

      const readmeExists = await getFileStatsIfExists(README_PATH)
      if (readmeExists) {
        filesToEmbed.push({ path: README_PATH, source: 'README.md' })
      }

      const dirContents = await listDirectoryContentsRecursive(DOCS_DIR)
      for (const entry of dirContents) {
        if (entry.type === 'file') {
          filesToEmbed.push({ path: entry.key, source: join('docs', entry.name) })
        }
      }

      logger.info(`[RAG] Discovered ${filesToEmbed.length} Nomad doc files to embed`)

      // Import EmbedFileJob dynamically to avoid circular dependencies
      const { EmbedFileJob } = await import('#jobs/embed_file_job')

      // Dispatch an EmbedFileJob for each discovered file
      for (const fileInfo of filesToEmbed) {
        try {
          logger.info(`[RAG] Dispatching embed job for: ${fileInfo.source}`)
          await EmbedFileJob.dispatch({
            filePath: fileInfo.path,
            fileName: fileInfo.source,
          })
          logger.info(`[RAG] Successfully dispatched job for ${fileInfo.source}`)
        } catch (fileError) {
          logger.error(`[RAG] Error dispatching job for file ${fileInfo.source}:`, fileError)
        }
      }

      // Update KV store to mark docs as discovered so we don't redo this unnecessarily
      await KVStore.setValue('rag.docsEmbedded', true)

      return {
        success: true,
        message: `Nomad docs discovery completed. Dispatched ${filesToEmbed.length} embedding jobs.`,
      }
    } catch (error) {
      logger.error('Error discovering Nomad docs:', error)
      return { success: false, message: 'Error discovering Nomad docs.' }
    }
  }

  /**
   * Absolute roots that _discoverKbFiles() scans. Shared with the orphan
   * sweep in scanAndSyncStorage() so it can allowlist candidates to sources
   * actually reachable by that scan, rather than denylisting every other
   * known source root by hand — a source embedded under a future root
   * outside kb_uploads/zim would otherwise be silently misclassified as
   * orphaned and purged the first time it's added (see _nomadDocsRoots()
   * for exactly that lesson learned with README.md/docs).
   */
  private _kbScanRoots(): { kbUploadsPath: string; zimPath: string } {
    return {
      kbUploadsPath: join(process.cwd(), RagService.UPLOADS_STORAGE_PATH),
      zimPath: join(process.cwd(), ZIM_STORAGE_PATH),
    }
  }

  /**
   * Walk kb_uploads and zim storage directories, returning the full path of
   * every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are
   * filtered out so they aren't dispatched only to fail with "Unsupported file
   * type" and retry on every sync.
   */
  private async _discoverKbFiles(): Promise<string[]> {
    return (await this._discoverKbFilesWithRoots()).files
  }

  /**
   * As _discoverKbFiles(), but also reports which roots were actually walked.
   *
   * A root that isn't there is skipped rather than fatal, because a fresh
   * install legitimately has no kb_uploads until the first upload. That makes
   * an absent root indistinguishable from a present-but-empty one by looking
   * at the file list alone — and the orphan sweep in scanAndSyncStorage()
   * cannot afford to confuse the two. "This root wasn't there" means we know
   * nothing about it and must not touch it. A root that was there but empty
   * is not much better evidence: boot creates the zim directory, so an
   * unmounted volume shows up as an empty one (#1378). decideOrphans()
   * therefore also refuses to purge under a walked root with no files.
   *
   * That distinction is the whole reason this variant exists. If the zim root
   * is missing, renamed, or not yet mounted (see #1050 — relocating the data
   * path is easy to get subtly wrong), the scan still returns a non-empty
   * list from kb_uploads. A guard that only checks "did the scan come back
   * empty" therefore passes, and every ZIM in the index gets purged in one
   * batch. Reporting the roots lets the sweep confine itself to ground it
   * actually stood on.
   */
  private async _discoverKbFilesWithRoots(): Promise<{
    files: string[]
    scannedRoots: string[]
  }> {
    const { kbUploadsPath: KB_UPLOADS_PATH, zimPath: ZIM_PATH } = this._kbScanRoots()
    const filesInStorage: string[] = []
    const scannedRoots: string[] = []

    for (const [label, dirPath] of [
      [RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const,
      [ZIM_STORAGE_PATH, ZIM_PATH] as const,
    ]) {
      try {
        const contents = await listDirectoryContentsRecursive(dirPath)
        contents.forEach((entry) => {
          if (entry.type === 'file') filesInStorage.push(entry.key)
        })
        scannedRoots.push(dirPath)
        logger.debug(`[RAG] Found ${contents.length} files in ${label}`)
      } catch (error) {
        if (error.code === 'ENOENT') {
          logger.debug(`[RAG] ${label} directory does not exist, skipping`)
        } else {
          throw error
        }
      }
    }

    return {
      files: filesInStorage.filter((f) => determineFileType(f) !== 'unknown'),
      scannedRoots,
    }
  }

  /**
   * Dispatch one EmbedFileJob per file path. Returns honest counts: `queuedCount`
   * is jobs newly enqueued, `dedupedCount` is jobs that hit BullMQ's per-file
   * jobId dedupe (an existing :completed/:waiting/etc. entry was returned
   * instead of a new enqueue), and `failedPaths` lists files whose dispatch
   * threw. Pass `force: true` for bulk callers that need to bypass dedupe
   * entirely. Per-file errors are logged but don't abort the batch — callers
   * must inspect `failedPaths` to surface partial failure to the operator.
   */
  private async _dispatchEmbedJobsFor(
    filePaths: string[],
    options?: { force?: boolean }
  ): Promise<{ queuedCount: number; dedupedCount: number; failedPaths: string[] }> {
    const { EmbedFileJob } = await import('#jobs/embed_file_job')
    let queuedCount = 0
    let dedupedCount = 0
    const failedPaths: string[] = []
    for (const filePath of filePaths) {
      try {
        const fileName = filePath.split(/[/\\]/).pop() || filePath
        const stats = await getFileStatsIfExists(filePath)
        const result = await EmbedFileJob.dispatch(
          {
            filePath,
            fileName,
            fileSize: stats?.size,
          },
          { force: options?.force }
        )
        if (result.created) {
          queuedCount++
        } else {
          dedupedCount++
        }
      } catch (fileError) {
        failedPaths.push(filePath)
        logger.error(`[RAG] Error dispatching job for file ${filePath}:`, fileError)
      }
    }
    return { queuedCount, dedupedCount, failedPaths }
  }

  /**
   * Dispatch an embed job for a single stored file. Wraps `_dispatchEmbedJobsFor`
   * with the safety checks needed for a user-triggered per-row action:
   *   1. The source must be known to the scanner OR have a state row — prevents
   *      arbitrary path dispatch from the public API.
   *   2. We refuse if any inflight job (waiting/active/delayed/paused) already
   *      targets this filePath. Otherwise a double-click or a rapid retry could
   *      enqueue duplicate jobs, producing duplicate chunks.
   *   3. When `force` is true (Re-embed of an already-indexed file), we
   *      pre-delete the prior Qdrant points so the new run doesn't stack on
   *      top of the old ones. For force=false (Index of a never-embedded file),
   *      there's nothing to clear.
   */
  public async embedSingleFile(
    source: string,
    force: boolean = false
  ): Promise<EmbedSingleFileResult> {
    const stateRow = await KbIngestState.query().where('file_path', source).first()
    if (!stateRow) {
      const knownFiles = await this._discoverKbFiles()
      if (!knownFiles.includes(source)) {
        return {
          success: false,
          code: 'not_found',
          message: 'File is not a tracked knowledge-base source.',
        }
      }
    }

    const { EmbedFileJob } = await import('#jobs/embed_file_job')
    const { QueueService } = await import('#services/queue_service')
    const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue)
    const inflight = await queue.getJobs(['waiting', 'active', 'delayed', 'paused'])
    if (inflight.some((j) => j.data?.filePath === source)) {
      return {
        success: false,
        code: 'inflight',
        message:
          'A job for this file is already in progress. Wait for it to finish before re-queuing.',
      }
    }

    if (force) {
      try {
        await this._deletePointsBySource(source)
      } catch (err) {
        logger.error(`[RAG] Failed to delete prior points for ${source}; aborting re-embed:`, err)
        return {
          success: false,
          code: 'delete_failed',
          message: 'Failed to clear prior embeddings before re-embed.',
        }
      }
    }

    const result = await this._dispatchEmbedJobsFor([source], { force })
    if (result.failedPaths.length > 0) {
      return {
        success: false,
        code: 'dispatch_failed',
        message: 'Failed to dispatch embed job for this file.',
      }
    }
    return {
      success: true,
      message: force ? 'Re-embed queued for this file.' : 'Indexing queued for this file.',
    }
  }

  /**
   * Delete all Qdrant points whose `source` payload matches the given path.
   * Unlike deleteFileBySource(), this does NOT touch the file on disk — used
   * by reembedAll() where the file must remain so it can be re-ingested.
   */
  private async _deletePointsBySource(source: string): Promise<void> {
    await this._deletePointsBySources([source])
  }

  /**
   * Same as _deletePointsBySource(), but for many sources in one round trip
   * via Qdrant's `match.any` filter — used by the orphan sweep below, where
   * purging one-by-one would mean a separate delete call (plus a separate
   * _ensureCollection check) per orphan.
   */
  private async _deletePointsBySources(sources: string[]): Promise<void> {
    if (sources.length === 0) return
    await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
    await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, {
      filter: { must: [{ key: 'source', match: { any: sources } }] },
    })
  }

  /**
   * Purge a source's Qdrant points and its `KbIngestState` row without
   * touching the file on disk. For callers where the file is already gone
   * (or never existed as a knowledge-base upload) — `ZimService.delete()`
   * (#1170) and the orphan sweep in `scanAndSyncStorage()` below. Does NOT
   * attempt to delete a physical file; use `deleteFileBySource()` for the
   * user-triggered "remove this file" action instead.
   *
   * State row removed first, points second: orphan detection in
   * scanAndSyncStorage() only looks at what's still in Qdrant, so if a
   * failure lands between the two steps, leftover points stay visible to
   * the next sync's reverse sweep and get retried there. Deleting the
   * points first would instead risk an orphaned state row that nothing
   * ever looks at again. Both steps are individually idempotent, so a
   * retry from either partial state is safe.
   */
  public async purgeIndexedSource(source: string): Promise<void> {
    await this.purgeIndexedSources([source])
  }

  /**
   * Batched form of purgeIndexedSource() — one KbIngestState delete query
   * and one Qdrant delete call for the whole list, instead of a per-source
   * round trip to each. Used by the orphan sweep in scanAndSyncStorage(),
   * which can otherwise find dozens of orphans after a bulk content change.
   */
  public async purgeIndexedSources(sources: string[]): Promise<void> {
    if (sources.length === 0) return
    await KbIngestState.query().whereIn('file_path', sources).delete()
    await this._deletePointsBySources(sources)
  }

  /**
   * Returns true if the file-embeddings queue has any in-flight work
   * (waiting, active, delayed, or paused). Bulk re-embed actions use this
   * to refuse mid-flight to avoid racing with deletes/dispatches already
   * in progress.
   */
  private async _hasInflightEmbedJobs(): Promise<boolean> {
    const { EmbedFileJob } = await import('#jobs/embed_file_job')
    const { QueueService } = await import('#services/queue_service')
    const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue)
    const counts = await queue.getJobCounts('waiting', 'active', 'delayed', 'paused')
    return (
      (counts.waiting || 0) + (counts.active || 0) + (counts.delayed || 0) + (counts.paused || 0) >
      0
    )
  }

  /**
   * Scans the knowledge base storage directories and syncs with Qdrant.
   * Identifies files that exist in storage but haven't been embedded yet,
   * and dispatches EmbedFileJob for each missing file.
   */
  public async scanAndSyncStorage(): Promise<{
    success: boolean
    message: string
    filesScanned?: number
    filesQueued?: number
  }> {
    try {
      logger.info('[RAG] Starting knowledge base sync scan')

      await this.discoverNomadDocs(true).catch((error) => {
        logger.error('[RAG] Error during Nomad docs discovery in sync process:', error)
      })

      const { files: filesInStorage, scannedRoots } = await this._discoverKbFilesWithRoots()
      logger.info(`[RAG] Found ${filesInStorage.length} embeddable files in storage`)

      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      // Collect every unique `source` already in Qdrant so we can skip files
      // that have already been embedded. Facet returns the distinct values in
      // a single call rather than scrolling the whole collection.
      const sourcesInQdrant = new Set<string>()
      const facetResult = await this.qdrant!.facet(RagService.CONTENT_COLLECTION_NAME, {
        key: 'source',
        limit: RagService.FACET_SOURCE_LIMIT,
        exact: true,
      })
      for (const hit of facetResult.hits) {
        if (typeof hit.value === 'string') sourcesInQdrant.add(hit.value)
      }

      logger.info(`[RAG] Found ${sourcesInQdrant.size} unique sources in Qdrant`)

      // Load all known per-file ingest states. The state row is authoritative
      // over the "any chunks in Qdrant" heuristic — it captures user choices
      // (browse_only) and terminal outcomes (failed, stalled) that aren't visible
      // from Qdrant alone. See RFC #883 for the full state machine.
      const stateRows = await KbIngestState.all()
      const stateByPath = new Map(stateRows.map((row) => [row.file_path, row]))

      // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise
      // be dispatched to EmbedFileJob, fail with "Unsupported file type", and retry
      // on every sync — filter them out before state decisions.
      const embeddableFiles = filesInStorage.filter(
        (filePath) => determineFileType(filePath) !== 'unknown'
      )

      // Reverse sweep (#1170): sourcesInQdrant and embeddableFiles are both
      // already known at this point — the forward loop below only ever asks
      // "is this on-disk file already embedded?" This closes the other
      // direction: a source in Qdrant with no corresponding file on disk is a
      // leftover from ZimService.delete() (which never touched Qdrant) or
      // from reconcileReplacedContentFile's qdrant_not_running no-op. Running
      // this in sync (rather than only in the delete path) also self-heals
      // installs already in this state.
      //
      // Confined to `scannedRoots`, the roots the scan above actually walked,
      // not the roots it meant to walk. Nomad's own bundled docs (README.md +
      // docs/) live outside these roots, so they're left alone without being
      // named here, and a root missing at scan time contributes nothing
      // (#1050). Within each walked root, decideOrphans also withholds the
      // purge when the root holds no embeddable files (an unmounted volume
      // leaves an empty mountpoint behind, #1378) or when it would remove most
      // of the root at once. Withheld roots are reported back to the operator
      // rather than silently skipped.
      const { orphans, withheld } = decideOrphans(
        [...sourcesInQdrant],
        embeddableFiles,
        scannedRoots
      )
      for (const w of withheld) {
        logger.warn(
          `[RAG] Withheld purge of ${w.count} indexed source(s) under ${w.root} (${w.reason}); the directory may be unmounted or pointing at the wrong location`
        )
      }
      let orphansPurged = 0
      if (orphans.length > 0) {
        logger.info(
          `[RAG] Found ${orphans.length} orphaned source(s) with no corresponding file on disk`
        )
        try {
          await this.purgeIndexedSources(orphans)
          orphansPurged = orphans.length
        } catch (error) {
          logger.error(`[RAG] Failed to purge orphaned sources:`, error)
        }
        logger.info(`[RAG] Purged ${orphansPurged}/${orphans.length} orphaned source(s)`)
      }

      // Read the global ingest policy. Unset is treated as 'Always' so legacy
      // installs keep their current behavior until the user explicitly opts
      // into Manual mode from the KB panel.
      const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy')
      const policy: IngestPolicy = policyRaw === 'Manual' ? 'Manual' : 'Always'

      const filesToEmbed: string[] = []
      let backfilled = 0
      let createdRows = 0
      let createdPending = 0
      let skipped = 0

      for (const filePath of embeddableFiles) {
        const stateRow = stateByPath.get(filePath) ?? null
        const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath), policy)

        switch (action.kind) {
          case 'skip':
            skipped++
            break
          case 'backfill_indexed':
            // Pre-RFC install (or a fresh admin pointed at an existing Qdrant volume):
            // chunks already exist with no state row, so trust Qdrant and record
            // `indexed` without re-embedding. chunks_embedded is left 0 because
            // we don't count points-per-source during the scroll above.
            await KbIngestState.create({
              file_path: filePath,
              state: 'indexed',
              chunks_embedded: 0,
            })
            backfilled++
            break
          case 'create_pending':
            // Manual mode: record that we've seen the file but don't dispatch.
            // The KB panel surfaces a per-card "Index" affordance for these.
            await KbIngestState.create({
              file_path: filePath,
              state: 'pending_decision',
              chunks_embedded: 0,
            })
            createdPending++
            break
          case 'dispatch':
            if (action.createStateRow) {
              await KbIngestState.create({
                file_path: filePath,
                state: 'pending_decision',
                chunks_embedded: 0,
              })
              createdRows++
            }
            filesToEmbed.push(filePath)
            break
        }
      }

      logger.info(
        `[RAG] Scan results (policy=${policy}): ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${createdPending} waiting on user, ${skipped} skipped`
      )

      const withheldNote = withheld
        .map(
          (w) =>
            `; left ${w.count} indexed source${w.count !== 1 ? 's' : ''} under ${relative(process.cwd(), w.root)} untouched because ${
              w.reason === 'empty_root'
                ? 'that folder has no files (is the drive mounted?)'
                : 'removing them would clear most of that folder (is it pointing at the right drive?)'
            }`
        )
        .join('')
      const orphanNote =
        (orphansPurged > 0
          ? `; purged ${orphansPurged} orphaned source${orphansPurged !== 1 ? 's' : ''}`
          : '') + withheldNote

      if (filesToEmbed.length === 0) {
        return {
          success: true,
          message: `Knowledge base is already in sync${orphanNote}`,
          filesScanned: filesInStorage.length,
          filesQueued: 0,
        }
      }

      const { queuedCount, dedupedCount } = await this._dispatchEmbedJobsFor(filesToEmbed)
      const dedupeNote = dedupedCount > 0 ? ` (${dedupedCount} already queued)` : ''
      return {
        success: true,
        message: `Scanned ${filesInStorage.length} files, queued ${queuedCount} for embedding${dedupeNote}${orphanNote}`,
        filesScanned: filesInStorage.length,
        filesQueued: queuedCount,
      }
    } catch (error) {
      logger.error('[RAG] Error scanning and syncing knowledge base:', error)
      return { success: false, message: 'Error scanning and syncing knowledge base' }
    }
  }

  /**
   * Re-embed every file on disk (per-file replace). For each discovered file:
   * delete its existing Qdrant points by `source` match, then dispatch a fresh
   * EmbedFileJob. Files are NOT removed from disk. Any orphan points (points
   * whose source file no longer exists) are intentionally preserved — use
   * resetAndRebuild() if a clean slate is required.
   *
   * Refuses to run if the embeddings queue already has in-flight work.
   */
  public async reembedAll(): Promise<{
    success: boolean
    message: string
    filesScanned?: number
    filesQueued?: number
    failedPaths?: string[]
  }> {
    try {
      if (await this._hasInflightEmbedJobs()) {
        return {
          success: false,
          message:
            'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a bulk re-embed.',
        }
      }

      logger.info('[RAG] Starting full re-embed (per-file replace)')

      await this.discoverNomadDocs(true).catch((error) => {
        logger.error('[RAG] Error re-running Nomad docs discovery during re-embed:', error)
      })

      const filesInStorage = await this._discoverKbFiles()

      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      // Per-file: delete-then-dispatch. We tried dispatch-then-delete but that
      // opens a race where a fast worker can write new points before our
      // delete-by-source runs, wiping both. Instead we delete first, then
      // dispatch — and if dispatch fails, we surface the failed paths in the
      // response so the operator knows which files dropped out (rather than
      // silently leaving them unindexed). A subsequent sync rescan picks them
      // back up. Note: a delete-failure aborts the per-file pair (we don't
      // dispatch a job whose old points are still present, since they'd live
      // alongside the new vectors forever).
      const { EmbedFileJob } = await import('#jobs/embed_file_job')
      let queuedCount = 0
      const failedPaths: string[] = []
      for (const filePath of filesInStorage) {
        try {
          await this._deletePointsBySource(filePath)
        } catch (err) {
          logger.error(
            `[RAG] Failed to delete prior points for ${filePath}; skipping dispatch:`,
            err
          )
          failedPaths.push(filePath)
          continue
        }
        try {
          const fileName = filePath.split(/[/\\]/).pop() || filePath
          const stats = await getFileStatsIfExists(filePath)
          const result = await EmbedFileJob.dispatch(
            { filePath, fileName, fileSize: stats?.size },
            { force: true }
          )
          if (result.created) queuedCount++
        } catch (fileError) {
          // Old points already deleted but the new job never made it onto the
          // queue. Logged + surfaced so an operator can rerun a sync.
          logger.error(
            `[RAG] Re-embed dispatch failed for ${filePath} after delete; file is now unindexed until next sync:`,
            fileError
          )
          failedPaths.push(filePath)
        }
      }

      logger.info(
        `[RAG] Re-embed dispatched ${queuedCount}/${filesInStorage.length} files` +
          (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '')
      )

      const failureSuffix =
        failedPaths.length > 0
          ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.`
          : ''

      return {
        success: failedPaths.length === 0,
        message:
          `Re-embedding ${queuedCount} file${queuedCount === 1 ? '' : 's'}. Existing points were replaced.` +
          failureSuffix,
        filesScanned: filesInStorage.length,
        filesQueued: queuedCount,
        ...(failedPaths.length > 0 ? { failedPaths } : {}),
      }
    } catch (error) {
      logger.error('[RAG] Error during re-embed:', error)
      return { success: false, message: 'Error during re-embed' }
    }
  }

  /**
   * Destructive rebuild. Drops the entire Qdrant collection (wiping every
   * point including orphans), recreates it with the correct dimension, clears
   * the Nomad-docs discovery flag, then dispatches an EmbedFileJob for every
   * file currently on disk.
   *
   * Refuses to run if the embeddings queue already has in-flight work.
   */
  public async resetAndRebuild(): Promise<{
    success: boolean
    message: string
    filesScanned?: number
    filesQueued?: number
    failedPaths?: string[]
  }> {
    try {
      if (await this._hasInflightEmbedJobs()) {
        return {
          success: false,
          message:
            'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a reset.',
        }
      }

      logger.info('[RAG] Starting destructive reset & rebuild')

      await this._initializeQdrantClient()
      try {
        await this.qdrant!.deleteCollection(RagService.CONTENT_COLLECTION_NAME)
        logger.info(`[RAG] Dropped collection ${RagService.CONTENT_COLLECTION_NAME}`)
      } catch (err) {
        // Collection may not exist yet on a fresh install — log and continue.
        logger.warn(`[RAG] deleteCollection failed (may not exist): ${(err as Error).message}`)
      }

      // The collection is gone — drop it from the ensured cache so the
      // _ensureCollection call below actually recreates it
      this.ensuredCollections.delete(RagService.CONTENT_COLLECTION_NAME)

      await this._ensureCollection(
        RagService.CONTENT_COLLECTION_NAME,
        RagService.EMBEDDING_DIMENSION
      )

      // Force Nomad docs to be re-dispatched.
      await KVStore.setValue('rag.docsEmbedded', false)
      await this.discoverNomadDocs(true).catch((error) => {
        logger.error('[RAG] Error re-running Nomad docs discovery after reset:', error)
      })

      const filesInStorage = await this._discoverKbFiles()
      const { queuedCount, failedPaths } = await this._dispatchEmbedJobsFor(filesInStorage, {
        force: true,
      })

      logger.info(
        `[RAG] Reset complete — dispatched ${queuedCount}/${filesInStorage.length} files` +
          (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '')
      )

      // Collection was already dropped, so dispatch failures here mean the
      // file is gone from Qdrant with no pending job to repopulate it. Surface
      // the count + paths so the operator can rerun a sync rescan to recover.
      const failureSuffix =
        failedPaths.length > 0
          ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.`
          : ''

      return {
        success: failedPaths.length === 0,
        message:
          `Collection wiped. Queued ${queuedCount} file${queuedCount === 1 ? '' : 's'} for a full rebuild.` +
          failureSuffix,
        filesScanned: filesInStorage.length,
        filesQueued: queuedCount,
        ...(failedPaths.length > 0 ? { failedPaths } : {}),
      }
    } catch (error) {
      logger.error('[RAG] Error during reset & rebuild:', error)
      return { success: false, message: 'Error during reset & rebuild' }
    }
  }
}