Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript
2.1 KB
Raw Blame
import { preprocessFile } from "@/commands/fs"
import { MAX_USER_CONCURRENCY, MIN_USER_CONCURRENCY } from "@/lib/concurrency-limits"

export const MIN_PARSING_CONCURRENCY = MIN_USER_CONCURRENCY
export const MAX_PARSING_CONCURRENCY = MAX_USER_CONCURRENCY

export function normalizeParsingConcurrency(value: number): number {
  if (!Number.isFinite(value)) return MIN_PARSING_CONCURRENCY
  return Math.max(
    MIN_PARSING_CONCURRENCY,
    Math.min(MAX_PARSING_CONCURRENCY, Math.floor(value)),
  )
}

interface WaitingParser {
  limit: number
  resolve: () => void
}

let activeParsers = 0
const waitingParsers: WaitingParser[] = []

function drainWaitingParsers(): void {
  while (waitingParsers.length > 0 && activeParsers < waitingParsers[0].limit) {
    const waiter = waitingParsers.shift()
    if (!waiter) return
    activeParsers += 1
    waiter.resolve()
  }
}

async function acquireParser(limit: number): Promise<() => void> {
  await new Promise<void>((resolve) => {
    waitingParsers.push({ limit, resolve })
    drainWaitingParsers()
  })
  let released = false
  return () => {
    if (released) return
    released = true
    activeParsers -= 1
    drainWaitingParsers()
  }
}

/**
 * Pre-extract source text with bounded concurrency before LLM work begins.
 * Individual extractor failures are intentionally non-fatal: autoIngest will
 * retry through read_file and surface a real source-read error if necessary.
 */
export async function preprocessSourceFiles(
  paths: readonly string[],
  concurrency: number,
): Promise<void> {
  const uniquePaths = [...new Set(paths)]
  if (uniquePaths.length === 0) return
  const limit = normalizeParsingConcurrency(concurrency)

  await Promise.all(uniquePaths.map(async (path) => {
    const release = await acquireParser(limit)
    try {
      try {
        await preprocessFile(path)
      } catch (err) {
        console.warn(
          `[source-preprocess] failed to pre-process "${path}"; ingest will retry when reading it:`,
          err instanceof Error ? err.message : err,
        )
      }
    } finally {
      release()
    }
  }))
}