Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

13.0 KB
Raw Blame
import type { KnowledgeProcessOverrides } from '@/types/knowledgeProcess'
import {
  classifyUploadResult,
  itemPhase,
  needsParsePolling,
  pickNextToStart,
  type UploadItem,
} from './uploadTasksState'

/**
 * The upload queue's behaviour — bounded concurrent transfers, cancel, retry
 * and polling the created rows until parsing settles — with the network and
 * timers injected, so the store only wires it to the API and Vue.
 *
 * `items` and `batches` are mutated in place (never reassigned) so a reactive
 * array handed in by the store stays the one the panel renders.
 */

export interface UploadBatch {
  id: string
  kbId: string
  kbName: string
  /** Destination folder, '' for the knowledge base root. */
  targetFolder: string
  tagIds?: string[]
  processConfig?: KnowledgeProcessOverrides
}

export interface UploadPayload {
  file: File
  /** Path-qualified name the backend splits into folder + file name. */
  fileName?: string
}

export interface KnowledgeStatusRow {
  id: string
  parse_status?: string
  error_message?: string
}

export interface TransferEnd {
  /** The file reached the server, so the knowledge base has a new row. */
  uploaded: boolean
  /** Nothing is queued or uploading for this knowledge base any more. */
  settled: boolean
}

export interface Timers {
  setTimer: (fn: () => void, ms: number) => unknown
  clearTimer: (handle: unknown) => void
}

export const defaultTimers: Timers = {
  setTimer: (fn, ms) => setTimeout(fn, ms),
  clearTimer: handle => clearTimeout(handle as ReturnType<typeof setTimeout>),
}

export interface UploadQueueDeps {
  concurrency: number
  pollIntervalMs: number
  /** Most knowledge ids asked about in one status request. */
  pollChunkSize: number
  /**
   * Most status requests one poll keeps in flight. Each one holds a database
   * connection on the server, so thousands of files still parsing must not
   * turn into a burst of hundreds of parallel requests.
   */
  pollConcurrency: number
  /** Resolves or rejects with the API's response body; the queue classifies either. */
  upload: (
    batch: UploadBatch,
    payload: UploadPayload,
    onProgress: (ratio: number) => void,
    signal: AbortSignal,
  ) => Promise<unknown>
  /** Current rows for the ids, or null when the request failed. */
  queryStatus: (kbId: string, knowledgeIds: string[]) => Promise<KnowledgeStatusRow[] | null>
  /** Called whenever a transfer ends, including one that outlived clear(). */
  onTransferEnd: (kbId: string, end: TransferEnd) => void
  timers?: Timers
}

/**
 * Consecutive successful polls a row must be missing from before it counts as
 * deleted, so one odd answer can't strand a live document as "Deleted".
 */
const MISSING_POLLS_BEFORE_DELETED = 3

let idSeq = 0
const nextId = (prefix: string) => `${prefix}-${Date.now().toString(36)}-${(idSeq++).toString(36)}`

export function createUploadQueue(items: UploadItem[], batches: UploadBatch[], deps: UploadQueueDeps) {
  const { setTimer, clearTimer } = deps.timers ?? defaultTimers

  // File payloads and abort handles stay out of the (possibly reactive) items:
  // proxying a File buys nothing and the panel never renders them.
  const payloads = new Map<string, UploadPayload>()
  const controllers = new Map<string, AbortController>()
  const missingPolls = new Map<string, number>()
  // Bumped by clear(), so work started before it cannot touch what comes after.
  let generation = 0
  let pollTimer: unknown = null

  const batchOf = (item: UploadItem) => batches.find(batch => batch.id === item.batchId)
  const findItem = (id: string) => items.find(item => item.id === id)

  const isBusy = (kbId: string) => items.some(item =>
    (item.transfer === 'queued' || item.transfer === 'uploading') && batchOf(item)?.kbId === kbId)

  // ---- transfer -----------------------------------------------------------

  // Identical files always have the same size. Keeping same-size files for one
  // knowledge base out of flight together lets the server's content-hash dedup,
  // which only sees rows already written, catch the second copy.
  const dedupKey = (item: UploadItem) => `${batchOf(item)?.kbId}:${item.size}`

  const pump = () => {
    for (const item of pickNextToStart(items, deps.concurrency, dedupKey)) void transfer(item)
  }

  const transfer = async (item: UploadItem) => {
    const batch = batchOf(item)
    const payload = payloads.get(item.id)
    if (!batch || !payload) {
      item.transfer = 'failed'
      return
    }
    const startedIn = generation
    const controller = new AbortController()
    controllers.set(item.id, controller)
    item.transfer = 'uploading'
    item.loaded = 0
    item.error = undefined

    let result: unknown
    try {
      result = await deps.upload(batch, payload, ratio => {
        if (item.transfer === 'uploading') item.loaded = Math.min(item.size, Math.round(ratio * item.size))
      }, controller.signal)
    } catch (error) {
      result = error
    }
    // A retry after a cancel starts a new attempt under the same id; only the
    // latest attempt may write the item.
    const latestAttempt = controllers.get(item.id) === controller
    if (latestAttempt) controllers.delete(item.id)
    const aborted = controller.signal.aborted

    // Outlived clear(): the row is gone from the panel, but a file that still
    // made it (clear() lets fully sent files finish) is new in the list.
    if (startedIn !== generation) {
      if (!aborted && classifyUploadResult(result).kind === 'uploaded') {
        deps.onTransferEnd(batch.kbId, { uploaded: true, settled: !isBusy(batch.kbId) })
      }
      return
    }
    if (!latestAttempt) return

    if (aborted || item.transfer !== 'uploading') {
      // Cancelled mid-flight; the abort rejection carries nothing worth showing.
      if (item.transfer === 'uploading') item.transfer = 'cancelled'
    } else {
      const outcome = classifyUploadResult(result)
      if (outcome.kind === 'uploaded') {
        item.transfer = 'uploaded'
        item.loaded = item.size
        item.knowledgeId = outcome.knowledgeId
        item.parseStatus = outcome.parseStatus || 'pending'
        payloads.delete(item.id)
        schedulePoll()
      } else if (outcome.kind === 'duplicate') {
        item.transfer = 'duplicate'
        item.knowledgeId = outcome.knowledgeId
        payloads.delete(item.id)
      } else {
        item.transfer = 'failed'
        item.error = outcome.message
      }
    }
    deps.onTransferEnd(batch.kbId, { uploaded: item.transfer === 'uploaded', settled: !isBusy(batch.kbId) })
    pump()
  }

  const add = (batchInput: Omit<UploadBatch, 'id'>, uploads: UploadPayload[]) => {
    const batch: UploadBatch = { ...batchInput, id: nextId('batch') }
    batches.push(batch)
    for (const upload of uploads) {
      const id = nextId('upload')
      payloads.set(id, upload)
      items.push({
        id,
        batchId: batch.id,
        name: upload.file.name,
        relativePath: upload.file.webkitRelativePath || '',
        size: upload.file.size,
        loaded: 0,
        transfer: 'queued',
      })
    }
    pump()
    return batch
  }

  const cancel = (id: string) => {
    const item = findItem(id)
    if (!item || (item.transfer !== 'queued' && item.transfer !== 'uploading')) return
    // Once the whole body is sent the server is already storing the file;
    // aborting now would hide a row that still gets created.
    if (itemPhase(item) === 'saving') return
    item.transfer = 'cancelled'
    item.loaded = 0
    const controller = controllers.get(id)
    if (controller) {
      // Its transfer reports the end once the abort lands.
      controller.abort()
      return
    }
    const kbId = batchOf(item)?.kbId
    if (kbId && !isBusy(kbId)) deps.onTransferEnd(kbId, { uploaded: false, settled: true })
  }

  const cancelAll = () => {
    for (const item of items) cancel(item.id)
  }

  const retry = (id: string) => {
    const item = findItem(id)
    if (!item || (item.transfer !== 'failed' && item.transfer !== 'cancelled') || !payloads.has(id)) return
    item.transfer = 'queued'
    item.loaded = 0
    item.error = undefined
    pump()
  }

  const retryFailed = () => {
    for (const item of items) retry(item.id)
  }

  /** Drop batches whose every file is searchable; anything with an issue stays. */
  const pruneFinished = () => {
    for (const batch of [...batches]) {
      const own = items.filter(item => item.batchId === batch.id)
      if (!own.every(item => itemPhase(item) === 'ready')) continue
      for (const item of own) {
        items.splice(items.indexOf(item), 1)
        missingPolls.delete(item.id)
      }
      batches.splice(batches.indexOf(batch), 1)
    }
  }

  // ---- parse polling ------------------------------------------------------

  function schedulePoll() {
    if (pollTimer !== null || !items.some(needsParsePolling)) return
    pollTimer = setTimer(() => {
      pollTimer = null
      void pollNow()
    }, deps.pollIntervalMs)
  }

  const pollNow = async () => {
    const startedIn = generation
    const byKb = new Map<string, UploadItem[]>()
    for (const item of items) {
      const kbId = needsParsePolling(item) ? batchOf(item)?.kbId : undefined
      if (!kbId) continue
      if (!byKb.has(kbId)) byKb.set(kbId, [])
      byKb.get(kbId)!.push(item)
    }

    const chunks: Array<{ kbId: string; chunk: UploadItem[] }> = []
    for (const [kbId, group] of byKb) {
      for (let i = 0; i < group.length; i += deps.pollChunkSize) {
        chunks.push({ kbId, chunk: group.slice(i, i + deps.pollChunkSize) })
      }
    }

    const queryChunk = async ({ kbId, chunk }: { kbId: string; chunk: UploadItem[] }) => {
      const rows = await deps.queryStatus(kbId, chunk.map(item => item.knowledgeId!)).catch(() => null)
      // A failed request just waits for the next tick.
      if (!rows || startedIn !== generation) return
      const byId = new Map(rows.map(row => [row.id, row]))
      for (const item of chunk) {
        const row = byId.get(item.knowledgeId!)
        if (!row) {
          const misses = (missingPolls.get(item.id) ?? 0) + 1
          missingPolls.set(item.id, misses)
          if (misses >= MISSING_POLLS_BEFORE_DELETED) item.parseStatus = 'deleted'
          continue
        }
        missingPolls.delete(item.id)
        item.parseStatus = row.parse_status
        item.parseError = row.error_message || undefined
      }
    }

    let next = 0
    const worker = async () => {
      while (next < chunks.length && startedIn === generation) await queryChunk(chunks[next++])
    }
    await Promise.all(Array.from({ length: Math.min(Math.max(1, deps.pollConcurrency), chunks.length) }, worker))
    if (startedIn === generation) schedulePoll()
  }

  // ---- lifecycle ----------------------------------------------------------

  /**
   * Forget everything. Transfers still sending are aborted; ones whose body is
   * fully sent are left to finish, since the server stores them regardless.
   */
  const clear = () => {
    generation++
    for (const item of items) {
      if (itemPhase(item) !== 'saving') controllers.get(item.id)?.abort()
    }
    const busyKbs = new Set(items.filter(item => item.transfer === 'queued' || item.transfer === 'uploading')
      .map(item => batchOf(item)?.kbId)
      .filter((kbId): kbId is string => !!kbId))
    controllers.clear()
    payloads.clear()
    missingPolls.clear()
    items.splice(0)
    batches.splice(0)
    if (pollTimer !== null) {
      clearTimer(pollTimer)
      pollTimer = null
    }
    // Files that landed before the clear may still be waiting for a refresh.
    for (const kbId of busyKbs) deps.onTransferEnd(kbId, { uploaded: false, settled: true })
  }

  return { add, cancel, cancelAll, retry, retryFailed, pruneFinished, pollNow, clear }
}

/**
 * Turns transfer ends into `knowledgeFileUploaded` refreshes: while a knowledge
 * base still has work queued, at most one mid-batch (`settled: false`) refresh
 * per `intervalMs`; once it settles, a single final (`settled: true`) one.
 * Ends that settle within `coalesceMs` of each other (cancel-all aborting
 * several requests) produce one final refresh.
 */
export function createListRefreshThrottle(
  emit: (kbId: string, settled: boolean) => void,
  intervalMs: number,
  coalesceMs: number,
  timers: Timers = defaultTimers,
) {
  const throttles = new Map<string, unknown>()
  const pending = new Set<string>()
  const finals = new Map<string, unknown>()

  return (kbId: string, end: TransferEnd) => {
    if (end.settled) {
      const throttle = throttles.get(kbId)
      if (throttle !== undefined) timers.clearTimer(throttle)
      throttles.delete(kbId)
      pending.delete(kbId)
      if (finals.has(kbId)) return
      finals.set(kbId, timers.setTimer(() => {
        finals.delete(kbId)
        emit(kbId, true)
      }, coalesceMs))
      return
    }
    if (!end.uploaded) return
    if (throttles.has(kbId)) {
      pending.add(kbId)
      return
    }
    emit(kbId, false)
    const tick = () => {
      if (pending.delete(kbId)) {
        emit(kbId, false)
        throttles.set(kbId, timers.setTimer(tick, intervalMs))
      } else {
        throttles.delete(kbId)
      }
    }
    throttles.set(kbId, timers.setTimer(tick, intervalMs))
  }
}