Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
10.3 KB
Raw Blame
import type { File as SearchFile } from "@/app/api/queries/useGetSearchQuery";
import type { TaskFile } from "@/contexts/task-context";

export interface KnowledgeSourceOption {
  value: string;
  label: string;
  count?: number;
}

export function getKnowledgeFileIdentity(file?: {
  filename?: string;
  source_url?: string;
}) {
  if (!file) {
    return "";
  }

  const normalizedFilename = file.filename?.trim();
  if (normalizedFilename) {
    return normalizedFilename;
  }

  const normalizedSourceUrl = file.source_url?.trim();
  if (normalizedSourceUrl) {
    return normalizedSourceUrl;
  }

  return "";
}

function looksLikeHttpUrl(value: string): boolean {
  return /^https?:\/\//i.test(value.trim());
}

/** Filename variants for overlay matching (mirrors backend `get_filename_aliases`). */
function getKnowledgeFilenameAliases(filename?: string): string[] {
  const normalized = filename?.trim() ?? "";
  if (!normalized) {
    return [];
  }
  if (looksLikeHttpUrl(normalized)) {
    return [normalized];
  }
  const aliases = [normalized];
  const lower = normalized.toLowerCase();
  if (lower.endsWith(".txt")) {
    aliases.push(`${normalized.slice(0, -4)}.md`);
  } else if (lower.endsWith(".md")) {
    aliases.push(`${normalized.slice(0, -3)}.txt`);
  }
  for (const name of [...aliases]) {
    aliases.push(name.replace(/ /g, "_").replace(/\//g, "_"));
  }
  return [...new Set(aliases)];
}

function addFilenameAliasKeys(keys: Set<string>, filename?: string): void {
  for (const alias of getKnowledgeFilenameAliases(filename)) {
    keys.add(alias);
  }
}

/** Lookup keys for matching task overlays to indexed rows (filename, path, basename). */
export function getKnowledgeFileAliasKeys(file?: {
  filename?: string;
  source_url?: string;
}): string[] {
  const keys = new Set<string>();
  addFilenameAliasKeys(keys, file?.filename);
  const sourceUrl = file?.source_url?.trim();
  if (sourceUrl) {
    keys.add(sourceUrl);
    addFilenameAliasKeys(keys, sourceUrl.split("/").pop());
  }
  return [...keys];
}

function isMeaningfulConnectorType(connectorType?: string): boolean {
  const normalized = connectorType?.trim();
  return Boolean(normalized && normalized !== "local");
}

/** Infer connector_type for task overlays when the API does not return it. */
export function inferTaskFileConnectorType(
  filePath: string,
  fileName?: string,
  taskConnectorType?: string,
): string {
  if (isMeaningfulConnectorType(taskConnectorType)) {
    return taskConnectorType!.trim();
  }

  for (const candidate of [filePath, fileName ?? ""]) {
    const normalized = candidate.trim();
    if (!normalized) {
      continue;
    }
    if (looksLikeHttpUrl(normalized)) {
      return normalized.includes("openr.ag") ? "openrag_docs" : "url";
    }
  }

  // Bucket connectors (aws_s3, ibm_cos, azure_blob) key task files by a bare
  // object key or composite "<container>::<blob>" id — none of which are
  // distinguishable here — so we rely on the connector_type captured at ingest
  // time (taskConnectorTypesRef) and fall through to "local" otherwise. The
  // processing-row icon for these is tracked as a follow-up
  // (see [[task-connector-type-processing-icon]]).
  return "local";
}

/** Pick the connector icon source when merging backend rows with task overlays. */
export function resolveKnowledgeRowConnectorType(
  backendType?: string,
  taskType?: string,
  status: SearchFile["status"] = "active",
): string {
  const rowStatus = status ?? "active";

  if (rowStatus === "active") {
    if (isMeaningfulConnectorType(backendType)) {
      return backendType!.trim();
    }
    if (isMeaningfulConnectorType(taskType)) {
      return taskType!.trim();
    }
    return backendType?.trim() || taskType?.trim() || "local";
  }

  if (isMeaningfulConnectorType(taskType)) {
    return taskType!.trim();
  }
  if (isMeaningfulConnectorType(backendType)) {
    return backendType!.trim();
  }
  return taskType?.trim() || backendType?.trim() || "local";
}

function taskOverlayPriority(status?: string): number {
  switch (status) {
    case "processing":
      return 3;
    case "failed":
      return 2;
    case "cancelled":
      return 2;
    case "skipped":
      return 2;
    case "active":
      return 1;
    default:
      return 0;
  }
}

function indexTaskFileOverlays(
  taskFilesAsFiles: SearchFile[],
): Map<string, SearchFile> {
  const map = new Map<string, SearchFile>();
  for (const file of taskFilesAsFiles) {
    for (const key of getKnowledgeFileAliasKeys(file)) {
      const existing = map.get(key);
      if (
        !existing ||
        taskOverlayPriority(file.status) >= taskOverlayPriority(existing.status)
      ) {
        map.set(key, file);
      }
    }
  }
  return map;
}

function lookupTaskFileOverlay(
  map: Map<string, SearchFile>,
  file: SearchFile,
): SearchFile | undefined {
  for (const key of getKnowledgeFileAliasKeys(file)) {
    const match = map.get(key);
    if (match) {
      return match;
    }
  }
  return undefined;
}

export function buildKnowledgeTableRows(
  searchData: SearchFile[],
  taskFiles: TaskFile[],
  hasActiveFilter = false,
  dateRange?: { from?: Date; to?: Date } | null,
  bounds?: { createdAfter?: string; createdBefore?: string },
): SearchFile[] {
  const taskFilesAsFiles: SearchFile[] = taskFiles.map((taskFile) => {
    const normalizedFilename =
      taskFile.filename?.trim() ||
      taskFile.source_url?.trim() ||
      "Untitled source";

    return {
      filename: normalizedFilename,
      indexed_time: taskFile.created_at || undefined,
      mimetype: taskFile.mimetype,
      source_url: taskFile.source_url || "",
      size: taskFile.size,
      connector_type: taskFile.connector_type,
      status: taskFile.status,
      error: taskFile.error,
      skip_reason: taskFile.skip_reason,
      warning: taskFile.warning,
      embedding_model: taskFile.embedding_model,
      embedding_dimensions: taskFile.embedding_dimensions,
    };
  });

  const taskFileMap = indexTaskFileOverlays(taskFilesAsFiles);

  const backendFiles = searchData.map((file) => {
    if (file.connector_type === "openrag_docs") {
      return file;
    }
    const taskFile = lookupTaskFileOverlay(taskFileMap, file);
    if (taskFile) {
      const backendStatus = file.status ?? "active";
      const status =
        taskFile.status === "processing" ||
        taskFile.status === "failed" ||
        taskFile.status === "cancelled" ||
        taskFile.status === "skipped"
          ? taskFile.status
          : backendStatus;
      return {
        ...file,
        // Indexed row identity: prefer backend fields so filename and source_url stay paired.
        filename: file.filename || taskFile.filename,
        source_url: file.source_url || taskFile.source_url,
        connector_type: isMeaningfulConnectorType(file.connector_type)
          ? file.connector_type!.trim()
          : resolveKnowledgeRowConnectorType(
              file.connector_type,
              taskFile.connector_type,
              status,
            ),
        status,
        error: taskFile.error,
        skip_reason: taskFile.skip_reason,
        warning: taskFile.warning,
        embedding_model: taskFile.embedding_model ?? file.embedding_model,
        embedding_dimensions:
          taskFile.embedding_dimensions ?? file.embedding_dimensions,
        // Preserve task_id so getTaskIdForRow can match correctly for cancellation
        task_id: taskFile.task_id,
      };
    }
    return file;
  });

  const backendIdentityKeys = new Set<string>();
  for (const file of searchData) {
    for (const key of getKnowledgeFileAliasKeys(file)) {
      backendIdentityKeys.add(key);
    }
  }

  const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {
    if (
      taskFile.filename === "OpenRAG docs refresh" ||
      (taskFile.source_url ?? "").includes("openr.ag")
    ) {
      return false;
    }
    if (taskFile.connector_type === "openrag_docs") {
      return false;
    }
    if (
      getKnowledgeFileAliasKeys(taskFile).some((key) =>
        backendIdentityKeys.has(key),
      )
    ) {
      return false;
    }
    // Keep "active" overlays until the index lists the file (task drops key before refetch).
    return true;
  });

  if (hasActiveFilter) {
    return backendFiles;
  }

  // Effective UTC bounds win over the display days so in-flight rows agree with
  // the server results even when the range was saved in another timezone.
  const boundFrom = bounds?.createdAfter
    ? new Date(bounds.createdAfter).getTime()
    : undefined;
  const boundTo = bounds?.createdBefore
    ? new Date(bounds.createdBefore).getTime()
    : undefined;
  const rangeFrom = dateRange?.from;
  const rangeTo = dateRange?.to;
  const useBounds = bounds !== undefined;
  const hasRange = useBounds
    ? boundFrom !== undefined || boundTo !== undefined
    : Boolean(rangeFrom ?? rangeTo);
  const dateFilteredTaskFiles = hasRange
    ? filteredTaskFiles.filter((f) => {
        if (!f.indexed_time) return true;
        const d = new Date(f.indexed_time);
        if (Number.isNaN(d.getTime())) return true;
        if (useBounds) {
          if (boundFrom !== undefined && d.getTime() < boundFrom) return false;
          if (boundTo !== undefined && d.getTime() > boundTo) return false;
          return true;
        }
        if (rangeFrom) {
          const start = new Date(
            rangeFrom.getFullYear(),
            rangeFrom.getMonth(),
            rangeFrom.getDate(),
          );
          if (d < start) return false;
        }
        if (rangeTo) {
          const end = new Date(
            rangeTo.getFullYear(),
            rangeTo.getMonth(),
            rangeTo.getDate(),
            23,
            59,
            59,
            999,
          );
          if (d > end) return false;
        }
        return true;
      })
    : filteredTaskFiles;

  return [...backendFiles, ...dateFilteredTaskFiles];
}

export function buildActiveSourceOptions(
  rows: SearchFile[],
): KnowledgeSourceOption[] {
  const seen = new Set<string>();
  const options: KnowledgeSourceOption[] = [];

  for (const file of rows) {
    if ((file.status || "active") !== "active") continue;
    const source = file.filename?.trim() || file.source_url?.trim();
    if (!source || seen.has(source)) continue;
    seen.add(source);
    options.push({ value: source, label: source });
  }

  return options.sort((a, b) => a.label.localeCompare(b.label));
}