Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
11.7 KB
Raw Blame
import {
  type UseQueryOptions,
  useQuery,
  useQueryClient,
} from "@tanstack/react-query";
import type { ParsedQueryData } from "@/contexts/knowledge-filter-context";
import { apiClient, isRequestCancelled } from "@/lib/api-client";
import { SEARCH_CONSTANTS } from "@/lib/constants";
import {
  buildSearchPayloadFilters,
  type FilterInput,
} from "@/lib/filter-normalization";

export interface SearchPayload {
  query: string;
  limit: number;
  scoreThreshold: number;
  filters?: FilterInput;
  created_after?: string;
  created_before?: string;
  resultMode?: "chunks" | "knowledge_sources" | "website_pages";
}

export interface ChunkResult {
  filename: string;
  mimetype: string;
  page: number;
  text: string;
  /** Keyword-match highlight fragments from OpenSearch. Each fragment is a
   *  substring of `text` with matched terms wrapped in `<mark>` tags.
   *  Empty array when no keyword match exists (pure semantic/KNN hit). */
  highlights?: string[];
  score: number;
  source_url?: string;
  owner?: string;
  owner_name?: string;
  owner_email?: string;
  file_size?: number;
  connector_type?: string;
  embedding_model?: string;
  embedding_dimensions?: number;
  parser?: string;
  chunk_size?: number;
  chunk_overlap?: number;
  chunk_id?: string;
  id?: string;
  index?: number;
  allowed_users?: string[];
  allowed_groups?: string[];
  indexed_time?: string;
  document_id?: string;
  web_source_id?: string;
  web_page_id?: string;
  web_page_depth?: number;
  web_child_count?: number;
  canonical_url?: string;
  status?: File["status"];
  error?: string;
  chunk_count?: number;
}

export interface File {
  filename: string;
  mimetype: string;
  chunkCount?: number;
  avgScore?: number;
  source_url: string;
  owner?: string;
  owner_name?: string;
  owner_email?: string;
  size: number;
  connector_type: string;
  embedding_model?: string;
  embedding_dimensions?: number;
  status?:
    | "processing"
    | "active"
    | "unavailable"
    | "failed"
    | "cancelled"
    | "skipped"
    | "hidden"
    | "sync"
    | "disabled";
  error?: string;
  /**
   * Skip reason forwarded from result.reason (e.g. "duplicate_content",
   * "deleted_at_source"). Only set when status === "skipped".
   */
  skip_reason?: string;
  /** Warning message for skipped rows (e.g. duplicate_content). */
  warning?: string;
  task_id?: string; // Task ID for file-level cancellation
  chunks?: ChunkResult[];
  allowed_users?: string[];
  allowed_groups?: string[];
  indexed_time?: string;
  document_id?: string;
  web_source_id?: string;
  web_page_id?: string;
  web_page_depth?: number;
  web_child_count?: number;
}

// Non-fatal signal from the backend — e.g. an embedding provider was removed
// so some models in the corpus can't be queried semantically. Results still
// come back via keyword matching.
export interface SearchWarning {
  code: string;
  models?: string[];
  semantic_search_available?: boolean;
  message?: string;
}

export interface SearchResult {
  files: File[];
  warnings: SearchWarning[];
}

export interface SearchResultDisplayOptions {
  groupBy?: "filename" | "document_id";
  resultMode?: "chunks" | "knowledge_sources" | "website_pages";
}

const EMPTY_SEARCH_RESULT: SearchResult = { files: [], warnings: [] };

export { EMPTY_SEARCH_RESULT };

const getFileIdentity = (
  chunk: ChunkResult,
  groupBy: SearchResultDisplayOptions["groupBy"],
): string => {
  if (groupBy === "document_id" && chunk.document_id) {
    return chunk.document_id;
  }
  const normalizedFilename = chunk.filename?.trim();
  if (normalizedFilename) {
    return normalizedFilename;
  }

  const normalizedSourceUrl = chunk.source_url?.trim();
  if (normalizedSourceUrl) {
    return normalizedSourceUrl;
  }

  return "Untitled source";
};

/** Short single-token queries get a lower threshold so partial words still match. */
export function resolveScoreThreshold(
  query: string,
  baseScoreThreshold: number,
): number {
  const normalized = query.trim();
  const isShortSingleToken =
    normalized !== "*" &&
    normalized.length > 0 &&
    normalized.length <= 4 &&
    !normalized.includes(" ");
  return isShortSingleToken
    ? Math.min(baseScoreThreshold, 1.0)
    : baseScoreThreshold;
}

export const useGetSearchQuery = (
  query: string,
  queryData?: ParsedQueryData | null,
  options?: Omit<
    UseQueryOptions<SearchResult, Error, SearchResult, unknown[]>,
    "queryKey" | "queryFn"
  >,
  displayOptions: SearchResultDisplayOptions = {},
  // Effective UTC bounds from the caller. Saved dates inside `queryData` are
  // deliberately not sent so a cleared temporary range cannot fall back to them.
  dateRange?: { createdAfter?: string; createdBefore?: string },
) => {
  const queryClient = useQueryClient();

  // Normalize the query to match what will actually be searched
  const effectiveQuery = query || queryData?.query || "*";
  const normalizedQuery = effectiveQuery.trim();

  async function getFiles(): Promise<SearchResult> {
    try {
      // For wildcard queries, use a high limit to get all files
      // Otherwise use the limit from queryData or default to 100
      const isWildcardQuery =
        effectiveQuery.trim() === "*" || effectiveQuery.trim() === "";
      const searchLimit = isWildcardQuery
        ? SEARCH_CONSTANTS.WILDCARD_QUERY_LIMIT
        : queryData?.limit || 100;

      const baseScoreThreshold =
        queryData?.scoreThreshold ?? SEARCH_CONSTANTS.DEFAULT_SCORE_THRESHOLD;
      const dynamicScoreThreshold = resolveScoreThreshold(
        normalizedQuery,
        baseScoreThreshold,
      );

      const searchPayload: SearchPayload = {
        query: effectiveQuery,
        limit: searchLimit,
        scoreThreshold: dynamicScoreThreshold,
        resultMode: displayOptions.resultMode,
      };
      if (queryData?.filters) {
        searchPayload.filters =
          buildSearchPayloadFilters(queryData.filters, {
            includeDates: false,
          }) ?? undefined;
      }
      if (dateRange?.createdAfter) {
        searchPayload.created_after = dateRange.createdAfter;
      }
      if (dateRange?.createdBefore) {
        searchPayload.created_before = dateRange.createdBefore;
      }

      const response = await apiClient.post(`/search`, searchPayload);

      if (response.status < 200 || response.status >= 300) {
        const errorData =
          response.data && typeof response.data === "object"
            ? (response.data as { error?: string })
            : { error: "Unknown error" };
        throw new Error(
          errorData.error || `Search failed with status ${response.status}`,
        );
      }

      const data = response.data;
      // Group chunks by filename to create file results similar to page.tsx
      const fileMap = new Map<
        string,
        {
          filename: string;
          mimetype: string;
          chunks: ChunkResult[];
          totalScore: number;
          source_url?: string;
          owner?: string;
          owner_name?: string;
          owner_email?: string;
          file_size?: number;
          connector_type?: string;
          embedding_model?: string;
          embedding_dimensions?: number;
          allowed_users?: string[];
          allowed_groups?: string[];
          indexed_time?: string;
          document_id?: string;
          web_source_id?: string;
          web_page_id?: string;
          web_page_depth?: number;
          web_child_count?: number;
          canonical_url?: string;
          status?: File["status"];
          error?: string;
          chunk_count?: number;
        }
      >();

      (data.results || []).forEach((chunk: ChunkResult) => {
        const fileIdentity = getFileIdentity(chunk, displayOptions.groupBy);
        // Preserve highlights on the chunk object itself — they are per-chunk,
        // not per-file, so we carry them through rather than aggregating.
        const chunkWithHighlights: ChunkResult = {
          ...chunk,
          highlights: Array.isArray(chunk.highlights) ? chunk.highlights : [],
        };
        const existing = fileMap.get(fileIdentity);
        if (existing) {
          existing.chunks.push(chunkWithHighlights);
          existing.totalScore += chunk.score;
          if (!existing.embedding_model && chunk.embedding_model) {
            existing.embedding_model = chunk.embedding_model;
          }
          if (
            existing.embedding_dimensions == null &&
            typeof chunk.embedding_dimensions === "number"
          ) {
            existing.embedding_dimensions = chunk.embedding_dimensions;
          }
          if (typeof chunk.chunk_count === "number") {
            existing.chunk_count = chunk.chunk_count;
          }
          if (typeof chunk.web_child_count === "number") {
            existing.web_child_count = chunk.web_child_count;
          }
        } else {
          fileMap.set(fileIdentity, {
            filename:
              displayOptions.groupBy === "document_id"
                ? chunk.filename || fileIdentity
                : fileIdentity,
            mimetype: chunk.mimetype,
            chunks: [chunkWithHighlights],
            totalScore: chunk.score,
            source_url: chunk.source_url,
            owner: chunk.owner,
            owner_name: chunk.owner_name,
            owner_email: chunk.owner_email,
            file_size: chunk.file_size,
            connector_type: chunk.connector_type,
            embedding_model: chunk.embedding_model,
            embedding_dimensions: chunk.embedding_dimensions,
            allowed_users: chunk.allowed_users || [],
            allowed_groups: chunk.allowed_groups || [],
            indexed_time: chunk.indexed_time,
            document_id: chunk.document_id,
            web_source_id: chunk.web_source_id,
            web_page_id: chunk.web_page_id,
            web_page_depth: chunk.web_page_depth,
            web_child_count: chunk.web_child_count,
            canonical_url: chunk.canonical_url,
            status: chunk.status,
            error: chunk.error,
            chunk_count: chunk.chunk_count,
          });
        }
      });

      const files: File[] = Array.from(fileMap.values()).map((file) => ({
        filename: file.filename,
        mimetype: file.mimetype,
        chunkCount: file.chunk_count ?? file.chunks.length,
        avgScore: file.totalScore / file.chunks.length,
        source_url: file.source_url || "",
        owner: file.owner || "",
        owner_name: file.owner_name || "",
        owner_email: file.owner_email || "",
        size: file.file_size || 0,
        connector_type: file.connector_type || "local",
        embedding_model: file.embedding_model,
        embedding_dimensions: file.embedding_dimensions,
        chunks: file.chunks,
        allowed_users: file.allowed_users || [],
        allowed_groups: file.allowed_groups || [],
        indexed_time: file.indexed_time,
        document_id: file.document_id,
        web_source_id: file.web_source_id,
        web_page_id: file.web_page_id,
        web_page_depth: file.web_page_depth,
        web_child_count: file.web_child_count,
        status: file.status,
        error: file.error,
      }));

      const warnings: SearchWarning[] = Array.isArray(data.warnings)
        ? data.warnings
        : [];

      return { files, warnings };
    } catch (error) {
      // Cancellation is expected (not an error); still re-throw it below
      if (!isRequestCancelled(error)) {
        console.error("Error getting files", error);
      }
      // Re-throw the error so React Query can handle it and trigger onError callbacks
      throw error;
    }
  }

  return useQuery(
    {
      queryKey: ["search", queryData, query, displayOptions, dateRange],
      placeholderData: (prev) => prev,
      staleTime: 0,
      queryFn: getFiles,
      retry: false, // Don't retry on errors - show them immediately
      ...options,
    },
    queryClient,
  );
};