Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
4.0 KB
Raw Blame
import {
  type QueryClient,
  useMutation,
  useQueryClient,
} from "@tanstack/react-query";
import { TASKS_QUERY_KEY } from "@/app/api/queries/useGetTasksQuery";
import { websiteSourceQueryKey } from "@/app/api/queries/useGetWebsiteSourceQuery";
import { websiteSourcesQueryKey } from "@/app/api/queries/useGetWebsiteSourcesQuery";
import type { WebsiteSource } from "@/components/website-pages/types";

export interface CreateWebsiteSourceRequest {
  name: string;
  starting_url: string;
  scope: "path" | "page" | "site";
  allow_subdomains: boolean;
  additional_hosts: string[];
  include_paths: string[];
  exclude_paths: string[];
  max_pages: number;
  max_depth: number;
  max_downloaded_mb: number;
  max_crawl_minutes: number;
  change_detection: "normalized_content_hash" | "always_reingest";
  resync_behavior: "full" | "root";
  removed_page_behavior: "retain" | "delete";
}

export interface WebsiteSourceTaskResponse {
  last_task_id?: string;
  task_id?: string;
}

export interface WebsiteSourcePageRequest {
  sourceId: string;
  pageId?: string;
}

async function readError(response: Response, fallback: string) {
  const data = (await response.json().catch(() => ({}))) as {
    detail?: string;
    error?: string;
  };
  return data.detail || data.error || fallback;
}

async function createWebsiteSource(
  request: CreateWebsiteSourceRequest,
): Promise<WebsiteSource & WebsiteSourceTaskResponse> {
  const response = await fetch("/api/connectors/url/sources", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(request),
  });
  if (!response.ok) {
    throw new Error(
      await readError(response, "Could not create website source"),
    );
  }
  return response.json();
}

async function syncWebsiteSource({
  sourceId,
  pageId,
}: WebsiteSourcePageRequest): Promise<WebsiteSourceTaskResponse> {
  const endpoint = pageId
    ? `/api/connectors/url/sources/${sourceId}/pages/${pageId}/sync`
    : `/api/connectors/url/sources/${sourceId}/sync`;
  const response = await fetch(endpoint, { method: "POST" });
  if (!response.ok) {
    throw new Error(await readError(response, "Could not start website sync"));
  }
  return response.json();
}

async function deleteWebsiteSourcePage({
  sourceId,
  pageId,
}: WebsiteSourcePageRequest): Promise<void> {
  const endpoint = pageId
    ? `/api/connectors/url/sources/${sourceId}/pages/${pageId}`
    : `/api/connectors/url/sources/${sourceId}`;
  const response = await fetch(endpoint, { method: "DELETE" });
  if (!response.ok) {
    throw new Error(
      await readError(response, "Could not delete website source"),
    );
  }
}

function invalidateWebsiteSourceQueries(
  queryClient: QueryClient,
  sourceId: string,
) {
  queryClient.invalidateQueries({ queryKey: websiteSourceQueryKey(sourceId) });
  queryClient.invalidateQueries({ queryKey: websiteSourcesQueryKey });
  queryClient.invalidateQueries({ queryKey: ["search"] });
  queryClient.invalidateQueries({ queryKey: [...TASKS_QUERY_KEY] });
}

export function useCreateWebsiteSourceMutation() {
  const queryClient = useQueryClient();

  return useMutation({
    mutationFn: createWebsiteSource,
    onSuccess: () => {
      queryClient.invalidateQueries({ queryKey: ["search"] });
      queryClient.invalidateQueries({ queryKey: ["listFiles"] });
      queryClient.invalidateQueries({ queryKey: websiteSourcesQueryKey });
      queryClient.invalidateQueries({ queryKey: [...TASKS_QUERY_KEY] });
    },
  });
}

export function useSyncWebsiteSourceMutation() {
  const queryClient = useQueryClient();

  return useMutation({
    mutationFn: syncWebsiteSource,
    onSuccess: (_data, { sourceId }) => {
      invalidateWebsiteSourceQueries(queryClient, sourceId);
    },
  });
}

export function useDeleteWebsiteSourcePageMutation() {
  const queryClient = useQueryClient();

  return useMutation({
    mutationFn: deleteWebsiteSourcePage,
    onSuccess: (_data, { sourceId }) => {
      invalidateWebsiteSourceQueries(queryClient, sourceId);
    },
  });
}