OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
update ingest page
Mike Fortman57c454614bd2
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex 1b43564df..ace13be5a 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -1,17 +1,18 @@"use client";+import { useQueryClient } from "@tanstack/react-query";import { AlertCircle, ArrowLeft, RefreshCw } from "lucide-react";import { useParams, useRouter } from "next/navigation";import { useState } from "react";import { toast } from "sonner";-import { useQueryClient } from "@tanstack/react-query";import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";import { useGetConnectorTokenQuery } from "@/app/api/queries/useGetConnectorTokenQuery";import { useIBMCOSBucketStatusQuery } from "@/app/api/queries/useIBMCOSBucketStatusQuery";import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQuery";import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";-import type { IngestSettings } from "@/components/cloud-picker/types";+import { IngestSettings } from "@/components/cloud-picker/ingest-settings";+import type { IngestSettings as IngestSettingsType } from "@/components/cloud-picker/types";import { Button } from "@/components/ui/button";import {Tooltip,@@ -24,32 +25,45 @@ import { useTask } from "@/contexts/task-context";const DIRECT_SYNC_PROVIDERS = ["ibm_cos", "aws_s3"];// ----------------------------------------------------------------------------// IBM COS bucket list with sync status+// Shared bucket view — used by both IBM COS and S3// ----------------------------------------------------------------------------function IBMCOSBucketView({+function BucketView({connector,+ buckets,+ isLoading,+ bucketsError,+ onRefetch,+ invalidateQueryKey,syncMutation,addTask,onBack,onDone,}: {connector: any;+ buckets: Array<{ name: string; ingested_count: number }> | undefined;+ isLoading: boolean;+ bucketsError?: Error | null;+ onRefetch: () => void;+ invalidateQueryKey: readonly unknown[];syncMutation: ReturnType<typeof useSyncConnector>;addTask: (id: string) => void;onBack: () => void;onDone: () => void;}) {const queryClient = useQueryClient();- const { data: buckets, isLoading, refetch } = useIBMCOSBucketStatusQuery(- connector.connectionId,- { enabled: true },- );-const [syncingBucket, setSyncingBucket] = useState<string | null>(null);+ const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({+ chunkSize: 1000,+ chunkOverlap: 200,+ ocr: false,+ pictureDescriptions: false,+ embeddingModel: "text-embedding-3-small",+ });+ const [isSettingsOpen, setIsSettingsOpen] = useState(false);- const invalidateBucketStatus = () => {- queryClient.invalidateQueries({ queryKey: ["ibm-cos-bucket-status", connector.connectionId] });+ const invalidate = () => {+ queryClient.invalidateQueries({ queryKey: invalidateQueryKey });};const syncAll = () => {@@ -60,11 +74,12 @@ function IBMCOSBucketView({connection_id: connector.connectionId!,selected_files: [],sync_all: true,+ settings: ingestSettings,},},{onSuccess: (result) => {- invalidateBucketStatus();+ invalidate();if (result.task_ids?.length) {addTask(result.task_ids[0]);onDone();@@ -88,12 +103,13 @@ function IBMCOSBucketView({connection_id: connector.connectionId!,selected_files: [],bucket_filter: [bucketName],+ settings: ingestSettings,},},{onSuccess: (result) => {setSyncingBucket(null);- invalidateBucketStatus();+ invalidate();if (result.task_ids?.length) {addTask(result.task_ids[0]);onDone();@@ -125,35 +141,29 @@ function IBMCOSBucketView({<p className="text-sm text-muted-foreground">Select a bucket to ingest, or sync everything at once.</p>- <div className="flex gap-2">- <Button- variant="ghost"- size="icon"- onClick={() => refetch()}- disabled={isLoading}- title="Refresh bucket status"- >- <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />- </Button>- <Button- className="bg-foreground text-background hover:bg-foreground/90 font-semibold"- onClick={syncAll}- disabled={syncMutation.isPending}- >- {syncMutation.isPending && !syncingBucket- ? "Ingesting…"- : "Sync All Buckets"}- </Button>- </div>+ <Button+ variant="ghost"+ size="icon"+ onClick={onRefetch}+ disabled={isLoading}+ title="Refresh bucket status"+ >+ <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />+ </Button></div>{isLoading ? (<div className="flex justify-center py-8"><div className="animate-spin rounded-full h-6 w-6 border-b-2 border-primary" /></div>+ ) : bucketsError ? (+ <div className="rounded-lg border border-destructive/50 p-6 text-center text-destructive text-sm">+ {bucketsError.message ||+ "Failed to load buckets. Check your credentials and endpoint."}+ </div>) : !buckets?.length ? (<div className="rounded-lg border p-6 text-center text-muted-foreground text-sm">- No buckets found. Check your IBM COS credentials and endpoint.+ No buckets found. Check your credentials and endpoint.</div>) : (<div className="rounded-lg border divide-y">@@ -162,15 +172,14 @@ function IBMCOSBucketView({key={bucket.name}className="flex items-center justify-between px-4 py-3">- <div className="flex items-center gap-3">- <div>- <p className="font-medium text-sm">{bucket.name}</p>- {bucket.ingested_count > 0 && (- <p className="text-xs text-muted-foreground">- {bucket.ingested_count} document{bucket.ingested_count !== 1 ? "s" : ""} ingested- </p>- )}- </div>+ <div>+ <p className="font-medium text-sm">{bucket.name}</p>+ {bucket.ingested_count > 0 && (+ <p className="text-xs text-muted-foreground">+ {bucket.ingested_count} document+ {bucket.ingested_count !== 1 ? "s" : ""} ingested+ </p>+ )}</div><Buttonvariant="outline"@@ -188,16 +197,45 @@ function IBMCOSBucketView({))}</div>)}++ <IngestSettings+ isOpen={isSettingsOpen}+ onOpenChange={setIsSettingsOpen}+ settings={ingestSettings}+ onSettingsChange={setIngestSettings}+ />+ </div>++ <div className="max-w-3xl mx-auto mt-6 sticky bottom-0 left-0 right-0 pb-6 bg-background pt-4">+ <div className="flex justify-between gap-3">+ <Button+ variant="ghost"+ className="border bg-transparent border-border rounded-lg text-secondary-foreground"+ onClick={onBack}+ >+ Back+ </Button>+ <Button+ className="bg-foreground text-background hover:bg-foreground/90 font-semibold"+ onClick={syncAll}+ disabled={syncMutation.isPending}+ loading={syncMutation.isPending && !syncingBucket}+ >+ {syncMutation.isPending && !syncingBucket+ ? "Ingesting…"+ : "Sync All Buckets"}+ </Button>+ </div></div></>);}// ----------------------------------------------------------------------------// Amazon S3 bucket list with sync status+// IBM COS wrapper// ----------------------------------------------------------------------------function S3BucketView({+function IBMCOSBucketView({connector,syncMutation,addTask,@@ -210,160 +248,62 @@ function S3BucketView({onBack: () => void;onDone: () => void;}) {- const queryClient = useQueryClient();- const { data: buckets, isLoading, error: bucketsError, refetch } = useS3BucketStatusQuery(- connector.connectionId,- { enabled: true },+ const {+ data: buckets,+ isLoading,+ refetch,+ } = useIBMCOSBucketStatusQuery(connector.connectionId, { enabled: true });+ return (+ <BucketView+ connector={connector}+ buckets={buckets}+ isLoading={isLoading}+ onRefetch={refetch}+ invalidateQueryKey={["ibm-cos-bucket-status", connector.connectionId]}+ syncMutation={syncMutation}+ addTask={addTask}+ onBack={onBack}+ onDone={onDone}+ />);+}- const [syncingBucket, setSyncingBucket] = useState<string | null>(null);-- const invalidateBucketStatus = () => {- queryClient.invalidateQueries({ queryKey: ["s3-bucket-status", connector.connectionId] });- };-- const syncAll = () => {- syncMutation.mutate(- {- connectorType: connector.type,- body: {- connection_id: connector.connectionId!,- selected_files: [],- sync_all: true,- },- },- {- onSuccess: (result) => {- invalidateBucketStatus();- if (result.task_ids?.length) {- addTask(result.task_ids[0]);- onDone();- } else {- toast.info("No files found in any bucket.");- }- },- onError: (err) => {- toast.error(err instanceof Error ? err.message : "Sync failed");- },- },- );- };-- const syncBucket = (bucketName: string) => {- setSyncingBucket(bucketName);- syncMutation.mutate(- {- connectorType: connector.type,- body: {- connection_id: connector.connectionId!,- selected_files: [],- bucket_filter: [bucketName],- },- },- {- onSuccess: (result) => {- setSyncingBucket(null);- invalidateBucketStatus();- if (result.task_ids?.length) {- addTask(result.task_ids[0]);- onDone();- } else {- toast.info(`No files found in bucket "${bucketName}".`);- }- },- onError: (err) => {- setSyncingBucket(null);- toast.error(err instanceof Error ? err.message : "Sync failed");- },- },- );- };+// ---------------------------------------------------------------------------+// Amazon S3 wrapper+// ---------------------------------------------------------------------------+function S3BucketView({+ connector,+ syncMutation,+ addTask,+ onBack,+ onDone,+}: {+ connector: any;+ syncMutation: ReturnType<typeof useSyncConnector>;+ addTask: (id: string) => void;+ onBack: () => void;+ onDone: () => void;+}) {+ const {+ data: buckets,+ isLoading,+ error: bucketsError,+ refetch,+ } = useS3BucketStatusQuery(connector.connectionId, { enabled: true });return (- <>- <div className="mb-8 flex gap-2 items-center">- <Button variant="ghost" onClick={onBack} size="icon">- <ArrowLeft size={18} />- </Button>- <h2 className="text-xl text-[18px] font-semibold">- Add from {connector.name}- </h2>- </div>-- <div className="max-w-3xl mx-auto space-y-4">- <div className="flex items-center justify-between">- <p className="text-sm text-muted-foreground">- Select a bucket to ingest, or sync everything at once.- </p>- <div className="flex gap-2">- <Button- variant="ghost"- size="icon"- onClick={() => refetch()}- disabled={isLoading}- title="Refresh bucket status"- >- <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />- </Button>- <Button- className="bg-foreground text-background hover:bg-foreground/90 font-semibold"- onClick={syncAll}- disabled={syncMutation.isPending}- >- {syncMutation.isPending && !syncingBucket- ? "Ingesting…"- : "Sync All Buckets"}- </Button>- </div>- </div>-- {isLoading ? (- <div className="flex justify-center py-8">- <div className="animate-spin rounded-full h-6 w-6 border-b-2 border-primary" />- </div>- ) : bucketsError ? (- <div className="rounded-lg border border-destructive/50 p-6 text-center text-destructive text-sm">- {(bucketsError as Error).message || "Failed to load buckets. Check your S3 credentials and endpoint."}- </div>- ) : !buckets?.length ? (- <div className="rounded-lg border p-6 text-center text-muted-foreground text-sm">- No buckets found. Check your S3 credentials and endpoint.- </div>- ) : (- <div className="rounded-lg border divide-y">- {buckets.map((bucket) => (- <div- key={bucket.name}- className="flex items-center justify-between px-4 py-3"- >- <div className="flex items-center gap-3">- <div>- <p className="font-medium text-sm">{bucket.name}</p>- {bucket.ingested_count > 0 && (- <p className="text-xs text-muted-foreground">- {bucket.ingested_count} document{bucket.ingested_count !== 1 ? "s" : ""} ingested- </p>- )}- </div>- </div>- <Button- variant="outline"- size="sm"- onClick={() => syncBucket(bucket.name)}- disabled={syncMutation.isPending}- >- {syncingBucket === bucket.name- ? "Ingesting…"- : bucket.ingested_count > 0- ? "Re-sync"- : "Ingest"}- </Button>- </div>- ))}- </div>- )}- </div>- </>+ <BucketView+ connector={connector}+ buckets={buckets}+ isLoading={isLoading}+ bucketsError={bucketsError as Error | null}+ onRefetch={refetch}+ invalidateQueryKey={["s3-bucket-status", connector.connectionId]}+ syncMutation={syncMutation}+ addTask={addTask}+ onBack={onBack}+ onDone={onDone}+ />);}@@ -373,7 +313,7 @@ export default function UploadProviderPage() {const params = useParams();const router = useRouter();const provider = params.provider as string;- const { addTask, tasks } = useTask();+ const { addTask } = useTask();const {data: connectors = [],@@ -406,10 +346,7 @@ export default function UploadProviderPage() {const syncMutation = useSyncConnector();const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]);- const [currentSyncTaskId, setCurrentSyncTaskId] = useState<string | null>(- null,- );- const [ingestSettings, setIngestSettings] = useState<IngestSettings>({+ const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({chunkSize: 1000,chunkOverlap: 200,ocr: false,@@ -459,7 +396,6 @@ export default function UploadProviderPage() {if (taskIds && taskIds.length > 0) {const taskId = taskIds[0]; // Use the first task IDaddTask(taskId);- setCurrentSyncTaskId(taskId);// Redirect to knowledge page already to show the syncing documentrouter.push("/knowledge");}