Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

update ingest page

Mike Fortman57c454614bd2
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index 1b43564df..ace13be5a 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -1,17 +1,18 @@
"use client";
+import { useQueryClient } from "@tanstack/react-query";
import { AlertCircle, ArrowLeft, RefreshCw } from "lucide-react";
import { useParams, useRouter } from "next/navigation";
import { useState } from "react";
import { toast } from "sonner";
-import { useQueryClient } from "@tanstack/react-query";
import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";
import { useGetConnectorTokenQuery } from "@/app/api/queries/useGetConnectorTokenQuery";
import { useIBMCOSBucketStatusQuery } from "@/app/api/queries/useIBMCOSBucketStatusQuery";
import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQuery";
import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";
-import type { IngestSettings } from "@/components/cloud-picker/types";
+import { IngestSettings } from "@/components/cloud-picker/ingest-settings";
+import type { IngestSettings as IngestSettingsType } from "@/components/cloud-picker/types";
import { Button } from "@/components/ui/button";
import {
Tooltip,
@@ -24,32 +25,45 @@ import { useTask } from "@/contexts/task-context";
const DIRECT_SYNC_PROVIDERS = ["ibm_cos", "aws_s3"];
// ---------------------------------------------------------------------------
-// IBM COS bucket list with sync status
+// Shared bucket view — used by both IBM COS and S3
// ---------------------------------------------------------------------------
-function IBMCOSBucketView({
+function BucketView({
connector,
+ buckets,
+ isLoading,
+ bucketsError,
+ onRefetch,
+ invalidateQueryKey,
syncMutation,
addTask,
onBack,
onDone,
}: {
connector: any;
+ buckets: Array<{ name: string; ingested_count: number }> | undefined;
+ isLoading: boolean;
+ bucketsError?: Error | null;
+ onRefetch: () => void;
+ invalidateQueryKey: readonly unknown[];
syncMutation: ReturnType<typeof useSyncConnector>;
addTask: (id: string) => void;
onBack: () => void;
onDone: () => void;
}) {
const queryClient = useQueryClient();
- const { data: buckets, isLoading, refetch } = useIBMCOSBucketStatusQuery(
- connector.connectionId,
- { enabled: true },
- );
-
const [syncingBucket, setSyncingBucket] = useState<string | null>(null);
+ const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({
+ chunkSize: 1000,
+ chunkOverlap: 200,
+ ocr: false,
+ pictureDescriptions: false,
+ embeddingModel: "text-embedding-3-small",
+ });
+ const [isSettingsOpen, setIsSettingsOpen] = useState(false);
- const invalidateBucketStatus = () => {
- queryClient.invalidateQueries({ queryKey: ["ibm-cos-bucket-status", connector.connectionId] });
+ const invalidate = () => {
+ queryClient.invalidateQueries({ queryKey: invalidateQueryKey });
};
const syncAll = () => {
@@ -60,11 +74,12 @@ function IBMCOSBucketView({
connection_id: connector.connectionId!,
selected_files: [],
sync_all: true,
+ settings: ingestSettings,
},
},
{
onSuccess: (result) => {
- invalidateBucketStatus();
+ invalidate();
if (result.task_ids?.length) {
addTask(result.task_ids[0]);
onDone();
@@ -88,12 +103,13 @@ function IBMCOSBucketView({
connection_id: connector.connectionId!,
selected_files: [],
bucket_filter: [bucketName],
+ settings: ingestSettings,
},
},
{
onSuccess: (result) => {
setSyncingBucket(null);
- invalidateBucketStatus();
+ invalidate();
if (result.task_ids?.length) {
addTask(result.task_ids[0]);
onDone();
@@ -125,35 +141,29 @@ function IBMCOSBucketView({
<p className="text-sm text-muted-foreground">
Select a bucket to ingest, or sync everything at once.
</p>
- <div className="flex gap-2">
- <Button
- variant="ghost"
- size="icon"
- onClick={() => refetch()}
- disabled={isLoading}
- title="Refresh bucket status"
- >
- <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />
- </Button>
- <Button
- className="bg-foreground text-background hover:bg-foreground/90 font-semibold"
- onClick={syncAll}
- disabled={syncMutation.isPending}
- >
- {syncMutation.isPending && !syncingBucket
- ? "Ingesting…"
- : "Sync All Buckets"}
- </Button>
- </div>
+ <Button
+ variant="ghost"
+ size="icon"
+ onClick={onRefetch}
+ disabled={isLoading}
+ title="Refresh bucket status"
+ >
+ <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />
+ </Button>
</div>
{isLoading ? (
<div className="flex justify-center py-8">
<div className="animate-spin rounded-full h-6 w-6 border-b-2 border-primary" />
</div>
+ ) : bucketsError ? (
+ <div className="rounded-lg border border-destructive/50 p-6 text-center text-destructive text-sm">
+ {bucketsError.message ||
+ "Failed to load buckets. Check your credentials and endpoint."}
+ </div>
) : !buckets?.length ? (
<div className="rounded-lg border p-6 text-center text-muted-foreground text-sm">
- No buckets found. Check your IBM COS credentials and endpoint.
+ No buckets found. Check your credentials and endpoint.
</div>
) : (
<div className="rounded-lg border divide-y">
@@ -162,15 +172,14 @@ function IBMCOSBucketView({
key={bucket.name}
className="flex items-center justify-between px-4 py-3"
>
- <div className="flex items-center gap-3">
- <div>
- <p className="font-medium text-sm">{bucket.name}</p>
- {bucket.ingested_count > 0 && (
- <p className="text-xs text-muted-foreground">
- {bucket.ingested_count} document{bucket.ingested_count !== 1 ? "s" : ""} ingested
- </p>
- )}
- </div>
+ <div>
+ <p className="font-medium text-sm">{bucket.name}</p>
+ {bucket.ingested_count > 0 && (
+ <p className="text-xs text-muted-foreground">
+ {bucket.ingested_count} document
+ {bucket.ingested_count !== 1 ? "s" : ""} ingested
+ </p>
+ )}
</div>
<Button
variant="outline"
@@ -188,16 +197,45 @@ function IBMCOSBucketView({
))}
</div>
)}
+
+ <IngestSettings
+ isOpen={isSettingsOpen}
+ onOpenChange={setIsSettingsOpen}
+ settings={ingestSettings}
+ onSettingsChange={setIngestSettings}
+ />
+ </div>
+
+ <div className="max-w-3xl mx-auto mt-6 sticky bottom-0 left-0 right-0 pb-6 bg-background pt-4">
+ <div className="flex justify-between gap-3">
+ <Button
+ variant="ghost"
+ className="border bg-transparent border-border rounded-lg text-secondary-foreground"
+ onClick={onBack}
+ >
+ Back
+ </Button>
+ <Button
+ className="bg-foreground text-background hover:bg-foreground/90 font-semibold"
+ onClick={syncAll}
+ disabled={syncMutation.isPending}
+ loading={syncMutation.isPending && !syncingBucket}
+ >
+ {syncMutation.isPending && !syncingBucket
+ ? "Ingesting…"
+ : "Sync All Buckets"}
+ </Button>
+ </div>
</div>
</>
);
}
// ---------------------------------------------------------------------------
-// Amazon S3 bucket list with sync status
+// IBM COS wrapper
// ---------------------------------------------------------------------------
-function S3BucketView({
+function IBMCOSBucketView({
connector,
syncMutation,
addTask,
@@ -210,160 +248,62 @@ function S3BucketView({
onBack: () => void;
onDone: () => void;
}) {
- const queryClient = useQueryClient();
- const { data: buckets, isLoading, error: bucketsError, refetch } = useS3BucketStatusQuery(
- connector.connectionId,
- { enabled: true },
+ const {
+ data: buckets,
+ isLoading,
+ refetch,
+ } = useIBMCOSBucketStatusQuery(connector.connectionId, { enabled: true });
+ return (
+ <BucketView
+ connector={connector}
+ buckets={buckets}
+ isLoading={isLoading}
+ onRefetch={refetch}
+ invalidateQueryKey={["ibm-cos-bucket-status", connector.connectionId]}
+ syncMutation={syncMutation}
+ addTask={addTask}
+ onBack={onBack}
+ onDone={onDone}
+ />
);
+}
- const [syncingBucket, setSyncingBucket] = useState<string | null>(null);
-
- const invalidateBucketStatus = () => {
- queryClient.invalidateQueries({ queryKey: ["s3-bucket-status", connector.connectionId] });
- };
-
- const syncAll = () => {
- syncMutation.mutate(
- {
- connectorType: connector.type,
- body: {
- connection_id: connector.connectionId!,
- selected_files: [],
- sync_all: true,
- },
- },
- {
- onSuccess: (result) => {
- invalidateBucketStatus();
- if (result.task_ids?.length) {
- addTask(result.task_ids[0]);
- onDone();
- } else {
- toast.info("No files found in any bucket.");
- }
- },
- onError: (err) => {
- toast.error(err instanceof Error ? err.message : "Sync failed");
- },
- },
- );
- };
-
- const syncBucket = (bucketName: string) => {
- setSyncingBucket(bucketName);
- syncMutation.mutate(
- {
- connectorType: connector.type,
- body: {
- connection_id: connector.connectionId!,
- selected_files: [],
- bucket_filter: [bucketName],
- },
- },
- {
- onSuccess: (result) => {
- setSyncingBucket(null);
- invalidateBucketStatus();
- if (result.task_ids?.length) {
- addTask(result.task_ids[0]);
- onDone();
- } else {
- toast.info(`No files found in bucket "${bucketName}".`);
- }
- },
- onError: (err) => {
- setSyncingBucket(null);
- toast.error(err instanceof Error ? err.message : "Sync failed");
- },
- },
- );
- };
+// ---------------------------------------------------------------------------
+// Amazon S3 wrapper
+// ---------------------------------------------------------------------------
+function S3BucketView({
+ connector,
+ syncMutation,
+ addTask,
+ onBack,
+ onDone,
+}: {
+ connector: any;
+ syncMutation: ReturnType<typeof useSyncConnector>;
+ addTask: (id: string) => void;
+ onBack: () => void;
+ onDone: () => void;
+}) {
+ const {
+ data: buckets,
+ isLoading,
+ error: bucketsError,
+ refetch,
+ } = useS3BucketStatusQuery(connector.connectionId, { enabled: true });
return (
- <>
- <div className="mb-8 flex gap-2 items-center">
- <Button variant="ghost" onClick={onBack} size="icon">
- <ArrowLeft size={18} />
- </Button>
- <h2 className="text-xl text-[18px] font-semibold">
- Add from {connector.name}
- </h2>
- </div>
-
- <div className="max-w-3xl mx-auto space-y-4">
- <div className="flex items-center justify-between">
- <p className="text-sm text-muted-foreground">
- Select a bucket to ingest, or sync everything at once.
- </p>
- <div className="flex gap-2">
- <Button
- variant="ghost"
- size="icon"
- onClick={() => refetch()}
- disabled={isLoading}
- title="Refresh bucket status"
- >
- <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />
- </Button>
- <Button
- className="bg-foreground text-background hover:bg-foreground/90 font-semibold"
- onClick={syncAll}
- disabled={syncMutation.isPending}
- >
- {syncMutation.isPending && !syncingBucket
- ? "Ingesting…"
- : "Sync All Buckets"}
- </Button>
- </div>
- </div>
-
- {isLoading ? (
- <div className="flex justify-center py-8">
- <div className="animate-spin rounded-full h-6 w-6 border-b-2 border-primary" />
- </div>
- ) : bucketsError ? (
- <div className="rounded-lg border border-destructive/50 p-6 text-center text-destructive text-sm">
- {(bucketsError as Error).message || "Failed to load buckets. Check your S3 credentials and endpoint."}
- </div>
- ) : !buckets?.length ? (
- <div className="rounded-lg border p-6 text-center text-muted-foreground text-sm">
- No buckets found. Check your S3 credentials and endpoint.
- </div>
- ) : (
- <div className="rounded-lg border divide-y">
- {buckets.map((bucket) => (
- <div
- key={bucket.name}
- className="flex items-center justify-between px-4 py-3"
- >
- <div className="flex items-center gap-3">
- <div>
- <p className="font-medium text-sm">{bucket.name}</p>
- {bucket.ingested_count > 0 && (
- <p className="text-xs text-muted-foreground">
- {bucket.ingested_count} document{bucket.ingested_count !== 1 ? "s" : ""} ingested
- </p>
- )}
- </div>
- </div>
- <Button
- variant="outline"
- size="sm"
- onClick={() => syncBucket(bucket.name)}
- disabled={syncMutation.isPending}
- >
- {syncingBucket === bucket.name
- ? "Ingesting…"
- : bucket.ingested_count > 0
- ? "Re-sync"
- : "Ingest"}
- </Button>
- </div>
- ))}
- </div>
- )}
- </div>
- </>
+ <BucketView
+ connector={connector}
+ buckets={buckets}
+ isLoading={isLoading}
+ bucketsError={bucketsError as Error | null}
+ onRefetch={refetch}
+ invalidateQueryKey={["s3-bucket-status", connector.connectionId]}
+ syncMutation={syncMutation}
+ addTask={addTask}
+ onBack={onBack}
+ onDone={onDone}
+ />
);
}
@@ -373,7 +313,7 @@ export default function UploadProviderPage() {
const params = useParams();
const router = useRouter();
const provider = params.provider as string;
- const { addTask, tasks } = useTask();
+ const { addTask } = useTask();
const {
data: connectors = [],
@@ -406,10 +346,7 @@ export default function UploadProviderPage() {
const syncMutation = useSyncConnector();
const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]);
- const [currentSyncTaskId, setCurrentSyncTaskId] = useState<string | null>(
- null,
- );
- const [ingestSettings, setIngestSettings] = useState<IngestSettings>({
+ const [ingestSettings, setIngestSettings] = useState<IngestSettingsType>({
chunkSize: 1000,
chunkOverlap: 200,
ocr: false,
@@ -459,7 +396,6 @@ export default function UploadProviderPage() {
if (taskIds && taskIds.length > 0) {
const taskId = taskIds[0]; // Use the first task ID
addTask(taskId);
- setCurrentSyncTaskId(taskId);
// Redirect to knowledge page already to show the syncing document
router.push("/knowledge");
}