Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Add Amazon S3 connector and UI

Edwin Jose671f294ba18c
diff --git a/frontend/app/api/mutations/useS3ConfigureMutation.ts b/frontend/app/api/mutations/useS3ConfigureMutation.ts
new file mode 100644
index 000000000..1e4e16a5e
--- /dev/null
+++ b/frontend/app/api/mutations/useS3ConfigureMutation.ts
@@ -0,0 +1,33 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+export interface S3ConfigurePayload {
+ access_key?: string;
+ secret_key?: string;
+ endpoint_url?: string;
+ region?: string;
+ bucket_names?: string[];
+ connection_id?: string;
+}
+
+async function configureS3(payload: S3ConfigurePayload) {
+ const res = await fetch("/api/connectors/aws_s3/configure", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(payload),
+ });
+ const data = await res.json();
+ if (!res.ok) throw new Error(data.error || "Failed to configure S3");
+ return data as { connection_id: string; status: string };
+}
+
+export function useS3ConfigureMutation() {
+ const queryClient = useQueryClient();
+
+ return useMutation({
+ mutationFn: configureS3,
+ onSuccess: () => {
+ queryClient.invalidateQueries({ queryKey: ["connectors"] });
+ queryClient.invalidateQueries({ queryKey: ["s3-defaults"] });
+ },
+ });
+}
diff --git a/frontend/app/api/queries/useS3BucketStatusQuery.ts b/frontend/app/api/queries/useS3BucketStatusQuery.ts
new file mode 100644
index 000000000..595920933
--- /dev/null
+++ b/frontend/app/api/queries/useS3BucketStatusQuery.ts
@@ -0,0 +1,30 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface S3BucketStatus {
+ name: string;
+ ingested_count: number;
+ is_synced: boolean;
+}
+
+async function fetchS3BucketStatus(connectionId: string): Promise<S3BucketStatus[]> {
+ const res = await fetch(`/api/connectors/aws_s3/${connectionId}/bucket-status`);
+ if (!res.ok) {
+ const err = await res.json().catch(() => ({}));
+ throw new Error(err.error || "Failed to fetch bucket status");
+ }
+ const data = await res.json();
+ return data.buckets as S3BucketStatus[];
+}
+
+export function useS3BucketStatusQuery(
+ connectionId: string | null | undefined,
+ options?: { enabled?: boolean },
+) {
+ return useQuery<S3BucketStatus[]>({
+ queryKey: ["s3-bucket-status", connectionId],
+ queryFn: () => fetchS3BucketStatus(connectionId!),
+ enabled: (options?.enabled ?? true) && !!connectionId,
+ staleTime: 0,
+ refetchOnMount: "always",
+ });
+}
diff --git a/frontend/app/api/queries/useS3DefaultsQuery.ts b/frontend/app/api/queries/useS3DefaultsQuery.ts
new file mode 100644
index 000000000..9c7d24560
--- /dev/null
+++ b/frontend/app/api/queries/useS3DefaultsQuery.ts
@@ -0,0 +1,25 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface S3Defaults {
+ access_key_set: boolean;
+ secret_key_set: boolean;
+ endpoint: string;
+ region: string;
+ bucket_names: string[];
+ connection_id: string | null;
+}
+
+async function fetchS3Defaults(): Promise<S3Defaults> {
+ const res = await fetch("/api/connectors/aws_s3/defaults");
+ if (!res.ok) throw new Error("Failed to fetch S3 defaults");
+ return res.json();
+}
+
+export function useS3DefaultsQuery(options?: { enabled?: boolean }) {
+ return useQuery<S3Defaults>({
+ queryKey: ["s3-defaults"],
+ queryFn: fetchS3Defaults,
+ enabled: options?.enabled ?? true,
+ staleTime: 0,
+ });
+}
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index 6a99deab8..a17f95f8a 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -45,6 +45,7 @@ import IBMLogo from "../../components/icons/ibm-logo";
import IBMCOSIcon from "../../components/icons/ibm-cos-icon";
import OneDriveIcon from "../../components/icons/one-drive-logo";
import SharePointIcon from "../../components/icons/share-point-logo";
+import AwsLogo from "../../components/icons/aws-logo";
import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";
@@ -67,6 +68,8 @@ function getSourceIcon(connectorType?: string) {
return <Cloud className="h-4 w-4 text-foreground flex-shrink-0" />;
case "ibm_cos":
return <IBMCOSIcon className="h-4 w-4 flex-shrink-0" />;
+ case "aws_s3":
+ return <AwsLogo className="h-4 w-4 flex-shrink-0" />;
default:
return (
<FileIcon className="h-4 w-4 text-muted-foreground flex-shrink-0" />
diff --git a/frontend/app/settings/_components/connector-cards.tsx b/frontend/app/settings/_components/connector-cards.tsx
index 00f207584..1121fad94 100644
--- a/frontend/app/settings/_components/connector-cards.tsx
+++ b/frontend/app/settings/_components/connector-cards.tsx
@@ -12,15 +12,18 @@ import GoogleDriveIcon from "@/components/icons/google-drive-logo";
import IBMCOSIcon from "@/components/icons/ibm-cos-icon";
import OneDriveIcon from "@/components/icons/one-drive-logo";
import SharePointIcon from "@/components/icons/share-point-logo";
+import AwsLogo from "@/components/icons/aws-logo";
import { useAuth } from "@/contexts/auth-context";
import ConnectorCard, { type Connector } from "./connector-card";
import ConnectorsSkeleton from "./connectors-skeleton";
import IBMCOSSettingsDialog from "./ibm-cos-settings-dialog";
+import S3SettingsDialog from "./s3-settings-dialog";
export default function ConnectorCards() {
const { isAuthenticated, isNoAuthMode } = useAuth();
const router = useRouter();
const [ibmCOSDialogOpen, setIBMCOSDialogOpen] = useState(false);
+ const [s3DialogOpen, setS3DialogOpen] = useState(false);
const { data: queryConnectors = [], isLoading: connectorsLoading } =
useGetConnectorsQuery({
@@ -36,6 +39,7 @@ export default function ConnectorCards() {
sharepoint: <SharePointIcon />,
onedrive: <OneDriveIcon />,
"ibm-cos": <IBMCOSIcon />,
+ "aws-s3": <AwsLogo />,
};
return (
iconMap[iconName] || (
@@ -72,6 +76,9 @@ export default function ConnectorCards() {
if (connector.type === "ibm_cos") {
return () => setIBMCOSDialogOpen(true);
}
+ if (connector.type === "aws_s3") {
+ return () => setS3DialogOpen(true);
+ }
return undefined;
};
@@ -114,6 +121,10 @@ export default function ConnectorCards() {
open={ibmCOSDialogOpen}
setOpen={setIBMCOSDialogOpen}
/>
+ <S3SettingsDialog
+ open={s3DialogOpen}
+ setOpen={setS3DialogOpen}
+ />
</>
);
}
diff --git a/frontend/app/settings/_components/s3-settings-dialog.tsx b/frontend/app/settings/_components/s3-settings-dialog.tsx
new file mode 100644
index 000000000..d2ee705ee
--- /dev/null
+++ b/frontend/app/settings/_components/s3-settings-dialog.tsx
@@ -0,0 +1,179 @@
+"use client";
+
+import { useState } from "react";
+import { FormProvider, useForm } from "react-hook-form";
+import { toast } from "sonner";
+import { useQueryClient } from "@tanstack/react-query";
+import AwsLogo from "@/components/icons/aws-logo";
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { S3SettingsForm, type S3FormData } from "./s3-settings-form";
+import { useS3DefaultsQuery } from "@/app/api/queries/useS3DefaultsQuery";
+import { useS3ConfigureMutation } from "@/app/api/mutations/useS3ConfigureMutation";
+
+interface S3SettingsDialogProps {
+ open: boolean;
+ setOpen: (open: boolean) => void;
+}
+
+export default function S3SettingsDialog({
+ open,
+ setOpen,
+}: S3SettingsDialogProps) {
+ const queryClient = useQueryClient();
+
+ const { data: defaults } = useS3DefaultsQuery({ enabled: open });
+
+ const methods = useForm<S3FormData>({
+ mode: "onSubmit",
+ values: {
+ access_key: "",
+ secret_key: "",
+ endpoint_url: defaults?.endpoint ?? "",
+ region: defaults?.region ?? "",
+ },
+ });
+
+ const { handleSubmit } = methods;
+
+ const [buckets, setBuckets] = useState<string[] | null>(
+ defaults?.bucket_names?.length ? defaults.bucket_names : null,
+ );
+ const [selectedBuckets, setSelectedBuckets] = useState<string[]>(
+ defaults?.bucket_names ?? [],
+ );
+ const [isFetchingBuckets, setIsFetchingBuckets] = useState(false);
+ const [bucketsError, setBucketsError] = useState<string | null>(null);
+ const [formError, setFormError] = useState<string | null>(null);
+
+ const configureMutation = useS3ConfigureMutation();
+
+ const handleTestConnection = handleSubmit(async (data) => {
+ setIsFetchingBuckets(true);
+ setBucketsError(null);
+ setFormError(null);
+
+ try {
+ const result = await configureMutation.mutateAsync({
+ access_key: data.access_key || undefined,
+ secret_key: data.secret_key || undefined,
+ endpoint_url: data.endpoint_url || undefined,
+ region: data.region || undefined,
+ connection_id: defaults?.connection_id ?? undefined,
+ });
+
+ const res = await fetch(
+ `/api/connectors/aws_s3/${result.connection_id}/buckets`,
+ );
+ const json = await res.json();
+ if (!res.ok) throw new Error(json.error || "Failed to list buckets");
+
+ const fetchedBuckets: string[] = json.buckets;
+ setBuckets(fetchedBuckets);
+
+ setSelectedBuckets((prev) =>
+ prev.filter((b) => fetchedBuckets.includes(b)),
+ );
+
+ queryClient.invalidateQueries({ queryKey: ["s3-defaults"] });
+ } catch (err: any) {
+ setBucketsError(err.message ?? "Connection failed");
+ } finally {
+ setIsFetchingBuckets(false);
+ }
+ });
+
+ const onSubmit = handleSubmit(async (data) => {
+ setFormError(null);
+ if (buckets === null) {
+ setFormError("Test the connection first to validate credentials.");
+ return;
+ }
+
+ try {
+ const latestDefaults = await queryClient.fetchQuery({
+ queryKey: ["s3-defaults"],
+ queryFn: async () => {
+ const res = await fetch("/api/connectors/aws_s3/defaults");
+ return res.json();
+ },
+ staleTime: 0,
+ });
+
+ await configureMutation.mutateAsync({
+ access_key: data.access_key || undefined,
+ secret_key: data.secret_key || undefined,
+ endpoint_url: data.endpoint_url || undefined,
+ region: data.region || undefined,
+ bucket_names: selectedBuckets,
+ connection_id: latestDefaults?.connection_id ?? defaults?.connection_id ?? undefined,
+ });
+
+ toast.success("Amazon S3 configured", {
+ description:
+ selectedBuckets.length > 0
+ ? `Will ingest from: ${selectedBuckets.join(", ")}`
+ : "Will auto-discover and ingest all accessible buckets.",
+ icon: <AwsLogo className="w-4 h-4" />,
+ });
+
+ queryClient.invalidateQueries({ queryKey: ["connectors"] });
+ setOpen(false);
+ } catch (err: any) {
+ setFormError(err.message ?? "Failed to save configuration");
+ }
+ });
+
+ return (
+ <Dialog open={open} onOpenChange={setOpen}>
+ <DialogContent autoFocus={false} className="max-w-2xl max-h-[90vh] overflow-y-auto">
+ <FormProvider {...methods}>
+ <form onSubmit={onSubmit} className="grid gap-4">
+ <DialogHeader className="mb-2">
+ <DialogTitle className="flex items-center gap-3">
+ <div className="w-8 h-8 rounded flex items-center justify-center bg-white border">
+ <AwsLogo className="text-black" />
+ </div>
+ Amazon S3 Setup
+ </DialogTitle>
+ </DialogHeader>
+
+ <S3SettingsForm
+ buckets={buckets}
+ selectedBuckets={selectedBuckets}
+ onSelectedBucketsChange={setSelectedBuckets}
+ isFetchingBuckets={isFetchingBuckets}
+ bucketsError={bucketsError}
+ onTestConnection={handleTestConnection as () => void}
+ accessKeySet={defaults?.access_key_set}
+ secretKeySet={defaults?.secret_key_set}
+ formError={formError}
+ />
+
+ <DialogFooter className="mt-4">
+ <Button
+ variant="outline"
+ type="button"
+ onClick={() => setOpen(false)}
+ >
+ Cancel
+ </Button>
+ <Button
+ type="submit"
+ disabled={configureMutation.isPending || isFetchingBuckets}
+ >
+ {configureMutation.isPending ? "Saving…" : "Save"}
+ </Button>
+ </DialogFooter>
+ </form>
+ </FormProvider>
+ </DialogContent>
+ </Dialog>
+ );
+}
diff --git a/frontend/app/settings/_components/s3-settings-form.tsx b/frontend/app/settings/_components/s3-settings-form.tsx
new file mode 100644
index 000000000..22d36ac50
--- /dev/null
+++ b/frontend/app/settings/_components/s3-settings-form.tsx
@@ -0,0 +1,223 @@
+"use client";
+
+import { useFormContext } from "react-hook-form";
+import { LabelWrapper } from "@/components/label-wrapper";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Loader2, RefreshCcw } from "lucide-react";
+import { Button } from "@/components/ui/button";
+
+export interface S3FormData {
+ access_key: string;
+ secret_key: string;
+ endpoint_url: string;
+ region: string;
+}
+
+interface S3SettingsFormProps {
+ /** Available buckets after a successful test — null means not yet tested */
+ buckets: string[] | null;
+ selectedBuckets: string[];
+ onSelectedBucketsChange: (buckets: string[]) => void;
+ isFetchingBuckets: boolean;
+ bucketsError: string | null;
+ onTestConnection: () => void;
+ accessKeySet?: boolean;
+ secretKeySet?: boolean;
+ formError?: string | null;
+}
+
+export function S3SettingsForm({
+ buckets,
+ selectedBuckets,
+ onSelectedBucketsChange,
+ isFetchingBuckets,
+ bucketsError,
+ onTestConnection,
+ accessKeySet,
+ secretKeySet,
+ formError,
+}: S3SettingsFormProps) {
+ const { register } = useFormContext<S3FormData>();
+
+ const toggleBucket = (name: string, checked: boolean) => {
+ if (checked) {
+ onSelectedBucketsChange([...selectedBuckets, name]);
+ } else {
+ onSelectedBucketsChange(selectedBuckets.filter((b) => b !== name));
+ }
+ };
+
+ const toggleAll = (checked: boolean) => {
+ onSelectedBucketsChange(checked ? (buckets ?? []) : []);
+ };
+
+ return (
+ <div className="space-y-5">
+ {/* Access Key ID */}
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Access Key ID"
+ helperText="From AWS IAM → Users → Security credentials. Or set the AWS_ACCESS_KEY_ID env var."
+ id="s3-access-key"
+ required
+ >
+ <Input
+ {...register("access_key", { setValueAs: (v) => v?.trim() })}
+ id="s3-access-key"
+ type="password"
+ placeholder={
+ accessKeySet
+ ? "•••••••• (loaded from env)"
+ : "AKIAIOSFODNN7EXAMPLE"
+ }
+ autoComplete="off"
+ />
+ </LabelWrapper>
+ </div>
+
+ {/* Secret Access Key */}
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Secret Access Key"
+ helperText="Or set the AWS_SECRET_ACCESS_KEY env var."
+ id="s3-secret-key"
+ required
+ >
+ <Input
+ {...register("secret_key", { setValueAs: (v) => v?.trim() })}
+ id="s3-secret-key"
+ type="password"
+ placeholder={
+ secretKeySet
+ ? "•••••••• (loaded from env)"
+ : "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+ }
+ autoComplete="off"
+ />
+ </LabelWrapper>
+ </div>
+
+ {/* Endpoint URL (optional) */}
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Endpoint URL"
+ helperText="Leave blank for AWS S3. For MinIO, Cloudflare R2, or other S3-compatible services, enter the endpoint URL. Or set the AWS_S3_ENDPOINT env var."
+ id="s3-endpoint"
+ >
+ <Input
+ {...register("endpoint_url", { setValueAs: (v) => v?.trim() })}
+ id="s3-endpoint"
+ placeholder="https://your-minio.example.com"
+ autoComplete="off"
+ />
+ </LabelWrapper>
+ </div>
+
+ {/* Region (optional) */}
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Region"
+ helperText="AWS region (e.g. us-east-1, eu-west-1). Default: us-east-1. Or set the AWS_REGION env var."
+ id="s3-region"
+ >
+ <Input
+ {...register("region", { setValueAs: (v) => v?.trim() })}
+ id="s3-region"
+ placeholder="us-east-1"
+ autoComplete="off"
+ />
+ </LabelWrapper>
+ </div>
+
+ {/* Test connection */}
+ <Button
+ type="button"
+ variant="outline"
+ onClick={onTestConnection}
+ disabled={isFetchingBuckets}
+ className="w-full"
+ >
+ {isFetchingBuckets ? (
+ <>
+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />
+ Connecting…
+ </>
+ ) : buckets !== null ? (
+ <>
+ <RefreshCcw className="h-4 w-4 mr-2" />
+ Refresh Buckets
+ </>
+ ) : (
+ "Test Connection & List Buckets"
+ )}
+ </Button>
+
+ {bucketsError && (
+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">
+ {bucketsError}
+ </p>
+ )}
+
+ {formError && (
+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">
+ {formError}
+ </p>
+ )}
+
+ {/* Bucket selector */}
+ {buckets !== null && (
+ <div className="space-y-2">
+ <div className="flex items-center justify-between">
+ <Label className="text-sm font-medium">
+ Select Buckets to Ingest
+ <span className="ml-1 text-muted-foreground font-normal">
+ (leave all unchecked to ingest everything)
+ </span>
+ </Label>
+ {buckets.length > 1 && (
+ <button
+ type="button"
+ className="text-xs text-primary underline-offset-2 hover:underline"
+ onClick={() =>
+ toggleAll(selectedBuckets.length !== buckets.length)
+ }
+ >
+ {selectedBuckets.length === buckets.length
+ ? "Deselect all"
+ : "Select all"}
+ </button>
+ )}
+ </div>
+
+ {buckets.length === 0 ? (
+ <p className="text-sm text-muted-foreground">
+ No buckets found for this account.
+ </p>
+ ) : (
+ <div className="max-h-48 overflow-y-auto rounded-lg border p-3 space-y-2">
+ {buckets.map((bucket) => (
+ <label
+ key={bucket}
+ htmlFor={`s3-bucket-${bucket}`}
+ className="flex items-center gap-2.5 cursor-pointer select-none"
+ >
+ <input
+ id={`s3-bucket-${bucket}`}
+ type="checkbox"
+ className="h-4 w-4 rounded border-border accent-primary"
+ checked={selectedBuckets.includes(bucket)}
+ onChange={(e) =>
+ toggleBucket(bucket, e.target.checked)
+ }
+ />
+ <span className="text-sm">{bucket}</span>
+ </label>
+ ))}
+ </div>
+ )}
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index 277fc965e..464e7c283 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -9,6 +9,7 @@ import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";
import { useGetConnectorTokenQuery } from "@/app/api/queries/useGetConnectorTokenQuery";
import { useIBMCOSBucketStatusQuery } from "@/app/api/queries/useIBMCOSBucketStatusQuery";
+import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQuery";
import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";
import type { IngestSettings } from "@/components/cloud-picker/types";
import { Button } from "@/components/ui/button";
@@ -20,7 +21,7 @@ import {
import { useTask } from "@/contexts/task-context";
// Connectors that sync entire buckets/repositories without a file picker
-const DIRECT_SYNC_PROVIDERS = ["ibm_cos"];
+const DIRECT_SYNC_PROVIDERS = ["ibm_cos", "aws_s3"];
// ---------------------------------------------------------------------------
// IBM COS bucket list with sync status
@@ -192,6 +193,176 @@ function IBMCOSBucketView({
);
}
+// ---------------------------------------------------------------------------
+// Amazon S3 bucket list with sync status
+// ---------------------------------------------------------------------------
+
+function S3BucketView({
+ connector,
+ syncMutation,
+ addTask,
+ onBack,
+ onDone,
+}: {
+ connector: any;
+ syncMutation: ReturnType<typeof useSyncConnector>;
+ addTask: (id: string) => void;
+ onBack: () => void;
+ onDone: () => void;
+}) {
+ const queryClient = useQueryClient();
+ const { data: buckets, isLoading, refetch } = useS3BucketStatusQuery(
+ connector.connectionId,
+ { enabled: true },
+ );
+
+ const [syncingBucket, setSyncingBucket] = useState<string | null>(null);
+
+ const invalidateBucketStatus = () => {
+ queryClient.invalidateQueries({ queryKey: ["s3-bucket-status", connector.connectionId] });
+ };
+
+ const syncAll = () => {
+ syncMutation.mutate(
+ {
+ connectorType: connector.type,
+ body: {
+ connection_id: connector.connectionId!,
+ selected_files: [],
+ sync_all: true,
+ },
+ },
+ {
+ onSuccess: (result) => {
+ invalidateBucketStatus();
+ if (result.task_ids?.length) {
+ addTask(result.task_ids[0]);
+ onDone();
+ } else {
+ toast.info("No files found in any bucket.");
+ }
+ },
+ onError: (err) => {
+ toast.error(err instanceof Error ? err.message : "Sync failed");
+ },
+ },
+ );
+ };
+
+ const syncBucket = (bucketName: string) => {
+ setSyncingBucket(bucketName);
+ syncMutation.mutate(
+ {
+ connectorType: connector.type,
+ body: {
+ connection_id: connector.connectionId!,
+ selected_files: [],
+ bucket_filter: [bucketName],
+ },
+ },
+ {
+ onSuccess: (result) => {
+ setSyncingBucket(null);
+ invalidateBucketStatus();
+ if (result.task_ids?.length) {
+ addTask(result.task_ids[0]);
+ onDone();
+ } else {
+ toast.info(`No files found in bucket "${bucketName}".`);
+ }
+ },
+ onError: (err) => {
+ setSyncingBucket(null);
+ toast.error(err instanceof Error ? err.message : "Sync failed");
+ },
+ },
+ );
+ };
+
+ return (
+ <>
+ <div className="mb-8 flex gap-2 items-center">
+ <Button variant="ghost" onClick={onBack} size="icon">
+ <ArrowLeft size={18} />
+ </Button>
+ <h2 className="text-xl text-[18px] font-semibold">
+ Add from {connector.name}
+ </h2>
+ </div>
+
+ <div className="max-w-3xl mx-auto space-y-4">
+ <div className="flex items-center justify-between">
+ <p className="text-sm text-muted-foreground">
+ Select a bucket to ingest, or sync everything at once.
+ </p>
+ <div className="flex gap-2">
+ <Button
+ variant="ghost"
+ size="icon"
+ onClick={() => refetch()}
+ disabled={isLoading}
+ title="Refresh bucket status"
+ >
+ <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />
+ </Button>
+ <Button
+ className="bg-foreground text-background hover:bg-foreground/90 font-semibold"
+ onClick={syncAll}
+ disabled={syncMutation.isPending}
+ >
+ {syncMutation.isPending && !syncingBucket
+ ? "Ingesting…"
+ : "Sync All Buckets"}
+ </Button>
+ </div>
+ </div>
+
+ {isLoading ? (
+ <div className="flex justify-center py-8">
+ <div className="animate-spin rounded-full h-6 w-6 border-b-2 border-primary" />
+ </div>
+ ) : !buckets?.length ? (
+ <div className="rounded-lg border p-6 text-center text-muted-foreground text-sm">
+ No buckets found. Check your S3 credentials and endpoint.
+ </div>
+ ) : (
+ <div className="rounded-lg border divide-y">
+ {buckets.map((bucket) => (
+ <div
+ key={bucket.name}
+ className="flex items-center justify-between px-4 py-3"
+ >
+ <div className="flex items-center gap-3">
+ <div>
+ <p className="font-medium text-sm">{bucket.name}</p>
+ {bucket.ingested_count > 0 && (
+ <p className="text-xs text-muted-foreground">
+ {bucket.ingested_count} document{bucket.ingested_count !== 1 ? "s" : ""} ingested
+ </p>
+ )}
+ </div>
+ </div>
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => syncBucket(bucket.name)}
+ disabled={syncMutation.isPending}
+ >
+ {syncingBucket === bucket.name
+ ? "Ingesting…"
+ : bucket.ingested_count > 0
+ ? "Re-sync"
+ : "Ingest"}
+ </Button>
+ </div>
+ ))}
+ </div>
+ )}
+ </div>
+ </>
+ );
+}
+
// CloudFile interface is now imported from the unified cloud picker
export default function UploadProviderPage() {
@@ -376,8 +547,19 @@ export default function UploadProviderPage() {
);
}
- // Direct-sync providers (e.g. IBM COS) show a bucket list with sync status.
+ // Direct-sync providers show a bucket list with sync status.
if (isDirectSyncProvider && connector.status === "connected") {
+ if (provider === "aws_s3") {
+ return (
+ <S3BucketView
+ connector={connector}
+ syncMutation={syncMutation}
+ addTask={addTask}
+ onBack={() => router.back()}
+ onDone={() => router.push("/knowledge")}
+ />
+ );
+ }
return (
<IBMCOSBucketView
connector={connector}
diff --git a/frontend/components/cloud-picker/file-item.tsx b/frontend/components/cloud-picker/file-item.tsx
index fa4d079fc..b71792685 100644
--- a/frontend/components/cloud-picker/file-item.tsx
+++ b/frontend/components/cloud-picker/file-item.tsx
@@ -5,6 +5,7 @@ import GoogleDriveIcon from "@/components/icons/google-drive-logo";
import IBMCOSIcon from "@/components/icons/ibm-cos-icon";
import OneDriveIcon from "@/components/icons/one-drive-logo";
import SharePointIcon from "@/components/icons/share-point-logo";
+import AwsLogo from "@/components/icons/aws-logo";
import { Button } from "@/components/ui/button";
import type { CloudFile } from "./types";
@@ -57,6 +58,8 @@ const getProviderIcon = (provider: string) => {
return <SharePointIcon />;
case "ibm_cos":
return <IBMCOSIcon />;
+ case "aws_s3":
+ return <AwsLogo />;
default:
return <FileText className="h-6 w-6" />;
}
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 921df2bd8..8f4f4130b 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -916,6 +916,205 @@ async def ibm_cos_bucket_status(
return JSONResponse({"buckets": result})
+# ---------------------------------------------------------------------------
+# Amazon S3 / S3-compatible endpoints
+# ---------------------------------------------------------------------------
+
+class S3ConfigureBody(BaseModel):
+ access_key: Optional[str] = None
+ secret_key: Optional[str] = None
+ endpoint_url: Optional[str] = None
+ region: Optional[str] = None
+ bucket_names: Optional[List[str]] = None
+ connection_id: Optional[str] = None
+
+
+async def s3_defaults(
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Return current S3 env-var defaults for pre-filling the config dialog.
+
+ Sensitive values (secret key) are masked — only whether they are set is returned.
+ """
+ import os
+
+ access_key = os.getenv("AWS_ACCESS_KEY_ID", "")
+ secret_key = os.getenv("AWS_SECRET_ACCESS_KEY", "")
+ endpoint_url = os.getenv("AWS_S3_ENDPOINT", "")
+ region = os.getenv("AWS_REGION", "")
+
+ connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="aws_s3"
+ )
+ conn_config = {}
+ if connections:
+ conn_config = connections[0].config or {}
+
+ def _pick(conn_key, env_val):
+ return conn_config.get(conn_key) or env_val
+
+ return JSONResponse({
+ "access_key_set": bool(access_key or conn_config.get("access_key")),
+ "secret_key_set": bool(secret_key or conn_config.get("secret_key")),
+ "endpoint": _pick("endpoint_url", endpoint_url),
+ "region": _pick("region", region),
+ "bucket_names": conn_config.get("bucket_names", []),
+ "connection_id": connections[0].connection_id if connections else None,
+ })
+
+
+async def s3_configure(
+ body: S3ConfigureBody,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Create or update an S3 connection with explicit credentials.
+
+ Tests the credentials by listing buckets, then persists the connection.
+ """
+ import os
+ from connectors.aws_s3.auth import create_s3_resource
+
+ access_key = body.access_key or os.getenv("AWS_ACCESS_KEY_ID")
+ secret_key = body.secret_key or os.getenv("AWS_SECRET_ACCESS_KEY")
+
+ # Fall back to existing connection config
+ existing_connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="aws_s3"
+ )
+ if not access_key and existing_connections:
+ access_key = existing_connections[0].config.get("access_key")
+ if not secret_key and existing_connections:
+ secret_key = existing_connections[0].config.get("secret_key")
+
+ if not access_key or not secret_key:
+ return JSONResponse(
+ {"error": "access_key and secret_key are required"},
+ status_code=400,
+ )
+
+ conn_config: dict = {
+ "access_key": access_key.strip(),
+ "secret_key": secret_key.strip(),
+ }
+ if body.endpoint_url:
+ conn_config["endpoint_url"] = body.endpoint_url.strip()
+ if body.region:
+ conn_config["region"] = body.region.strip()
+ if body.bucket_names is not None:
+ conn_config["bucket_names"] = body.bucket_names
+
+ # Test credentials
+ try:
+ s3 = create_s3_resource(conn_config)
+ list(s3.buckets.all())
+ except Exception as exc:
+ return JSONResponse(
+ {"error": f"Could not connect to S3: {exc}"},
+ status_code=400,
+ )
+
+ # Persist: update existing connection or create a new one
+ if body.connection_id:
+ existing = await connector_service.connection_manager.get_connection(body.connection_id)
+ if existing and existing.user_id == user.user_id:
+ await connector_service.connection_manager.update_connection(
+ connection_id=body.connection_id,
+ config=conn_config,
+ )
+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)
+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})
+
+ connection_id = await connector_service.connection_manager.create_connection(
+ connector_type="aws_s3",
+ name="Amazon S3",
+ config=conn_config,
+ user_id=user.user_id,
+ )
+ return JSONResponse({"connection_id": connection_id, "status": "connected"})
+
+
+async def s3_list_buckets(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """List all buckets accessible with the stored S3 credentials."""
+ from connectors.aws_s3.auth import create_s3_resource
+
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "aws_s3":
+ return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
+
+ try:
+ s3 = create_s3_resource(connection.config)
+ buckets = [b.name for b in s3.buckets.all()]
+ return JSONResponse({"buckets": buckets})
+ except Exception as exc:
+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)
+
+
+async def s3_bucket_status(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(get_current_user),
+):
+ """Return all buckets for an S3 connection with their ingestion status."""
+ from connectors.aws_s3.auth import create_s3_resource
+
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "aws_s3":
+ return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
+
+ # 1. List all buckets from S3
+ try:
+ s3 = create_s3_resource(connection.config)
+ all_buckets = [b.name for b in s3.buckets.all()]
+ except Exception as exc:
+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)
+
+ # 2. Count indexed documents per bucket from OpenSearch
+ ingested_counts: dict = {}
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(
+ user.user_id, user.jwt_token
+ )
+ query_body = {
+ "size": 0,
+ "query": {"term": {"connector_type": "aws_s3"}},
+ "aggs": {
+ "doc_ids": {
+ "terms": {"field": "document_id", "size": 50000}
+ }
+ },
+ }
+ index_name = get_index_name(user.user_id)
+ os_resp = opensearch_client.search(index=index_name, body=query_body)
+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):
+ doc_id = bucket_entry["key"]
+ if "::" in doc_id:
+ bucket_name = doc_id.split("::")[0]
+ ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1
+ except Exception:
+ pass # OpenSearch unavailable — show zero counts
+
+ result = [
+ {
+ "name": bucket,
+ "ingested_count": ingested_counts.get(bucket, 0),
+ "is_synced": ingested_counts.get(bucket, 0) > 0,
+ }
+ for bucket in all_buckets
+ ]
+ return JSONResponse({"buckets": result})
+
+
# ---------------------------------------------------------------------------
async def sync_all_connectors(
@@ -931,7 +1130,7 @@ async def sync_all_connectors(
jwt_token = user.jwt_token
# Cloud connector types to sync
- cloud_connector_types = ["google_drive", "onedrive", "sharepoint", "ibm_cos"]
+ cloud_connector_types = ["google_drive", "onedrive", "sharepoint", "ibm_cos", "aws_s3"]
all_task_ids = []
synced_connectors = []
diff --git a/src/connectors/__init__.py b/src/connectors/__init__.py
index a49dff219..323a76285 100644
--- a/src/connectors/__init__.py
+++ b/src/connectors/__init__.py
@@ -3,6 +3,7 @@ from .google_drive import GoogleDriveConnector
from .sharepoint import SharePointConnector
from .onedrive import OneDriveConnector
from .ibm_cos import IBMCOSConnector
+from .aws_s3 import S3Connector
__all__ = [
"BaseConnector",
@@ -10,4 +11,5 @@ __all__ = [
"SharePointConnector",
"OneDriveConnector",
"IBMCOSConnector",
+ "S3Connector",
]
diff --git a/src/connectors/aws_s3/__init__.py b/src/connectors/aws_s3/__init__.py
new file mode 100644
index 000000000..ee9a5ccab
--- /dev/null
+++ b/src/connectors/aws_s3/__init__.py
@@ -0,0 +1,5 @@
+"""Amazon S3 / S3-compatible connector for OpenRAG."""
+
+from .connector import S3Connector
+
+__all__ = ["S3Connector"]
diff --git a/src/connectors/aws_s3/auth.py b/src/connectors/aws_s3/auth.py
new file mode 100644
index 000000000..5542debf7
--- /dev/null
+++ b/src/connectors/aws_s3/auth.py
@@ -0,0 +1,96 @@
+"""Amazon S3 / S3-compatible storage authentication and client factory."""
+
+import os
+from typing import Any, Dict, Optional
+
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+_DEFAULT_REGION = "us-east-1"
+
+
+def _resolve_credentials(config: Dict[str, Any]) -> Dict[str, Any]:
+ """Resolve S3 credentials from config dict with environment variable fallback.
+
+ Resolution order for each value: config dict → environment variable → default.
+
+ Raises:
+ ValueError: If access_key or secret_key cannot be resolved.
+ """
+ access_key: Optional[str] = config.get("access_key") or os.getenv("AWS_ACCESS_KEY_ID")
+ secret_key: Optional[str] = config.get("secret_key") or os.getenv("AWS_SECRET_ACCESS_KEY")
+
+ if not access_key or not secret_key:
+ raise ValueError(
+ "S3 credentials are required. Provide 'access_key' and 'secret_key' in the "
+ "connector config, or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars."
+ )
+
+ # endpoint_url is optional — only inject when non-empty (real AWS users don't set it)
+ endpoint_url: Optional[str] = config.get("endpoint_url") or os.getenv("AWS_S3_ENDPOINT") or None
+
+ region: str = config.get("region") or os.getenv("AWS_REGION") or _DEFAULT_REGION
+
+ return {
+ "access_key": access_key,
+ "secret_key": secret_key,
+ "endpoint_url": endpoint_url,
+ "region": region,
+ }
+
+
+def _build_boto3_kwargs(creds: Dict[str, Any]) -> Dict[str, Any]:
+ """Build the keyword arguments for boto3.resource / boto3.client."""
+ kwargs: Dict[str, Any] = {
+ "aws_access_key_id": creds["access_key"],
+ "aws_secret_access_key": creds["secret_key"],
+ "region_name": creds["region"],
+ }
+ if creds["endpoint_url"]:
+ kwargs["endpoint_url"] = creds["endpoint_url"]
+ return kwargs
+
+
+def create_s3_resource(config: Dict[str, Any]):
+ """Return a boto3 S3 resource (high-level API) for bucket/object access.
+
+ Works with AWS S3, MinIO, Cloudflare R2, and any S3-compatible service.
+ """
+ try:
+ import boto3
+ except ImportError as exc:
+ raise ImportError(
+ "boto3 is required for the S3 connector. "
+ "Install it with: pip install boto3"
+ ) from exc
+
+ creds = _resolve_credentials(config)
+ kwargs = _build_boto3_kwargs(creds)
+ logger.debug(
+ "Creating S3 resource with HMAC authentication (boto3), "
+ f"endpoint={creds['endpoint_url']!r}, region={creds['region']!r}"
+ )
+ return boto3.resource("s3", **kwargs)
+
+
+def create_s3_client(config: Dict[str, Any]):
+ """Return a boto3 S3 low-level client.
+
+ Used for operations such as list_buckets() and get_object_acl().
+ """
+ try:
+ import boto3
+ except ImportError as exc:
+ raise ImportError(
+ "boto3 is required for the S3 connector. "
+ "Install it with: pip install boto3"
+ ) from exc
+
+ creds = _resolve_credentials(config)
+ kwargs = _build_boto3_kwargs(creds)
+ logger.debug(
+ "Creating S3 client with HMAC authentication (boto3), "
+ f"endpoint={creds['endpoint_url']!r}, region={creds['region']!r}"
+ )
+ return boto3.client("s3", **kwargs)
diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.py
new file mode 100644
index 000000000..917e002bd
--- /dev/null
+++ b/src/connectors/aws_s3/connector.py
@@ -0,0 +1,260 @@
+"""Amazon S3 / S3-compatible storage connector for OpenRAG."""
+
+import mimetypes
+import os
+from datetime import datetime, timezone
+from posixpath import basename
+from typing import Any, Dict, List, Optional
+
+from connectors.base import BaseConnector, ConnectorDocument, DocumentACL
+from utils.logging_config import get_logger
+
+from .auth import create_s3_client, create_s3_resource
+
+logger = get_logger(__name__)
+
+# Separator used in composite file IDs: "<bucket>::<key>"
+_ID_SEPARATOR = "::"
+
+
+def _make_file_id(bucket: str, key: str) -> str:
+ return f"{bucket}{_ID_SEPARATOR}{key}"
+
+
+def _split_file_id(file_id: str):
+ """Split a composite file ID into (bucket, key). Raises ValueError if invalid."""
+ if _ID_SEPARATOR not in file_id:
+ raise ValueError(f"Invalid S3 file ID (missing separator): {file_id!r}")
+ bucket, key = file_id.split(_ID_SEPARATOR, 1)
+ return bucket, key
+
+
+class S3Connector(BaseConnector):
+ """Connector for Amazon S3 and S3-compatible object storage.
+
+ Uses HMAC (Access Key + Secret Key) authentication. Supports AWS S3,
+ MinIO, Cloudflare R2, and any service that speaks the S3 API.
+
+ Config dict keys:
+ access_key (str): Overrides AWS_ACCESS_KEY_ID.
+ secret_key (str): Overrides AWS_SECRET_ACCESS_KEY.
+ endpoint_url (str): Optional; overrides AWS_S3_ENDPOINT. Leave empty for AWS S3.
+ region (str): Optional; overrides AWS_REGION. Default: us-east-1.
+ bucket_names (list[str]): Buckets to ingest from. If empty, all accessible buckets are used.
+ connection_id (str): Connection identifier used for logging.
+ """
+
+ CONNECTOR_NAME = "Amazon S3"
+ CONNECTOR_DESCRIPTION = "Add knowledge from Amazon S3 or any S3-compatible storage"
+ CONNECTOR_ICON = "aws-s3"
+
+ CLIENT_ID_ENV_VAR = "AWS_ACCESS_KEY_ID"
+ CLIENT_SECRET_ENV_VAR = "AWS_SECRET_ACCESS_KEY"
+
+ def __init__(self, config: Dict[str, Any]):
+ if config is None:
+ config = {}
+ super().__init__(config)
+
+ self.bucket_names: List[str] = config.get("bucket_names") or []
+ self.prefix: str = config.get("prefix", "")
+ self.connection_id: str = config.get("connection_id", "default")
+
+ self._resource = None # Lazy-initialised on first use
+ self._client = None
+
+ def _get_resource(self):
+ if self._resource is None:
+ self._resource = create_s3_resource(self.config)
+ return self._resource
+
+ def _get_client(self):
+ if self._client is None:
+ self._client = create_s3_client(self.config)
+ return self._client
+
+ # ------------------------------------------------------------------
+ # BaseConnector abstract method implementations
+ # ------------------------------------------------------------------
+
+ async def authenticate(self) -> bool:
+ """Validate credentials by listing accessible buckets."""
+ try:
+ resource = self._get_resource()
+ list(resource.buckets.all())
+ self._authenticated = True
+ logger.debug(f"S3 authenticated for connection {self.connection_id}")
+ return True
+ except Exception as exc:
+ logger.warning(f"S3 authentication failed: {exc}")
+ self._authenticated = False
+ return False
+
+ def _resolve_bucket_names(self) -> List[str]:
+ """Return configured bucket names, or auto-discover all accessible buckets."""
+ if self.bucket_names:
+ return self.bucket_names
+ try:
+ resource = self._get_resource()
+ buckets = [b.name for b in resource.buckets.all()]
+ logger.debug(f"S3 auto-discovered {len(buckets)} bucket(s): {buckets}")
+ return buckets
+ except Exception as exc:
+ logger.warning(f"S3 could not auto-discover buckets: {exc}")
+ return []
+
+ async def list_files(
+ self,
+ page_token: Optional[str] = None,
+ max_files: Optional[int] = None,
+ **kwargs,
+ ) -> Dict[str, Any]:
+ """List objects across all configured (or auto-discovered) buckets.
+
+ Uses the boto3 resource API: Bucket.objects.all() handles pagination
+ internally so all objects are returned without manual continuation tokens.
+
+ Returns:
+ dict with keys:
+ "files": list of file dicts (id, name, bucket, size, modified_time)
+ "next_page_token": always None (SDK handles pagination internally)
+ """
+ resource = self._get_resource()
+ files: List[Dict[str, Any]] = []
+ bucket_names = self._resolve_bucket_names()
+
+ for bucket_name in bucket_names:
+ try:
+ bucket = resource.Bucket(bucket_name)
+ objects = (
+ bucket.objects.filter(Prefix=self.prefix)
+ if self.prefix
+ else bucket.objects.all()
+ )
+ for obj in objects:
+ if obj.key.endswith("/"):
+ continue
+ files.append(
+ {
+ "id": _make_file_id(bucket_name, obj.key),
+ "name": basename(obj.key) or obj.key,
+ "bucket": bucket_name,
+ "key": obj.key,
+ "size": obj.size,
+ "modified_time": obj.last_modified.isoformat()
+ if obj.last_modified
+ else None,
+ }
+ )
+ if max_files and len(files) >= max_files:
+ return {"files": files, "next_page_token": None}
+ except Exception as exc:
+ logger.error(f"Failed to list objects in S3 bucket {bucket_name!r}: {exc}")
+ continue
+
+ return {"files": files, "next_page_token": None}
+
+ async def get_file_content(self, file_id: str) -> ConnectorDocument:
+ """Download an object from S3 and return a ConnectorDocument.
+
+ Args:
+ file_id: Composite ID in the form "<bucket>::<key>".
+
+ Returns:
+ ConnectorDocument with content bytes, ACL, and metadata.
+ """
+ bucket_name, key = _split_file_id(file_id)
+ resource = self._get_resource()
+
+ response = resource.Object(bucket_name, key).get()
+ content: bytes = response["Body"].read()
+
+ last_modified: datetime = response.get("LastModified") or datetime.now(timezone.utc)
+ size: int = response.get("ContentLength", len(content))
+
+ # Prefer filename extension over generic S3 content-type (often application/octet-stream)
+ raw_content_type = response.get("ContentType", "")
+ if raw_content_type and raw_content_type != "application/octet-stream":
+ mime_type: str = raw_content_type
+ else:
+ mime_type = mimetypes.guess_type(key)[0] or "application/octet-stream"
+
+ filename = basename(key) or key
+ acl = await self._extract_acl(bucket_name, key)
+
+ return ConnectorDocument(
+ id=file_id,
+ filename=filename,
+ mimetype=mime_type,
+ content=content,
+ source_url=f"s3://{bucket_name}/{key}",
+ acl=acl,
+ modified_time=last_modified,
+ created_time=last_modified, # S3 does not expose creation time
+ metadata={
+ "s3_bucket": bucket_name,
+ "s3_key": key,
+ "size": size,
+ },
+ )
+
+ async def _extract_acl(self, bucket: str, key: str) -> DocumentACL:
+ """Fetch object ACL from S3 and map it to DocumentACL.
+
+ Falls back to a minimal ACL on failure (e.g. ACLs disabled on the bucket).
+ """
+ try:
+ client = self._get_client()
+ acl_response = client.get_object_acl(Bucket=bucket, Key=key)
+
+ owner_id: str = (
+ acl_response.get("Owner", {}).get("DisplayName")
+ or acl_response.get("Owner", {}).get("ID")
+ or ""
+ )
+
+ allowed_users: List[str] = []
+ for grant in acl_response.get("Grants", []):
+ grantee = grant.get("Grantee", {})
+ permission = grant.get("Permission", "")
+ if permission in ("FULL_CONTROL", "READ"):
+ user_id = (
+ grantee.get("DisplayName")
+ or grantee.get("ID")
+ or grantee.get("EmailAddress")
+ )
+ if user_id and user_id not in allowed_users:
+ allowed_users.append(user_id)
+
+ return DocumentACL(
+ owner=owner_id or None,
+ allowed_users=allowed_users,
+ allowed_groups=[],
+ )
+ except Exception as exc:
+ logger.warning(
+ f"Could not fetch ACL for s3://{bucket}/{key}: {exc}. "
+ "Using fallback ACL."
+ )
+ return DocumentACL(owner=None, allowed_users=[], allowed_groups=[])
+
+ # ------------------------------------------------------------------
+ # Webhook / subscription stubs (S3 event notifications are out of scope)
+ # ------------------------------------------------------------------
+
+ async def setup_subscription(self) -> str:
+ """No-op: S3 event notifications are out of scope for this connector."""
+ return ""
+
+ async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:
+ """No-op: webhooks are not supported in this connector version."""
+ return []
+
+ def extract_webhook_channel_id(
+ self, payload: Dict[str, Any], headers: Dict[str, str]
+ ) -> Optional[str]:
+ return None
+
+ async def cleanup_subscription(self, subscription_id: str) -> bool:
+ """No-op: no subscription to clean up."""
+ return True
diff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.py
index 35016e389..c172a93ca 100644
--- a/src/connectors/connection_manager.py
+++ b/src/connectors/connection_manager.py
@@ -14,6 +14,7 @@ from .google_drive import GoogleDriveConnector
from .sharepoint import SharePointConnector
from .onedrive import OneDriveConnector
from .ibm_cos import IBMCOSConnector
+from .aws_s3 import S3Connector
@dataclass
@@ -358,6 +359,12 @@ class ConnectionManager:
"icon": IBMCOSConnector.CONNECTOR_ICON,
"available": self._is_connector_available("ibm_cos"),
},
+ "aws_s3": {
+ "name": S3Connector.CONNECTOR_NAME,
+ "description": S3Connector.CONNECTOR_DESCRIPTION,
+ "icon": S3Connector.CONNECTOR_ICON,
+ "available": self._is_connector_available("aws_s3"),
+ },
}
def _is_connector_available(self, connector_type: str) -> bool:
@@ -388,6 +395,8 @@ class ConnectionManager:
return OneDriveConnector(config.config)
elif config.connector_type == "ibm_cos":
return IBMCOSConnector(config.config)
+ elif config.connector_type == "aws_s3":
+ return S3Connector(config.config)
elif config.connector_type == "box":
raise NotImplementedError("Box connector not implemented yet")
elif config.connector_type == "dropbox":
diff --git a/src/main.py b/src/main.py
index 016a86ff7..8f61723b5 100644
--- a/src/main.py
+++ b/src/main.py
@@ -767,6 +767,10 @@ async def create_app():
app.add_api_route("/connectors/ibm_cos/configure", connectors.ibm_cos_configure, methods=["POST"], tags=["internal"])
app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", connectors.ibm_cos_list_buckets, methods=["GET"], tags=["internal"])
app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", connectors.ibm_cos_bucket_status, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/defaults", connectors.s3_defaults, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/configure", connectors.s3_configure, methods=["POST"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/{connection_id}/buckets", connectors.s3_list_buckets, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/{connection_id}/bucket-status", connectors.s3_bucket_status, methods=["GET"], tags=["internal"])
app.add_api_route("/connectors/{connector_type}/sync", connectors.connector_sync, methods=["POST"], tags=["internal"])
app.add_api_route("/connectors/sync-all", connectors.sync_all_connectors, methods=["POST"], tags=["internal"])
app.add_api_route("/connectors/{connector_type}/status", connectors.connector_status, methods=["GET"], tags=["internal"])
diff --git a/src/tui/config_fields.py b/src/tui/config_fields.py
index 6a78a883e..184365a32 100644
--- a/src/tui/config_fields.py
+++ b/src/tui/config_fields.py
@@ -181,6 +181,17 @@ CONFIG_SECTIONS: list[ConfigSection] = [
"aws_secret_access_key", "AWS_SECRET_ACCESS_KEY", "Secret Access Key",
placeholder="", secret=True,
),
+ ConfigField(
+ "aws_s3_endpoint", "AWS_S3_ENDPOINT", "S3 Endpoint URL (optional)",
+ placeholder="",
+ helper_text="Leave empty for AWS S3. For MinIO, R2, or other S3-compatible services, enter the endpoint URL.",
+ ),
+ ConfigField(
+ "aws_region", "AWS_REGION", "AWS Region (optional)",
+ placeholder="us-east-1",
+ default="us-east-1",
+ helper_text="AWS region (e.g. us-east-1, eu-west-1). Default: us-east-1.",
+ ),
], advanced=True, gate_prompt="Configure AWS credentials?"),
# ── IBM Cloud Object Storage ─────────────────────────────────
diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.py
index b16c38af2..7fba12917 100644
--- a/src/tui/managers/env_manager.py
+++ b/src/tui/managers/env_manager.py
@@ -57,6 +57,8 @@ class EnvConfig:
webhook_base_url: str = ""
aws_access_key_id: str = ""
aws_secret_access_key: str = ""
+ aws_s3_endpoint: str = ""
+ aws_region: str = ""
langflow_public_url: str = ""
# IBM Cloud Object Storage settings
@@ -200,6 +202,8 @@ class EnvManager:
"WEBHOOK_BASE_URL": "webhook_base_url",
"AWS_ACCESS_KEY_ID": "aws_access_key_id",
"AWS_SECRET_ACCESS_KEY": "aws_secret_access_key", # pragma: allowlist secret
+ "AWS_S3_ENDPOINT": "aws_s3_endpoint",
+ "AWS_REGION": "aws_region",
"LANGFLOW_PUBLIC_URL": "langflow_public_url",
"IBM_COS_API_KEY": "ibm_cos_api_key", # pragma: allowlist secret
"IBM_COS_SERVICE_INSTANCE_ID": "ibm_cos_service_instance_id",