OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Integrate IBM COS connector with HMAC as default connection method
Edwin Jose0e25241b6855
diff --git a/frontend/app/api/mutations/useConnectConnectorMutation.ts b/frontend/app/api/mutations/useConnectConnectorMutation.tsindex 893a90086..6e0faf4b8 100644--- a/frontend/app/api/mutations/useConnectConnectorMutation.ts+++ b/frontend/app/api/mutations/useConnectConnectorMutation.ts@@ -81,6 +81,11 @@ export const useConnectConnectorMutation = () => {`state=${result.connection_id}`;window.location.href = authUrl;+ } else {+ // Direct-auth connector (e.g. IBM COS) — credentials already verified,+ // no OAuth redirect needed. Refresh connector status.+ queryClient.invalidateQueries({ queryKey: ["connectors"] });+ toast.success(`${connector.name} connected successfully`);}},});diff --git a/frontend/app/api/mutations/useIBMCOSConfigureMutation.ts b/frontend/app/api/mutations/useIBMCOSConfigureMutation.tsnew file mode 100644index 000000000..89e0e23f4--- /dev/null+++ b/frontend/app/api/mutations/useIBMCOSConfigureMutation.ts@@ -0,0 +1,40 @@+import { useMutation, useQueryClient } from "@tanstack/react-query";++export interface IBMCOSConfigurePayload {+ auth_mode: "iam" | "hmac";+ endpoint: string;+ // IAM+ api_key?: string;+ service_instance_id?: string;+ auth_endpoint?: string;+ // HMAC+ hmac_access_key?: string;+ hmac_secret_key?: string;+ // Bucket selection+ bucket_names?: string[];+ // Updating an existing connection+ connection_id?: string;+}++async function configureIBMCOS(payload: IBMCOSConfigurePayload) {+ const res = await fetch("/api/connectors/ibm_cos/configure", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(payload),+ });+ const data = await res.json();+ if (!res.ok) throw new Error(data.error || "Failed to configure IBM COS");+ return data as { connection_id: string; status: string };+}++export function useIBMCOSConfigureMutation() {+ const queryClient = useQueryClient();++ return useMutation({+ mutationFn: configureIBMCOS,+ onSuccess: () => {+ queryClient.invalidateQueries({ queryKey: ["connectors"] });+ queryClient.invalidateQueries({ queryKey: ["ibm-cos-defaults"] });+ },+ });+}diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.tsindex 9ff22e47e..fe2c7ef2e 100644--- a/frontend/app/api/mutations/useSyncConnector.ts+++ b/frontend/app/api/mutations/useSyncConnector.ts@@ -47,6 +47,10 @@ const syncConnector = async ({size?: number;}>;settings?: any;+ /** When true, ingest all files from the connector (bypasses the re-sync gate). */+ sync_all?: boolean;+ /** Restrict ingest to these bucket names (IBM COS). */+ bucket_filter?: string[];};}): Promise<SyncResponse> => {const response = await fetch(`/api/connectors/${connectorType}/sync`, {diff --git a/frontend/app/api/queries/useIBMCOSBucketStatusQuery.ts b/frontend/app/api/queries/useIBMCOSBucketStatusQuery.tsnew file mode 100644index 000000000..2a4b81b89--- /dev/null+++ b/frontend/app/api/queries/useIBMCOSBucketStatusQuery.ts@@ -0,0 +1,34 @@+import { useQuery } from "@tanstack/react-query";++export interface IBMCOSBucketStatus {+ name: string;+ ingested_count: number;+ is_synced: boolean;+}++async function fetchIBMCOSBucketStatus(+ connectionId: string,+): Promise<IBMCOSBucketStatus[]> {+ const res = await fetch(+ `/api/connectors/ibm_cos/${connectionId}/bucket-status`,+ );+ if (!res.ok) {+ const err = await res.json().catch(() => ({}));+ throw new Error(err.error || "Failed to fetch bucket status");+ }+ const data = await res.json();+ return data.buckets as IBMCOSBucketStatus[];+}++export function useIBMCOSBucketStatusQuery(+ connectionId: string | null | undefined,+ options?: { enabled?: boolean },+) {+ return useQuery<IBMCOSBucketStatus[]>({+ queryKey: ["ibm-cos-bucket-status", connectionId],+ queryFn: () => fetchIBMCOSBucketStatus(connectionId!),+ enabled: (options?.enabled ?? true) && !!connectionId,+ staleTime: 0,+ refetchOnMount: "always",+ });+}diff --git a/frontend/app/api/queries/useIBMCOSBucketsQuery.ts b/frontend/app/api/queries/useIBMCOSBucketsQuery.tsnew file mode 100644index 000000000..a7daa619f--- /dev/null+++ b/frontend/app/api/queries/useIBMCOSBucketsQuery.ts@@ -0,0 +1,23 @@+import { useQuery } from "@tanstack/react-query";++async function fetchIBMCOSBuckets(connectionId: string): Promise<string[]> {+ const res = await fetch(`/api/connectors/ibm_cos/${connectionId}/buckets`);+ if (!res.ok) {+ const err = await res.json().catch(() => ({}));+ throw new Error(err.error || "Failed to list buckets");+ }+ const data = await res.json();+ return data.buckets as string[];+}++export function useIBMCOSBucketsQuery(+ connectionId: string | null | undefined,+ options?: { enabled?: boolean },+) {+ return useQuery<string[]>({+ queryKey: ["ibm-cos-buckets", connectionId],+ queryFn: () => fetchIBMCOSBuckets(connectionId!),+ enabled: (options?.enabled ?? true) && !!connectionId,+ staleTime: 30_000,+ });+}diff --git a/frontend/app/api/queries/useIBMCOSDefaultsQuery.ts b/frontend/app/api/queries/useIBMCOSDefaultsQuery.tsnew file mode 100644index 000000000..b44ee4c5b--- /dev/null+++ b/frontend/app/api/queries/useIBMCOSDefaultsQuery.ts@@ -0,0 +1,28 @@+import { useQuery } from "@tanstack/react-query";++export interface IBMCOSDefaults {+ api_key_set: boolean;+ service_instance_id: string;+ endpoint: string;+ hmac_access_key_set: boolean;+ hmac_secret_key_set: boolean;+ auth_mode: "iam" | "hmac";+ bucket_names: string[];+ connection_id: string | null;+ disable_iam: boolean;+}++async function fetchIBMCOSDefaults(): Promise<IBMCOSDefaults> {+ const res = await fetch("/api/connectors/ibm_cos/defaults");+ if (!res.ok) throw new Error("Failed to fetch IBM COS defaults");+ return res.json();+}++export function useIBMCOSDefaultsQuery(options?: { enabled?: boolean }) {+ return useQuery<IBMCOSDefaults>({+ queryKey: ["ibm-cos-defaults"],+ queryFn: fetchIBMCOSDefaults,+ enabled: options?.enabled ?? true,+ staleTime: 0,+ });+}diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex a634423e8..6a99deab8 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -41,6 +41,8 @@ import {formatFilesToDelete,} from "../../components/delete-confirmation-dialog";import GoogleDriveIcon from "../../components/icons/google-drive-logo";+import IBMLogo from "../../components/icons/ibm-logo";+import IBMCOSIcon from "../../components/icons/ibm-cos-icon";import OneDriveIcon from "../../components/icons/one-drive-logo";import SharePointIcon from "../../components/icons/share-point-logo";import { useDeleteDocument } from "../api/mutations/useDeleteDocument";@@ -63,6 +65,8 @@ function getSourceIcon(connectorType?: string) {return <Globe className="h-4 w-4 text-muted-foreground flex-shrink-0" />;case "s3":return <Cloud className="h-4 w-4 text-foreground flex-shrink-0" />;+ case "ibm_cos":+ return <IBMCOSIcon className="h-4 w-4 flex-shrink-0" />;default:return (<FileIcon className="h-4 w-4 text-muted-foreground flex-shrink-0" />diff --git a/frontend/app/settings/_components/connector-card.tsx b/frontend/app/settings/_components/connector-card.tsxindex aa132d99d..37c08dc77 100644--- a/frontend/app/settings/_components/connector-card.tsx+++ b/frontend/app/settings/_components/connector-card.tsx@@ -1,6 +1,6 @@"use client";-import { Loader2, Plus, RefreshCcw, Trash2 } from "lucide-react";+import { Loader2, Plus, RefreshCcw, Settings2, Trash2 } from "lucide-react";import Link from "next/link";import { Button } from "@/components/ui/button";import {@@ -29,6 +29,8 @@ interface ConnectorCardProps {onConnect: (connector: Connector) => void;onDisconnect: (connector: Connector) => void;onNavigateToKnowledge: (connector: Connector) => void;+ /** Optional: open a connector-specific settings/edit dialog */+ onConfigure?: (connector: Connector) => void;}export default function ConnectorCard({@@ -38,6 +40,7 @@ export default function ConnectorCard({onConnect,onDisconnect,onNavigateToKnowledge,+ onConfigure,}: ConnectorCardProps) {console.log(connector);const isConnected =@@ -91,19 +94,32 @@ export default function ConnectorCard({<Plus className="h-4 w-4" /><span className="text-mmd truncate">Add Knowledge</span></Button>- <Button- variant="outline"- onClick={() => onConnect(connector)}- disabled={isConnecting || isDisconnecting}- className="cursor-pointer"- size="iconMd"- >- {isConnecting ? (- <RefreshCcw className="h-4 w-4 animate-spin" />- ) : (- <RefreshCcw className="h-4 w-4" />- )}- </Button>+ {onConfigure ? (+ <Button+ variant="outline"+ onClick={() => onConfigure(connector)}+ disabled={isConnecting || isDisconnecting}+ className="cursor-pointer"+ size="iconMd"+ title="Edit configuration"+ >+ <Settings2 className="h-4 w-4" />+ </Button>+ ) : (+ <Button+ variant="outline"+ onClick={() => onConnect(connector)}+ disabled={isConnecting || isDisconnecting}+ className="cursor-pointer"+ size="iconMd"+ >+ {isConnecting ? (+ <RefreshCcw className="h-4 w-4 animate-spin" />+ ) : (+ <RefreshCcw className="h-4 w-4" />+ )}+ </Button>+ )}<Buttonvariant="outline"onClick={() => onDisconnect(connector)}@@ -120,7 +136,9 @@ export default function ConnectorCard({</div>) : (<Button- onClick={() => onConnect(connector)}+ onClick={() =>+ onConfigure ? onConfigure(connector) : onConnect(connector)+ }disabled={isConnecting}className="w-full cursor-pointer group-hover:bg-background group-hover:border-zinc-700 group-hover:text-primary"size="sm"@@ -136,6 +154,17 @@ export default function ConnectorCard({</Button>)}</div>+ ) : onConfigure ? (+ // Connector supports UI-based configuration (e.g. IBM COS)+ <Button+ variant="outline"+ onClick={() => onConfigure(connector)}+ className="w-full cursor-pointer"+ size="sm"+ >+ <Settings2 className="h-4 w-4 mr-2" />+ Configure+ </Button>) : (<div className="text-sm text-muted-foreground"><p>diff --git a/frontend/app/settings/_components/connector-cards.tsx b/frontend/app/settings/_components/connector-cards.tsxindex 2e9f31a73..00f207584 100644--- a/frontend/app/settings/_components/connector-cards.tsx+++ b/frontend/app/settings/_components/connector-cards.tsx@@ -1,7 +1,7 @@"use client";+import { useCallback, useState } from "react";import { useRouter } from "next/navigation";-import { useCallback } from "react";import { useConnectConnectorMutation } from "@/app/api/mutations/useConnectConnectorMutation";import { useDisconnectConnectorMutation } from "@/app/api/mutations/useDisconnectConnectorMutation";import {@@ -9,15 +9,18 @@ import {useGetConnectorsQuery,} from "@/app/api/queries/useGetConnectorsQuery";import GoogleDriveIcon from "@/components/icons/google-drive-logo";+import IBMCOSIcon from "@/components/icons/ibm-cos-icon";import OneDriveIcon from "@/components/icons/one-drive-logo";import SharePointIcon from "@/components/icons/share-point-logo";import { useAuth } from "@/contexts/auth-context";import ConnectorCard, { type Connector } from "./connector-card";import ConnectorsSkeleton from "./connectors-skeleton";+import IBMCOSSettingsDialog from "./ibm-cos-settings-dialog";export default function ConnectorCards() {const { isAuthenticated, isNoAuthMode } = useAuth();const router = useRouter();+ const [ibmCOSDialogOpen, setIBMCOSDialogOpen] = useState(false);const { data: queryConnectors = [], isLoading: connectorsLoading } =useGetConnectorsQuery({@@ -32,6 +35,7 @@ export default function ConnectorCards() {"google-drive": <GoogleDriveIcon />,sharepoint: <SharePointIcon />,onedrive: <OneDriveIcon />,+ "ibm-cos": <IBMCOSIcon />,};return (iconMap[iconName] || (@@ -63,37 +67,53 @@ export default function ConnectorCards() {router.push(`/upload/${provider}`);};+ // Connectors that use a settings dialog instead of OAuth for configuration+ const getConfigureHandler = (connector: Connector) => {+ if (connector.type === "ibm_cos") {+ return () => setIBMCOSDialogOpen(true);+ }+ return undefined;+ };+if (!connectorsLoading && connectors.length === 0) {return null;}return (- <div className="grid gap-6 md:grid-cols-2 lg:grid-cols-3">- {connectorsLoading ? (- <>- <ConnectorsSkeleton />- <ConnectorsSkeleton />- <ConnectorsSkeleton />- </>- ) : (- connectors.map((connector) => (- <ConnectorCard- key={connector.id}- connector={connector}- isConnecting={- connectMutation.isPending &&- connectMutation.variables?.connector.id === connector.id- }- isDisconnecting={- disconnectMutation.isPending &&- (disconnectMutation.variables as any)?.type === connector.type- }- onConnect={handleConnect}- onDisconnect={handleDisconnect}- onNavigateToKnowledge={navigateToKnowledgePage}- />- ))- )}- </div>+ <>+ <div className="grid gap-6 md:grid-cols-2 lg:grid-cols-3">+ {connectorsLoading ? (+ <>+ <ConnectorsSkeleton />+ <ConnectorsSkeleton />+ <ConnectorsSkeleton />+ </>+ ) : (+ connectors.map((connector) => (+ <ConnectorCard+ key={connector.id}+ connector={connector}+ isConnecting={+ connectMutation.isPending &&+ connectMutation.variables?.connector.id === connector.id+ }+ isDisconnecting={+ disconnectMutation.isPending &&+ (disconnectMutation.variables as any)?.type === connector.type+ }+ onConnect={handleConnect}+ onDisconnect={handleDisconnect}+ onNavigateToKnowledge={navigateToKnowledgePage}+ onConfigure={getConfigureHandler(connector)}+ />+ ))+ )}+ </div>++ <IBMCOSSettingsDialog+ open={ibmCOSDialogOpen}+ setOpen={setIBMCOSDialogOpen}+ />+ </>);}diff --git a/frontend/app/settings/_components/ibm-cos-settings-dialog.tsx b/frontend/app/settings/_components/ibm-cos-settings-dialog.tsxnew file mode 100644index 000000000..0d6a9b367--- /dev/null+++ b/frontend/app/settings/_components/ibm-cos-settings-dialog.tsx@@ -0,0 +1,198 @@+"use client";++import { useState } from "react";+import { FormProvider, useForm } from "react-hook-form";+import { toast } from "sonner";+import { useQueryClient } from "@tanstack/react-query";+import IBMCOSIcon from "@/components/icons/ibm-cos-icon";+import { Button } from "@/components/ui/button";+import {+ Dialog,+ DialogContent,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog";+import { IBMCOSSettingsForm, type IBMCOSFormData } from "./ibm-cos-settings-form";+import { useIBMCOSDefaultsQuery } from "@/app/api/queries/useIBMCOSDefaultsQuery";+import { useIBMCOSConfigureMutation } from "@/app/api/mutations/useIBMCOSConfigureMutation";++interface IBMCOSSettingsDialogProps {+ open: boolean;+ setOpen: (open: boolean) => void;+}++export default function IBMCOSSettingsDialog({+ open,+ setOpen,+}: IBMCOSSettingsDialogProps) {+ const queryClient = useQueryClient();++ // Fetch env-based defaults to pre-fill the form+ const { data: defaults } = useIBMCOSDefaultsQuery({ enabled: open });++ const disableIam = defaults?.disable_iam ?? false;++ const methods = useForm<IBMCOSFormData>({+ mode: "onSubmit",+ values: {+ auth_mode: disableIam ? "hmac" : (defaults?.auth_mode ?? "hmac"),+ endpoint: defaults?.endpoint ?? "",+ api_key: "",+ service_instance_id: defaults?.service_instance_id ?? "",+ hmac_access_key: "",+ hmac_secret_key: "",+ },+ });++ const { handleSubmit } = methods;++ // Bucket state+ const [buckets, setBuckets] = useState<string[] | null>(+ defaults?.bucket_names?.length ? defaults.bucket_names : null,+ );+ const [selectedBuckets, setSelectedBuckets] = useState<string[]>(+ defaults?.bucket_names ?? [],+ );+ const [isFetchingBuckets, setIsFetchingBuckets] = useState(false);+ const [bucketsError, setBucketsError] = useState<string | null>(null);+ const [formError, setFormError] = useState<string | null>(null);++ const configureMutation = useIBMCOSConfigureMutation();++ // Inline bucket test: calls the configure endpoint without saving,+ // then uses the returned connection_id to list buckets.+ const handleTestConnection = handleSubmit(async (data) => {+ setIsFetchingBuckets(true);+ setBucketsError(null);+ setFormError(null);++ try {+ // First configure (creates/updates connection) to get a connection_id+ const result = await configureMutation.mutateAsync({+ auth_mode: data.auth_mode,+ endpoint: data.endpoint,+ api_key: data.api_key || undefined,+ service_instance_id: data.service_instance_id || undefined,+ hmac_access_key: data.hmac_access_key || undefined,+ hmac_secret_key: data.hmac_secret_key || undefined,+ connection_id: defaults?.connection_id ?? undefined,+ });++ // Then list buckets using the connection+ const res = await fetch(+ `/api/connectors/ibm_cos/${result.connection_id}/buckets`,+ );+ const json = await res.json();+ if (!res.ok) throw new Error(json.error || "Failed to list buckets");++ const fetchedBuckets: string[] = json.buckets;+ setBuckets(fetchedBuckets);++ // Keep any previously selected buckets that still exist+ setSelectedBuckets((prev) =>+ prev.filter((b) => fetchedBuckets.includes(b)),+ );++ // Refresh defaults so we have the new connection_id+ queryClient.invalidateQueries({ queryKey: ["ibm-cos-defaults"] });+ } catch (err: any) {+ setBucketsError(err.message ?? "Connection failed");+ } finally {+ setIsFetchingBuckets(false);+ }+ });++ const onSubmit = handleSubmit(async (data) => {+ setFormError(null);+ if (buckets === null) {+ setFormError("Test the connection first to validate credentials.");+ return;+ }++ try {+ // Refresh defaults to get latest connection_id+ const latestDefaults = await queryClient.fetchQuery({+ queryKey: ["ibm-cos-defaults"],+ queryFn: async () => {+ const res = await fetch("/api/connectors/ibm_cos/defaults");+ return res.json();+ },+ staleTime: 0,+ });++ await configureMutation.mutateAsync({+ auth_mode: data.auth_mode,+ endpoint: data.endpoint,+ api_key: data.api_key || undefined,+ service_instance_id: data.service_instance_id || undefined,+ hmac_access_key: data.hmac_access_key || undefined,+ hmac_secret_key: data.hmac_secret_key || undefined,+ bucket_names: selectedBuckets,+ connection_id: latestDefaults?.connection_id ?? defaults?.connection_id ?? undefined,+ });++ toast.success("IBM Cloud Object Storage configured", {+ description:+ selectedBuckets.length > 0+ ? `Will ingest from: ${selectedBuckets.join(", ")}`+ : "Will auto-discover and ingest all accessible buckets.",+ icon: <IBMCOSIcon className="w-4 h-4" />,+ });++ queryClient.invalidateQueries({ queryKey: ["connectors"] });+ setOpen(false);+ } catch (err: any) {+ setFormError(err.message ?? "Failed to save configuration");+ }+ });++ return (+ <Dialog open={open} onOpenChange={setOpen}>+ <DialogContent autoFocus={false} className="max-w-2xl max-h-[90vh] overflow-y-auto">+ <FormProvider {...methods}>+ <form onSubmit={onSubmit} className="grid gap-4">+ <DialogHeader className="mb-2">+ <DialogTitle className="flex items-center gap-3">+ <div className="w-8 h-8 rounded flex items-center justify-center bg-white border">+ <IBMCOSIcon className="text-black" />+ </div>+ IBM Cloud Object Storage Setup+ </DialogTitle>+ </DialogHeader>++ <IBMCOSSettingsForm+ buckets={buckets}+ selectedBuckets={selectedBuckets}+ onSelectedBucketsChange={setSelectedBuckets}+ isFetchingBuckets={isFetchingBuckets}+ bucketsError={bucketsError}+ onTestConnection={handleTestConnection as () => void}+ apiKeySet={defaults?.api_key_set}+ hmacAccessKeySet={defaults?.hmac_access_key_set}+ hmacSecretKeySet={defaults?.hmac_secret_key_set}+ formError={formError}+ disableIam={disableIam}+ />++ <DialogFooter className="mt-4">+ <Button+ variant="outline"+ type="button"+ onClick={() => setOpen(false)}+ >+ Cancel+ </Button>+ <Button+ type="submit"+ disabled={configureMutation.isPending || isFetchingBuckets}+ >+ {configureMutation.isPending ? "Saving…" : "Save"}+ </Button>+ </DialogFooter>+ </form>+ </FormProvider>+ </DialogContent>+ </Dialog>+ );+}diff --git a/frontend/app/settings/_components/ibm-cos-settings-form.tsx b/frontend/app/settings/_components/ibm-cos-settings-form.tsxnew file mode 100644index 000000000..481aad35b--- /dev/null+++ b/frontend/app/settings/_components/ibm-cos-settings-form.tsx@@ -0,0 +1,305 @@+"use client";++import { useFormContext, Controller } from "react-hook-form";+import { LabelWrapper } from "@/components/label-wrapper";+import { Input } from "@/components/ui/input";+import { Label } from "@/components/ui/label";+import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs";+import { Loader2, RefreshCcw } from "lucide-react";+import { Button } from "@/components/ui/button";++export interface IBMCOSFormData {+ auth_mode: "iam" | "hmac";+ endpoint: string;+ // IAM+ api_key: string;+ service_instance_id: string;+ // HMAC+ hmac_access_key: string;+ hmac_secret_key: string;+}++interface IBMCOSSettingsFormProps {+ /** Available buckets after a successful test — null means not yet tested */+ buckets: string[] | null;+ selectedBuckets: string[];+ onSelectedBucketsChange: (buckets: string[]) => void;+ isFetchingBuckets: boolean;+ bucketsError: string | null;+ onTestConnection: () => void;+ apiKeySet?: boolean;+ hmacAccessKeySet?: boolean;+ hmacSecretKeySet?: boolean;+ formError?: string | null;+ /** When true, IAM tab is greyed out and HMAC is the only selectable option */+ disableIam?: boolean;+}++export function IBMCOSSettingsForm({+ buckets,+ selectedBuckets,+ onSelectedBucketsChange,+ isFetchingBuckets,+ bucketsError,+ onTestConnection,+ apiKeySet,+ hmacAccessKeySet,+ hmacSecretKeySet,+ formError,+ disableIam = false,+}: IBMCOSSettingsFormProps) {+ const {+ register,+ control,+ formState: { errors },+ } = useFormContext<IBMCOSFormData>();++ const toggleBucket = (name: string, checked: boolean) => {+ if (checked) {+ onSelectedBucketsChange([...selectedBuckets, name]);+ } else {+ onSelectedBucketsChange(selectedBuckets.filter((b) => b !== name));+ }+ };++ const toggleAll = (checked: boolean) => {+ onSelectedBucketsChange(checked ? (buckets ?? []) : []);+ };++ return (+ <div className="space-y-5">+ {/* Auth mode selector using Tabs */}+ <div className="space-y-2">+ <Label className="text-sm font-medium">Authentication Method</Label>+ <Controller+ control={control}+ name="auth_mode"+ render={({ field }) => (+ <Tabs+ value={disableIam && field.value === "iam" ? "hmac" : field.value}+ onValueChange={(v) => {+ if (disableIam && v === "iam") return;+ field.onChange(v);+ }}+ >+ <TabsList className="w-full">+ <TabsTrigger value="hmac">+ <span className="font-semibold text-sm">HMAC</span>+ <span className="text-xs text-muted-foreground font-normal">+ Access Key + Secret Key+ </span>+ </TabsTrigger>+ <TabsTrigger+ value="iam"+ disabled={disableIam}+ className={disableIam ? "opacity-40 cursor-not-allowed" : ""}+ title={disableIam ? "IAM mode is disabled. Set OPENRAG_IBM_COS_IAM_UI=true to enable." : undefined}+ >+ <span className="font-semibold text-sm">IAM</span>+ <span className="text-xs text-muted-foreground font-normal">+ API Key + Resource Instance ID+ </span>+ </TabsTrigger>+ </TabsList>++ {/* HMAC fields — first tab */}+ <TabsContent value="hmac">+ <div className="space-y-4">+ <div className="space-y-1">+ <LabelWrapper+ label="Access Key ID"+ helperText={'Copy "cos_hmac_keys.access_key_id" from your IBM COS Service Credentials JSON'}+ id="ibm-cos-hmac-key"+ required+ >+ <Input+ {...register("hmac_access_key", { setValueAs: (v) => v?.trim() })}+ id="ibm-cos-hmac-key"+ type="password"+ placeholder={+ hmacAccessKeySet+ ? "•••••••• (loaded from env)"+ : "cos_hmac_keys.access_key_id"+ }+ autoComplete="off"+ />+ </LabelWrapper>+ </div>+ <div className="space-y-1">+ <LabelWrapper+ label="Secret Access Key"+ helperText={'Copy "cos_hmac_keys.secret_access_key" from your IBM COS Service Credentials JSON'}+ id="ibm-cos-hmac-secret"+ required+ >+ <Input+ {...register("hmac_secret_key", { setValueAs: (v) => v?.trim() })}+ id="ibm-cos-hmac-secret"+ type="password"+ placeholder={+ hmacSecretKeySet+ ? "•••••••• (loaded from env)"+ : "cos_hmac_keys.secret_access_key"+ }+ autoComplete="off"+ />+ </LabelWrapper>+ </div>+ </div>+ </TabsContent>++ {/* IAM fields — second tab */}+ <TabsContent value="iam">+ <div className="space-y-4">+ <div className="space-y-1">+ <LabelWrapper+ label="API Key"+ helperText={'Copy the "apikey" field from your IBM COS Service Credentials JSON'}+ id="ibm-cos-api-key"+ required+ >+ <Input+ {...register("api_key", { setValueAs: (v) => v?.trim() })}+ id="ibm-cos-api-key"+ type="password"+ placeholder={+ apiKeySet+ ? "•••••••• (loaded from env)"+ : 'apikey value from Service Credentials'+ }+ autoComplete="off"+ />+ </LabelWrapper>+ </div>+ <div className="space-y-1">+ <LabelWrapper+ label="Resource Instance ID"+ helperText={'Copy the "resource_instance_id" field from your IBM COS Service Credentials JSON'}+ id="ibm-cos-svc-id"+ required+ >+ <Input+ {...register("service_instance_id", { setValueAs: (v) => v?.trim() })}+ id="ibm-cos-svc-id"+ placeholder="crn:v1:bluemix:public:cloud-object-storage:..."+ />+ </LabelWrapper>+ </div>+ </div>+ </TabsContent>+ </Tabs>+ )}+ />+ </div>++ {/* Endpoint — shared by both auth modes */}+ <div className="space-y-1">+ <LabelWrapper+ label="Service Endpoint"+ helperText="Public endpoint for your bucket region, e.g. https://s3.us-south.cloud-object-storage.appdomain.cloud — find yours at IBM Cloud → COS → Buckets → Configuration → Endpoints"+ id="ibm-cos-endpoint"+ required+ >+ <Input+ {...register("endpoint", { required: "Endpoint is required", setValueAs: (v) => v?.trim() })}+ id="ibm-cos-endpoint"+ placeholder="https://s3.us-south.cloud-object-storage.appdomain.cloud"+ className={errors.endpoint ? "!border-destructive" : ""}+ />+ </LabelWrapper>+ {errors.endpoint && (+ <p className="text-sm text-destructive">{errors.endpoint.message}</p>+ )}+ </div>++ {/* Test connection */}+ <Button+ type="button"+ variant="outline"+ onClick={onTestConnection}+ disabled={isFetchingBuckets}+ className="w-full"+ >+ {isFetchingBuckets ? (+ <>+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />+ Connecting…+ </>+ ) : buckets !== null ? (+ <>+ <RefreshCcw className="h-4 w-4 mr-2" />+ Refresh Buckets+ </>+ ) : (+ "Test Connection & List Buckets"+ )}+ </Button>++ {bucketsError && (+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">+ {bucketsError}+ </p>+ )}++ {formError && (+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">+ {formError}+ </p>+ )}++ {/* Bucket selector — native checkboxes styled with Tailwind */}+ {buckets !== null && (+ <div className="space-y-2">+ <div className="flex items-center justify-between">+ <Label className="text-sm font-medium">+ Select Buckets to Ingest+ <span className="ml-1 text-muted-foreground font-normal">+ (leave all unchecked to ingest everything)+ </span>+ </Label>+ {buckets.length > 1 && (+ <button+ type="button"+ className="text-xs text-primary underline-offset-2 hover:underline"+ onClick={() =>+ toggleAll(selectedBuckets.length !== buckets.length)+ }+ >+ {selectedBuckets.length === buckets.length+ ? "Deselect all"+ : "Select all"}+ </button>+ )}+ </div>++ {buckets.length === 0 ? (+ <p className="text-sm text-muted-foreground">+ No buckets found for this account.+ </p>+ ) : (+ <div className="max-h-48 overflow-y-auto rounded-lg border p-3 space-y-2">+ {buckets.map((bucket) => (+ <label+ key={bucket}+ htmlFor={`bucket-${bucket}`}+ className="flex items-center gap-2.5 cursor-pointer select-none"+ >+ <input+ id={`bucket-${bucket}`}+ type="checkbox"+ className="h-4 w-4 rounded border-border accent-primary"+ checked={selectedBuckets.includes(bucket)}+ onChange={(e) =>+ toggleBucket(bucket, e.target.checked)+ }+ />+ <span className="text-sm">{bucket}</span>+ </label>+ ))}+ </div>+ )}+ </div>+ )}+ </div>+ );+}diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex bf96214f7..277fc965e 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -1,11 +1,14 @@"use client";-import { AlertCircle, ArrowLeft } from "lucide-react";+import { AlertCircle, ArrowLeft, RefreshCw } from "lucide-react";import { useParams, useRouter } from "next/navigation";-import { useEffect, useState } from "react";+import { useState } from "react";+import { toast } from "sonner";+import { useQueryClient } from "@tanstack/react-query";import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";import { useGetConnectorTokenQuery } from "@/app/api/queries/useGetConnectorTokenQuery";+import { useIBMCOSBucketStatusQuery } from "@/app/api/queries/useIBMCOSBucketStatusQuery";import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";import type { IngestSettings } from "@/components/cloud-picker/types";import { Button } from "@/components/ui/button";@@ -16,6 +19,179 @@ import {} from "@/components/ui/tooltip";import { useTask } from "@/contexts/task-context";+// Connectors that sync entire buckets/repositories without a file picker+const DIRECT_SYNC_PROVIDERS = ["ibm_cos"];++// ---------------------------------------------------------------------------+// IBM COS bucket list with sync status+// ---------------------------------------------------------------------------++function IBMCOSBucketView({+ connector,+ syncMutation,+ addTask,+ onBack,+ onDone,+}: {+ connector: any;+ syncMutation: ReturnType<typeof useSyncConnector>;+ addTask: (id: string) => void;+ onBack: () => void;+ onDone: () => void;+}) {+ const queryClient = useQueryClient();+ const { data: buckets, isLoading, refetch } = useIBMCOSBucketStatusQuery(+ connector.connectionId,+ { enabled: true },+ );++ const [syncingBucket, setSyncingBucket] = useState<string | null>(null);++ const invalidateBucketStatus = () => {+ queryClient.invalidateQueries({ queryKey: ["ibm-cos-bucket-status", connector.connectionId] });+ };++ const syncAll = () => {+ syncMutation.mutate(+ {+ connectorType: connector.type,+ body: {+ connection_id: connector.connectionId!,+ selected_files: [],+ sync_all: true,+ },+ },+ {+ onSuccess: (result) => {+ invalidateBucketStatus();+ if (result.task_ids?.length) {+ addTask(result.task_ids[0]);+ onDone();+ } else {+ toast.info("No files found in any bucket.");+ }+ },+ onError: (err) => {+ toast.error(err instanceof Error ? err.message : "Sync failed");+ },+ },+ );+ };++ const syncBucket = (bucketName: string) => {+ setSyncingBucket(bucketName);+ syncMutation.mutate(+ {+ connectorType: connector.type,+ body: {+ connection_id: connector.connectionId!,+ selected_files: [],+ bucket_filter: [bucketName],+ },+ },+ {+ onSuccess: (result) => {+ setSyncingBucket(null);+ invalidateBucketStatus();+ if (result.task_ids?.length) {+ addTask(result.task_ids[0]);+ onDone();+ } else {+ toast.info(`No files found in bucket "${bucketName}".`);+ }+ },+ onError: (err) => {+ setSyncingBucket(null);+ toast.error(err instanceof Error ? err.message : "Sync failed");+ },+ },+ );+ };++ return (+ <>+ <div className="mb-8 flex gap-2 items-center">+ <Button variant="ghost" onClick={onBack} size="icon">+ <ArrowLeft size={18} />+ </Button>+ <h2 className="text-xl text-[18px] font-semibold">+ Add from {connector.name}+ </h2>+ </div>++ <div className="max-w-3xl mx-auto space-y-4">+ <div className="flex items-center justify-between">+ <p className="text-sm text-muted-foreground">+ Select a bucket to ingest, or sync everything at once.+ </p>+ <div className="flex gap-2">+ <Button+ variant="ghost"+ size="icon"+ onClick={() => refetch()}+ disabled={isLoading}+ title="Refresh bucket status"+ >+ <RefreshCw size={16} className={isLoading ? "animate-spin" : ""} />+ </Button>+ <Button+ className="bg-foreground text-background hover:bg-foreground/90 font-semibold"+ onClick={syncAll}+ disabled={syncMutation.isPending}+ >+ {syncMutation.isPending && !syncingBucket+ ? "Ingesting…"+ : "Sync All Buckets"}+ </Button>+ </div>+ </div>++ {isLoading ? (+ <div className="flex justify-center py-8">+ <div className="animate-spin rounded-full h-6 w-6 border-b-2 border-primary" />+ </div>+ ) : !buckets?.length ? (+ <div className="rounded-lg border p-6 text-center text-muted-foreground text-sm">+ No buckets found. Check your IBM COS credentials and endpoint.+ </div>+ ) : (+ <div className="rounded-lg border divide-y">+ {buckets.map((bucket) => (+ <div+ key={bucket.name}+ className="flex items-center justify-between px-4 py-3"+ >+ <div className="flex items-center gap-3">+ <div>+ <p className="font-medium text-sm">{bucket.name}</p>+ {bucket.ingested_count > 0 && (+ <p className="text-xs text-muted-foreground">+ {bucket.ingested_count} document{bucket.ingested_count !== 1 ? "s" : ""} ingested+ </p>+ )}+ </div>+ </div>+ <Button+ variant="outline"+ size="sm"+ onClick={() => syncBucket(bucket.name)}+ disabled={syncMutation.isPending}+ >+ {syncingBucket === bucket.name+ ? "Ingesting…"+ : bucket.ingested_count > 0+ ? "Re-sync"+ : "Ingest"}+ </Button>+ </div>+ ))}+ </div>+ )}+ </div>+ </>+ );+}+// CloudFile interface is now imported from the unified cloud pickerexport default function UploadProviderPage() {@@ -31,6 +207,8 @@ export default function UploadProviderPage() {} = useGetConnectorsQuery();const connector = connectors.find((c) => c.type === provider);+ const isDirectSyncProvider = DIRECT_SYNC_PROVIDERS.includes(provider);+const { data: tokenData, isLoading: tokenLoading } =useGetConnectorTokenQuery({@@ -42,7 +220,11 @@ export default function UploadProviderPage() {: undefined,},{- enabled: !!connector && connector.status === "connected",+ // Direct-sync providers (e.g. IBM COS) don't use OAuth tokens+ enabled:+ !!connector &&+ connector.status === "connected" &&+ !isDirectSyncProvider,},);@@ -61,7 +243,8 @@ export default function UploadProviderPage() {});const accessToken = tokenData?.access_token || null;- const isLoading = connectorsLoading || tokenLoading;+ const isLoading =+ connectorsLoading || (!isDirectSyncProvider && tokenLoading);const isIngesting = syncMutation.isPending;// Error handling@@ -193,6 +376,19 @@ export default function UploadProviderPage() {);}+ // Direct-sync providers (e.g. IBM COS) show a bucket list with sync status.+ if (isDirectSyncProvider && connector.status === "connected") {+ return (+ <IBMCOSBucketView+ connector={connector}+ syncMutation={syncMutation}+ addTask={addTask}+ onBack={() => router.back()}+ onDone={() => router.push("/knowledge")}+ />+ );+ }+if (!accessToken) {return (<>diff --git a/frontend/components/cloud-picker/file-item.tsx b/frontend/components/cloud-picker/file-item.tsxindex 617e54433..fa4d079fc 100644--- a/frontend/components/cloud-picker/file-item.tsx+++ b/frontend/components/cloud-picker/file-item.tsx@@ -2,6 +2,7 @@import { FileText, Folder, Trash2 } from "lucide-react";import GoogleDriveIcon from "@/components/icons/google-drive-logo";+import IBMCOSIcon from "@/components/icons/ibm-cos-icon";import OneDriveIcon from "@/components/icons/one-drive-logo";import SharePointIcon from "@/components/icons/share-point-logo";import { Button } from "@/components/ui/button";@@ -54,6 +55,8 @@ const getProviderIcon = (provider: string) => {return <OneDriveIcon />;case "sharepoint":return <SharePointIcon />;+ case "ibm_cos":+ return <IBMCOSIcon />;default:return <FileText className="h-6 w-6" />;}diff --git a/frontend/components/icons/ibm-cos-icon.tsx b/frontend/components/icons/ibm-cos-icon.tsxnew file mode 100644index 000000000..a8fc4b324--- /dev/null+++ b/frontend/components/icons/ibm-cos-icon.tsx@@ -0,0 +1,63 @@+export default function IBMCOSIcon(props: React.SVGProps<SVGSVGElement>) {+ return (+ <svg+ xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 32 32"+ width="16"+ height="16"+ {...props}+ >+ <defs>+ <linearGradient+ id="ibm-cos-grad-a"+ x1="827.5"+ y1="4690.5"+ x2="838.5"+ y2="4690.5"+ gradientTransform="translate(-823.5 -4669.5)"+ gradientUnits="userSpaceOnUse"+ >+ <stop offset=".1" />+ <stop offset=".888" stopOpacity="0" />+ </linearGradient>+ <linearGradient+ id="ibm-cos-grad-b"+ y1="32"+ x2="32"+ gradientUnits="userSpaceOnUse"+ >+ <stop offset=".1" stopColor="#a56eff" />+ <stop offset=".9" stopColor="#0f62fe" />+ </linearGradient>+ <mask+ id="ibm-cos-mask"+ x="0"+ y="0"+ width="32"+ height="32"+ maskUnits="userSpaceOnUse"+ >+ <circle cx="7" cy="25" r="1" fill="#fff" />+ <path+ d="M28 20h-2v2h2v6H4v-6h10v-2H4a2 2 0 0 0-2 2v6a2 2 0 0 0 2 2h24a2 2 0 0 0 2-2v-6a2 2 0 0 0-2-2z"+ fill="#fff"+ />+ <path+ transform="rotate(180 9.5 21)"+ fill="url(#ibm-cos-grad-a)"+ d="M4 19h11v4H4z"+ />+ </mask>+ </defs>+ <g>+ <g mask="url(#ibm-cos-mask)">+ <path fill="url(#ibm-cos-grad-b)" d="M0 0h32v32H0z" />+ </g>+ <path+ d="M18 10h-8V2h8zm-6-2h4V4h-4zm10 6h-6v8h8v-6h6V8h-8zm0 6h-4v-4h4zm6-10v4h-4v-4z"+ fill="#001d6c"+ />+ </g>+ </svg>+ );+}diff --git a/pyproject.toml b/pyproject.tomlindex 422c052bd..dfb583aac 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -23,6 +23,7 @@ dependencies = ["fastapi>=0.115.0","uvicorn>=0.35.0","boto3>=1.35.0",+ "ibm-cos-sdk>=2.13.0","psutil>=7.0.0","rich>=13.0.0","textual>=0.45.0",diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 73c096d44..921df2bd8 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -88,6 +88,11 @@ async def get_synced_file_ids_for_connector(class ConnectorSyncBody(BaseModel):max_files: Optional[int] = Noneselected_files: Optional[List[Any]] = None+ # When True, ingest ALL files from the connector (bypasses the existing-files gate).+ # Used by direct-sync providers like IBM COS on initial ingest.+ sync_all: bool = False+ # When set, only ingest files from these buckets (IBM COS specific).+ bucket_filter: Optional[List[str]] = Noneasync def list_connectors(@@ -200,6 +205,51 @@ async def connector_sync(jwt_token=jwt_token,file_infos=file_infos,)+ elif body.sync_all or body.bucket_filter:+ # Full ingest: discover and ingest all files (or files from specific buckets).+ # Used by direct-sync providers (IBM COS) on initial ingest or per-bucket sync.+ logger.info(+ "Full connector ingest requested",+ connector_type=connector_type,+ bucket_filter=body.bucket_filter,+ )+ connector = await connector_service.get_connector(working_connection.connection_id)+ if body.bucket_filter:+ # List only files from the requested buckets, then sync_specific_files+ original_buckets = connector.bucket_names+ connector.bucket_names = body.bucket_filter+ try:+ all_file_ids = []+ page_token = None+ while True:+ result = await connector.list_files(page_token=page_token)+ for f in result.get("files", []):+ all_file_ids.append(f["id"])+ page_token = result.get("next_page_token")+ if not page_token:+ break+ finally:+ connector.bucket_names = original_buckets++ if not all_file_ids:+ return JSONResponse(+ {"status": "no_files", "message": "No files found in the selected buckets."},+ status_code=200,+ )+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ all_file_ids,+ jwt_token=jwt_token,+ )+ else:+ # sync_all: ingest everything the connector can see+ task_id = await connector_service.sync_connector_files(+ working_connection.connection_id,+ user.user_id,+ max_files=max_files,+ jwt_token=jwt_token,+ )else:# No files specified - sync only files already in OpenSearch for this connector# This ensures deleted files stay deleted@@ -209,7 +259,7 @@ async def connector_sync(session_manager=session_manager,jwt_token=jwt_token,)-+if not existing_file_ids and not existing_filenames:return JSONResponse({@@ -218,7 +268,7 @@ async def connector_sync(},status_code=200,)-+# If we have document_ids (connector file IDs), use sync_specific_files# Otherwise, use filename filtering with sync_connector_filesif existing_file_ids:@@ -602,6 +652,272 @@ async def connector_disconnect()+# ---------------------------------------------------------------------------+# IBM COS-specific endpoints+# ---------------------------------------------------------------------------++class IBMCOSConfigureBody(BaseModel):+ auth_mode: str # "iam" or "hmac"+ endpoint: str+ # IAM fields+ api_key: Optional[str] = None+ service_instance_id: Optional[str] = None+ auth_endpoint: Optional[str] = None+ # HMAC fields+ hmac_access_key: Optional[str] = None+ hmac_secret_key: Optional[str] = None+ # Optional bucket selection+ bucket_names: Optional[List[str]] = None+ # Optional: update an existing connection+ connection_id: Optional[str] = None+++async def ibm_cos_defaults(+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Return current IBM COS env-var defaults for pre-filling the config dialog.++ Sensitive values (API key, HMAC secret) are masked — only whether they are+ set is returned, not the actual values.+ """+ import os++ api_key = os.getenv("IBM_COS_API_KEY", "")+ service_instance_id = os.getenv("IBM_COS_SERVICE_INSTANCE_ID", "")+ endpoint = os.getenv("IBM_COS_ENDPOINT", "")+ hmac_access_key = os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID", "")+ hmac_secret_key = os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY", "")+ disable_iam = os.getenv("OPENRAG_IBM_COS_IAM_UI", "").lower() not in ("1", "true", "yes")++ # Try to read existing connection config for this user too+ connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="ibm_cos"+ )+ conn_config = {}+ if connections:+ conn_config = connections[0].config or {}++ def _pick(conn_key, env_val):+ """Prefer connection config value over env var."""+ return conn_config.get(conn_key) or env_val++ return JSONResponse({+ "api_key_set": bool(api_key or conn_config.get("api_key")),+ "service_instance_id": _pick("service_instance_id", service_instance_id),+ "endpoint": _pick("endpoint_url", endpoint),+ "hmac_access_key_set": bool(hmac_access_key or conn_config.get("hmac_access_key")),+ "hmac_secret_key_set": bool(hmac_secret_key or conn_config.get("hmac_secret_key")),+ # Return which auth mode was previously used; default to hmac when IAM is disabled+ "auth_mode": conn_config.get("auth_mode", "hmac" if (disable_iam or not (api_key or conn_config.get("api_key"))) else "iam"),+ "disable_iam": disable_iam,+ # Return bucket_names from existing connection (if any)+ "bucket_names": conn_config.get("bucket_names", []),+ # Return connection_id if an existing connection exists+ "connection_id": connections[0].connection_id if connections else None,+ })+++async def ibm_cos_configure(+ body: IBMCOSConfigureBody,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Create or update an IBM COS connection with explicit credentials.++ Tests the credentials by listing buckets, then persists the connection.+ Credentials are stored in the connection config dict (not env vars) so+ the connector works even without system-level env vars.+ """+ import os+ from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource++ # Build the config dict that will be stored in the connection+ conn_config: dict = {+ "auth_mode": body.auth_mode,+ "endpoint_url": body.endpoint,+ }++ if body.auth_mode == "iam":+ # Resolve: use supplied value, fall back to env var, fall back to existing connection+ api_key = body.api_key or os.getenv("IBM_COS_API_KEY")+ svc_id = body.service_instance_id or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")++ # If still empty, pull from existing connection config+ existing_connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="ibm_cos"+ )+ if not api_key and existing_connections:+ api_key = existing_connections[0].config.get("api_key")+ if not svc_id and existing_connections:+ svc_id = existing_connections[0].config.get("service_instance_id")++ if not api_key or not svc_id:+ return JSONResponse(+ {"error": "IAM mode requires api_key and service_instance_id"},+ status_code=400,+ )+ conn_config["api_key"] = api_key+ conn_config["service_instance_id"] = svc_id+ if body.auth_endpoint:+ conn_config["auth_endpoint"] = body.auth_endpoint+ else:+ # HMAC mode+ hmac_access = body.hmac_access_key or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")+ hmac_secret = body.hmac_secret_key or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")++ existing_connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="ibm_cos"+ )+ if not hmac_access and existing_connections:+ hmac_access = existing_connections[0].config.get("hmac_access_key")+ if not hmac_secret and existing_connections:+ hmac_secret = existing_connections[0].config.get("hmac_secret_key")++ if not hmac_access or not hmac_secret:+ return JSONResponse(+ {"error": "HMAC mode requires hmac_access_key and hmac_secret_key"},+ status_code=400,+ )+ conn_config["hmac_access_key"] = hmac_access+ conn_config["hmac_secret_key"] = hmac_secret++ if body.bucket_names is not None:+ conn_config["bucket_names"] = body.bucket_names++ # Test credentials — IAM uses client (avoids ibm_botocore discovery-call bug),+ # HMAC uses resource (S3-compatible, works with MinIO).+ try:+ if conn_config.get("auth_mode", "iam") == "hmac":+ cos = create_ibm_cos_resource(conn_config)+ list(cos.buckets.all())+ else:+ cos = create_ibm_cos_client(conn_config)+ cos.list_buckets()+ except Exception as exc:+ return JSONResponse(+ {"error": f"Could not connect to IBM COS: {exc}"},+ status_code=400,+ )++ # Persist: update existing connection or create a new one+ if body.connection_id:+ existing = await connector_service.connection_manager.get_connection(body.connection_id)+ if existing and existing.user_id == user.user_id:+ await connector_service.connection_manager.update_connection(+ connection_id=body.connection_id,+ config=conn_config,+ )+ # Evict cached connector so next call gets a fresh instance+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})++ # Create a fresh connection+ connection_id = await connector_service.connection_manager.create_connection(+ connector_type="ibm_cos",+ name="IBM Cloud Object Storage",+ config=conn_config,+ user_id=user.user_id,+ )+ return JSONResponse({"connection_id": connection_id, "status": "connected"})+++async def ibm_cos_list_buckets(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """List all buckets accessible with the stored IBM COS credentials."""+ from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource++ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "ibm_cos":+ return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)++ try:+ cfg = connection.config+ if cfg.get("auth_mode", "iam") == "hmac":+ cos = create_ibm_cos_resource(cfg)+ buckets = [b.name for b in cos.buckets.all()]+ else:+ cos = create_ibm_cos_client(cfg)+ buckets = [b["Name"] for b in cos.list_buckets().get("Buckets", [])]+ return JSONResponse({"buckets": buckets})+ except Exception as exc:+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)+++async def ibm_cos_bucket_status(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(get_current_user),+):+ """Return all buckets for an IBM COS connection with their ingestion status.++ Each entry includes the bucket name, whether it has been ingested (is_synced),+ and the count of indexed documents from that bucket.+ """+ from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource++ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "ibm_cos":+ return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)++ # 1. List all buckets from COS+ try:+ cfg = connection.config+ if cfg.get("auth_mode", "iam") == "hmac":+ cos = create_ibm_cos_resource(cfg)+ all_buckets = [b.name for b in cos.buckets.all()]+ else:+ cos = create_ibm_cos_client(cfg)+ all_buckets = [b["Name"] for b in cos.list_buckets().get("Buckets", [])]+ except Exception as exc:+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)++ # 2. Count indexed documents per bucket from OpenSearch+ ingested_counts: dict = {}+ try:+ opensearch_client = session_manager.get_user_opensearch_client(+ user.user_id, user.jwt_token+ )+ query_body = {+ "size": 0,+ "query": {"term": {"connector_type": "ibm_cos"}},+ "aggs": {+ "doc_ids": {+ "terms": {"field": "document_id", "size": 50000}+ }+ },+ }+ index_name = get_index_name(user.user_id)+ os_resp = opensearch_client.search(index=index_name, body=query_body)+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):+ doc_id = bucket_entry["key"]+ if "::" in doc_id:+ bucket_name = doc_id.split("::")[0]+ ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1+ except Exception:+ pass # OpenSearch unavailable — show zero counts++ result = [+ {+ "name": bucket,+ "ingested_count": ingested_counts.get(bucket, 0),+ "is_synced": ingested_counts.get(bucket, 0) > 0,+ }+ for bucket in all_buckets+ ]+ return JSONResponse({"buckets": result})+++# ---------------------------------------------------------------------------+async def sync_all_connectors(connector_service=Depends(get_connector_service),session_manager=Depends(get_session_manager),@@ -615,7 +931,7 @@ async def sync_all_connectors(jwt_token = user.jwt_token# Cloud connector types to sync- cloud_connector_types = ["google_drive", "onedrive", "sharepoint"]+ cloud_connector_types = ["google_drive", "onedrive", "sharepoint", "ibm_cos"]all_task_ids = []synced_connectors = []diff --git a/src/connectors/__init__.py b/src/connectors/__init__.pyindex 83e77b8cd..a49dff219 100644--- a/src/connectors/__init__.py+++ b/src/connectors/__init__.py@@ -2,10 +2,12 @@ from .base import BaseConnectorfrom .google_drive import GoogleDriveConnectorfrom .sharepoint import SharePointConnectorfrom .onedrive import OneDriveConnector+from .ibm_cos import IBMCOSConnector__all__ = ["BaseConnector","GoogleDriveConnector","SharePointConnector","OneDriveConnector",+ "IBMCOSConnector",]diff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.pyindex fd207be6e..35016e389 100644--- a/src/connectors/connection_manager.py+++ b/src/connectors/connection_manager.py@@ -13,6 +13,7 @@ from .base import BaseConnectorfrom .google_drive import GoogleDriveConnectorfrom .sharepoint import SharePointConnectorfrom .onedrive import OneDriveConnector+from .ibm_cos import IBMCOSConnector@dataclass@@ -351,6 +352,12 @@ class ConnectionManager:"icon": OneDriveConnector.CONNECTOR_ICON,"available": self._is_connector_available("onedrive"),},+ "ibm_cos": {+ "name": IBMCOSConnector.CONNECTOR_NAME,+ "description": IBMCOSConnector.CONNECTOR_DESCRIPTION,+ "icon": IBMCOSConnector.CONNECTOR_ICON,+ "available": self._is_connector_available("ibm_cos"),+ },}def _is_connector_available(self, connector_type: str) -> bool:@@ -379,6 +386,8 @@ class ConnectionManager:return SharePointConnector(config.config)elif config.connector_type == "onedrive":return OneDriveConnector(config.config)+ elif config.connector_type == "ibm_cos":+ return IBMCOSConnector(config.config)elif config.connector_type == "box":raise NotImplementedError("Box connector not implemented yet")elif config.connector_type == "dropbox":diff --git a/src/connectors/ibm_cos/__init__.py b/src/connectors/ibm_cos/__init__.pynew file mode 100644index 000000000..32f19defc--- /dev/null+++ b/src/connectors/ibm_cos/__init__.py@@ -0,0 +1,3 @@+from .connector import IBMCOSConnector++__all__ = ["IBMCOSConnector"]diff --git a/src/connectors/ibm_cos/auth.py b/src/connectors/ibm_cos/auth.pynew file mode 100644index 000000000..54012d724--- /dev/null+++ b/src/connectors/ibm_cos/auth.py@@ -0,0 +1,183 @@+"""IBM Cloud Object Storage authentication and client factory."""++import os+from typing import Dict, Any++from utils.logging_config import get_logger++logger = get_logger(__name__)++# IAM auth endpoint default+_DEFAULT_AUTH_ENDPOINT = "https://iam.cloud.ibm.com/identity/token"+++def _resolve_credentials(config: Dict[str, Any]):+ """Resolve IBM COS credentials from config dict → environment variable fallback.++ Returns a dict with the resolved values needed to build a boto3 client/resource.+ Raises ValueError if neither IAM nor HMAC credentials are available.+ """+ endpoint_url = config.get("endpoint_url") or os.getenv("IBM_COS_ENDPOINT")+ if not endpoint_url:+ raise ValueError(+ "IBM COS endpoint URL is required. Set IBM_COS_ENDPOINT or provide "+ "'endpoint_url' in the connector config."+ )++ api_key = config.get("api_key") or os.getenv("IBM_COS_API_KEY")+ service_instance_id = (+ config.get("service_instance_id") or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")+ )+ hmac_access_key = (+ config.get("hmac_access_key") or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")+ )+ hmac_secret_key = (+ config.get("hmac_secret_key") or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")+ )+ auth_endpoint = (+ config.get("auth_endpoint")+ or os.getenv("IBM_COS_AUTH_ENDPOINT")+ or _DEFAULT_AUTH_ENDPOINT+ )++ return {+ "endpoint_url": endpoint_url,+ "api_key": api_key,+ "service_instance_id": service_instance_id,+ "hmac_access_key": hmac_access_key,+ "hmac_secret_key": hmac_secret_key,+ "auth_endpoint": auth_endpoint,+ }+++def _build_resource(config: Dict[str, Any], creds: Dict[str, Any]):+ """Build an S3-compatible resource using resolved credentials.++ HMAC mode uses standard boto3 (no IBM-specific calls, pure S3 protocol).+ IAM mode uses ibm_boto3 with OAuth signature.+ """+ auth_mode = config.get("auth_mode", "iam")++ if auth_mode == "hmac":+ if not (creds["hmac_access_key"] and creds["hmac_secret_key"]):+ raise ValueError(+ "HMAC mode requires hmac_access_key and hmac_secret_key."+ )+ try:+ import boto3+ except ImportError as exc:+ raise ImportError(+ "boto3 is required for IBM COS HMAC mode. "+ "Install it with: pip install boto3"+ ) from exc+ logger.debug("Creating IBM COS resource with HMAC authentication (boto3)")+ return boto3.resource(+ "s3",+ aws_access_key_id=creds["hmac_access_key"],+ aws_secret_access_key=creds["hmac_secret_key"],+ endpoint_url=creds["endpoint_url"],+ )++ # IAM mode (default) — requires ibm_boto3 for OAuth token handling+ try:+ import ibm_boto3+ from ibm_botocore.client import Config+ except ImportError as exc:+ raise ImportError(+ "ibm-cos-sdk is required for IBM COS IAM mode. "+ "Install it with: pip install ibm-cos-sdk"+ ) from exc+ if not (creds["api_key"] and creds["service_instance_id"]):+ raise ValueError(+ "IAM mode requires api_key and service_instance_id."+ )+ logger.debug("Creating IBM COS resource with IAM authentication (ibm_boto3)")+ return ibm_boto3.resource(+ "s3",+ ibm_api_key_id=creds["api_key"],+ ibm_service_instance_id=creds["service_instance_id"],+ ibm_auth_endpoint=creds["auth_endpoint"],+ config=Config(signature_version="oauth"),+ endpoint_url=creds["endpoint_url"],+ )+++def _build_client(config: Dict[str, Any], creds: Dict[str, Any]):+ """Build an S3-compatible client using resolved credentials.++ HMAC mode uses standard boto3 (no IBM-specific calls, pure S3 protocol).+ IAM mode uses ibm_boto3 with OAuth signature.+ """+ auth_mode = config.get("auth_mode", "iam")++ if auth_mode == "hmac":+ if not (creds["hmac_access_key"] and creds["hmac_secret_key"]):+ raise ValueError(+ "HMAC mode requires hmac_access_key and hmac_secret_key."+ )+ try:+ import boto3+ except ImportError as exc:+ raise ImportError(+ "boto3 is required for IBM COS HMAC mode. "+ "Install it with: pip install boto3"+ ) from exc+ logger.debug("Creating IBM COS client with HMAC authentication (boto3)")+ return boto3.client(+ "s3",+ aws_access_key_id=creds["hmac_access_key"],+ aws_secret_access_key=creds["hmac_secret_key"],+ endpoint_url=creds["endpoint_url"],+ )++ # IAM mode (default) — requires ibm_boto3 for OAuth token handling+ try:+ import ibm_boto3+ from ibm_botocore.client import Config+ except ImportError as exc:+ raise ImportError(+ "ibm-cos-sdk is required for IBM COS IAM mode. "+ "Install it with: pip install ibm-cos-sdk"+ ) from exc+ if not (creds["api_key"] and creds["service_instance_id"]):+ raise ValueError(+ "IAM mode requires api_key and service_instance_id."+ )+ logger.debug("Creating IBM COS client with IAM authentication (ibm_boto3)")+ return ibm_boto3.client(+ "s3",+ ibm_api_key_id=creds["api_key"],+ ibm_service_instance_id=creds["service_instance_id"],+ ibm_auth_endpoint=creds["auth_endpoint"],+ config=Config(signature_version="oauth"),+ endpoint_url=creds["endpoint_url"],+ )+++def create_ibm_cos_resource(config: Dict[str, Any]):+ """Return an S3 resource handle (high-level API).++ HMAC mode returns a standard boto3.resource (pure S3, no IBM discovery calls).+ IAM mode returns an ibm_boto3.resource (OAuth token handling).++ Auth mode is determined by config["auth_mode"]:+ - "iam" (default): IBM_COS_API_KEY + IBM_COS_SERVICE_INSTANCE_ID+ - "hmac": IBM_COS_HMAC_ACCESS_KEY_ID + IBM_COS_HMAC_SECRET_ACCESS_KEY++ Resolution order for each credential: config dict → environment variable.+ """+ creds = _resolve_credentials(config)+ return _build_resource(config, creds)+++def create_ibm_cos_client(config: Dict[str, Any]):+ """Return an S3 low-level client.++ HMAC mode returns a standard boto3.client (pure S3, no IBM discovery calls).+ IAM mode returns an ibm_boto3.client (OAuth token handling).++ Used by API endpoints that need raw client operations (e.g. get_object_acl).+ For bucket/object listing and download, prefer create_ibm_cos_resource().+ """+ creds = _resolve_credentials(config)+ return _build_client(config, creds)diff --git a/src/connectors/ibm_cos/connector.py b/src/connectors/ibm_cos/connector.pynew file mode 100644index 000000000..4a41a5b7d--- /dev/null+++ b/src/connectors/ibm_cos/connector.py@@ -0,0 +1,368 @@+"""IBM Cloud Object Storage connector for OpenRAG."""++import mimetypes+import os+from datetime import datetime, timezone+from posixpath import basename+from typing import Any, Dict, List, Optional++from connectors.base import BaseConnector, ConnectorDocument, DocumentACL+from utils.logging_config import get_logger++from .auth import create_ibm_cos_client, create_ibm_cos_resource++logger = get_logger(__name__)++# Separator used in composite file IDs: "<bucket>::<key>"+_ID_SEPARATOR = "::"+++def _make_file_id(bucket: str, key: str) -> str:+ return f"{bucket}{_ID_SEPARATOR}{key}"+++def _split_file_id(file_id: str):+ """Split a composite file ID into (bucket, key). Raises ValueError if invalid."""+ if _ID_SEPARATOR not in file_id:+ raise ValueError(f"Invalid IBM COS file ID (missing separator): {file_id!r}")+ bucket, key = file_id.split(_ID_SEPARATOR, 1)+ return bucket, key+++class IBMCOSConnector(BaseConnector):+ """Connector for IBM Cloud Object Storage.++ Supports IAM (API key) and HMAC credential modes. Credentials are read+ from the connector config dict first, then from environment variables.++ Config dict keys:+ bucket_names (list[str]): Buckets to ingest from. Required.+ prefix (str): Optional object key prefix filter.+ endpoint_url (str): Overrides IBM_COS_ENDPOINT.+ api_key (str): Overrides IBM_COS_API_KEY.+ service_instance_id (str): Overrides IBM_COS_SERVICE_INSTANCE_ID.+ hmac_access_key (str): HMAC mode – overrides IBM_COS_HMAC_ACCESS_KEY_ID.+ hmac_secret_key (str): HMAC mode – overrides IBM_COS_HMAC_SECRET_ACCESS_KEY.+ connection_id (str): Connection identifier used for logging.+ """++ CONNECTOR_NAME = "IBM Cloud Object Storage"+ CONNECTOR_DESCRIPTION = "Add knowledge from IBM Cloud Object Storage"+ CONNECTOR_ICON = "ibm-cos"++ # BaseConnector uses these to check env-var availability for IAM mode.+ # HMAC-only setups will show as "unavailable" in the UI but can still be+ # used when credentials are supplied in the config dict directly.+ CLIENT_ID_ENV_VAR = "IBM_COS_API_KEY"+ CLIENT_SECRET_ENV_VAR = "IBM_COS_SERVICE_INSTANCE_ID"++ def get_client_id(self) -> str:+ """Return IAM API key, or HMAC access key ID as fallback."""+ val = os.getenv("IBM_COS_API_KEY") or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")+ if val:+ return val+ raise ValueError(+ "IBM COS credentials not set. Provide IBM_COS_API_KEY (IAM) "+ "or IBM_COS_HMAC_ACCESS_KEY_ID (HMAC)."+ )++ def get_client_secret(self) -> str:+ """Return IAM service instance ID, or HMAC secret key as fallback."""+ val = os.getenv("IBM_COS_SERVICE_INSTANCE_ID") or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")+ if val:+ return val+ raise ValueError(+ "IBM COS credentials not set. Provide IBM_COS_SERVICE_INSTANCE_ID (IAM) "+ "or IBM_COS_HMAC_SECRET_ACCESS_KEY (HMAC)."+ )++ def __init__(self, config: Dict[str, Any]):+ if config is None:+ config = {}+ super().__init__(config)++ self.bucket_names: List[str] = config.get("bucket_names") or []+ self.prefix: str = config.get("prefix", "")+ self.connection_id: str = config.get("connection_id", "default")++ # Resolved service instance ID used as ACL owner fallback+ self._service_instance_id: str = (+ config.get("service_instance_id")+ or os.getenv("IBM_COS_SERVICE_INSTANCE_ID", "")+ )++ self._handle = None # Lazy-initialised on first use+ # IAM mode uses ibm_boto3.client to avoid internal service-instance+ # discovery calls that cause XML-parse errors against the real IBM COS API.+ # HMAC mode uses ibm_boto3.resource (confirmed working with MinIO and S3).+ self._is_hmac: bool = (config.get("auth_mode", "iam") == "hmac")++ def _get_handle(self):+ """Return (and cache) the appropriate boto3 handle for the configured auth mode.++ - HMAC → ibm_boto3.resource (S3-compatible, works with MinIO)+ - IAM → ibm_boto3.client (avoids ibm_botocore service-discovery calls+ that break against the real IBM COS API)+ """+ if self._handle is None:+ if self._is_hmac:+ self._handle = create_ibm_cos_resource(self.config)+ else:+ self._handle = create_ibm_cos_client(self.config)+ return self._handle++ # ------------------------------------------------------------------+ # BaseConnector abstract method implementations+ # ------------------------------------------------------------------++ async def authenticate(self) -> bool:+ """Validate credentials by listing buckets on the COS service."""+ try:+ handle = self._get_handle()+ if self._is_hmac:+ list(handle.buckets.all()) # resource API+ else:+ handle.list_buckets() # client API+ self._authenticated = True+ logger.debug(f"IBM COS authenticated for connection {self.connection_id}")+ return True+ except Exception as exc:+ logger.warning(f"IBM COS authentication failed: {exc}")+ self._authenticated = False+ return False++ def _resolve_bucket_names(self) -> List[str]:+ """Return configured bucket names, or auto-discover all accessible buckets."""+ if self.bucket_names:+ return self.bucket_names+ try:+ handle = self._get_handle()+ if self._is_hmac:+ buckets = [b.name for b in handle.buckets.all()]+ else:+ resp = handle.list_buckets()+ buckets = [b["Name"] for b in resp.get("Buckets", [])]+ logger.debug(f"IBM COS auto-discovered {len(buckets)} bucket(s): {buckets}")+ return buckets+ except Exception as exc:+ logger.warning(f"IBM COS could not auto-discover buckets: {exc}")+ return []++ async def list_files(+ self,+ page_token: Optional[str] = None,+ max_files: Optional[int] = None,+ **kwargs,+ ) -> Dict[str, Any]:+ """List objects across all configured (or auto-discovered) buckets.++ Uses the ibm_boto3 resource API: Bucket.objects.all() handles pagination+ internally so all objects are returned without manual continuation tokens.++ If no bucket_names are configured, all accessible buckets are used.++ Returns:+ dict with keys:+ "files": list of file dicts (id, name, bucket, size, modified_time)+ "next_page_token": always None (SDK handles pagination internally)+ """+ handle = self._get_handle()+ files: List[Dict[str, Any]] = []+ bucket_names = self._resolve_bucket_names()++ for bucket_name in bucket_names:+ try:+ if self._is_hmac:+ # resource API: Bucket.objects.all() handles pagination internally+ bucket = handle.Bucket(bucket_name)+ objects = (+ bucket.objects.filter(Prefix=self.prefix)+ if self.prefix+ else bucket.objects.all()+ )+ for obj in objects:+ if obj.key.endswith("/"):+ continue+ files.append(+ {+ "id": _make_file_id(bucket_name, obj.key),+ "name": basename(obj.key) or obj.key,+ "bucket": bucket_name,+ "key": obj.key,+ "size": obj.size,+ "modified_time": obj.last_modified.isoformat()+ if obj.last_modified+ else None,+ }+ )+ if max_files and len(files) >= max_files:+ return {"files": files, "next_page_token": None}+ else:+ # client API: list_objects_v2 with manual pagination+ kwargs: Dict[str, Any] = {"Bucket": bucket_name}+ if self.prefix:+ kwargs["Prefix"] = self.prefix+ while True:+ resp = handle.list_objects_v2(**kwargs)+ for obj in resp.get("Contents", []):+ key = obj["Key"]+ if key.endswith("/"):+ continue+ files.append(+ {+ "id": _make_file_id(bucket_name, key),+ "name": basename(key) or key,+ "bucket": bucket_name,+ "key": key,+ "size": obj.get("Size", 0),+ "modified_time": obj["LastModified"].isoformat()+ if obj.get("LastModified")+ else None,+ }+ )+ if max_files and len(files) >= max_files:+ return {"files": files, "next_page_token": None}+ if resp.get("IsTruncated"):+ kwargs["ContinuationToken"] = resp["NextContinuationToken"]+ else:+ break++ except Exception as exc:+ logger.error(f"Failed to list objects in bucket {bucket_name!r}: {exc}")+ continue++ return {"files": files, "next_page_token": None}++ async def get_file_content(self, file_id: str) -> ConnectorDocument:+ """Download an object from IBM COS and return a ConnectorDocument.++ Uses the ibm_boto3 resource API: Object.get() downloads content and+ returns all metadata (ContentType, ContentLength, LastModified) in one call.++ Args:+ file_id: Composite ID in the form "<bucket>::<key>".++ Returns:+ ConnectorDocument with content bytes, ACL, and metadata.+ """+ bucket_name, key = _split_file_id(file_id)+ handle = self._get_handle()++ # Both client.get_object() and resource.Object().get() return the same+ # response dict: Body stream + ContentType, ContentLength, LastModified.+ if self._is_hmac:+ response = handle.Object(bucket_name, key).get() # resource+ else:+ response = handle.get_object(Bucket=bucket_name, Key=key) # client+ content: bytes = response["Body"].read()++ last_modified: datetime = response.get("LastModified") or datetime.now(timezone.utc)+ size: int = response.get("ContentLength", len(content))++ # MIME type detection: prefer filename extension over generic S3 content-type.+ # IBM COS often stores "application/octet-stream" for all objects regardless+ # of their real type, so we treat that as "unknown" and fall back to the+ # extension-based guess which is more reliable for named files.+ raw_content_type = response.get("ContentType", "")+ if raw_content_type and raw_content_type != "application/octet-stream":+ mime_type: str = raw_content_type+ else:+ mime_type = mimetypes.guess_type(key)[0] or "application/octet-stream"++ filename = basename(key) or key++ acl = await self._extract_acl(bucket_name, key)++ return ConnectorDocument(+ id=file_id,+ filename=filename,+ mimetype=mime_type,+ content=content,+ source_url=f"cos://{bucket_name}/{key}",+ acl=acl,+ modified_time=last_modified,+ created_time=last_modified, # IBM COS does not expose creation time+ metadata={+ "ibm_cos_bucket": bucket_name,+ "ibm_cos_key": key,+ "size": size,+ },+ )++ async def _extract_acl(self, bucket: str, key: str) -> DocumentACL:+ """Fetch object ACL from IBM COS and map it to DocumentACL.++ Falls back to a minimal ACL (owner = service instance ID) on failure.+ """+ try:+ handle = self._get_handle()+ # For resource (HMAC), access the underlying client via meta.client.+ # For client (IAM), call directly.+ client = handle.meta.client if self._is_hmac else handle+ acl_response = client.get_object_acl(Bucket=bucket, Key=key)++ owner_id: str = (+ acl_response.get("Owner", {}).get("DisplayName")+ or acl_response.get("Owner", {}).get("ID")+ or self._service_instance_id+ )++ allowed_users: List[str] = []+ for grant in acl_response.get("Grants", []):+ grantee = grant.get("Grantee", {})+ permission = grant.get("Permission", "")+ if permission in ("FULL_CONTROL", "READ"):+ user_id = (+ grantee.get("DisplayName")+ or grantee.get("ID")+ or grantee.get("EmailAddress")+ )+ if user_id and user_id not in allowed_users:+ allowed_users.append(user_id)++ return DocumentACL(+ owner=owner_id,+ allowed_users=allowed_users,+ allowed_groups=[],+ )+ except Exception as exc:+ logger.warning(+ f"Could not fetch ACL for cos://{bucket}/{key}: {exc}. "+ "Using fallback ACL."+ )+ return DocumentACL(+ owner=self._service_instance_id or None,+ allowed_users=[],+ allowed_groups=[],+ )++ # ------------------------------------------------------------------+ # Webhook / subscription (stub — IBM COS events require IBM Event+ # Notifications service; not in scope for this connector version)+ # ------------------------------------------------------------------++ async def setup_subscription(self) -> str:+ """No-op: IBM COS event notifications are out of scope for this connector."""+ return ""++ async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:+ """No-op: webhooks are not supported in this connector version."""+ return []++ def extract_webhook_channel_id(+ self, payload: Dict[str, Any], headers: Dict[str, str]+ ) -> Optional[str]:+ return None++ async def cleanup_subscription(self, subscription_id: str) -> bool:+ """No-op: no subscription to clean up."""+ return True+++++if __name__ == "__main__":+ connector = IBMCOSConnector({})+ print(connector.authenticate())+ print(connector.list_files())+ # print(connector.get_file_content("test_cos.py"))\ No newline at end of filediff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex f80df0b02..1c981ba5a 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -331,39 +331,40 @@ class LangflowConnectorService:original_folder_ids = getattr(cfg, "folder_ids", None)expanded_file_ids = file_ids # Default to original IDs-- try:- # Set the file_ids we want to sync in the connector's config- if cfg is not None:++ # Only attempt folder expansion for connectors that use cfg-based filtering+ # (Google Drive, OneDrive, SharePoint). Connectors without a cfg attribute+ # (e.g. IBM COS) receive pre-filtered file IDs and must NOT call list_files()+ # here — doing so would re-list all files from all buckets, overwriting the+ # carefully selected IDs passed in.+ if cfg is not None:+ try:cfg.file_ids = file_ids # type: ignorecfg.folder_ids = None # type: ignore- # Get the expanded list of file IDs (folders will be expanded to their contents)- # This uses the connector's list_files() which calls _iter_selected_items()- result = await connector.list_files()- expanded_file_ids = [f["id"] for f in result.get("files", [])]+ # Expand file IDs — folders become their individual file contents+ result = await connector.list_files()+ expanded_file_ids = [f["id"] for f in result.get("files", [])]++ if not expanded_file_ids:+ logger.warning(+ f"No files found after expanding file_ids. "+ f"Original IDs: {file_ids}. This may indicate all IDs were folders "+ f"with no contents, or files that were filtered out."+ )+ # If we have file_infos with download URLs, use original file_ids+ # (OneDrive sharing IDs can't be expanded but can be downloaded directly)+ if file_infos:+ logger.info("Using original file IDs with cached download URLs")+ expanded_file_ids = file_ids+ else:+ raise ValueError("No files to sync after expanding folders")- if not expanded_file_ids:- logger.warning(- f"No files found after expanding file_ids. "- f"Original IDs: {file_ids}. This may indicate all IDs were folders "- f"with no contents, or files that were filtered out."- )- # If we have file_infos with download URLs, use original file_ids- # (OneDrive sharing IDs can't be expanded but can be downloaded directly)- if file_infos:- logger.info("Using original file IDs with cached download URLs")- expanded_file_ids = file_ids- else:- raise ValueError("No files to sync after expanding folders")-- except Exception as e:- logger.error(f"Failed to expand file_ids via list_files(): {e}")- # Fallback to original file_ids if expansion fails- expanded_file_ids = file_ids- finally:- # Restore original config values- if cfg is not None:+ except Exception as e:+ logger.error(f"Failed to expand file_ids via list_files(): {e}")+ # Fallback to original file_ids if expansion fails+ expanded_file_ids = file_ids+ finally:cfg.file_ids = original_file_ids # type: ignorecfg.folder_ids = original_folder_ids # type: ignorediff --git a/src/main.py b/src/main.pyindex 598263f78..016a86ff7 100644--- a/src/main.py+++ b/src/main.py@@ -762,6 +762,11 @@ async def create_app():# Connector endpointsapp.add_api_route("/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"])+ # IBM COS-specific routes (registered before generic /{connector_type}/... to avoid shadowing)+ app.add_api_route("/connectors/ibm_cos/defaults", connectors.ibm_cos_defaults, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/configure", connectors.ibm_cos_configure, methods=["POST"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", connectors.ibm_cos_list_buckets, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", connectors.ibm_cos_bucket_status, methods=["GET"], tags=["internal"])app.add_api_route("/connectors/{connector_type}/sync", connectors.connector_sync, methods=["POST"], tags=["internal"])app.add_api_route("/connectors/sync-all", connectors.sync_all_connectors, methods=["POST"], tags=["internal"])app.add_api_route("/connectors/{connector_type}/status", connectors.connector_status, methods=["GET"], tags=["internal"])diff --git a/src/services/auth_service.py b/src/services/auth_service.pyindex d2fe43971..277638c6a 100644--- a/src/services/auth_service.py+++ b/src/services/auth_service.py@@ -1,4 +1,3 @@-import osimport uuidimport jsonimport httpx@@ -20,6 +19,9 @@ from connectors.google_drive import GoogleDriveConnectorfrom connectors.onedrive import OneDriveConnectorfrom connectors.sharepoint import SharePointConnector+# Connectors that authenticate directly (no OAuth redirect required)+_DIRECT_AUTH_CONNECTORS = {"ibm_cos"}+class AuthService:def __init__(self, session_manager: SessionManager, connector_service=None, langflow_mcp_service: LangflowMCPService | None = None):@@ -56,6 +58,7 @@ class AuthService:"google_drive","onedrive","sharepoint",+ "ibm_cos",]:raise ValueError(f"Unsupported connector type: {connector_type}")elif purpose not in ["app_auth", "data_source"]:@@ -91,6 +94,10 @@ class AuthService:))+ # Direct-auth connectors (HMAC/API-key based, no OAuth redirect)+ if connector_type in _DIRECT_AUTH_CONNECTORS:+ return await self._init_direct_connection(connector_type, connection_id)+# Get OAuth configuration from connector and OAuth classesimport os@@ -147,6 +154,61 @@ class AuthService:return {"connection_id": connection_id, "oauth_config": oauth_config}+ async def _init_direct_connection(+ self, connector_type: str, connection_id: str+ ) -> dict:+ """Authenticate a non-OAuth connector immediately using env var credentials.++ Creates the connection record (already done by the caller) and verifies+ that the credentials work by calling authenticate() on the connector.+ Returns a response without oauth_config so the frontend knows no redirect+ is needed.+ """+ try:+ connection_config = (+ await self.connector_service.connection_manager.get_connection(+ connection_id+ )+ )+ if not connection_config:+ raise ValueError("Connection not found")++ connector = self.connector_service.connection_manager._create_connector(+ connection_config+ )+ authenticated = await connector.authenticate()+ if not authenticated:+ # Remove the connection so the user can retry after fixing credentials+ await self.connector_service.connection_manager.delete_connection(+ connection_id+ )+ raise ValueError(+ f"Could not authenticate with {connector_type}. "+ "Check that your credentials and endpoint are correct."+ )++ # Cache the authenticated connector+ self.connector_service.connection_manager.active_connectors[+ connection_id+ ] = connector++ except ValueError:+ raise+ except Exception as exc:+ await self.connector_service.connection_manager.delete_connection(+ connection_id+ )+ raise ValueError(+ f"Failed to connect {connector_type}: {exc}"+ ) from exc++ return {+ "connection_id": connection_id,+ "status": "connected",+ "connector_type": connector_type,+ # No oauth_config — frontend must not attempt an OAuth redirect+ }+async def handle_oauth_callback(self,connection_id: str,@@ -414,7 +476,7 @@ class AuthService:else:logger.warning("_handle_data_source_auth: _detect_base_url returned None")else:- logger.warning(f"_handle_data_source_auth: Connector not available or doesn't have _detect_base_url")+ logger.warning("_handle_data_source_auth: Connector not available or doesn't have _detect_base_url")# Clear the cached connector so next get_connector() creates a fresh instance# with the updated config (including base_url)diff --git a/src/tui/config_fields.py b/src/tui/config_fields.pyindex c13e2e126..6a78a883e 100644--- a/src/tui/config_fields.py+++ b/src/tui/config_fields.py@@ -183,6 +183,36 @@ CONFIG_SECTIONS: list[ConfigSection] = [),], advanced=True, gate_prompt="Configure AWS credentials?"),+ # ── IBM Cloud Object Storage ─────────────────────────────────+ ConfigSection("IBM Cloud Object Storage", [+ ConfigField(+ "ibm_cos_api_key", "IBM_COS_API_KEY", "API Key",+ placeholder="",+ helper_text="Create API key at https://cloud.ibm.com/iam/apikeys",+ secret=True,+ ),+ ConfigField(+ "ibm_cos_service_instance_id", "IBM_COS_SERVICE_INSTANCE_ID",+ "Service Instance ID (CRN)",+ placeholder="crn:v1:bluemix:...",+ ),+ ConfigField(+ "ibm_cos_endpoint", "IBM_COS_ENDPOINT", "Service Endpoint",+ placeholder="https://s3.us-south.cloud-object-storage.appdomain.cloud",+ helper_text="Endpoints: https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-endpoints",+ ),+ ConfigField(+ "ibm_cos_hmac_access_key_id", "IBM_COS_HMAC_ACCESS_KEY_ID",+ "HMAC Access Key ID (optional)",+ placeholder="",+ ),+ ConfigField(+ "ibm_cos_hmac_secret_access_key", "IBM_COS_HMAC_SECRET_ACCESS_KEY",+ "HMAC Secret Access Key (optional)",+ placeholder="", secret=True,+ ),+ ], advanced=True, gate_prompt="Configure IBM Cloud Object Storage?"),+# ── Langfuse ────────────────────────────────────────────────ConfigSection("Langfuse", [ConfigField(diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.pyindex b940f2a0e..b16c38af2 100644--- a/src/tui/managers/env_manager.py+++ b/src/tui/managers/env_manager.py@@ -59,6 +59,13 @@ class EnvConfig:aws_secret_access_key: str = ""langflow_public_url: str = ""+ # IBM Cloud Object Storage settings+ ibm_cos_api_key: str = ""+ ibm_cos_service_instance_id: str = ""+ ibm_cos_endpoint: str = ""+ ibm_cos_hmac_access_key_id: str = ""+ ibm_cos_hmac_secret_access_key: str = ""+# Langfuse settings (optional)langfuse_secret_key: str = ""langfuse_public_key: str = ""@@ -194,6 +201,11 @@ class EnvManager:"AWS_ACCESS_KEY_ID": "aws_access_key_id","AWS_SECRET_ACCESS_KEY": "aws_secret_access_key", # pragma: allowlist secret"LANGFLOW_PUBLIC_URL": "langflow_public_url",+ "IBM_COS_API_KEY": "ibm_cos_api_key", # pragma: allowlist secret+ "IBM_COS_SERVICE_INSTANCE_ID": "ibm_cos_service_instance_id",+ "IBM_COS_ENDPOINT": "ibm_cos_endpoint",+ "IBM_COS_HMAC_ACCESS_KEY_ID": "ibm_cos_hmac_access_key_id",+ "IBM_COS_HMAC_SECRET_ACCESS_KEY": "ibm_cos_hmac_secret_access_key", # pragma: allowlist secret"OPENRAG_DOCUMENTS_PATHS": "openrag_documents_paths","OPENRAG_DOCUMENTS_PATH": "openrag_documents_path","OPENRAG_KEYS_PATH": "openrag_keys_path",@@ -521,6 +533,11 @@ class EnvManager:("AWS_ACCESS_KEY_ID", self.config.aws_access_key_id),("AWS_SECRET_ACCESS_KEY", self.config.aws_secret_access_key),("LANGFLOW_PUBLIC_URL", self.config.langflow_public_url),+ ("IBM_COS_API_KEY", self.config.ibm_cos_api_key),+ ("IBM_COS_SERVICE_INSTANCE_ID", self.config.ibm_cos_service_instance_id),+ ("IBM_COS_ENDPOINT", self.config.ibm_cos_endpoint),+ ("IBM_COS_HMAC_ACCESS_KEY_ID", self.config.ibm_cos_hmac_access_key_id),+ ("IBM_COS_HMAC_SECRET_ACCESS_KEY", self.config.ibm_cos_hmac_secret_access_key),]optional_written = Falsediff --git a/src/utils/file_utils.py b/src/utils/file_utils.pyindex 764f90694..44d0bb98e 100644--- a/src/utils/file_utils.py+++ b/src/utils/file_utils.py@@ -61,7 +61,7 @@ def safe_unlink(path: str) -> None:def get_file_extension(mimetype: str) -> str:- """Get file extension based on MIME type"""+ """Get file extension based on MIME type. Returns None if the type is unknown."""mime_to_ext = {"application/pdf": ".pdf","application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",@@ -69,19 +69,33 @@ def get_file_extension(mimetype: str) -> str:"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx","application/vnd.ms-powerpoint": ".ppt","text/plain": ".txt",+ "text/markdown": ".md",+ "text/x-markdown": ".md","text/html": ".html",+ "text/csv": ".csv",+ "application/json": ".json",+ "application/xml": ".xml",+ "text/xml": ".xml","application/rtf": ".rtf","application/vnd.google-apps.document": ".pdf", # Exported as PDF"application/vnd.google-apps.presentation": ".pdf","application/vnd.google-apps.spreadsheet": ".pdf",}- return mime_to_ext.get(mimetype, ".bin")+ return mime_to_ext.get(mimetype)def clean_connector_filename(filename: str, mimetype: str) -> str:- """Clean filename and ensure correct extension"""- suffix = get_file_extension(mimetype)+ """Clean filename and ensure correct extension.++ If the MIME type maps to a known extension, it is enforced.+ If the MIME type is unknown, the original filename (and its extension) is kept as-is+ rather than appending a meaningless .bin suffix.+ """clean_name = filename.replace(" ", "_").replace("/", "_")+ suffix = get_file_extension(mimetype)+ if suffix is None:+ # Unknown type — keep whatever extension the file already has+ return clean_nameif not clean_name.lower().endswith(suffix.lower()):return clean_name + suffixreturn clean_name\ No newline at end of filediff --git a/uv.lock b/uv.lockindex 8b98c160b..56f6a9646 100644--- a/uv.lock+++ b/uv.lock@@ -819,6 +819,38 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/d5/ae/2f6d96b4e6c5478d87d606a1934b5d436c4a2bce6bb7c6fdece891c128e3/huggingface_hub-1.4.1-py3-none-any.whl", hash = "sha256:9931d075fb7a79af5abc487106414ec5fba2c0ae86104c0c62fd6cae38873d18", size = 553326, upload-time = "2026-02-06T09:20:00.728Z" },]+[[package]]+name = "ibm-cos-sdk"+version = "2.16.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "ibm-cos-sdk-core" },+ { name = "ibm-cos-sdk-s3transfer" },+ { name = "jmespath" },+]+sdist = { url = "https://files.pythonhosted.org/packages/d8/6c/ffbe556fd148e58d5b8e4b1f71fce604dcf531d791a8599cc8a7a1ee2a09/ibm_cos_sdk-2.16.0.tar.gz", hash = "sha256:ef11ceb121dc5c90050e87a82b6d27e67394c58f1c7abcc991a3d9a0964a290a", size = 58939, upload-time = "2026-01-06T10:30:11.235Z" }++[[package]]+name = "ibm-cos-sdk-core"+version = "2.16.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "jmespath" },+ { name = "python-dateutil" },+ { name = "requests" },+ { name = "urllib3" },+]+sdist = { url = "https://files.pythonhosted.org/packages/26/bb/4d6908fd4fe9d1994ad6aae483e352859b4ae7e501b8743f5db5c4840124/ibm_cos_sdk_core-2.16.0.tar.gz", hash = "sha256:2707d0ca62dd6f85455c4ac02ed37aa35ea1f2b143666271237f979d4fe904f7", size = 1119533, upload-time = "2026-01-06T10:30:01.931Z" }++[[package]]+name = "ibm-cos-sdk-s3transfer"+version = "2.16.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "ibm-cos-sdk-core" },+]+sdist = { url = "https://files.pythonhosted.org/packages/b3/6f/660a6e0b7015a512304487c17c4ac87c280180b0a6142ee196570efc221e/ibm_cos_sdk_s3transfer-2.16.0.tar.gz", hash = "sha256:5cca69c48dcb7a1442b39cadb3a635162fbc2cf002820c8cd6cb170056d27c5c", size = 141138, upload-time = "2026-01-06T10:30:06.375Z" }+[[package]]name = "idna"version = "3.11"@@ -914,11 +946,11 @@ wheels = [[[package]]name = "jmespath"-version = "1.1.0"+version = "1.0.1"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" }+sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },+ { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },][[package]]@@ -1265,6 +1297,7 @@ dependencies = [{ name = "google-auth-httplib2" },{ name = "google-auth-oauthlib" },{ name = "httpx" },+ { name = "ibm-cos-sdk" },{ name = "msal" },{ name = "opensearch-py", extra = ["async"] },{ name = "psutil" },@@ -1298,6 +1331,7 @@ requires-dist = [{ name = "google-auth-httplib2", specifier = ">=0.2.0" },{ name = "google-auth-oauthlib", specifier = ">=1.2.0" },{ name = "httpx", specifier = ">=0.27.0" },+ { name = "ibm-cos-sdk", specifier = ">=2.13.0" },{ name = "msal", specifier = ">=1.29.0" },{ name = "opensearch-py", extras = ["async"], specifier = ">=3.0.0" },{ name = "psutil", specifier = ">=7.0.0" },