Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

13.6 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package storage

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"net/http"
	"ragflow/internal/common"
	"ragflow/internal/server/config"
	"time"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"go.uber.org/zap"
)

// MinioStorage implements Storage interface for MinIO
type MinioStorage struct {
	client     *minio.Client
	bucket     string // default bucket
	prefixPath string // default prefix path
	config     config.MinioConfig
}

// NewMinioStorage creates a new MinIO storage instance
func NewMinioStorage(config config.MinioConfig) (*MinioStorage, error) {
	storage := &MinioStorage{
		bucket:     config.Bucket,
		prefixPath: config.PrefixPath,
		config:     config,
	}

	if err := storage.connect(); err != nil {
		return nil, err
	}

	return storage, nil
}

func newMinioTransport(cfg config.MinioConfig) (*http.Transport, error) {
	transport, err := minio.DefaultTransport(cfg.Secure)
	if err != nil {
		return nil, fmt.Errorf("failed to create default transport: %w", err)
	}

	if cfg.Secure {
		if transport.TLSClientConfig == nil {
			transport.TLSClientConfig = &tls.Config{}
		}
		transport.TLSClientConfig.InsecureSkipVerify = !cfg.Verify
	}

	return transport, nil
}

func (m *MinioStorage) connect() error {
	transport, err := newMinioTransport(m.config)
	if err != nil {
		return fmt.Errorf("failed to create MinIO transport: %w", err)
	}

	client, err := minio.New(m.config.Host, &minio.Options{
		Creds:     credentials.NewStaticV4(m.config.User, m.config.Password, ""),
		Secure:    m.config.Secure,
		Transport: transport,
		Region:    m.config.Region,
	})
	if err != nil {
		return fmt.Errorf("failed to connect to MinIO: %w", err)
	}

	m.client = client
	return nil
}

func (m *MinioStorage) resolveBucketAndPath(bucket, fnm string) (string, string) {
	actualBucket := bucket
	if m.bucket != "" {
		actualBucket = m.bucket
	}

	actualPath := fnm
	if m.bucket != "" {
		if m.prefixPath != "" {
			actualPath = fmt.Sprintf("%s/%s/%s", m.prefixPath, bucket, fnm)
		} else {
			actualPath = fmt.Sprintf("%s/%s", bucket, fnm)
		}
	} else if m.prefixPath != "" {
		actualPath = fmt.Sprintf("%s/%s", m.prefixPath, fnm)
	}

	return actualBucket, actualPath
}

func (m *MinioStorage) Type() string { return "minio" }

// Health checks endpoint liveness with a read-only probe, not bucket permissions.
func (m *MinioStorage) Health(ctx context.Context) bool {
	probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	if probeCtx.Err() != nil {
		return false
	}

	// A missing or inaccessible probe bucket still proves the endpoint responds.
	_, err := m.client.BucketExists(probeCtx, "ragflow-health-check")
	if probeCtx.Err() != nil {
		return false
	}
	if err != nil && minio.ToErrorResponse(err).Code != minio.AccessDenied {
		common.Warn("Failed to check MinIO health", zap.Error(err))
		return false
	}

	return true
}

// Put uploads an object to MinIO
func (m *MinioStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
	bucket, fnm = m.resolveBucketAndPath(bucket, fnm)

	return retryObjectOp(ctx, 3, nil, func() error {
		// Ensure bucket exists
		if m.bucket == "" {
			exists, err := m.client.BucketExists(ctx, bucket)
			if err != nil {
				common.Warn("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err))
				return err
			}
			if !exists {
				if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
					common.Warn("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))
					return err
				}
			}
		}

		reader := bytes.NewReader(binary)
		if _, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{}); err != nil {
			const warnMessage = "Failed to put object"
			common.Warn(warnMessage, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
			return fmt.Errorf("%s: %w", warnMessage, err)
		}

		return nil
	})
}

// Get retrieves an object from MinIO
func (m *MinioStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
	bucket, fnm = m.resolveBucketAndPath(bucket, fnm)

	var data []byte
	err := retryObjectOp(ctx, 2, nil, func() error {
		obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{})
		if err != nil {
			common.Warn("failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
			return err
		}
		buf := new(bytes.Buffer)

		readErr := func() error {
			defer obj.Close()
			_, err = buf.ReadFrom(obj)
			return err
		}()
		if readErr != nil {
			common.Error("failed to read object data", err, zap.String("bucket", bucket), zap.String("key", fnm))
			return readErr
		}
		data = buf.Bytes()

		return nil
	})
	if err != nil {
		return nil, err
	}

	return data, nil
}

// Remove removes an object from MinIO
func (m *MinioStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {
	bucket, fnm = m.resolveBucketAndPath(bucket, fnm)

	if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil {
		code := minio.ToErrorResponse(err).Code
		if code == "NoSuchKey" || code == "NoSuchBucket" {
			return nil
		}
		common.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
		return err
	}

	return nil
}

// ObjExist checks if an object exists in MinIO
func (m *MinioStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {
	bucket, fnm = m.resolveBucketAndPath(bucket, fnm)

	exists, err := m.client.BucketExists(ctx, bucket)
	if err != nil || !exists {
		return false
	}

	_, err = m.client.StatObject(ctx, bucket, fnm, minio.StatObjectOptions{})
	if err != nil {
		errResponse := minio.ToErrorResponse(err)
		if errResponse.Code == "NoSuchKey" || errResponse.Code == "NoSuchBucket" {
			return false
		}
		common.Warn("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
		return false
	}

	return true
}

func (m *MinioStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
	bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
	_, err := m.client.StatObject(ctx, bucket, fnm, minio.StatObjectOptions{})
	if err == nil {
		return true, nil
	}
	code := minio.ToErrorResponse(err).Code
	if code == "NoSuchKey" || code == "NoSuchBucket" {
		return false, nil
	}
	return false, err
}

// GetPresignedURL generates a presigned URL for accessing an object
func (m *MinioStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
	bucket, fnm = m.resolveBucketAndPath(bucket, fnm)

	var url string
	err := retryObjectOp(ctx, 10, nil, func() error {
		u, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil)
		if err != nil {
			common.Warn("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
			return err
		}
		url = u.String()

		return nil
	})
	if err != nil {
		return "", fmt.Errorf("failed to get presigned URL after 10 retries: %w", err)
	}

	return url, nil
}

// BucketExists checks if a bucket exists
func (m *MinioStorage) BucketExists(ctx context.Context, bucket string) bool {
	actualBucket := bucket
	if m.bucket != "" {
		actualBucket = m.bucket
	}

	exists, err := m.client.BucketExists(ctx, actualBucket)
	if err != nil {
		common.Warn("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err))
		return false
	}

	return exists
}

func (m *MinioStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
	if m.bucket != "" {
		bucket = m.bucket
	}
	return m.client.BucketExists(ctx, bucket)
}

func (m *MinioStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {

	var objects []string
	for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{
		Prefix:    "",
		Recursive: true,
	}) {
		if obj.Err != nil {
			common.Warn("Failed to list objects", zap.Error(obj.Err))
			return nil, obj.Err
		}
		objects = append(objects, obj.Key)
	}

	return objects, nil
}

// RemoveBucket removes a bucket and all its objects
func (m *MinioStorage) RemoveBucket(ctx context.Context, bucket string) error {
	actualBucket := bucket
	origBucket := bucket

	if m.bucket != "" {
		actualBucket = m.bucket
	}
	exists, err := m.client.BucketExists(ctx, actualBucket)
	if err != nil {
		return fmt.Errorf("failed to check bucket %s: %w", actualBucket, err)
	}
	if !exists {
		return nil
	}

	// Build prefix for single-bucket mode
	prefix := ""
	if m.bucket != "" {
		if m.prefixPath != "" {
			prefix = fmt.Sprintf("%s/", m.prefixPath)
		}
		prefix += fmt.Sprintf("%s/", origBucket)
	}

	// Include versions and delete markers so versioned buckets can be emptied.
	removeCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	objectsCh := make(chan minio.ObjectInfo)
	listErrCh := make(chan error, 1)

	go func() {
		defer close(objectsCh)
		defer close(listErrCh)
		for obj := range m.client.ListObjects(removeCtx, actualBucket, minio.ListObjectsOptions{
			Prefix:       prefix,
			Recursive:    true,
			WithVersions: true,
		}) {
			if obj.Err != nil {
				common.Warn("Failed to list objects", zap.Error(obj.Err))
				listErrCh <- obj.Err
				return
			}
			select {
			case objectsCh <- obj:
			case <-removeCtx.Done():
				return
			}
		}
	}()

	var removeErr error
	for objErr := range m.client.RemoveObjects(removeCtx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) {
		common.Warn(fmt.Sprintf("Failed to remove object, key: %s", objErr.ObjectName), zap.Error(objErr.Err))
		if removeErr == nil {
			removeErr = fmt.Errorf("failed to remove object %s: %w", objErr.ObjectName, objErr.Err)
		}
	}
	cancel()
	if removeErr != nil {
		return removeErr
	}
	if err := <-listErrCh; err != nil {
		return fmt.Errorf("failed to list objects in bucket %s: %w", actualBucket, err)
	}
	if err := ctx.Err(); err != nil {
		return err
	}

	// Only remove the actual bucket if not in single-bucket mode
	if m.bucket == "" {
		if err := m.client.RemoveBucket(ctx, actualBucket); err != nil {
			common.Warn("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err))
			return err
		}
	}

	return nil
}

// RemoveEmptyBucket removes a bucket only when it contains no object versions.
func (m *MinioStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
	actualBucket := bucket
	prefix := ""
	if m.bucket != "" {
		actualBucket = m.bucket
		if m.prefixPath != "" {
			prefix = m.prefixPath + "/"
		}
		prefix += bucket + "/"
	}
	exists, err := m.client.BucketExists(ctx, actualBucket)
	if err != nil {
		return err
	}
	if !exists {
		return nil
	}
	listCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	for object := range m.client.ListObjects(listCtx, actualBucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true, WithVersions: true}) {
		if object.Err != nil {
			return object.Err
		}
		return fmt.Errorf("bucket %s is not empty", bucket)
	}
	if err := ctx.Err(); err != nil {
		return err
	}
	if m.bucket == "" {
		return m.client.RemoveBucket(ctx, actualBucket)
	}
	return nil
}

// Copy copies an object from source to destination
func (m *MinioStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
	srcBucket, srcPath = m.resolveBucketAndPath(srcBucket, srcPath)
	destBucket, destPath = m.resolveBucketAndPath(destBucket, destPath)

	// Ensure destination bucket exists
	if m.bucket == "" {
		exists, err := m.client.BucketExists(ctx, destBucket)
		if err != nil {
			common.Warn("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err))
			return false
		}
		if !exists {
			if err = m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil {
				common.Warn("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err))
				return false
			}
		}
	}

	// Check if source object exists
	_, err := m.client.StatObject(ctx, srcBucket, srcPath, minio.StatObjectOptions{})
	if err != nil {
		common.Warn("Failed to stat source object", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
		return false
	}

	// Copy object
	srcOpts := minio.CopySrcOptions{
		Bucket: srcBucket,
		Object: srcPath,
	}
	destOpts := minio.CopyDestOptions{
		Bucket: destBucket,
		Object: destPath,
	}

	_, err = m.client.CopyObject(ctx, destOpts, srcOpts)
	if err != nil {
		common.Warn("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
		return false
	}

	return true
}

// Move moves an object from source to destination
func (m *MinioStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
	return moveWithRollback(ctx, srcBucket, srcPath, destBucket, destPath,
		func() bool { return m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) },
		func(rctx context.Context) error { return m.Remove(rctx, srcBucket, srcPath) },
		func(rctx context.Context) error { return m.Remove(rctx, destBucket, destPath) },
	)
}

func (m *MinioStorage) Close() error { return nil }