Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

3.1 KB
Raw Blame
package file

import (
	"context"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"

	"ragflow/internal/permission"
	"ragflow/internal/storage"
	"ragflow/internal/utility"
)

// maxUploadFilenameBytes is the longest name checkUploadInfoHealth accepts.
const maxUploadFilenameBytes = 255

// UploadFromURL fetches a remote URL, saves the content to the current user's
// downloads bucket, and returns the file metadata map.
//
// The remote fetch is SSRF-guarded (mirrors Python's assert_url_is_safe): the
// scheme must be http/https and every address the host resolves to must be
// globally routable; the validated IP is pinned for the actual connection — and
// re-validated on each redirect hop — to defeat DNS-rebinding. The HTTP client
// carries connect and overall timeouts, and the response body is bounded with
// truncation detection so an oversized file is rejected rather than silently
// clipped.
func (s *FileService) UploadFromURL(ctx context.Context, rawURL string) (map[string]interface{}, error) {
	if permission.UserIDFromContext(ctx) == "" {
		return nil, permission.ErrUnauthenticated
	}
	if rawURL == "" {
		return nil, fmt.Errorf("url is required")
	}
	parsed, err := url.Parse(rawURL)
	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" {
		return nil, fmt.Errorf("invalid or unsafe URL")
	}

	data, headers, finalURL, err := utility.FetchRemoteFileSafely(ctx, rawURL, maxRemoteFileSize)
	if err != nil {
		return nil, err
	}

	storageImpl := storage.GetStorageFactory().GetStorage()
	if storageImpl == nil {
		return nil, fmt.Errorf("storage not initialized")
	}

	contentType := headers.Get("Content-Type")
	filename := normalizeRemoteUploadFilename(finalURL, contentType, data)
	if err = s.checkUploadInfoHealth(ctx, filename); err != nil {
		return nil, err
	}
	filename, contentType, data = utility.NormalizeUploadInfoContent(filename, contentType, data)
	return s.storeUploadInfoBlob(ctx, storageImpl, filename, contentType, data)
}

func normalizeRemoteUploadFilename(rawURL, contentType string, data []byte) string {
	parsed, err := url.Parse(rawURL)
	filename := "download"
	if err == nil {
		filename = sanitizeFilename(filepath.Base(parsed.Path))
	}
	ct := strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0]))
	if ct == "application/pdf" || utility.BytesLooksLikePDF(data) {
		if !strings.HasSuffix(strings.ToLower(filename), ".pdf") {
			filename += ".pdf"
		}
	}
	// The appended ".pdf" can push a name that sanitizeFilename already
	// shortened back over the limit.
	return utility.TruncateFilename(filename, maxUploadFilenameBytes)
}

func sanitizeFilename(name string) string {
	name = filepath.Base(name)
	name = strings.TrimSpace(name)

	name = strings.Map(func(r rune) rune {
		switch r {
		case '/', '\\', ':', '*', '?', '"', '<', '>', '|', 0:
			return '_'
		}
		if r < 0x20 {
			return '_'
		}
		return r
	}, name)

	name = strings.Trim(name, ". ")

	if name == "" || name == "." || name == ".." {
		return "download"
	}
	if stem := strings.SplitN(strings.ToUpper(name), ".", 2)[0]; reservedDeviceNames[stem] {
		return "download"
	}
	return utility.TruncateFilename(name, maxUploadFilenameBytes)
}