Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

17.6 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package utility

import (
	"fmt"
	"net/http"
	"net/url"
	"path/filepath"
	"regexp"
	"strings"
	"unicode/utf8"
)

type FileType string

const (
	FileTypePDF      FileType = "pdf"
	FileTypeDOC      FileType = "doc"
	FileTypeDOCX     FileType = "docx"
	FileTypePPT      FileType = "ppt"
	FileTypePPTX     FileType = "pptx"
	FileTypeXLS      FileType = "xls"
	FileTypeXLSX     FileType = "xlsx"
	FileTypeCSV      FileType = "csv"
	FileTypeHTML     FileType = "html"
	FileTypeMarkdown FileType = "md"
	FileTypeTXT      FileType = "txt"
	FileTypeEPUB     FileType = "epub"
	FileTypeJSON     FileType = "json"
	FileTypeVIDEO    FileType = "video"
	FileTypeEMAIL    FileType = "email"
	FileTypeVISUAL   FileType = "visual"
	FileTypeAURAL    FileType = "aural"
	FileTypeFOLDER   FileType = "folder"
	FileTypeOTHER    FileType = "other"
)

var (
	filenameLenLimit = 255

	// videoExtensions is the single source of truth for video containers.
	// GetFileType (runtime dispatch) and the ingestion parser-family mapping
	// read this set. FilenameType keeps the legacy upload bucket (video is
	// admitted as visual and routed to the picture pipeline), so every video
	// container accepted below must also be present here.
	videoExtensions = map[string]struct{}{
		"mp4": {}, "avi": {}, "mkv": {}, "mov": {}, "webm": {}, "flv": {},
		"mpeg": {}, "mpg": {}, "wmv": {}, "3gp": {}, "3gpp": {},
		"rm": {}, "rmvb": {}, "asf": {}, "dat": {}, "asx": {}, "wvx": {},
		"mpe": {}, "mpa": {},
	}
)

// IsVideoExtension reports whether ext (with or without a leading dot,
// case-insensitive) is a video container.
func IsVideoExtension(ext string) bool {
	normalized := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(ext), "."))
	_, ok := videoExtensions[normalized]
	return ok
}

// IsVideoFilename reports whether filename has a video extension.
func IsVideoFilename(filename string) bool {
	return IsVideoExtension(filepath.Ext(filename))
}

func init() {
}

func normalizeFilename(filename string) (string, bool) {
	if filename == "" {
		return "", false
	}
	base := filepath.Base(filename)
	base = strings.TrimSpace(base)
	if base == "" || utf8.RuneCountInString(base) > filenameLenLimit {
		return "", false
	}
	return strings.ToLower(base), true
}

// GetFileType classifies a filename by its case-insensitive extension.
// It returns FileTypeOTHER when the extension is missing or unsupported.
func GetFileType(filename string) FileType {

	ext := filepath.Ext(filename)
	var suffix string
	if len(ext) > 0 && ext[0] == '.' {
		suffix = strings.ToLower(ext[1:])
	} else {
		suffix = strings.ToLower(ext)
	}

	if IsVideoExtension(suffix) {
		return FileTypeVIDEO
	}

	switch suffix {
	case "pdf":
		return FileTypePDF
	case "xls", "et", "ett":
		return FileTypeXLS
	case "xlsx":
		return FileTypeXLSX
	case "csv":
		return FileTypeCSV
	case "doc", "wps", "wpt":
		return FileTypeDOC
	case "docx":
		return FileTypeDOCX
	case "ppt", "dps", "dpt":
		return FileTypePPT
	case "pptx":
		return FileTypePPTX
	case "html", "htm":
		return FileTypeHTML
	case "md", "markdown", "mdx":
		return FileTypeMarkdown
	case "txt", "py", "js", "java", "c", "cpp", "h", "php",
		"go", "ts", "sh", "cs", "kt", "sql":
		return FileTypeTXT
	case "epub":
		return FileTypeEPUB
	case "json", "jsonl", "ldjson":
		return FileTypeJSON
	case "eml", "msg":
		return FileTypeEMAIL
	case "da", "wave", "wav", "mp3", "aac", "flac", "ogg",
		"aiff", "au", "midi", "wma", "ape", "alac", "wv", "opus":
		return FileTypeAURAL
	case "png", "jpg", "jpeg", "gif", "bmp":
		return FileTypeVISUAL
	case "tiff", "tif", "webp", "svg", "ico":
		return FileTypeVISUAL
	case "avif", "heic", "apng":
		return FileTypeVISUAL
	default:
		return FileTypeOTHER
	}
}

// FilenameType returns the upload category for a filename.
// It classifies supported extensions and returns FileTypeOTHER for invalid
// filenames or unsupported extensions.
func FilenameType(filename string) FileType {
	normalized, ok := normalizeFilename(filename)
	if !ok {
		return FileTypeOTHER
	}

	// Video is admitted at upload in the legacy visual bucket and routed to the
	// picture pipeline; parsing is blocked later (no parser implements video
	// analysis). Keep the admission here so every video container in the shared
	// set passes the upload gate, including webm/flv/3gp/3gpp which the old
	// bucket missed.
	if IsVideoFilename(normalized) {
		return FileTypeVISUAL
	}

	if matched, _ := regexp.MatchString(`.*\.pdf$`, normalized); matched {
		return FileTypePDF
	}

	docExtensions := []string{
		"msg", "eml", "doc", "docx", "ppt", "pptx", "yml", "xml", "htm", "json", "jsonl", "ldjson",
		"csv", "txt", "ini", "xls", "xlsx", "wps", "wpt", "et", "ett", "dps", "dpt", "rtf", "hlp", "pages", "numbers", "key",
		"md", "markdown", "mdx", "py", "js", "java", "c", "cpp", "h", "php", "go", "ts", "sh", "cs", "kt",
		"html", "sql", "epub",
	}
	for _, ext := range docExtensions {
		if strings.HasSuffix(normalized, "."+ext) {
			return FileTypeDOC
		}
	}

	audioExtensions := []string{
		"wav", "flac", "ape", "alac", "wv", "mp3", "aac", "ogg", "vorbis", "opus",
	}
	for _, ext := range audioExtensions {
		if strings.HasSuffix(normalized, "."+ext) {
			return FileTypeAURAL
		}
	}

	visualExtensions := []string{
		"jpg", "jpeg", "png", "tif", "tiff", "gif", "bmp", "pcx", "tga", "exif", "fpx", "svg", "psd", "cdr",
		"pcd", "dxf", "ufo", "eps", "ai", "raw", "wmf", "webp", "avif", "apng", "icon", "ico",
	}
	for _, ext := range visualExtensions {
		if strings.HasSuffix(normalized, "."+ext) {
			return FileTypeVISUAL
		}
	}

	return FileTypeOTHER
}

func SanitizeFilename(filename string) string {
	if filename == "" {
		return ""
	}
	filename = strings.TrimSpace(filename)
	if filename == "" {
		return ""
	}

	filename = strings.ReplaceAll(filename, "\\", "/")
	filename = strings.Trim(filename, "/")

	parts := strings.Split(filename, "/")
	var sanitizedParts []string
	for _, part := range parts {
		if part != "" && part != "." && part != ".." {
			sanitizedParts = append(sanitizedParts, part)
		}
	}

	unsafeRegex := regexp.MustCompile(`[^A-Za-z0-9_\-/]`)
	for i, part := range sanitizedParts {
		sanitizedParts[i] = unsafeRegex.ReplaceAllString(part, "")
	}

	result := strings.Join(sanitizedParts, "/")
	return result
}

func GetFileExtension(filename string) string {
	ext := filepath.Ext(filename)
	if len(ext) > 0 && ext[0] == '.' {
		return strings.ToLower(ext[1:])
	}
	return strings.ToLower(ext)
}

// TruncateFilename shortens name to at most maxBytes bytes. It keeps the
// extension when it fits and cuts on a rune boundary, so the result stays
// valid UTF-8 and keeps the file type the extension implies.
func TruncateFilename(name string, maxBytes int) string {
	if len(name) <= maxBytes {
		return name
	}
	ext := filepath.Ext(name)
	if len(ext) >= maxBytes {
		ext = ""
	}
	end := maxBytes - len(ext)
	for end > 0 && !utf8.RuneStart(name[end]) {
		end--
	}
	return name[:end] + ext
}

// StorageKeyMaxBytes is the maximum size of one object-store path segment.
// Filesystem-backed backends such as Silo and MinIO store each segment as a
// file name and reject anything longer.
const StorageKeyMaxBytes = 255

// StorageObjectName returns a storage key segment for filename that fits
// within maxBytes. Display names use Unicode code points, but object keys must
// stay within the byte limit of the storage backend. attempt > 0 adds a
// collision suffix before the extension without exceeding the budget.
func StorageObjectName(filename string, attempt, maxBytes int) string {
	if maxBytes <= 0 {
		return ""
	}
	suffix := ""
	if attempt > 0 {
		suffix = fmt.Sprintf("(%d)", attempt)
	}
	if len(suffix) >= maxBytes {
		return suffix[:maxBytes]
	}
	ext := filepath.Ext(filename)
	if len(ext)+len(suffix) >= maxBytes {
		ext = ""
	}
	budget := maxBytes - len(ext) - len(suffix)
	base := strings.TrimSuffix(filename, ext)
	if budget <= 0 {
		base = ""
	} else if len(base) > budget {
		end := budget
		for end > 0 && !utf8.RuneStart(base[end]) {
			end--
		}
		base = base[:end]
	}
	return base + suffix + ext
}

// CONTENT_TYPE_MAP maps file extensions to MIME content types
var CONTENT_TYPE_MAP = map[string]string{
	// Office
	"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
	"doc":  "application/msword",
	"pdf":  "application/pdf",
	"csv":  "text/csv",
	"xls":  "application/vnd.ms-excel",
	"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
	// Text/code
	"txt":  "text/plain",
	"py":   "text/plain",
	"js":   "text/plain",
	"java": "text/plain",
	"c":    "text/plain",
	"cpp":  "text/plain",
	"h":    "text/plain",
	"php":  "text/plain",
	"go":   "text/plain",
	"ts":   "text/plain",
	"sh":   "text/plain",
	"cs":   "text/plain",
	"kt":   "text/plain",
	"sql":  "text/plain",
	// Web
	"md":       "text/markdown",
	"markdown": "text/markdown",
	"mdx":      "text/markdown",
	"htm":      "text/html",
	"html":     "text/html",
	"json":     "application/json",
	// Image formats
	"png":  "image/png",
	"jpg":  "image/jpeg",
	"jpeg": "image/jpeg",
	"gif":  "image/gif",
	"bmp":  "image/bmp",
	"tiff": "image/tiff",
	"tif":  "image/tiff",
	"webp": "image/webp",
	"svg":  "image/svg+xml",
	"ico":  "image/x-icon",
	"avif": "image/avif",
	"heic": "image/heic",
	// PPTX
	"ppt":  "application/vnd.ms-powerpoint",
	"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
	// Video formats
	"mp4":  "video/mp4",
	"mov":  "video/quicktime",
	"avi":  "video/x-msvideo",
	"mpg":  "video/mpeg",
	"mpeg": "video/mpeg",
	"mkv":  "video/x-matroska",
	"wmv":  "video/x-ms-wmv",
	"webm": "video/webm",
	"rm":   "application/vnd.rn-realmedia",
	"rmvb": "application/vnd.rn-realmedia",
	"dat":  "video/mpeg",
	"asx":  "video/x-ms-asf",
	"wvx":  "video/x-ms-wvx",
	"mpe":  "video/mpeg",
	"mpa":  "video/mpeg",
}

// FORCE_ATTACHMENT_EXTENSIONS are extensions that should always be downloaded as attachments
var FORCE_ATTACHMENT_EXTENSIONS = map[string]bool{
	"htm":   true,
	"html":  true,
	"shtml": true,
	"xht":   true,
	"xhtml": true,
	"xml":   true,
	"mhtml": true,
	"svg":   true,
}

// FORCE_ATTACHMENT_CONTENT_TYPES are content types that should always be downloaded as attachments
var FORCE_ATTACHMENT_CONTENT_TYPES = map[string]bool{
	"text/html":             true,
	"image/svg+xml":         true,
	"application/xhtml+xml": true,
	"text/xml":              true,
	"application/xml":       true,
	"multipart/related":     true,
}

// stripContentTypeParams strips "; charset=..." and similar parameters
// from a content type string.  Mirrors Python's .split(";")[0].strip().
func stripContentTypeParams(ct string) string {
	if before, _, found := strings.Cut(ct, ";"); found {
		return strings.TrimSpace(before)
	}
	return strings.TrimSpace(ct)
}

// ShouldForceAttachment determines if the file should be forced as attachment
func ShouldForceAttachment(ext string, contentType string) bool {
	normalizedExt := strings.ToLower(strings.TrimPrefix(ext, "."))
	if normalizedExt != "" && FORCE_ATTACHMENT_EXTENSIONS[normalizedExt] {
		return true
	}
	normalizedType := strings.ToLower(stripContentTypeParams(contentType))
	return FORCE_ATTACHMENT_CONTENT_TYPES[normalizedType]
}

// GetContentType determines the content type based on extension and file type
// fallbackPrefix is "image" for visual files, "application" for others
func GetContentType(ext string, fileType string) string {
	if ext == "" {
		return ""
	}
	normalizedExt := strings.ToLower(strings.TrimPrefix(ext, "."))
	if contentType, ok := CONTENT_TYPE_MAP[normalizedExt]; ok {
		return contentType
	}
	fallbackPrefix := "application"
	if fileType == string(FileTypeVISUAL) {
		fallbackPrefix = "image"
	}
	return fallbackPrefix + "/" + normalizedExt
}

// contentDispositionBasename extracts the final path segment, treating both
// "/" and "\" as separators. Mirrors Python file_response.py basename logic.
func contentDispositionBasename(filename string) string {
	base := filename
	if idx := strings.LastIndex(base, "/"); idx >= 0 {
		base = base[idx+1:]
	}
	if idx := strings.LastIndex(base, "\\"); idx >= 0 {
		base = base[idx+1:]
	}
	return base
}

// rfc8187Encode percent-encodes a UTF-8 string for RFC 8187 filename*.
// Mirrors Python urllib.parse.quote(s, safe="").
func rfc8187Encode(s string) string {
	var buf strings.Builder
	for _, b := range []byte(s) {
		if (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9') ||
			b == '-' || b == '.' || b == '_' || b == '~' {
			buf.WriteByte(b)
		} else {
			buf.WriteString(fmt.Sprintf("%%%02X", b))
		}
	}
	return buf.String()
}

// SanitizeContentDispositionFilename builds the ASCII filename fallback for
// Content-Disposition headers. Mirrors Python file_response.py:
// ascii_content_disposition_filename().
func SanitizeContentDispositionFilename(filename string) string {
	if filename == "" {
		return "file"
	}
	var asciiOnly strings.Builder
	for _, r := range filename {
		if r < 0x80 {
			asciiOnly.WriteRune(r)
		}
	}
	var sanitized strings.Builder
	for _, r := range asciiOnly.String() {
		if (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') ||
			r == '_' || r == '.' || r == '-' {
			sanitized.WriteRune(r)
		} else {
			sanitized.WriteRune('_')
		}
	}
	result := strings.Trim(sanitized.String(), "._")
	if result == "" {
		return "file"
	}
	return result
}

// ResolveAttachmentContentType resolves a content type and extension from
// query-parameter values. When mimeType is non-empty it is preferred and
// reverse-looked up in CONTENT_TYPE_MAP to also resolve the extension;
// otherwise the extension is looked up in CONTENT_TYPE_MAP, falling back
// to "application/<ext>". Returns (contentType, ext). Mirrors Python
// file_response.py: resolve_attachment_content_type().
func ResolveAttachmentContentType(ext string, mimeType string) (string, string) {
	ext = strings.ToLower(strings.TrimSpace(ext))
	ext = strings.TrimPrefix(ext, ".")
	mimeType = strings.TrimSpace(mimeType)

	contentType := ""
	if mimeType != "" {
		normalizedType := strings.ToLower(stripContentTypeParams(mimeType))
		contentType = normalizedType
		// Reverse-lookup extension from CONTENT_TYPE_MAP only when
		// no explicit ext was provided. Never overwrite a caller-supplied extension (e.g. ext=svg&mime_type=image/png must
		// stay ext=svg for the force-attachment check).
		if ext == "" {
			for knownExt, knownType := range CONTENT_TYPE_MAP {
				if knownType == normalizedType {
					ext = knownExt
					break
				}
			}
		}
	} else if ext != "" {
		if ct, ok := CONTENT_TYPE_MAP[ext]; ok {
			contentType = ct
		} else {
			contentType = "application/" + ext
		}
	}
	return contentType, ext
}

// FormatContentDisposition builds a Content-Disposition value with an ASCII
// filename fallback and an RFC 5987 UTF-8 filename* parameter.
func FormatContentDisposition(disposition, filename string) string {
	base := contentDispositionBasename(filename)
	if base == "" {
		return disposition
	}
	safe := SanitizeContentDispositionFilename(base)
	encoded := rfc8187Encode(base)
	return fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, safe, encoded)
}

// SetPreviewFileResponseHeaders sets response headers for inline file
// preview. For force-attachment types (HTML, SVG, XML) it falls back to
// attachment disposition with nosniff. Mirrors Python file_response.py:
// apply_preview_file_response_headers().
func SetPreviewFileResponseHeaders(h http.Header, contentType, ext, filename string) {
	if contentType != "" {
		h.Set("Content-Type", contentType)
	}
	if ShouldForceAttachment(ext, contentType) {
		h.Set("X-Content-Type-Options", "nosniff")
		if filename != "" {
			h.Set("Content-Disposition", FormatContentDisposition("attachment", filename))
		} else {
			h.Set("Content-Disposition", "attachment")
		}
		return
	}

	if filename != "" {
		h.Set("Content-Disposition", FormatContentDisposition("inline", filename))
	} else {
		h.Set("Content-Disposition", "inline")
	}
}

// SetDownloadFileResponseHeaders sets response headers for file download
// (always attachment disposition). Mirrors Python file_response.py:
// apply_download_file_response_headers().
func SetDownloadFileResponseHeaders(h http.Header, contentType, ext, filename string) {
	if contentType != "" {
		h.Set("Content-Type", contentType)
	}
	if ShouldForceAttachment(ext, contentType) {
		h.Set("X-Content-Type-Options", "nosniff")
		if filename != "" {
			h.Set("Content-Disposition", FormatContentDisposition("attachment", filename))
		} else {
			h.Set("Content-Disposition", "attachment")
		}
		return
	}
	if filename != "" {
		h.Set("Content-Disposition", FormatContentDisposition("attachment", filename))
	} else {
		h.Set("Content-Disposition", "attachment")
	}
}

// AgentAttachmentPreviewPath builds the preview URL path for an agent
// attachment. Query parameters ext and mime_type are URL-encoded when
// provided. Mirrors Python file_response.py:
// agent_attachment_preview_path().
func AgentAttachmentPreviewPath(attachmentID, ext, mimeType string) string {
	path := "/api/v1/agents/attachments/" + attachmentID + "/preview"
	params := url.Values{}
	if ext != "" {
		params.Set("ext", ext)
	}
	if mimeType != "" {
		params.Set("mime_type", mimeType)
	}
	if qs := params.Encode(); qs != "" {
		return path + "?" + qs
	}
	return path
}