Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

42.5 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package handler

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"ragflow/internal/engine"
	"ragflow/internal/engine/types"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"

	"ragflow/internal/common"
	"ragflow/internal/dao"
	"ragflow/internal/permission"
	"ragflow/internal/service"
	"ragflow/internal/service/billing"
	dataset "ragflow/internal/service/dataset"
)

// DatasetsHandler handles the RESTful dataset endpoints.
type DatasetsHandler struct {
	datasetsService       *dataset.DatasetService
	metadataService       *service.MetadataService
	searchDatasetsService searchDatasetsService
	searchDatasetService  searchDatasetService
}

type searchDatasetsService interface {
	SearchDatasets(ctx context.Context, req *service.SearchDatasetsRequest) (*service.SearchDatasetsResponse, error)
}

type searchDatasetService interface {
	SearchDataset(ctx context.Context, datasetID string, req *service.SearchDatasetRequest) (*service.SearchDatasetsResponse, error)
}

// NewDatasetsHandler creates a new datasets' handler.
func NewDatasetsHandler(datasetsService *dataset.DatasetService, metadataService *service.MetadataService) *DatasetsHandler {
	h := &DatasetsHandler{
		datasetsService: datasetsService,
		metadataService: metadataService,
	}
	if datasetsService != nil {
		h.searchDatasetsService = datasetsService
		h.searchDatasetService = datasetsService
	}
	return h
}

// ListDatasets handles GET /api/v1/datasets.
func (h *DatasetsHandler) ListDatasets(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	page, pageSize, err := common.ParsePagination(c.Request.URL.Query())
	if err != nil {
		common.RespondError(c, common.CodeArgumentError, err.Error())
		return
	}

	if c.Query("type") == "filter" {
		ctx := c.Request.Context()
		data, code, err := h.datasetsService.ListDatasetFilters(ctx, c.Query("canvas_id"))
		if err != nil {
			common.RespondError(c, code, err.Error())
			return
		}
		common.RespondSuccess(c, data, "success")
		return
	}

	// Mirror pydantic's extra="forbid" on ListDatasetReq.
	for param := range c.Request.URL.Query() {
		if !listDatasetsAllowedParams[param] {
			common.RespondError(c, common.CodeArgumentError, fmt.Sprintf("Extra inputs are not permitted: %s", param))
			return
		}
	}

	// `sort` supersedes the older pair, so a request it can order is not rejected
	// for the spelling of an `orderby` or `desc` that will not be read.
	sortTerms := sortTermsFromQuery(c)
	orderby := "create_time"
	if queryOrderby, exists := c.GetQuery("orderby"); exists {
		if queryOrderby != "create_time" && queryOrderby != "update_time" {
			if len(sortTerms) == 0 {
				common.RespondError(c, common.CodeArgumentError, "Input should be 'create_time' or 'update_time'")
				return
			}
		} else {
			orderby = queryOrderby
		}
	}

	desc := true
	if descStr := c.Query("desc"); descStr != "" {
		parsed, ok := parsePythonBool(descStr)
		if !ok {
			if len(sortTerms) == 0 {
				common.RespondError(c, common.CodeArgumentError, "Input should be a valid boolean, unable to interpret input")
				return
			}
		} else {
			desc = parsed
		}
	}
	terms := orderTerms(sortTerms, orderby, desc)

	keywords := c.Query("keywords")
	parserID := c.Query("parser_id")
	var ownerIDs []string
	for _, item := range c.QueryArray("owner_ids") {
		for _, ownerID := range strings.Split(item, ",") {
			if ownerID = strings.TrimSpace(ownerID); ownerID != "" {
				ownerIDs = append(ownerIDs, ownerID)
			}
		}
	}

	// Mirror pydantic: a present-but-empty id fails UUID validation.
	if rawID, exists := c.GetQuery("id"); exists {
		if _, err := dataset.NormalizeDatasetID(strings.TrimSpace(rawID)); err != nil {
			common.RespondError(c, common.CodeArgumentError, err.Error())
			return
		}
	}

	// Mirror pydantic's ListDatasetReq.ids: each occurrence is comma-split,
	// every value must be a valid UUID, and duplicates are rejected.
	var ids []string
	if rawIDs, exists := c.Request.URL.Query()["ids"]; exists {
		seen := make(map[string]int)
		for _, item := range rawIDs {
			for _, value := range strings.Split(item, ",") {
				if value == "" {
					continue
				}
				normalizedID, err := dataset.NormalizeDatasetID(strings.TrimSpace(value))
				if err != nil {
					common.RespondError(c, common.CodeArgumentError, err.Error())
					return
				}
				seen[normalizedID]++
				ids = append(ids, normalizedID)
			}
		}
		duplicates := make([]string, 0, len(ids))
		reported := make(map[string]bool)
		for _, normalizedID := range ids {
			if seen[normalizedID] > 1 && !reported[normalizedID] {
				reported[normalizedID] = true
				duplicates = append(duplicates, normalizedID)
			}
		}
		if len(duplicates) > 0 {
			common.RespondError(c, common.CodeArgumentError, fmt.Sprintf("duplicate ids: '%s'", strings.Join(duplicates, ", ")))
			return
		}
	}

	ctx := c.Request.Context()
	data, total, code, err := h.datasetsService.ListDatasets(
		ctx,
		c.Query("id"),
		c.Query("name"),
		page,
		pageSize,
		terms,
		keywords,
		ownerIDs,
		parserID,
		ids,
		c.Query("canvas_id"),
	)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	c.JSON(http.StatusOK, gin.H{
		"code":           common.CodeSuccess,
		"data":           data,
		"message":        "success",
		"total_datasets": total,
	})
}

// CreateDataset handles POST /api/v1/datasets.
func (h *DatasetsHandler) CreateDataset(c *gin.Context) {
	startAt := time.Now()
	operationLog := &common.OperationLog{
		EventTime:  startAt,
		Operation:  "create_dataset",
		APIPath:    c.FullPath(),
		HTTPMethod: c.Request.Method,
		IPAddress:  c.ClientIP(),
	}
	defer func() {
		operationLog.DurationMS = time.Since(startAt).Milliseconds()
		common.Audit(operationLog)
	}()

	user, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		operationLog.Code = int(errorCode)
		operationLog.Error = errors.New(errorMessage)
		return
	}
	operationLog.UserID = user.ID

	bodyBytes, raw, err := parseJSONRequestObject(c)
	if err != nil {
		operationLog.Code = int(common.CodeArgumentError)
		operationLog.Error = err
		return
	}
	if _, exists := raw["ext"]; exists {
		err = fmt.Errorf("extra inputs are not permitted: ext")
		common.RespondError(c, common.CodeArgumentError, err.Error())
		operationLog.Code = int(common.CodeArgumentError)
		operationLog.Error = err
		return
	}
	for field := range raw {
		if !createDatasetAllowedFields[field] {
			err = fmt.Errorf("extra inputs are not permitted: %s", field)
			common.RespondError(c, common.CodeArgumentError, err.Error())
			operationLog.Code = int(common.CodeArgumentError)
			operationLog.Error = err
			return
		}
	}

	var req service.CreateDatasetRequest
	if err = json.Unmarshal(bodyBytes, &req); err != nil {
		common.RespondError(c, common.CodeDataError, err.Error())
		operationLog.Code = int(common.CodeDataError)
		operationLog.Error = err
		return
	}
	if req.ParserConfig == nil && req.PipelineID == nil {
		req.ParserConfig = map[string]interface{}{}
	}

	if req.Name == "" || (len(bodyBytes) > 0 && jsonNullValue(bodyBytes, "name")) {
		err = fmt.Errorf("field validation for 'name' failed on the 'required' tag")
		common.RespondError(c, common.CodeDataError, err.Error())
		operationLog.Code = int(common.CodeDataError)
		operationLog.Error = err
		return
	}

	ctx := c.Request.Context()

	tenantID := strings.TrimSpace(c.GetString("tenant_id"))
	if tenantID == "" {
		tenantID = user.ID
	}

	// Application quota: every new dataset counts as one app, matching
	// Python's check_dynamic_resources(apps=1).
	if billingService := billing.Get(); billingService != nil && billingService.Enabled() {
		if quotaErr := billingService.CheckResources(ctx, tenantID, billing.ResourceDelta{Apps: 1}); quotaErr != nil {
			var insufficient *billing.InsufficientResourceError
			if errors.As(quotaErr, &insufficient) {
				common.RespondError(c, common.CodeBillingAppsInsufficient, quotaErr.Error(), common.ResponseOptions{Detail: gin.H{
					"resource": "apps",
					"current":  insufficient.Current,
					"limit":    insufficient.Limit,
				}})
				operationLog.Code = int(common.CodeBillingAppsInsufficient)
				operationLog.Error = insufficient
				return
			}
			common.RespondError(c, common.CodeServerError, quotaErr.Error())
			operationLog.Code = int(common.CodeServerError)
			operationLog.Error = quotaErr
			return
		}
	}

	result, code, err := h.datasetsService.CreateDataset(ctx, &req)
	if err != nil {
		common.RespondError(c, code, err.Error())
		operationLog.Code = int(code)
		operationLog.Error = err
		return
	}

	common.RespondSuccess(c, result, "success")
}

// GetDataset handles GET /api/v1/datasets/:dataset_id.
func (h *DatasetsHandler) GetDataset(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	ctx := c.Request.Context()

	datasetID := c.Param("dataset_id")
	result, code, err := h.datasetsService.GetDataset(ctx, datasetID)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	common.RespondSuccess(c, result, "success")
}

// UpdateDataset Update a dataset.
// parsePythonBool mirrors pydantic's boolean coercion for query params:
// accepts true/false/1/0/yes/no/y/n (case-insensitive).
func parsePythonBool(s string) (bool, bool) {
	switch strings.ToLower(s) {
	case "true", "1", "yes", "y", "on":
		return true, true
	case "false", "0", "no", "n", "off":
		return false, true
	}
	return false, false
}

// parseJSONRequestObject mirrors Python's validate_and_parse_json_request:
// content-type check first, then JSON syntax, then object shape. On failure, it
// writes the error response and returns ok=false.
func parseJSONRequestObject(c *gin.Context) ([]byte, map[string]interface{}, error) {
	if c.ContentType() != "application/json" {
		common.RespondError(c, common.CodeArgumentError, fmt.Sprintf("Unsupported content type: Expected application/json, got %s", c.GetHeader("Content-Type")))
		return nil, nil, fmt.Errorf("unsupported content type: Expected application/json, got %s", c.GetHeader("Content-Type"))
	}

	body, err := c.GetRawData()
	if err != nil || len(body) == 0 {
		common.RespondError(c, common.CodeArgumentError, "Malformed JSON syntax: Missing commas/brackets or invalid encoding")
		return nil, nil, fmt.Errorf("malformed JSON syntax: Missing commas/brackets or invalid encoding")
	}

	var payload interface{}
	if err = json.Unmarshal(body, &payload); err != nil {
		common.RespondError(c, common.CodeArgumentError, "Malformed JSON syntax: Missing commas/brackets or invalid encoding")
		return nil, nil, err
	}
	raw, isObject := payload.(map[string]interface{})
	if !isObject {
		common.RespondError(c, common.CodeArgumentError, fmt.Sprintf("Invalid request payload: expected object, got %s", pythonJSONTypeName(payload)))
		return nil, nil, fmt.Errorf("invalid request payload: expected object, got %s", pythonJSONTypeName(payload))
	}
	return body, raw, nil
}

// jsonNullValue checks if a field is explicitly null in the JSON body.
func jsonNullValue(body []byte, field string) bool {
	var m map[string]json.RawMessage
	if err := json.Unmarshal(body, &m); err != nil {
		return false
	}
	raw, ok := m[field]
	if !ok {
		return false
	}
	return strings.TrimSpace(string(raw)) == "null"
}

// pythonJSONTypeName maps a decoded JSON value to the Python type name used in
// the "Invalid request payload" contract message.
func pythonJSONTypeName(v interface{}) string {
	switch v.(type) {
	case string:
		return "str"
	case []interface{}:
		return "list"
	case float64:
		return "float"
	case bool:
		return "bool"
	case nil:
		return "NoneType"
	default:
		return "object"
	}
}

// listDatasetsAllowedParams mirrors the query field set of Python's
// ListDatasetReq (BaseListReq + dataset filters; `type` is handled
// before validation in the Python endpoint).
var listDatasetsAllowedParams = map[string]bool{
	"id": true, "ids": true, "name": true, "page": true, "page_size": true,
	"orderby": true, "desc": true, "sort": true, "include_parsing_status": true,
	"keywords": true, "owner_ids": true, "parser_id": true, "type": true,
	"canvas_id": true,
}

var createDatasetAllowedFields = map[string]bool{
	"name": true, "embedding_model": true, "parser_config": true,
	"language": true, "permission": true, "parser_id": true,
	"pipeline_id": true, "pipeline_version_id": true, "parse_type": true,
}

// updateDatasetAllowedFields mirrors the field set of Python's UpdateDatasetReq
// (CreateDatasetReq fields + dataset_id/pagerank/language/connectors).
var updateDatasetAllowedFields = map[string]bool{
	"name": true, "avatar": true, "description": true, "embedding_model": true,
	"permission": true, "parse_type": true, "pipeline_id": true, "pipeline_version_id": true, "chunk_method": true,
	"parser_id": true, "parser_config": true, "auto_metadata_config": true,
	"dataset_id": true, "pagerank": true, "language": true, "connectors": true,
}

func (h *DatasetsHandler) UpdateDataset(c *gin.Context) {
	startAt := time.Now()
	operationLog := &common.OperationLog{
		EventTime:  startAt,
		Operation:  "create_dataset",
		APIPath:    c.FullPath(),
		HTTPMethod: c.Request.Method,
		IPAddress:  c.ClientIP(),
	}
	defer func() {
		operationLog.DurationMS = time.Since(startAt).Milliseconds()
		common.Audit(operationLog)
	}()

	user, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		operationLog.Code = int(errorCode)
		operationLog.Error = errors.New(errorMessage)
		return
	}

	userID := strings.TrimSpace(user.ID)
	if userID == "" {
		common.RespondError(c, common.CodeDataError, "user id is required")
		operationLog.Code = int(common.CodeDataError)
		operationLog.Error = errors.New("user id is required")
		return
	}

	datasetID := strings.TrimSpace(c.Param("dataset_id"))
	if datasetID == "" {
		common.RespondError(c, common.CodeBadRequest, "dataset id is required")
		operationLog.Code = int(common.CodeBadRequest)
		operationLog.Error = errors.New("dataset id is required")
		return
	}

	if _, err := dataset.NormalizeDatasetID(datasetID); err != nil {
		common.RespondError(c, common.CodeArgumentError, err.Error())
		operationLog.Code = int(common.CodeArgumentError)
		operationLog.Error = err
		return
	}

	bodyBytes, _, err := parseJSONRequestObject(c)
	if err != nil {
		// parseJSONRequestObject already wrote the error response.
		operationLog.Code = int(common.CodeArgumentError)
		operationLog.Error = err
		return
	}

	var req service.UpdateDatasetRequest
	if err = json.Unmarshal(bodyBytes, &req); err != nil {
		common.RespondError(c, common.CodeDataError, err.Error())
		operationLog.Code = int(common.CodeDataError)
		operationLog.Error = err
		return
	}

	// Detect an explicitly provided parser_config key (even {} or null) so it is not
	// rejected as "no properties were modified", mirroring the Python contract.
	var providedFields map[string]json.RawMessage
	if err = json.Unmarshal(bodyBytes, &providedFields); err == nil {
		_, req.ParserConfigProvided = providedFields["parser_config"]
		// Mirror pydantic's extra="forbid" on UpdateDatasetReq.
		for field := range providedFields {
			if !updateDatasetAllowedFields[field] {
				common.RespondError(c, common.CodeArgumentError, fmt.Sprintf("Extra inputs are not permitted: %s", field))
				operationLog.Code = int(common.CodeArgumentError)
				operationLog.Error = fmt.Errorf("extra inputs are not permitted: %s", field)
				return
			}
		}
		if parserConfig, ok := providedFields["parser_config"]; ok {
			var config map[string]interface{}
			if json.Unmarshal(parserConfig, &config) == nil {
				if _, ok = config["ext"]; ok {
					common.RespondError(c, common.CodeArgumentError, "parser_config.ext is not supported; send parser configuration fields directly")
					operationLog.Code = int(common.CodeArgumentError)
					operationLog.Error = errors.New("parser_config.ext is not supported; send parser configuration fields directly")
					return
				}
			}
		}
	}

	ctx := c.Request.Context()

	result, code, err := h.datasetsService.UpdateDataset(ctx, datasetID, req)
	if err != nil {
		common.RespondError(c, code, err.Error())
		operationLog.Code = int(code)
		operationLog.Error = err
		return
	}
	if code != common.CodeSuccess {
		common.RespondError(c, code, "dataset updated failed")
		operationLog.Code = int(code)
		operationLog.Error = errors.New("dataset updated failed")
		return
	}

	common.RespondSuccess(c, result, "success")
}

func (h *DatasetsHandler) GetMetadataConfig(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	ctx := c.Request.Context()

	datasetID := c.Param("dataset_id")
	result, code, err := h.datasetsService.GetMetadataConfig(ctx, datasetID)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	common.RespondSuccess(c, result, "success")
}

func (h *DatasetsHandler) UpdateMetadataConfig(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	datasetID := c.Param("dataset_id")
	var req service.MetadataConfigRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		common.RespondError(c, common.CodeDataError, err.Error())
		return
	}

	ctx := c.Request.Context()

	result, code, err := h.datasetsService.UpdateMetadataConfig(ctx, datasetID, &req)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	common.RespondSuccess(c, result, "success")
}

// GetIngestionSummary handles GET /api/v1/datasets/:dataset_id/ingestions/summary.
func (h *DatasetsHandler) GetIngestionSummary(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}
	ctx := c.Request.Context()
	datasetID := c.Param("dataset_id")
	result, code, err := h.datasetsService.GetIngestionSummary(ctx, datasetID)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	common.RespondSuccess(c, result, "success")
}

// ListIngestionLogs handles GET /api/v1/datasets/:dataset_id/ingestions.
// The log_type=datasource query returns logs for connectors linked to the dataset.
func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	datasetID := c.Param("dataset_id")

	page, pageSize, err := common.ParsePagination(c.Request.URL.Query())
	if err != nil {
		common.RespondError(c, common.CodeArgumentError, err.Error())
		return
	}

	ctx := c.Request.Context()
	if strings.EqualFold(strings.TrimSpace(c.Query("log_type")), "datasource") {
		result, code, err := h.datasetsService.ListSyncLogs(ctx, datasetID, page, pageSize)
		if err != nil {
			common.RespondError(c, code, err.Error())
			return
		}
		common.RespondSuccess(c, result, "success")
		return
	}

	orderby := c.DefaultQuery("orderby", "create_time")
	// desc defaults to true and is only disabled by the literal value "false".
	desc := strings.ToLower(c.DefaultQuery("desc", "true")) != "false"
	terms := orderTermsFromQuery(c, orderby, desc)
	operationStatus := c.QueryArray("operation_status")
	createDateFrom := c.Query("create_date_from")
	createDateTo := c.Query("create_date_to")
	logType := c.DefaultQuery("log_type", "dataset")
	keywords := c.Query("keywords")
	// Exact per-document filter for the file-log list. Python's endpoint has no
	// equivalent; the frontend only sends it on the Go backend.
	documentID := c.Query("document_id")

	result, code, err := h.datasetsService.ListIngestionLogs(ctx, datasetID, page, pageSize, terms, operationStatus, createDateFrom, createDateTo, logType, keywords, documentID)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	common.RespondSuccess(c, result, "success")
}

// GetIngestionLog handles GET /api/v1/datasets/:dataset_id/ingestions/:log_id.
func (h *DatasetsHandler) GetIngestionLog(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	ctx := c.Request.Context()

	datasetID := c.Param("dataset_id")
	logID := c.Param("log_id")
	result, code, err := h.datasetsService.GetIngestionLog(ctx, datasetID, logID)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	common.RespondSuccess(c, result, "success")
}

// ListIngestionMessages handles GET
// /api/v1/datasets/:dataset_id/ingestions/:log_id/messages.
func (h *DatasetsHandler) ListIngestionMessages(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	limit := 0
	if rawLimit := c.Query("limit"); rawLimit != "" {
		parsed, err := strconv.Atoi(rawLimit)
		if err != nil || parsed <= 0 {
			common.RespondError(c, common.CodeArgumentError, "limit must be a positive integer")
			return
		}
		limit = parsed
	}
	afterID, ok := ingestionEventCursor(c, "after_id")
	if !ok {
		return
	}
	beforeID, ok := ingestionEventCursor(c, "before_id")
	if !ok {
		return
	}

	result, code, err := h.datasetsService.ListIngestionMessages(
		c.Request.Context(), c.Param("dataset_id"), c.Param("log_id"), limit, afterID, beforeID,
	)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}
	common.RespondSuccess(c, result, "success")
}

func ingestionEventCursor(c *gin.Context, name string) (*int, bool) {
	raw := c.Query(name)
	if raw == "" {
		return nil, true
	}
	value, err := strconv.Atoi(raw)
	if err != nil || value <= 0 {
		common.RespondError(c, common.CodeArgumentError, name+" must be a positive integer")
		return nil, false
	}
	return &value, true
}

// DeleteDatasets handles DELETE /api/v1/datasets.
func (h *DatasetsHandler) DeleteDatasets(c *gin.Context) {
	startAt := time.Now()
	operationLog := &common.OperationLog{
		EventTime:  startAt,
		Operation:  "delete_dataset",
		APIPath:    c.FullPath(),
		HTTPMethod: c.Request.Method,
		IPAddress:  c.ClientIP(),
	}
	defer func() {
		operationLog.DurationMS = time.Since(startAt).Milliseconds()
		common.Audit(operationLog)
	}()
	user, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		operationLog.Code = int(errorCode)
		operationLog.Error = errors.New(errorMessage)
		return
	}
	operationLog.UserID = user.ID

	bodyBytes, rawPayload, err := parseJSONRequestObject(c)
	if err != nil {
		// parseJSONRequestObject already wrote the error response.
		operationLog.Code = int(common.CodeArgumentError)
		operationLog.Error = err
		return
	}

	// Mirror pydantic's extra="forbid" on DeleteDatasetReq.
	for field := range rawPayload {
		if field != "ids" && field != "delete_all" {
			err = fmt.Errorf("extra inputs are not permitted: %s", field)
			common.RespondError(c, common.CodeArgumentError, err.Error())
			operationLog.Code = int(common.CodeArgumentError)
			operationLog.Error = err
			return
		}
	}

	var req struct {
		IDs       *[]string `json:"ids"`
		DeleteAll bool      `json:"delete_all,omitempty"`
	}
	if err = json.Unmarshal(bodyBytes, &req); err != nil {
		common.RespondError(c, common.CodeDataError, err.Error())
		operationLog.Code = int(common.CodeDataError)
		operationLog.Error = err
		return
	}

	// Mirror DeleteReq duplicate detection.
	if req.IDs != nil {
		seen := make(map[string]int, len(*req.IDs))
		duplicates := make([]string, 0)
		for _, id := range *req.IDs {
			seen[id]++
			if seen[id] == 2 {
				duplicates = append(duplicates, id)
			}
		}
		if len(duplicates) > 0 {
			err = fmt.Errorf("duplicate ids: '%s'", strings.Join(duplicates, ", "))
			common.RespondError(c, common.CodeArgumentError, err.Error())
			operationLog.Code = int(common.CodeArgumentError)
			operationLog.Error = err
			return
		}
	}

	var ids []string
	if req.IDs != nil {
		ids = *req.IDs
	}

	ctx := c.Request.Context()
	result, code, err := h.datasetsService.DeleteDatasets(ctx, ids, req.DeleteAll)
	if err != nil {
		common.RespondError(c, code, err.Error())
		operationLog.Code = int(code)
		operationLog.Error = err
		return
	}

	common.RespondSuccess(c, result, "success")
}

// GetKnowledgeGraph handles GET /api/v1/datasets/:dataset_id/graph.
func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	datasetID := strings.TrimSpace(c.Param("dataset_id"))
	if datasetID == "" {
		common.RespondError(c, common.CodeDataError, "dataset_id is required")
		return
	}

	ctx := c.Request.Context()
	datasetInstance, code, err := h.datasetsService.GetDataset(ctx, datasetID)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	tenantID, _ := datasetInstance["tenant_id"].(string)
	if tenantID == "" {
		common.RespondError(c, common.CodeDataError, "tenant_id is required")
		return
	}

	docEngine := engine.Get()
	if docEngine == nil {
		common.RespondError(c, common.CodeServerError, "Document engine is not initialized")
		return
	}

	indexName := fmt.Sprintf("ragflow_%s", tenantID)
	exists, err := docEngine.ChunkStoreExists(c.Request.Context(), indexName, datasetID)
	if err != nil {
		jsonInternalError(c, err)
		return
	}

	result := gin.H{
		"graph":    map[string]interface{}{},
		"mind_map": map[string]interface{}{},
	}
	if !exists {
		common.RespondSuccess(c, result, "success")
		return
	}

	searchResult, err := docEngine.Search(c.Request.Context(), &types.SearchRequest{
		IndexNames:   []string{indexName},
		KbIDs:        []string{datasetID},
		Offset:       0,
		Limit:        1,
		SelectFields: []string{"content_with_weight", "type_kwd", "knowledge_graph_kwd"},
		Filter: map[string]interface{}{
			"kb_id":               []string{datasetID},
			"knowledge_graph_kwd": []string{"graph"},
		},
	})
	if err != nil {
		jsonInternalError(c, err)
		return
	}
	if searchResult == nil || len(searchResult.Chunks) == 0 {
		common.RespondSuccess(c, result, "success")
		return
	}

	chunk := searchResult.Chunks[0]
	graphType := firstStringValue(chunk["type_kwd"])
	if graphType == "" {
		graphType = firstStringValue(chunk["knowledge_graph_kwd"])
	}
	contentWithWeight, _ := chunk["content_with_weight"].(string)
	if strings.TrimSpace(contentWithWeight) == "" {
		common.RespondSuccess(c, result, "success")
		return
	}

	var graphData map[string]interface{}
	if err = json.Unmarshal([]byte(contentWithWeight), &graphData); err != nil {
		common.RespondSuccess(c, result, "success")
		return
	}
	if len(graphData) == 0 {
		common.RespondSuccess(c, result, "success")
		return
	}

	if graphType == "" {
		graphType = "graph"
	}
	if graphType == "graph" {
		sortKnowledgeGraph(graphData)
		result["graph"] = graphData
	} else {
		result[graphType] = graphData
	}

	common.RespondSuccess(c, result, "success")
}

// DeleteKnowledgeGraph handles DELETE /api/v1/datasets/:dataset_id/graph.
func (h *DatasetsHandler) DeleteKnowledgeGraph(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	datasetID := strings.TrimSpace(c.Param("dataset_id"))
	if datasetID == "" {
		common.RespondError(c, common.CodeDataError, "dataset_id is required")
		return
	}

	ctx := c.Request.Context()
	datasetInstance, code, err := h.datasetsService.GetDataset(ctx, datasetID)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	tenantID, _ := datasetInstance["tenant_id"].(string)
	if tenantID == "" {
		common.RespondError(c, common.CodeDataError, "tenant_id is required")
		return
	}

	docEngine := engine.Get()
	if docEngine == nil {
		common.RespondError(c, common.CodeServerError, "Document engine is not initialized")
		return
	}

	indexName := fmt.Sprintf("ragflow_%s", tenantID)
	if _, err := docEngine.DeleteChunks(c.Request.Context(), map[string]interface{}{
		"knowledge_graph_kwd": []interface{}{"graph", "subgraph", "entity", "relation", "community_report"},
		"kb_id":               datasetID,
	}, indexName, datasetID); err != nil {
		jsonInternalError(c, err)
		return
	}

	common.RespondSuccess(c, nil, "success")
}

// CheckEmbedding Check embedding model compatibility by sampling random chunks,
// re-embedding them with the new model, and computing cosine similarity.
func (h *DatasetsHandler) CheckEmbedding(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	datasetID := strings.TrimSpace(c.Param("dataset_id"))
	if datasetID == "" {
		common.RespondError(c, common.CodeDataError, "dataset_id is required")
		return
	}

	var req service.CheckEmbeddingRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		common.RespondError(c, common.CodeDataError, err.Error())
		return
	}
	if strings.TrimSpace(req.EmbeddingID) == "" {
		common.RespondError(c, common.CodeDataError, "`embd_id` is required.")
		return
	}

	ctx := c.Request.Context()
	data, code, err := h.datasetsService.CheckEmbedding(ctx, datasetID, &req)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}
	common.RespondSuccess(c, data, "success")
}

// AggregateTags handles GET /api/v1/datasets/tags/aggregation.
// @Summary Aggregate dataset tags
// @Description Aggregate tags across multiple datasets
// @Tags datasets
// @Produce json
// @Security ApiKeyAuth
// @Param dataset_ids query string true "Comma-separated dataset IDs"
// @Success 200 {object} map[string]interface{}
// @Router /api/v1/datasets/tags/aggregation [get]
func (h *DatasetsHandler) AggregateTags(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	rawIDs := strings.Split(c.Query("dataset_ids"), ",")
	datasetIDs := make([]string, 0, len(rawIDs))

	for _, rawID := range rawIDs {
		tempID := strings.TrimSpace(rawID)
		if tempID != "" {
			datasetIDs = append(datasetIDs, tempID)
		}
	}
	if len(datasetIDs) == 0 {
		common.RespondError(c, common.CodeDataError, "Lack of dataset_ids in query parameters")
		return
	}

	ctx := c.Request.Context()

	result, code, err := h.datasetsService.AggregateTags(ctx, datasetIDs)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}
	common.RespondSuccessList(c, result, "success")
}

// GetCompilationStatus returns the dataset-level knowledge-compile lifecycle
// state. The optional `kind` query parameter scopes the status to one compile
// type.
func (h *DatasetsHandler) GetCompilationStatus(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}
	datasetID := strings.TrimSpace(c.Param("dataset_id"))
	if datasetID == "" {
		common.RespondError(c, common.CodeDataError, "dataset_id is required")
		return
	}
	ctx := c.Request.Context()
	kind := strings.TrimSpace(c.Query("kind"))
	st, code, err := h.datasetsService.GetDatasetCompilationStatus(ctx, datasetID, kind)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}
	common.RespondSuccess(c, st, "success")
}

// ListMetadataFlattened handles GET /api/v1/datasets/metadata/flattened.
// @Summary List flattened metadata for datasets
// @Description Get flattened metadata for multiple datasets
// @Tags datasets
// @Produce json
// @Security ApiKeyAuth
// @Param dataset_ids query string true "Comma-separated dataset IDs"
// @Success 200 {object} map[string]interface{}
// @Router /api/v1/datasets/metadata/flattened [get]
func (h *DatasetsHandler) ListMetadataFlattened(c *gin.Context) {

	datasetIDsStr := c.Query("dataset_ids")
	if datasetIDsStr == "" {
		common.RespondError(c, common.CodeDataError, "dataset_ids is required")
		return
	}

	rawIDs := strings.Split(datasetIDsStr, ",")
	datasetIDs := make([]string, 0, len(rawIDs))
	for _, id := range rawIDs {
		id = strings.TrimSpace(id)
		if id != "" {
			datasetIDs = append(datasetIDs, id)
		}
	}
	if len(datasetIDs) == 0 {
		common.RespondError(c, common.CodeDataError, "dataset_ids is required")
		return
	}

	ctx := c.Request.Context()
	// Check access for each dataset
	for _, datasetID := range datasetIDs {
		if err := permission.NewDatabaseChecker(dao.DB).CheckResource(ctx, permission.ResourceRef{Kind: permission.ResourceKindDataset, ID: datasetID}, permission.OperationRead); err != nil {
			respondPermissionError(c, err, false)
			return
		}
	}

	flattenedMeta, err := h.metadataService.GetFlattedMetaByKBs(ctx, datasetIDs)
	if err != nil {
		common.RespondError(c, common.CodeServerError, "Failed to get metadata: "+err.Error())
		return
	}

	common.RespondSuccess(c, flattenedMeta, "success")
}

func (h *DatasetsHandler) UpdateDocumentMetadataConfig(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	datasetID := strings.TrimSpace(c.Param("dataset_id"))
	if datasetID == "" {
		common.RespondError(c, common.CodeArgumentError, "dataset_id is required")
		return
	}
	documentID := strings.TrimSpace(c.Param("document_id"))
	if documentID == "" {
		common.RespondError(c, common.CodeArgumentError, "document_id is required")
		return
	}
	var req map[string]interface{}
	if err := c.ShouldBindJSON(&req); err != nil {
		common.RespondError(c, common.CodeDataError, err.Error())
		return
	}

	ctx := c.Request.Context()
	data, code, err := h.datasetsService.UpdateDocumentMetadataConfig(ctx, datasetID, documentID, req)
	if err != nil {
		common.RespondError(c, code, err.Error())
		return
	}

	common.RespondSuccess(c, data, "success")
}

// SearchDatasets searches chunks across datasets based on a question
// @Summary Search Datasets
// @Description Search for relevant chunks across one or more datasets based on a question
// @Tags datasets
// @Param request body service.SearchDatasetsRequest true "search parameters"
// @Success 200 {object} map[string]interface{}
// @Router /api/v1/datasets/search [post]
func (h *DatasetsHandler) SearchDatasets(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	var req service.SearchDatasetsRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		common.RespondError(c, common.CodeArgumentError, err.Error())
		return
	}

	if req.Page == nil {
		defaultPage := common.DefaultPage
		req.Page = &defaultPage
	}
	if req.PageSize == nil && req.Size == nil {
		defaultSize := common.DefaultPageSize
		req.PageSize = &defaultSize
	}
	if req.KNNTopK == nil && req.TopK == nil {
		defaultTopK := 1024
		req.KNNTopK = &defaultTopK
	}
	if req.KNNNumCandidates == nil {
		defaultNumCandidates := 2048
		req.KNNNumCandidates = &defaultNumCandidates
	}
	if req.UseKG == nil {
		defaultUseKG := false
		req.UseKG = &defaultUseKG
	}

	req.Question = strings.TrimSpace(req.Question)
	if req.Question == "" {
		common.RespondError(c, common.CodeArgumentError, "question is required")
		return
	}
	if req.DatasetIDs == nil {
		common.RespondError(c, common.CodeArgumentError, "dataset_ids is required")
		return
	}

	if len(req.DatasetIDs) == 0 {
		common.RespondError(c, common.CodeArgumentError, "dataset_ids array cannot be empty")
		return
	}
	if err := validateSearchDatasetsRequest(&req); err != nil {
		common.RespondError(c, common.CodeArgumentError, err.Error())
		return
	}

	searchService := h.searchDatasetsService
	if searchService == nil {
		searchService = h.datasetsService
	}
	if searchService == nil {
		common.RespondError(c, common.CodeDataError, "dataset service is not initialized")
		return
	}

	ctx := c.Request.Context()

	resp, err := searchService.SearchDatasets(ctx, &req)
	if err != nil {
		if respondPermissionErrorIf(c, err, false) {
			return
		}
		common.RespondError(c, common.CodeDataError, err.Error())
		return
	}

	common.RespondSuccess(c, resp, "success")
}

// SearchDataset searches chunks within a single dataset based on a question.
// @Summary Search Dataset
// @Description Search for relevant chunks within one dataset based on a question
// @Tags datasets
// @Param dataset_id path string true "dataset id"
// @Param request body service.SearchDatasetRequest true "search parameters"
// @Success 200 {object} map[string]interface{}
// @Router /api/v1/datasets/{dataset_id}/search [post]
func (h *DatasetsHandler) SearchDataset(c *gin.Context) {
	_, errorCode, errorMessage := GetUser(c)
	if errorCode != common.CodeSuccess {
		common.RespondError(c, errorCode, errorMessage)
		return
	}

	datasetID := c.Param("dataset_id")
	if datasetID == "" {
		common.RespondError(c, common.CodeDataError, "dataset_id is required")
		return
	}

	var req service.SearchDatasetRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		common.RespondError(c, common.CodeArgumentError, err.Error())
		return
	}
	req.Question = strings.TrimSpace(req.Question)
	if req.Question == "" {
		common.RespondError(c, common.CodeArgumentError, "question is required")
		return
	}
	if err := validateSearchDatasetRequest(&req); err != nil {
		common.RespondError(c, common.CodeArgumentError, err.Error())
		return
	}

	searchService := h.searchDatasetService
	if searchService == nil {
		searchService = h.datasetsService
	}
	if searchService == nil {
		common.RespondError(c, common.CodeDataError, "dataset service is not initialized")
		return
	}

	ctx := c.Request.Context()

	resp, err := searchService.SearchDataset(ctx, datasetID, &req)
	if err != nil {
		if respondPermissionErrorIf(c, err, false) {
			return
		}
		common.RespondError(c, common.CodeDataError, err.Error())
		return
	}

	common.RespondSuccess(c, resp, "success")
}

func validateSearchDatasetsRequest(req *service.SearchDatasetsRequest) error {
	if err := service.ValidateKnowledgeCompilationScope(req.KnowledgeCompilationScope); err != nil {
		return err
	}
	vectorSimilarityWeight, err := service.ResolveVectorSimilarityWeight(req.KeywordsSimilarityWeight, req.VectorSimilarityWeight)
	if err != nil {
		return err
	}
	req.VectorSimilarityWeight = vectorSimilarityWeight
	return validateSearchParams(req.Page, req.PageSize, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)
}

func validateSearchDatasetRequest(req *service.SearchDatasetRequest) error {
	if err := service.ValidateKnowledgeCompilationScope(req.KnowledgeCompilationScope); err != nil {
		return err
	}
	vectorSimilarityWeight, err := service.ResolveVectorSimilarityWeight(req.KeywordsSimilarityWeight, req.VectorSimilarityWeight)
	if err != nil {
		return err
	}
	req.VectorSimilarityWeight = vectorSimilarityWeight
	return validateSearchParams(req.Page, req.PageSize, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)
}

func validateSearchParams(page, pageSize, size, knnTopK, topK, knnNumCandidates *int, similarityThreshold, vectorSimilarityWeight *float64) error {
	if err := common.ValidatePagination(page, pageSize, size); err != nil {
		return err
	}
	effectiveKNNTopK := 1024
	knnTopKName := "knn_top_k"
	if knnTopK != nil {
		effectiveKNNTopK = *knnTopK
	} else if topK != nil {
		effectiveKNNTopK = *topK
		knnTopKName = "top_k (alias for knn_top_k)"
	}
	if effectiveKNNTopK < 1 || effectiveKNNTopK > 2048 {
		return fmt.Errorf("%s must be between 1 and 2048", knnTopKName)
	}
	effectiveKNNNumCandidates := 2048
	if knnNumCandidates != nil {
		effectiveKNNNumCandidates = *knnNumCandidates
	}
	if effectiveKNNNumCandidates < 1 || effectiveKNNNumCandidates > 10000 {
		return fmt.Errorf("knn_num_candidates must be between 1 and 10000")
	}
	if effectiveKNNNumCandidates < effectiveKNNTopK {
		return fmt.Errorf("knn_num_candidates must be greater than or equal to knn_top_k")
	}
	if similarityThreshold != nil && (*similarityThreshold < 0 || *similarityThreshold > 1) {
		return fmt.Errorf("similarity_threshold must be between 0 and 1")
	}
	if vectorSimilarityWeight != nil && (*vectorSimilarityWeight < 0 || *vectorSimilarityWeight > 1) {
		return fmt.Errorf("vector_similarity_weight must be between 0 and 1")
	}
	return nil
}

func firstStringValue(value interface{}) string {
	switch v := value.(type) {
	case string:
		return strings.TrimSpace(v)
	case []string:
		if len(v) > 0 {
			return strings.TrimSpace(v[0])
		}
	case []interface{}:
		for _, item := range v {
			if s, ok := item.(string); ok {
				s = strings.TrimSpace(s)
				if s != "" {
					return s
				}
			}
		}
	}
	return ""
}

func sortKnowledgeGraph(graphData map[string]interface{}) {
	nodes := mapSlice(graphData["nodes"])
	if len(nodes) > 0 {
		sort.Slice(nodes, func(i, j int) bool {
			return numericValue(nodes[i]["pagerank"]) > numericValue(nodes[j]["pagerank"])
		})
		if len(nodes) > 256 {
			nodes = nodes[:256]
		}
		graphData["nodes"] = nodes
	}

	edges := mapSlice(graphData["edges"])
	if len(edges) > 0 {
		nodeIDSet := make(map[string]struct{}, len(nodes))
		for _, node := range nodes {
			if id, ok := node["id"].(string); ok {
				nodeIDSet[id] = struct{}{}
			}
		}
		filteredEdges := make([]map[string]interface{}, 0, len(edges))
		for _, edge := range edges {
			source, _ := edge["source"].(string)
			target, _ := edge["target"].(string)
			if source == "" || target == "" || source == target {
				continue
			}
			if _, ok := nodeIDSet[source]; !ok {
				continue
			}
			if _, ok := nodeIDSet[target]; !ok {
				continue
			}
			filteredEdges = append(filteredEdges, edge)
		}
		sort.Slice(filteredEdges, func(i, j int) bool {
			return numericValue(filteredEdges[i]["weight"]) > numericValue(filteredEdges[j]["weight"])
		})
		if len(filteredEdges) > 128 {
			filteredEdges = filteredEdges[:128]
		}
		graphData["edges"] = filteredEdges
	}
}

func mapSlice(value interface{}) []map[string]interface{} {
	raw, ok := value.([]interface{})
	if !ok {
		return nil
	}
	result := make([]map[string]interface{}, 0, len(raw))
	for _, item := range raw {
		if m, ok := item.(map[string]interface{}); ok {
			result = append(result, m)
		}
	}
	return result
}

func numericValue(value interface{}) float64 {
	switch v := value.(type) {
	case float64:
		return v
	case float32:
		return float64(v)
	case int:
		return float64(v)
	case int64:
		return float64(v)
	case json.Number:
		f, _ := v.Float64()
		return f
	default:
		return 0
	}
}