Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

67.7 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package chunk

import (
	"bytes"
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"image"
	"image/color"
	"image/draw"
	"image/jpeg"
	"math"

	"ragflow/internal/common"
	"ragflow/internal/entity"
	"ragflow/internal/entity/models"
	"ragflow/internal/permission"
	permissionresponse "ragflow/internal/permission/response"
	"ragflow/internal/service"
	"strings"
	"sync"
	"time"

	_ "image/gif"
	_ "image/png"

	"go.uber.org/zap"
	"gorm.io/gorm"

	"ragflow/internal/dao"
	"ragflow/internal/dao/transaction"
	"ragflow/internal/engine"
	"ragflow/internal/engine/types"
	"ragflow/internal/ingestion/knowledge_compile"
	"ragflow/internal/service/document"
	"ragflow/internal/service/nlp"
	"ragflow/internal/storage"
	"ragflow/internal/tokenizer"
	"ragflow/internal/utility"
)

var chunkImageLocks = struct {
	sync.Mutex
	locks map[string]*chunkImageLock
}{locks: make(map[string]*chunkImageLock)}

type chunkImageLock struct {
	mu   sync.Mutex
	refs int
}

func searchConfigMap(value interface{}) (map[string]interface{}, bool) {
	switch typed := value.(type) {
	case entity.JSONMap:
		return map[string]interface{}(typed), true
	case map[string]interface{}:
		return typed, true
	default:
		return nil, false
	}
}

// ChunkService chunk service
type ChunkService struct {
	docEngine        engine.DocEngine
	embeddingCache   *utility.EmbeddingLRU
	kbDAO            *dao.KnowledgebaseDAO
	documentDAO      *dao.DocumentDAO
	taskDAO          *dao.TaskDAO
	ingestionTaskDAO *dao.IngestionTaskDAO
	searchService    *service.SearchService

	checkDatasetAccessFunc   func(string) error
	getKnowledgebaseByIDFunc func(string) (*entity.Knowledgebase, error)
	getDocumentsByIDsFunc    func([]string) ([]*entity.Document, error)
	// startParseDocumentsFunc overrides the DSL start-parse flow. Production
	// uses service.DocumentService.StartParseDocuments; tests inject a fake
	// to avoid the MQ publisher.
	startParseDocumentsFunc func(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, opts document.StartParseOptions) error
	// cancelIngestionTaskFunc overrides the document-parsing cancellation.
	// Production uses service.DocumentService.CancelDocParse; tests inject
	// a fake to avoid the MQ publisher.
	cancelIngestionTaskFunc func(ctx context.Context, doc *entity.Document) error
	getEmbeddingModelFunc   func(string, string) (*models.EmbeddingModel, error)
	incrementChunkStatsFunc func(string, string, int64, int64, float64) error
	decrementChunkStatsFunc func(string, string, int64, int64, float64) error
	storeChunkImageFunc     func(string, string, []byte, string) error
	removeChunkImageFunc    func(string, string) error
	markWikiDirtyFunc       func(tenantID, datasetID, documentID string, chunkIDs []string)
	tokenizeFunc            func(text, language string) (string, error)
	fineGrainedTokenizeFunc func(text, language string) (string, error)
	numTokensFunc           func(string) int
}

// NewChunkService creates chunk service
func NewChunkService() *ChunkService {
	return &ChunkService{
		docEngine:        engine.Get(),
		embeddingCache:   utility.NewEmbeddingLRU(1000), // default capacity
		kbDAO:            dao.NewKnowledgebaseDAO(),
		documentDAO:      dao.NewDocumentDAO(),
		taskDAO:          dao.NewTaskDAO(),
		ingestionTaskDAO: dao.NewIngestionTaskDAO(),
		searchService:    service.NewSearchService(),
	}
}

// RetrievalTest performs retrieval test for a given question against specified knowledge bases.
//
// Flow:
//  1. Validate kbs permissions and embedding model
//  2. Apply metadata filter if specified (auto/semi_auto uses LLM, manual uses provided conditions)
//  3. Apply cross_languages transformation if requested (translate question)
//  4. Apply keyword extraction if requested (append keywords to question)
//  5. Get rank features via LabelQuestion() - tag-based weights or pagerank_fld fallback
//  6. Call RetrievalService.Retrieval() which:
//     - Computes query embedding
//     - Performs hybrid search (text + vector) with rank features
//     - Reranks results
//     - Builds doc_aggs by aggregating chunks per document
//  7. knowledge graph retrieval (not implemented)
//  8. Apply retrieval by children to group child chunks under parent chunks
func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.RetrievalTestRequest) (*service.RetrievalTestResponse, error) {
	common.Info("RetrievalTest started", zap.String("userID", permission.UserIDFromContext(ctx)), zap.Any("kbID", req.Datasets), zap.String("question", req.Question))

	common.Debug(fmt.Sprintf("RetrievalTest request:\n"+
		"    kbID=%v\n"+
		"    question=%s\n"+
		"    page=%v, size=%v\n"+
		"    docIDs=%v\n"+
		"    useKG=%v, topK=%v\n"+
		"    crossLanguages=%v\n"+
		"    searchID=%v\n"+
		"    filter=%v\n"+
		"    tenantRerankID=%v\n"+
		"    rerankID=%v\n"+
		"    keyword=%v\n"+
		"    similarityThreshold=%v, vectorSimilarityWeight=%v",
		req.Datasets, req.Question,
		common.PtrString(req.Page), common.PtrString(req.Size), req.DocIDs,
		common.PtrString(req.UseKG), common.PtrString(req.TopK), req.CrossLanguages, common.PtrString(req.SearchID),
		req.Filter,
		common.PtrString(req.TenantRerankID), common.PtrString(req.RerankID),
		common.PtrString(req.Keyword),
		common.PtrString(req.SimilarityThreshold), common.PtrString(req.VectorSimilarityWeight)))

	if req.Question == "" {
		return nil, fmt.Errorf("question is required")
	}
	if len(req.Datasets) == 0 {
		return nil, fmt.Errorf("dataset_ids is required")
	}
	modelFactory := service.NewModelFactory()

	var tenantIDs []string
	var kbRecords []*entity.Knowledgebase
	for _, datasetID := range req.Datasets {
		if err := permission.NewDatabaseChecker(dao.DB).CheckResource(ctx, permission.ResourceRef{
			Kind: permission.ResourceKindDataset,
			ID:   datasetID,
		}, permission.OperationUse); err != nil {
			_, permissionErr := permissionresponse.NormalizeHidden(err)
			return nil, permissionErr
		}
		kb, err := s.getKnowledgebaseByID(ctx, datasetID)
		if err != nil {
			return nil, fmt.Errorf("failed to load dataset %s: %w", datasetID, err)
		}
		if kb == nil {
			return nil, fmt.Errorf("dataset %s not found", datasetID)
		}
		common.Debug("Found knowledge base in database",
			zap.String("datasetID", datasetID),
			zap.String("tenantID", kb.TenantID),
			zap.String("kbName", kb.Name),
			zap.String("embdID", kb.EmbdID))
		tenantIDs = append(tenantIDs, kb.TenantID)
		kbRecords = append(kbRecords, kb)
	}

	// Check if all kbs resolve to the same base embedding model
	if len(kbRecords) > 1 {
		embdNames := transaction.NewEmbeddingBaseNameResolver(dao.NewKnowledgebaseDAO().EmbeddingBaseName)
		firstEmbeddingKey := knowledgebaseEmbeddingKey(ctx, dao.DB, kbRecords[0], tenantIDs[0], embdNames)
		for i := 1; i < len(kbRecords); i++ {
			if knowledgebaseEmbeddingKey(ctx, dao.DB, kbRecords[i], tenantIDs[i], embdNames) != firstEmbeddingKey {
				return nil, fmt.Errorf("cannot retrieve across datasets with different embedding models")
			}
		}
	}

	// Determine meta_data_filter
	var chatID string
	var chatModelForFilter *models.ChatModel
	// Filled by the search_id block below, then used to resolve retrieval
	// parameters.
	var searchConfig map[string]interface{}
	filter := req.Filter
	tenantRerankID := req.TenantRerankID
	rerankID := req.RerankID
	rerankCandidatesCount := 64
	if req.RerankCandidatesCount != nil {
		rerankCandidatesCount = *req.RerankCandidatesCount
	}

	if req.SearchID != nil && *req.SearchID != "" {
		// If search_id is set, get meta_data_filter and chat_id from search_config
		if s.searchService == nil {
			return nil, fmt.Errorf("search service not configured")
		}
		searchDetail, err := s.searchService.GetDetail(ctx, *req.SearchID)
		if err != nil {
			return nil, fmt.Errorf("failed to get search app %s: %w", *req.SearchID, err)
		}
		if cfg, ok := searchConfigMap(searchDetail["search_config"]); ok && cfg != nil {
			searchConfig = cfg
			if req.RerankCandidatesCount == nil || *req.RerankCandidatesCount == 0 {
				rerankCandidatesCount = 100
				if configuredRerankCandidatesCount, ok := common.GetInt(searchConfig["rerank_candidates_count"]); ok {
					rerankCandidatesCount = configuredRerankCandidatesCount
				}
			}
			if searchMetaFilter, ok := searchConfigMap(searchConfig["meta_data_filter"]); ok {
				filter = searchMetaFilter
			}
			if tenantRerankID == nil || strings.TrimSpace(*tenantRerankID) == "" {
				if configuredTenantRerankID, ok := searchConfig["tenant_rerank_id"].(string); ok && strings.TrimSpace(configuredTenantRerankID) != "" {
					tenantRerankID = &configuredTenantRerankID
				}
			}
			if rerankID == nil || strings.TrimSpace(*rerankID) == "" {
				if configuredRerankID, ok := searchConfig["rerank_id"].(string); ok && strings.TrimSpace(configuredRerankID) != "" {
					rerankID = &configuredRerankID
				}
			}
			chatID, _ = searchConfig["chat_id"].(string)
		} else {
			common.Warn("No search_config found in search detail", zap.String("searchID", *req.SearchID))
		}
	}

	// If meta_data_filter method is auto/semi_auto, get chat model
	var getErr error
	if filter != nil {
		method, _ := filter["method"].(string)
		if method == "auto" || method == "semi_auto" {
			modelCtx := permission.WithTenantID(ctx, tenantIDs[0])
			if chatID != "" {
				chatModelForFilter, getErr = modelFactory.NewChatModel(modelCtx, chatID)
				if getErr != nil {
					common.Warn("Failed to get chat model from search_config chat_id, using tenant default", zap.String("chatID", chatID), zap.Error(getErr))
				} else {
					common.Info("Fetched chat model (from search_config) for metadata filter",
						zap.String("chatID", chatID),
						zap.String("tenantID", tenantIDs[0]))
				}
			}

			if chatModelForFilter == nil {
				chatModelForFilter, getErr = modelFactory.NewDefaultChatModel(modelCtx)
				if getErr != nil {
					common.Warn("Failed to get tenant default chat model for meta_data_filter", zap.Error(getErr))
				}
			}
		}
	}

	// Apply meta_data_filter to get filtered doc_ids (filter by metadata before retrieval)
	docIDs := make([]string, len(req.DocIDs))
	copy(docIDs, req.DocIDs)
	if filter != nil {
		// Get flattened metadata
		metadataSvc := service.NewMetadataService()
		flattedMeta, err := metadataSvc.GetFlattedMetaByKBs(ctx, []string(req.Datasets))
		if err != nil {
			common.Warn("Failed to get flatted metadata", zap.Error(err))
		} else {
			common.Info("metadata filter conditions", zap.Any("filter", filter))
			// nil means no metadata narrowing: search the caller's scope
			// unfiltered rather than scoping to nothing.
			if filteredDocIDs := service.ApplyMetaDataFilter(ctx, filter, flattedMeta, req.Question, chatModelForFilter, req.DocIDs, []string(req.Datasets)); filteredDocIDs != nil {
				docIDs = filteredDocIDs
			}
			common.Info("ApplyMetaDataFilter result", zap.Strings("docIDs", docIDs))
		}
	}

	// Apply cross_languages and keyword extraction with tenant default chat model
	modifiedQuestion := req.Question
	var chatModel *models.ChatModel

	// Get chat model for cross_languages and keyword_extraction
	var llmModelName string
	if len(req.CrossLanguages) > 0 || (req.Keyword != nil && *req.Keyword) {
		var err error
		chatModel, err = modelFactory.NewDefaultChatModel(permission.WithTenantID(ctx, tenantIDs[0]))
		if err != nil {
			common.Warn("Failed to get default chat model name for LLM transformations", zap.Error(err))
		} else {
			if info := chatModel.Info(); info != nil {
				llmModelName = service.ModelReference(info)
			}
			common.Info("Fetched chat model (tenant default) for cross_languages/keyword_extraction",
				zap.String("tenantID", tenantIDs[0]),
				zap.String("modelName", llmModelName))
		}
	}

	// Apply cross_languages on the question (translate question)
	if len(req.CrossLanguages) > 0 {
		translated, err := service.CrossLanguages(ctx, tenantIDs[0], llmModelName, req.Question, req.CrossLanguages)
		if err != nil {
			common.Warn("Failed to translate question", zap.Error(err))
		} else {
			modifiedQuestion = translated
		}
	}

	// Apply keyword extraction on the question (append keywords to question)
	if chatModel != nil && req.Keyword != nil && *req.Keyword {
		extractedKeywords, err := service.KeywordExtraction(ctx, chatModel, modifiedQuestion, 3)
		if err != nil {
			common.Warn("Failed to extract keywords from question", zap.Error(err))
		} else {
			modifiedQuestion = service.AppendKeywords(modifiedQuestion, extractedKeywords)
		}
	}

	if modifiedQuestion != req.Question {
		common.Info("Modified question after transformations",
			zap.String("originalQuestion", req.Question),
			zap.String("modifiedQuestion", modifiedQuestion),
			zap.Strings("crossLanguages", req.CrossLanguages),
			zap.Bool("keywordExtraction", req.Keyword != nil && *req.Keyword))
	}

	// Get tag-based rank features via LabelQuestion. The result is logged inside
	// LabelQuestion (info on match, debug when skipped), so no log here.
	metadataSvc := service.NewMetadataService()
	labels := metadataSvc.LabelQuestion(ctx, modifiedQuestion, kbRecords)

	// Determine embedding model.
	modelCtx := permission.WithTenantID(ctx, tenantIDs[0])
	var embeddingModel *models.EmbeddingModel
	var embdID string
	if kbRecords[0].TenantEmbdID != nil && *kbRecords[0].TenantEmbdID != "" {
		embeddingModel, getErr = modelFactory.NewEmbeddingModel(modelCtx, *kbRecords[0].TenantEmbdID)
		if getErr != nil {
			return nil, fmt.Errorf("failed to get embedding model by tenant_embd_id: %w", getErr)
		}
	} else if kbRecords[0].EmbdID != "" {
		embdID = kbRecords[0].EmbdID
		embeddingModel, getErr = modelFactory.NewEmbeddingModel(modelCtx, embdID)
		if getErr != nil {
			return nil, fmt.Errorf("failed to get embedding model by embd_id: %w", getErr)
		}
	} else {
		embeddingModel, getErr = modelFactory.NewDefaultEmbeddingModel(modelCtx)
		if getErr != nil {
			return nil, fmt.Errorf("failed to get tenant default embedding model: %w", getErr)
		}
	}

	if embeddingModel == nil {
		return nil, fmt.Errorf("no embedding model found for tenant %s", tenantIDs[0])
	}
	modelName := ""
	if info := embeddingModel.Info(); info != nil {
		modelName = info.Name
	}
	common.Info("Fetched embedding model for retrieval",
		zap.String("tenantID", tenantIDs[0]),
		zap.String("modelName", modelName))

	// Get rerank model if RerankID is specified
	var rerankModel *models.RerankModel
	if tenantRerankID != nil && *tenantRerankID != "" {
		rerankModel, getErr = modelFactory.NewRerankModel(modelCtx, *tenantRerankID)
		if getErr != nil {
			return nil, fmt.Errorf("failed to get rerank model by tenant_rerank_id: %w", getErr)
		}
	} else if rerankID != nil && *rerankID != "" {
		rerankModel, getErr = modelFactory.NewRerankModel(modelCtx, *rerankID)
		if getErr != nil {
			return nil, fmt.Errorf("failed to get rerank model by rerank_id: %w", getErr)
		}
	}

	// Resolved here because search_config is only known after the search_id
	// block above.
	similarityThreshold, vectorSimilarityWeight, topK := resolveSearchbotRetrievalParams(req, searchConfig)

	retrievalReq := &nlp.RetrievalRequest{
		TenantIDs:              tenantIDs,
		Question:               modifiedQuestion,
		KbIDs:                  []string(req.Datasets),
		DocIDs:                 docIDs,
		Page:                   common.CoalesceInt(req.Page, common.DefaultPage),
		PageSize:               common.CoalesceInt(req.Size, common.DefaultPageSize),
		RerankCandidatesCount:  &rerankCandidatesCount,
		KNNTopK:                &topK,
		SimilarityThreshold:    &similarityThreshold,
		VectorSimilarityWeight: &vectorSimilarityWeight,
		RerankModel:            rerankModel,
		RankFeature:            &labels,
		EmbeddingModel:         embeddingModel,
		Highlight:              req.Highlight,
		Language:               entity.KnowledgebasesLanguage(kbRecords),
	}

	// Call RetrievalService to perform retrieval
	retrievalResult, err := nlp.NewRetrievalService(s.docEngine, s.documentDAO).Retrieval(ctx, retrievalReq)
	if err != nil {
		return nil, fmt.Errorf("retrieval search failed: %w", err)
	}

	filteredChunks := retrievalResult.Chunks

	// Handle knowledge graph retrieval
	// TODO: KG retrieval requires GraphRAG infrastructure which is not yet implemented in Go
	if req.UseKG != nil && *req.UseKG {
		common.Warn("use_kg is not yet implemented in Go - skipping KG retrieval")
	}

	// Apply retrieval_by_children - aggregate child chunks into parent chunks
	filteredChunks = nlp.RetrievalByChildren(filteredChunks, tenantIDs, s.docEngine, ctx)

	// Hydrate: ES returns zero vectors; replace with real vectors from FetchChunkVectors.
	// Infinity/OceanBase chunks already carry real vectors and are left unchanged.
	hydrateChunkVectors(ctx, s.docEngine, filteredChunks, req.Datasets, tenantIDs)

	common.Info("RetrievalTest completed", zap.String("userID", permission.UserIDFromContext(ctx)), zap.Any("kbID", req.Datasets), zap.String("question", req.Question), zap.Int64("chunkCount", int64(len(filteredChunks))))

	return &service.RetrievalTestResponse{
		Chunks:  filteredChunks,
		DocAggs: retrievalResult.DocAggs,
		Labels:  &labels,
		Total:   retrievalResult.Total,
	}, nil
}

// Endpoint defaults for the searchbot retrieval_test path, used when neither
// the request nor the saved search_config provides a value.
const (
	searchbotDefaultSimilarityThreshold    = 0.0
	searchbotDefaultVectorSimilarityWeight = 0.3
	searchbotDefaultTopK                   = 1024
)

// resolveSearchbotRetrievalParams resolves similarity_threshold,
// vector_similarity_weight and top_k from the request, falling back to the
// search app's saved search_config and then to the endpoint defaults. A nil
// pointer means the field was omitted, so an explicit zero is kept — it is how
// a caller disables the threshold floor or selects a zero vector weight.
// searchConfig may be nil.
//
// vector_similarity_weight also derives from keywords_similarity_weight
// (vector = 1 - keywords) when the config carries only the keyword form.
func resolveSearchbotRetrievalParams(req *service.RetrievalTestRequest, searchConfig map[string]interface{}) (similarityThreshold, vectorSimilarityWeight float64, topK int) {
	similarityThreshold = searchbotDefaultSimilarityThreshold
	if req.SimilarityThreshold != nil {
		similarityThreshold = *req.SimilarityThreshold
	} else if configured, ok := common.ToFloat(searchConfig["similarity_threshold"]); ok {
		similarityThreshold = configured
	}

	vectorSimilarityWeight = searchbotDefaultVectorSimilarityWeight
	if req.VectorSimilarityWeight != nil {
		vectorSimilarityWeight = *req.VectorSimilarityWeight
	} else if configured, ok := common.ToFloat(searchConfig["vector_similarity_weight"]); ok {
		vectorSimilarityWeight = configured
	} else if keywords, ok := common.ToFloat(searchConfig["keywords_similarity_weight"]); ok {
		vectorSimilarityWeight = 1 - keywords
	}

	topK = searchbotDefaultTopK
	if req.TopK != nil {
		topK = *req.TopK
	} else if configured, ok := common.GetInt(searchConfig["top_k"]); ok {
		topK = configured
	}
	return similarityThreshold, vectorSimilarityWeight, topK
}

// knowledgebaseEmbeddingKey groups datasets by their resolved base embedding
// model name (e.g. "BAAI/bge-m3") so datasets pointing at the same model
// through different provider instances or storage forms (tenant_model id vs
// legacy composite name) retrieve together.
func knowledgebaseEmbeddingKey(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase, tenantID string, names *transaction.EmbeddingBaseNameResolver) string {
	if strings.TrimSpace(kb.EmbdID) == "" && (kb.TenantEmbdID == nil || strings.TrimSpace(*kb.TenantEmbdID) == "") {
		return fmt.Sprintf("default:%s", tenantID)
	}
	return "embd:" + names.Resolve(ctx, db, kb)
}

// hydrateChunkVectors replaces zero (placeholder) vectors in chunks with real
// vectors fetched from the engine.  Infinity and OceanBase already ship real
// vectors with chunks, so this is a no-op for those engines; for ES it queries
// the engine by chunk ID list.  No if/else on engine type — just replaces
// whatever is missing or zero.
func hydrateChunkVectors(ctx context.Context, engine engine.DocEngine, chunks []map[string]interface{}, kbIDs []string, tenantIDs []string) {
	if len(chunks) == 0 {
		return
	}

	// Collect chunk IDs whose vectors are missing or all-zero.
	var missingIDs []string
	missingIdx := make(map[string]int)
	for i, ck := range chunks {
		id, _ := ck["id"].(string)
		if id == "" {
			continue
		}
		v, _ := ck["vector"].([]float64)
		if len(v) == 0 || common.IsZeroVector(v) {
			missingIDs = append(missingIDs, id)
			missingIdx[id] = i
		}
	}
	if len(missingIDs) == 0 {
		return
	}

	dim := 0
	for _, ck := range chunks {
		if v, _ := ck["vector"].([]float64); len(v) > 0 {
			dim = len(v)
			break
		}
	}
	if dim == 0 {
		return
	}

	vectors := FetchChunkVectors(ctx, engine, missingIDs, tenantIDs, kbIDs, dim)
	for id, v := range vectors {
		if idx, ok := missingIdx[id]; ok && !common.IsZeroVector(v) {
			chunks[idx]["vector"] = v
		}
	}
}

// Get retrieves a chunk by ID
func (s *ChunkService) Get(ctx context.Context, req *service.GetChunkRequest) (*service.GetChunkResponse, error) {
	if s.docEngine == nil {
		return nil, fmt.Errorf("doc engine not initialized")
	}

	if req.ChunkID == "" {
		return nil, fmt.Errorf("chunk_id is required")
	}

	kb, err := s.getAccessibleKnowledgebase(ctx, req.DatasetID, permission.OperationRead)
	if err != nil {
		return nil, err
	}
	targetTenantID := kb.TenantID

	// Verify the document belongs to the dataset, mirroring Python's get_chunk
	// (DocumentService.query(id=document_id, kb_id=dataset_id)).
	doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, req.DocumentID)
	if err != nil || doc == nil {
		return nil, fmt.Errorf("document not found")
	}
	if doc.KbID != req.DatasetID {
		return nil, fmt.Errorf("document does not belong to this dataset")
	}

	// The lookup stays inside the dataset named in the route, and the row must
	// resolve to the document named in the route: a chunk id alone is not
	// authority to read another dataset's (or another document's) chunk.
	indexName := fmt.Sprintf("ragflow_%s", targetTenantID)
	rawChunk, err := s.docEngine.GetChunk(ctx, indexName, req.ChunkID, []string{req.DatasetID})
	if err != nil {
		return nil, fmt.Errorf("chunk not found")
	}
	chunk, ok := rawChunk.(map[string]interface{})
	if !ok || chunk == nil {
		return nil, fmt.Errorf("chunk not found")
	}
	if documentID, _ := chunk["doc_id"].(string); documentID != req.DocumentID {
		return nil, fmt.Errorf("chunk not found")
	}
	if !utility.IsEmpty(chunk["compile_kwd"]) {
		return nil, fmt.Errorf("chunk not found")
	}

	// Return the stored row without tokenized/vector runtime fields.
	result := make(map[string]interface{}, len(chunk))
	for k, v := range chunk {
		if isChunkRuntimeField(k) {
			continue
		}
		result[k] = v
	}
	return &service.GetChunkResponse{Chunk: result}, nil
}

// isChunkRuntimeField identifies internal fields omitted from chunk responses.
// The _vec suffix is anchored at the end; token field names use substring matches.
func isChunkRuntimeField(name string) bool {
	return strings.HasSuffix(name, "_vec") ||
		strings.Contains(name, "_sm_") ||
		strings.Contains(name, "_tks") ||
		strings.Contains(name, "_ltks")
}

const (
	docStopParsingInvalidStateMessage   = "Can't stop parsing document that has not started or already completed"
	docStopParsingInvalidStateErrorCode = "DOC_STOP_PARSING_INVALID_STATE"
)

func (s *ChunkService) cancelAllTasksOfDoc(ctx context.Context, doc *entity.Document) error {
	cancel := s.cancelIngestionTaskFunc
	if cancel == nil {
		cancel = document.NewDocumentService().CancelDocParse
	}
	return cancel(ctx, doc)
}

func (s *ChunkService) StopParsing(ctx context.Context, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error) {
	if err := permission.NewDatabaseChecker(dao.DB).CheckResource(ctx, permission.ResourceRef{Kind: permission.ResourceKindDataset, ID: datasetID}, permission.OperationRun); err != nil {
		code, permissionErr := permissionresponse.Normalize(err)
		return nil, code, permissionErr
	}

	if len(req.DocumentIDs) == 0 {
		return nil, common.CodeDataError, fmt.Errorf("`document_ids` is required")
	}

	_, err := s.kbDAO.GetByID(ctx, dao.DB, datasetID)
	if err != nil {
		return nil, common.CodeDataError, fmt.Errorf("you don't own the dataset %s", datasetID)
	}

	docIDs, duplicateMessages := service.CheckDuplicateIDs(req.DocumentIDs, "document")
	successCount := 0

	for _, docID := range docIDs {
		doc, err := s.documentDAO.GetByDocumentIDAndDatasetID(ctx, dao.DB, docID, datasetID)
		if err != nil || doc == nil {
			return nil, common.CodeDataError, fmt.Errorf("you don't own the document %s", docID)
		}

		if err = s.cancelAllTasksOfDoc(ctx, doc); err != nil {
			if errors.Is(err, document.ErrParseNotRunning) {
				return &service.StopParsingResponse{
					Data: map[string]interface{}{"error_code": docStopParsingInvalidStateErrorCode},
				}, common.CodeDataError, fmt.Errorf("%s", docStopParsingInvalidStateMessage)
			}
			return nil, common.CodeServerError, err
		}
		// CancelDocParse (inside cancelAllTasksOfDoc) already issues
		// RequestStop (STOPPING). Defer destruction (chunk deletion, counter
		// reset) until the worker detects STOPPING and reaches a terminal state.

		successCount++
	}

	if len(duplicateMessages) > 0 {
		if successCount > 0 {
			return &service.StopParsingResponse{
				Message: fmt.Sprintf("Partially stopped %d documents with %d errors", successCount, len(duplicateMessages)),
				Data: map[string]interface{}{
					"success_count": successCount,
					"errors":        duplicateMessages,
				},
			}, common.CodeSuccess, nil
		}
		return nil, common.CodeDataError, fmt.Errorf("%s", strings.Join(duplicateMessages, ";"))
	}

	return nil, common.CodeSuccess, nil
}

func checkDuplicateIDs(documentIDs []string, idTypes string) ([]string, []string) {
	idCount := make(map[string]int, len(documentIDs))
	duplicateMessages := make([]string, 0)
	uniqueDocIDs := make([]string, 0, len(documentIDs))

	for _, id := range documentIDs {
		idCount[id]++
	}
	for id, count := range idCount {
		if count > 1 {
			duplicateMessages = append(duplicateMessages, fmt.Sprintf("Duplicate %s ids: %s ", idTypes, id))
		}
		uniqueDocIDs = append(uniqueDocIDs, id)
	}
	return uniqueDocIDs, duplicateMessages
}

func (s *ChunkService) checkDatasetAccess(ctx context.Context, datasetID string, operation permission.Operation) error {
	if s.checkDatasetAccessFunc != nil {
		return s.checkDatasetAccessFunc(datasetID)
	}
	return permission.NewDatabaseChecker(dao.DB).CheckResource(ctx, permission.ResourceRef{Kind: permission.ResourceKindDataset, ID: datasetID}, operation)
}

func (s *ChunkService) getKnowledgebaseByID(ctx context.Context, datasetID string) (*entity.Knowledgebase, error) {
	if s.getKnowledgebaseByIDFunc != nil {
		return s.getKnowledgebaseByIDFunc(datasetID)
	}
	return s.kbDAO.GetByID(ctx, dao.DB, datasetID)
}

func (s *ChunkService) getAccessibleKnowledgebase(ctx context.Context, datasetID string, operation permission.Operation) (*entity.Knowledgebase, error) {
	if err := s.checkDatasetAccess(ctx, datasetID, operation); err != nil {
		return nil, err
	}
	kb, err := s.getKnowledgebaseByID(ctx, datasetID)
	if err != nil || kb == nil {
		return nil, fmt.Errorf("knowledge base not found")
	}
	return kb, nil
}

func (s *ChunkService) getDocumentsByIDs(ctx context.Context, docIDs []string) ([]*entity.Document, error) {
	if s.getDocumentsByIDsFunc != nil {
		return s.getDocumentsByIDsFunc(docIDs)
	}
	return s.documentDAO.GetByIDs(ctx, dao.DB, docIDs)
}

func (s *ChunkService) Parse(ctx context.Context, datasetID string, req *service.ParseFileRequest) (map[string]interface{}, common.ErrorCode, error) {
	if err := s.checkDatasetAccess(ctx, datasetID, permission.OperationRun); err != nil {
		code, permissionErr := permissionresponse.Normalize(err)
		return nil, code, permissionErr
	}
	if req == nil || len(req.DocumentIDs) == 0 {
		return nil, common.CodeDataError, fmt.Errorf("`document_ids` is required")
	}

	kb, err := s.getKnowledgebaseByID(ctx, datasetID)
	if err != nil || kb == nil {
		return nil, common.CodeDataError, fmt.Errorf("dataset not found")
	}

	docIDs, duplicateMessages := checkDuplicateIDs(req.DocumentIDs, "document")
	notFound := make([]string, 0)

	docs, err := s.getDocumentsByIDs(ctx, docIDs)
	if err != nil {
		return nil, common.CodeServerError, err
	}
	docByID := make(map[string]*entity.Document, len(docs))
	for _, doc := range docs {
		docByID[doc.ID] = doc
	}
	for _, docID := range docIDs {
		doc := docByID[docID]
		if doc == nil || doc.KbID != datasetID {
			notFound = append(notFound, docID)
		}
	}
	if len(notFound) > 0 {
		return nil, common.CodeDataError, fmt.Errorf("documents not found: %v", notFound)
	}

	// Batch pre-check: refuse the whole request if any document's ingestion
	// task is non-terminal (RUNNING/STOPPING), so we never partially clean.
	if err = (document.NewDocumentService().AssertIngestionTasksTerminal(ctx, docIDs)); err != nil {
		return nil, common.CodeDataError, err
	}

	startParse := s.startParseDocumentsFunc
	if startParse == nil {
		docSvc := document.NewDocumentService()
		startParse = docSvc.StartParseDocuments
	}

	successCount := 0

	for _, docID := range docIDs {
		doc := docByID[docID]
		if err = startParse(ctx, doc, kb, document.StartParseOptions{RerunWithDelete: true}); err != nil {
			return nil, common.CodeServerError, err
		}
		successCount++
	}

	if len(duplicateMessages) > 0 {
		if successCount > 0 {
			return map[string]interface{}{
				"success_count": successCount,
				"errors":        duplicateMessages,
			}, common.CodeSuccess, fmt.Errorf("partially parsed %d documents with %d errors", successCount, len(duplicateMessages))
		}
		return nil, common.CodeDataError, fmt.Errorf("%s", strings.Join(duplicateMessages, ";"))
	}
	return nil, common.CodeSuccess, nil
}

// List retrieves chunks for a document
func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest) (*service.ListChunksResponse, error) {
	if s.docEngine == nil {
		return nil, fmt.Errorf("doc engine not initialized")
	}

	if req.DocID == "" {
		return nil, fmt.Errorf("doc_id is required")
	}

	// Get document to find its tenant
	docDAO := dao.NewDocumentDAO()
	doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)
	if err != nil || doc == nil {
		return nil, fmt.Errorf("document not found")
	}
	if req.DatasetID != "" && doc.KbID != req.DatasetID {
		return nil, fmt.Errorf("document not found")
	}

	kb, err := s.getAccessibleKnowledgebase(ctx, doc.KbID, permission.OperationRead)
	if err != nil {
		return nil, err
	}
	targetTenantID := kb.TenantID

	indexName := fmt.Sprintf("ragflow_%s", targetTenantID)

	page := common.CoalesceInt(req.Page, common.DefaultPage)
	size := common.CoalesceInt(req.Size, common.DefaultPageSize)
	keywords := strings.TrimSpace(req.Keywords)
	var orderBy *types.OrderByExpr
	matchExprs := make([]interface{}, 0, 1)
	if keywords == "" {
		orderBy = (&types.OrderByExpr{}).
			Asc("chunk_order_int").
			Asc("page_num_int").
			Asc("top_int").
			Desc("create_timestamp_flt")
	} else {
		queryBuilder := nlp.GetQueryBuilder()
		if queryBuilder == nil {
			queryBuilder = nlp.NewQueryBuilder()
		}
		if matchText, _ := queryBuilder.Question(keywords, "", 0.3, datasetLanguageOf(kb)); matchText != nil {
			matchExprs = append(matchExprs, matchText)
		}
	}

	// Build search request - same as retrieval test but filtered by doc_id.
	// KbIDs is pinned to the document's own dataset: searching every KB of the
	// tenant also returned rows whose KB context is a different dataset (chunks
	// carry img_id "<kb_id>-<chunk_id>"), so a document of dataset A could list
	// dataset B's chunks. Mirrors Python's [dataset_id] scope.
	searchReq := &types.SearchRequest{
		IndexNames:         []string{indexName},
		MatchExprs:         matchExprs,
		KbIDs:              []string{doc.KbID},
		Offset:             (page - 1) * size,
		Limit:              size,
		OrderBy:            orderBy,
		IncludeUnavailable: req.AvailableInt == nil,
		SelectFields: []string{
			"id",
			"content_with_weight",
			"img_id",
			"position_int",
			// Read back only to emit the `available` boolean.
			"available_int",
			// Call for the ES field names: Elasticsearch does not translate
			// SelectFields (see elasticsearch/chunk.go), while Infinity's
			// convertSelectFields maps these names onto its own docnm /
			// important_keywords / questions columns. Asking for the Python-style
			// aliases here returned no document name at all on ES.
			"docnm_kwd",
			"important_kwd",
			"question_kwd",
			"entities_kwd",
			"entity_kwd",
			"entity_type_kwd",
			"from_entity_kwd",
			"name_kwd",
			"raptor_kwd",
			"removed_kwd",
			"source_id",
			"tag_kwd",
			"to_entity_kwd",
			"toc_kwd",
			"doc_type_kwd",
		},
		Filter: map[string]interface{}{
			"doc_id":   req.DocID,
			"must_not": map[string]interface{}{"exists": "compile_kwd"},
		},
	}
	if len(req.ChunkIDs) > 0 {
		searchReq.Filter["id"] = req.ChunkIDs
	}

	// Add available_int filter if specified
	if req.AvailableInt != nil {
		searchReq.Filter["available_int"] = *req.AvailableInt
	}

	// Execute search through unified engine interface
	searchResp, err := s.docEngine.Search(ctx, searchReq)
	if err != nil {
		return nil, fmt.Errorf("search failed: %w", err)
	}

	chunks := make([]map[string]interface{}, 0, len(searchResp.Chunks))
	for _, chunk := range searchResp.Chunks {
		// Inline formatChunkForList
		result := make(map[string]interface{})
		// `available`: available_int=0 → false, missing → true; the frontend uses
		// it to render disabled chunks, incl. parent rows.
		availableInt := 1
		if stored, ok := common.GetInt(chunk["available_int"]); ok {
			availableInt = stored
		}
		result["available"] = availableInt != 0
		skipFields := map[string]bool{
			"_id": true, "authors": true, "_score": true, "SCORE": true, "available_int": true,
			"important_kwd_empty_count": true, "kb_id": true, "mom_id": true, "page_num_int": true,
		}
		for k, v := range chunk {
			if skipFields[k] || strings.HasSuffix(k, "_vec") || strings.Contains(k, "_sm_") || strings.HasSuffix(k, "_ltks") || strings.HasSuffix(k, "_tks") {
				continue
			}
			switch k {
			case "img_id":
				if strVal, ok := v.(string); ok {
					result["image_id"] = strVal
				} else {
					result["image_id"] = ""
				}
			case "position_int":
				result["positions"] = v
			case "id":
				result["chunk_id"] = v
			case "content_with_weight":
				result["content_with_weight"] = v
			case "content":
				if _, ok := result["content_with_weight"]; !ok {
					result["content_with_weight"] = v
				}
			case "docnm":
				result["docnm_kwd"] = v
			case "important_keywords":
				utility.SetFieldArray(result, "important_kwd", v)
			case "questions":
				utility.SetFieldArray(result, "question_kwd", v)
			case "entities_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd",
				"name_kwd", "raptor_kwd", "removed_kwd",
				"source_id", "tag_kwd", "to_entity_kwd", "toc_kwd", "doc_type_kwd":
				if utility.IsEmpty(v) {
					result[k] = []interface{}{}
				} else {
					result[k] = v
				}
			default:
				// Handle _kwd fields that need "###" splitting
				if strings.HasSuffix(k, "_kwd") && k != "knowledge_graph_kwd" {
					if strVal, ok := v.(string); ok && strings.Contains(strVal, "###") {
						parts := strings.Split(strVal, "###")
						var filtered []interface{}
						for _, p := range parts {
							if p != "" {
								filtered = append(filtered, p)
							}
						}
						result[k] = filtered
					} else {
						result[k] = v
					}
				} else {
					result[k] = v
				}
			}
		}
		result["id"] = result["chunk_id"]
		result["content"] = result["content_with_weight"]
		utility.SetFieldArray(result, "important_keywords", result["important_kwd"])
		utility.SetFieldArray(result, "questions", result["question_kwd"])
		result["document_id"] = req.DocID
		result["dataset_id"] = doc.KbID
		chunks = append(chunks, result)
	}

	ingestionStatus := "UNSTART"
	taskDAO := s.ingestionTaskDAO
	if taskDAO == nil {
		taskDAO = dao.NewIngestionTaskDAO()
	}
	task, err := taskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
	if err != nil {
		return nil, fmt.Errorf("failed to get ingestion task for document %s: %w", doc.ID, err)
	}
	if task != nil && task.Status != "" {
		ingestionStatus = task.Status
	}

	// Build document info, mirroring Python's _map_doc key renames:
	// kb_id→dataset_id, parser_id→chunk_method, token_num→token_count,
	// chunk_num→chunk_count.
	timeFormat := "2006-01-02T15:04:05"
	docInfo := map[string]interface{}{
		"id":               doc.ID,
		"thumbnail":        doc.Thumbnail,
		"dataset_id":       doc.KbID,
		"chunk_method":     doc.ParserID,
		"pipeline_id":      doc.PipelineID,
		"parser_config":    doc.ParserConfig,
		"source_type":      doc.SourceType,
		"type":             doc.Type,
		"created_by":       doc.CreatedBy,
		"name":             doc.Name,
		"location":         doc.Location,
		"size":             doc.Size,
		"token_count":      doc.TokenNum,
		"chunk_count":      doc.ChunkNum,
		"progress":         utility.JSONFloat64(doc.Progress),
		"progress_msg":     doc.ProgressMsg,
		"process_begin_at": utility.FormatTimeToString(doc.ProcessBeginAt, timeFormat),
		"process_duration": doc.ProcessDuration,
		"content_hash":     doc.ContentHash,
		"suffix":           doc.Suffix,
		"ingestion_status": ingestionStatus,
		"status":           doc.Status,
		"create_time":      doc.CreateTime,
		"create_date":      utility.FormatTimeToString(doc.CreateDate, timeFormat),
		"update_time":      doc.UpdateTime,
		"update_date":      utility.FormatTimeToString(doc.UpdateDate, timeFormat),
	}

	return &service.ListChunksResponse{
		Total:  searchResp.Total,
		Chunks: chunks,
		Doc:    docInfo,
	}, nil
}

func (s *ChunkService) SwitchChunks(ctx context.Context, datasetID, documentID string, availableInt int, chunkIDs []string) error {
	if s.docEngine == nil {
		return fmt.Errorf("doc engine not initialized")
	}

	if availableInt != 0 && availableInt != 1 {
		return fmt.Errorf("available_int should be 0 or 1")
	}

	if chunkIDs == nil || len(chunkIDs) == 0 {
		return fmt.Errorf("req is null")
	}

	kb, err := s.getAccessibleKnowledgebase(ctx, datasetID, permission.OperationUpdate)
	if err != nil {
		return err
	}
	targetTenantID := kb.TenantID

	docDAO := dao.NewDocumentDAO()
	doc, err := docDAO.GetByID(ctx, dao.DB, documentID)
	if err != nil || doc == nil {
		return fmt.Errorf("document not found")
	}
	if doc.KbID != datasetID {
		return fmt.Errorf("document does not belong to this dataset")
	}

	for _, cid := range chunkIDs {
		indexName := fmt.Sprintf("ragflow_%s", targetTenantID)

		if err = s.docEngine.UpdateChunks(ctx, map[string]interface{}{
			"id":     cid,
			"doc_id": documentID,
		}, map[string]interface{}{
			"id":            cid,
			"available_int": availableInt,
		}, indexName, datasetID); err != nil {
			return err
		}
	}
	s.markWikiDirty(ctx, targetTenantID, datasetID, documentID, chunkIDs)

	return nil
}

func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunkRequest) error {
	if s.docEngine == nil {
		return fmt.Errorf("doc engine not initialized")
	}

	if req.ChunkID == "" {
		return fmt.Errorf("chunk_id is required")
	}

	kb, err := s.getAccessibleKnowledgebase(ctx, req.DatasetID, permission.OperationUpdate)
	if err != nil {
		return err
	}
	targetTenantID := kb.TenantID
	language := datasetLanguageOf(kb)

	// Verify document belongs to dataset
	docDAO := dao.NewDocumentDAO()
	doc, err := docDAO.GetByID(ctx, dao.DB, req.DocumentID)
	if err != nil || doc == nil {
		return fmt.Errorf("document not found")
	}
	if doc.KbID != req.DatasetID {
		return fmt.Errorf("document does not belong to this dataset")
	}

	// Fetch existing chunk first
	indexName := fmt.Sprintf("ragflow_%s", targetTenantID)
	existingChunk, err := s.docEngine.GetChunk(ctx, indexName, req.ChunkID, []string{req.DatasetID})
	if err != nil {
		return fmt.Errorf("failed to get existing chunk: %w", err)
	}

	existing, ok := existingChunk.(map[string]interface{})
	if !ok {
		return fmt.Errorf("invalid chunk format")
	}
	existingDocumentID, ok := existing["doc_id"].(string)
	if !ok || existingDocumentID != req.DocumentID {
		return fmt.Errorf("chunk not found")
	}

	// Build update dict
	d := make(map[string]interface{})

	// Content - use new value or existing
	if req.Content != nil {
		d["content_with_weight"] = *req.Content
	} else {
		if v, ok := existing["content_with_weight"].(string); ok {
			d["content_with_weight"] = v
		} else if v, ok := existing["content"].(string); ok {
			d["content_with_weight"] = v
		} else {
			d["content_with_weight"] = ""
		}
	}

	// Tokenize content
	contentStr := d["content_with_weight"].(string)
	d["content_ltks"], _ = s.tokenize(contentStr, language)
	d["content_sm_ltks"], _ = s.fineGrainedTokenize(d["content_ltks"].(string), language)

	// Important keywords - convert []string to []interface{} for transformChunkFields
	if req.ImportantKwd != nil {
		impKwd := make([]interface{}, len(req.ImportantKwd))
		for i, v := range req.ImportantKwd {
			impKwd[i] = v
		}
		d["important_kwd"] = impKwd
	}

	// Questions
	if req.Questions != nil {
		// Filter out empty questions and trim
		filteredQuestions := []string{}
		for _, q := range req.Questions {
			q = strings.TrimSpace(q)
			if q != "" {
				filteredQuestions = append(filteredQuestions, q)
			}
		}
		d["question_kwd"] = filteredQuestions
	}

	// Available
	if req.Available != nil {
		if *req.Available {
			d["available_int"] = 1
		} else {
			d["available_int"] = 0
		}
	}

	// Positions
	if req.Positions != nil {
		d["position_int"] = req.Positions
	}

	if req.TagKwd != nil {
		d["tag_kwd"] = req.TagKwd
	}

	// Tag features
	if req.TagFeas != nil {
		tagFeas, err := validateTagFeatures(req.TagFeas)
		if err != nil {
			return updateChunkError{code: common.CodeArgumentError, message: "`tag_feas` " + err.Error()}
		}
		d["tag_feas"] = tagFeas
	}

	// Image (Python: chunk_api.update_chunk). The image is stored before the
	// index update and dropped from storage only after it, so a failed update
	// never leaves the index pointing at a deleted image.
	removeImageAfterUpdate := false
	if req.ImageUpdateMode != nil || req.ImageBase64 != nil {
		imageMode, err := parseImageUpdateMode(req.ImageUpdateMode)
		if err != nil {
			return updateChunkError{code: common.CodeDataError, message: err.Error()}
		}
		switch {
		case imageMode == imageUpdateModeRemove:
			d["img_id"] = ""
			d["doc_type_kwd"] = "text"
			removeImageAfterUpdate = true
		case req.ImageBase64 != nil:
			imageBinary, err := decodeChunkImageBase64(*req.ImageBase64)
			if err != nil {
				return updateChunkError{code: common.CodeDataError, message: err.Error()}
			}
			if err = s.storeChunkImage(ctx, req.DatasetID, req.ChunkID, imageBinary, imageMode); err != nil {
				common.Error("failed to store chunk image", err,
					zap.String("dataset_id", req.DatasetID),
					zap.String("chunk_id", req.ChunkID),
					zap.String("mode", imageMode))
				return updateChunkError{code: common.CodeDataError, message: "Failed to store chunk image"}
			}
			d["img_id"] = fmt.Sprintf("%s-%s", req.DatasetID, req.ChunkID)
			d["doc_type_kwd"] = "image"
		default:
			return updateChunkError{code: common.CodeDataError, message: "`image_base64` is required when `image_update_mode` is `append` or `replace`"}
		}
	}

	// Always include id
	d["id"] = req.ChunkID

	// Call update
	condition := map[string]interface{}{
		"id":     req.ChunkID,
		"doc_id": req.DocumentID,
	}

	err = s.docEngine.UpdateChunks(ctx, condition, d, indexName, req.DatasetID)
	if err != nil {
		return fmt.Errorf("failed to update chunk: %w", err)
	}
	// The index update above already committed, so a failing image removal must
	// not skip the Wiki refresh for a content/availability change in the same
	// request. The request still answers with the removal error (the Python
	// reference does the same); the stored object is an orphan by then.
	if req.Content != nil || req.Available != nil {
		s.markWikiDirty(ctx, targetTenantID, req.DatasetID, req.DocumentID, []string{req.ChunkID})
	}
	if removeImageAfterUpdate {
		if err = s.removeChunkImage(ctx, req.DatasetID, req.ChunkID); err != nil {
			common.Error("failed to remove chunk image", err,
				zap.String("dataset_id", req.DatasetID),
				zap.String("chunk_id", req.ChunkID))
			return updateChunkError{code: common.CodeDataError, message: "Failed to remove chunk image"}
		}
	}

	return nil
}
func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChunksRequest) (int64, error) {
	if s.docEngine == nil {
		return 0, fmt.Errorf("doc engine not initialized")
	}

	if req.DatasetID == "" || req.DocID == "" {
		return 0, fmt.Errorf("dataset_id and doc_id are required")
	}

	// Bind deletion to the dataset and document named in the route.
	docDAO := dao.NewDocumentDAO()
	doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)
	if err != nil || doc == nil {
		return 0, fmt.Errorf("document not found")
	}
	if doc.KbID != req.DatasetID {
		return 0, permission.ErrResourceNotFound
	}

	kb, err := s.getAccessibleKnowledgebase(ctx, doc.KbID, permission.OperationUpdate)
	if err != nil {
		return 0, err
	}
	targetTenantID := kb.TenantID

	indexName := fmt.Sprintf("ragflow_%s", targetTenantID)

	// Build condition
	condition := make(map[string]interface{})
	switch {
	case len(req.ChunkIDs) > 0 && req.DeleteAll:
		return 0, fmt.Errorf("chunk_ids and delete_all are mutually exclusive")
	case len(req.ChunkIDs) > 0:
		// Delete specific chunks - convert []string to []interface{} for buildFilterFromCondition
		chunkIDsIf := make([]interface{}, len(req.ChunkIDs))
		for i, id := range req.ChunkIDs {
			chunkIDsIf[i] = id
		}
		condition["id"] = chunkIDsIf
		condition["doc_id"] = req.DocID
	case req.DeleteAll:
		// Delete all chunks for this document
		condition["doc_id"] = req.DocID
	default:
		return 0, nil
	}

	deletedCount, err := s.docEngine.DeleteChunks(ctx, condition, indexName, doc.KbID)
	if err != nil {
		return 0, fmt.Errorf("failed to delete chunks: %w", err)
	}

	if deletedCount > 0 {
		if err = s.decrementChunkStats(req.DocID, doc.KbID, 0, deletedCount, 0); err != nil {
			return deletedCount, fmt.Errorf("failed to update chunk stats: %w", err)
		}
		s.markWikiDirty(ctx, targetTenantID, doc.KbID, req.DocID, req.ChunkIDs)
	}

	return deletedCount, nil
}

// nextSourceChunkOrder returns the chunk_order_int a newly added source chunk
// should carry: one past the highest order already used by the document's source
// chunks, so manual chunks append after the ingested ones. Parent rows live in
// the reserved range starting at common.ParentChunkOrderBase, so the new chunk
// still lists before every parent of the document — including when the document
// has been parsed (and its parents materialized) long before the chunk was added.
//
// The lookup asks the engine for the document's highest-numbered source chunk.
// doc_type_kwd separates the two ranges without a range condition: every chunker
// output carries it (itemDocType defaults to "text"), it survives the tokenizer,
// and this API writes it below, while parent rows never copy it — so descending
// by chunk_order_int among those rows lands on the last source chunk rather than
// on a parent. A pipeline that emits chunks without doc_type_kwd (custom
// pipelines that never set it) makes the lookup come back empty, and the new
// chunk falls back to order 0, still inside the source range.
func (s *ChunkService) nextSourceChunkOrder(ctx context.Context, indexName, datasetID, documentID string) int {
	// A small page rather than a single row: the lookup only needs the first row
	// that actually carries a source order, so being tolerant of rows without one
	// (products written after the tokenizer, e.g. RAPTOR summaries) keeps this
	// independent of how an engine orders rows missing the sort key in DESC.
	searchResp, err := s.docEngine.Search(ctx, &types.SearchRequest{
		IndexNames:         []string{indexName},
		KbIDs:              []string{datasetID},
		Limit:              16,
		IncludeUnavailable: true, // a disabled source chunk still holds its order
		SelectFields:       []string{"id", "chunk_order_int"},
		OrderBy:            (&types.OrderByExpr{}).Desc("chunk_order_int"),
		Filter: map[string]interface{}{
			"doc_id":   documentID,
			"exists":   "doc_type_kwd",
			"must_not": map[string]interface{}{"exists": "compile_kwd"},
		},
	})
	if err != nil || searchResp == nil {
		// The order only decides where the chunk lists: never fail a write over
		// it. 0 keeps the chunk inside the source range, ahead of the parents.
		return 0
	}
	for _, row := range searchResp.Chunks {
		order, ok := common.GetInt(row["chunk_order_int"])
		if !ok || order < 0 || order >= common.ParentChunkOrderBase {
			continue
		}
		return order + 1
	}
	return 0
}

func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequest) (*service.AddChunkResponse, error) {
	if s.docEngine == nil {
		return nil, addChunkError{code: common.CodeServerError, message: "doc engine not initialized"}
	}
	if req == nil {
		return nil, addChunkError{code: common.CodeDataError, message: "invalid request payload"}
	}
	if err := s.checkDatasetAccess(ctx, req.DatasetID, permission.OperationUpdate); err != nil {
		code, permissionErr := permissionresponse.Normalize(err)
		return nil, addChunkError{code: code, message: permissionErr.Error(), cause: permissionErr}
	}

	kb, err := s.getKnowledgebaseByID(ctx, req.DatasetID)
	if err != nil || kb == nil {
		return nil, addChunkError{code: common.CodeDataError, message: fmt.Sprintf("You don't own the dataset %s.", req.DatasetID)}
	}

	doc, err := s.documentDAO.GetByDocumentIDAndDatasetID(ctx, dao.DB, req.DocumentID, req.DatasetID)
	if err != nil || doc == nil {
		return nil, addChunkError{code: common.CodeDataError, message: fmt.Sprintf("you don't own the document %s", req.DocumentID)}
	}

	content := strings.TrimSpace(req.Content)
	if content == "" {
		return nil, addChunkError{code: common.CodeDataError, message: "`content` is required"}
	}

	var tagFeas map[string]float64
	if req.TagFeas != nil {
		tagFeas, err = validateTagFeatures(req.TagFeas)
		if err != nil {
			return nil, addChunkError{code: common.CodeDataError, message: "`tag_feas` " + err.Error()}
		}
	}

	chunkID := common.ChunkID(req.DocumentID, req.Content)
	indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)
	language := datasetLanguageOf(kb)
	contentLtks, err := s.tokenize(req.Content, language)
	if err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize content: %v", err)}
	}
	contentSmLtks, err := s.fineGrainedTokenize(contentLtks, language)
	if err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize content fine-grained: %v", err)}
	}
	importantTks, err := s.tokenize(strings.Join(req.ImportantKeywords, " "), language)
	if err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize important keywords: %v", err)}
	}
	questionKwd := filterTrimmedStrings(req.Questions)
	questionTks, err := s.tokenize(strings.Join(req.Questions, "\n"), language)
	if err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize questions: %v", err)}
	}

	now := time.Now()
	docName := ""
	if doc.Name != nil {
		docName = *doc.Name
	}
	importantKeywords := req.ImportantKeywords
	if importantKeywords == nil {
		importantKeywords = []string{}
	}

	// Append the manual chunk after the document's existing source chunks. The
	// value stays inside the source range [0, common.ParentChunkOrderBase) so the
	// new chunk keeps listing before the parent rows MaterializeParentChunks
	// created for this document.
	chunkOrder := s.nextSourceChunkOrder(ctx, indexName, req.DatasetID, req.DocumentID)

	chunkData := map[string]interface{}{
		"id":                   chunkID,
		"content_with_weight":  req.Content,
		"doc_type_kwd":         "text", // Python chunk_api.py:1225; the image branch below overrides it
		"chunk_order_int":      chunkOrder,
		"content_ltks":         contentLtks,
		"content_sm_ltks":      contentSmLtks,
		"important_kwd":        importantKeywords,
		"important_tks":        importantTks,
		"question_kwd":         questionKwd,
		"question_tks":         questionTks,
		"create_time":          now.Format("2006-01-02 15:04:05"),
		"create_timestamp_flt": float64(now.UnixNano()) / float64(time.Second),
		"kb_id":                req.DatasetID,
		"docnm_kwd":            docName,
		"doc_id":               req.DocumentID,
	}
	if req.TagKwd != nil {
		// Mirror Python chunk_api.py: `tag_kwd` is only persisted when the
		// caller actually supplied it. Omitting the field when absent avoids
		// a Go/Python behavioural divergence on chunks that were not tagged
		// at write time. See issue #20138.
		chunkData["tag_kwd"] = req.TagKwd
	}
	if tagFeas != nil {
		chunkData["tag_feas"] = tagFeas
	}
	if kb.Pagerank != 0 {
		// Python's chunk_api.create stamps the dataset's pagerank on a manually
		// added chunk (`if kb.pagerank: d["pagerank_fea"] = kb.pagerank`); a
		// chunk that skipped it would rank at 0 until the setting is re-saved.
		chunkData[common.PAGERANK_FLD] = kb.Pagerank
	}

	if req.ImageBase64 != nil {
		var imageBinary []byte
		imageBinary, err = decodeChunkImageBase64(*req.ImageBase64)
		if err != nil {
			return nil, addChunkError{code: common.CodeDataError, message: err.Error()}
		}
		if err = s.storeChunkImage(ctx, req.DatasetID, chunkID, imageBinary, imageUpdateModeAppend); err != nil {
			common.Error("failed to store chunk image", err,
				zap.String("dataset_id", req.DatasetID),
				zap.String("chunk_id", chunkID),
				zap.String("mode", imageUpdateModeAppend))
			return nil, addChunkError{code: common.CodeDataError, message: "Failed to store chunk image"}
		}
		chunkData["img_id"] = fmt.Sprintf("%s-%s", req.DatasetID, chunkID)
		chunkData["doc_type_kwd"] = "image"
	}

	embeddingModel, err := s.getEmbeddingModel(ctx, kb.TenantID, kb.EmbdID)
	if err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("get embedding model: %v", err)}
	}
	embeddingText := req.Content
	if len(questionKwd) > 0 {
		embeddingText = strings.Join(questionKwd, "\n")
	}
	// Embed inside the model's window: Content arrives from the API and can be
	// arbitrarily long, and the provider answers 400/20015 instead of truncating it.
	embeddings, err := embeddingModel.Embed(ctx, models.EmbedRequest{Texts: []string{docName, embeddingText}}, &models.EmbeddingConfig{Dimension: 0}, nil)
	if err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("encode chunk embedding: %v", err)}
	}
	if len(embeddings) != 2 {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("unexpected embedding count: %d", len(embeddings))}
	}
	mergedVec, err := mergeChunkEmbeddings(embeddings[0].Embedding, embeddings[1].Embedding)
	if err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: err.Error()}
	}
	chunkData[fmt.Sprintf("q_%d_vec", len(mergedVec))] = mergedVec

	ctx, cancel := context.WithTimeout(ctx, 600*time.Second)
	defer cancel()
	if _, err = s.docEngine.InsertChunks(ctx, []map[string]interface{}{chunkData}, indexName, req.DatasetID, datasetLanguageOf(kb)); err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("insert chunk: %v", err)}
	}

	tokenNum := int64(s.numTokens(req.Content))
	if err = s.incrementChunkStats(req.DocumentID, req.DatasetID, tokenNum, 1, 0); err != nil {
		return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("increment chunk stats: %v", err)}
	}
	s.markWikiDirty(ctx, kb.TenantID, req.DatasetID, req.DocumentID, []string{chunkID})

	renamedChunk := map[string]interface{}{
		"id":                 chunkID,
		"content":            req.Content,
		"document_id":        req.DocumentID,
		"document":           docName,
		"important_keywords": importantKeywords,
		"questions":          questionKwd,
		"dataset_id":         req.DatasetID,
		"create_timestamp":   chunkData["create_timestamp_flt"],
		"create_time":        chunkData["create_time"],
	}
	if tagKwd, ok := chunkData["tag_kwd"]; ok {
		renamedChunk["tag_kwd"] = tagKwd
	}
	if imgID, ok := chunkData["img_id"]; ok {
		renamedChunk["image_id"] = imgID
	}

	return &service.AddChunkResponse{Chunk: renamedChunk}, nil
}

func (s *ChunkService) markWikiDirty(ctx context.Context, tenantID, datasetID, documentID string, chunkIDs []string) {
	if s.markWikiDirtyFunc != nil {
		s.markWikiDirtyFunc(tenantID, datasetID, documentID, chunkIDs)
		return
	}
	markCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
	defer cancel()
	if err := knowledge_compile.MarkWikiDocumentDirty(markCtx, tenantID, datasetID, documentID, chunkIDs); err != nil {
		common.Warn("chunk mutation: failed to schedule Wiki refresh",
			zap.String("document_id", documentID), zap.Error(err))
	}
}

type addChunkError struct {
	code    common.ErrorCode
	message string
	cause   error
}

type updateChunkError struct {
	code    common.ErrorCode
	message string
}

func (e updateChunkError) Error() string {
	return e.message
}

func (e updateChunkError) Code() common.ErrorCode {
	return e.code
}

func (e addChunkError) Error() string {
	return e.message
}

func (e addChunkError) Unwrap() error {
	return e.cause
}

func (e addChunkError) Code() common.ErrorCode {
	return e.code
}

func validateTagFeatures(raw interface{}) (map[string]float64, error) {
	parsed, ok := raw.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("must be an object mapping string tags to finite numeric scores")
	}
	cleaned := make(map[string]float64, len(parsed))
	for key, value := range parsed {
		key = strings.TrimSpace(key)
		if key == "" {
			return nil, fmt.Errorf("keys must be non-empty strings")
		}
		switch typed := value.(type) {
		case float64:
			if math.IsNaN(typed) || math.IsInf(typed, 0) || typed <= 0 {
				return nil, fmt.Errorf("values must be finite numbers greater than 0")
			}
			cleaned[key] = typed
		case float32:
			if math.IsNaN(float64(typed)) || math.IsInf(float64(typed), 0) || typed <= 0 {
				return nil, fmt.Errorf("values must be finite numbers greater than 0")
			}
			cleaned[key] = float64(typed)
		case int:
			if typed <= 0 {
				return nil, fmt.Errorf("values must be finite numbers greater than 0")
			}
			cleaned[key] = float64(typed)
		case int8:
			if typed <= 0 {
				return nil, fmt.Errorf("values must be finite numbers greater than 0")
			}
			cleaned[key] = float64(typed)
		case int16:
			if typed <= 0 {
				return nil, fmt.Errorf("values must be finite numbers greater than 0")
			}
			cleaned[key] = float64(typed)
		case int32:
			if typed <= 0 {
				return nil, fmt.Errorf("values must be finite numbers greater than 0")
			}
			cleaned[key] = float64(typed)
		case int64:
			if typed <= 0 {
				return nil, fmt.Errorf("values must be finite numbers greater than 0")
			}
			cleaned[key] = float64(typed)
		default:
			return nil, fmt.Errorf("values must be finite numbers greater than 0")
		}
	}
	return cleaned, nil
}

func decodeChunkImageBase64(raw string) ([]byte, error) {
	if strings.TrimSpace(raw) == "" {
		return nil, fmt.Errorf("`image_base64` must be a non-empty string")
	}
	imageBinary, err := base64.StdEncoding.Strict().DecodeString(raw)
	if err != nil {
		return nil, fmt.Errorf("invalid `image_base64`")
	}
	if len(imageBinary) == 0 {
		return nil, fmt.Errorf("`image_base64` is empty")
	}
	return imageBinary, nil
}

const (
	imageUpdateModeAppend  = "append"
	imageUpdateModeReplace = "replace"
	imageUpdateModeRemove  = "remove"
)

// parseImageUpdateMode resolves the image update mode, defaulting to append
// when the caller leaves it out (Python: chunk_api._parse_image_update_mode).
func parseImageUpdateMode(raw *string) (string, error) {
	if raw == nil {
		return imageUpdateModeAppend, nil
	}
	switch mode := strings.ToLower(strings.TrimSpace(*raw)); mode {
	case imageUpdateModeAppend, imageUpdateModeReplace, imageUpdateModeRemove:
		return mode, nil
	default:
		return "", fmt.Errorf("`image_update_mode` must be one of: append, replace, remove")
	}
}

func mergeChunkEmbeddings(a, b []float64) ([]float64, error) {
	if len(a) == 0 || len(b) == 0 || len(a) != len(b) {
		return nil, fmt.Errorf("unexpected embedding dimensions")
	}
	merged := make([]float64, len(a))
	for i := range a {
		merged[i] = 0.1*a[i] + 0.9*b[i]
	}
	return merged, nil
}

func filterTrimmedStrings(values []string) []string {
	filtered := make([]string, 0, len(values))
	for _, value := range values {
		trimmed := strings.TrimSpace(value)
		if trimmed != "" {
			filtered = append(filtered, trimmed)
		}
	}
	return filtered
}

// tokenize tokenizes a manually written chunk field with the dataset language,
// so it is indexed the way the dataset's parsed chunks are and its queries fold.
func (s *ChunkService) tokenize(text, language string) (string, error) {
	if s.tokenizeFunc != nil {
		return s.tokenizeFunc(text, language)
	}
	return tokenizer.New(language).Tokenize(text)
}

func (s *ChunkService) fineGrainedTokenize(text, language string) (string, error) {
	if s.fineGrainedTokenizeFunc != nil {
		return s.fineGrainedTokenizeFunc(text, language)
	}
	return tokenizer.New(language).FineGrainedTokenize(text)
}

func (s *ChunkService) numTokens(text string) int {
	if s.numTokensFunc != nil {
		return s.numTokensFunc(text)
	}
	return tokenizer.NumTokensFromString(text)
}

func (s *ChunkService) getEmbeddingModel(ctx context.Context, tenantID, embdID string) (*models.EmbeddingModel, error) {
	if s.getEmbeddingModelFunc != nil {
		return s.getEmbeddingModelFunc(tenantID, embdID)
	}
	embeddingModel, err := service.NewModelFactory().NewEmbeddingModel(permission.WithTenantID(ctx, tenantID), embdID)
	if err != nil {
		return nil, err
	}
	return embeddingModel, nil
}

func (s *ChunkService) incrementChunkStats(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {
	if s.incrementChunkStatsFunc != nil {
		return s.incrementChunkStatsFunc(docID, kbID, tokenNum, chunkNum, duration)
	}
	return dao.DB.Transaction(func(tx *gorm.DB) error {
		result := tx.Model(&entity.Document{}).
			Where("id = ? AND kb_id = ?", docID, kbID).
			Updates(map[string]interface{}{
				"token_num":        gorm.Expr("token_num + ?", tokenNum),
				"chunk_num":        gorm.Expr("chunk_num + ?", chunkNum),
				"process_duration": gorm.Expr("process_duration + ?", duration),
			})
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected == 0 {
			return fmt.Errorf("document not found")
		}

		result = tx.Model(&entity.Knowledgebase{}).
			Where("id = ?", kbID).
			Updates(map[string]interface{}{
				"token_num": gorm.Expr("token_num + ?", tokenNum),
				"chunk_num": gorm.Expr("chunk_num + ?", chunkNum),
			})
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected == 0 {
			return fmt.Errorf("knowledgebase not found")
		}
		return nil
	})
}

func (s *ChunkService) decrementChunkStats(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {
	if s.decrementChunkStatsFunc != nil {
		return s.decrementChunkStatsFunc(docID, kbID, tokenNum, chunkNum, duration)
	}
	return dao.DB.Transaction(func(tx *gorm.DB) error {
		result := tx.Model(&entity.Document{}).
			Where("id = ? AND kb_id = ?", docID, kbID).
			Updates(map[string]interface{}{
				"token_num":        gorm.Expr("CASE WHEN token_num - ? >= 0 THEN token_num - ? ELSE 0 END", tokenNum, tokenNum),
				"chunk_num":        gorm.Expr("CASE WHEN chunk_num - ? >= 0 THEN chunk_num - ? ELSE 0 END", chunkNum, chunkNum),
				"process_duration": gorm.Expr("CASE WHEN process_duration + ? >= 0 THEN process_duration + ? ELSE 0 END", duration, duration),
			})
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected == 0 {
			return fmt.Errorf("document not found")
		}

		result = tx.Model(&entity.Knowledgebase{}).
			Where("id = ?", kbID).
			Updates(map[string]interface{}{
				"token_num": gorm.Expr("CASE WHEN token_num - ? >= 0 THEN token_num - ? ELSE 0 END", tokenNum, tokenNum),
				"chunk_num": gorm.Expr("CASE WHEN chunk_num - ? >= 0 THEN chunk_num - ? ELSE 0 END", chunkNum, chunkNum),
			})
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected == 0 {
			return fmt.Errorf("knowledgebase not found")
		}
		return nil
	})
}

// storeChunkImage writes imageBinary under (bucket, chunkID). In append mode an
// existing image is stacked below the new one; replace mode overwrites it
// (Python: api/utils/image_utils.store_chunk_image). Every mode holds the same
// per-chunk lock: a replace landing between an append's read and write would
// otherwise be overwritten by the merged image.
func (s *ChunkService) storeChunkImage(ctx context.Context, bucket, chunkID string, imageBinary []byte, mode string) error {
	if s.storeChunkImageFunc != nil {
		return s.storeChunkImageFunc(bucket, chunkID, imageBinary, mode)
	}
	storageImpl := storage.GetStorageFactory().GetStorage()
	if storageImpl == nil {
		return fmt.Errorf("storage not initialized")
	}
	lockKey := bucket + "/" + chunkID
	lock := acquireChunkImageLock(lockKey)
	lock.mu.Lock()
	defer func() {
		lock.mu.Unlock()
		releaseChunkImageLock(lockKey)
	}()

	if mode == imageUpdateModeReplace {
		return storageImpl.Put(ctx, bucket, chunkID, imageBinary)
	}

	if !storageImpl.ObjExist(ctx, bucket, chunkID) {
		return storageImpl.Put(ctx, bucket, chunkID, imageBinary)
	}

	oldBinary, err := storageImpl.Get(ctx, bucket, chunkID)
	if err != nil {
		return err
	}
	oldImage, _, err := image.Decode(bytes.NewReader(oldBinary))
	if err != nil {
		return err
	}
	newImage, _, err := image.Decode(bytes.NewReader(imageBinary))
	if err != nil {
		return err
	}
	oldBounds, newBounds := oldImage.Bounds(), newImage.Bounds()
	width := oldBounds.Dx()
	if newBounds.Dx() > width {
		width = newBounds.Dx()
	}
	height := oldBounds.Dy() + newBounds.Dy()
	combined := image.NewRGBA(image.Rect(0, 0, width, height))
	draw.Draw(combined, combined.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
	draw.Draw(combined, oldBounds, oldImage, oldBounds.Min, draw.Src)
	draw.Draw(combined, image.Rect(0, oldBounds.Dy(), newBounds.Dx(), oldBounds.Dy()+newBounds.Dy()), newImage, newBounds.Min, draw.Src)

	var buf bytes.Buffer
	if err = jpeg.Encode(&buf, combined, nil); err != nil {
		return err
	}
	return storageImpl.Put(ctx, bucket, chunkID, buf.Bytes())
}

// removeChunkImage drops a chunk's stored image, tolerating a missing object
// (Python: api/utils/image_utils.remove_chunk_image). It takes the same
// per-chunk lock as storeChunkImage, so a delete cannot slip into an append's
// read-modify-write and leave the index pointing at a deleted object.
func (s *ChunkService) removeChunkImage(ctx context.Context, bucket, chunkID string) error {
	if s.removeChunkImageFunc != nil {
		return s.removeChunkImageFunc(bucket, chunkID)
	}
	storageImpl := storage.GetStorageFactory().GetStorage()
	if storageImpl == nil {
		return fmt.Errorf("storage not initialized")
	}
	lockKey := bucket + "/" + chunkID
	lock := acquireChunkImageLock(lockKey)
	lock.mu.Lock()
	defer func() {
		lock.mu.Unlock()
		releaseChunkImageLock(lockKey)
	}()

	if !storageImpl.ObjExist(ctx, bucket, chunkID) {
		return nil
	}
	return storageImpl.Remove(ctx, bucket, chunkID)
}

func acquireChunkImageLock(key string) *chunkImageLock {
	chunkImageLocks.Lock()
	defer chunkImageLocks.Unlock()

	lock := chunkImageLocks.locks[key]
	if lock == nil {
		lock = &chunkImageLock{}
		chunkImageLocks.locks[key] = lock
	}
	lock.refs++
	return lock
}

func releaseChunkImageLock(key string) {
	chunkImageLocks.Lock()
	defer chunkImageLocks.Unlock()

	lock := chunkImageLocks.locks[key]
	if lock == nil {
		return
	}
	lock.refs--
	if lock.refs == 0 {
		delete(chunkImageLocks.locks, key)
	}
}

// datasetLanguageOf is the dataset's language, or "" when unset. A query is
// folded the way its dataset was indexed, and on engines whose fulltext
// analyzer is fixed when the chunk store is created a write that may create it
// has to carry the language too.
func datasetLanguageOf(kb *entity.Knowledgebase) string {
	if kb == nil || kb.Language == nil {
		return ""
	}
	return *kb.Language
}