Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

60.2 KB
Raw Blame
package service

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/Tencent/WeKnora/internal/application/access"
	"github.com/Tencent/WeKnora/internal/application/repository"
	"github.com/Tencent/WeKnora/internal/application/service/retriever"
	"github.com/Tencent/WeKnora/internal/datasource"
	apperrors "github.com/Tencent/WeKnora/internal/errors"
	"github.com/Tencent/WeKnora/internal/logger"
	"github.com/Tencent/WeKnora/internal/storageallowlist"
	"github.com/Tencent/WeKnora/internal/tracing/langfuse"
	"github.com/Tencent/WeKnora/internal/types"
	"github.com/Tencent/WeKnora/internal/types/interfaces"
	secutils "github.com/Tencent/WeKnora/internal/utils"
	"github.com/google/uuid"
	"github.com/hibiken/asynq"
)

// ErrInvalidTenantID represents an error for invalid tenant ID
var ErrInvalidTenantID = errors.New("invalid tenant ID")

const kbTaskCleanupTimeout = 5 * time.Second

// kbWikiCleanupTimeout bounds the wiki cleanup step (pages, folders,
// revisions, issues) in the ProcessKBDelete defer. It is deliberately much
// longer than kbTaskCleanupTimeout: the wiki DELETE statements themselves are
// fast once unblocked (idx_wiki_pages_kb_id covers them), but the KB's wiki
// workers may still be holding row locks at delete time. With a 5s budget the
// statements were killed while waiting on those locks and all asynq retries
// landed in the same lock window, leaving thousands of wiki rows orphaned
// after MaxRetry was exhausted (see issue #4288).
const kbWikiCleanupTimeout = 2 * time.Minute

// kbDeleteTaskMaxRetry is the asynq retry budget for the KB delete task,
// mirrored here so the defer can detect the final attempt and escalate
// logging when the task is about to land in the dead letter queue.
const kbDeleteTaskMaxRetry = 3

// knowledgeBaseService implements the knowledge base service interface
type knowledgeBaseService struct {
	repo            interfaces.KnowledgeBaseRepository
	kgRepo          interfaces.KnowledgeRepository
	chunkRepo       interfaces.ChunkRepository
	shareRepo       interfaces.KBShareRepository
	kbShareService  interfaces.KBShareService
	modelService    interfaces.ModelService
	retrieveEngine  interfaces.RetrieveEngineRegistry
	ownership       retriever.TenantStoreOwnership
	tenantRepo      interfaces.TenantRepository
	fileSvc         interfaces.FileService
	storageResolver interfaces.StorageBackendResolver
	graphEngine     interfaces.RetrieveGraphRepository
	asynqClient     interfaces.TaskEnqueuer
	taskInspector   interfaces.TaskInspector
	taskPendingRepo interfaces.TaskPendingOpsRepository
	dsRepo          interfaces.DataSourceRepository
	syncLogRepo     interfaces.SyncLogRepository
	dsScheduler     *datasource.Scheduler
	audit           interfaces.AuditLogService
	resourceCatalog interfaces.ResourceCatalog
	wikiRepo        interfaces.WikiPageRepository
}

// NewKnowledgeBaseService creates a new knowledge base service
func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
	kgRepo interfaces.KnowledgeRepository,
	chunkRepo interfaces.ChunkRepository,
	shareRepo interfaces.KBShareRepository,
	kbShareService interfaces.KBShareService,
	modelService interfaces.ModelService,
	retrieveEngine interfaces.RetrieveEngineRegistry,
	ownership retriever.TenantStoreOwnership,
	tenantRepo interfaces.TenantRepository,
	fileSvc interfaces.FileService,
	storageResolver interfaces.StorageBackendResolver,
	graphEngine interfaces.RetrieveGraphRepository,
	asynqClient interfaces.TaskEnqueuer,
	taskInspector interfaces.TaskInspector,
	taskPendingRepo interfaces.TaskPendingOpsRepository,
	dsRepo interfaces.DataSourceRepository,
	syncLogRepo interfaces.SyncLogRepository,
	dsScheduler *datasource.Scheduler,
	audit interfaces.AuditLogService,
	resourceCatalog interfaces.ResourceCatalog,
	wikiRepo interfaces.WikiPageRepository,
) interfaces.KnowledgeBaseService {
	return &knowledgeBaseService{
		repo:            repo,
		kgRepo:          kgRepo,
		chunkRepo:       chunkRepo,
		shareRepo:       shareRepo,
		kbShareService:  kbShareService,
		modelService:    modelService,
		retrieveEngine:  retrieveEngine,
		ownership:       ownership,
		tenantRepo:      tenantRepo,
		fileSvc:         fileSvc,
		storageResolver: storageResolver,
		graphEngine:     graphEngine,
		asynqClient:     asynqClient,
		taskInspector:   taskInspector,
		taskPendingRepo: taskPendingRepo,
		dsRepo:          dsRepo,
		syncLogRepo:     syncLogRepo,
		dsScheduler:     dsScheduler,
		audit:           audit,
		resourceCatalog: resourceCatalog,
		wikiRepo:        wikiRepo,
	}
}

// GetRepository gets the knowledge base repository
// Parameters:
//   - ctx: Context with authentication and request information
//
// Returns:
//   - interfaces.KnowledgeBaseRepository: Knowledge base repository
func (s *knowledgeBaseService) GetRepository() interfaces.KnowledgeBaseRepository {
	return s.repo
}

// CreateKnowledgeBase creates a new knowledge base.
//
// When VectorStoreID is set, the binding is validated against the caller's
// tenant scope and the engine registry before persisting. A nil or
// empty-string VectorStoreID is normalized to nil ("use the tenant's
// effective engines") to match the retrieve-engine factory's pre-condition.
func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
	kb *types.KnowledgeBase,
) (*types.KnowledgeBase, error) {
	// Generate UUID and set creation timestamps
	if kb.ID == "" {
		kb.ID = uuid.New().String()
	}
	kb.CreatedAt = time.Now()
	kb.TenantID = types.MustTenantIDFromContext(ctx)
	kb.UpdatedAt = time.Now()
	// Record the creator so RBAC's RequireOwnershipOrRole can let
	// Contributors edit their own KBs without granting them tenant-wide
	// edit rights. The X-API-Key auth path attaches a synthetic
	// `system-<tenantID>` user; we deliberately skip those so the KB
	// stays tenant-owned (CreatorID == ""), which matches the original
	// API-key semantics (any human Admin can manage it) and prevents a
	// later "list KBs by creator" feature from surfacing rows nobody can
	// re-attribute.
	if uid, ok := types.UserIDFromContext(ctx); ok && !types.IsSyntheticUserID(uid) {
		kb.CreatorID = uid
	}
	kb.EnsureDefaults()
	applyTenantDefaultStorageProvider(ctx, kb)
	if err := s.applyAndValidateStorageBackend(ctx, kb); err != nil {
		return nil, err
	}

	// Fold empty-string vector_store_id into nil so this path and the
	// retrieve-engine factory's pre-condition share a single representation.
	wasEmpty := kb.VectorStoreID != nil && *kb.VectorStoreID == ""
	kb.Normalize()
	if wasEmpty {
		logger.Debugf(ctx,
			"[kb.create] empty vector_store_id normalized to nil for tenant=%d",
			kb.TenantID)
	}

	if kb.HasVectorStore() {
		if err := s.validateVectorStoreBinding(ctx, kb.TenantID, *kb.VectorStoreID); err != nil {
			return nil, err
		}
	}

	// The image settings have to survive the same check the settings panel
	// makes: a pipeline whose every action switch is off would process images
	// by ignoring them.
	if err := s.validateImagePipelineConfig(ctx, kb); err != nil {
		return nil, err
	}

	logger.Infof(ctx, "Creating knowledge base, ID: %s, tenant ID: %d, name: %s", kb.ID, kb.TenantID, kb.Name)

	if err := s.repo.CreateKnowledgeBase(ctx, kb); err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": kb.ID,
			"tenant_id":         kb.TenantID,
		})
		return nil, err
	}
	recordKBActivity(ctx, s.audit, kb.TenantID, kb.ID, types.AuditActionKBCreated,
		"knowledge_base", kb.ID, types.AuditOutcomeSuccess, map[string]any{
			"name": kb.Name, "type": kb.Type,
		})

	logger.Infof(ctx, "Knowledge base created successfully, ID: %s, name: %s", kb.ID, kb.Name)
	return kb, nil
}

func (s *knowledgeBaseService) applyAndValidateStorageBackend(ctx context.Context, kb *types.KnowledgeBase) error {
	if s.storageResolver == nil || kb == nil {
		return nil
	}
	tenant, _ := types.TenantInfoFromContext(ctx)
	if tenant == nil {
		return apperrors.NewBadRequestError("workspace context missing")
	}
	id := ""
	if kb.StorageBackendID != nil {
		id = strings.TrimSpace(*kb.StorageBackendID)
	}
	provider := kb.GetStorageProvider()
	// A newly created KB without an explicit instance follows the concrete
	// tenant default. The legacy provider is only a fallback for workspaces
	// that have not been migrated yet.
	if id == "" && tenant.DefaultStorageBackendID != nil && strings.TrimSpace(*tenant.DefaultStorageBackendID) != "" {
		provider = ""
	}
	backend, err := s.storageResolver.ResolveBackend(ctx, tenant, id, provider)
	if err != nil {
		return apperrors.NewBadRequestError("storage backend is unavailable").WithDetails(err.Error())
	}
	if backend == nil {
		return nil
	}
	kb.StorageBackendID = &backend.ID
	kb.SetStorageProvider(backend.Provider)
	return nil
}

// applyTenantDefaultStorageProvider fills an empty KB storage provider from the
// tenant's global default (Settings → Storage engine). Frontend should send the
// same value; this keeps API clients and legacy UIs consistent.
func applyTenantDefaultStorageProvider(ctx context.Context, kb *types.KnowledgeBase) {
	if kb == nil || strings.TrimSpace(kb.GetStorageProvider()) != "" {
		return
	}
	tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
	provider := ""
	if tenant != nil && tenant.StorageEngineConfig != nil {
		provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider))
	}
	if provider == "" || !storageallowlist.IsAllowed(provider) {
		provider = storageallowlist.FirstAllowed()
	}
	if provider == "" {
		return
	}
	kb.SetStorageProvider(provider)
}

// validateVectorStoreBinding routes through retriever.VerifyBinding so the
// ownership + registry sentinel hierarchy stays the single source of truth.
// The service layer's responsibility is to:
//
//  1. fast-reject malformed UUIDs (cheap pre-flight that also avoids a DB
//     round trip for type-confusion inputs like "' OR 1=1 --"),
//  2. translate retriever sentinels into user-facing AppErrors with
//     generic messages and the typed error codes.
//
// UUID parse failures map to the same "vector store not found" message as
// cross-tenant attempts to avoid an enumeration oracle that distinguishes
// "malformed input" from "non-existent UUID".
func (s *knowledgeBaseService) validateVectorStoreBinding(
	ctx context.Context, tenantID uint64, storeID string,
) error {
	sanitized := secutils.SanitizeForLog(storeID)

	if _, err := uuid.Parse(storeID); err != nil {
		logger.WarnWithFields(ctx, logger.Fields{
			"tenant_id": tenantID,
			"store_id":  sanitized,
			"reason":    "malformed vector_store_id",
		}, "[kb.create] vector store id is not a valid UUID")
		return apperrors.NewVectorStoreBindingInvalidError("vector store not found")
	}

	switch err := retriever.VerifyBinding(
		ctx, s.retrieveEngine, s.ownership, tenantID, storeID,
	); {
	case err == nil:
		return nil
	case errors.Is(err, retriever.ErrVectorStoreForbidden):
		logger.WarnWithFields(ctx, logger.Fields{
			"tenant_id": tenantID,
			"store_id":  sanitized,
			"reason":    "cross-tenant or unknown store",
		}, "[kb.create] vector store not owned by tenant")
		return apperrors.NewVectorStoreBindingInvalidError("vector store not found")
	case errors.Is(err, retriever.ErrVectorStoreNotFound),
		errors.Is(err, retriever.ErrVectorStoreUnavailable):
		logger.WarnWithFields(ctx, logger.Fields{
			"tenant_id": tenantID,
			"store_id":  sanitized,
			"reason":    "store recorded in DB but no engine could be resolved",
		}, "[kb.create] vector store currently unavailable")
		return apperrors.NewVectorStoreUnavailableError(
			"vector store is currently unavailable; check its connection configuration")
	case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
		// The caller went away or ran out of time while the binding was being
		// verified, which can now include rebuilding the store's engine. That
		// is not a server fault, so it must not be logged and answered as one.
		return err
	default:
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"tenant_id": tenantID,
			"store_id":  sanitized,
			"reason":    "binding verification failed",
		})
		return apperrors.NewInternalServerError("failed to verify vector store binding")
	}
}

// GetKnowledgeBaseByID retrieves a knowledge base by its ID
func (s *knowledgeBaseService) GetKnowledgeBaseByID(ctx context.Context, id string) (*types.KnowledgeBase, error) {
	if id == "" {
		logger.Error(ctx, "Knowledge base ID is empty")
		return nil, errors.New("knowledge base ID cannot be empty")
	}

	kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
		})
		return nil, err
	}

	kb.EnsureDefaults()
	return kb, nil
}

// GetKnowledgeBaseByIDOnly retrieves knowledge base by ID without tenant filter
// Used for cross-tenant shared KB access where permission is checked elsewhere
func (s *knowledgeBaseService) GetKnowledgeBaseByIDOnly(ctx context.Context, id string) (*types.KnowledgeBase, error) {
	if id == "" {
		logger.Error(ctx, "Knowledge base ID is empty")
		return nil, errors.New("knowledge base ID cannot be empty")
	}

	kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
		})
		return nil, err
	}

	kb.EnsureDefaults()
	return kb, nil
}

// GetKnowledgeBasesByIDsOnly retrieves knowledge bases by IDs without tenant filter (batch).
func (s *knowledgeBaseService) GetKnowledgeBasesByIDsOnly(ctx context.Context, ids []string) ([]*types.KnowledgeBase, error) {
	if len(ids) == 0 {
		return nil, nil
	}
	kbs, err := s.repo.GetKnowledgeBaseByIDs(ctx, ids)
	if err != nil {
		return nil, err
	}
	for _, kb := range kbs {
		if kb != nil {
			kb.EnsureDefaults()
		}
	}
	return kbs, nil
}

// ListKnowledgeBases returns all knowledge bases for a tenant
func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) {
	tenantID := types.MustTenantIDFromContext(ctx)

	kbs, err := s.repo.ListKnowledgeBasesByTenantID(ctx, tenantID)
	if err != nil {
		for _, kb := range kbs {
			kb.EnsureDefaults()
		}

		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"tenant_id": tenantID,
		})
		return nil, err
	}

	// Query knowledge count and chunk count for each knowledge base
	for _, kb := range kbs {
		kb.EnsureDefaults()

		// Get knowledge count
		switch kb.Type {
		case types.KnowledgeBaseTypeDocument:
			knowledgeCount, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID)
			if err != nil {
				logger.Warnf(ctx, "Failed to get knowledge count for knowledge base %s: %v", kb.ID, err)
			} else {
				kb.KnowledgeCount = knowledgeCount
			}
		case types.KnowledgeBaseTypeFAQ:
			// Get chunk count
			chunkCount, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID)
			if err != nil {
				logger.Warnf(ctx, "Failed to get chunk count for knowledge base %s: %v", kb.ID, err)
			} else {
				kb.ChunkCount = chunkCount
			}
		}

		// Check if there is a processing import task
		processingCount, err := s.kgRepo.CountKnowledgeByStatus(
			ctx,
			tenantID,
			kb.ID,
			[]string{"pending", "processing"},
		)
		if err != nil {
			logger.Warnf(ctx, "Failed to check processing status for knowledge base %s: %v", kb.ID, err)
		} else {
			kb.IsProcessing = processingCount > 0
			kb.ProcessingCount = processingCount
		}
	}

	// Per-user pin stamping + ordering. The "main" list view is the
	// only path that needs to honour the caller's personal pin set;
	// agent/share/IM callers go through ListKnowledgeBasesByTenantID
	// which also enriches but keys off the user in their own context.
	if userID, ok := types.UserIDFromContext(ctx); ok && userID != "" {
		s.applyUserKBPins(ctx, tenantID, userID, kbs)
	}
	return kbs, nil
}

// ListKnowledgeBasesByTenantID returns all knowledge bases for the given tenant (e.g. for shared agent context).
func (s *knowledgeBaseService) ListKnowledgeBasesByTenantID(ctx context.Context, tenantID uint64) ([]*types.KnowledgeBase, error) {
	kbs, err := s.repo.ListKnowledgeBasesByTenantID(ctx, tenantID)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"tenant_id": tenantID,
		})
		return nil, err
	}
	for _, kb := range kbs {
		kb.EnsureDefaults()
		switch kb.Type {
		case types.KnowledgeBaseTypeDocument:
			if cnt, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID); err == nil {
				kb.KnowledgeCount = cnt
			}
		case types.KnowledgeBaseTypeFAQ:
			if cnt, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID); err == nil {
				kb.ChunkCount = cnt
			}
		}
		if processingCount, err := s.kgRepo.CountKnowledgeByStatus(ctx, tenantID, kb.ID, []string{"pending", "processing"}); err == nil {
			kb.IsProcessing = processingCount > 0
			kb.ProcessingCount = processingCount
		}
	}

	// Stamp pin state from the caller's perspective. The tenantID
	// argument may not match the caller's own tenant (this method is
	// also used to list a shared-agent's source-tenant KBs); we still
	// scope user_kb_pins by `tenantID` since a pin tied to one tenant
	// shouldn't surface when browsing another tenant's KBs.
	if userID, ok := types.UserIDFromContext(ctx); ok && userID != "" {
		s.applyUserKBPins(ctx, tenantID, userID, kbs)
	}
	return kbs, nil
}

// FillKnowledgeBaseCounts fills KnowledgeCount, ChunkCount, IsProcessing, ProcessingCount for the given KB using kb.TenantID.
func (s *knowledgeBaseService) FillKnowledgeBaseCounts(ctx context.Context, kb *types.KnowledgeBase) error {
	if kb == nil {
		return nil
	}
	tenantID := kb.TenantID
	kb.EnsureDefaults()
	switch kb.Type {
	case types.KnowledgeBaseTypeDocument:
		if cnt, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, tenantID, kb.ID); err == nil {
			kb.KnowledgeCount = cnt
		}
	case types.KnowledgeBaseTypeFAQ:
		if cnt, err := s.chunkRepo.CountChunksByKnowledgeBaseID(ctx, tenantID, kb.ID); err == nil {
			kb.ChunkCount = cnt
		}
	}
	if processingCount, err := s.kgRepo.CountKnowledgeByStatus(ctx, tenantID, kb.ID, []string{"pending", "processing"}); err == nil {
		kb.IsProcessing = processingCount > 0
		kb.ProcessingCount = processingCount
	}
	return nil
}

// resolveUpdatedVLMConfig builds the VLM config an update request asks for.
// Only the model-managed fields are taken from the request: the legacy
// ModelName/BaseURL/APIKey/InterfaceType fields make the VLM client call
// BaseURL directly, bypassing model management and its SSRF checks, so they
// are never written from the request. A referenced model must exist and be a
// VLM model.
//
// Stored legacy fields keep VLMConfig.IsEnabled() true on their own, so they
// are cleared when the request enables a managed model, or when it disables
// VLM without echoing back the stored model_name and base_url (the shape a
// GET-then-PUT round trip sends). Request legacy fields are only compared.
//
// Only a pure legacy KB (no stored ModelID) keeps its legacy endpoint on an
// echo or on enabled=true without a model_id. A config that carries a managed
// ModelID next to legacy fields (CreateKnowledgeBase stores the request as
// is) is treated as managed: clearing ModelID while keeping the legacy
// fields would silently move image calls to the stored BaseURL.
func (s *knowledgeBaseService) resolveUpdatedVLMConfig(
	ctx context.Context, current, requested types.VLMConfig,
) (types.VLMConfig, error) {
	next := current
	next.Enabled = requested.Enabled
	next.ModelID = strings.TrimSpace(requested.ModelID)
	next.DescriptionLanguage = strings.TrimSpace(requested.DescriptionLanguage)
	next.CustomInstructions = strings.TrimSpace(requested.CustomInstructions)
	if err := types.ValidateKnowledgeBasePromptInstructions(&types.KnowledgeBase{VLMConfig: next}); err != nil {
		return types.VLMConfig{}, apperrors.NewBadRequestError("vlm_config: " + err.Error())
	}
	pureLegacy := current.ModelID == "" && current.ModelName != "" && current.BaseURL != ""
	if !next.Enabled {
		next.ModelID = ""
		legacyEcho := pureLegacy &&
			requested.ModelName == current.ModelName && requested.BaseURL == current.BaseURL
		if !legacyEcho {
			clearLegacyVLMFields(&next)
		}
		return next, nil
	}
	if next.ModelID == "" {
		if !pureLegacy {
			return types.VLMConfig{}, apperrors.NewBadRequestError(
				"vlm_config.model_id is required when enabled")
		}
		return next, nil
	}
	model, err := s.modelService.GetModelByID(ctx, next.ModelID)
	if errors.Is(err, ErrModelNotFound) {
		return types.VLMConfig{}, apperrors.NewBadRequestError("vlm_config.model_id: model not found")
	}
	if err != nil {
		return types.VLMConfig{}, fmt.Errorf("get vlm model %s: %w", next.ModelID, err)
	}
	if !model.SupportsType(types.ModelTypeVLLM) {
		return types.VLMConfig{}, apperrors.NewBadRequestError(
			fmt.Sprintf("vlm_config.model_id: model type is %s, want %s", model.Type, types.ModelTypeVLLM))
	}
	clearLegacyVLMFields(&next)
	return next, nil
}

// clearLegacyVLMFields drops the pre-model-management inline VLM endpoint.
func clearLegacyVLMFields(cfg *types.VLMConfig) {
	cfg.ModelName = ""
	cfg.BaseURL = ""
	cfg.APIKey = ""
	cfg.InterfaceType = ""
}

// UpdateKnowledgeBase updates a knowledge base's mutable properties.
//
// IMPORTANT — vector_store_id immutability contract:
// The vector_store_id binding is deliberately not accepted by this method.
// Two layers enforce immutability:
//
//  1. ORM layer: the GORM tag `<-:create` on KnowledgeBase.VectorStoreID
//     makes every UPDATE path (Save / Updates / Select-Updates) a no-op for
//     that column. Verified by repository/knowledgebase_sqlite_test.go.
//  2. Service layer: this method intentionally omits VectorStoreID from its
//     parameter list, and the matching handler DTO UpdateKnowledgeBaseRequest
//     omits the field as well. A reflection-based regression test
//     (handler/knowledgebase_request_test.go) fails if either DTO field
//     is added back, alerting future maintainers.
//
// Any future cross-store rebind workflow must use raw SQL through a
// dedicated repository method — the only sanctioned write path post-creation.
func (s *knowledgeBaseService) UpdateKnowledgeBase(ctx context.Context,
	id string,
	name string,
	description string,
	config *types.KnowledgeBaseConfig,
	vlmConfig *types.VLMConfig,
) (*types.KnowledgeBase, error) {
	if id == "" {
		logger.Error(ctx, "Knowledge base ID is empty")
		return nil, errors.New("knowledge base ID cannot be empty")
	}

	logger.Infof(ctx, "Updating knowledge base, ID: %s, name: %s", id, name)

	// Get existing knowledge base
	kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
		})
		return nil, err
	}

	changedFields := make([]string, 0, 3)
	profileWasEnabled := kb.ProfileConfig.IsEnabled()
	if kb.Name != name {
		changedFields = append(changedFields, "name")
	}
	if kb.Description != description {
		changedFields = append(changedFields, "description")
	}
	if config != nil {
		changedFields = append(changedFields, "config")
	}

	// Update the knowledge base properties
	kb.Name = name
	kb.Description = description
	if config != nil {
		kb.ChunkingConfig = config.ChunkingConfig
		// Replaced only when the request carried one, so a caller that does not
		// know about the image settings cannot wipe them.
		if config.ImageProcessingConfig != nil {
			kb.ImageProcessingConfig = *config.ImageProcessingConfig
			// Same rule as on create: the update lands only if the pipeline the
			// request names can actually run what it declares.
			if err := s.validateImagePipelineConfig(ctx, kb); err != nil {
				return nil, err
			}
		}
		if config.FAQConfig != nil {
			kb.FAQConfig = config.FAQConfig
		}
		if config.WikiConfig != nil {
			kb.WikiConfig = config.WikiConfig
		}
		if config.AutoTagConfig != nil {
			config.AutoTagConfig.Normalize()
			kb.AutoTagConfig = config.AutoTagConfig
		}
		if config.ProfileConfig != nil {
			profileWasEnabled = kb.ProfileConfig.IsEnabled()
			kb.ProfileConfig = config.ProfileConfig
		}
		if config.ExtractConfig != nil {
			kb.ExtractConfig = config.ExtractConfig
			changedFields = append(changedFields, "extract_config")
		}
		// Update indexing strategy — syncs to ExtractConfig for backward compat
		if config.IndexingStrategy != nil {
			if !config.IndexingStrategy.HasAnyIndexing() {
				return nil, errors.New("at least one indexing strategy must be enabled")
			}
			kb.IndexingStrategy = *config.IndexingStrategy
			// Ensure WikiConfig exists when wiki indexing is enabled so that
			// wiki-specific tunables (synthesis model, granularity, …) have a home.
			if kb.WikiConfig == nil && config.IndexingStrategy.WikiEnabled {
				kb.WikiConfig = &types.WikiConfig{}
			}
			// Sync GraphEnabled → ExtractConfig
			if kb.ExtractConfig != nil {
				kb.ExtractConfig.Enabled = config.IndexingStrategy.GraphEnabled
			} else if config.IndexingStrategy.GraphEnabled {
				kb.ExtractConfig = &types.ExtractConfig{Enabled: true}
			}
		}
		if config.EmbeddingModelID != nil {
			next := strings.TrimSpace(*config.EmbeddingModelID)
			if next != kb.EmbeddingModelID {
				if err := s.rejectEmbeddingModelChange(ctx, kb, next); err != nil {
					return nil, err
				}
				changedFields = append(changedFields, "embedding_model_id")
				kb.EmbeddingModelID = next
			}
		}
		if config.SummaryModelID != nil {
			next := strings.TrimSpace(*config.SummaryModelID)
			if err := s.requireModelType(ctx, next, types.ModelTypeKnowledgeQA, "summary_model_id"); err != nil {
				return nil, err
			}
			if kb.SummaryModelID != next {
				changedFields = append(changedFields, "summary_model_id")
			}
			kb.SummaryModelID = next
		}
		if config.StorageBackendID != nil {
			changed, err := s.updateStorageBackend(ctx, kb, *config.StorageBackendID)
			if err != nil {
				return nil, err
			}
			if changed {
				changedFields = append(changedFields, "storage_backend_id")
			}
		}
		if config.ASRConfig != nil {
			if config.ASRConfig.Enabled {
				modelID := strings.TrimSpace(config.ASRConfig.ModelID)
				if err := s.requireModelType(ctx, modelID, types.ModelTypeASR, "asr_model_id"); err != nil {
					return nil, err
				}
			}
			kb.ASRConfig = *config.ASRConfig
			changedFields = append(changedFields, "asr_config")
		}
		if config.QuestionGenerationConfig != nil {
			kb.QuestionGenerationConfig = config.QuestionGenerationConfig
			changedFields = append(changedFields, "question_generation_config")
		}
	}
	// Apply multimodal (vision) config only when the caller provided it,
	// mirroring the nil-means-no-change semantics used above.
	if vlmConfig != nil {
		next, err := s.resolveUpdatedVLMConfig(ctx, kb.VLMConfig, *vlmConfig)
		if err != nil {
			return nil, err
		}
		if kb.VLMConfig != next {
			changedFields = append(changedFields, "vlm_config")
		}
		kb.VLMConfig = next
	}
	kb.UpdatedAt = time.Now()
	kb.EnsureDefaults()

	logger.Info(ctx, "Saving knowledge base update")
	if err := s.repo.UpdateKnowledgeBase(ctx, kb); err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
		})
		return nil, err
	}
	recordKBActivity(ctx, s.audit, kb.TenantID, kb.ID, types.AuditActionKBUpdated,
		"knowledge_base", kb.ID, types.AuditOutcomeSuccess, map[string]any{
			"name": kb.Name, "changed_fields": changedFields,
		})
	// Turning automatic description generation on should produce a
	// description now, not after the next upload.
	if !profileWasEnabled && kb.ProfileConfig.IsEnabled() {
		_ = requestKnowledgeBaseProfileRefresh(ctx, s.asynqClient, kb, false)
	}

	logger.Infof(ctx, "Knowledge base updated successfully, ID: %s, name: %s", kb.ID, kb.Name)
	return kb, nil
}

// updateStorageBackend preserves existing file locations. Echoing a binding is
// a no-op, including for shared KBs whose storage belongs to another workspace.
func (s *knowledgeBaseService) updateStorageBackend(
	ctx context.Context, kb *types.KnowledgeBase, id string,
) (bool, error) {
	id = strings.TrimSpace(id)
	if id == "" {
		return false, apperrors.NewBadRequestError("storage_backend_id cannot be empty")
	}
	if kb.StorageBackendID != nil && strings.TrimSpace(*kb.StorageBackendID) == id {
		return false, nil
	}
	if types.CallerFromContext(ctx).TenantID != kb.TenantID {
		return false, apperrors.NewForbiddenError("只有知识库所属空间可以修改存储配置")
	}
	total, err := s.kgRepo.CountKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID)
	if err != nil {
		return false, fmt.Errorf("check files before changing storage backend: %w", err)
	}
	if total > 0 {
		return false, apperrors.NewBadRequestError(
			"Storage backend cannot be changed while the knowledge base contains files; migrate storage first")
	}
	next := *kb
	next.StorageBackendID = &id
	if err := s.applyAndValidateStorageBackend(ctx, &next); err != nil {
		return false, err
	}
	kb.StorageBackendID = next.StorageBackendID
	kb.StorageProviderConfig = next.StorageProviderConfig
	return true, nil
}

func (s *knowledgeBaseService) rejectEmbeddingModelChange(
	ctx context.Context, kb *types.KnowledgeBase, next string,
) error {
	if next == "" {
		return apperrors.NewBadRequestError("embedding_model_id cannot be empty")
	}
	if err := s.requireModelType(ctx, next, types.ModelTypeEmbedding, "embedding_model_id"); err != nil {
		return err
	}
	if kb.EmbeddingModelID == "" {
		return nil
	}
	_, total, err := s.kgRepo.ListPagedKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID,
		&types.Pagination{Page: 1, PageSize: 1}, types.KnowledgeListFilter{})
	if err != nil {
		return err
	}
	if total > 0 {
		return apperrors.NewBadRequestError("知识库中已有文件,无法修改Embedding模型")
	}
	return nil
}

func (s *knowledgeBaseService) requireModelType(
	ctx context.Context, id string, want types.ModelType, field string,
) error {
	if strings.TrimSpace(id) == "" {
		return apperrors.NewBadRequestError(field + " cannot be empty")
	}
	if s.modelService == nil {
		return apperrors.NewInternalServerError("model service is not configured")
	}
	model, err := s.modelService.GetModelByID(ctx, id)
	if err != nil || model == nil {
		return apperrors.NewBadRequestError(field + " does not match a model")
	}
	if model.Type != want {
		return apperrors.NewBadRequestError(field + " must be a " + string(want) + " model")
	}
	return nil
}

// TogglePinKnowledgeBase toggles whether the calling user has pinned
// this knowledge base. Pin state is per-(user, kb) as of migration
// 000050; previously this method flipped a tenant-wide column on the
// KB row which broke down under RBAC (only Admin/creator could pin,
// and the pin reordered the list for everyone in the tenant). The
// public signature is unchanged so the HTTP handler / CLI / SDK don't
// move.
//
// The KB still has to belong to the caller's tenant — the route is
// already gated behind KBAccessRead, but we re-check via
// GetKnowledgeBaseByIDAndTenant so a stale param survives a tenant
// switch cleanly.
func (s *knowledgeBaseService) TogglePinKnowledgeBase(
	ctx context.Context, id string,
) (*types.KnowledgeBase, error) {
	if id == "" {
		return nil, errors.New("knowledge base ID cannot be empty")
	}
	tenantID := types.MustTenantIDFromContext(ctx)
	userID, ok := types.UserIDFromContext(ctx)
	if !ok || userID == "" {
		// API-key callers without a user identity can't have a personal
		// pin set. We surface this rather than silently flipping a
		// shared-tenant flag like the old behaviour.
		return nil, errors.New("pin requires an authenticated user")
	}

	// Look the KB up without a tenant filter: the route's KBAccessRead
	// guard already validated that this caller can see this KB (own,
	// org-shared, or agent-shared). Filtering by the caller's tenant
	// here would 404 every legitimate pin against a shared KB whose
	// owning tenant differs from the caller's active tenant.
	kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
			"tenant_id":         tenantID,
		})
		return nil, err
	}

	// Read current pin state to decide direction. ListUserKBPinIDs is
	// already optimised for the "many KBs at once" path; for a single-id
	// check the round-trip is acceptable and avoids leaking a second
	// repository method just for this.
	pins, err := s.repo.ListUserKBPinIDs(ctx, tenantID, userID)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
			"tenant_id":         tenantID,
			"user_id":           userID,
		})
		return nil, err
	}
	_, currentlyPinned := pins[id]

	pinnedAt, err := s.repo.SetUserKBPin(ctx, tenantID, userID, id, !currentlyPinned)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
			"tenant_id":         tenantID,
			"user_id":           userID,
			"target_pinned":     !currentlyPinned,
		})
		return nil, err
	}

	kb.EnsureDefaults()
	kb.IsPinned = !currentlyPinned
	kb.PinnedAt = pinnedAt
	logger.Infof(ctx, "Knowledge base pin toggled, ID: %s, user: %s, is_pinned: %v",
		id, userID, kb.IsPinned)
	return kb, nil
}

// applyUserKBPins stamps IsPinned / PinnedAt onto each KB in the slice
// from the caller's perspective and sorts the slice so pinned rows
// float to the top (newest pin first, ties broken by created_at desc).
// Safe to call with an empty userID (no-op stamp; default sort by
// created_at preserved).
func (s *knowledgeBaseService) applyUserKBPins(
	ctx context.Context, tenantID uint64, userID string, kbs []*types.KnowledgeBase,
) {
	if len(kbs) == 0 || userID == "" {
		return
	}
	pins, err := s.repo.ListUserKBPinIDs(ctx, tenantID, userID)
	if err != nil {
		// Pin enrichment is best-effort: a transient DB blip here
		// should not break listing KBs. Log and bail without altering
		// the slice — caller still gets a valid list, just unsorted by
		// pin.
		logger.Warnf(ctx, "applyUserKBPins: failed to load pins for tenant=%d user=%s: %v",
			tenantID, userID, err)
		return
	}
	if len(pins) == 0 {
		return
	}
	for _, kb := range kbs {
		if ts, ok := pins[kb.ID]; ok {
			kb.IsPinned = true
			t := ts
			kb.PinnedAt = &t
		}
	}
	sort.SliceStable(kbs, func(i, j int) bool {
		a, b := kbs[i], kbs[j]
		if a.IsPinned != b.IsPinned {
			return a.IsPinned
		}
		if a.IsPinned && b.IsPinned {
			at, bt := a.PinnedAt, b.PinnedAt
			if at != nil && bt != nil && !at.Equal(*bt) {
				return at.After(*bt)
			}
		}
		return a.CreatedAt.After(b.CreatedAt)
	})
}

// DeleteKnowledgeBase deletes a knowledge base by its ID
// This method marks the knowledge base as deleted and enqueues an async task
// to handle the heavy cleanup operations (embeddings, chunks, files, graph data)
func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id string) error {
	if id == "" {
		logger.Error(ctx, "Knowledge base ID is empty")
		return errors.New("knowledge base ID cannot be empty")
	}

	logger.Infof(ctx, "Deleting knowledge base, ID: %s", id)

	// Get tenant ID from context
	tenantID := types.MustTenantIDFromContext(ctx)
	tenantInfo, _ := types.TenantInfoFromContext(ctx)

	// Load the KB before soft-delete so we can snapshot its VectorStoreID
	// into the async cleanup payload. GORM's soft-delete filter hides the
	// row from subsequent reads, so this read must happen first.
	kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
		})
		return err
	}
	var vectorStoreIDSnapshot *string
	if kb != nil {
		vectorStoreIDSnapshot = kb.VectorStoreID
	}

	// Step 1: Delete the knowledge base record first (mark as deleted)
	logger.Infof(ctx, "Deleting knowledge base from database")
	err = s.repo.DeleteKnowledgeBase(ctx, id)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
		})
		return err
	}
	deletedName := ""
	if kb != nil {
		deletedName = kb.Name
	}
	recordKBActivity(ctx, s.audit, tenantID, id, types.AuditActionKBDeleted,
		"knowledge_base", id, types.AuditOutcomeSuccess, map[string]any{"name": deletedName})

	// Stop both ephemeral queue work and durable wiki operations that target
	// the now-deleted KB. ProcessKBDelete repeats this with document IDs and
	// performs one final scrub after heavy cleanup to close enqueue races.
	//
	// Run detached with a bounded timeout so a disconnecting API client cannot
	// truncate this best-effort scrub mid-scan, matching ProcessKBDelete's
	// cleanup semantics. The KB row is already soft-deleted, so the async
	// delete task remains the durable backstop even if this pass is cut short.
	kbCleanupCtx, cancelKBCleanup := context.WithTimeout(
		context.WithoutCancel(ctx), kbTaskCleanupTimeout,
	)
	s.cleanupTasksForKnowledgeBase(kbCleanupCtx, id, nil, nil)
	cancelKBCleanup()

	// Step 1b: Remove all organization shares for this KB so org settings no longer show them
	if s.shareRepo != nil {
		if delErr := s.shareRepo.DeleteByKnowledgeBaseID(ctx, id); delErr != nil {
			logger.Warnf(ctx, "Failed to delete KB shares for knowledge base %s: %v", id, delErr)
		}
	}

	// Step 1c: Stop and soft-delete all data sources bound to this KB so cron
	// schedules and in-flight sync logs do not keep running against a deleted KB.
	dataSourceIDs := s.deleteDataSourcesForKnowledgeBase(ctx, id)
	if len(dataSourceIDs) > 0 {
		dsCancelCtx, cancelDSCancel := context.WithTimeout(
			context.WithoutCancel(ctx), kbTaskCleanupTimeout,
		)
		s.cancelTasksForKnowledgeBase(dsCancelCtx, id, nil, dataSourceIDs)
		cancelDSCancel()
	}

	// Step 2: Enqueue async task for heavy cleanup operations
	payload := types.KBDeletePayload{
		TenantID:         tenantID,
		KnowledgeBaseID:  id,
		DataSourceIDs:    dataSourceIDs,
		EffectiveEngines: tenantInfo.GetEffectiveEngines(),
		VectorStoreID:    vectorStoreIDSnapshot, // snapshot taken before soft-delete
	}
	langfuse.InjectTracing(ctx, &payload)

	payloadBytes, err := json.Marshal(payload)
	if err != nil {
		logger.Warnf(ctx, "Failed to marshal KB delete payload: %v", err)
		// Don't fail the request, the KB record is already deleted
		return nil
	}

	task := asynq.NewTask(types.TypeKBDelete, payloadBytes,
		asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(kbDeleteTaskMaxRetry), asynq.Timeout(2*time.Hour))
	info, err := s.asynqClient.Enqueue(task)
	if err != nil {
		logger.Warnf(ctx, "Failed to enqueue KB delete task: %v", err)
		// Don't fail the request, the KB record is already deleted
		return nil
	}

	logger.Infof(ctx, "KB delete task enqueued: %s, knowledge base ID: %s", info.ID, id)
	logger.Infof(ctx, "Knowledge base deleted successfully, ID: %s", id)
	return nil
}

// ProcessKBDelete handles async knowledge base deletion task
// This method performs heavy cleanup operations: deleting embeddings, chunks, files, and graph data
func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Task) (err error) {
	var payload types.KBDeletePayload
	if unmarshalErr := json.Unmarshal(t.Payload(), &payload); unmarshalErr != nil {
		logger.Errorf(ctx, "Failed to unmarshal KB delete payload: %v", unmarshalErr)
		return unmarshalErr
	}

	tenantID := payload.TenantID
	kbID := payload.KnowledgeBaseID
	var knowledgeIDs []string

	// Set tenant context for downstream services
	ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)
	defer func() {
		// Wiki rows are independent of the vector engine. Run this on every
		// return path — including SkipRetry — with its own timeout so a slow
		// Redis queue scan cannot starve the SQL deletes.
		//
		// The budget is kbWikiCleanupTimeout (not the shared 5s
		// kbTaskCleanupTimeout): the wiki DELETEs are fast once unblocked,
		// but in-flight wiki workers may hold row locks at delete time and a
		// 5s budget was being exhausted while waiting on them, orphaning
		// thousands of wiki rows after retries ran out (issue #4288).
		wikiCtx, cancelWiki := context.WithTimeout(context.WithoutCancel(ctx), kbWikiCleanupTimeout)
		wikiErr := s.cleanupWikiForKnowledgeBase(wikiCtx, tenantID, kbID)
		cancelWiki()
		if wikiErr != nil {
			if retryCount, _ := asynq.GetRetryCount(ctx); retryCount >= kbDeleteTaskMaxRetry {
				// Retries are exhausted; the returned error sends this task
				// to the dead letter queue. Log at ERROR so the residual wiki
				// rows are visible to operators instead of silently orphaned.
				logger.Errorf(ctx,
					"Wiki cleanup for KB %s failed on final retry (attempt %d): %v. "+
						"Wiki pages/folders/revisions/issues may be orphaned — manual cleanup required.",
					kbID, retryCount+1, wikiErr)
			} else {
				logger.Warnf(ctx, "Failed to clean wiki data for KB %s: %v", kbID, wikiErr)
			}
			// SkipRetry would otherwise drop the task while wiki orphans
			// remain. Promote the wiki error so asynq retries; a later
			// attempt that succeeds at wiki cleanup can still SkipRetry.
			if err == nil || errors.Is(err, asynq.SkipRetry) {
				err = wikiErr
			}
		} else if err == nil {
			logger.Infof(ctx, "KB delete task completed successfully, knowledge base ID: %s", kbID)
		}

		// Workers may enqueue downstream work while the delete task performs
		// heavy storage cleanup. A detached, bounded final scrub runs on every
		// return path, including retryable failures and cancellation.
		cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kbTaskCleanupTimeout)
		defer cancel()
		s.cleanupTasksForKnowledgeBase(cleanupCtx, kbID, knowledgeIDs, payload.DataSourceIDs)
	}()

	logger.Infof(ctx, "Processing KB delete task for knowledge base: %s", kbID)

	// Step 1: Get all knowledge entries in this knowledge base
	logger.Infof(ctx, "Fetching all knowledge entries in knowledge base, ID: %s", kbID)
	knowledgeList, err := s.kgRepo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": kbID,
		})
		return err
	}
	logger.Infof(ctx, "Found %d knowledge entries to delete", len(knowledgeList))
	knowledgeIDs = make([]string, 0, len(knowledgeList))
	for _, knowledge := range knowledgeList {
		knowledgeIDs = append(knowledgeIDs, knowledge.ID)
	}

	// Repeat the best-effort queue scrub with document IDs. Some batch tasks
	// only carry knowledge_id(s), and active work from the first pass may have
	// enqueued another downstream task before cancellation reached it.
	s.cleanupTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, payload.DataSourceIDs)

	// Step 2: Delete all knowledge entries and their resources
	if len(knowledgeList) > 0 {
		logger.Infof(ctx, "Deleting all knowledge entries and their resources")

		// Delete embeddings from vector store.
		// Resolve the engine via the factory, using the VectorStoreID captured
		// at enqueue time (may be nil → falls back to payload.EffectiveEngines).
		// If the payload references a store no longer owned/registered
		// (e.g. tampered queue entry or a store that was deleted while the
		// task sat in the queue), the factory returns a sentinel and we
		// SkipRetry to avoid burning retries on an unrecoverable situation.
		logger.Infof(ctx, "Deleting embeddings from vector store")
		retrieveEngine, err := retriever.CreateRetrieveEngineFromPayload(
			ctx,
			s.retrieveEngine,
			s.ownership,
			payload.TenantID,
			payload.EffectiveEngines,
			payload.VectorStoreID,
		)
		if errors.Is(err, retriever.ErrVectorStoreForbidden) ||
			errors.Is(err, retriever.ErrVectorStoreNotFound) {
			logger.Errorf(ctx, "KB delete task aborted: %v (tenant=%d, kb=%s)", err, payload.TenantID, payload.KnowledgeBaseID)
			return asynq.SkipRetry
		}
		if err != nil {
			// Transient failures — store temporarily unavailable, request
			// cancellation during resolution, or other retryable errors —
			// must not fall through and report success while embeddings remain.
			logger.Errorf(ctx, "KB delete task deferred: %v (tenant=%d, kb=%s)", err, payload.TenantID, payload.KnowledgeBaseID)
			return err
		} else {
			// Group knowledge by embedding model and type
			type groupKey struct {
				EmbeddingModelID string
				Type             string
			}
			embeddingGroups := make(map[groupKey][]string)
			for _, knowledge := range knowledgeList {
				key := groupKey{EmbeddingModelID: knowledge.EmbeddingModelID, Type: knowledge.Type}
				embeddingGroups[key] = append(embeddingGroups[key], knowledge.ID)
			}

			for key, knowledgeGroup := range embeddingGroups {
				embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, key.EmbeddingModelID)
				if err != nil {
					logger.Warnf(ctx, "Failed to get embedding model %s: %v", key.EmbeddingModelID, err)
					continue
				}
				if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeGroup, embeddingModel.GetDimensions(), key.Type); err != nil {
					logger.Warnf(ctx, "Failed to delete embeddings for model %s: %v", key.EmbeddingModelID, err)
				}
			}
		}

		// Collect image URLs before chunks are deleted
		chunkImageInfos, imgErr := s.chunkRepo.ListImageInfoByKnowledgeIDs(ctx, tenantID, knowledgeIDs)
		if imgErr != nil {
			logger.Warnf(ctx, "Failed to collect image URLs for KB delete: %v", imgErr)
		}
		var imageInfoStrs []string
		for _, ci := range chunkImageInfos {
			imageInfoStrs = append(imageInfoStrs, ci.ImageInfo)
		}
		imageURLs := mergeKnowledgeReleaseURLs(
			ctx, s.resourceCatalog, knowledgeIDs, collectImageURLs(ctx, imageInfoStrs))

		// Delete all chunks
		logger.Infof(ctx, "Deleting all chunks in knowledge base")
		for _, knowledgeID := range knowledgeIDs {
			if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, tenantID, knowledgeID); err != nil {
				logger.Warnf(ctx, "Failed to delete chunks for knowledge %s: %v", knowledgeID, err)
			}
		}

		// Delete physical files, extracted images, and adjust storage
		logger.Infof(ctx, "Deleting physical files and extracted images")
		storageAdjust := int64(0)
		for _, knowledge := range knowledgeList {
			if knowledge.FilePath != "" {
				if err := s.fileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil {
					logger.Warnf(ctx, "Failed to delete file %s: %v", knowledge.FilePath, err)
				}
			}
			storageAdjust -= knowledge.StorageSize
		}
		deleteExtractedImages(ctx, s.fileSvc, knowledgeResourceOwners(s.resourceCatalog, knowledgeIDs...), imageURLs)
		if storageAdjust != 0 {
			if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantID, storageAdjust); err != nil {
				logger.Warnf(ctx, "Failed to adjust tenant storage: %v", err)
			}
		}

		// Delete knowledge graph data
		logger.Infof(ctx, "Deleting knowledge graph data")
		namespaces := make([]types.NameSpace, 0, len(knowledgeList))
		for _, knowledge := range knowledgeList {
			namespaces = append(namespaces, types.NameSpace{
				KnowledgeBase: knowledge.KnowledgeBaseID,
				Knowledge:     knowledge.ID,
			})
		}
		if s.graphEngine != nil && len(namespaces) > 0 {
			if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil {
				logger.Warnf(ctx, "Failed to delete knowledge graph: %v", err)
			}
		}

		// Delete all knowledge entries from database
		logger.Infof(ctx, "Deleting knowledge entries from database")
		if err := s.kgRepo.DeleteKnowledgeList(ctx, tenantID, knowledgeIDs); err != nil {
			logger.ErrorWithFields(ctx, err, map[string]interface{}{
				"knowledge_base_id": kbID,
			})
			return err
		}
	}

	logger.Infof(ctx, "KB resource cleanup finished, knowledge base ID: %s", kbID)
	return nil
}

// cleanupWikiForKnowledgeBase removes wiki pages, folders, revisions, and
// issues for a deleted knowledge base. Pages, folders, and issues are
// soft-deleted (they carry DeletedAt); revisions are hard-deleted (no
// deleted_at column — they are immutable snapshots).
//
// nil-safe for tests that construct knowledgeBaseService without wikiRepo.
// Failures are returned so the KB delete task can retry; the caller logs.
func (s *knowledgeBaseService) cleanupWikiForKnowledgeBase(
	ctx context.Context, tenantID uint64, kbID string,
) error {
	if s.wikiRepo == nil || kbID == "" {
		return nil
	}
	logger.Infof(ctx, "Cleaning up wiki data for knowledge base")
	var errs error
	if err := s.wikiRepo.DeleteByKnowledgeBaseID(ctx, tenantID, kbID); err != nil {
		logger.Warnf(ctx, "Failed to delete wiki pages for KB %s: %v", kbID, err)
		errs = errors.Join(errs, err)
	}
	if err := s.wikiRepo.DeleteFoldersByKnowledgeBaseID(ctx, tenantID, kbID); err != nil {
		logger.Warnf(ctx, "Failed to delete wiki folders for KB %s: %v", kbID, err)
		errs = errors.Join(errs, err)
	}
	if err := s.wikiRepo.DeleteRevisionsByKnowledgeBaseID(ctx, tenantID, kbID); err != nil {
		logger.Warnf(ctx, "Failed to delete wiki revisions for KB %s: %v", kbID, err)
		errs = errors.Join(errs, err)
	}
	if err := s.wikiRepo.DeleteIssuesByKnowledgeBaseID(ctx, tenantID, kbID); err != nil {
		logger.Warnf(ctx, "Failed to delete wiki issues for KB %s: %v", kbID, err)
		errs = errors.Join(errs, err)
	}
	return errs
}

// cancelTasksForKnowledgeBase removes queue work for a deleted KB when the
// configured task backend supports knowledge-base-wide inspection. Queue
// cleanup is an optimization: the soft-deleted database row remains the
// durable source of truth, so backend failures must not fail KB deletion.
func (s *knowledgeBaseService) cancelTasksForKnowledgeBase(
	ctx context.Context,
	kbID string,
	knowledgeIDs []string,
	dataSourceIDs []string,
) {
	canceller, ok := s.taskInspector.(interfaces.KnowledgeBaseTaskCanceller)
	if !ok || kbID == "" {
		return
	}
	if _, _, err := canceller.CancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs); err != nil {
		logger.Warnf(ctx, "Failed to cancel queued tasks for deleted KB %s: %v", kbID, err)
	}
}

// cleanupTasksForKnowledgeBase removes both asynq records and durable wiki
// operations. The latter must be cleared as well or startup recovery can
// recreate Redis triggers for a KB that no longer exists.
func (s *knowledgeBaseService) cleanupTasksForKnowledgeBase(
	ctx context.Context,
	kbID string,
	knowledgeIDs []string,
	dataSourceIDs []string,
) {
	if kbID == "" {
		return
	}
	cleaner, ok := s.taskPendingRepo.(interfaces.TaskPendingOpsScopeCleaner)
	if ok {
		// Clear durable work before scanning Redis. A large or degraded queue
		// must not consume the caller's entire deadline and starve the database
		// fence that prevents startup recovery from reviving this KB.
		if err := cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, kbID); err != nil {
			logger.Warnf(ctx, "Failed to clear durable tasks for deleted KB %s: %v", kbID, err)
		}
	}
	s.cancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs)
}

// deleteDataSourcesForKnowledgeBase mirrors DataSourceService.DeleteDataSource for
// every data source attached to the KB. Errors on individual sources are logged
// but do not fail KB deletion — the KB record is already soft-deleted.
func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Context, kbID string) []string {
	if s.dsRepo == nil {
		return nil
	}

	dataSources, err := s.dsRepo.FindByKnowledgeBase(ctx, kbID)
	if err != nil {
		logger.Warnf(ctx, "Failed to list data sources for deleted KB %s: %v", kbID, err)
		return nil
	}
	dataSourceIDs := make([]string, 0, len(dataSources))
	for _, ds := range dataSources {
		if ds == nil || ds.ID == "" {
			continue
		}
		dataSourceIDs = append(dataSourceIDs, ds.ID)
		if err := s.dsRepo.Delete(ctx, ds.ID); err != nil {
			logger.Warnf(ctx, "Failed to delete data source %s for KB %s: %v", ds.ID, kbID, err)
			continue
		}
		if s.dsScheduler != nil {
			s.dsScheduler.Remove(ds.ID)
		}
		if s.syncLogRepo != nil {
			if err := s.syncLogRepo.CancelPendingByDataSource(ctx, ds.ID); err != nil {
				logger.Warnf(ctx, "Failed to cancel pending sync logs for ds=%s (kb=%s): %v", ds.ID, kbID, err)
			}
		}
		logger.Infof(ctx, "Data source deleted with knowledge base: ds=%s kb=%s", ds.ID, kbID)
	}
	return dataSourceIDs
}

// SetEmbeddingModel sets the embedding model for a knowledge base
func (s *knowledgeBaseService) SetEmbeddingModel(ctx context.Context, id string, modelID string) error {
	if id == "" {
		logger.Error(ctx, "Knowledge base ID is empty")
		return errors.New("knowledge base ID cannot be empty")
	}

	if modelID == "" {
		logger.Error(ctx, "Model ID is empty")
		return errors.New("model ID cannot be empty")
	}

	logger.Infof(ctx, "Setting embedding model for knowledge base, knowledge base ID: %s, model ID: %s", id, modelID)

	// Get the knowledge base
	kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id": id,
		})
		return err
	}

	// Update the knowledge base's embedding model
	kb.EmbeddingModelID = modelID
	kb.UpdatedAt = time.Now()

	logger.Info(ctx, "Saving knowledge base embedding model update")
	err = s.repo.UpdateKnowledgeBase(ctx, kb)
	if err != nil {
		logger.ErrorWithFields(ctx, err, map[string]interface{}{
			"knowledge_base_id":  id,
			"embedding_model_id": modelID,
		})
		return err
	}

	logger.Infof(
		ctx,
		"Knowledge base embedding model set successfully, knowledge base ID: %s, model ID: %s",
		id,
		modelID,
	)
	return nil
}

// CopyKnowledgeBase copies a knowledge base to a new knowledge base (shallow copy).
// Source and target must belong to the tenant in context; cross-tenant access is rejected.
//
// The transfer grant fixes both resource IDs and whether a new destination
// may be created. Existing targets must be compatible with the source. A
// worker retry reloads the reserved target instead of creating another KB.
func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
	srcKB string, dstKB string,
) (*types.KnowledgeBase, *types.KnowledgeBase, error) {
	destinationID, create, creatorID, err := access.CloneDestination(ctx, srcKB)
	if err != nil {
		return nil, nil, err
	}
	if dstKB != "" && dstKB != destinationID {
		return nil, nil, access.ErrForbidden
	}
	tenantID := types.MustTenantIDFromContext(ctx)
	sourceKB, err := s.repo.GetKnowledgeBaseByIDAndTenant(ctx, srcKB, tenantID)
	if err != nil {
		return nil, nil, err
	}
	if sourceKB == nil || sourceKB.ID != srcKB || sourceKB.TenantID != tenantID {
		return nil, nil, access.ErrForbidden
	}
	sourceCopy := *sourceKB
	sourceKB = &sourceCopy
	sourceKB.EnsureDefaults()
	targetKB, err := s.repo.GetKnowledgeBaseByIDAndTenant(ctx, destinationID, tenantID)
	if err != nil && (!create || !errors.Is(err, repository.ErrKnowledgeBaseNotFound)) {
		return nil, nil, err
	}
	if targetKB != nil {
		targetCopy := *targetKB
		targetKB = &targetCopy
		targetKB.EnsureDefaults()
		if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferClone); err != nil {
			return nil, nil, err
		}
		tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
		if err := access.ValidateKBTransferCompatibility(sourceKB,
			targetKB,
			access.KBTransferClone,
			"",
			tenant); err != nil {
			return nil, nil, apperrors.NewBadRequestError(err.Error())
		}
	} else {
		reserved := &types.KnowledgeBase{ID: destinationID, TenantID: tenantID}
		if !create {
			return nil, nil, access.ErrNotFound
		}
		if err := access.RequireKBTransfer(ctx, sourceKB, reserved, access.KBTransferClone); err != nil {
			return nil, nil, err
		}
		var faqConfig *types.FAQConfig
		if sourceKB.FAQConfig != nil {
			cfg := *sourceKB.FAQConfig
			faqConfig = &cfg
		}
		var profileConfig *types.KnowledgeBaseProfileConfig
		if sourceKB.ProfileConfig != nil {
			cfg := *sourceKB.ProfileConfig
			profileConfig = &cfg
		}
		// Preserve VectorStoreID so the cloned KB lands on the same
		// physical index. GORM `<-:create` permits the value at INSERT.
		targetKB = &types.KnowledgeBase{
			ID:                    destinationID,
			CreatorID:             creatorID,
			Name:                  sourceKB.Name,
			Type:                  sourceKB.Type,
			Description:           sourceKB.Description,
			TenantID:              tenantID,
			ChunkingConfig:        sourceKB.ChunkingConfig,
			ImageProcessingConfig: sourceKB.ImageProcessingConfig,
			EmbeddingModelID:      sourceKB.EmbeddingModelID,
			SummaryModelID:        sourceKB.SummaryModelID,
			VLMConfig:             sourceKB.VLMConfig,
			StorageProviderConfig: sourceKB.StorageProviderConfig,
			StorageBackendID:      sourceKB.StorageBackendID,
			StorageConfig:         sourceKB.StorageConfig,
			FAQConfig:             faqConfig,
			ProfileConfig:         profileConfig,
			VectorStoreID:         sourceKB.VectorStoreID,
		}
		// The clone is owned by the caller, not the original creator —
		// otherwise a Contributor copying someone else's KB would still
		// not be able to edit the result. Skip synthetic API-key users
		// (see CreateKnowledgeBase for the same reasoning).
		if uid, ok := types.UserIDFromContext(ctx); ok && !types.IsSyntheticUserID(uid) {
			targetKB.CreatorID = uid
		}
		targetKB.EnsureDefaults()
		if err := s.repo.CreateKnowledgeBase(ctx, targetKB); err != nil {
			return nil, nil, err
		}
	}
	return sourceKB, targetKB, nil
}

// DuplicateKnowledgeBase creates a new KB from the source KB's settings only.
// Runtime/content state is deliberately reset so this path never copies
// knowledge entries, chunks, FAQ content, wiki pages, indexes, shares or pins.
func (s *knowledgeBaseService) DuplicateKnowledgeBase(
	ctx context.Context,
	srcKB string,
) (*types.KnowledgeBase, error) {
	srcKB = strings.TrimSpace(srcKB)
	if srcKB == "" {
		return nil, apperrors.NewBadRequestError("source knowledge base ID cannot be empty")
	}

	tenantID := types.MustTenantIDFromContext(ctx)
	sourceKB, err := s.repo.GetKnowledgeBaseByIDAndTenant(ctx, srcKB, tenantID)
	if err != nil {
		logger.Errorf(ctx, "Get source knowledge base failed: %v", err)
		return nil, err
	}
	sourceKB.EnsureDefaults()

	targetKB, err := cloneKnowledgeBaseConfiguration(sourceKB)
	if err != nil {
		return nil, err
	}
	// A duplicate copies settings, not content. The generated description is
	// derived from the source's documents, so carrying it over would describe
	// documents the copy does not have; ProfileConfig is kept so the copy
	// produces its own once documents arrive.
	targetKB.GeneratedProfile = nil
	targetKB.ID = uuid.New().String()
	targetKB.TenantID = tenantID
	targetKB.Name = s.buildDuplicateKnowledgeBaseName(ctx, tenantID, sourceKB.Name)
	targetKB.CreatorID = ""
	if uid, ok := types.UserIDFromContext(ctx); ok && !types.IsSyntheticUserID(uid) {
		targetKB.CreatorID = uid
	}
	now := time.Now()
	targetKB.CreatedAt = now
	targetKB.UpdatedAt = now
	targetKB.DeletedAt.Valid = false
	targetKB.DeletedAt.Time = time.Time{}
	targetKB.IsTemporary = false
	targetKB.IsPinned = false
	targetKB.PinnedAt = nil
	targetKB.KnowledgeCount = 0
	targetKB.ChunkCount = 0
	targetKB.IsProcessing = false
	targetKB.ProcessingCount = 0
	targetKB.ShareCount = 0
	targetKB.CreatorName = ""
	targetKB.EnsureDefaults()
	targetKB.Normalize()

	if targetKB.HasVectorStore() {
		if err := s.validateVectorStoreBinding(ctx, tenantID, *targetKB.VectorStoreID); err != nil {
			return nil, err
		}
	}

	if err := s.repo.CreateKnowledgeBase(ctx, targetKB); err != nil {
		return nil, err
	}
	recordKBActivity(ctx, s.audit, tenantID, targetKB.ID, types.AuditActionKBDuplicated,
		"knowledge_base", targetKB.ID, types.AuditOutcomeSuccess, map[string]any{
			"source_kb_id": sourceKB.ID, "name": targetKB.Name,
		})
	return targetKB, nil
}

func duplicateKBCopySuffix(locale string) string {
	locale = strings.ToLower(locale)
	switch {
	case strings.HasPrefix(locale, "zh"):
		return " 副本"
	case strings.HasPrefix(locale, "ko"):
		return " 사본"
	case strings.HasPrefix(locale, "ru"):
		return " копия"
	default:
		return " Copy"
	}
}

func duplicateKBDefaultName(locale string) string {
	locale = strings.ToLower(locale)
	switch {
	case strings.HasPrefix(locale, "zh"):
		return "知识库"
	case strings.HasPrefix(locale, "ko"):
		return "지식베이스"
	case strings.HasPrefix(locale, "ru"):
		return "База знаний"
	default:
		return "Knowledge Base"
	}
}

func (s *knowledgeBaseService) buildDuplicateKnowledgeBaseName(
	ctx context.Context,
	tenantID uint64,
	sourceName string,
) string {
	locale := types.LanguageFromContextOrDefault(ctx)
	suffix := duplicateKBCopySuffix(locale)

	baseName := strings.TrimSpace(sourceName)
	if baseName == "" {
		baseName = duplicateKBDefaultName(locale)
	}

	kbs, err := s.repo.ListKnowledgeBasesByTenantID(ctx, tenantID)
	if err != nil {
		logger.Warnf(ctx, "List tenant knowledge bases failed while building duplicate name: %v", err)
		return baseName + suffix
	}

	existing := make(map[string]struct{}, len(kbs))
	for _, kb := range kbs {
		if kb == nil {
			continue
		}
		existing[kb.Name] = struct{}{}
	}

	candidate := baseName + suffix
	if _, ok := existing[candidate]; !ok {
		return candidate
	}
	for i := 2; ; i++ {
		candidate = fmt.Sprintf("%s%s %d", baseName, suffix, i)
		if _, ok := existing[candidate]; !ok {
			return candidate
		}
	}
}

func cloneKnowledgeBaseConfiguration(sourceKB *types.KnowledgeBase) (*types.KnowledgeBase, error) {
	if sourceKB == nil {
		return nil, apperrors.NewBadRequestError("source knowledge base cannot be empty")
	}
	data, err := json.Marshal(sourceKB)
	if err != nil {
		return nil, err
	}
	var targetKB types.KnowledgeBase
	if err := json.Unmarshal(data, &targetKB); err != nil {
		return nil, err
	}
	return &targetKB, nil
}