Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/agent/tools"
"github.com/Tencent/WeKnora/internal/application/access"
"github.com/Tencent/WeKnora/internal/application/repository"
chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/common"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/sourceloc"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
func (s *knowledgeService) cloneKnowledge(
ctx context.Context,
src *types.Knowledge,
targetKB *types.KnowledgeBase,
) (err error) {
sourceKB, err := knowledgeWriteKB(ctx, s.kbService, src)
if err != nil {
return err
}
if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferClone); err != nil {
return err
}
if src.ParseStatus != types.ParseStatusCompleted {
return fmt.Errorf("source knowledge %s is not completed", src.ID)
}
if _, err := s.transferChunks(ctx, src, sourceKB.ID); err != nil {
return err
}
dst := &types.Knowledge{
ID: uuid.NewString(),
TenantID: targetKB.TenantID,
KnowledgeBaseID: targetKB.ID,
Type: src.Type,
Channel: src.Channel,
Title: src.Title,
Description: src.Description,
Profile: src.Profile.Clone(),
Source: src.Source,
ParseStatus: "processing",
EnableStatus: "disabled",
EmbeddingModelID: targetKB.EmbeddingModelID,
FileName: src.FileName,
FolderPath: src.FolderPath,
FileType: src.FileType,
FileSize: src.FileSize,
FileHash: src.FileHash,
FilePath: src.FilePath,
StorageSize: 0,
Metadata: src.Metadata,
CustomMetadata: src.CustomMetadata,
}
state := knowledgeTransferState{
TaskID: access.TransferTaskID(ctx),
Operation: access.KBTransferClone,
SourceKB: sourceKB.ID,
TargetKB: targetKB.ID,
SourceID: src.ID,
Phase: "cloning",
}
if err := setTransferState(dst, state); err != nil {
return err
}
// Deep-copy the source document file into an object owned by the destination
// knowledge. Without this the clone only shares the source's storage path, so
// deleting the source knowledge would destroy the clone's file too. The new
// object is tracked for cleanup if the clone fails downstream.
var copiedFilePaths []string
if src.FilePath != "" {
srcKB, kbErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID)
if kbErr != nil {
return fmt.Errorf("clone knowledge: failed to load source knowledge base: %w", kbErr)
}
srcSvc := s.resolveFileServiceForPath(ctx, srcKB, src.FilePath)
dstSvc := s.resolveFileService(ctx, targetKB)
newPath, copyErr := copyOwnedObject(
ctx, srcSvc, dstSvc, src.FilePath, targetKB.TenantID, dst.ID,
s.resourceCatalog, types.ResourceRelationSourceFile)
if copyErr != nil {
return fmt.Errorf("clone knowledge file copy failed: %w", copyErr)
}
dst.FilePath = newPath
copiedFilePaths = append(copiedFilePaths, newPath)
}
defer func() {
if err == nil {
return
}
stored, loadErr := s.repo.GetKnowledgeByID(ctx, dst.TenantID, dst.ID)
if errors.Is(loadErr, repository.ErrKnowledgeNotFound) {
cleanupCopiedObjects(
ctx, s.resolveFileService(ctx, targetKB),
knowledgeResourceOwners(s.resourceCatalog, dst.ID), copiedFilePaths,
)
return
}
// An uncertain save/read must not delete objects that may be referenced.
if loadErr != nil || stored == nil || validateTransferKnowledge(stored, targetKB) != nil {
return
}
marker, markerErr := transferState(stored)
if markerErr != nil || !matchesTransfer(ctx, marker, sourceKB, targetKB, access.KBTransferClone, src.ID, "") ||
stored.ParseStatus == types.ParseStatusCompleted {
return
}
before, after := *stored, *stored
after.ParseStatus = types.ParseStatusFailed
after.ErrorMessage = err.Error()
_ = s.repo.UpdateKnowledgeForTransfer(ctx, &before, &after)
}()
if err = s.repo.CreateKnowledge(ctx, dst); err != nil {
return err
}
// Create timestamps can be rounded by the database before the first CAS.
persisted, err := s.repo.GetKnowledgeByID(ctx, dst.TenantID, dst.ID)
if err != nil {
return err
}
if err := validateTransferKnowledge(persisted, targetKB); err != nil {
return err
}
dst.UpdatedAt = persisted.UpdatedAt
if err = s.CloneChunk(ctx, src, dst); err != nil {
return err
}
before := *dst
state.Phase = "done"
if err = setTransferState(dst, state); err != nil {
return err
}
dst.ParseStatus = types.ParseStatusCompleted
dst.EnableStatus = "enabled"
dst.StorageSize = src.StorageSize
return s.repo.UpdateKnowledgeForTransfer(ctx, &before, dst)
}
// processDocumentFromPassage handles asynchronous processing of text passages
func (s *knowledgeService) processDocumentFromPassage(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string,
) {
// Update status to processing
knowledge.ParseStatus = "processing"
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
return
}
// Convert passages to chunks
chunks := make([]types.ParsedChunk, 0, len(passage))
start, end := 0, 0
for i, p := range passage {
if p == "" {
continue
}
end += len([]rune(p))
chunks = append(chunks, types.ParsedChunk{
Content: p,
Seq: i,
Start: start,
End: end,
})
start = end
}
// Process and store chunks
var opts ProcessChunksOptions
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
opts.EnableQuestionGeneration = true
opts.QuestionCount = kb.QuestionGenerationConfig.QuestionCount
if opts.QuestionCount <= 0 {
opts.QuestionCount = 3
}
}
if err := s.processChunks(ctx, kb, knowledge, chunks, opts); err != nil {
logger.Warnf(ctx, "process passages for knowledge %s: %v", knowledge.ID, err)
}
}
// ProcessChunksOptions contains options for processing chunks
type ProcessChunksOptions struct {
EnableQuestionGeneration bool
QuestionCount int
EnableMultimodel bool
StoredImages []docparser.StoredImage
// ParentChunks holds parent chunk data when parent-child chunking is enabled.
// When set, the chunks passed to processChunks are child chunks, and each
// child's ParentIndex references an entry in this slice.
ParentChunks []types.ParsedParentChunk
Metadata map[string]string
}
func imagesForMultimodal(images []docparser.StoredImage) []docparser.StoredImage {
selected := make([]docparser.StoredImage, 0, len(images))
for _, image := range images {
if !image.SkipMultimodal {
selected = append(selected, image)
}
}
return selected
}
func cleanupUnboundInlineImages(
ctx context.Context, fileSvc interfaces.FileService, images []docparser.StoredImage,
) {
if fileSvc == nil {
return
}
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
deleted := make(map[string]struct{})
for _, image := range images {
if (!image.Inline && !image.NewlyStored) || image.ServingURL == "" {
continue
}
if _, exists := deleted[image.ServingURL]; exists {
continue
}
deleted[image.ServingURL] = struct{}{}
if err := fileSvc.DeleteFile(cleanupCtx, image.ServingURL); err != nil {
logger.Warnf(ctx, "Failed to clean up unbound inline image %s: %v", image.ServingURL, err)
}
}
}
// finalizeIndexedKnowledgeState makes a document retrievable as soon as chunks
// and indexes are persisted (enable_status=enabled), but it deliberately does
// NOT mark the row completed when enrichment is still expected. Whenever the
// document still has work to fan out — pending multimodal image tasks, or text
// chunks that feed summary/question/graph generation — parse_status stays
// "processing" so KnowledgePostProcess remains the single authority that drives
// processing → finalizing → completed. Marking the row completed here would make
// post-process hit its "non-processing status" guard and skip the summary
// fan-out, stranding summary_status on "pending" forever.
func finalizeIndexedKnowledgeState(
knowledge *types.Knowledge,
totalStorageSize int64,
textChunkCount int,
hasPendingMultimodal bool,
now time.Time,
) {
if hasPendingMultimodal || textChunkCount > 0 {
knowledge.ParseStatus = types.ParseStatusProcessing
knowledge.SummaryStatus = types.SummaryStatusNone
} else {
// No text chunks and no pending multimodal work: there is nothing for
// post-process to enrich, so complete immediately. This is the only
// route to 'completed' that bypasses FinalizeSubtask, so it has to
// clear error_message itself — otherwise a successfully indexed row
// keeps reporting a failure from an earlier attempt.
knowledge.ParseStatus = types.ParseStatusCompleted
knowledge.SummaryStatus = types.SummaryStatusNone
knowledge.ErrorMessage = ""
}
knowledge.EnableStatus = "enabled"
knowledge.StorageSize = totalStorageSize
knowledge.ProcessedAt = &now
knowledge.UpdatedAt = now
}
// markKnowledgeProcessing makes the top-level knowledge state describe the
// attempt that is starting now. Clearing error_message is part of that: the
// column is only meaningful for the attempt that produced it, so a row
// re-entering processing must not keep surfacing the previous attempt's
// failure while it is visibly running again.
func markKnowledgeProcessing(knowledge *types.Knowledge, now time.Time) {
knowledge.ParseStatus = types.ParseStatusProcessing
knowledge.ErrorMessage = ""
knowledge.UpdatedAt = now
}
// failKnowledgeOnEmbeddingModel records an unresolvable embedding model as this
// attempt's terminal state.
//
// It re-reads the row instead of trusting the entry guard, because resolving a
// model reads the database: a cancel or a ReplaceKnowledgeFile can land in the
// window between that guard and this write. Both would be clobbered otherwise —
// UpdateKnowledge is a full-row Save and file_path is not among the omitted
// columns, so a stale in-memory row would roll file_path back over the
// replacement and mark the replacement's live attempt failed.
func (s *knowledgeService) failKnowledgeOnEmbeddingModel(
ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, cause error,
) error {
return s.failKnowledgeAtEmbedding(ctx, kb, knowledge, werrors.ErrCodeEmbeddingProviderFail,
"failed to get embedding model", cause)
}
// failKnowledgeAtEmbedding records cause as this attempt's terminal state at
// the embedding stage; see failKnowledgeOnEmbeddingModel for the guards. It
// returns nil once the attempt is settled (failure recorded, or the row was
// cancelled / deleted / replaced) and an error when nothing could be
// recorded, so the task is retried instead of acked with the row in flight.
func (s *knowledgeService) failKnowledgeAtEmbedding(
ctx context.Context, kb *types.KnowledgeBase, knowledge *types.Knowledge, code, what string, cause error,
) error {
// A cancelled or expired context means the run was interrupted — the user
// cancelled (asynq CancelProcessing cancels the handler context), the
// worker was preempted, the process is shutting down. The model itself is
// not implicated, so recording "failed to get embedding model" would both
// mislabel the cause and overwrite the cancelled status the abort path
// just wrote. Leave the row to the abort path, or to the housekeeping
// sweep when nothing else claims it.
if ctx.Err() != nil || errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {
logger.Infof(ctx,
"%s interrupted for %s (%v); leaving parse status untouched",
what, knowledge.ID, cause)
return fmt.Errorf("%s: %w", what, cause)
}
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx,
"Knowledge aborted (%s), not recording %q: %s", status, what, knowledge.ID)
return abortRetryErr(ctx, knowledge.ID, status)
}
knowledge.ParseStatus = types.ParseStatusFailed
knowledge.ErrorMessage = fmt.Sprintf("%s: %v", what, cause)
knowledge.UpdatedAt = time.Now()
if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
logger.Errorf(ctx, "failed to persist %q for %s: %v", what, knowledge.ID, err)
return fmt.Errorf("persist %s: %w", what, err)
}
s.beginStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{
"model_id": kb.EmbeddingModelID,
})
// The span's error_message is what the timeline renders, and its
// error_detail is withheld from non-admin responses — so carry the same
// text the document list shows rather than a fixed generic string.
s.failStage(ctx, knowledge.ID, types.StageEmbedding, code, knowledge.ErrorMessage, cause)
return nil
}
// buildSplitterConfigFromChunking normalizes effective chunking settings with
// the shared chunker defaults.
func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterConfig {
return chunker.NormalizeSplitterConfig(chunker.SplitterConfig{
ChunkSize: cc.ChunkSize,
ChunkOverlap: cc.ChunkOverlap,
Separators: cc.Separators,
Strategy: cc.Strategy,
TokenLimit: cc.TokenLimit,
Languages: cc.Languages,
})
}
// buildParentChildConfigs derives parent and child SplitterConfig from ChunkingConfig.
// The base config (already validated with defaults) is used for separators and
// the splitting strategy. Strategy must be propagated: an empty Strategy resolves
// to the legacy tier (see resolveChainWithProfile), which never runs the heading
// splitter, so parent-child chunks would silently lose heading alignment and
// ContextHeader breadcrumbs regardless of the configured strategy.
func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) {
return chunker.DeriveParentChildConfigs(base, cc.ParentChunkSize, cc.ChildChunkSize)
}
// deleteUnindexedChunks drops the chunks processChunks wrote for a knowledge
// that failed before BatchIndex ran. Nothing reached the vector store yet, so
// only the chunk rows need removing; left alone they would stay active under a
// failed knowledge, the same leftovers the BatchIndex failure path cleans up.
func (s *knowledgeService) deleteUnindexedChunks(ctx context.Context, knowledge *types.Knowledge) {
if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil {
logger.Errorf(ctx, "Delete chunks failed: %v", err)
}
}
// supersededChunkPurger is intentionally narrower than ChunkRepository: only
// persistent repositories that can prove a newer active replacement should
// physically reclaim soft-deleted chunk history.
type supersededChunkPurger interface {
PurgeSupersededChunksByKnowledgeID(context.Context, uint64, string) (int64, error)
}
// processChunks processes chunks and creates embeddings for knowledge content
func (s *knowledgeService) processChunks(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []types.ParsedChunk,
opts ...ProcessChunksOptions,
) error {
// Get options
var options ProcessChunksOptions
if len(opts) > 0 {
options = opts[0]
}
// Parser output and manually supplied passages can contain malformed byte
// sequences. Clean them before logging, chunk persistence, or embedding;
// the embedding provider and tracing/database drivers expect valid UTF-8.
for i := range chunks {
chunks[i].Content = common.CleanInvalidUTF8(chunks[i].Content)
chunks[i].ContextHeader = common.CleanInvalidUTF8(chunks[i].ContextHeader)
for j := range chunks[i].Images {
chunks[i].Images[j].URL = common.CleanInvalidUTF8(chunks[i].Images[j].URL)
chunks[i].Images[j].Caption = common.CleanInvalidUTF8(chunks[i].Images[j].Caption)
chunks[i].Images[j].OCRText = common.CleanInvalidUTF8(chunks[i].Images[j].OCRText)
chunks[i].Images[j].OriginalURL = common.CleanInvalidUTF8(chunks[i].Images[j].OriginalURL)
}
}
for i := range options.ParentChunks {
options.ParentChunks[i].Content = common.CleanInvalidUTF8(options.ParentChunks[i].Content)
}
// Check if knowledge is being deleted/cancelled before processing.
// Both statuses short-circuit identically here — there's nothing to clean
// up yet so the branch is purely "stop early".
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx, "Knowledge aborted (%s), skipping chunk processing: %s", status, knowledge.ID)
return abortRetryErr(ctx, knowledge.ID, status)
}
if s.isKnowledgeSourceReplaced(ctx, knowledge) {
logger.Infof(ctx, "Knowledge source replaced, skipping chunk processing: %s", knowledge.ID)
return nil
}
// A reparse started after this run: the cleanup below would wipe the
// new attempt's chunks and the final save would put its status back.
if s.currentAttemptSuperseded(ctx, knowledge.ID) {
logger.Infof(ctx, "Parse attempt superseded, skipping chunk processing: %s", knowledge.ID)
return nil
}
// Get embedding model for vectorization — only needed when vector/keyword indexing is enabled
var embeddingModel embedding.Embedder
if kb.NeedsEmbeddingModel() {
var err error
embeddingModel, err = s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
// Terminal for this attempt, and it has to be recorded as such.
// A KB that indexes vectors cannot proceed without an embedder;
// returning without a status leaves parse_status on "processing",
// so the row hangs until the housekeeping sweep fails it hours
// later with a generic "stuck in processing" message. Failing
// here names the real cause instead: a model row that was
// deleted or deactivated, or a configuration the embedder factory
// refuses. (Credential and connectivity problems surface later, on
// the first BatchIndex round-trip — resolving a model only reads
// the row and constructs a client.)
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get embedding model failed")
return s.failKnowledgeOnEmbeddingModel(ctx, kb, knowledge, err)
}
} else {
logger.Infof(ctx, "Vector/keyword indexing disabled for KB %s, skipping embedding model", kb.ID)
}
// Resolve the vector store before deleting anything: failing after the
// cleanup below would leave the document with neither old nor new chunks.
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil && embeddingModel != nil {
// Indexing below dereferences the engine; a nil one would panic.
logger.Errorf(ctx, "processChunks resolve vector store for KB %s failed: %v", kb.ID, err)
if errors.Is(err, retriever.ErrVectorStoreUnavailable) {
// The lookup failed, not the store: retry rather than fail.
return fmt.Errorf("resolve vector store: %w", err)
}
return s.failKnowledgeAtEmbedding(ctx, kb, knowledge, werrors.ErrCodeVectorStoreWriteFailed,
"failed to resolve vector store", err)
}
// 幂等性处理:清理旧的chunks和索引数据,避免重复数据
logger.Infof(ctx, "Cleaning up existing chunks and index data for knowledge: %s", knowledge.ID)
// 删除旧的chunks
if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil {
logger.Warnf(ctx, "Failed to delete existing chunks (may not exist): %v", err)
// 不返回错误,继续处理(可能没有旧数据)
}
// 删除旧的索引数据 — only when vector/keyword indexing is enabled
if embeddingModel != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err)
// 不返回错误,继续处理(可能没有旧数据)
} else {
logger.Infof(ctx, "Successfully deleted existing index data for knowledge: %s", knowledge.ID)
}
}
// 删除知识图谱数据(如果存在)
namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}
if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil {
logger.Warnf(ctx, "Failed to delete existing graph data (may not exist): %v", err)
// 不返回错误,继续处理
}
logger.Infof(ctx, "Cleanup completed, starting to process new chunks")
// ========== DocReader 解析结果日志 ==========
logger.Infof(ctx, "[DocReader] ========== 解析结果概览 ==========")
logger.Infof(ctx, "[DocReader] 知识ID: %s, 知识库ID: %s", knowledge.ID, knowledge.KnowledgeBaseID)
logger.Infof(ctx, "[DocReader] 总Chunk数量: %d", len(chunks))
// 统计图片信息
totalImages := 0
chunksWithImages := 0
for _, chunkData := range chunks {
if len(chunkData.Images) > 0 {
chunksWithImages++
totalImages += len(chunkData.Images)
}
}
logger.Infof(ctx, "[DocReader] 包含图片的Chunk数: %d, 总图片数: %d", chunksWithImages, totalImages)
// 打印每个Chunk的详细信息
for idx, chunkData := range chunks {
contentPreview := chunkData.Content
if len(contentPreview) > 200 {
contentPreview = contentPreview[:200] + "..."
}
logger.Infof(ctx, "[DocReader] Chunk #%d (seq=%d): 内容长度=%d, 图片数=%d, 范围=[%d-%d]",
idx, chunkData.Seq, len(chunkData.Content), len(chunkData.Images), chunkData.Start, chunkData.End)
logger.Debugf(ctx, "[DocReader] Chunk #%d 内容预览: %s", idx, contentPreview)
// 打印图片详细信息
for imgIdx, img := range chunkData.Images {
logger.Infof(ctx, "[DocReader] 图片 #%d: URL=%s", imgIdx, img.URL)
logger.Infof(ctx, "[DocReader] 图片 #%d: OriginalURL=%s", imgIdx, img.OriginalURL)
if img.Caption != "" {
captionPreview := img.Caption
if len(captionPreview) > 100 {
captionPreview = captionPreview[:100] + "..."
}
logger.Infof(ctx, "[DocReader] 图片 #%d: Caption=%s", imgIdx, captionPreview)
}
if img.OCRText != "" {
ocrPreview := img.OCRText
if len(ocrPreview) > 100 {
ocrPreview = ocrPreview[:100] + "..."
}
logger.Infof(ctx, "[DocReader] 图片 #%d: OCRText=%s", imgIdx, ocrPreview)
}
logger.Infof(ctx, "[DocReader] 图片 #%d: 位置=[%d-%d]", imgIdx, img.Start, img.End)
}
}
logger.Infof(ctx, "[DocReader] ========== 解析结果概览结束 ==========")
// Create chunk objects from proto chunks
maxSeq := 0
// 统计图片相关的子Chunk数量,用于扩展insertChunks的容量
imageChunkCount := 0
for _, chunkData := range chunks {
if len(chunkData.Images) > 0 {
// 为每个图片的OCR和Caption分别创建一个Chunk
imageChunkCount += len(chunkData.Images) * 2
}
if int(chunkData.Seq) > maxSeq {
maxSeq = int(chunkData.Seq)
}
}
// === Parent-Child Chunking: create parent chunks first ===
hasParentChild := len(options.ParentChunks) > 0
var parentDBChunks []*types.Chunk // indexed by ParsedParentChunk position
if hasParentChild {
parentDBChunks = make([]*types.Chunk, len(options.ParentChunks))
for i, pc := range options.ParentChunks {
parentDBChunks[i] = &types.Chunk{
ID: uuid.New().String(),
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: pc.Content,
ChunkIndex: pc.Seq,
IsEnabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
StartAt: pc.Start,
EndAt: pc.End,
ChunkType: types.ChunkTypeParentText,
SourceLocators: pc.SourceLocators,
}
}
// Set prev/next links for parent chunks
for i := range parentDBChunks {
if i > 0 {
parentDBChunks[i-1].NextChunkID = parentDBChunks[i].ID
parentDBChunks[i].PreChunkID = parentDBChunks[i-1].ID
}
}
logger.Infof(ctx, "Created %d parent chunks for parent-child strategy", len(parentDBChunks))
}
// 重新分配容量,考虑图片相关的Chunk + parent chunks
parentCount := len(options.ParentChunks)
insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount+parentCount)
// Add parent chunks first (they go into DB but NOT into the vector index)
if hasParentChild {
insertChunks = append(insertChunks, parentDBChunks...)
}
for idx, chunkData := range chunks {
if strings.TrimSpace(chunkData.Content) == "" {
continue
}
// 创建主文本Chunk
textChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: chunkData.Content,
ContextHeader: chunkData.ContextHeader,
ChunkIndex: int(chunkData.Seq),
IsEnabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
StartAt: int(chunkData.Start),
EndAt: int(chunkData.End),
ChunkType: types.ChunkTypeText,
SourceLocators: chunkData.SourceLocators,
}
// Wire up ParentChunkID for child chunks
if hasParentChild && chunkData.ParentIndex >= 0 && chunkData.ParentIndex < len(parentDBChunks) {
textChunk.ParentChunkID = parentDBChunks[chunkData.ParentIndex].ID
}
chunks[idx].ChunkID = textChunk.ID
insertChunks = append(insertChunks, textChunk)
}
// Sort chunks by index for proper ordering
sort.Slice(insertChunks, func(i, j int) bool {
return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex
})
// Collect retrievable text chunks only. ParentChunkID only controls parent expansion after retrieval.
// When ParentChunkID is empty, retrieval keeps the standalone child content without loading a parent.
textChunks := make([]*types.Chunk, 0, len(chunks))
for _, chunk := range insertChunks {
if chunk.ChunkType == types.ChunkTypeText {
textChunks = append(textChunks, chunk)
}
}
// 设置文本Chunk之间的前后关系 (skip if parent-child, children don't need prev/next links)
if !hasParentChild {
for i, chunk := range textChunks {
if i > 0 {
textChunks[i-1].NextChunkID = chunk.ID
}
if i < len(textChunks)-1 {
textChunks[i+1].PreChunkID = chunk.ID
}
}
}
// Check if knowledge is being deleted/cancelled before writing chunks.
// Nothing has been persisted yet, so both branches just bail.
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx, "Knowledge aborted (%s), skipping chunk write: %s", status, knowledge.ID)
return abortRetryErr(ctx, knowledge.ID, status)
}
if s.isKnowledgeSourceReplaced(ctx, knowledge) {
logger.Infof(ctx, "Knowledge source replaced, skipping chunk write: %s", knowledge.ID)
return nil
}
// Save chunks to database — ALWAYS, regardless of indexing strategy.
// Chunks are needed for wiki generation, graph extraction, and summary generation
// even when vector/keyword indexing is disabled.
s.beginStage(ctx, knowledge.ID, types.StageChunking, types.JSONMap{
"chunks_planned": len(insertChunks),
})
if err := s.chunkRepo.CreateChunks(ctx, insertChunks); err != nil {
knowledge.ParseStatus = types.ParseStatusFailed
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
s.failStage(ctx, knowledge.ID, types.StageChunking,
werrors.ErrCodeChunkingFailed, "create chunks failed", err)
return nil
}
totalChunkChars := 0
for _, c := range insertChunks {
totalChunkChars += len(c.Content)
}
s.endStage(ctx, knowledge.ID, types.StageChunking, types.JSONMap{
"chunks_written": len(insertChunks),
"total_text_chars": totalChunkChars,
})
// Create index information and perform vector indexing — only when vector/keyword is enabled.
// Chunks are ALWAYS saved to DB (above) because wiki and graph need them even without vector indexing.
var totalStorageSize int64
if kb.NeedsEmbeddingModel() && embeddingModel != nil {
embedInput := types.JSONMap{
"chunks_to_embed": len(textChunks),
"model_id": kb.EmbeddingModelID,
}
if dim := embeddingModel.GetDimensions(); dim > 0 {
embedInput["dim"] = dim
}
s.beginStage(ctx, knowledge.ID, types.StageEmbedding, embedInput)
// Create index information — only for child/flat chunks, NOT parent chunks.
// Parent chunks are stored for context retrieval but do not need vector embeddings.
// Prepend the document title to improve semantic alignment between
// question-style queries and statement-style chunk content.
indexInfoList := make([]*types.IndexInfo, 0, len(textChunks))
for _, chunk := range textChunks {
// chunk.EmbeddingContent prepends ContextHeader (heading breadcrumb)
// when the chunker populated it during Tier-1 splitting; falls back
// to plain Content otherwise. The document title sits outermost;
// custom metadata remains document-scoped model context.
indexContent := buildKnowledgeIndexContent(knowledge, chunk.EmbeddingContent())
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: indexContent,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
IsEnabled: true,
})
}
// Calculate storage size required for embeddings
totalStorageSize = retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfoList)
if tenantInfo.StorageQuota > 0 {
// Re-fetch tenant storage information
tenantInfo, err = s.tenantRepo.GetTenantByID(ctx, tenantInfo.ID)
if err != nil {
knowledge.ParseStatus = types.ParseStatusFailed
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
s.deleteUnindexedChunks(ctx, knowledge)
return nil
}
// Check if there's enough storage quota available
if tenantInfo.StorageUsed+totalStorageSize > tenantInfo.StorageQuota {
knowledge.ParseStatus = types.ParseStatusFailed
knowledge.ErrorMessage = "存储空间不足"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
s.deleteUnindexedChunks(ctx, knowledge)
return nil
}
}
// Check again before batch indexing (heavy operation).
// deleting → row is going away anyway, drop the chunks we just wrote.
// cancelled → user wants to keep what was already persisted, just stop.
if s.isKnowledgeSourceReplaced(ctx, knowledge) {
logger.Infof(ctx, "Knowledge source replaced, skipping indexing: %s", knowledge.ID)
return nil
}
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx, "Knowledge aborted (%s) before indexing: %s", status, knowledge.ID)
if status == types.ParseStatusDeleting {
if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil {
logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err)
}
}
return abortRetryErr(ctx, knowledge.ID, status)
}
err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList)
if err != nil {
// Embedding and vector-store errors are often transient. While the
// task has attempts left, leave the row processing and hand the
// error to the queue so the idempotent parse runs again; only the
// final attempt (or a caller without a retry loop) records failure.
willRetry := summaryTaskWillRetry(ctx)
if !willRetry {
knowledge.ParseStatus = types.ParseStatusFailed
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
if updateErr := s.repo.UpdateKnowledge(ctx, knowledge); updateErr != nil {
logger.Errorf(ctx, "Update knowledge status failed after batch index error: %v", updateErr)
}
}
// delete failed chunks
if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil {
logger.Errorf(ctx, "Delete chunks failed: %v", err)
}
// delete index
if err := retrieveEngine.DeleteByKnowledgeIDList(
ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type,
); err != nil {
logger.Errorf(ctx, "Delete index failed: %v", err)
}
// Map vector store / embedding rate-limit errors to a
// stable code so the UI can offer "retry later" hints.
code := werrors.ErrCodeVectorStoreWriteFailed
if isLikelyRateLimitError(err) {
code = werrors.ErrCodeEmbeddingRateLimit
}
if willRetry {
s.failStageRetryable(ctx, knowledge.ID, types.StageEmbedding,
code, "batch index failed", err)
return fmt.Errorf("batch index: %w", err)
}
s.failStage(ctx, knowledge.ID, types.StageEmbedding,
code, "batch index failed", err)
return nil
}
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
s.endStage(ctx, knowledge.ID, types.StageEmbedding, types.JSONMap{
"vectors_written": len(indexInfoList),
"storage_bytes": totalStorageSize,
})
// Final check before marking as completed.
// deleting → drop chunks+index we just wrote.
// cancelled → keep persisted data; the row stays in cancelled status
// and downstream stages skip via the entry guards.
if s.isKnowledgeSourceReplaced(ctx, knowledge) {
logger.Infof(ctx, "Knowledge source replaced, skipping completion: %s", knowledge.ID)
return nil
}
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx, "Knowledge aborted (%s) after indexing: %s", status, knowledge.ID)
if status == types.ParseStatusDeleting {
if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil {
logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err)
}
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type); err != nil {
logger.Warnf(ctx, "Failed to cleanup index after deletion detected: %v", err)
}
}
return abortRetryErr(ctx, knowledge.ID, status)
}
} else {
logger.Infof(ctx, "Vector/keyword indexing disabled for KB %s, skipping BatchIndex", kb.ID)
s.skipStage(ctx, knowledge.ID, types.StageEmbedding, "skipped")
}
// Check if this document has extracted images that will be processed asynchronously
isImage := IsImageType(knowledge.FileType)
isVideo := IsVideoType(knowledge.FileType)
pendingMultimodal := isImage && options.EnableMultimodel && len(options.StoredImages) > 0
pendingPDFMultimodal := !isImage && !isVideo && options.EnableMultimodel && len(options.StoredImages) > 0
if s.currentAttemptSuperseded(ctx, knowledge.ID) {
logger.Infof(ctx, "Parse attempt superseded, skipping completion: %s", knowledge.ID)
return nil
}
// Reparse has now written the replacement chunks and, when enabled, their
// replacement indexes. Only at this successful point is it safe to reclaim
// old soft-deleted rows. Keep this optional at the interface boundary so
// existing non-SQL repositories and narrow test doubles retain their
// lifecycle behavior; the concrete SQL chunk repository implements it.
if purger, ok := s.chunkRepo.(supersededChunkPurger); ok {
purged, purgeErr := purger.PurgeSupersededChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID)
if purgeErr != nil {
logger.Warnf(ctx, "Failed to purge superseded chunks for knowledge %s: %v", knowledge.ID, purgeErr)
} else if purged > 0 {
logger.Infof(ctx, "Purged %d superseded chunks for knowledge %s", purged, knowledge.ID)
}
}
now := time.Now()
finalizeIndexedKnowledgeState(
knowledge,
totalStorageSize,
len(textChunks),
pendingMultimodal || pendingPDFMultimodal,
now,
)
if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update knowledge failed")
}
// Enqueue multimodal tasks for images (async, non-blocking)
if options.EnableMultimodel && len(options.StoredImages) > 0 {
// Counts only, no policy claims: whether a given image is OCR'd is
// decided per image once its attributes are observed, so a boolean here
// would assert something the pipeline has not decided yet — and would
// drift from what actually happened.
s.beginStage(ctx, knowledge.ID, types.StageMultimodal, types.JSONMap{
"image_count": len(options.StoredImages),
})
if err := s.enqueueImageMultimodalTasks(
ctx, knowledge, kb, options.StoredImages, chunks, options.Metadata,
); err != nil {
// Nothing else moves the row out of "processing"; let asynq
// run the (idempotent) parse again, or dead-letter it.
return err
}
} else {
s.skipStage(ctx, knowledge.ID, types.StageMultimodal, "skipped")
// If there are no multimodal tasks, enqueue the post process task immediately
if err := s.enqueueKnowledgePostProcessTask(ctx, knowledge); err != nil {
return err
}
}
// Update tenant's storage usage
tenantInfo.StorageUsed += totalStorageSize
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update tenant storage used failed")
}
logger.GetLogger(ctx).Infof("processChunks successfully")
return nil
}
// defaultMaxInputChars is the default maximum characters used as input for
// summary generation. The document profile (short summary, gist, topics,
// type, one question) is decided by the head of a document; 8k characters
// keeps the per-document cost a quarter of the previous 24k without changing
// what the profile can say.
const defaultMaxInputChars = 1024 * 8
// defaultSummaryMaxTokens bounds the JSON profile reply; every field is short.
const defaultSummaryMaxTokens = 1024
// documentSummaryResult is what one profiling call yields: the short summary
// stored in knowledge.Description plus the structured profile, when the model
// returned parseable JSON.
type documentSummaryResult struct {
Summary string
Profile *types.KnowledgeProfile
}
// documentProfileOutput mirrors the JSON contract of generate_summary.yaml.
type documentProfileOutput struct {
Summary string `json:"summary"`
Gist string `json:"gist"`
Topics []string `json:"topics"`
DocType string `json:"doc_type"`
TypicalQuestion string `json:"typical_question"`
}
// normalizeSummaryReplyContent strips a UTF-8 BOM, the surrounding whitespace
// and one markdown code fence from a summary reply, so shape detection and
// parsing look at the same text. A fence without a newline carries no payload
// we can classify, so it normalizes to the empty string.
func normalizeSummaryReplyContent(content string) string {
content = strings.TrimSpace(strings.TrimPrefix(content, "\uFEFF"))
if rest, ok := strings.CutPrefix(content, "```"); ok {
// Drop the fence's info string (```json) and the newline after it.
idx := strings.IndexByte(rest, '\n')
if idx < 0 {
return ""
}
content = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(rest[idx+1:]), "```"))
}
return content
}
// summaryReplyIsJSONShaped reports whether the reply really is a structured
// JSON payload, either directly or inside a markdown code fence. Only such
// replies are held to the structured contract below; anything else is a
// legacy plain-text summary.
//
// An object opening ("{") always counts, parseable or not: half an object is
// exactly the reply that must never be stored as a summary. A bracket opening
// is ambiguous — "[1] …" and "[文档摘要] …" are prose with a leading citation,
// not broken JSON — so it only counts when the reply is a JSON array or opens
// an array of objects/strings.
func summaryReplyIsJSONShaped(content string) bool {
content = normalizeSummaryReplyContent(content)
if content == "" {
return false
}
switch content[0] {
case '{':
return true
case '[':
var arr []json.RawMessage
if json.Unmarshal([]byte(content), &arr) == nil {
return true
}
first := strings.TrimLeft(content[1:], " \t\r\n")
return strings.HasPrefix(first, "{") ||
strings.HasPrefix(first, "[") ||
strings.HasPrefix(first, `"`)
default:
return false
}
}
// parseDocumentSummaryOutput accepts both the structured JSON reply and a
// legacy plain-text summary (custom templates, older models). Plain text is
// stored as the summary with no profile, so nothing that worked before
// regresses; only the knowledge-base aggregation loses that document's topics.
//
// A reply that opens like JSON is held to the JSON contract instead: a broken
// object (output cut off at MaxTokens, an unescaped newline inside a string)
// or an object with no usable text returns errSummaryOutputNotParsable. The
// raw reply used to be stored as the summary, which put half a JSON object in
// knowledges.description, marked the summary completed so Asynq never retried
// it, and embedded the same fragment into the RAG index.
func parseDocumentSummaryOutput(content string) (*documentSummaryResult, error) {
// Some providers prefix the reply with a UTF-8 BOM; TrimSpace alone would
// leave it in place, hide the "{" and let half an object through as prose.
content = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(content), "\uFEFF"))
if content == "" {
return nil, errEmptySummaryOutput
}
jsonShaped := summaryReplyIsJSONShaped(content)
var out documentProfileOutput
if err := common.ParseLLMJsonResponse(content, &out); err != nil {
if jsonShaped {
return nil, fmt.Errorf("%w: %w", errSummaryOutputNotParsable, err)
}
// Legacy plain text: keep it as the summary with no profile.
return &documentSummaryResult{Summary: content}, nil
}
summary := strings.TrimSpace(out.Summary)
profile := (&types.KnowledgeProfile{
Gist: out.Gist,
Topics: out.Topics,
DocType: out.DocType,
TypicalQuestion: out.TypicalQuestion,
}).Normalize()
if summary == "" && profile != nil {
summary = profile.Gist
}
if summary == "" {
if jsonShaped {
// JSON without usable text is as unusable as broken JSON: the
// caller must fail the attempt instead of storing the object.
return nil, errSummaryOutputNotParsable
}
// Plain text that happened to contain balanced JSON punctuation.
return &documentSummaryResult{Summary: content}, nil
}
return &documentSummaryResult{Summary: summary, Profile: profile}, nil
}
// buildSummaryChunkContent is the text embedded for the document-level
// summary chunk. The gist leads so the vector reflects the headline first.
func buildSummaryChunkContent(summary string, profile *types.KnowledgeProfile) string {
summary = strings.TrimSpace(summary)
if profile != nil {
gist := strings.TrimSpace(profile.Gist)
if gist != "" && !strings.EqualFold(gist, summary) {
return "# Summary\n" + gist + "\n\n" + summary
}
}
return "# Summary\n" + summary
}
// imageDominatedTextThreshold is the rune count below which a document is
// considered "image-dominated" — i.e. the body text is so sparse that we
// should fall back to full image enrichment (caption + OCR) for the summary
// LLM call. Above this threshold the document has enough native text that
// caption-only enrichment is preferable (OCR text from incidental figures
// would otherwise add noise without contributing to the main topic).
const imageDominatedTextThreshold = 200
// errInsufficientSummaryContent signals that getSummary refused to call the
// LLM because the document had no usable text after image markup was stripped
// (typical for scanned PDFs where VLM OCR yielded nothing). Callers should
// mark the knowledge's summary as failed instead of falling back to the first
// chunk's raw content (which would just be a bare image reference).
var (
errInsufficientSummaryContent = errors.New("insufficient text content for summary generation")
errEmptySummaryOutput = errors.New("summary model returned empty output")
// errSummaryOutputNotParsable marks a reply that is JSON-shaped but yields
// no usable summary text — broken JSON (cut off at the completion budget,
// unescaped newline) or an object without summary/gist. It is deliberately
// not stored: the raw reply is not a summary.
errSummaryOutputNotParsable = errors.New("summary model returned JSON without a usable summary")
// errSummaryOutputTruncated marks a reply the provider stopped at the
// completion budget, so its JSON can never be trusted as complete.
errSummaryOutputTruncated = errors.New("summary model output was truncated at the completion budget")
)
const summaryFallbackMaxRunes = 500
// validateSummaryOutput rejects successful model responses that contain no
// user-visible text. It only looks at the content shape: a provider that
// stopped at the completion budget can still return usable text, and the older
// plain-text consumers (custom summary templates, table/column descriptions)
// deliberately keep whatever the model produced.
func validateSummaryOutput(response *types.ChatResponse) (string, error) {
if response == nil {
return "", errEmptySummaryOutput
}
content := strings.TrimSpace(tools.StripThinkBlocks(response.Content))
if content == "" {
return "", errEmptySummaryOutput
}
return content, nil
}
// validateStructuredSummaryDocument is the validator for the document-summary
// entry point (getSummary). On top of validateSummaryOutput it rejects a reply
// the provider cut off at the completion budget — but only when the reply was
// producing a structured summary, because that is the case where the truncated
// text is half a JSON object. A plain-text summary that hit the budget is
// still usable text and keeps the legacy fallback.
//
// The shape is judged before the finish reason, so "finish_reason=length" can
// never turn a plain-text reply into a failure on its own.
func validateStructuredSummaryDocument(response *types.ChatResponse) (string, error) {
content, err := validateSummaryOutput(response)
if err != nil {
return "", err
}
structured := summaryReplyIsJSONShaped(content)
if structured && chatpipeline.IsLengthFinishReason(response.FinishReason) {
return "", errSummaryOutputTruncated
}
return content, nil
}
// firstTextChunkSummaryFallback preserves the existing deterministic fallback:
// use the first already-ordered text chunk and cap it by runes so Chinese and
// emoji are never cut in the middle of a UTF-8 sequence.
func firstTextChunkSummaryFallback(textChunks []*types.Chunk) string {
if len(textChunks) == 0 || textChunks[0] == nil {
return ""
}
fallback := strings.TrimSpace(textChunks[0].Content)
runes := []rune(fallback)
if len(runes) > summaryFallbackMaxRunes {
fallback = string(runes[:summaryFallbackMaxRunes])
}
return fallback
}
// applyRetryableSummaryFailureState keeps an existing description visible
// while another attempt is queued, then publishes the deterministic fallback
// and marks only the summary subtask failed after the retry budget is exhausted.
func applyRetryableSummaryFailureState(
knowledge *types.Knowledge, textChunks []*types.Chunk, willRetry bool,
) string {
knowledge.UpdatedAt = time.Now()
if willRetry {
knowledge.SummaryStatus = types.SummaryStatusPending
return ""
}
fallback := firstTextChunkSummaryFallback(textChunks)
knowledge.Description = fallback
knowledge.Profile = nil
knowledge.SummaryStatus = types.SummaryStatusFailed
return fallback
}
// saveSummaryState persists only the columns the summary task owns. The task
// holds a row snapshot across its LLM call; a full-row save of it wrote the
// parse status of that moment back over a cancel, a housekeeping failure or
// a reparse that landed meanwhile. Over a reparse it put "finalizing" on the
// new run's "processing" row, whose post-process then found the row no
// longer processing, skipped its fan-out and left it stuck.
func (s *knowledgeService) saveSummaryState(ctx context.Context, knowledge *types.Knowledge) error {
if knowledge.UpdatedAt.IsZero() {
knowledge.UpdatedAt = time.Now()
}
return s.repo.UpdateKnowledgeColumns(ctx, knowledge.ID, map[string]interface{}{
"description": knowledge.Description,
"profile": knowledge.Profile,
"summary_status": knowledge.SummaryStatus,
"updated_at": knowledge.UpdatedAt,
})
}
// summaryTaskWillRetry reports whether the current Asynq delivery has another
// configured attempt remaining. Calls outside an Asynq worker are terminal.
// Despite the name it is not summary-specific; processChunks uses it too.
func summaryTaskWillRetry(ctx context.Context) bool {
retried, retryOK := asynq.GetRetryCount(ctx)
maxRetry, maxRetryOK := asynq.GetMaxRetry(ctx)
if retryOK && maxRetryOK {
return retried < maxRetry
}
retried, maxRetry, ok := types.TaskRetryMetadataFromContext(ctx)
return ok && retried < maxRetry
}
// checkSufficientSummaryContent returns errInsufficientSummaryContent if the
// given content does not carry enough real text (after stripping image markup)
// for an LLM summary call, and logs a warning at the call site. Returns nil
// when the content passes the threshold.
//
// Extracted so the threshold gate can be unit-tested without standing up the
// full ProcessSummaryGeneration dependency graph.
func checkSufficientSummaryContent(ctx context.Context, knowledgeID, content string) error {
realTextLen := realTextRuneCount(content)
if realTextLen < minTextContentRunes {
logger.GetLogger(ctx).Warnf(
"summary content check: knowledge %s has insufficient text after stripping image markup (real_text_runes=%d, min=%d); skipping LLM call",
knowledgeID, realTextLen, minTextContentRunes,
)
return errInsufficientSummaryContent
}
return nil
}
// sortChunksForSummary orders chunks for document reconstruction. Parser
// StartAt offsets stay authoritative until any chunk has been manually edited;
// after that ChunkIndex is the safer reading order.
func sortChunksForSummary(chunks []*types.Chunk) []*types.Chunk {
sorted := make([]*types.Chunk, len(chunks))
copy(sorted, chunks)
edited := false
for _, chunk := range sorted {
if chunk.ContentRevision > 0 {
edited = true
break
}
}
sort.Slice(sorted, func(i, j int) bool {
if edited {
if sorted[i].ChunkIndex != sorted[j].ChunkIndex {
return sorted[i].ChunkIndex < sorted[j].ChunkIndex
}
return sorted[i].ID < sorted[j].ID
}
return sorted[i].StartAt < sorted[j].StartAt
})
return sorted
}
// getSummary generates a summary for knowledge content using an AI model
func (s *knowledgeService) getSummary(ctx context.Context,
summaryModel chat.Chat, knowledge *types.Knowledge, chunks []*types.Chunk,
) (*documentSummaryResult, error) {
// Get knowledge info from the first chunk
if len(chunks) == 0 {
return nil, fmt.Errorf("no chunks provided for summary generation")
}
// Determine max input chars from config
maxInputChars := defaultMaxInputChars
if s.config.Conversation.Summary != nil && s.config.Conversation.Summary.MaxInputChars > 0 {
maxInputChars = s.config.Conversation.Summary.MaxInputChars
}
// Sort chunks for reconstruction. Parser offsets remain authoritative until
// any chunk has been manually edited; after that ChunkIndex is safer.
sortedChunks := sortChunksForSummary(chunks)
// Reconstruct the document before enriching it with image info. Enrichment
// must happen AFTER reconstruction because StartAt is based on original
// document offsets; enriched (longer) content would break the positioning.
chunkContents := ""
hasEditedChunk := false
for _, chunk := range sortedChunks {
if chunk.ContentRevision > 0 {
hasEditedChunk = true
break
}
}
if hasEditedChunk {
// Parser offsets describe the immutable source. Once a replacement has
// changed length they can no longer be applied to the effective content;
// concatenate current chunks instead of truncating at stale offsets.
parts := make([]string, 0, len(sortedChunks))
for _, chunk := range sortedChunks {
if chunk.IsEnabled && strings.TrimSpace(chunk.Content) != "" {
parts = append(parts, chunk.Content)
}
}
chunkContents = strings.Join(parts, "\n\n")
} else {
// Synthetic table headers are not represented in StartAt/EndAt. Match
// real text overlap instead of slicing by source offsets.
chunkContents = searchutil.MergeTextChunks(sortedChunks, "")
}
// Collect image_info from image_ocr/image_caption children and enrich
chunkIDs := make([]string, len(sortedChunks))
for i, c := range sortedChunks {
chunkIDs[i] = c.ID
}
imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, s.chunkRepo, knowledge.TenantID, chunkIDs)
mergedImageInfo := searchutil.MergeImageInfoJSON(imageInfoMap)
if mergedImageInfo != "" {
// For image-dominated documents (e.g. a docx whose only payload is a
// single embedded picture, or a screenshot-only file), captions alone
// often carry too little signal — the real content lives in OCR text.
// Detect that case by measuring the document's real (non-image-markup)
// text BEFORE enrichment, and switch to full enrichment (caption + OCR)
// when the body is essentially empty. Text-heavy documents stay on the
// caption-only path to avoid OCR noise (page headers/footers/watermarks
// from many figures diluting the main topic).
if realTextRuneCount(chunkContents) < imageDominatedTextThreshold {
// Caption + OCR (no URL/original wrappers — those are pure noise
// for the summary LLM and have been observed to trigger the
// "image reference with no extracted text" refusal heuristic).
chunkContents = searchutil.EnrichContentCaptionAndOCR(chunkContents, mergedImageInfo)
} else {
chunkContents = searchutil.EnrichContentCaptionOnly(chunkContents, mergedImageInfo)
}
}
// Apply length limit: sample long content to fit within maxInputChars
chunkContents = sampleLongContent(chunkContents, maxInputChars)
logger.GetLogger(ctx).Infof("getSummary: content length=%d chars (max=%d) for knowledge %s",
len([]rune(chunkContents)), maxInputChars, knowledge.ID)
// Bail out before the LLM call when there is not enough actual text to
// summarise. We deliberately do not pass filename/file-type metadata to the
// LLM: scanned PDFs frequently carry filenames like "MX5280.pdf" (the
// scanner model), and feeding that to the model would invite it to
// hallucinate a scanner manual instead of admitting the document had no
// extractable text.
if err := checkSufficientSummaryContent(ctx, knowledge.ID, chunkContents); err != nil {
return nil, err
}
// User-authored metadata is trusted document context. Internal ingestion
// metadata remains excluded because it contains IDs and pipeline controls.
contentWithMetadata := chunkContents
if custom := knowledge.CustomMetadataText(); custom != "" {
contentWithMetadata = "Document metadata:\n" + custom + "\n\nDocument content:\n" + chunkContents
}
contentWithMetadata = sampleLongContent(contentWithMetadata, maxInputChars)
// Determine max output tokens from config
maxTokens := defaultSummaryMaxTokens
if s.config.Conversation.Summary != nil && s.config.Conversation.Summary.MaxCompletionTokens > 0 {
maxTokens = s.config.Conversation.Summary.MaxCompletionTokens
}
// Generate summary using AI model
summaryPrompt := types.RenderPromptPlaceholders(s.config.Conversation.GenerateSummaryPrompt, types.PlaceholderValues{
"language": types.LanguageNameFromContext(ctx),
})
thinking := false
modelCtx := types.WithLLMCallMetadata(ctx, "document_summary", "")
summary, err := summaryModel.Chat(modelCtx, []chat.Message{
{
Role: "system",
Content: summaryPrompt,
},
{
Role: "user",
Content: contentWithMetadata,
},
}, &chat.ChatOptions{
Temperature: 0.3,
MaxTokens: maxTokens,
Thinking: &thinking,
})
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("GetSummary failed")
return nil, err
}
content, err := validateStructuredSummaryDocument(summary)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Warnf("GetSummary returned no usable content")
return nil, err
}
result, err := parseDocumentSummaryOutput(content)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Warnf("GetSummary returned unusable content")
return nil, err
}
logger.GetLogger(ctx).WithField("summary", result.Summary).
WithField("has_profile", result.Profile != nil).Infof("GetSummary success")
return result, nil
}
// sampleLongContent returns content that fits within maxChars.
// For short content (≤ maxChars), it is returned as-is.
// For long content, it samples: head (60%), tail (20%), and evenly-spaced middle (20%),
// joined by "[...content omitted...]" markers so the LLM knows content was skipped.
func sampleLongContent(content string, maxChars int) string {
runes := []rune(content)
if len(runes) <= maxChars {
return content
}
const omitMarker = "\n\n[...content omitted...]\n\n"
omitRunes := len([]rune(omitMarker))
// Reserve space for two omit markers (head→middle, middle→tail)
usable := maxChars - 2*omitRunes
if usable < 100 {
// Fallback: just truncate
return string(runes[:maxChars])
}
headLen := usable * 60 / 100
tailLen := usable * 20 / 100
midLen := usable - headLen - tailLen
head := string(runes[:headLen])
tail := string(runes[len(runes)-tailLen:])
// Sample middle portion: take a contiguous block from the center of the document
midStart := len(runes)/2 - midLen/2
if midStart < headLen {
midStart = headLen
}
midEnd := midStart + midLen
if midEnd > len(runes)-tailLen {
midEnd = len(runes) - tailLen
midStart = midEnd - midLen
if midStart < headLen {
midStart = headLen
}
}
middle := string(runes[midStart:midEnd])
return head + omitMarker + middle + omitMarker + tail
}
// ProcessSummaryGeneration handles async summary generation task
func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) (retErr error) {
var payload types.SummaryGenerationPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal summary generation payload: %v", err)
return nil // Don't retry on unmarshal error
}
logger.Infof(ctx, "Processing summary generation for knowledge: %s", payload.KnowledgeID)
// Set tenant and language context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// A newer attempt (re-upload / edit / reparse) has superseded this one:
// skip before opening the span or registering the FinalizeSubtask defer
// so we neither read stale chunks nor decrement the new attempt's counter.
if attemptSuperseded(ctx, s.tracker(), payload.KnowledgeID, payload.Attempt) {
logger.Infof(ctx, "summary: attempt %d superseded for %s, skipping stale enrichment",
payload.Attempt, payload.KnowledgeID)
return nil
}
if payload.Refresh {
var err error
ctx, err = restoreSummaryRefreshTenantInfo(ctx, s.tenantRepo, payload.TenantID)
if err == nil {
_, err = s.RegenerateKnowledgeSummary(ctx, payload.KnowledgeID)
}
if err != nil {
if errors.Is(err, ErrSummaryRefreshStale) {
logger.Infof(ctx, "Discarding stale summary refresh for knowledge %s", payload.KnowledgeID)
return nil
}
logger.Warnf(ctx, "Summary refresh failed for knowledge %s: %v", payload.KnowledgeID, err)
if errors.Is(err, errInsufficientSummaryContent) {
return nil
}
return err
}
return nil
}
// Open a subspan under the parent attempt's postprocess stage so the
// trace surface shows the real summary-generation duration (LLM call
// + chunk write + index) instead of just the upstream's enqueue time.
// Closes via the deferred handler below — every return path lands in
// the defer, including the early returns ahead.
span := s.beginPostprocessSubspan(ctx, payload.KnowledgeID, payload.Attempt, "postprocess.summary",
types.JSONMap{
"language": payload.Language,
})
var summaryErr error
summaryOut := types.JSONMap{}
// profileKB is the knowledge base once loaded; the deferred handler uses
// it to schedule the knowledge-base description refresh on terminal exit.
var profileKB *types.KnowledgeBase
defer func() {
// Decrement the parent's enrichment counter on terminal exit.
// "Terminal" is keyed on the value RETURNED to asynq, not on
// summaryErr: several branches record a failure on the span
// (summaryErr != nil) yet deliberately `return nil` so asynq does
// NOT retry (e.g. insufficient text content, KB/knowledge fetch
// failures). Those are terminal and must drain — keying on
// summaryErr would skip them and leave the row stuck in
// "finalizing". When we DO return an error asynq will retry, so
// we only drain on the final attempt.
finalizeSubtaskDetached(ctx, s.repo, payload.KnowledgeID, "summary",
retErr, false, isFinalAsynqAttempt(ctx))
// Every terminal exit changes what the knowledge-base aggregation
// sees (a new profile, a cleared one, or a document that will never
// get one), so the debounced refresh is requested regardless of
// outcome. It is a no-op unless the KB opted in.
if retErr == nil && profileKB != nil {
_ = requestKnowledgeBaseProfileRefresh(ctx, s.task, profileKB, false)
}
if span == nil {
return
}
if summaryErr != nil {
s.failPostprocessSubspan(ctx, span, "SUMMARY_FAILED", summaryErr.Error(), summaryErr)
} else {
s.endPostprocessSubspan(ctx, span, summaryOut)
}
}()
// Get knowledge base
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
summaryErr = err
return nil
}
// Capture the resolved model id on the span output the moment we
// know it — debugging "summary stage took 60s" benefits hugely from
// seeing WHICH chat model was actually used (kb config drift, fall-
// throughs to a slow upstream, etc.).
summaryOut["model_id"] = kb.SummaryModelID
profileKB = kb
if kb.SummaryModelID == "" {
logger.Warn(ctx, "Knowledge base summary model ID is empty, skipping summary generation")
_ = s.repo.UpdateKnowledgeColumn(ctx, payload.KnowledgeID, "summary_status", types.SummaryStatusFailed)
summaryOut["skipped"] = "no_summary_model"
return nil
}
// Get knowledge
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
summaryErr = err
return nil
}
// Short-circuit when the user cancelled parsing or the row is being deleted.
if knowledge != nil {
switch knowledge.ParseStatus {
case types.ParseStatusCancelled, types.ParseStatusDeleting:
logger.Infof(ctx, "Summary generation: knowledge aborted (%s), skipping: %s",
knowledge.ParseStatus, payload.KnowledgeID)
summaryOut["skipped"] = "knowledge_" + knowledge.ParseStatus
return nil
}
}
// Update summary status to processing
knowledge.SummaryStatus = types.SummaryStatusProcessing
knowledge.UpdatedAt = time.Now()
if err := s.saveSummaryState(ctx, knowledge); err != nil {
logger.Warnf(ctx, "Failed to update summary status to processing: %v", err)
}
// Helper function to mark summary as failed
markSummaryFailed := func() {
knowledge.SummaryStatus = types.SummaryStatusFailed
knowledge.UpdatedAt = time.Now()
if err := s.saveSummaryState(ctx, knowledge); err != nil {
logger.Warnf(ctx, "Failed to update summary status to failed: %v", err)
}
}
// Get text chunks for this knowledge
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to get chunks: %v", err)
markSummaryFailed()
summaryErr = err
return nil
}
// Filter text chunks only
textChunks := make([]*types.Chunk, 0)
for _, chunk := range chunks {
if chunk.ChunkType == types.ChunkTypeText {
textChunks = append(textChunks, chunk)
}
}
summaryOut["text_chunks"] = len(textChunks)
if len(textChunks) == 0 {
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
knowledge.Description = ""
knowledge.Profile = nil
knowledge.SummaryStatus = types.SummaryStatusFailed
knowledge.UpdatedAt = time.Now()
if err := s.saveSummaryState(ctx, knowledge); err != nil {
logger.Warnf(ctx, "Failed to mark summary failed for knowledge without text chunks: %v", err)
}
summaryOut["skipped"] = "no_text_chunks"
return nil
}
// Sort chunks by ChunkIndex for proper ordering
sort.Slice(textChunks, func(i, j int) bool {
return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex
})
summaryMetadataVersion := string(knowledge.CustomMetadata)
handleRetryableSummaryFailure := func(generationErr error) error {
summaryErr = generationErr
summaryOut["error"] = previewText(generationErr.Error(), 500)
summaryOut["error_type"] = fmt.Sprintf("%T", generationErr)
if summaryTaskWillRetry(ctx) {
applyRetryableSummaryFailureState(knowledge, textChunks, true)
if updateErr := s.saveSummaryState(ctx, knowledge); updateErr != nil {
logger.Warnf(ctx, "Failed to mark summary pending for retry: %v", updateErr)
}
summaryOut["retrying"] = true
return fmt.Errorf("summary generation attempt failed: %w", generationErr)
}
// Before publishing the terminal fallback, make sure its source still
// matches the chunks and metadata captured for this attempt.
stale, staleErr := summarySourceChanged(
ctx, s.repo, s.chunkRepo, payload.TenantID, payload.KnowledgeID,
summaryMetadataVersion, textChunks,
)
if staleErr != nil {
logger.Errorf(ctx, "Failed to verify summary fallback freshness for knowledge %s: %v",
payload.KnowledgeID, staleErr)
markSummaryFailed()
summaryErr = staleErr
return fmt.Errorf("verify summary fallback freshness: %w", staleErr)
}
if stale {
logger.Infof(ctx, "Discarding stale summary fallback for knowledge %s", payload.KnowledgeID)
summaryOut["skipped"] = "content_revision_changed"
return nil
}
fallback := applyRetryableSummaryFailureState(knowledge, textChunks, false)
if updateErr := s.saveSummaryState(ctx, knowledge); updateErr != nil {
logger.Errorf(ctx, "Failed to save terminal summary fallback: %v", updateErr)
summaryErr = updateErr
return fmt.Errorf("save terminal summary fallback: %w", updateErr)
}
if fallback == "" {
summaryOut["fallback"] = "empty"
} else {
summaryOut["fallback"] = "first_chunk"
}
summaryOut["fallback_chars"] = len([]rune(fallback))
return fmt.Errorf("summary generation exhausted retries: %w", generationErr)
}
// Initialize chat model for summary. Model resolution failures use the same
// retry budget and terminal first-chunk fallback as LLM request failures.
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return handleRetryableSummaryFailure(fmt.Errorf("get chat model: %w", err))
}
// Generate summary
summaryResult, err := s.getSummary(ctx, chatModel, knowledge, textChunks)
if err != nil {
logger.Errorf(ctx, "Failed to generate summary for knowledge %s: %v", payload.KnowledgeID, err)
// Surface the underlying LLM/IO error on the span so the trace UI
// can explain "why did this stage take 60s and then fall back?"
// without forcing the operator to grep worker logs. We also capture
// the error type to disambiguate timeouts from upstream HTTP errors
// (deadline exceeded vs unexpected EOF vs 5xx, etc.).
summaryOut["error"] = previewText(err.Error(), 500)
summaryOut["error_type"] = fmt.Sprintf("%T", err)
// For the insufficient-content case (scanned PDF without OCR, etc.)
// we deliberately do NOT fall back to the first chunk's raw content,
// since that chunk is typically just a bare markdown image reference
// and surfacing it in the description is misleading.
if errors.Is(err, errInsufficientSummaryContent) {
knowledge.Description = ""
knowledge.Profile = nil
knowledge.SummaryStatus = types.SummaryStatusFailed
knowledge.UpdatedAt = time.Now()
if updateErr := s.saveSummaryState(ctx, knowledge); updateErr != nil {
logger.Errorf(ctx, "Failed to mark summary as failed: %v", updateErr)
summaryErr = updateErr
return fmt.Errorf("failed to update knowledge: %w", updateErr)
}
summaryOut["fallback"] = "insufficient_content"
summaryErr = err
return nil
}
return handleRetryableSummaryFailure(err)
}
// Do not publish an answer derived from a superseded chunk or metadata
// version. A user can explicitly refresh again from the latest revision.
staleSummary, staleErr := summarySourceChanged(
ctx, s.repo, s.chunkRepo, payload.TenantID, payload.KnowledgeID, summaryMetadataVersion, textChunks,
)
if staleErr != nil {
logger.Errorf(ctx, "Failed to verify summary freshness for knowledge %s: %v", payload.KnowledgeID, staleErr)
markSummaryFailed()
summaryErr = staleErr
return fmt.Errorf("verify summary freshness: %w", staleErr)
}
if staleSummary {
logger.Infof(ctx, "Discarding stale summary for knowledge %s", payload.KnowledgeID)
summaryOut["skipped"] = "content_revision_changed"
return nil
}
// Update knowledge description and structured profile
summary := summaryResult.Summary
knowledge.Description = summary
knowledge.Profile = summaryResult.Profile
knowledge.SummaryStatus = types.SummaryStatusCompleted
knowledge.UpdatedAt = time.Now()
summaryOut["summary_chars"] = len([]rune(summary))
summaryOut["has_profile"] = summaryResult.Profile != nil
// Preview the generated summary on the span output so the trace
// viewer can show "this is what the LLM produced" at a glance,
// without hopping to the knowledge-detail page. Capped to keep
// span rows compact.
summaryOut["summary_preview"] = previewText(summary, 240)
if err := s.saveSummaryState(ctx, knowledge); err != nil {
logger.Errorf(ctx, "Failed to update knowledge description: %v", err)
summaryErr = err
return fmt.Errorf("failed to update knowledge: %w", err)
}
// Create summary chunk and index it — only when RAG indexing is enabled.
// Wiki-only KBs don't need summary chunks in the vector index.
if strings.TrimSpace(summary) != "" && kb.NeedsEmbeddingModel() {
// Get max chunk index
maxChunkIndex := 0
for _, chunk := range chunks {
if chunk.ChunkIndex > maxChunkIndex {
maxChunkIndex = chunk.ChunkIndex
}
}
// Embed only the LLM-generated summary in the indexed chunk.
// We deliberately omit knowledge.FileName here: filenames are an
// unreliable signal (e.g. "MX5280.pdf" for a scanned legal letter)
// and surfacing them in retrieved RAG context can re-introduce the
// hallucination vector this branch is meant to close.
summaryChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: buildSummaryChunkContent(summary, summaryResult.Profile),
ChunkIndex: maxChunkIndex + 1,
IsEnabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
StartAt: 0,
EndAt: 0,
ChunkType: types.ChunkTypeSummary,
ParentChunkID: textChunks[0].ID,
}
// Save summary chunk
if err := s.chunkRepo.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil {
logger.Errorf(ctx, "Failed to create summary chunk: %v", err)
summaryErr = err
return fmt.Errorf("failed to create summary chunk: %w", err)
}
// Index summary chunk
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
summaryErr = err
return fmt.Errorf("failed to get tenant info: %w", err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil {
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
summaryErr = err
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
summaryErr = err
return fmt.Errorf("failed to get embedding model: %w", err)
}
indexInfo := []*types.IndexInfo{{
Content: summaryChunk.Content,
SourceID: summaryChunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: summaryChunk.ID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
IsEnabled: true,
}}
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil {
logger.Errorf(ctx, "Failed to index summary chunk: %v", err)
summaryErr = err
return fmt.Errorf("failed to index summary chunk: %w", err)
}
logger.Infof(ctx, "Successfully created and indexed summary chunk for knowledge: %s", payload.KnowledgeID)
summaryOut["summary_chunk_indexed"] = true
}
logger.Infof(ctx, "Successfully generated summary for knowledge: %s", payload.KnowledgeID)
summaryOut["status"] = "completed"
return nil
}
// ProcessQuestionGeneration handles async question generation task. It
// dispatches between the batched fan-out path (current: one task per window of
// text chunks, payload.ChunkIDs set) and the legacy whole-knowledge path (kept
// for tasks enqueued before fan-out shipped, no chunk ids). A lone ChunkID
// (from an interim per-chunk build) is treated as a one-element batch.
func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) (retErr error) {
var payload types.QuestionGenerationPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal question generation payload: %v", err)
return nil // Don't retry on unmarshal error
}
if len(payload.ChunkIDs) > 0 || payload.ChunkID != "" {
return s.processQuestionGenerationForChunks(ctx, t, payload)
}
return s.processQuestionGenerationForKnowledge(ctx, t, payload)
}
// processQuestionGenerationForKnowledge is the legacy whole-knowledge handler:
// it iterates every text chunk of the knowledge in one task. Retained for
// in-flight tasks queued before per-chunk fan-out; new enqueues always set
// payload.ChunkID and take the per-chunk path instead.
func (s *knowledgeService) processQuestionGenerationForKnowledge(ctx context.Context, t *asynq.Task, payload types.QuestionGenerationPayload) (retErr error) {
taskStartedAt := time.Now()
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
exitStatus := "success"
totalChunks := 0
totalTextChunks := 0
emptyContentChunks := 0
llmCallAttempts := 0
llmCallSuccess := 0
llmCallFailed := 0
llmCallEmpty := 0
generatedQuestionsTotal := 0
chunkMetadataSetFailed := 0
chunkUpdateFailed := 0
indexEntriesPrepared := 0
indexBatchAttempted := false
indexBatchSucceeded := false
// Sample question + model id surfaced on the span output so the
// trace viewer can answer "what did the LLM actually produce?" and
// "which model did it run on?" without joining back to the chunk
// store. Captured the first time we see a non-empty question batch.
var sampleQuestion string
var resolvedModelID string
// Postprocess subspan for the trace viewer. Opened lazily after we
// unmarshal the payload (so we have payload.Attempt) and closed in
// the defer below alongside the stats log so the span output mirrors
// what we already log to stdout.
var qSpan *Span
var qErr error
// Set when a newer attempt supersedes this run; suppresses the
// FinalizeSubtask decrement so a stale task can't drain the new
// attempt's counter.
superseded := false
// Decrement enrichment counter on terminal exit. Keyed on the value
// RETURNED to asynq (retErr), not qErr: some branches record a span
// failure (qErr != nil) yet `return nil` so asynq won't retry (KB /
// knowledge fetch failures); those are terminal and must drain.
// Keying on qErr would skip them and strand the row in "finalizing".
// When we return an error asynq retries, so we only drain on the
// final attempt. Runs AFTER the stats-log defer below — defers
// unwind LIFO, so this one declared first executes last.
defer func() {
finalizeSubtaskDetached(ctx, s.repo, payload.KnowledgeID, "question_legacy",
retErr, superseded, isFinalAsynqAttempt(ctx))
}()
defer func() {
logger.Infof(
ctx,
"Question generation stats: knowledge=%s kb=%s retry=%d/%d status=%s elapsed=%s chunks(total=%d,text=%d,empty_text=%d) llm(attempt=%d,success=%d,empty=%d,failed=%d) generated_questions=%d chunk_update_failed=%d metadata_set_failed=%d index(prepared=%d,attempted=%v,succeeded=%v)",
payload.KnowledgeID,
payload.KnowledgeBaseID,
retryCount,
maxRetry,
exitStatus,
time.Since(taskStartedAt).Round(time.Millisecond),
totalChunks,
totalTextChunks,
emptyContentChunks,
llmCallAttempts,
llmCallSuccess,
llmCallEmpty,
llmCallFailed,
generatedQuestionsTotal,
chunkUpdateFailed,
chunkMetadataSetFailed,
indexEntriesPrepared,
indexBatchAttempted,
indexBatchSucceeded,
)
if llmCallEmpty > 0 {
// Empty completions are counted, not logged, at the call site; a
// run where every chunk came back empty is indistinguishable from
// a healthy run in the application log. Reasoning-style models
// burning the whole budget on thinking are the known cause —
// point operators at the knob instead of leaving them to guess.
logger.Warnf(ctx,
"Question generation: %d of %d model calls returned empty completions "+
"(reasoning-style models can spend the completion budget on thinking even when disabled; "+
"raise %s to give the reply room)",
llmCallEmpty, llmCallAttempts, generateQuestionsMaxTokensEnv)
}
if qSpan != nil {
out := types.JSONMap{
"status": exitStatus,
"total_chunks": totalChunks,
"text_chunks": totalTextChunks,
"empty_content_chunks": emptyContentChunks,
"llm_attempts": llmCallAttempts,
"llm_success": llmCallSuccess,
"llm_empty": llmCallEmpty,
"llm_failed": llmCallFailed,
"questions_generated": generatedQuestionsTotal,
"chunk_update_failed": chunkUpdateFailed,
"metadata_set_failed": chunkMetadataSetFailed,
"index_entries_prepared": indexEntriesPrepared,
"index_batch_attempted": indexBatchAttempted,
"index_batch_succeeded": indexBatchSucceeded,
"retry": retryCount,
"max_retry": maxRetry,
}
// Surface the resolved model id and a sample question on the
// span output. These help debugging "why is question generation
// slow" — both questions ("which model was hit?") and ("what
// did it produce?") are hard to answer from logs alone.
if resolvedModelID != "" {
out["model_id"] = resolvedModelID
}
if sampleQuestion != "" {
out["sample_question"] = sampleQuestion
}
// Treat any non-success exitStatus as a failed run; the
// existing stats-string already enumerates them. qErr stays
// optional for callers that want to surface a Go error.
if exitStatus != "success" || qErr != nil {
msg := exitStatus
var detailErr error = qErr
if qErr != nil {
msg = qErr.Error()
}
s.failPostprocessSubspan(ctx, qSpan, "QUESTION_FAILED", msg, detailErr)
} else {
s.endPostprocessSubspan(ctx, qSpan, out)
}
}
}()
logger.Infof(ctx, "Processing question generation for knowledge: %s", payload.KnowledgeID)
// A newer attempt has superseded this one: skip before opening the span
// so we don't read stale chunks. superseded suppresses the counter
// decrement in the defer above; qSpan stays nil so the stats defer no-ops.
if attemptSuperseded(ctx, s.tracker(), payload.KnowledgeID, payload.Attempt) {
superseded = true
exitStatus = "superseded"
logger.Infof(ctx, "question: attempt %d superseded for %s, skipping stale enrichment",
payload.Attempt, payload.KnowledgeID)
return nil
}
// Open the postprocess.question subspan now that we have payload.Attempt.
// Closes via the defer above.
qSpan = s.beginPostprocessSubspan(ctx, payload.KnowledgeID, payload.Attempt, "postprocess.question",
types.JSONMap{
"question_count": payload.QuestionCount,
"language": payload.Language,
})
// Set tenant context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
if strings.TrimSpace(s.config.Conversation.GenerateQuestionsPrompt) == "" {
exitStatus = "prompt_not_configured"
logger.Errorf(ctx, "GenerateQuestionsPrompt is empty: configure conversation.generate_questions_prompt_id")
qErr = fmt.Errorf("generate questions prompt not configured")
return qErr
}
// Get knowledge base
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
exitStatus = "kb_not_found"
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
qErr = err
return nil
}
// Get knowledge
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
exitStatus = "knowledge_not_found"
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
qErr = err
return nil
}
// Short-circuit when the user cancelled parsing or the row is being deleted.
if knowledge != nil {
switch knowledge.ParseStatus {
case types.ParseStatusCancelled, types.ParseStatusDeleting:
exitStatus = "knowledge_" + knowledge.ParseStatus
logger.Infof(ctx, "Question generation: knowledge aborted (%s), skipping: %s",
knowledge.ParseStatus, payload.KnowledgeID)
return nil
}
}
// Get text chunks for this knowledge
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
exitStatus = "list_chunks_failed"
logger.Errorf(ctx, "Failed to get chunks: %v", err)
return nil
}
totalChunks = len(chunks)
// Filter text chunks only
textChunks := make([]*types.Chunk, 0)
for _, chunk := range chunks {
if chunk.ChunkType == types.ChunkTypeText {
textChunks = append(textChunks, chunk)
}
}
totalTextChunks = len(textChunks)
if len(textChunks) == 0 {
exitStatus = "no_text_chunks"
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
return nil
}
// Sort chunks by StartAt for context building
sort.Slice(textChunks, func(i, j int) bool {
return textChunks[i].StartAt < textChunks[j].StartAt
})
// Initialize chat model
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
exitStatus = "get_chat_model_failed"
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return fmt.Errorf("failed to get chat model: %w", err)
}
resolvedModelID = kb.SummaryModelID
// Initialize embedding model and retrieval engine
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
exitStatus = "get_embedding_model_failed"
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
return fmt.Errorf("failed to get embedding model: %w", err)
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
exitStatus = "get_tenant_failed"
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
return fmt.Errorf("failed to get tenant info: %w", err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil {
exitStatus = "init_retrieve_engine_failed"
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
questionCount := payload.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
processOverrides, _ := knowledge.ProcessOverrides()
questionGenCfg := ResolveProcessConfig(kb, processOverrides).QuestionGenerationConfig
customInstructions := questionGenCfg.CustomInstructions
// Collect image info for all text chunks so question generation can
// see caption / OCR text instead of bare image links.
textChunkIDs := make([]string, len(textChunks))
for i, c := range textChunks {
textChunkIDs[i] = c.ID
}
imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, s.chunkRepo, payload.TenantID, textChunkIDs)
enrichContent := func(chunk *types.Chunk) string {
if info, ok := imageInfoMap[chunk.ID]; ok && info != "" {
return searchutil.EnrichContentWithImageInfo(chunk.Content, info)
}
return chunk.Content
}
// Generate questions for each chunk with context
var indexInfoList []*types.IndexInfo
for i, chunk := range textChunks {
if strings.TrimSpace(chunk.Content) == "" {
emptyContentChunks++
continue
}
// Build context from adjacent chunks
var prevContent, nextContent string
if i > 0 {
prevContent = enrichContent(textChunks[i-1])
}
if i < len(textChunks)-1 {
nextContent = enrichContent(textChunks[i+1])
}
generationRevision := chunk.ContentRevision
llmCallAttempts++
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent,
knowledge.Title, questionCount, customInstructions)
if err != nil {
llmCallFailed++
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err)
continue
}
if len(questions) == 0 {
llmCallEmpty++
continue
}
latestChunk, latestErr := s.chunkRepo.GetChunkByID(ctx, payload.TenantID, chunk.ID)
if latestErr != nil || latestChunk.ContentRevision != generationRevision {
logger.Infof(ctx, "Skipping stale generated questions for chunk %s (revision changed)", chunk.ID)
continue
}
chunk = latestChunk
llmCallSuccess++
generatedQuestionsTotal += len(questions)
if sampleQuestion == "" && len(questions) > 0 {
sampleQuestion = previewText(questions[0], 200)
}
// Update chunk metadata with unique IDs for each question
generatedQuestions := make([]types.GeneratedQuestion, len(questions))
questionRevision := chunk.ContentRevision
for j, question := range questions {
questionID := fmt.Sprintf("q%d", time.Now().UnixNano()+int64(j))
generatedQuestions[j] = types.GeneratedQuestion{
ID: questionID,
Question: question,
ContentRevision: &questionRevision,
}
}
meta := &types.DocumentChunkMetadata{
GeneratedQuestions: generatedQuestions, GeneratedQuestionsRevision: chunk.ContentRevision,
}
if err := chunk.SetDocumentMetadata(meta); err != nil {
chunkMetadataSetFailed++
logger.Warnf(ctx, "Failed to set document metadata for chunk %s: %v", chunk.ID, err)
continue
}
// Update chunk in database
if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {
chunkUpdateFailed++
logger.Warnf(ctx, "Failed to update chunk %s: %v", chunk.ID, err)
continue
}
// Create index entries for generated questions
for _, gq := range generatedQuestions {
sourceID := types.GeneratedQuestionSourceID(chunk.ID, gq.ID)
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: buildKnowledgeIndexContent(knowledge, gq.Question),
SourceID: sourceID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
IsEnabled: true,
})
}
logger.Debugf(ctx, "Generated %d questions for chunk %s", len(questions), chunk.ID)
}
indexEntriesPrepared = len(indexInfoList)
// Index generated questions
if len(indexInfoList) > 0 {
indexBatchAttempted = true
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
exitStatus = "index_questions_failed"
logger.Errorf(ctx, "Failed to index generated questions: %v", err)
return fmt.Errorf("failed to index questions: %w", err)
}
indexBatchSucceeded = true
logger.Infof(ctx, "Successfully indexed %d generated questions for knowledge: %s", len(indexInfoList), payload.KnowledgeID)
}
return nil
}
// processQuestionGenerationForChunks generates questions for a batch (window)
// of text chunks. This is the batched fan-out path (one asynq task per
// questionGenChunkBatchSize chunks), aligned with the graph-extract
// TypeChunkExtract pattern: independent retry, per-batch cancellation, and a
// postprocess.question.batch[i] subspan. The payload carries only chunk ids
// (never content); content is read fresh here, and all questions for the batch
// are indexed in a single embedding BatchIndex call.
func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Context, t *asynq.Task, payload types.QuestionGenerationPayload) (retErr error) {
taskStartedAt := time.Now()
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
// Normalize the batch: prefer ChunkIDs, fall back to a lone ChunkID
// (interim per-chunk build) so those in-flight tasks still run.
batchIDs := payload.ChunkIDs
if len(batchIDs) == 0 && payload.ChunkID != "" {
batchIDs = []string{payload.ChunkID}
}
exitStatus := "success"
chunksInBatch := len(batchIDs)
chunksProcessed := 0
emptyChunks := 0
llmCallEmpty := 0
llmCallFailed := 0
generatedQuestionsTotal := 0
indexEntriesPrepared := 0
indexBatchSucceeded := false
var sampleQuestion string
var resolvedModelID string
var qSpan *Span
var qErr error
// Suppresses the FinalizeSubtask drain when a newer attempt superseded
// this run, so a stale task can't decrement the new attempt's counter.
superseded := false
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// Drain the parent's enrichment counter on terminal exit. Keyed on the
// value RETURNED to asynq (retErr), not qErr: some branches record a
// span failure yet `return nil` (terminal, must drain). Declared first
// so it runs LAST (after the stats/span defer below).
defer func() {
finalizeSubtaskDetached(ctx, s.repo, payload.KnowledgeID,
fmt.Sprintf("question_batch[%d]", payload.BatchIndex),
retErr, superseded, isFinalAsynqAttempt(ctx))
}()
defer func() {
logger.Infof(ctx,
"Question generation (batch) stats: knowledge=%s batch=%d "+
"chunks(in_batch=%d,processed=%d,empty=%d,llm_empty=%d) llm_failed=%d retry=%d/%d "+
"status=%s elapsed=%s generated_questions=%d index(entries=%d,succeeded=%v)",
payload.KnowledgeID, payload.BatchIndex, chunksInBatch, chunksProcessed, emptyChunks,
llmCallEmpty, llmCallFailed, retryCount, maxRetry, exitStatus,
time.Since(taskStartedAt).Round(time.Millisecond),
generatedQuestionsTotal, indexEntriesPrepared, indexBatchSucceeded,
)
if llmCallEmpty > 0 {
// See the matching warning in processQuestionGenerationForKnowledge:
// empty completions point at a completion budget spent on thinking.
logger.Warnf(ctx,
"Question generation (batch): %d model calls returned empty completions "+
"(reasoning-style models can spend the completion budget on thinking even when disabled; "+
"raise %s to give the reply room)",
llmCallEmpty, generateQuestionsMaxTokensEnv)
}
if qSpan != nil {
out := types.JSONMap{
"status": exitStatus,
"batch_index": payload.BatchIndex,
"chunks_in_batch": chunksInBatch,
"chunks_processed": chunksProcessed,
"empty_chunks": emptyChunks,
"llm_failed": llmCallFailed,
"questions_generated": generatedQuestionsTotal,
"index_entries_prepared": indexEntriesPrepared,
"index_batch_succeeded": indexBatchSucceeded,
"retry": retryCount,
"max_retry": maxRetry,
}
if resolvedModelID != "" {
out["model_id"] = resolvedModelID
}
if sampleQuestion != "" {
out["sample_question"] = sampleQuestion
}
if exitStatus != "success" || qErr != nil {
msg := exitStatus
if qErr != nil {
msg = qErr.Error()
}
s.failPostprocessSubspan(ctx, qSpan, "QUESTION_FAILED", msg, qErr)
} else {
s.endPostprocessSubspan(ctx, qSpan, out)
}
}
}()
logger.Infof(ctx, "Processing question generation for knowledge=%s batch=%d chunks=%d",
payload.KnowledgeID, payload.BatchIndex, chunksInBatch)
if chunksInBatch == 0 {
exitStatus = "empty_batch"
return nil
}
// A newer attempt has superseded this one: skip before opening the span
// so we don't read stale chunks and don't drain the new attempt.
if attemptSuperseded(ctx, s.tracker(), payload.KnowledgeID, payload.Attempt) {
superseded = true
exitStatus = "superseded"
logger.Infof(ctx, "question: attempt %d superseded for %s, skipping stale enrichment",
payload.Attempt, payload.KnowledgeID)
return nil
}
qSpan = s.beginQuestionBatchSubspan(ctx, payload.KnowledgeID, payload.Attempt,
fmt.Sprintf("postprocess.question.batch[%d]", payload.BatchIndex),
types.JSONMap{
"batch_index": payload.BatchIndex,
"chunks": chunksInBatch,
"question_count": payload.QuestionCount,
"language": payload.Language,
})
if strings.TrimSpace(s.config.Conversation.GenerateQuestionsPrompt) == "" {
exitStatus = "prompt_not_configured"
logger.Errorf(ctx, "GenerateQuestionsPrompt is empty: configure conversation.generate_questions_prompt_id")
qErr = fmt.Errorf("generate questions prompt not configured")
return qErr
}
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
exitStatus = "kb_not_found"
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
qErr = err
return nil
}
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
exitStatus = "knowledge_not_found"
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
qErr = err
return nil
}
// Short-circuit when the user cancelled parsing or the row is being
// deleted — batched fan-out means we get this check for free on every
// batch, so a cancel stops burning LLM quota on the remaining batches.
if knowledge != nil {
switch knowledge.ParseStatus {
case types.ParseStatusCancelled, types.ParseStatusDeleting:
exitStatus = "knowledge_" + knowledge.ParseStatus
logger.Infof(ctx, "Question generation: knowledge aborted (%s), skipping batch %d",
knowledge.ParseStatus, payload.BatchIndex)
return nil
}
}
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
exitStatus = "get_chat_model_failed"
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return fmt.Errorf("failed to get chat model: %w", err)
}
resolvedModelID = kb.SummaryModelID
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
exitStatus = "get_embedding_model_failed"
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
return fmt.Errorf("failed to get embedding model: %w", err)
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
exitStatus = "get_tenant_failed"
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
return fmt.Errorf("failed to get tenant info: %w", err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil {
exitStatus = "init_retrieve_engine_failed"
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
questionCount := payload.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
processOverrides, _ := knowledge.ProcessOverrides()
questionGenCfg := ResolveProcessConfig(kb, processOverrides).QuestionGenerationConfig
customInstructions := questionGenCfg.CustomInstructions
// Fetch the batch chunks (in payload order) plus the two boundary
// neighbors so we can rebuild the same surrounding context the legacy
// loop used, all enriched with image OCR / caption info. A vanished
// chunk degrades gracefully (skipped / empty context).
getChunk := func(id string) *types.Chunk {
if id == "" {
return nil
}
c, gerr := s.chunkRepo.GetChunkByID(ctx, payload.TenantID, id)
if gerr != nil {
return nil
}
return c
}
batchChunks := make([]*types.Chunk, len(batchIDs))
for i, id := range batchIDs {
batchChunks[i] = getChunk(id)
}
prevChunk := getChunk(payload.PrevChunkID)
nextChunk := getChunk(payload.NextChunkID)
infoIDs := make([]string, 0, len(batchIDs)+2)
infoIDs = append(infoIDs, batchIDs...)
if payload.PrevChunkID != "" {
infoIDs = append(infoIDs, payload.PrevChunkID)
}
if payload.NextChunkID != "" {
infoIDs = append(infoIDs, payload.NextChunkID)
}
imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, s.chunkRepo, payload.TenantID, infoIDs)
enrich := func(c *types.Chunk) string {
if c == nil {
return ""
}
if info, ok := imageInfoMap[c.ID]; ok && info != "" {
return searchutil.EnrichContentWithImageInfo(c.Content, info)
}
return c.Content
}
// neighborContent returns the context content for position i within the
// batch: the in-batch neighbor when present, else the boundary chunk.
prevContentAt := func(i int) string {
if i > 0 {
return enrich(batchChunks[i-1])
}
return enrich(prevChunk)
}
nextContentAt := func(i int) string {
if i < len(batchChunks)-1 {
return enrich(batchChunks[i+1])
}
return enrich(nextChunk)
}
var indexInfoList []*types.IndexInfo
for i, chunk := range batchChunks {
if chunk == nil || strings.TrimSpace(chunk.Content) == "" {
emptyChunks++
continue
}
generationRevision := chunk.ContentRevision
questions, gerr := s.generateQuestionsWithContext(
ctx, chatModel, enrich(chunk), prevContentAt(i), nextContentAt(i), knowledge.Title, questionCount,
customInstructions)
if gerr != nil {
llmCallFailed++
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, gerr)
continue
}
if len(questions) == 0 {
llmCallEmpty++
continue
}
latestChunk, latestErr := s.chunkRepo.GetChunkByID(ctx, payload.TenantID, chunk.ID)
if latestErr != nil || latestChunk.ContentRevision != generationRevision {
logger.Infof(ctx, "Skipping stale generated questions for chunk %s (revision changed)", chunk.ID)
continue
}
chunk = latestChunk
chunksProcessed++
generatedQuestionsTotal += len(questions)
if sampleQuestion == "" {
sampleQuestion = previewText(questions[0], 200)
}
generatedQuestions := make([]types.GeneratedQuestion, len(questions))
questionRevision := chunk.ContentRevision
for j, question := range questions {
generatedQuestions[j] = types.GeneratedQuestion{
ID: fmt.Sprintf("q%d", time.Now().UnixNano()+int64(j)),
Question: question,
ContentRevision: &questionRevision,
}
}
meta := &types.DocumentChunkMetadata{
GeneratedQuestions: generatedQuestions, GeneratedQuestionsRevision: chunk.ContentRevision,
}
if err := chunk.SetDocumentMetadata(meta); err != nil {
logger.Warnf(ctx, "Failed to set document metadata for chunk %s: %v", chunk.ID, err)
continue
}
if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "Failed to update chunk %s: %v", chunk.ID, err)
continue
}
for _, gq := range generatedQuestions {
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: buildKnowledgeIndexContent(knowledge, gq.Question),
SourceID: types.GeneratedQuestionSourceID(chunk.ID, gq.ID),
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
IsEnabled: true,
})
}
}
indexEntriesPrepared = len(indexInfoList)
if len(indexInfoList) > 0 {
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
exitStatus = "index_questions_failed"
qErr = err
logger.Errorf(ctx, "Failed to index generated questions for batch %d: %v", payload.BatchIndex, err)
return fmt.Errorf("failed to index questions: %w", err)
}
indexBatchSucceeded = true
logger.Infof(ctx, "Indexed %d generated questions for knowledge=%s batch=%d",
len(indexInfoList), payload.KnowledgeID, payload.BatchIndex)
}
return nil
}
const (
// generateQuestionsDefaultMaxTokens is the per-chunk completion budget for
// question generation. The reply itself is a handful of short lines, but
// reasoning-style models can spend a small budget on thinking even when
// thinking is disabled and come back with an empty completion (the same
// failure mode the description path grew KNOWLEDGE_BASE_PROFILE_MAX_TOKENS
// for, issue #4221) — so the default keeps headroom.
generateQuestionsDefaultMaxTokens = 2048
// generateQuestionsMaxTokensEnv lets operators raise (or lower) the budget
// without a rebuild; invalid or non-positive values fall back to the
// default so one bad variable cannot break question generation.
generateQuestionsMaxTokensEnv = "GENERATE_QUESTIONS_MAX_TOKENS"
)
func generateQuestionsMaxTokens() int {
if v := strings.TrimSpace(os.Getenv(generateQuestionsMaxTokensEnv)); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return generateQuestionsDefaultMaxTokens
}
// generateQuestionsWithContext generates questions for a chunk with surrounding context
func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
chatModel chat.Chat, content, prevContent, nextContent, docName string, questionCount int,
customInstructions string,
) ([]string, error) {
if content == "" || questionCount <= 0 {
return nil, nil
}
prompt := strings.TrimSpace(s.config.Conversation.GenerateQuestionsPrompt)
if prompt == "" {
return nil, fmt.Errorf("generate questions prompt not configured")
}
// Build context section
var contextSection string
if prevContent != "" || nextContent != "" {
contextSection = "<surrounding_context>\n"
if prevContent != "" {
contextSection += fmt.Sprintf("<preceding_content>\n%s\n\n</preceding_content>\n\n", prevContent)
}
if nextContent != "" {
contextSection += fmt.Sprintf("<following_content>\n%s\n\n</following_content>\n\n", nextContent)
}
contextSection += "</surrounding_context>\n\n"
}
langName := types.LanguageNameFromContext(ctx)
prompt = types.RenderPromptPlaceholders(prompt, types.PlaceholderValues{
"question_count": fmt.Sprintf("%d", questionCount),
"content": content,
"context": contextSection,
"doc_name": docName,
"language": langName,
})
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "question_generation")
thinking := false
modelCtx := types.WithLLMCallMetadata(ctx, "question_generation", "")
response, err := chatModel.Chat(modelCtx, []chat.Message{
{
Role: "user",
Content: prompt,
},
}, &chat.ChatOptions{
Temperature: 0.7,
MaxTokens: generateQuestionsMaxTokens(),
Thinking: &thinking,
})
if err != nil {
return nil, fmt.Errorf("failed to generate questions: %w", err)
}
return parseGeneratedQuestions(response.Content, questionCount), nil
}
// parseGeneratedQuestions drops inline <think> reasoning first: servers that
// ignore Thinking=false still emit it in content, and each reasoning line
// would otherwise be indexed as a question.
func parseGeneratedQuestions(content string, questionCount int) []string {
lines := strings.Split(tools.StripThinkBlocks(content), "\n")
questions := make([]string, 0, questionCount)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
line = strings.TrimLeft(line, "0123456789.-*) ")
line = strings.TrimSpace(line)
if line != "" && len(line) > 5 {
questions = append(questions, line)
if len(questions) >= questionCount {
break
}
}
}
return questions
}
// RegenerateChunkQuestions synchronously refreshes Doc2Query-style auxiliary
// questions for the current chunk revision and atomically replaces their
// retrieval entries through updateChunkVector.
func (s *knowledgeService) RegenerateChunkQuestions(
ctx context.Context, chunkID string,
) ([]types.GeneratedQuestion, error) {
tenantID := types.MustTenantIDFromContext(ctx)
chunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, chunkID)
if err != nil {
return nil, err
}
if chunk.ChunkType != types.ChunkTypeText {
return nil, fmt.Errorf("questions can only be generated for text chunks")
}
generationRevision := chunk.ContentRevision
knowledge, kb, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, chunk.KnowledgeID)
if err != nil {
return nil, err
}
if knowledge.KnowledgeBaseID != chunk.KnowledgeBaseID || chunk.TenantID != knowledge.TenantID {
return nil, werrors.NewForbiddenError("chunk does not belong to its knowledge document")
}
if kb.SummaryModelID == "" {
return nil, fmt.Errorf("summary model is required for question generation")
}
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
return nil, err
}
resolveNeighbor := func(id string) string {
if id == "" {
return ""
}
neighbor, getErr := s.chunkRepo.GetChunkByID(ctx, tenantID, id)
if getErr != nil || !sameChunkDocument(chunk, neighbor) {
return ""
}
return neighbor.Content
}
overrides, _ := knowledge.ProcessOverrides()
config := ResolveProcessConfig(kb, overrides).QuestionGenerationConfig
count := config.QuestionCount
if count <= 0 {
count = 3
}
if count > 10 {
count = 10
}
questions, err := s.generateQuestionsWithContext(
ctx, chatModel, chunk.Content, resolveNeighbor(chunk.PreChunkID),
resolveNeighbor(chunk.NextChunkID), knowledge.Title, count, config.CustomInstructions,
)
if err != nil {
return nil, err
}
latestChunk, err := s.chunkRepo.GetChunkByID(ctx, tenantID, chunkID)
if err != nil {
return nil, err
}
if latestChunk.ContentRevision != generationRevision {
return nil, ErrChunkRevisionConflict
}
chunk = latestChunk
generated := make([]types.GeneratedQuestion, 0, len(questions))
questionRevision := chunk.ContentRevision
for _, question := range questions {
generated = append(generated, types.GeneratedQuestion{
ID: uuid.NewString(), Question: question, ContentRevision: &questionRevision,
})
}
meta := &types.DocumentChunkMetadata{
GeneratedQuestions: generated, GeneratedQuestionsRevision: chunk.ContentRevision,
}
if err := chunk.SetDocumentMetadata(meta); err != nil {
return nil, err
}
if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {
return nil, err
}
if err := s.updateChunkVector(ctx, chunk.KnowledgeBaseID, []*types.Chunk{chunk}); err != nil {
return nil, err
}
return generated, nil
}
// RegenerateKnowledgeSummary refreshes both the knowledge description and the
// summary chunk(s), then reindexes only those summary chunks. It is
// intentionally idempotent.
func (s *knowledgeService) RegenerateKnowledgeSummary(
ctx context.Context, knowledgeID string,
) (*types.Knowledge, error) {
tenantID := types.MustTenantIDFromContext(ctx)
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil {
return nil, err
}
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
if err != nil {
return nil, err
}
if kb.SummaryModelID == "" {
return nil, fmt.Errorf("summary model is not configured")
}
allChunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, tenantID, knowledgeID)
if err != nil {
return nil, err
}
textChunks := make([]*types.Chunk, 0)
for _, chunk := range allChunks {
if chunk.ChunkType == types.ChunkTypeText && chunk.IsEnabled {
textChunks = append(textChunks, chunk)
}
}
if len(textChunks) == 0 {
knowledge.Description = ""
knowledge.Profile = nil
knowledge.SummaryStatus = types.SummaryStatusFailed
knowledge.UpdatedAt = time.Now()
if updateErr := s.saveSummaryState(ctx, knowledge); updateErr != nil {
return knowledge, updateErr
}
_ = requestKnowledgeBaseProfileRefresh(ctx, s.task, kb, false)
return knowledge, errInsufficientSummaryContent
}
sort.Slice(textChunks, func(i, j int) bool {
return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex
})
metadataVersion := string(knowledge.CustomMetadata)
knowledge.SummaryStatus = types.SummaryStatusProcessing
if err := s.saveSummaryState(ctx, knowledge); err != nil {
return nil, err
}
handleGenerationFailure := func(generationErr error) (*types.Knowledge, error) {
if errors.Is(generationErr, errInsufficientSummaryContent) {
knowledge.Description = ""
knowledge.Profile = nil
knowledge.SummaryStatus = types.SummaryStatusFailed
knowledge.UpdatedAt = time.Now()
if updateErr := s.saveSummaryState(ctx, knowledge); updateErr != nil {
return knowledge, updateErr
}
return knowledge, generationErr
}
if summaryTaskWillRetry(ctx) {
applyRetryableSummaryFailureState(knowledge, textChunks, true)
if updateErr := s.saveSummaryState(ctx, knowledge); updateErr != nil {
logger.Warnf(ctx, "Failed to mark summary refresh pending for retry: %v", updateErr)
}
return knowledge, generationErr
}
stale, staleErr := summarySourceChanged(
ctx, s.repo, s.chunkRepo, tenantID, knowledgeID, metadataVersion, textChunks,
)
if staleErr != nil {
knowledge.SummaryStatus = types.SummaryStatusFailed
_ = s.saveSummaryState(ctx, knowledge)
return knowledge, fmt.Errorf("verify summary fallback freshness: %w", staleErr)
}
if stale {
return knowledge, ErrSummaryRefreshStale
}
applyRetryableSummaryFailureState(knowledge, textChunks, false)
if updateErr := s.saveSummaryState(ctx, knowledge); updateErr != nil {
return knowledge, updateErr
}
return knowledge, generationErr
}
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
return handleGenerationFailure(fmt.Errorf("get chat model: %w", err))
}
summaryResult, err := s.getSummary(ctx, chatModel, knowledge, textChunks)
if err != nil {
return handleGenerationFailure(err)
}
stale, err := summarySourceChanged(
ctx, s.repo, s.chunkRepo, tenantID, knowledgeID, metadataVersion, textChunks,
)
if err != nil {
return nil, fmt.Errorf("verify summary freshness: %w", err)
}
if stale {
logger.Infof(ctx, "Discarding stale summary refresh for knowledge %s", knowledgeID)
return nil, ErrSummaryRefreshStale
}
summary := summaryResult.Summary
knowledge.Description = summary
knowledge.Profile = summaryResult.Profile
knowledge.SummaryStatus = types.SummaryStatusCompleted
knowledge.UpdatedAt = time.Now()
if err := s.saveSummaryState(ctx, knowledge); err != nil {
return nil, err
}
summaryChunkContent := buildSummaryChunkContent(summary, summaryResult.Profile)
defer func() { _ = requestKnowledgeBaseProfileRefresh(ctx, s.task, kb, false) }()
if kb.NeedsEmbeddingModel() {
maxIndex := 0
for _, chunk := range allChunks {
if chunk.ChunkIndex > maxIndex {
maxIndex = chunk.ChunkIndex
}
}
// allChunks holds text chunks only, so it can never carry the existing
// summary chunk. Scanning it for one always came up empty, which left
// every refresh appending a new summary chunk beside the stale one --
// and a stale summary stays enabled and indexed, so content the user
// edited out of the document kept being retrievable through it.
existingSummaries, err := s.chunkRepo.ListChunksByKnowledgeIDAndTypes(
ctx, tenantID, knowledgeID, []types.ChunkType{types.ChunkTypeSummary},
)
if err != nil {
return nil, err
}
summaryChunks := make([]*types.Chunk, 0, len(existingSummaries))
for _, chunk := range existingSummaries {
chunk.Content = summaryChunkContent
chunk.SourceContent = chunk.Content
chunk.IsEnabled = true
chunk.UpdatedAt = time.Now()
if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {
return nil, err
}
summaryChunks = append(summaryChunks, chunk)
}
if len(summaryChunks) == 0 {
summaryChunk := &types.Chunk{
ID: uuid.NewString(), TenantID: tenantID, KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: summaryChunkContent,
ChunkIndex: maxIndex + 1, IsEnabled: true, ChunkType: types.ChunkTypeSummary,
ParentChunkID: textChunks[0].ID, CreatedAt: time.Now(), UpdatedAt: time.Now(),
}
if err := s.chunkRepo.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil {
return nil, err
}
summaryChunks = append(summaryChunks, summaryChunk)
}
if err := s.updateChunkVector(ctx, knowledge.KnowledgeBaseID, summaryChunks); err != nil {
return nil, err
}
}
return knowledge, nil
}
// ReparseKnowledge deletes existing document content and re-parses the knowledge asynchronously.
// This method reuses the logic from UpdateManualKnowledge for resource cleanup and async parsing.
func (s *knowledgeService) ReparseKnowledge(
ctx context.Context,
knowledgeID string,
processOverrides *types.KnowledgeProcessOverrides,
) (*types.Knowledge, error) {
return s.reparseKnowledge(ctx, knowledgeID, processOverrides, false)
}
// reparseKnowledge implements ReparseKnowledge. tasksDequeued tells it the
// caller already dropped the knowledge's queued tasks (ReplaceKnowledgeFile
// does, before pointing the row at the new file), which saves a second scan
// of every queue.
func (s *knowledgeService) reparseKnowledge(
ctx context.Context,
knowledgeID string,
processOverrides *types.KnowledgeProcessOverrides,
tasksDequeued bool,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start re-parsing knowledge")
existing, kb, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to load knowledge: %v", err)
return nil, err
}
tenantID := existing.TenantID
// When the caller supplies new overrides (e.g. via the reparse confirm
// dialog), validate them against this knowledge's file type, then persist
// to metadata so both this call's enqueue and the worker re-read the same
// config. nil keeps whatever was stored at upload time.
if processOverrides != nil {
if err := ValidateProcessOverrides(ctx, kb, processOverrides, reparseFileTypes(existing)); err != nil {
return nil, err
}
if err := existing.SetProcessOverrides(processOverrides); err != nil {
logger.Errorf(ctx, "Failed to set process overrides on reparse: %v", err)
return nil, err
}
if err := s.repo.UpdateKnowledgeColumn(ctx, existing.ID, "metadata", existing.Metadata); err != nil {
logger.Errorf(ctx, "Failed to persist process overrides on reparse: %v", err)
return nil, err
}
}
processOverrides, _ = existing.ProcessOverrides()
reparseEff := ResolveProcessConfig(kb, processOverrides)
var manualContent string
if existing.IsManual() {
meta, metaErr := existing.ManualMetadata()
if metaErr != nil || meta == nil {
logger.Errorf(ctx, "Failed to get manual metadata for reparse: %v", metaErr)
return nil, werrors.NewBadRequestError("无法获取手工知识内容")
}
manualContent = meta.Content
} else {
// Cleanup synchronously; manual knowledge is cleaned up by its worker.
logger.Infof(ctx, "Cleaning up existing resources for knowledge: %s", knowledgeID)
if err := s.cleanupKnowledgeResources(ctx, existing); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": knowledgeID,
})
return nil, err
}
}
// Everything that can reject the reparse has run. From here on the
// previous run is superseded, so stop it before announcing the new one.
//
// A reparse may land while the previous run is still in flight (batch
// reparse and the API do not gate on status). Its queued tasks would
// otherwise run against the new attempt: an old ProcessDocument wipes the
// new chunks and writes "processing" back, an old post-process seeds the
// counter for subtasks that all drop themselves as superseded.
if !tasksDequeued && isInFlightParseStatus(existing.ParseStatus) {
s.dequeueKnowledgeTasks(ctx, existing.ID)
}
// Allocate the new span tree attempt only now. Opening it earlier and
// then rejecting the reparse left the previous run's subtasks seeing a
// newer attempt: they skipped their drain as superseded and stranded the
// row in "finalizing". Opening it before the enqueue still means the UI
// shows the fresh attempt immediately and the worker does not allocate a
// second one.
reparseAttempt := 0
if root, n, err := s.tracker().OpenAttempt(ctx, existing.ID, ""); err == nil && root != nil {
reparseAttempt = n
} else if err != nil {
logger.Warnf(ctx, "[Reparse] OpenAttempt failed for %s: %v (will fall back in worker)", existing.ID, err)
}
// Keep wiki's pending queue consistent across both manual and non-manual
// paths. The destructive work (swapping old wiki contributions for new)
// happens asynchronously inside mapOneDocument — see its oldPageSlugs
// handling — once post-process re-enqueues wiki ingest. All we need to
// do here is stop any stale pending ingest op from firing against the
// pre-reparse chunk set. The op's finalizing slot is not released: the
// reset below zeroes the counter. Scrub even when wiki is off now: an op
// left by an earlier wiki-enabled run would otherwise drain a slot of
// the new attempt.
s.prepareWikiForReparse(ctx, existing)
recordReparseStarted := func() {
recordKBActivity(ctx, s.audit, tenantID, existing.KnowledgeBaseID, types.AuditActionKnowledgeReparseStarted,
"knowledge", existing.ID, types.AuditOutcomeAccepted,
map[string]any{"title": existing.Title, "type": existing.Type, "attempt": reparseAttempt})
}
// For manual knowledge, use async manual processing (cleanup + re-indexing in worker)
if existing.IsManual() {
resetKnowledgeForReparse(existing, kb)
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
logger.Errorf(ctx, "Failed to update knowledge status before reparse: %v", err)
return nil, err
}
if err := s.repo.UpdateKnowledgeColumn(ctx, existing.ID, "pending_subtasks_count", 0); err != nil {
logger.Errorf(ctx, "Failed to reset pending_subtasks_count before reparse: %v", err)
return nil, err
}
if _, err := s.enqueueManualProcessing(ctx, existing, manualContent, true); err != nil {
logger.Errorf(ctx, "Failed to enqueue manual reparse task: %v", err)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
} else {
recordReparseStarted()
}
return existing, nil
}
// Step 2: Update knowledge status and metadata
resetKnowledgeForReparse(existing, kb)
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
logger.Errorf(ctx, "Failed to update knowledge status before reparse: %v", err)
return nil, err
}
if err := s.repo.UpdateKnowledgeColumn(ctx, existing.ID, "pending_subtasks_count", 0); err != nil {
logger.Errorf(ctx, "Failed to reset pending_subtasks_count before reparse: %v", err)
return nil, err
}
// Step 3: Trigger async re-parsing based on knowledge type
logger.Infof(ctx, "Knowledge status updated, scheduling async reparse, ID: %s, Type: %s", existing.ID, existing.Type)
// For file-based knowledge, enqueue document processing task
if existing.FilePath != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enableMultimodel := reparseEff.EnableMultimodel
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang := types.LanguageFromContextOrDefault(ctx)
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: existing.ID,
KnowledgeBaseID: existing.KnowledgeBaseID,
FilePath: existing.FilePath,
FileName: existing.FileName,
FileType: getFileType(existing.FileName),
EnableMultimodel: enableMultimodel,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
Language: lang,
Attempt: reparseAttempt,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal reparse task payload: %v", err)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue reparse task: %v", err)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
logger.Infof(ctx, "Enqueued reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
recordReparseStarted()
// For data tables (csv, xlsx, xls), also enqueue summary task
enqueueDataTableSummaryIfNeeded(ctx, s.task, tenantID, existing.ID, existing.FileName, existing.FileType, kb.SummaryModelID, kb.EmbeddingModelID)
return existing, nil
}
// For file-URL-based knowledge, enqueue document processing task with FileURL field
if existing.Type == "file_url" && existing.Source != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enableMultimodel := reparseEff.EnableMultimodel
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang := types.LanguageFromContextOrDefault(ctx)
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: existing.ID,
KnowledgeBaseID: existing.KnowledgeBaseID,
FileURL: existing.Source,
FileName: existing.FileName,
FileType: existing.FileType,
EnableMultimodel: enableMultimodel,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
Language: lang,
Attempt: reparseAttempt,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal file URL reparse task payload: %v", err)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config)...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue file URL reparse task: %v", err)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
logger.Infof(ctx, "Enqueued file URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
recordReparseStarted()
enqueueDataTableSummaryIfNeeded(ctx, s.task, tenantID, existing.ID, existing.FileName, existing.FileType, kb.SummaryModelID, kb.EmbeddingModelID)
return existing, nil
}
// For URL-based knowledge, enqueue URL processing task
if existing.Type == "url" && existing.Source != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enableMultimodel := reparseEff.EnableMultimodel
enableQuestionGeneration := reparseEff.QuestionGenerationConfig.Enabled
questionCount := reparseEff.QuestionGenerationConfig.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
lang := types.LanguageFromContextOrDefault(ctx)
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: existing.ID,
KnowledgeBaseID: existing.KnowledgeBaseID,
URL: existing.Source,
EnableMultimodel: enableMultimodel,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
Language: lang,
Attempt: reparseAttempt,
}
langfuse.InjectTracing(ctx, &taskPayload)
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal URL reparse task payload: %v", err)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
task := asynq.NewTask(
types.TypeDocumentProcess,
payloadBytes,
documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...,
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue URL reparse task: %v", err)
s.markKnowledgeEnqueueFailed(ctx, existing)
return existing, werrors.NewInternalServerError("Failed to submit processing task")
}
logger.Infof(ctx, "Enqueued URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
recordReparseStarted()
return existing, nil
}
logger.Warnf(ctx, "Knowledge %s has no parseable content (no file, URL, or manual content)", knowledgeID)
existing.ParseStatus = types.ParseStatusFailed
existing.ErrorMessage = "Knowledge has no parseable content"
if err := s.repo.UpdateKnowledge(ctx, existing); err != nil {
logger.Errorf(ctx, "Failed to persist unparseable knowledge state for %s: %v",
secutils.SanitizeForLog(knowledgeID), err)
}
return existing, werrors.NewBadRequestError("Knowledge has no parseable content")
}
// resetKnowledgeForReparse makes the top-level knowledge state describe the
// new processing attempt rather than retaining terminal state from the
// previous one.
func resetKnowledgeForReparse(knowledge *types.Knowledge, kb *types.KnowledgeBase) {
knowledge.ParseStatus = types.ParseStatusPending
knowledge.EnableStatus = "disabled"
knowledge.Description = ""
knowledge.ProcessedAt = nil
knowledge.ErrorMessage = ""
knowledge.EmbeddingModelID = kb.EmbeddingModelID
// UpdateKnowledge deliberately omits pending_subtasks_count, so callers
// must still persist this reset through an explicit column update.
knowledge.PendingSubtasksCount = 0
}
// CancelKnowledgeParse marks an in-progress parse as cancelled by the user.
//
// Semantics (kept aligned with the existing deleting path, but partial work
// is preserved instead of cleaned up):
// - parse_status is set to "cancelled"; partial chunks/index already written
// to the database remain on disk. The user can re-trigger parsing via the
// existing ReparseKnowledge API, which overwrites status back to pending.
// - Any in-flight worker reads the new status at its next checkpoint and
// bails (see processChunks / ProcessDocument / downstream handlers).
// - The asynq inspector (if available) dequeues pending / scheduled / retry
// tasks for this knowledge_id across all queues used by the pipeline
// and signals active workers to stop. Lite mode (no Redis) skips the
// dequeue step — the checkpoint-based abort is the only stop signal there.
// - Idempotent: re-calling on an already-cancelled row is a no-op.
//
// Errors:
// - ParseStatusCompleted / ParseStatusFailed: the parse has already finished.
// - ParseStatusDeleting: a delete is in progress; cancel cannot supersede it.
func (s *knowledgeService) CancelKnowledgeParse(
ctx context.Context, knowledgeID string,
) (*types.Knowledge, error) {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil {
logger.Errorf(ctx, "CancelKnowledgeParse: failed to load knowledge: %v", err)
return nil, err
}
if existing == nil {
return nil, werrors.NewNotFoundError("knowledge not found")
}
switch existing.ParseStatus {
case types.ParseStatusCancelled:
// Idempotent — still attempt the dequeue in case earlier calls
// raced an enqueue, but skip the row update / span close path.
s.dequeueKnowledgeTasks(ctx, knowledgeID)
return existing, nil
case types.ParseStatusCompleted, types.ParseStatusFailed:
return nil, werrors.NewBadRequestError("解析已结束,无法取消")
case types.ParseStatusDeleting:
return nil, werrors.NewBadRequestError("知识正在删除中,无法取消解析")
case types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing:
// Cancellable. `finalizing` is the post-process fan-out window
// where graph-extract / summary / question subtasks are still
// running; cancel here stops the LLM cost they would burn.
default:
// Unknown status — let it through but log. Should never happen
// outside test fixtures or hand-edited rows.
logger.Warnf(ctx, "CancelKnowledgeParse: unexpected status %q for %s, proceeding",
existing.ParseStatus, knowledgeID)
}
// Flip the row to cancelled and zero the enrichment counter in one
// update so a late subtask FinalizeSubtask call can't race-promote
// the row back to completed. Persisted partial data is left in
// place — the user can reuse it on the next reparse attempt.
now := time.Now()
if err := s.repo.UpdateKnowledgeColumns(ctx, existing.ID, map[string]interface{}{
"parse_status": types.ParseStatusCancelled,
"error_message": "用户已取消解析",
"pending_subtasks_count": 0,
// The summary task is dequeued (or will bail) with the rest.
"summary_status": summaryStatusClosedExpr(types.SummaryStatusNone),
"updated_at": now,
}); err != nil {
logger.Errorf(ctx, "CancelKnowledgeParse: failed to mark knowledge cancelled: %v", err)
return nil, err
}
existing.ParseStatus = types.ParseStatusCancelled
existing.ErrorMessage = "用户已取消解析"
existing.PendingSubtasksCount = 0
if existing.SummaryStatus == types.SummaryStatusPending || existing.SummaryStatus == types.SummaryStatusProcessing {
existing.SummaryStatus = types.SummaryStatusNone
}
existing.UpdatedAt = now
logger.Infof(ctx, "Knowledge %s marked as cancelled by user", knowledgeID)
// Close the active attempt span tree so the UI stops showing "进行中"
// for the cancelled run. AbortAttempt cascade-cancels every still-
// running descendant (multimodal per-image, postprocess subtasks,
// graph chunks) BEFORE closing the root, otherwise the trace
// viewer would leave those striped/running bars hanging forever
// because workers exit via their abort-guard without ever calling
// FailSpan on their own subspan. Best-effort: nil tracker / missing
// attempt no-ops.
if attempt := s.tracker().LatestAttempt(ctx, knowledgeID); attempt > 0 {
s.tracker().AbortAttempt(ctx, knowledgeID, attempt,
"USER_CANCELLED", "用户已取消解析", "用户已取消解析")
}
// Best-effort dequeue. Failures here don't block the cancel — the
// downstream tasks will still self-abort at their entry guards.
s.dequeueKnowledgeTasks(ctx, knowledgeID)
// Wiki ingest lives in its own per-KB pending queue (task_pending_ops)
// rather than asynq, so dequeueKnowledgeTasks above can't see it.
// Mirror the deletion path's scrub so a cancelled knowledge doesn't
// get picked up by the next 30s batch and burn a wiki LLM call on a
// doc the user already abandoned. The in-flight worker would skip it
// at isWikiKnowledgeAborted anyway, but scrubbing avoids waking the
// batch in the first place.
s.scrubWikiPendingIngest(ctx, existing.KnowledgeBaseID, knowledgeID, "cancel")
recordKBActivity(ctx, s.audit, tenantID, existing.KnowledgeBaseID, types.AuditActionKnowledgeParseCanceled,
"knowledge", existing.ID, types.AuditOutcomeCanceled,
map[string]any{"title": existing.Title, "type": existing.Type})
return existing, nil
}
// dequeueKnowledgeTasks asks the task inspector to remove any queued
// tasks for this knowledge and signal active workers to stop. Safe to
// call when the inspector is a no-op (Lite mode).
func (s *knowledgeService) dequeueKnowledgeTasks(ctx context.Context, knowledgeID string) {
if s.taskInspector == nil {
return
}
if _, _, err := s.taskInspector.CancelTasksForKnowledge(ctx, knowledgeID); err != nil {
logger.Warnf(ctx, "CancelKnowledgeParse: dequeue best-effort failed for %s: %v", knowledgeID, err)
}
}
func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, chunks []*types.Chunk) error {
// Get embedding model from knowledge base
sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
if err != nil {
return err
}
if !sourceKB.NeedsEmbeddingModel() {
return nil
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, sourceKB.EmbeddingModelID)
if err != nil {
return err
}
// Initialize composite retrieve engine from tenant configuration
indexInfo := make([]*types.IndexInfo, 0, len(chunks))
ids := make([]string, 0, len(chunks))
knowledgeCache := make(map[string]*types.Knowledge)
for _, chunk := range chunks {
if chunk.KnowledgeBaseID != kbID {
logger.Warnf(ctx, "Knowledge base ID mismatch: %s != %s", chunk.KnowledgeBaseID, kbID)
continue
}
// Its index row holds the image's own vector, which re-embedding
// Content here would replace with a text vector of the caption.
if chunk.ChunkType == types.ChunkTypeImageVector {
continue
}
ids = append(ids, chunk.ID)
if !chunk.IsEnabled || chunk.ChunkType == types.ChunkTypeParentText {
continue
}
knowledge := knowledgeCache[chunk.KnowledgeID]
if knowledge == nil {
knowledge, err = s.repo.GetKnowledgeByID(ctx, chunk.TenantID, chunk.KnowledgeID)
if err != nil {
return err
}
knowledgeCache[chunk.KnowledgeID] = knowledge
}
indexInfo = append(indexInfo, &types.IndexInfo{
Content: buildKnowledgeIndexContent(knowledge, chunk.EmbeddingContent()),
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
KnowledgeType: sourceKB.Type,
IsEnabled: chunk.IsEnabled,
})
meta, metaErr := chunk.DocumentMetadata()
if metaErr != nil {
return metaErr
}
if meta != nil {
for _, q := range meta.GeneratedQuestions {
if strings.TrimSpace(q.Question) != "" {
indexInfo = append(indexInfo, &types.IndexInfo{
Content: buildKnowledgeIndexContent(knowledge, q.Question), SourceID: types.GeneratedQuestionSourceID(chunk.ID, q.ID),
SourceType: types.ChunkSourceType, ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID,
KnowledgeType: sourceKB.Type, IsEnabled: true,
})
}
}
}
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceKB.VectorStoreID)
if err != nil {
return err
}
// Delete old vector representation of the chunk
err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions(), sourceKB.Type)
if err != nil {
return err
}
// Index updated chunk content with new vector representation
err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo)
if err != nil {
return err
}
return nil
}
func (s *knowledgeService) UpdateImageInfo(
ctx context.Context,
knowledgeID string,
chunkID string,
imageInfo string,
) error {
knowledge, _, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledgeID)
if err != nil {
return err
}
imageInfo = common.CleanInvalidUTF8(imageInfo)
var images []*types.ImageInfo
if err := json.Unmarshal([]byte(imageInfo), &images); err != nil {
logger.Errorf(ctx, "Failed to unmarshal image info: %v", err)
return err
}
if len(images) != 1 {
logger.Warnf(ctx, "Expected exactly one image info, got %d", len(images))
return nil
}
image := images[0]
// Retrieve all chunks with the given parent chunk ID
chunk, err := s.chunkService.GetChunkByID(ctx, chunkID)
if err != nil {
logger.Errorf(ctx, "Failed to get chunk: %v", err)
return err
}
if chunk == nil || chunk.ID != chunkID || chunk.KnowledgeID != knowledge.ID ||
chunk.TenantID != knowledge.TenantID ||
chunk.KnowledgeBaseID != knowledge.KnowledgeBaseID {
return werrors.NewForbiddenError("chunk does not belong to its knowledge document")
}
chunk.ImageInfo = imageInfo
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
chunkChildren, err := s.chunkService.ListChunkByParentID(ctx, tenantID, chunkID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"parent_chunk_id": chunkID,
"tenant_id": tenantID,
})
return err
}
for _, child := range chunkChildren {
if !sameChunkDocument(chunk, child) {
return werrors.NewForbiddenError("image child does not belong to its document")
}
}
logger.Infof(ctx, "Found %d chunks with parent chunk ID: %s", len(chunkChildren), chunkID)
// Iterate through each chunk and update its content based on the image information
updateChunk := []*types.Chunk{chunk}
var addChunk []*types.Chunk
// Track whether we've found OCR and caption child chunks for this image
hasOCRChunk := false
hasCaptionChunk := false
for i, child := range chunkChildren {
// Skip chunks that are not image types
var cImageInfo []*types.ImageInfo
err = json.Unmarshal([]byte(child.ImageInfo), &cImageInfo)
if err != nil {
logger.Warnf(ctx, "Failed to unmarshal image %s info: %v", child.ID, err)
continue
}
if len(cImageInfo) == 0 {
continue
}
if cImageInfo[0].OriginalURL != image.OriginalURL {
logger.Warnf(ctx, "Skipping chunk ID: %s, image URL mismatch: %s != %s",
child.ID, cImageInfo[0].OriginalURL, image.OriginalURL)
continue
}
// Mark that we've found chunks for this image
switch child.ChunkType {
case types.ChunkTypeImageCaption:
hasCaptionChunk = true
// Update caption if it has changed
if image.Caption != cImageInfo[0].Caption {
child.Content = image.Caption
child.ImageInfo = imageInfo
updateChunk = append(updateChunk, chunkChildren[i])
}
case types.ChunkTypeImageOCR:
hasOCRChunk = true
// Update OCR if it has changed
if image.OCRText != cImageInfo[0].OCRText {
child.Content = image.OCRText
child.ImageInfo = imageInfo
updateChunk = append(updateChunk, chunkChildren[i])
}
case types.ChunkTypeImageVector:
// The image's vector does not change with its caption, only the
// text shown and reranked for it; updateChunkVector leaves the
// vector alone.
if image.Caption != "" && image.Caption != child.Content {
child.Content = image.Caption
child.ImageInfo = imageInfo
updateChunk = append(updateChunk, chunkChildren[i])
}
}
}
// Create a new caption chunk if it doesn't exist and we have caption data
if !hasCaptionChunk && image.Caption != "" {
captionChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: tenantID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
Content: image.Caption,
ChunkType: types.ChunkTypeImageCaption,
ParentChunkID: chunk.ID,
ImageInfo: imageInfo,
// CreateChunks inserts with Select("*"), so the gorm default:true never
// applies -- an unset IsEnabled lands in the database as false and the
// chunk is silently excluded from retrieval and model context.
IsEnabled: true,
}
addChunk = append(addChunk, captionChunk)
logger.Infof(ctx, "Created new caption chunk ID: %s for image URL: %s", captionChunk.ID, image.OriginalURL)
}
// Create a new OCR chunk if it doesn't exist and we have OCR data
if !hasOCRChunk && image.OCRText != "" {
ocrChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: tenantID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
Content: image.OCRText,
ChunkType: types.ChunkTypeImageOCR,
ParentChunkID: chunk.ID,
ImageInfo: imageInfo,
IsEnabled: true,
}
addChunk = append(addChunk, ocrChunk)
logger.Infof(ctx, "Created new OCR chunk ID: %s for image URL: %s", ocrChunk.ID, image.OriginalURL)
}
logger.Infof(ctx, "Updated %d chunks out of %d total chunks", len(updateChunk), len(chunkChildren)+1)
if len(addChunk) > 0 {
err := s.chunkRepo.CreateChunks(ctx, addChunk)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"add_chunk_size": len(addChunk),
})
return err
}
}
// Update the chunks
for _, c := range updateChunk {
err := s.chunkRepo.UpdateChunk(ctx, c)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"chunk_id": c.ID,
"knowledge_id": c.KnowledgeID,
})
return err
}
}
// Update the chunk vector
err = s.updateChunkVector(ctx, chunk.KnowledgeBaseID, append(updateChunk, addChunk...))
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"chunk_id": chunk.ID,
"knowledge_id": chunk.KnowledgeID,
})
return err
}
// Update the knowledge file hash
knowledge, err = s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
return err
}
fileHash := calculateStr(knowledgeID, knowledge.FileHash, imageInfo)
knowledge.FileHash = fileHash
err = s.repo.UpdateKnowledge(ctx, knowledge)
if err != nil {
logger.Warnf(ctx, "Failed to update knowledge file hash: %v", err)
}
logger.Infof(ctx, "Updated chunk successfully, chunk ID: %s, knowledge ID: %s", chunk.ID, chunk.KnowledgeID)
return nil
}
// ProcessManualUpdate handles Asynq manual knowledge update tasks.
// It performs cleanup of old indexes/chunks (when NeedCleanup is true) and re-indexes the content.
func (s *knowledgeService) ProcessManualUpdate(ctx context.Context, t *asynq.Task) error {
var payload types.ManualProcessPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "failed to unmarshal manual process task payload: %v", err)
return nil
}
ctx = logger.WithRequestID(ctx, payload.RequestId)
ctx = logger.WithField(ctx, "manual_process", payload.KnowledgeID)
ctx = types.WithExecutionTenant(ctx, payload.TenantID)
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
// Retry: acking leaves the row pending with no task behind it.
logger.Errorf(ctx, "ProcessManualUpdate: failed to get tenant: %v", err)
return fmt.Errorf("get tenant %d: %w", payload.TenantID, err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if errors.Is(err, repository.ErrKnowledgeNotFound) {
logger.Warnf(ctx, "ProcessManualUpdate: knowledge not found: %s", payload.KnowledgeID)
return nil
}
if err != nil {
logger.Errorf(ctx, "ProcessManualUpdate: failed to get knowledge: %v", err)
return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err)
}
if knowledge == nil {
logger.Warnf(ctx, "ProcessManualUpdate: knowledge not found: %s", payload.KnowledgeID)
return nil
}
if err := validateProcessingKnowledge(knowledge,
payload.TenantID,
payload.KnowledgeBaseID,
payload.KnowledgeID); err != nil {
return err
}
// Skip if already completed or being deleted
if knowledge.ParseStatus == types.ParseStatusCompleted {
logger.Infof(ctx, "ProcessManualUpdate: already completed, skipping: %s", payload.KnowledgeID)
return nil
}
if knowledge.ParseStatus == types.ParseStatusDeleting {
logger.Infof(ctx, "ProcessManualUpdate: being deleted, skipping: %s", payload.KnowledgeID)
return nil
}
if knowledge.ParseStatus == types.ParseStatusCancelled {
logger.Infof(ctx, "ProcessManualUpdate: cancelled by user, skipping: %s", payload.KnowledgeID)
return nil
}
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "ProcessManualUpdate: failed to get knowledge base: %v", err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("failed to get knowledge base: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
if kb == nil || kb.ID != payload.KnowledgeBaseID || kb.TenantID != payload.TenantID {
return fmt.Errorf("processing task KB owner changed: %w", asynq.SkipRetry)
}
ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID)
if err != nil {
return fmt.Errorf("invalid processing scope: %v: %w", err, asynq.SkipRetry)
}
// Re-check abort status right before marking processing — see the same
// note in ProcessDocument for the cancel race this guards.
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx, "ProcessManualUpdate: knowledge aborted (%s), skipping: %s", status, knowledge.ID)
return abortRetryErr(ctx, knowledge.ID, status)
}
// Update status to processing
markKnowledgeProcessing(knowledge, time.Now())
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Errorf(ctx, "ProcessManualUpdate: failed to update status to processing: %v", err)
return fmt.Errorf("mark knowledge %s processing: %w", knowledge.ID, err)
}
// Allocate a fresh span-tracking attempt for this manual (re)index.
// Without it attemptFromCtx stays 0, so processChunks drops all stage
// spans and KnowledgePostProcess falls back to LatestAttempt — piling
// this run's summary/wiki subspans onto the previous attempt's trace.
attempt := 0
if root, n, err := s.tracker().OpenAttempt(ctx, knowledge.ID, payload.LangfuseTraceID); err == nil && root != nil {
attempt = n
} else if err != nil {
logger.Warnf(ctx, "ProcessManualUpdate: OpenAttempt failed for %s: %v", knowledge.ID, err)
}
ctx = withAttempt(ctx, attempt)
// Cleanup old resources (indexes, chunks, graph) for update operations
if payload.NeedCleanup {
if err := s.cleanupKnowledgeResources(ctx, knowledge); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": payload.KnowledgeID,
})
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("failed to cleanup old resources: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
}
// Run manual processing (image resolution + chunking + embedding) synchronously within the worker
return s.triggerManualProcessing(ctx, kb, knowledge, payload.Content, true)
}
// ProcessDocument handles Asynq document processing tasks
func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) error {
var payload types.DocumentProcessPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "failed to unmarshal document process task payload: %v", err)
return nil
}
ctx = logger.WithRequestID(ctx, payload.RequestId)
ctx = logger.WithField(ctx, "document_process", payload.KnowledgeID)
ctx = types.WithExecutionTenant(ctx, payload.TenantID)
if payload.Language != "" {
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// 获取任务重试信息,用于判断是否是最后一次重试。Lite 模式的执行器不提供
// asynq 的重试计数,改用它注入的 TaskRetryMetadata;否则每次都算最后一次。
retryCount, _ := asynq.GetRetryCount(ctx)
maxRetry, _ := asynq.GetMaxRetry(ctx)
if retried, limit, ok := types.TaskRetryMetadataFromContext(ctx); ok {
retryCount, maxRetry = retried, limit
}
isLastRetry := retryCount >= maxRetry
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
// Retry: acking leaves the row pending with no task behind it.
logger.Errorf(ctx, "failed to get tenant: %v", err)
return fmt.Errorf("get tenant %d: %w", payload.TenantID, err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
logger.Infof(ctx, "Processing document task: knowledge_id=%s, file_path=%s, retry=%d/%d",
payload.KnowledgeID, payload.FilePath, retryCount, maxRetry)
// 幂等性检查:获取knowledge记录
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if errors.Is(err, repository.ErrKnowledgeNotFound) {
return nil
}
if err != nil {
logger.Errorf(ctx, "failed to get knowledge: %v", err)
return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err)
}
if knowledge == nil {
return nil
}
if err := validateProcessingKnowledge(knowledge,
payload.TenantID,
payload.KnowledgeBaseID,
payload.KnowledgeID); err != nil {
return err
}
// A reparse opened a newer attempt after this task was queued (or while
// an earlier delivery of it was running). Its row, chunks and counter
// belong to the new run; this task must not touch them.
if attemptSuperseded(ctx, s.tracker(), payload.KnowledgeID, payload.Attempt) {
logger.Infof(ctx, "Parse attempt %d superseded, skipping stale process task: %s",
payload.Attempt, payload.KnowledgeID)
return nil
}
// 检查是否正在删除 / 已被用户取消 - 如果是则直接退出
if knowledge.ParseStatus == types.ParseStatusDeleting {
logger.Infof(ctx, "Knowledge is being deleted, aborting processing: %s", payload.KnowledgeID)
return nil
}
if knowledge.ParseStatus == types.ParseStatusCancelled {
logger.Infof(ctx, "Knowledge cancelled by user, aborting processing: %s", payload.KnowledgeID)
return nil
}
// 检查任务状态 - 幂等性处理
if knowledge.ParseStatus == types.ParseStatusCompleted {
logger.Infof(ctx, "Document already completed, skipping: %s", payload.KnowledgeID)
return nil // 幂等:已完成的任务直接返回
}
if knowledge.ParseStatus == types.ParseStatusFailed {
// 检查是否可恢复(例如:超时、临时错误等)
// 对于不可恢复的错误,直接返回
logger.Warnf(
ctx,
"Document processing previously failed: %s, error: %s",
payload.KnowledgeID,
knowledge.ErrorMessage,
)
// 这里可以根据错误类型判断是否可恢复,暂时允许重试
}
// 检查是否有部分处理(有chunks但状态不是completed)
if knowledge.ParseStatus != "completed" && knowledge.ParseStatus != "pending" &&
knowledge.ParseStatus != "processing" {
// 状态异常,记录日志但继续处理
logger.Warnf(ctx, "Unexpected parse status: %s for knowledge: %s", knowledge.ParseStatus, payload.KnowledgeID)
}
// 获取知识库信息
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("failed to get knowledge base: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
if kb == nil || kb.ID != payload.KnowledgeBaseID || kb.TenantID != payload.TenantID {
return fmt.Errorf("processing task KB owner changed: %w", asynq.SkipRetry)
}
ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID)
if err != nil {
return fmt.Errorf("invalid processing scope: %v: %w", err, asynq.SkipRetry)
}
processOverrides, _ := knowledge.ProcessOverrides()
eff := ResolveProcessConfig(kb, processOverrides)
// Re-check abort status right before flipping to "processing" — closes
// the race where the user cancels between the entry guard above and
// this write (otherwise the worker would overwrite cancelled→processing
// and downstream checkpoints would treat the run as live).
if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted {
logger.Infof(ctx, "Knowledge aborted (%s) before marking processing: %s", status, knowledge.ID)
return abortRetryErr(ctx, knowledge.ID, status)
}
if payload.FilePath != "" && knowledge.FilePath != "" && payload.FilePath != knowledge.FilePath {
logger.Infof(ctx, "Document source replaced, skipping stale process task: %s", payload.KnowledgeID)
return nil
}
if s.isKnowledgeSourceReplaced(ctx, knowledge) {
logger.Infof(ctx, "Document source replaced, skipping stale process task: %s", payload.KnowledgeID)
return nil
}
markKnowledgeProcessing(knowledge, time.Now())
if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
logger.Errorf(ctx, "failed to update knowledge status to processing: %v", err)
return fmt.Errorf("mark knowledge %s processing: %w", knowledge.ID, err)
}
if s.isKnowledgeSourceReplaced(ctx, knowledge) {
logger.Infof(ctx, "Document source replaced, aborting after status update: %s", payload.KnowledgeID)
return nil
}
// Resolve the attempt for span tracking. The enqueue site sets
// payload.Attempt to a fresh number for the initial parse and to
// max+1 for each user-initiated reparse. Asynq retries within a
// single user action keep the same payload (so retries record
// onto the same attempt). For payloads predating this code we
// fall back to OpenAttempt.
attempt := payload.Attempt
if attempt <= 0 {
if root, n, err := s.tracker().OpenAttempt(ctx, knowledge.ID, payload.LangfuseTraceID); err == nil && root != nil {
attempt = n
}
}
ctx = withAttempt(ctx, attempt)
// 检查多模态配置(仅对文件导入)
if payload.FilePath != "" && !payload.EnableMultimodel && IsImageType(payload.FileType) {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = ErrImageNotParse.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// 检查音频ASR配置(仅对文件导入)
if payload.FilePath != "" && IsAudioType(payload.FileType) && !eff.ASRConfig.IsASREnabled() {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Errorf("processDocument audio without ASR model configured")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "上传音频文件需要设置ASR语音识别模型"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// 视频文件不再支持入库解析
if payload.FilePath != "" && IsVideoType(payload.FileType) {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
Errorf("processDocument video not supported")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "暂不支持视频文件"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// New pipeline: convert -> store images -> chunk -> vectorize -> multimodal tasks
var convertResult *types.ReadResult
var chunks []types.ParsedChunk
if payload.FileURL != "" {
// file_url import: SSRF re-check (防 DNS 重绑定), download, persist, then delegate to convert()
if err := secutils.ValidateURLForSSRF(payload.FileURL); err != nil {
logger.Errorf(ctx, "File URL rejected for SSRF protection in ProcessDocument: %s, err: %v", payload.FileURL, err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "File URL is not allowed for security reasons"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
resolvedFileName := payload.FileName
resolvedFileType := payload.FileType
contentBytes, err := downloadFileFromURL(ctx, payload.FileURL, &resolvedFileName, &resolvedFileType)
if err != nil {
logger.Errorf(ctx, "Failed to download file from URL: %s, error: %v", payload.FileURL, err)
if isLastRetry {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
}
return fmt.Errorf("failed to download file from URL: %w", err)
}
if resolvedFileType != "" && !isSupportedImportExtension(resolvedFileType) {
logger.Errorf(ctx, "Unsupported file type resolved from file URL: %s", resolvedFileType)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("unsupported file type: %s", resolvedFileType)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
if resolvedFileName != "" && knowledge.FileName == "" {
knowledge.FileName = resolvedFileName
}
if resolvedFileType != "" && knowledge.FileType == "" {
knowledge.FileType = resolvedFileType
s.repo.UpdateKnowledge(ctx, knowledge)
}
fileSvc := s.resolveFileService(ctx, kb)
filePath, err := fileSvc.SaveBytes(ctx, contentBytes, payload.TenantID, resolvedFileName, true)
if err != nil {
if isLastRetry {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
}
return fmt.Errorf("failed to save downloaded file: %w", err)
}
payload.FilePath = filePath
payload.FileName = resolvedFileName
payload.FileType = resolvedFileType
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
if err != nil {
return err
}
if convertResult == nil {
return nil
}
} else if payload.URL != "" {
// URL import
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
if err != nil {
return err
}
if convertResult == nil {
return nil
}
// Update knowledge title from extracted page title if not already set
if knowledge.Title == "" || knowledge.Title == payload.URL {
if extractedTitle := convertResult.Metadata["title"]; extractedTitle != "" {
knowledge.Title = extractedTitle
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Warnf(ctx, "Failed to update knowledge title from extracted page title: %v", err)
} else {
logger.Infof(ctx, "Updated knowledge title to extracted page title: %s", extractedTitle)
}
}
}
} else if len(payload.Passages) > 0 {
// Text passage import - direct chunking, no conversion needed
passageChunks := make([]types.ParsedChunk, 0, len(payload.Passages))
start, end := 0, 0
for i, p := range payload.Passages {
if p == "" {
continue
}
end += len([]rune(p))
passageChunks = append(passageChunks, types.ParsedChunk{
Content: p,
Seq: i,
Start: start,
End: end,
})
start = end
}
passageOpts := ProcessChunksOptions{
EnableQuestionGeneration: payload.EnableQuestionGeneration,
QuestionCount: payload.QuestionCount,
}
return s.processChunks(ctx, kb, knowledge, passageChunks, passageOpts)
} else {
// File import
convertResult, err = s.convert(ctx, payload, kb, knowledge, eff, isLastRetry)
if err != nil {
return err
}
if convertResult == nil {
return nil
}
}
// Step 1.5: ASR transcription for audio files
if convertResult != nil && convertResult.IsAudio && len(convertResult.AudioData) > 0 {
if !eff.ASRConfig.IsASREnabled() {
logger.Error(ctx, "Audio file detected but ASR is not configured")
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "ASR model is not configured for audio transcription"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
logger.Infof(ctx, "[ASR] Starting audio transcription for knowledge %s, audio size=%d bytes",
knowledge.ID, len(convertResult.AudioData))
asrModel, err := s.modelService.GetASRModel(ctx, eff.ASRConfig.ModelID)
if err != nil {
logger.Errorf(ctx, "[ASR] Failed to get ASR model: %v", err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("failed to get ASR model: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// The knowledge base's language hint; the vendor declares where it
// goes, or that it takes none.
asrCtx := asr.WithLanguage(ctx, eff.ASRConfig.Language)
transcriptionResult, err := asrModel.Transcribe(asrCtx, convertResult.AudioData, knowledge.FileName)
if err != nil {
logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)
if isLastRetry {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("audio transcription failed: %v", err)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
}
return fmt.Errorf("audio transcription failed: %w", err)
}
transcribedText, timeBlocks := transcriptWithSegments(transcriptionResult)
if transcribedText == "" {
logger.Warn(ctx, "[ASR] Transcription returned empty text")
transcribedText = "[No speech detected in audio file]"
timeBlocks = nil
}
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(transcribedText))
// Replace the audio placeholder with the transcribed text
convertResult.MarkdownContent = transcribedText
convertResult.SourceBlocks = timeBlocks
convertResult.IsAudio = false
convertResult.AudioData = nil
}
// Normalize inline HTML tables from any parser engine (MinerU, builtin,
// markitdown, paddleocr-vl, ...) into GFM, or row-newline-separated HTML
// for merged / non-GFM tables, so the chunker can split them.
// Runs before image resolution so <img> inside HTML tables becomes
// markdown the resolver can persist, instead of rewriting data-URI
// <img> tags to `` and then escaping them in a later
// HTML→GFM pass. Line endings are normalized first so injected row
// breaks are plain LF.
// Source blocks were recorded against the parser's markdown; keep it so
// they can be re-aimed after the rewrites below.
var parsedMarkdown string
var sourceBlocks []types.SourceBlock
if convertResult != nil {
parsedMarkdown = convertResult.MarkdownContent
sourceBlocks = convertResult.SourceBlocks
for i := range sourceBlocks {
sourceBlocks[i].Locator.SourceHash = knowledge.FileHash
}
}
sanitizeReadResult(convertResult)
normalizeStarted := time.Now()
if convertResult != nil {
convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
if convertResult.MarkdownContent != "" {
convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent)
}
}
normalizedBytes := 0
if convertResult != nil {
normalizedBytes = len(convertResult.MarkdownContent)
}
logger.Infof(ctx, "Document text normalization completed in %s (knowledge=%s input_bytes=%d output_bytes=%d)",
time.Since(normalizeStarted), knowledge.ID, len(parsedMarkdown), normalizedBytes)
// Step 2: Store images and update markdown references
var storedImages []docparser.StoredImage
var imageFileSvc interfaces.FileService
var inlineParseErr error
if s.imageResolver != nil && convertResult != nil {
imageFileSvc = s.resolveFileService(ctx, kb)
tenantID, _ := ctx.Value(types.TenantIDContextKey).(uint64)
updatedMarkdown, images, resolveErr := s.imageResolver.ResolveAndStore(
ctx, convertResult, imageFileSvc, tenantID)
if errors.Is(resolveErr, docparser.ErrAmbiguousInlineImage) {
inlineParseErr = resolveErr
images = nil // ResolveAndStore already rolled back this attempt's uploads.
} else if resolveErr != nil {
_, err := s.failKnowledge(ctx, knowledge, isLastRetry, "store inline image: %v", resolveErr)
return err
}
convertResult.MarkdownContent = updatedMarkdown
storedImages = images
}
if inlineParseErr != nil ||
(convertResult != nil && docparser.HasUnresolvedInlineImagePayload(convertResult.MarkdownContent)) {
message := "unresolved inline image base64 payload remains before chunking"
if inlineParseErr != nil {
message = inlineParseErr.Error()
}
logger.Errorf(ctx, "Knowledge %s: %s", knowledge.ID, message)
cleanupUnboundInlineImages(ctx, imageFileSvc, storedImages)
knowledge.ParseStatus = types.ParseStatusFailed
knowledge.ErrorMessage = message
knowledge.UpdatedAt = time.Now()
s.beginStage(ctx, knowledge.ID, types.StageChunking, nil)
s.failStage(ctx, knowledge.ID, types.StageChunking, werrors.ErrCodeChunkingFailed, message, nil)
if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
return fmt.Errorf("%w: persist inline image processing failure: %v", asynq.SkipRetry, err)
}
return fmt.Errorf("%s: %w", message, asynq.SkipRetry)
}
if s.imageResolver != nil && convertResult != nil {
tenantID, _ := ctx.Value(types.TenantIDContextKey).(uint64)
// Resolve remote http(s) images (e.g. markdown external URLs) → download + upload to storage.
// ResolveAndStore handles inline bytes and base64; ResolveRemoteImages handles http/https URLs.
updatedContent, remoteImages, remoteErr := s.imageResolver.ResolveRemoteImages(
ctx, convertResult.MarkdownContent, imageFileSvc, tenantID)
if remoteErr != nil {
logger.Warnf(ctx, "Remote image resolution partially failed: %v", remoteErr)
}
if len(remoteImages) > 0 {
logger.Infof(ctx, "Resolved %d remote images for knowledge %s", len(remoteImages), knowledge.ID)
convertResult.MarkdownContent = updatedContent
storedImages = append(storedImages, remoteImages...)
}
logger.Infof(ctx, "Resolved %d total images for knowledge %s", len(storedImages), knowledge.ID)
}
// Claim the stored images for this document before chunking proceeds:
// the file proxies authorize images through resource bindings, and an
// unbound extracted image renders broken for org-shared KB viewers (#3342).
s.bindStoredImages(ctx, knowledge, storedImages)
var sourceIndex *sourceloc.Index
if convertResult != nil {
sourceIndex = buildSourceIndex(parsedMarkdown, sourceBlocks, convertResult.MarkdownContent, storedImages)
}
// Step 3: Split into chunks using Go chunker. Line endings and inline
// HTML tables were normalized before image resolution above.
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
chunkStarted := time.Now()
multimodalImages := imagesForMultimodal(storedImages)
processOpts := ProcessChunksOptions{
EnableQuestionGeneration: payload.EnableQuestionGeneration,
QuestionCount: payload.QuestionCount,
EnableMultimodel: payload.EnableMultimodel,
StoredImages: multimodalImages,
}
if convertResult != nil {
processOpts.Metadata = convertResult.Metadata
}
if eff.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(eff.ChunkingConfig, chunkCfg)
pcResult := chunker.SplitParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
chunks = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
chunks[i] = types.ParsedChunk{
Content: c.Content,
ContextHeader: c.ContextHeader,
Seq: c.Seq,
Start: c.Start,
End: c.End,
ParentIndex: c.ParentIndex,
SourceLocators: sourceIndex.Locators(c.Start, c.End),
}
}
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
for i, p := range pcResult.Parents {
parentChunks[i] = types.ParsedParentChunk{
Content: p.Content, Seq: p.Seq, Start: p.Start, End: p.End,
SourceLocators: sourceIndex.Locators(p.Start, p.End),
}
}
processOpts.ParentChunks = parentChunks
logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s",
len(pcResult.Parents), len(pcResult.Children), knowledge.ID)
} else {
splitChunks := chunker.Split(convertResult.MarkdownContent, chunkCfg)
chunks = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
chunks[i] = types.ParsedChunk{
Content: c.Content,
ContextHeader: c.ContextHeader,
Seq: c.Seq,
Start: c.Start,
End: c.End,
SourceLocators: sourceIndex.Locators(c.Start, c.End),
}
}
logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)
}
logger.Infof(ctx, "Document chunking completed in %s (knowledge=%s input_bytes=%d chunks=%d)",
time.Since(chunkStarted), knowledge.ID, len(convertResult.MarkdownContent), len(chunks))
// Step 4: Process chunks (vectorize + index + enqueue async tasks)
return s.processChunks(ctx, kb, knowledge, chunks, processOpts)
}
// sanitizeReadResult protects every text field that can cross from a parser
// into the embedding, storage, or tracing layers. A parser may return a Go
// string containing arbitrary bytes even though the string type itself does
// not enforce UTF-8 validity.
func sanitizeReadResult(result *types.ReadResult) {
if result == nil {
return
}
result.MarkdownContent = common.CleanInvalidUTF8(result.MarkdownContent)
result.ImageDirPath = common.CleanInvalidUTF8(result.ImageDirPath)
result.Error = common.CleanInvalidUTF8(result.Error)
for key, value := range result.Metadata {
result.Metadata[key] = common.CleanInvalidUTF8(value)
}
for i := range result.ImageRefs {
result.ImageRefs[i].Filename = common.CleanInvalidUTF8(result.ImageRefs[i].Filename)
result.ImageRefs[i].OriginalRef = common.CleanInvalidUTF8(result.ImageRefs[i].OriginalRef)
result.ImageRefs[i].MimeType = common.CleanInvalidUTF8(result.ImageRefs[i].MimeType)
result.ImageRefs[i].StorageKey = common.CleanInvalidUTF8(result.ImageRefs[i].StorageKey)
}
}
// convert handles both file and URL reading using a unified ReadRequest.
func (s *knowledgeService) convert(
ctx context.Context,
payload types.DocumentProcessPayload,
kb *types.KnowledgeBase,
knowledge *types.Knowledge,
eff types.EffectiveProcessConfig,
isLastRetry bool,
) (*types.ReadResult, error) {
// Stage tracking: docreader. Mark the stage as running here so the
// timeline reflects "DocReader" the moment a worker picks the task
// up — before that, the stage stays "pending" from the initial
// upload. Failure/skip transitions are emitted at the specific
// failure points below; success is emitted at the bottom.
docInput := types.JSONMap{
"file_name": payload.FileName,
"file_type": payload.FileType,
"is_url": payload.URL != "",
}
if payload.URL != "" {
docInput["url"] = payload.URL
}
s.beginStage(ctx, knowledge.ID, types.StageDocReader, docInput)
isURL := payload.URL != ""
fileType := payload.FileType
tenantOverrides := s.getParserEngineOverridesFromContext(ctx)
var uploadOverrides map[string]string
if processOverrides, err := knowledge.ProcessOverrides(); err == nil && processOverrides != nil {
uploadOverrides = processOverrides.ParserEngineOverrides
}
mergedOverrides := MergeParserEngineOverrides(tenantOverrides, uploadOverrides)
applyParserRuleOverrides(mergedOverrides, eff.ChunkingConfig, fileType)
if err := validateParserEngineOverrideURLs(mergedOverrides); err != nil {
logger.Errorf(ctx, "Parser endpoint rejected for SSRF protection: %v", err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "Parser endpoint is not allowed for security reasons"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
s.failStage(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, knowledge.ErrorMessage, err)
return nil, nil
}
if isURL {
if err := secutils.ValidateURLForSSRF(payload.URL); err != nil {
logger.Errorf(ctx, "URL rejected for SSRF protection: %s, err: %v", payload.URL, err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "URL is not allowed for security reasons"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
s.failStage(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, "URL rejected for security reasons", err)
return nil, nil
}
}
parserEngine := eff.ChunkingConfig.ResolveParserEngine(fileType)
if isURL {
parserEngine = eff.ChunkingConfig.ResolveParserEngine("url")
}
logger.Infof(ctx, "[convert] kb=%s fileType=%s isURL=%v engine=%q rules=%+v",
kb.ID, fileType, isURL, parserEngine, eff.ChunkingConfig.ParserEngineRules)
var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, mergedOverrides)
if reader == nil {
logger.Errorf(ctx, "[convert] no doc reader for kb=%s knowledge=%s fileType=%s engine=%q isURL=%v",
kb.ID, knowledge.ID, fileType, parserEngine, isURL)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "Document parsing service is not configured. Please use text/paragraph import or set DOCREADER_ADDR."
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
s.failStage(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderUnavailable, knowledge.ErrorMessage, nil)
return nil, nil
}
req := &types.ReadRequest{
URL: payload.URL,
Title: knowledge.Title,
ParserEngine: parserEngine,
RequestID: payload.RequestId,
ParserEngineOverrides: mergedOverrides,
}
if !isURL {
fileReader, err := s.resolveFileServiceForPath(ctx, kb, payload.FilePath).GetFile(ctx, payload.FilePath)
if err != nil {
result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err)
s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, "failed to get file", err, taskErr != nil)
return result, taskErr
}
defer fileReader.Close()
contentBytes, err := io.ReadAll(fileReader)
if err != nil {
result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "failed to read file: %v", err)
s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, "failed to read file", err, taskErr != nil)
return result, taskErr
}
req.FileContent = contentBytes
req.FileName = payload.FileName
req.FileType = fileType
}
result, err := s.callDocReaderWithTimeout(ctx, reader, req)
// Parser resource limits are permanent document rejections, not transient reader failures.
if errors.Is(err, docparser.ErrCSVMarkdownTooLarge) || errors.Is(err, anydoc.ErrResourceLimit) {
result, err = &types.ReadResult{Error: err.Error()}, nil
}
if err != nil {
code, message := docReaderFailure(err)
logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err)
result, taskErr := s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)
s.failStageWithRetry(ctx, knowledge.ID, types.StageDocReader,
code, message, err, taskErr != nil)
return result, taskErr
}
sanitizeReadResult(result)
if result.Error != "" {
logger.Errorf(ctx, "[convert] parser returned error kb=%s knowledge=%s file=%q type=%s engine=%q: %s",
kb.ID, knowledge.ID, req.FileName, fileType, parserEngine, result.Error)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = result.Error
knowledge.UpdatedAt = time.Now()
if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
logger.Errorf(ctx, "failed to persist parser error for %s: %v", knowledge.ID, err)
}
s.failStage(ctx, knowledge.ID, types.StageDocReader,
werrors.ErrCodeDocReaderParseFailed, result.Error, nil)
return nil, nil
}
if !isURL {
attachStructureBlocks(ctx, fileType, req.FileContent, result)
}
docOutput := types.JSONMap{
"text_length": len(result.MarkdownContent),
"images_found": len(result.ImageRefs),
"is_audio": result.IsAudio,
}
if pages := result.Metadata["pages"]; pages != "" {
docOutput["pages"] = pages
}
s.endStage(ctx, knowledge.ID, types.StageDocReader, docOutput)
return result, nil
}
// callDocReaderWithTimeout wraps the DocReader RPC in a child context whose
// deadline is min(parent_deadline, DocReaderCallTimeout). Without this cap,
// a hung docreader (network partition, GC pause, OCR runaway) silently
// burns the whole DocumentProcessTimeout budget and pins a worker for hours
// — the #1 cause of "knowledge stuck in processing" reports.
//
// On timeout we annotate the error so retries / dead-letter consumers can
// distinguish "docreader was slow" from "docreader returned an error".
func (s *knowledgeService) callDocReaderWithTimeout(
ctx context.Context, reader interfaces.DocReader, req *types.ReadRequest,
) (*types.ReadResult, error) {
timeout := 30 * time.Minute
if s.config != nil && s.config.KnowledgeBase != nil && s.config.KnowledgeBase.DocReaderCallTimeout > 0 {
timeout = s.config.KnowledgeBase.DocReaderCallTimeout
}
callCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
start := time.Now()
result, err := reader.Read(callCtx, req)
elapsed := time.Since(start)
if err != nil {
// Promote DeadlineExceeded into a clearer message; retain underlying
// error via %w so errors.Is(callCtx.Err(), context.DeadlineExceeded)
// still works for upstream classification.
if errors.Is(callCtx.Err(), context.DeadlineExceeded) && !errors.Is(ctx.Err(), context.DeadlineExceeded) {
logger.Errorf(ctx, "[convert] docreader call timed out after %s (limit %s) for %q",
elapsed, timeout, req.FileName)
return nil, fmt.Errorf("docreader call timeout after %s: %w", timeout, err)
}
return nil, err
}
logger.Infof(ctx, "[convert] docreader call ok in %s for %q", elapsed, req.FileName)
return result, nil
}
// isLikelyRateLimitError performs a fuzzy classification of an error as a
// rate-limit / quota / backpressure failure. We only need a hint — the
// caller maps to one of two error_codes so the UI can offer "retry later"
// vs. "fix configuration" advice. False positives are harmless (the
// detail is preserved in error_detail anyway).
func isLikelyRateLimitError(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
for _, needle := range []string{"rate limit", "ratelimit", "429", "too many requests", "quota"} {
if strings.Contains(msg, needle) {
return true
}
}
return false
}
// resolveDocReader picks the reader for one parse request. The engine catalog
// itself lives in the docparser registry; this only supplies the dependencies
// the service owns. Returns nil when the chosen engine cannot run — an
// unconfigured cloud engine, a disconnected docreader — after logging why.
func (s *knowledgeService) resolveDocReader(
ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string,
) interfaces.DocReader {
reader, err := docparser.NewReader(ctx, engine, fileType, isURL, docparser.ReaderDeps{
Overrides: overrides,
Remote: s.documentReader,
WeKnoraCloudCredentials: s.tenantService.GetWeKnoraCloudCredentials,
})
if err != nil {
logger.Warnf(ctx, "[resolveDocReader] engine=%q fileType=%q unusable: %v", engine, fileType, err)
return nil
}
return reader
}
// failKnowledge marks knowledge as failed (only on last retry) and returns an error.
func (s *knowledgeService) failKnowledge(
ctx context.Context,
knowledge *types.Knowledge,
isLastRetry bool,
format string,
args ...interface{},
) (*types.ReadResult, error) {
errMsg := fmt.Sprintf(format, args...)
if s.isKnowledgeSourceReplaced(ctx, knowledge) {
logger.Infof(ctx, "Skip failing knowledge %s: source file was replaced", knowledge.ID)
return nil, nil
}
if isLastRetry {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = errMsg
knowledge.UpdatedAt = time.Now()
if err := s.updateKnowledgeUnlessSourceReplaced(ctx, knowledge); err != nil {
logger.Errorf(ctx, "failed to persist knowledge failure for %s: %v", knowledge.ID, err)
}
}
return nil, fmt.Errorf(format, args...)
}
// enqueueImageMultimodalTasks enqueues asynq tasks for multimodal image
// processing. It returns an error only when no path is left to move the
// knowledge out of "processing" (see releaseUnownedMultimodalSlots).
func (s *knowledgeService) enqueueImageMultimodalTasks(
ctx context.Context,
knowledge *types.Knowledge,
kb *types.KnowledgeBase,
images []docparser.StoredImage,
chunks []types.ParsedChunk,
metadata map[string]string,
) error {
if s.task == nil || len(images) == 0 {
return nil
}
attempt := attemptFromCtx(ctx)
lang := types.LanguageFromContextOrDefault(ctx)
redisKey := multimodalPendingKey(knowledge.ID)
// The counter has to be seeded BEFORE the first task is enqueued: an image
// task that finishes early would otherwise decrement a key we are about to
// overwrite, and its slot would be counted twice. The cost of seeding
// first is that a slot whose task never reaches the queue has no owner to
// drain it — releaseUnownedMultimodalSlots settles that after the fan-out.
counterSeeded := false
if s.redisClient != nil {
if err := s.redisClient.Set(ctx, redisKey, len(images), 24*time.Hour).Err(); err != nil {
logger.Warnf(ctx, "Failed to set multimodal pending count for %s: %v", knowledge.ID, err)
} else {
counterSeeded = true
}
} else {
liteMultimodalSet(redisKey, int64(len(images)))
counterSeeded = true
}
// Resolve the image pipeline settings once for the whole fan-out: the image
// action policy and the pipeline switch travel with each task so the worker
// needs no second config lookup, and so a task keeps the policy it was
// enqueued with even if the knowledge base is reconfigured meanwhile.
overrides, _ := knowledge.ProcessOverrides()
eff := ResolveProcessConfig(kb, overrides)
enqueued := 0
for idx, img := range images {
// Match image to the ParsedChunk whose content contains the image URL.
// ChunkID was populated by processChunks with the real DB UUID.
chunkID := imageChunkOwner(img.ServingURL, chunks)
if chunkID == "" {
logger.Warnf(ctx, "Image has no owning text chunk: knowledge=%s image=%s", knowledge.ID, img.ServingURL)
}
payload := types.ImageMultimodalPayload{
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kb.ID,
ChunkID: chunkID,
ImageURL: img.ServingURL,
ImagePipelineID: eff.ImagePipelineID,
ImagePipelineParams: eff.ImagePipelineParams,
ImageAttrsEnabled: eff.ImageAttrsEnabled,
ImageActions: eff.ImageActions,
Language: lang,
ImageSourceType: metadata["image_source_type"],
Attempt: attempt,
ImageIndex: idx,
SourceLocators: img.SourceLocators,
ImageVectorEnabled: &eff.ImageVectorEnabled,
}
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Warnf(ctx, "Failed to marshal image multimodal payload: %v", err)
continue
}
task := asynq.NewTask(types.TypeImageMultimodal, payloadBytes,
asynq.Queue(types.QueueMultimodal), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
if _, err := s.task.Enqueue(task); err != nil {
logger.Warnf(ctx, "Failed to enqueue image multimodal task for %s: %v", img.ServingURL, err)
continue
}
enqueued++
logger.Infof(ctx, "Enqueued image:multimodal task for %s", img.ServingURL)
}
return s.releaseUnownedMultimodalSlots(
ctx, knowledge, redisKey, counterSeeded, len(images), enqueued,
)
}
// releaseUnownedMultimodalSlots settles the multimodal fan-in counter against
// what actually reached the queue.
//
// Every enqueued image task drains exactly one slot when it exits terminally,
// and the knowledge only leaves "processing" once the counter reaches zero. A
// slot whose task was never enqueued (payload marshal failure, asynq refusing
// the write) has no owner, so without this release the counter stalls above
// zero, post-process is never triggered, and the row sits in "processing"
// until the housekeeping sweep fails it an hour later — which reaches the user
// as an unexplained parse failure. This mirrors the shortfall release
// KnowledgePostProcess already performs for its own enrichment subtasks.
func (s *knowledgeService) releaseUnownedMultimodalSlots(
ctx context.Context,
knowledge *types.Knowledge,
redisKey string,
counterSeeded bool,
planned, enqueued int,
) error {
if enqueued == 0 {
// No task exists to finalize this knowledge, whatever the counter
// says. Drive post-process directly so the row completes on the
// chunks that were already indexed instead of waiting to be swept.
logger.Warnf(ctx,
"[ImageMultimodal] No image task enqueued for %s (planned=%d); enqueueing post-process directly",
knowledge.ID, planned)
if counterSeeded {
// Detached like the shortfall path below: a cancelled parent
// context would otherwise leave the seeded key to expire on its
// own 24h TTL.
dctx, cancel := context.WithTimeout(
context.WithoutCancel(ctx), finalizeSubtaskDetachedTimeout)
s.delMultimodalCounter(dctx, redisKey)
cancel()
}
return s.enqueueKnowledgePostProcessTask(ctx, knowledge)
}
shortfall := planned - enqueued
if shortfall <= 0 {
return nil
}
if !counterSeeded {
// Seeding failed, so the key is absent and the first image to finish
// already drives post-process through the missing-key fallback in
// checkAndFinalizeAllImages. Decrementing here would only create the
// key below zero and finalize before the siblings are done.
return nil
}
logger.Warnf(ctx,
"[ImageMultimodal] Releasing %d un-enqueued image slot(s) for %s (planned=%d enqueued=%d)",
shortfall, knowledge.ID, planned, enqueued)
// Detached ctx, for the same reason KnowledgePostProcess detaches its own
// releases: these slots have no other path to drain, so a parent context
// cancelled mid-fan-out (graceful shutdown) must not skip the release.
rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), finalizeSubtaskDetachedTimeout)
defer cancel()
pending, err := s.decrMultimodalSlots(rctx, redisKey, int64(shortfall))
if err != nil {
// Giving up here puts the row back in the state this whole fix exists
// to remove: a counter no task can drain, recovered only by the
// housekeeping sweep an hour later. Say so at error level so it is
// greppable, rather than hiding it in a warning.
logger.Errorf(ctx,
"Failed to release %d multimodal slot(s) for %s after %d attempts: %v; "+
"row will be left to the housekeeping sweep",
shortfall, knowledge.ID, multimodalSlotReleaseAttempts, err)
return nil
}
if pending <= 0 {
// Every enqueued sibling had already finished, so the release owns the
// finalize. Ordering is safe either way: the slots we just drained have
// no task behind them, so exactly one path reaches zero.
if err := s.enqueueKnowledgePostProcessTask(ctx, knowledge); err != nil {
return err
}
s.delMultimodalCounter(rctx, redisKey)
}
return nil
}
// delMultimodalCounter removes the fan-in counter from Redis, or from the
// in-process stand-in in Lite mode.
func (s *knowledgeService) delMultimodalCounter(ctx context.Context, redisKey string) {
if s.redisClient == nil {
liteMultimodalDel(redisKey)
return
}
s.redisClient.Del(ctx, redisKey)
}
// multimodalSlotReleaseAttempts bounds the retry on the release decrement.
//
// The release is the only thing that can ever drain an un-owned slot, so a
// single transient Redis error must not be the end of it — that would strand
// the row exactly as it did before this fix. Attempts stay small because they
// share the detached context's budget, and the housekeeping sweep is still the
// backstop when Redis is genuinely down.
//
// DECRBY is not idempotent, so a retry after a lost reply can over-decrement.
// That is the safe direction to fail in: an over-decrement finalizes early
// (post-process may miss some OCR / caption chunks, and repeat deliveries are
// already handled by its non-processing-status branch), whereas not retrying
// strands the document in "processing" for an hour.
const multimodalSlotReleaseAttempts = 3
// multimodalSlotReleaseBackoff is the unit of linear backoff between release
// attempts, small enough that all attempts fit the detached 10s budget.
const multimodalSlotReleaseBackoff = 100 * time.Millisecond
// decrMultimodalSlots decrements the fan-in counter by `by`, retrying transient
// Redis failures, and returns the resulting count.
func (s *knowledgeService) decrMultimodalSlots(
ctx context.Context, redisKey string, by int64,
) (int64, error) {
if s.redisClient == nil {
return liteMultimodalDecrBy(redisKey, by), nil
}
var lastErr error
for attempt := 1; attempt <= multimodalSlotReleaseAttempts; attempt++ {
pending, err := s.redisClient.DecrBy(ctx, redisKey, by).Result()
if err == nil {
return pending, nil
}
lastErr = err
if attempt == multimodalSlotReleaseAttempts {
break
}
logger.Warnf(ctx, "multimodal slot release attempt %d/%d failed for %s: %v",
attempt, multimodalSlotReleaseAttempts, redisKey, err)
select {
case <-time.After(time.Duration(attempt) * multimodalSlotReleaseBackoff):
case <-ctx.Done():
return 0, ctx.Err()
}
}
return 0, lastErr
}
// enqueueKnowledgePostProcessTask hands a knowledge to the post-process
// orchestrator, which owns the enrichment fan-out and the promotion out of
// "processing". Shared by the no-multimodal path and by the multimodal slot
// release, so both build the same payload (attempt, language, tracing).
//
// The error must reach the caller's task result: once the chunks are indexed
// this enqueue is the only thing that moves the row out of "processing", so
// swallowing it acked the task with the row stranded until housekeeping.
func (s *knowledgeService) enqueueKnowledgePostProcessTask(
ctx context.Context, knowledge *types.Knowledge,
) error {
if s.task == nil {
return nil
}
payload := types.KnowledgePostProcessPayload{
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Language: types.LanguageFromContextOrDefault(ctx),
Attempt: attemptFromCtx(ctx),
}
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal knowledge post process payload: %v", err)
return fmt.Errorf("marshal post process payload: %w", err)
}
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
knowledgePostProcessTaskOptions()...)
if err := enqueueWithRetry(ctx, s.task, task); err != nil {
logger.Errorf(ctx, "Failed to enqueue knowledge post process task for %s: %v", knowledge.ID, err)
return err
}
logger.Infof(ctx, "Enqueued knowledge post process task for %s", knowledge.ID)
return nil
}
// ProcessKnowledgeListReparse handles Asynq knowledge list reparse tasks.
func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *asynq.Task) error {
var payload types.KnowledgeListReparsePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal knowledge list reparse payload: %v", err)
return err
}
ctx = payload.Initiator.Apply(ctx)
taskID, _ := asynq.GetTaskID(ctx)
ctx = withKBActivityTask(ctx, taskID, kbActivityTrigger(ctx))
logger.Infof(ctx, "Processing knowledge list reparse task for %d knowledge items", len(payload.KnowledgeIDs))
ctx, ids, err := s.reparseTaskScope(ctx, payload)
if err != nil || len(ids) == 0 {
return err
}
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "Failed to get tenant %d: %v", payload.TenantID, err)
return err
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
outcome, err := runKnowledgeListReparseSubmissions(ids, func(id string) error {
_, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig)
return err
})
logger.Infof(ctx, "Knowledge list reparse task finished: %d submitted, %d failed",
outcome.Submitted, outcome.Failed)
return err
}
type knowledgeListReparseOutcome struct {
Submitted int
Failed int
}
// runKnowledgeListReparseSubmissions attempts every item so one bad document
// cannot block the remainder of the batch. A partial failure is non-retryable:
// retrying the wrapper task would destructively reparse items that were already
// submitted successfully. Failed rows remain selectable for an explicit retry.
func runKnowledgeListReparseSubmissions(
ids []string,
submit func(string) error,
) (knowledgeListReparseOutcome, error) {
var outcome knowledgeListReparseOutcome
failures := make([]error, 0)
for _, id := range ids {
if err := submit(id); err != nil {
failures = append(failures, fmt.Errorf("knowledge %s: %w", secutils.SanitizeForLog(id), err))
outcome.Failed++
continue
}
outcome.Submitted++
}
if len(failures) == 0 {
return outcome, nil
}
return outcome, fmt.Errorf(
"%w: batch reparse submitted %d item(s) and failed %d: %w",
asynq.SkipRetry, outcome.Submitted, outcome.Failed, errors.Join(failures...),
)
}