Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(retrieval): 分段中删除的图片不再被检索到
lawrence28dd15ec30d8
diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.goindex 17be8e557..ccb9fe871 100644--- a/internal/application/repository/chunk.go+++ b/internal/application/repository/chunk.go@@ -160,6 +160,26 @@ func (r *chunkRepository) ListChunksByKnowledgeID(return chunks, nil}+// ListChunksByKnowledgeIDAndTypes lists a knowledge's chunks restricted to the+// given chunk types. ListChunksByKnowledgeID is text-only by design, so callers+// that also need summary / parent_text / image chunks come through here rather+// than widening that query underneath its existing callers.+func (r *chunkRepository) ListChunksByKnowledgeIDAndTypes(+ ctx context.Context, tenantID uint64, knowledgeID string, chunkTypes []types.ChunkType,+) ([]*types.Chunk, error) {+ if len(chunkTypes) == 0 {+ return nil, nil+ }+ var chunks []*types.Chunk+ if err := r.db.WithContext(ctx).+ Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN ?", tenantID, knowledgeID, chunkTypes).+ Order("chunk_index ASC").+ Find(&chunks).Error; err != nil {+ return nil, err+ }+ return chunks, nil+}+// ListPagedChunksByKnowledgeID lists chunks for a knowledge ID with paginationfunc (r *chunkRepository) ListPagedChunksByKnowledgeID(ctx context.Context,diff --git a/internal/application/service/chat_pipeline/merge_parent_resolve_test.go b/internal/application/service/chat_pipeline/merge_parent_resolve_test.goindex 67516a3ff..5ae6688d7 100644--- a/internal/application/service/chat_pipeline/merge_parent_resolve_test.go+++ b/internal/application/service/chat_pipeline/merge_parent_resolve_test.go@@ -26,6 +26,7 @@ func TestResolveParentChunksUsesCurrentContentAndImageURLs(t *testing.T) {{ID: "image", ParentChunkID: "child", ChunkType: types.ChunkTypeImageOCR,ImageInfo: string(imageInfo),+ IsEnabled: true,},},},@@ -73,6 +74,7 @@ func TestResolveImageChunkKeepsGrandparentContextWithoutCoordinateSlicing(t *tes"text": {{ID: "image", ParentChunkID: "text", ChunkType: types.ChunkTypeImageOCR,ImageInfo: string(imageInfo),+ IsEnabled: true,}},},}diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 3ac257843..6c2f7350b 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -2340,26 +2340,35 @@ func (s *knowledgeService) RegenerateKnowledgeSummary(return nil, err}if kb.NeedsEmbeddingModel() {- found := falsemaxIndex := 0- summaryChunks := make([]*types.Chunk, 0, 1)for _, chunk := range allChunks {if chunk.ChunkIndex > maxIndex {maxIndex = chunk.ChunkIndex}- if chunk.ChunkType == types.ChunkTypeSummary {- chunk.Content = "# Summary\n" + summary- chunk.SourceContent = chunk.Content- chunk.IsEnabled = true- chunk.UpdatedAt = time.Now()- if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {- return nil, err- }- summaryChunks = append(summaryChunks, chunk)- found = true+ }+ // allChunks holds text chunks only, so it can never carry the existing+ // summary chunk. Scanning it for one always came up empty, which left+ // every refresh appending a new summary chunk beside the stale one --+ // and a stale summary stays enabled and indexed, so content the user+ // edited out of the document kept being retrievable through it.+ existingSummaries, err := s.chunkRepo.ListChunksByKnowledgeIDAndTypes(+ ctx, tenantID, knowledgeID, []types.ChunkType{types.ChunkTypeSummary},+ )+ if err != nil {+ return nil, err+ }+ summaryChunks := make([]*types.Chunk, 0, len(existingSummaries))+ for _, chunk := range existingSummaries {+ chunk.Content = "# Summary\n" + summary+ chunk.SourceContent = chunk.Content+ chunk.IsEnabled = true+ chunk.UpdatedAt = time.Now()+ if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil {+ return nil, err}+ summaryChunks = append(summaryChunks, chunk)}- if !found {+ if len(summaryChunks) == 0 {summaryChunk := &types.Chunk{ID: uuid.NewString(), TenantID: tenantID, KnowledgeID: knowledge.ID,KnowledgeBaseID: knowledge.KnowledgeBaseID, Content: "# Summary\n" + summary,@@ -2973,6 +2982,10 @@ func (s *knowledgeService) UpdateImageInfo(ChunkType: types.ChunkTypeImageCaption,ParentChunkID: chunk.ID,ImageInfo: imageInfo,+ // CreateChunks inserts with Select("*"), so the gorm default:true never+ // applies -- an unset IsEnabled lands in the database as false and the+ // chunk is silently excluded from retrieval and model context.+ IsEnabled: true,}addChunk = append(addChunk, captionChunk)logger.Infof(ctx, "Created new caption chunk ID: %s for image URL: %s", captionChunk.ID, image.OriginalURL)@@ -2989,6 +3002,7 @@ func (s *knowledgeService) UpdateImageInfo(ChunkType: types.ChunkTypeImageOCR,ParentChunkID: chunk.ID,ImageInfo: imageInfo,+ IsEnabled: true,}addChunk = append(addChunk, ocrChunk)logger.Infof(ctx, "Created new OCR chunk ID: %s for image URL: %s", ocrChunk.ID, image.OriginalURL)diff --git a/internal/searchutil/imageinfo.go b/internal/searchutil/imageinfo.goindex 7cba563fd..487563aea 100644--- a/internal/searchutil/imageinfo.go+++ b/internal/searchutil/imageinfo.go@@ -46,6 +46,11 @@ const HTMLImageSrcURLGroup = 2// - If chunkIDs are parent_text chunks, their children are text chunks// whose children are image chunks → two queries.//+// Disabled children are skipped at both levels. Removing an image from a chunk+// disables its image_ocr / image_caption children (syncEditedChunkImages), and+// disabling a text chunk disables them too; honoring that flag here is what+// keeps a deleted image out of retrieval, summaries and model context.+//// Returns a map of input chunkID → merged image_info JSON string.func CollectImageInfoByChunkIDs(ctx context.Context,@@ -107,6 +112,9 @@ func CollectImageInfoByChunkIDs(textToParent := make(map[string]string)for _, child := range children {+ if !child.IsEnabled {+ continue+ }switch child.ChunkType {case types.ChunkTypeImageOCR, types.ChunkTypeImageCaption:addInfo(child.ParentChunkID, child)@@ -120,6 +128,9 @@ func CollectImageInfoByChunkIDs(grandChildren, err := chunkRepo.ListChunksByParentIDs(ctx, tenantID, textChildIDs)if err == nil {for _, gc := range grandChildren {+ if !gc.IsEnabled {+ continue+ }if gc.ChunkType != types.ChunkTypeImageOCR && gc.ChunkType != types.ChunkTypeImageCaption {continue}diff --git a/internal/searchutil/imageinfo_disabled_test.go b/internal/searchutil/imageinfo_disabled_test.gonew file mode 100644index 000000000..6d4e41495--- /dev/null+++ b/internal/searchutil/imageinfo_disabled_test.go@@ -0,0 +1,84 @@+package searchutil++import (+ "context"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// childrenByParentRepo serves chunk children from a static parent → children map.+type childrenByParentRepo struct {+ interfaces.ChunkRepository+ byParent map[string][]*types.Chunk+}++func (r childrenByParentRepo) ListChunksByParentIDs(+ _ context.Context, _ uint64, parentIDs []string,+) ([]*types.Chunk, error) {+ var out []*types.Chunk+ for _, id := range parentIDs {+ out = append(out, r.byParent[id]...)+ }+ return out, nil+}++func imageChild(id, parentID string, chunkType types.ChunkType, url string, enabled bool) *types.Chunk {+ return &types.Chunk{+ ID: id,+ ParentChunkID: parentID,+ ChunkType: chunkType,+ IsEnabled: enabled,+ ImageInfo: `[{"url":"` + url + `","caption":"a screenshot"}]`,+ }+}++// Deleting an image from a chunk disables its image children. Their image_info+// must not come back through enrichment, or the removed image reappears in+// summaries, tool output and model context.+func TestCollectImageInfoByChunkIDsSkipsDisabledImageChildren(t *testing.T) {+ repo := childrenByParentRepo{byParent: map[string][]*types.Chunk{+ "text-1": {+ imageChild("ocr-1", "text-1", types.ChunkTypeImageOCR, "resource://removed", false),+ imageChild("cap-1", "text-1", types.ChunkTypeImageCaption, "resource://removed", false),+ },+ "text-2": {+ imageChild("ocr-2", "text-2", types.ChunkTypeImageOCR, "resource://kept", true),+ },+ }}++ got := CollectImageInfoByChunkIDs(context.Background(), repo, 1, []string{"text-1", "text-2"})++ if _, ok := got["text-1"]; ok {+ t.Fatalf("disabled image children leaked image_info: %q", got["text-1"])+ }+ if got["text-2"] == "" {+ t.Fatal("enabled image child lost its image_info")+ }+}++// The parent_text → text → image path must apply the same rule at both levels.+func TestCollectImageInfoByChunkIDsSkipsDisabledGrandchildrenAndTextChildren(t *testing.T) {+ repo := childrenByParentRepo{byParent: map[string][]*types.Chunk{+ "parent-1": {+ {ID: "text-1", ParentChunkID: "parent-1", ChunkType: types.ChunkTypeText, IsEnabled: true},+ },+ "parent-2": {+ // Disabling a text chunk hides everything under it.+ {ID: "text-2", ParentChunkID: "parent-2", ChunkType: types.ChunkTypeText, IsEnabled: false},+ },+ "text-1": {+ imageChild("ocr-1", "text-1", types.ChunkTypeImageOCR, "resource://removed", false),+ },+ "text-2": {+ imageChild("ocr-2", "text-2", types.ChunkTypeImageOCR, "resource://hidden", true),+ },+ }}++ got := CollectImageInfoByChunkIDs(context.Background(), repo, 1, []string{"parent-1", "parent-2"})++ if len(got) != 0 {+ t.Fatalf("expected no image_info, got %v", got)+ }+}diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.goindex 9f6434d61..28cf8a4c6 100644--- a/internal/types/interfaces/chunk.go+++ b/internal/types/interfaces/chunk.go@@ -28,8 +28,15 @@ type ChunkRepository interface {ListChunksByIDOnly(ctx context.Context, ids []string) ([]*types.Chunk, error)// ListChunksBySeqID lists chunks by seq_idsListChunksBySeqID(ctx context.Context, tenantID uint64, seqIDs []int64) ([]*types.Chunk, error)- // ListChunksByKnowledgeID lists chunks by knowledge id+ // ListChunksByKnowledgeID lists the knowledge's text chunks. Despite the name+ // it filters to chunk_type = 'text'; reach for ListChunksByKnowledgeIDAndTypes+ // when summary, parent_text or image chunks are needed too.ListChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error)+ // ListChunksByKnowledgeIDAndTypes lists the knowledge's chunks restricted to+ // the given chunk types, ordered by chunk_index.+ ListChunksByKnowledgeIDAndTypes(+ ctx context.Context, tenantID uint64, knowledgeID string, chunkTypes []types.ChunkType,+ ) ([]*types.Chunk, error)// ListPagedChunksByKnowledgeID lists paged chunks by knowledge id.// When tagIDs is non-empty, results are filtered by tag_id (OR semantics).// knowledgeType: "faq" or "manual" - determines sort order and search behavior