RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(dataset): honor keywords filter in Go wiki artifact list APIs (#19310)
euvref4b047844473
diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.goindex 4b1a282a3b..aa637b9dc7 100644--- a/internal/handler/dataset_artifact.go+++ b/internal/handler/dataset_artifact.go@@ -105,6 +105,7 @@ func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {datasetID := c.Param("dataset_id")pageType := c.Query("page_type")topic := c.Query("topic")+ keywords := strings.TrimSpace(c.Query("keywords"))page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30"))if page < 1 {@@ -116,7 +117,7 @@ func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {if pageSize > maxArtifactPageSize {pageSize = maxArtifactPageSize}- items, total, err := h.svc.ListWikiPages(c.Request.Context(), tenantID, datasetID, pageType, topic, page, pageSize)+ items, total, err := h.svc.ListWikiPages(c.Request.Context(), tenantID, datasetID, pageType, topic, keywords, page, pageSize)if err != nil {common.ErrorWithCode(c, common.CodeDataError, err.Error())return@@ -240,7 +241,8 @@ func (h *DatasetArtifactHandler) ListArtifactTopics(c *gin.Context) {return}datasetID := c.Param("dataset_id")- items, total, err := h.svc.ListWikiTopics(c.Request.Context(), tenantID, datasetID)+ keywords := strings.TrimSpace(c.Query("keywords"))+ items, total, err := h.svc.ListWikiTopics(c.Request.Context(), tenantID, datasetID, keywords)if err != nil {common.ErrorWithCode(c, common.CodeDataError, err.Error())returndiff --git a/internal/service/dataset_artifact_keyword_integration_test.go b/internal/service/dataset_artifact_keyword_integration_test.gonew file mode 100644index 0000000000..0f3f1728dc--- /dev/null+++ b/internal/service/dataset_artifact_keyword_integration_test.go@@ -0,0 +1,152 @@+//go:build integration+// +build integration++package service++import (+ "context"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/engine"+ "ragflow/internal/server"+)++// wikiIntRow builds a merged dataset-level wiki_page row shaped like the+// knowledge-compiler writer output that the artifact APIs read.+func wikiIntRow(id, pageType, slug, title, topic, summary string, outlinks int) map[string]interface{} {+ fullSlug := pageType + "/" + slug+ return map[string]interface{}{+ "id": id,+ "doc_id": id,+ "available_int": 1,+ "scope_kwd": "dataset",+ "compile_kwd": CompileKwdWikiPage,+ "slug_kwd": fullSlug,+ "artifact_slug_kwd": fullSlug,+ "title_kwd": title,+ "page_type_kwd": pageType,+ "topic_kwd": topic,+ "summary_with_weight": summary,+ "outlinks_int": outlinks,+ }+}++// TestWikiKeywordFilter_Integration seeds wiki_page rows into a real document+// engine and asserts ListWikiPages/ListWikiTopics keyword semantics (mirrors+// Python list_wiki_pages/list_wiki_topics): pages match by title/slug/summary+// only; topics match by their own path or by holding a matching page.+//+// Run with: bash build.sh --test-integration ./internal/service/...+func TestWikiKeywordFilter_Integration(t *testing.T) {+ if err := common.InitLogger("info", common.FileOutput{}, ""); err != nil {+ t.Fatalf("init logger: %v", err)+ }+ if err := server.Init(""); err != nil {+ t.Fatalf("init service config: %v", err)+ }+ if err := engine.InitDocEngine(context.Background()); err != nil {+ t.Skipf("no document engine: %v", err)+ }+ de := engine.Get()+ if de == nil {+ t.Skip("no live document engine configured")+ }++ tenantID := "wikiint_t1"+ kbID := "wikiint_kb1"+ idx := wikiIndexName(tenantID)+ ctx := t.Context()++ rows := []map[string]interface{}{+ wikiIntRow("wikiint_p1", "entity", "Daisy", "Daisy", "General", "the field flower", 3),+ wikiIntRow("wikiint_p2", "entity", "Swallow", "Swallow", "General", "the little swallow", 5),+ wikiIntRow("wikiint_p3", "concept", "Love", "Love", "Themes", "devotion and kindness", 7),+ }+ for _, row := range rows {+ row["kb_id"] = kbID+ }+ // Best effort: the store may already exist from a previous run.+ _ = de.CreateChunkStore(ctx, idx, kbID, 1024, "naive")+ if _, err := de.InsertChunks(ctx, rows, idx, kbID); err != nil {+ t.Fatalf("seed wiki rows: %v", err)+ }+ t.Cleanup(func() {+ _, _ = de.DeleteChunks(context.Background(),+ map[string]interface{}{"kb_id": []string{kbID}}, idx, kbID)+ })++ svc := NewDatasetArtifactService()++ // Baseline: no keywords returns every seeded page.+ all, total, err := svc.ListWikiPages(ctx, tenantID, kbID, "", "", "", 1, 30)+ if err != nil {+ t.Fatalf("list pages: %v", err)+ }+ if total != 3 || len(all) != 3 {+ t.Fatalf("unfiltered pages = %#v (total %d), want 3", all, total)+ }++ // Keyword filters by title, case-insensitively.+ got, total, err := svc.ListWikiPages(ctx, tenantID, kbID, "", "", "dAiSy", 1, 30)+ if err != nil {+ t.Fatalf("list pages with keyword: %v", err)+ }+ if total != 1 || len(got) != 1 || got[0].Title != "Daisy" || got[0].Slug != "Daisy" {+ t.Fatalf("keyword dAiSy pages = %#v (total %d), want only Daisy", got, total)+ }++ // Keyword matches the summary too ("devotion" only occurs in Love's summary).+ got, total, err = svc.ListWikiPages(ctx, tenantID, kbID, "", "", "devotion", 1, 30)+ if err != nil {+ t.Fatalf("list pages with summary keyword: %v", err)+ }+ if total != 1 || len(got) != 1 || got[0].Title != "Love" {+ t.Fatalf("keyword devotion pages = %#v (total %d), want only Love", got, total)+ }++ // The page topic is navigation metadata, not a searchable page field:+ // "themes" occurs only in Love's topic, so it must match no page.+ got, total, err = svc.ListWikiPages(ctx, tenantID, kbID, "", "", "themes", 1, 30)+ if err != nil {+ t.Fatalf("list pages with topic keyword: %v", err)+ }+ if total != 0 || len(got) != 0 {+ t.Fatalf("keyword themes pages = %#v (total %d), want none", got, total)+ }++ topics, ttotal, err := svc.ListWikiTopics(ctx, tenantID, kbID, "")+ if err != nil {+ t.Fatalf("list topics: %v", err)+ }+ if ttotal != 2 || len(topics) != 2 {+ t.Fatalf("unfiltered topics = %#v (total %d), want General and Themes", topics, ttotal)+ }++ // A page-name keyword keeps the topic holding that page (Daisy under General).+ topics, ttotal, err = svc.ListWikiTopics(ctx, tenantID, kbID, "daisy")+ if err != nil {+ t.Fatalf("list topics with page keyword: %v", err)+ }+ if ttotal != 1 || len(topics) != 1 || topics[0].Topic != "General" || topics[0].PageCount != 2 {+ t.Fatalf("keyword daisy topics = %#v (total %d), want General with 2 pages", topics, ttotal)+ }++ // A keyword matching the topic path itself keeps that topic.+ topics, ttotal, err = svc.ListWikiTopics(ctx, tenantID, kbID, "themes")+ if err != nil {+ t.Fatalf("list topics with path keyword: %v", err)+ }+ if ttotal != 1 || len(topics) != 1 || topics[0].Topic != "Themes" {+ t.Fatalf("keyword themes topics = %#v (total %d), want Themes", topics, ttotal)+ }++ // No match anywhere yields an empty topic list.+ topics, ttotal, err = svc.ListWikiTopics(ctx, tenantID, kbID, "nonexistent")+ if err != nil {+ t.Fatalf("list topics with unknown keyword: %v", err)+ }+ if ttotal != 0 || len(topics) != 0 {+ t.Fatalf("keyword nonexistent topics = %#v (total %d), want none", topics, ttotal)+ }+}diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 6c0ecb4141..0498386d2c 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -159,9 +159,18 @@ type WikiPageItem struct {Summary string `json:"summary"`}-// ListWikiPages lists wiki pages for a dataset with optional page_type/topic-// filters and pagination.-func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic string, page, pageSize int) ([]WikiPageItem, int64, error) {+// wikiPageSelectFields are the engine fields ListWikiPages reads.+var wikiPageSelectFields = []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"}++// wikiPageOrderBy ranks wiki pages by connectivity then title.+func wikiPageOrderBy() *types.OrderByExpr {+ return (&types.OrderByExpr{}).Desc("outlinks_int").Asc("title_kwd")+}++// ListWikiPages lists wiki pages for a dataset with optional page_type/topic/+// keywords filters and pagination. keywords is a case-insensitive substring+// match over title/slug/summary, mirroring Python list_wiki_pages.+func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic, keywords string, page, pageSize int) ([]WikiPageItem, int64, error) {// Only surface the merged dataset-level pages. Each unique (page_type, slug)// can also have a per-document source row (available_int=0); without this// filter the same entity/concept would appear once per source doc. Python's@@ -178,31 +187,91 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, dafilter["topic_kwd"] = []string{kccommon.NormalizeWikiTopicPath(topic)}}offset := (page - 1) * pageSize- chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"},- offset, pageSize, (&types.OrderByExpr{}).Desc("outlinks_int").Asc("title_kwd"))- if err != nil {- return nil, 0, err+ if keywords == "" {+ chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,+ wikiPageSelectFields, offset, pageSize, wikiPageOrderBy())+ if err != nil {+ return nil, 0, err+ }+ return wikiPageItemsFromChunks(chunks), total, nil+ }+ // Wiki list search is intentionally metadata-based (substring over+ // title/slug/summary): those fields exist on every row and behave the same+ // across document-store backends, unlike full-text token fields. The engine+ // has no cross-field substring filter, so scan the filtered rows in+ // batches, match in memory, then apply the requested page slice — exactly+ // what Python list_wiki_pages does for keywords.+ kw := strings.ToLower(keywords)+ matched := make([]WikiPageItem, 0)+ const batchSize = 1000+ for scanOffset := 0; ; scanOffset += batchSize {+ batch, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,+ wikiPageSelectFields, scanOffset, batchSize, wikiPageOrderBy())+ if err != nil {+ return nil, 0, err+ }+ if len(batch) == 0 {+ break+ }+ for _, item := range wikiPageItemsFromChunks(batch) {+ if wikiPageItemMatchesKeyword(item, kw) {+ matched = append(matched, item)+ }+ }+ if len(batch) < batchSize {+ break+ }+ }+ total := int64(len(matched))+ if offset >= len(matched) {+ return []WikiPageItem{}, total, nil}+ end := offset + pageSize+ if end > len(matched) {+ end = len(matched)+ }+ return matched[offset:end], total, nil+}++// wikiPageItemsFromChunks maps engine rows to wiki page summaries.+func wikiPageItemsFromChunks(chunks []map[string]interface{}) []WikiPageItem {items := make([]WikiPageItem, 0, len(chunks))for _, c := range chunks {- pageType := firstStringValue(c["page_type_kwd"])- // slug_kwd is stored as the full "<page_type>/<slug>" form (Python- // contract); expose the bare slug to the frontend so it can be placed in- // a single URL path segment (gin :slug does not match '/').- bareSlug := firstStringValue(c["slug_kwd"])- if pageType != "" {- bareSlug = strings.TrimPrefix(bareSlug, pageType+"/")- }- items = append(items, WikiPageItem{- Slug: bareSlug,- Title: firstStringValue(c["title_kwd"]),- PageType: pageType,- Topic: kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"])),- Summary: firstStringValue(c["summary_with_weight"]),- })+ items = append(items, wikiPageItemFromChunk(c))+ }+ return items+}++func wikiPageItemFromChunk(c map[string]interface{}) WikiPageItem {+ pageType := firstStringValue(c["page_type_kwd"])+ // slug_kwd is stored as the full "<page_type>/<slug>" form (Python+ // contract); expose the bare slug to the frontend so it can be placed in+ // a single URL path segment (gin :slug does not match '/').+ bareSlug := firstStringValue(c["slug_kwd"])+ if pageType != "" {+ bareSlug = strings.TrimPrefix(bareSlug, pageType+"/")+ }+ return WikiPageItem{+ Slug: bareSlug,+ Title: firstStringValue(c["title_kwd"]),+ PageType: pageType,+ Topic: kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"])),+ Summary: firstStringValue(c["summary_with_weight"]),}- return items, total, nil+}++// wikiPageItemMatchesKeyword reports whether the lowercased keyword occurs in+// the page title, slug or summary.+func wikiPageItemMatchesKeyword(item WikiPageItem, lowerKeyword string) bool {+ return containsFold(item.Title, lowerKeyword) ||+ containsFold(item.Slug, lowerKeyword) ||+ containsFold(item.Summary, lowerKeyword)+}++// containsFold reports whether lowerKeyword (already lowercased) occurs in s,+// case-insensitively.+func containsFold(s, lowerKeyword string) bool {+ return strings.Contains(strings.ToLower(s), lowerKeyword)}// WikiPageDetail is the full wiki page payload. The content field is exposed as@@ -337,8 +406,10 @@ type WikiTopicItem struct {// ListWikiTopics aggregates materialized Wiki topic paths for a dataset. Topic// is the complete path and Title is its leaf segment; the frontend may derive a-// navigation tree by splitting Topic on '/'.-func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID string) ([]WikiTopicItem, int64, error) {+// navigation tree by splitting Topic on '/'. keywords keeps only topics whose+// own path matches or that hold a concept/entity page matching by+// title/slug/summary, mirroring Python list_wiki_topics.+func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID, keywords string) ([]WikiTopicItem, int64, error) {filter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{"concept", "entity", "topic"},@@ -358,6 +429,16 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, d}}items := aggregateWikiTopicItems(chunks)+ if keywords != "" {+ // The topic tree must survive searching by a page that lives under a+ // topic (e.g. "Daisy" under "General"), so topics also match when any+ // of their concept/entity pages matches the keyword.+ pageTopics, err := s.wikiTopicsWithKeywordPages(ctx, tenantID, datasetID, strings.ToLower(keywords))+ if err != nil {+ return nil, 0, err+ }+ items = filterWikiTopicItemsByKeyword(items, strings.ToLower(keywords), pageTopics)+ }// Sort topics by a deterministic rule. Plain UTF-8 byte order is chaotic for// CJK (it sorts by Unicode code point, unrelated to pinyin/stroke). We use a// CLDR-based collator (golang.org/x/text/collate) with the Chinese locale,@@ -369,6 +450,62 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, dreturn items, int64(len(items)), nil}+// wikiTopicsWithKeywordPages scans the dataset's concept/entity wiki pages and+// returns the normalized topic paths of the pages whose title, slug or summary+// contains the keyword.+func (s *DatasetArtifactService) wikiTopicsWithKeywordPages(ctx context.Context, tenantID, datasetID, lowerKeyword string) (map[string]bool, error) {+ filter := map[string]interface{}{+ "compile_kwd": []string{CompileKwdWikiPage},+ "page_type_kwd": []string{"concept", "entity"},+ }+ fields := []string{"topic_kwd", "title_kwd", "slug_kwd", "summary_with_weight"}+ matching := make(map[string]bool)+ const batchSize = 1000+ for offset := 0; ; offset += batchSize {+ batch, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, fields, offset, batchSize, nil)+ if err != nil {+ return nil, err+ }+ if len(batch) == 0 {+ break+ }+ for _, c := range batch {+ topic := kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"]))+ if topic == "" {+ continue+ }+ if containsFold(firstStringValue(c["title_kwd"]), lowerKeyword) ||+ containsFold(firstStringValue(c["slug_kwd"]), lowerKeyword) ||+ containsFold(firstStringValue(c["summary_with_weight"]), lowerKeyword) {+ matching[topic] = true+ }+ }+ if len(batch) < batchSize {+ break+ }+ }+ return matching, nil+}++// filterWikiTopicItemsByKeyword keeps topics whose path or leaf title contains+// the keyword, plus topics listed in pageMatched (topics holding a matching+// page).+func filterWikiTopicItemsByKeyword(items []WikiTopicItem, lowerKeyword string, pageMatched map[string]bool) []WikiTopicItem {+ filtered := make([]WikiTopicItem, 0, len(items))+ for _, item := range items {+ if pageMatched[item.Topic] || wikiTopicItemMatchesKeyword(item, lowerKeyword) {+ filtered = append(filtered, item)+ }+ }+ return filtered+}++// wikiTopicItemMatchesKeyword reports whether the lowercased keyword occurs in+// the topic path or its leaf title.+func wikiTopicItemMatchesKeyword(item WikiTopicItem, lowerKeyword string) bool {+ return containsFold(item.Topic, lowerKeyword) || containsFold(item.Title, lowerKeyword)+}+func aggregateWikiTopicItems(chunks []map[string]interface{}) []WikiTopicItem {type aggregate struct {topic stringdiff --git a/internal/service/dataset_artifact_topic_test.go b/internal/service/dataset_artifact_topic_test.goindex 60c397ffa3..05bb39d496 100644--- a/internal/service/dataset_artifact_topic_test.go+++ b/internal/service/dataset_artifact_topic_test.go@@ -1,6 +1,9 @@package service-import "testing"+import (+ "strings"+ "testing"+)func TestAggregateWikiTopicItemsUsesMaterializedPaths(t *testing.T) {items := aggregateWikiTopicItems([]map[string]interface{}{@@ -34,3 +37,57 @@ func TestAggregateWikiTopicItemsIgnoresPathCase(t *testing.T) {t.Fatalf("topics = %#v, want one case-insensitive path", items)}}++func TestWikiPageItemMatchesKeyword(t *testing.T) {+ item := WikiPageItem{+ Slug: "Daisy",+ Title: "Daisy",+ Topic: "General",+ Summary: "The field flower the Happy Prince met",+ }+ for _, kw := range []string{"daisy", "DAISY", "field flower"} {+ if !wikiPageItemMatchesKeyword(item, strings.ToLower(kw)) {+ t.Fatalf("keyword %q should match item %#v", kw, item)+ }+ }+ if wikiPageItemMatchesKeyword(item, "swallow") {+ t.Fatalf("keyword %q should not match item %#v", "swallow", item)+ }+ // The topic is navigation metadata, not a searchable page field.+ if wikiPageItemMatchesKeyword(item, "general") {+ t.Fatalf("keyword %q should not match item topic field %#v", "general", item)+ }+}++func TestWikiPageItemsFromChunksStripsPageTypePrefix(t *testing.T) {+ items := wikiPageItemsFromChunks([]map[string]interface{}{+ {+ "slug_kwd": "entity/Daisy",+ "title_kwd": "Daisy",+ "page_type_kwd": "entity",+ "topic_kwd": " General ",+ "summary_with_weight": "a summary",+ },+ })+ if len(items) != 1 {+ t.Fatalf("items = %#v, want 1", items)+ }+ got := items[0]+ if got.Slug != "Daisy" || got.Title != "Daisy" || got.PageType != "entity" || got.Topic != "General" || got.Summary != "a summary" {+ t.Fatalf("item = %#v, want bare slug and mapped fields", got)+ }+}++func TestFilterWikiTopicItemsByKeyword(t *testing.T) {+ items := []WikiTopicItem{+ {Topic: "General", Title: "General"},+ {Topic: "People/Writers", Title: "Writers"},+ {Topic: "Places", Title: "Places"},+ }+ // "General" holds the Daisy page; "People/Writers" matches by path; the+ // rest are dropped.+ got := filterWikiTopicItemsByKeyword(items, "writers", map[string]bool{"General": true})+ if len(got) != 2 || got[0].Topic != "General" || got[1].Topic != "People/Writers" {+ t.Fatalf("filtered = %#v, want General and People/Writers", got)+ }+}