Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(dataset): honor keywords filter in Go wiki artifact list APIs (#19310)

euvref4b047844473
diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.go
index 4b1a282a3b..aa637b9dc7 100644
--- a/internal/handler/dataset_artifact.go
+++ b/internal/handler/dataset_artifact.go
@@ -105,6 +105,7 @@ func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {
datasetID := c.Param("dataset_id")
pageType := c.Query("page_type")
topic := c.Query("topic")
+ keywords := strings.TrimSpace(c.Query("keywords"))
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30"))
if page < 1 {
@@ -116,7 +117,7 @@ func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {
if pageSize > maxArtifactPageSize {
pageSize = maxArtifactPageSize
}
- items, total, err := h.svc.ListWikiPages(c.Request.Context(), tenantID, datasetID, pageType, topic, page, pageSize)
+ items, total, err := h.svc.ListWikiPages(c.Request.Context(), tenantID, datasetID, pageType, topic, keywords, page, pageSize)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
@@ -240,7 +241,8 @@ func (h *DatasetArtifactHandler) ListArtifactTopics(c *gin.Context) {
return
}
datasetID := c.Param("dataset_id")
- items, total, err := h.svc.ListWikiTopics(c.Request.Context(), tenantID, datasetID)
+ keywords := strings.TrimSpace(c.Query("keywords"))
+ items, total, err := h.svc.ListWikiTopics(c.Request.Context(), tenantID, datasetID, keywords)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
diff --git a/internal/service/dataset_artifact_keyword_integration_test.go b/internal/service/dataset_artifact_keyword_integration_test.go
new file mode 100644
index 0000000000..0f3f1728dc
--- /dev/null
+++ b/internal/service/dataset_artifact_keyword_integration_test.go
@@ -0,0 +1,152 @@
+//go:build integration
+// +build integration
+
+package service
+
+import (
+ "context"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+ "ragflow/internal/server"
+)
+
+// wikiIntRow builds a merged dataset-level wiki_page row shaped like the
+// knowledge-compiler writer output that the artifact APIs read.
+func wikiIntRow(id, pageType, slug, title, topic, summary string, outlinks int) map[string]interface{} {
+ fullSlug := pageType + "/" + slug
+ return map[string]interface{}{
+ "id": id,
+ "doc_id": id,
+ "available_int": 1,
+ "scope_kwd": "dataset",
+ "compile_kwd": CompileKwdWikiPage,
+ "slug_kwd": fullSlug,
+ "artifact_slug_kwd": fullSlug,
+ "title_kwd": title,
+ "page_type_kwd": pageType,
+ "topic_kwd": topic,
+ "summary_with_weight": summary,
+ "outlinks_int": outlinks,
+ }
+}
+
+// TestWikiKeywordFilter_Integration seeds wiki_page rows into a real document
+// engine and asserts ListWikiPages/ListWikiTopics keyword semantics (mirrors
+// Python list_wiki_pages/list_wiki_topics): pages match by title/slug/summary
+// only; topics match by their own path or by holding a matching page.
+//
+// Run with: bash build.sh --test-integration ./internal/service/...
+func TestWikiKeywordFilter_Integration(t *testing.T) {
+ if err := common.InitLogger("info", common.FileOutput{}, ""); err != nil {
+ t.Fatalf("init logger: %v", err)
+ }
+ if err := server.Init(""); err != nil {
+ t.Fatalf("init service config: %v", err)
+ }
+ if err := engine.InitDocEngine(context.Background()); err != nil {
+ t.Skipf("no document engine: %v", err)
+ }
+ de := engine.Get()
+ if de == nil {
+ t.Skip("no live document engine configured")
+ }
+
+ tenantID := "wikiint_t1"
+ kbID := "wikiint_kb1"
+ idx := wikiIndexName(tenantID)
+ ctx := t.Context()
+
+ rows := []map[string]interface{}{
+ wikiIntRow("wikiint_p1", "entity", "Daisy", "Daisy", "General", "the field flower", 3),
+ wikiIntRow("wikiint_p2", "entity", "Swallow", "Swallow", "General", "the little swallow", 5),
+ wikiIntRow("wikiint_p3", "concept", "Love", "Love", "Themes", "devotion and kindness", 7),
+ }
+ for _, row := range rows {
+ row["kb_id"] = kbID
+ }
+ // Best effort: the store may already exist from a previous run.
+ _ = de.CreateChunkStore(ctx, idx, kbID, 1024, "naive")
+ if _, err := de.InsertChunks(ctx, rows, idx, kbID); err != nil {
+ t.Fatalf("seed wiki rows: %v", err)
+ }
+ t.Cleanup(func() {
+ _, _ = de.DeleteChunks(context.Background(),
+ map[string]interface{}{"kb_id": []string{kbID}}, idx, kbID)
+ })
+
+ svc := NewDatasetArtifactService()
+
+ // Baseline: no keywords returns every seeded page.
+ all, total, err := svc.ListWikiPages(ctx, tenantID, kbID, "", "", "", 1, 30)
+ if err != nil {
+ t.Fatalf("list pages: %v", err)
+ }
+ if total != 3 || len(all) != 3 {
+ t.Fatalf("unfiltered pages = %#v (total %d), want 3", all, total)
+ }
+
+ // Keyword filters by title, case-insensitively.
+ got, total, err := svc.ListWikiPages(ctx, tenantID, kbID, "", "", "dAiSy", 1, 30)
+ if err != nil {
+ t.Fatalf("list pages with keyword: %v", err)
+ }
+ if total != 1 || len(got) != 1 || got[0].Title != "Daisy" || got[0].Slug != "Daisy" {
+ t.Fatalf("keyword dAiSy pages = %#v (total %d), want only Daisy", got, total)
+ }
+
+ // Keyword matches the summary too ("devotion" only occurs in Love's summary).
+ got, total, err = svc.ListWikiPages(ctx, tenantID, kbID, "", "", "devotion", 1, 30)
+ if err != nil {
+ t.Fatalf("list pages with summary keyword: %v", err)
+ }
+ if total != 1 || len(got) != 1 || got[0].Title != "Love" {
+ t.Fatalf("keyword devotion pages = %#v (total %d), want only Love", got, total)
+ }
+
+ // The page topic is navigation metadata, not a searchable page field:
+ // "themes" occurs only in Love's topic, so it must match no page.
+ got, total, err = svc.ListWikiPages(ctx, tenantID, kbID, "", "", "themes", 1, 30)
+ if err != nil {
+ t.Fatalf("list pages with topic keyword: %v", err)
+ }
+ if total != 0 || len(got) != 0 {
+ t.Fatalf("keyword themes pages = %#v (total %d), want none", got, total)
+ }
+
+ topics, ttotal, err := svc.ListWikiTopics(ctx, tenantID, kbID, "")
+ if err != nil {
+ t.Fatalf("list topics: %v", err)
+ }
+ if ttotal != 2 || len(topics) != 2 {
+ t.Fatalf("unfiltered topics = %#v (total %d), want General and Themes", topics, ttotal)
+ }
+
+ // A page-name keyword keeps the topic holding that page (Daisy under General).
+ topics, ttotal, err = svc.ListWikiTopics(ctx, tenantID, kbID, "daisy")
+ if err != nil {
+ t.Fatalf("list topics with page keyword: %v", err)
+ }
+ if ttotal != 1 || len(topics) != 1 || topics[0].Topic != "General" || topics[0].PageCount != 2 {
+ t.Fatalf("keyword daisy topics = %#v (total %d), want General with 2 pages", topics, ttotal)
+ }
+
+ // A keyword matching the topic path itself keeps that topic.
+ topics, ttotal, err = svc.ListWikiTopics(ctx, tenantID, kbID, "themes")
+ if err != nil {
+ t.Fatalf("list topics with path keyword: %v", err)
+ }
+ if ttotal != 1 || len(topics) != 1 || topics[0].Topic != "Themes" {
+ t.Fatalf("keyword themes topics = %#v (total %d), want Themes", topics, ttotal)
+ }
+
+ // No match anywhere yields an empty topic list.
+ topics, ttotal, err = svc.ListWikiTopics(ctx, tenantID, kbID, "nonexistent")
+ if err != nil {
+ t.Fatalf("list topics with unknown keyword: %v", err)
+ }
+ if ttotal != 0 || len(topics) != 0 {
+ t.Fatalf("keyword nonexistent topics = %#v (total %d), want none", topics, ttotal)
+ }
+}
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index 6c0ecb4141..0498386d2c 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -159,9 +159,18 @@ type WikiPageItem struct {
Summary string `json:"summary"`
}
-// ListWikiPages lists wiki pages for a dataset with optional page_type/topic
-// filters and pagination.
-func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic string, page, pageSize int) ([]WikiPageItem, int64, error) {
+// wikiPageSelectFields are the engine fields ListWikiPages reads.
+var wikiPageSelectFields = []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"}
+
+// wikiPageOrderBy ranks wiki pages by connectivity then title.
+func wikiPageOrderBy() *types.OrderByExpr {
+ return (&types.OrderByExpr{}).Desc("outlinks_int").Asc("title_kwd")
+}
+
+// ListWikiPages lists wiki pages for a dataset with optional page_type/topic/
+// keywords filters and pagination. keywords is a case-insensitive substring
+// match over title/slug/summary, mirroring Python list_wiki_pages.
+func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic, keywords string, page, pageSize int) ([]WikiPageItem, int64, error) {
// Only surface the merged dataset-level pages. Each unique (page_type, slug)
// can also have a per-document source row (available_int=0); without this
// filter the same entity/concept would appear once per source doc. Python's
@@ -178,31 +187,91 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, da
filter["topic_kwd"] = []string{kccommon.NormalizeWikiTopicPath(topic)}
}
offset := (page - 1) * pageSize
- chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
- []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"},
- offset, pageSize, (&types.OrderByExpr{}).Desc("outlinks_int").Asc("title_kwd"))
- if err != nil {
- return nil, 0, err
+ if keywords == "" {
+ chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
+ wikiPageSelectFields, offset, pageSize, wikiPageOrderBy())
+ if err != nil {
+ return nil, 0, err
+ }
+ return wikiPageItemsFromChunks(chunks), total, nil
+ }
+ // Wiki list search is intentionally metadata-based (substring over
+ // title/slug/summary): those fields exist on every row and behave the same
+ // across document-store backends, unlike full-text token fields. The engine
+ // has no cross-field substring filter, so scan the filtered rows in
+ // batches, match in memory, then apply the requested page slice — exactly
+ // what Python list_wiki_pages does for keywords.
+ kw := strings.ToLower(keywords)
+ matched := make([]WikiPageItem, 0)
+ const batchSize = 1000
+ for scanOffset := 0; ; scanOffset += batchSize {
+ batch, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
+ wikiPageSelectFields, scanOffset, batchSize, wikiPageOrderBy())
+ if err != nil {
+ return nil, 0, err
+ }
+ if len(batch) == 0 {
+ break
+ }
+ for _, item := range wikiPageItemsFromChunks(batch) {
+ if wikiPageItemMatchesKeyword(item, kw) {
+ matched = append(matched, item)
+ }
+ }
+ if len(batch) < batchSize {
+ break
+ }
+ }
+ total := int64(len(matched))
+ if offset >= len(matched) {
+ return []WikiPageItem{}, total, nil
}
+ end := offset + pageSize
+ if end > len(matched) {
+ end = len(matched)
+ }
+ return matched[offset:end], total, nil
+}
+
+// wikiPageItemsFromChunks maps engine rows to wiki page summaries.
+func wikiPageItemsFromChunks(chunks []map[string]interface{}) []WikiPageItem {
items := make([]WikiPageItem, 0, len(chunks))
for _, c := range chunks {
- pageType := firstStringValue(c["page_type_kwd"])
- // slug_kwd is stored as the full "<page_type>/<slug>" form (Python
- // contract); expose the bare slug to the frontend so it can be placed in
- // a single URL path segment (gin :slug does not match '/').
- bareSlug := firstStringValue(c["slug_kwd"])
- if pageType != "" {
- bareSlug = strings.TrimPrefix(bareSlug, pageType+"/")
- }
- items = append(items, WikiPageItem{
- Slug: bareSlug,
- Title: firstStringValue(c["title_kwd"]),
- PageType: pageType,
- Topic: kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"])),
- Summary: firstStringValue(c["summary_with_weight"]),
- })
+ items = append(items, wikiPageItemFromChunk(c))
+ }
+ return items
+}
+
+func wikiPageItemFromChunk(c map[string]interface{}) WikiPageItem {
+ pageType := firstStringValue(c["page_type_kwd"])
+ // slug_kwd is stored as the full "<page_type>/<slug>" form (Python
+ // contract); expose the bare slug to the frontend so it can be placed in
+ // a single URL path segment (gin :slug does not match '/').
+ bareSlug := firstStringValue(c["slug_kwd"])
+ if pageType != "" {
+ bareSlug = strings.TrimPrefix(bareSlug, pageType+"/")
+ }
+ return WikiPageItem{
+ Slug: bareSlug,
+ Title: firstStringValue(c["title_kwd"]),
+ PageType: pageType,
+ Topic: kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"])),
+ Summary: firstStringValue(c["summary_with_weight"]),
}
- return items, total, nil
+}
+
+// wikiPageItemMatchesKeyword reports whether the lowercased keyword occurs in
+// the page title, slug or summary.
+func wikiPageItemMatchesKeyword(item WikiPageItem, lowerKeyword string) bool {
+ return containsFold(item.Title, lowerKeyword) ||
+ containsFold(item.Slug, lowerKeyword) ||
+ containsFold(item.Summary, lowerKeyword)
+}
+
+// containsFold reports whether lowerKeyword (already lowercased) occurs in s,
+// case-insensitively.
+func containsFold(s, lowerKeyword string) bool {
+ return strings.Contains(strings.ToLower(s), lowerKeyword)
}
// WikiPageDetail is the full wiki page payload. The content field is exposed as
@@ -337,8 +406,10 @@ type WikiTopicItem struct {
// ListWikiTopics aggregates materialized Wiki topic paths for a dataset. Topic
// is the complete path and Title is its leaf segment; the frontend may derive a
-// navigation tree by splitting Topic on '/'.
-func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID string) ([]WikiTopicItem, int64, error) {
+// navigation tree by splitting Topic on '/'. keywords keeps only topics whose
+// own path matches or that hold a concept/entity page matching by
+// title/slug/summary, mirroring Python list_wiki_topics.
+func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID, keywords string) ([]WikiTopicItem, int64, error) {
filter := map[string]interface{}{
"compile_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{"concept", "entity", "topic"},
@@ -358,6 +429,16 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, d
}
}
items := aggregateWikiTopicItems(chunks)
+ if keywords != "" {
+ // The topic tree must survive searching by a page that lives under a
+ // topic (e.g. "Daisy" under "General"), so topics also match when any
+ // of their concept/entity pages matches the keyword.
+ pageTopics, err := s.wikiTopicsWithKeywordPages(ctx, tenantID, datasetID, strings.ToLower(keywords))
+ if err != nil {
+ return nil, 0, err
+ }
+ items = filterWikiTopicItemsByKeyword(items, strings.ToLower(keywords), pageTopics)
+ }
// Sort topics by a deterministic rule. Plain UTF-8 byte order is chaotic for
// CJK (it sorts by Unicode code point, unrelated to pinyin/stroke). We use a
// CLDR-based collator (golang.org/x/text/collate) with the Chinese locale,
@@ -369,6 +450,62 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, d
return items, int64(len(items)), nil
}
+// wikiTopicsWithKeywordPages scans the dataset's concept/entity wiki pages and
+// returns the normalized topic paths of the pages whose title, slug or summary
+// contains the keyword.
+func (s *DatasetArtifactService) wikiTopicsWithKeywordPages(ctx context.Context, tenantID, datasetID, lowerKeyword string) (map[string]bool, error) {
+ filter := map[string]interface{}{
+ "compile_kwd": []string{CompileKwdWikiPage},
+ "page_type_kwd": []string{"concept", "entity"},
+ }
+ fields := []string{"topic_kwd", "title_kwd", "slug_kwd", "summary_with_weight"}
+ matching := make(map[string]bool)
+ const batchSize = 1000
+ for offset := 0; ; offset += batchSize {
+ batch, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, fields, offset, batchSize, nil)
+ if err != nil {
+ return nil, err
+ }
+ if len(batch) == 0 {
+ break
+ }
+ for _, c := range batch {
+ topic := kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"]))
+ if topic == "" {
+ continue
+ }
+ if containsFold(firstStringValue(c["title_kwd"]), lowerKeyword) ||
+ containsFold(firstStringValue(c["slug_kwd"]), lowerKeyword) ||
+ containsFold(firstStringValue(c["summary_with_weight"]), lowerKeyword) {
+ matching[topic] = true
+ }
+ }
+ if len(batch) < batchSize {
+ break
+ }
+ }
+ return matching, nil
+}
+
+// filterWikiTopicItemsByKeyword keeps topics whose path or leaf title contains
+// the keyword, plus topics listed in pageMatched (topics holding a matching
+// page).
+func filterWikiTopicItemsByKeyword(items []WikiTopicItem, lowerKeyword string, pageMatched map[string]bool) []WikiTopicItem {
+ filtered := make([]WikiTopicItem, 0, len(items))
+ for _, item := range items {
+ if pageMatched[item.Topic] || wikiTopicItemMatchesKeyword(item, lowerKeyword) {
+ filtered = append(filtered, item)
+ }
+ }
+ return filtered
+}
+
+// wikiTopicItemMatchesKeyword reports whether the lowercased keyword occurs in
+// the topic path or its leaf title.
+func wikiTopicItemMatchesKeyword(item WikiTopicItem, lowerKeyword string) bool {
+ return containsFold(item.Topic, lowerKeyword) || containsFold(item.Title, lowerKeyword)
+}
+
func aggregateWikiTopicItems(chunks []map[string]interface{}) []WikiTopicItem {
type aggregate struct {
topic string
diff --git a/internal/service/dataset_artifact_topic_test.go b/internal/service/dataset_artifact_topic_test.go
index 60c397ffa3..05bb39d496 100644
--- a/internal/service/dataset_artifact_topic_test.go
+++ b/internal/service/dataset_artifact_topic_test.go
@@ -1,6 +1,9 @@
package service
-import "testing"
+import (
+ "strings"
+ "testing"
+)
func TestAggregateWikiTopicItemsUsesMaterializedPaths(t *testing.T) {
items := aggregateWikiTopicItems([]map[string]interface{}{
@@ -34,3 +37,57 @@ func TestAggregateWikiTopicItemsIgnoresPathCase(t *testing.T) {
t.Fatalf("topics = %#v, want one case-insensitive path", items)
}
}
+
+func TestWikiPageItemMatchesKeyword(t *testing.T) {
+ item := WikiPageItem{
+ Slug: "Daisy",
+ Title: "Daisy",
+ Topic: "General",
+ Summary: "The field flower the Happy Prince met",
+ }
+ for _, kw := range []string{"daisy", "DAISY", "field flower"} {
+ if !wikiPageItemMatchesKeyword(item, strings.ToLower(kw)) {
+ t.Fatalf("keyword %q should match item %#v", kw, item)
+ }
+ }
+ if wikiPageItemMatchesKeyword(item, "swallow") {
+ t.Fatalf("keyword %q should not match item %#v", "swallow", item)
+ }
+ // The topic is navigation metadata, not a searchable page field.
+ if wikiPageItemMatchesKeyword(item, "general") {
+ t.Fatalf("keyword %q should not match item topic field %#v", "general", item)
+ }
+}
+
+func TestWikiPageItemsFromChunksStripsPageTypePrefix(t *testing.T) {
+ items := wikiPageItemsFromChunks([]map[string]interface{}{
+ {
+ "slug_kwd": "entity/Daisy",
+ "title_kwd": "Daisy",
+ "page_type_kwd": "entity",
+ "topic_kwd": " General ",
+ "summary_with_weight": "a summary",
+ },
+ })
+ if len(items) != 1 {
+ t.Fatalf("items = %#v, want 1", items)
+ }
+ got := items[0]
+ if got.Slug != "Daisy" || got.Title != "Daisy" || got.PageType != "entity" || got.Topic != "General" || got.Summary != "a summary" {
+ t.Fatalf("item = %#v, want bare slug and mapped fields", got)
+ }
+}
+
+func TestFilterWikiTopicItemsByKeyword(t *testing.T) {
+ items := []WikiTopicItem{
+ {Topic: "General", Title: "General"},
+ {Topic: "People/Writers", Title: "Writers"},
+ {Topic: "Places", Title: "Places"},
+ }
+ // "General" holds the Daisy page; "People/Writers" matches by path; the
+ // rest are dropped.
+ got := filterWikiTopicItemsByKeyword(items, "writers", map[string]bool{"General": true})
+ if len(got) != 2 || got[0].Topic != "General" || got[1].Topic != "People/Writers" {
+ t.Fatalf("filtered = %#v, want General and People/Writers", got)
+ }
+}