Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(ima): traverse subfolders using list response media IDs
yushangcanfca1cd06f818
diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.goindex 6111ffc69..4bf3ba17a 100644--- a/internal/datasource/connector/ima/connector.go+++ b/internal/datasource/connector/ima/connector.go@@ -362,34 +362,21 @@ func listAllKBFiles(return nil, nil, err}for _, raw := range resp.KnowledgeList {- // Probe each entry: an entry with a non-empty folder_id is a- // folder; otherwise it's a knowledge item (file / note / etc.).- var probe struct {- FolderID string `json:"folder_id"`- MediaID string `json:"media_id"`+ var ki knowledgeInfo+ if err := json.Unmarshal(raw, &ki); err != nil || ki.MediaID == "" {+ continue // unrecognized shape, skip defensively}- _ = json.Unmarshal(raw, &probe)-- if probe.FolderID != "" && probe.MediaID == "" {- var fi folderInfo- if err := json.Unmarshal(raw, &fi); err != nil {- continue- }+ if ki.MediaType == mediaTypeFolder {+ // List entries identify folders by media_type. Their full+ // media_id (including the folder_ prefix) is the recursion key.child := cur.pathif child == "" {- child = fi.Name+ child = ki.Title} else {- child = cur.path + "/" + fi.Name+ child = cur.path + "/" + ki.Title}- folderPath[fi.FolderID] = child- stack = append(stack, todo{folderID: fi.FolderID, path: child})- continue- }- if probe.MediaID == "" {- continue // unrecognized shape, skip defensively- }- var ki knowledgeInfo- if err := json.Unmarshal(raw, &ki); err != nil {+ folderPath[ki.MediaID] = child+ stack = append(stack, todo{folderID: ki.MediaID, path: child})continue}out = append(out, walkedFile{diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.goindex 42970c8ca..7a99f7904 100644--- a/internal/datasource/connector/ima/connector_test.go+++ b/internal/datasource/connector/ima/connector_test.go@@ -407,10 +407,11 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {f.setKB("kb1",[]fakeFile{{MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"},- {MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"},+ {MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_f2", MediaType: mediaTypeMarkdown, Body: "deep"},},- fakeFolder{FolderID: "f1", Name: "Outer"},- fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"},+ fakeFolder{FolderID: "folder_f1", Name: "Outer"},+ fakeFolder{FolderID: "folder_f2", Name: "Inner", ParentFolderID: "folder_f1"},+ fakeFolder{FolderID: "folder_empty", Name: "Empty"},)items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})@@ -418,7 +419,13 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {t.Fatalf("FetchAll: %v", err)}- deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep"))+ if len(items) != 2 {+ t.Fatalf("expected only the root and nested files, got %s", describeItems(items))+ }+ deep := mustFindItem(t, items, logicalKey("kb1", "folder_f2", "Deep"))+ if string(deep.Content) != "deep" {+ t.Errorf("nested content = %q, want deep", deep.Content)+ }if deep.Metadata["folder_path"] != "Outer/Inner" {t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])}@@ -426,6 +433,74 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {if root.Metadata["folder_path"] != "" {t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"])}+ for _, id := range []string{"folder_f1", "folder_f2", "folder_empty"} {+ if got := f.callCount("get_knowledge_list:" + id); got != 1 {+ t.Errorf("list folder %q calls = %d, want 1 (with the folder_ prefix preserved)", id, got)+ }+ if got := f.callCount("get_media_info:" + id); got != 0 {+ t.Errorf("folder %q was fetched as a file %d times", id, got)+ }+ }+}++func TestFetchIncremental_NestedFolders(t *testing.T) {+ f := newFakeIMA(t)+ folders := []fakeFolder{+ {FolderID: "folder_outer", Name: "Outer"},+ {FolderID: "folder_inner", Name: "Inner", ParentFolderID: "folder_outer"},+ }+ file := fakeFile{+ MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_inner",+ MediaType: mediaTypeMarkdown, Body: "deep",+ }+ f.setKB("kb1", []fakeFile{file}, folders...)+ c := NewConnector()+ cfg := f.config("kb1")+ key := logicalKey("kb1", "folder_inner", "Deep")++ items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)+ if err != nil {+ t.Fatalf("first FetchIncremental: %v", err)+ }+ deep := mustFindItem(t, items, key)+ if deep.Metadata["folder_path"] != "Outer/Inner" {+ t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])+ }+ state := decodeCursor(t, cursor).KBLogical["kb1"]+ if len(state) != 1 || state[key] != file.MediaID {+ t.Fatalf("cursor should contain only the nested file, got %#v", state)+ }++ items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)+ if err != nil {+ t.Fatalf("unchanged FetchIncremental: %v", err)+ }+ if len(items) != 0 {+ t.Fatalf("unchanged nested file must not be emitted or tombstoned, got %s", describeItems(items))+ }+ if got := f.callCount("get_media_info:" + file.MediaID); got != 1 {+ t.Errorf("nested file fetched %d times, want 1", got)+ }+ if decodeCursor(t, cursor).KBLogical["kb1"][key] != file.MediaID {+ t.Fatal("unchanged nested file must remain in the cursor")+ }++ f.setKB("kb1", nil, folders...)+ items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)+ if err != nil {+ t.Fatalf("deleted FetchIncremental: %v", err)+ }+ if len(items) != 1 || !mustFindItem(t, items, key).IsDeleted {+ t.Fatalf("removed nested file must be tombstoned, got %s", describeItems(items))+ }+ if state := decodeCursor(t, cursor).KBLogical["kb1"]; len(state) != 0 {+ t.Errorf("empty folders must not be recorded as files, got %#v", state)+ }+ for _, folder := range folders {+ if got := f.callCount("get_media_info:" + folder.FolderID); got != 0 {+ t.Errorf("folder %q was fetched as a file %d times", folder.FolderID, got)+ }+ }}func TestListResources_ReturnsAddableBases(t *testing.T) {diff --git a/internal/datasource/connector/ima/fake_ima_test.go b/internal/datasource/connector/ima/fake_ima_test.goindex 1b503ab6c..368945513 100644--- a/internal/datasource/connector/ima/fake_ima_test.go+++ b/internal/datasource/connector/ima/fake_ima_test.go@@ -30,7 +30,7 @@ type fakeFile struct {Title stringParentFolderID string- // MediaType is what get_media_info reports for this file.+ // MediaType is reported by both get_knowledge_list and get_media_info.MediaType int32// Body is what the download URL serves.Body string@@ -190,6 +190,7 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {case "get_knowledge_list":kbID, folderID := str("knowledge_base_id"), str("folder_id")+ f.record("get_knowledge_list:" + folderID)f.mu.Lock()files := f.files[kbID][folderID]folders := f.folders[kbID][folderID]@@ -197,19 +198,21 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {var list []json.RawMessagefor _, folder := range folders {- b, _ := json.Marshal(folderInfo{- FolderID: folder.FolderID,- Name: folder.Name,- ParentFolderID: folder.ParentFolderID,+ // Folder entries use the same fields as files in list responses.+ b, _ := json.Marshal(map[string]interface{}{+ "media_id": folder.FolderID,+ "title": folder.Name,+ "parent_folder_id": folder.ParentFolderID,+ "media_type": 99,})list = append(list, b)}for _, file := range files {- // IMA omits media_type in list responses.b, _ := json.Marshal(map[string]interface{}{"media_id": file.MediaID,"title": file.Title,"parent_folder_id": file.ParentFolderID,+ "media_type": file.MediaType,})list = append(list, b)}diff --git a/internal/datasource/connector/ima/types.go b/internal/datasource/connector/ima/types.goindex 48569f2e3..3e4fadbc4 100644--- a/internal/datasource/connector/ima/types.go+++ b/internal/datasource/connector/ima/types.go@@ -156,10 +156,9 @@ type getKnowledgeBaseResp struct {Infos map[string]knowledgeBaseInfo `json:"infos"`}-// knowledgeInfo mirrors IMA's KnowledgeInfo (files, notes, etc.).-// IMPORTANT: IMA does not expose media_type in the list response, we must call-// get_media_info to discover it. However, folders are distinguished by presence-// of a non-empty folder_id — see folderInfo below.+// knowledgeInfo mirrors entries in IMA's knowledge_list (files, notes, folders).+// Folders have media_type=99 and use media_id as the folder ID for child listings.+// File download details are resolved separately through get_media_info.type knowledgeInfo struct {MediaID string `json:"media_id"`Title string `json:"title"`@@ -167,7 +166,7 @@ type knowledgeInfo struct {MediaType int32 `json:"media_type,omitempty"`}-// folderInfo mirrors IMA's FolderInfo.+// folderInfo mirrors IMA's FolderInfo in current_path, not knowledge_list.type folderInfo struct {FolderID string `json:"folder_id"`Name string `json:"name"`@@ -202,9 +201,8 @@ func (f folderInfo) FolderCount() int64 {}// getKnowledgeListResp — get_knowledge_list response.-// According to `references/api.md`, the response returns knowledge_list mixed-// with folders; we decode a loose JSON view so entries without a folder_id are-// treated as knowledge and those with folder_id are treated as folders.+// The response returns files and folders together in knowledge_list;+// media_type distinguishes folders from knowledge items.type getKnowledgeListResp struct {// KnowledgeList is a loose slice of raw JSON objects because IMA returns// files and folders in the same array. We iterate and inspect fields.@@ -309,6 +307,7 @@ const (mediaTypeVideo int32 = 16mediaTypeHTML int32 = 20mediaTypeEPUB int32 = 21+ mediaTypeFolder int32 = 99)// extensionForMediaType maps IMA media types to a file extension.