Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(ima): traverse subfolders using list response media IDs

yushangcanfca1cd06f818
diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.go
index 6111ffc69..4bf3ba17a 100644
--- a/internal/datasource/connector/ima/connector.go
+++ b/internal/datasource/connector/ima/connector.go
@@ -362,34 +362,21 @@ func listAllKBFiles(
return nil, nil, err
}
for _, raw := range resp.KnowledgeList {
- // Probe each entry: an entry with a non-empty folder_id is a
- // folder; otherwise it's a knowledge item (file / note / etc.).
- var probe struct {
- FolderID string `json:"folder_id"`
- MediaID string `json:"media_id"`
+ var ki knowledgeInfo
+ if err := json.Unmarshal(raw, &ki); err != nil || ki.MediaID == "" {
+ continue // unrecognized shape, skip defensively
}
- _ = json.Unmarshal(raw, &probe)
-
- if probe.FolderID != "" && probe.MediaID == "" {
- var fi folderInfo
- if err := json.Unmarshal(raw, &fi); err != nil {
- continue
- }
+ if ki.MediaType == mediaTypeFolder {
+ // List entries identify folders by media_type. Their full
+ // media_id (including the folder_ prefix) is the recursion key.
child := cur.path
if child == "" {
- child = fi.Name
+ child = ki.Title
} else {
- child = cur.path + "/" + fi.Name
+ child = cur.path + "/" + ki.Title
}
- folderPath[fi.FolderID] = child
- stack = append(stack, todo{folderID: fi.FolderID, path: child})
- continue
- }
- if probe.MediaID == "" {
- continue // unrecognized shape, skip defensively
- }
- var ki knowledgeInfo
- if err := json.Unmarshal(raw, &ki); err != nil {
+ folderPath[ki.MediaID] = child
+ stack = append(stack, todo{folderID: ki.MediaID, path: child})
continue
}
out = append(out, walkedFile{
diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.go
index 42970c8ca..7a99f7904 100644
--- a/internal/datasource/connector/ima/connector_test.go
+++ b/internal/datasource/connector/ima/connector_test.go
@@ -407,10 +407,11 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
f.setKB("kb1",
[]fakeFile{
{MediaID: "m-root", Title: "Root", MediaType: mediaTypeMarkdown, Body: "root"},
- {MediaID: "m-deep", Title: "Deep", ParentFolderID: "f2", MediaType: mediaTypeMarkdown, Body: "deep"},
+ {MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_f2", MediaType: mediaTypeMarkdown, Body: "deep"},
},
- fakeFolder{FolderID: "f1", Name: "Outer"},
- fakeFolder{FolderID: "f2", Name: "Inner", ParentFolderID: "f1"},
+ fakeFolder{FolderID: "folder_f1", Name: "Outer"},
+ fakeFolder{FolderID: "folder_f2", Name: "Inner", ParentFolderID: "folder_f1"},
+ fakeFolder{FolderID: "folder_empty", Name: "Empty"},
)
items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
@@ -418,7 +419,13 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
t.Fatalf("FetchAll: %v", err)
}
- deep := mustFindItem(t, items, logicalKey("kb1", "f2", "Deep"))
+ if len(items) != 2 {
+ t.Fatalf("expected only the root and nested files, got %s", describeItems(items))
+ }
+ deep := mustFindItem(t, items, logicalKey("kb1", "folder_f2", "Deep"))
+ if string(deep.Content) != "deep" {
+ t.Errorf("nested content = %q, want deep", deep.Content)
+ }
if deep.Metadata["folder_path"] != "Outer/Inner" {
t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])
}
@@ -426,6 +433,74 @@ func TestFetchAll_ResolvesFolderPath(t *testing.T) {
if root.Metadata["folder_path"] != "" {
t.Errorf("root folder_path = %q, want empty", root.Metadata["folder_path"])
}
+ for _, id := range []string{"folder_f1", "folder_f2", "folder_empty"} {
+ if got := f.callCount("get_knowledge_list:" + id); got != 1 {
+ t.Errorf("list folder %q calls = %d, want 1 (with the folder_ prefix preserved)", id, got)
+ }
+ if got := f.callCount("get_media_info:" + id); got != 0 {
+ t.Errorf("folder %q was fetched as a file %d times", id, got)
+ }
+ }
+}
+
+func TestFetchIncremental_NestedFolders(t *testing.T) {
+ f := newFakeIMA(t)
+ folders := []fakeFolder{
+ {FolderID: "folder_outer", Name: "Outer"},
+ {FolderID: "folder_inner", Name: "Inner", ParentFolderID: "folder_outer"},
+ }
+ file := fakeFile{
+ MediaID: "m-deep", Title: "Deep", ParentFolderID: "folder_inner",
+ MediaType: mediaTypeMarkdown, Body: "deep",
+ }
+ f.setKB("kb1", []fakeFile{file}, folders...)
+ c := NewConnector()
+ cfg := f.config("kb1")
+ key := logicalKey("kb1", "folder_inner", "Deep")
+
+ items, cursor, err := c.FetchIncremental(context.Background(), cfg, nil)
+ if err != nil {
+ t.Fatalf("first FetchIncremental: %v", err)
+ }
+ deep := mustFindItem(t, items, key)
+ if deep.Metadata["folder_path"] != "Outer/Inner" {
+ t.Errorf("folder_path = %q, want Outer/Inner", deep.Metadata["folder_path"])
+ }
+ state := decodeCursor(t, cursor).KBLogical["kb1"]
+ if len(state) != 1 || state[key] != file.MediaID {
+ t.Fatalf("cursor should contain only the nested file, got %#v", state)
+ }
+
+ items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)
+ if err != nil {
+ t.Fatalf("unchanged FetchIncremental: %v", err)
+ }
+ if len(items) != 0 {
+ t.Fatalf("unchanged nested file must not be emitted or tombstoned, got %s", describeItems(items))
+ }
+ if got := f.callCount("get_media_info:" + file.MediaID); got != 1 {
+ t.Errorf("nested file fetched %d times, want 1", got)
+ }
+ if decodeCursor(t, cursor).KBLogical["kb1"][key] != file.MediaID {
+ t.Fatal("unchanged nested file must remain in the cursor")
+ }
+
+ f.setKB("kb1", nil, folders...)
+ items, cursor, err = c.FetchIncremental(context.Background(), cfg, cursor)
+ if err != nil {
+ t.Fatalf("deleted FetchIncremental: %v", err)
+ }
+ if len(items) != 1 || !mustFindItem(t, items, key).IsDeleted {
+ t.Fatalf("removed nested file must be tombstoned, got %s", describeItems(items))
+ }
+ if state := decodeCursor(t, cursor).KBLogical["kb1"]; len(state) != 0 {
+ t.Errorf("empty folders must not be recorded as files, got %#v", state)
+ }
+ for _, folder := range folders {
+ if got := f.callCount("get_media_info:" + folder.FolderID); got != 0 {
+ t.Errorf("folder %q was fetched as a file %d times", folder.FolderID, got)
+ }
+ }
}
func TestListResources_ReturnsAddableBases(t *testing.T) {
diff --git a/internal/datasource/connector/ima/fake_ima_test.go b/internal/datasource/connector/ima/fake_ima_test.go
index 1b503ab6c..368945513 100644
--- a/internal/datasource/connector/ima/fake_ima_test.go
+++ b/internal/datasource/connector/ima/fake_ima_test.go
@@ -30,7 +30,7 @@ type fakeFile struct {
Title string
ParentFolderID string
- // MediaType is what get_media_info reports for this file.
+ // MediaType is reported by both get_knowledge_list and get_media_info.
MediaType int32
// Body is what the download URL serves.
Body string
@@ -190,6 +190,7 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {
case "get_knowledge_list":
kbID, folderID := str("knowledge_base_id"), str("folder_id")
+ f.record("get_knowledge_list:" + folderID)
f.mu.Lock()
files := f.files[kbID][folderID]
folders := f.folders[kbID][folderID]
@@ -197,19 +198,21 @@ func (f *fakeIMA) handleAPI(w http.ResponseWriter, r *http.Request) {
var list []json.RawMessage
for _, folder := range folders {
- b, _ := json.Marshal(folderInfo{
- FolderID: folder.FolderID,
- Name: folder.Name,
- ParentFolderID: folder.ParentFolderID,
+ // Folder entries use the same fields as files in list responses.
+ b, _ := json.Marshal(map[string]interface{}{
+ "media_id": folder.FolderID,
+ "title": folder.Name,
+ "parent_folder_id": folder.ParentFolderID,
+ "media_type": 99,
})
list = append(list, b)
}
for _, file := range files {
- // IMA omits media_type in list responses.
b, _ := json.Marshal(map[string]interface{}{
"media_id": file.MediaID,
"title": file.Title,
"parent_folder_id": file.ParentFolderID,
+ "media_type": file.MediaType,
})
list = append(list, b)
}
diff --git a/internal/datasource/connector/ima/types.go b/internal/datasource/connector/ima/types.go
index 48569f2e3..3e4fadbc4 100644
--- a/internal/datasource/connector/ima/types.go
+++ b/internal/datasource/connector/ima/types.go
@@ -156,10 +156,9 @@ type getKnowledgeBaseResp struct {
Infos map[string]knowledgeBaseInfo `json:"infos"`
}
-// knowledgeInfo mirrors IMA's KnowledgeInfo (files, notes, etc.).
-// IMPORTANT: IMA does not expose media_type in the list response, we must call
-// get_media_info to discover it. However, folders are distinguished by presence
-// of a non-empty folder_id — see folderInfo below.
+// knowledgeInfo mirrors entries in IMA's knowledge_list (files, notes, folders).
+// Folders have media_type=99 and use media_id as the folder ID for child listings.
+// File download details are resolved separately through get_media_info.
type knowledgeInfo struct {
MediaID string `json:"media_id"`
Title string `json:"title"`
@@ -167,7 +166,7 @@ type knowledgeInfo struct {
MediaType int32 `json:"media_type,omitempty"`
}
-// folderInfo mirrors IMA's FolderInfo.
+// folderInfo mirrors IMA's FolderInfo in current_path, not knowledge_list.
type folderInfo struct {
FolderID string `json:"folder_id"`
Name string `json:"name"`
@@ -202,9 +201,8 @@ func (f folderInfo) FolderCount() int64 {
}
// getKnowledgeListResp — get_knowledge_list response.
-// According to `references/api.md`, the response returns knowledge_list mixed
-// with folders; we decode a loose JSON view so entries without a folder_id are
-// treated as knowledge and those with folder_id are treated as folders.
+// The response returns files and folders together in knowledge_list;
+// media_type distinguishes folders from knowledge items.
type getKnowledgeListResp struct {
// KnowledgeList is a loose slice of raw JSON objects because IMA returns
// files and folders in the same array. We iterate and inspect fields.
@@ -309,6 +307,7 @@ const (
mediaTypeVideo int32 = 16
mediaTypeHTML int32 = 20
mediaTypeEPUB int32 = 21
+ mediaTypeFolder int32 = 99
)
// extensionForMediaType maps IMA media types to a file extension.