Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(client): expose knowledge-base parser engine rules (#4753)

jiangjiang248ac282d77cdf6
diff --git a/client/knowledgebase.go b/client/knowledgebase.go
index d2b23aef3..9746b6870 100644
--- a/client/knowledgebase.go
+++ b/client/knowledgebase.go
@@ -54,6 +54,8 @@ type ChunkingConfig struct {
ChunkSize int `json:"chunk_size"` // Chunk size
ChunkOverlap int `json:"chunk_overlap"` // Overlap size
Separators []string `json:"separators"` // Separators
+ // ParserEngineRules selects the parser engine for each group of file types.
+ ParserEngineRules []ParserEngineRule `json:"parser_engine_rules,omitempty"`
}
// IndexingStrategy selects which indexing pipelines are active.
diff --git a/client/knowledgebase_parser_engine_rules_test.go b/client/knowledgebase_parser_engine_rules_test.go
new file mode 100644
index 000000000..d196f451e
--- /dev/null
+++ b/client/knowledgebase_parser_engine_rules_test.go
@@ -0,0 +1,127 @@
+package client
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "testing"
+)
+
+func TestChunkingConfigParserEngineRulesJSONRoundTrip(t *testing.T) {
+ const body = `{
+ "chunk_size": 512,
+ "chunk_overlap": 50,
+ "separators": ["\n\n"],
+ "parser_engine_rules": [
+ {"file_types": ["pdf", "docx"], "engine": "anydoc"},
+ {"file_types": ["txt"], "engine": "builtin"}
+ ]
+ }`
+ var config ChunkingConfig
+ if err := json.Unmarshal([]byte(body), &config); err != nil {
+ t.Fatal(err)
+ }
+ encoded, err := json.Marshal(config)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var want, got map[string]any
+ if err := json.Unmarshal([]byte(body), &want); err != nil {
+ t.Fatal(err)
+ }
+ if err := json.Unmarshal(encoded, &got); err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("chunking config changed during JSON round trip: got %#v, want %#v", got, want)
+ }
+}
+
+func TestKnowledgeBaseParserEngineRulesRequests(t *testing.T) {
+ rules := []ParserEngineRule{
+ {FileTypes: []string{"pdf", "docx"}, Engine: "anydoc"},
+ {FileTypes: []string{"txt"}, Engine: "builtin"},
+ }
+ config := ChunkingConfig{ChunkSize: 512, ChunkOverlap: 50, ParserEngineRules: rules}
+ for _, method := range []string{http.MethodPost, http.MethodGet, http.MethodPut} {
+ t.Run(method, func(t *testing.T) {
+ path := "/api/v1/knowledge-bases/kb-1"
+ if method == http.MethodPost {
+ path = "/api/v1/knowledge-bases"
+ }
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != method || r.URL.Path != path {
+ t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+ }
+ if method != http.MethodGet {
+ var request struct {
+ ChunkingConfig ChunkingConfig `json:"chunking_config"`
+ Config *KnowledgeBaseConfig `json:"config"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Error(err)
+ return
+ }
+ got := request.ChunkingConfig.ParserEngineRules
+ if method == http.MethodPut {
+ if request.Config == nil {
+ t.Error("update request is missing config")
+ return
+ }
+ got = request.Config.ChunkingConfig.ParserEngineRules
+ }
+ if !reflect.DeepEqual(got, rules) {
+ t.Errorf("request parser engine rules: got %#v, want %#v", got, rules)
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(KnowledgeBaseResponse{
+ Success: true,
+ Data: KnowledgeBase{ID: "kb-1", ChunkingConfig: config},
+ }); err != nil {
+ t.Error(err)
+ }
+ }))
+ defer srv.Close()
+ client := NewClient(srv.URL)
+ var kb *KnowledgeBase
+ var err error
+ switch method {
+ case http.MethodPost:
+ kb, err = client.CreateKnowledgeBase(context.Background(), &KnowledgeBase{
+ Name: "parser-config", ChunkingConfig: config,
+ })
+ case http.MethodGet:
+ kb, err = client.GetKnowledgeBase(context.Background(), "kb-1")
+ case http.MethodPut:
+ kb, err = client.UpdateKnowledgeBase(context.Background(), "kb-1", &UpdateKnowledgeBaseRequest{
+ Config: &KnowledgeBaseConfig{ChunkingConfig: config},
+ })
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(kb.ChunkingConfig.ParserEngineRules, rules) {
+ t.Errorf("response parser engine rules: got %#v, want %#v", kb.ChunkingConfig.ParserEngineRules, rules)
+ }
+ })
+ }
+}
+
+func TestChunkingConfigOmitsUnsetParserEngineRules(t *testing.T) {
+ for _, rules := range [][]ParserEngineRule{nil, {}} {
+ encoded, err := json.Marshal(ChunkingConfig{ParserEngineRules: rules})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var fields map[string]json.RawMessage
+ if err := json.Unmarshal(encoded, &fields); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := fields["parser_engine_rules"]; ok {
+ t.Errorf("unset parser engine rules should be omitted: %s", encoded)
+ }
+ }
+}