Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(document): honour apply_kb so reparse applies the dataset config (#20149)

Bruce-Yii66bd0554b394
diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.go
index 7acc009553..c6d876ab01 100644
--- a/internal/service/component_scoped_parser_config.go
+++ b/internal/service/component_scoped_parser_config.go
@@ -79,7 +79,36 @@ func ApplyComponentScopedParserConfig(
func cloneJSONMap(in map[string]any) map[string]any {
out := make(map[string]any, len(in))
for k, v := range in {
- out[k] = v
+ out[k] = cloneValue(v)
}
return out
}
+
+func cloneValue(v any) any {
+ switch val := v.(type) {
+ case nil:
+ return nil
+ case map[string]any:
+ return cloneJSONMap(val)
+ case entity.JSONMap:
+ out := make(map[string]any, len(val))
+ for k, vv := range val {
+ out[k] = cloneValue(vv)
+ }
+ return out
+ case []any:
+ out := make([]any, len(val))
+ for i, vv := range val {
+ out[i] = cloneValue(vv)
+ }
+ return out
+ case []map[string]any:
+ out := make([]map[string]any, len(val))
+ for i, vv := range val {
+ out[i] = cloneValue(vv).(map[string]any)
+ }
+ return out
+ default:
+ return v
+ }
+}
diff --git a/internal/service/document/deep_copy.go b/internal/service/document/deep_copy.go
new file mode 100644
index 0000000000..5b2aed9d73
--- /dev/null
+++ b/internal/service/document/deep_copy.go
@@ -0,0 +1,51 @@
+package document
+
+import "ragflow/internal/entity"
+
+// deepCopyJSONMap returns an independent copy of in, including all nested
+// maps and slices. A nil input yields nil.
+func deepCopyJSONMap(in entity.JSONMap) entity.JSONMap {
+ if in == nil {
+ return nil
+ }
+ out := make(entity.JSONMap, len(in))
+ for k, v := range in {
+ out[k] = deepCopyValue(v)
+ }
+ return out
+}
+
+// deepCopyValue recursively copies maps and slices; scalar values are shared
+// (they are immutable from the caller's perspective).
+func deepCopyValue(v any) any {
+ switch val := v.(type) {
+ case nil:
+ return nil
+ case entity.JSONMap:
+ out := make(entity.JSONMap, len(val))
+ for k, vv := range val {
+ out[k] = deepCopyValue(vv)
+ }
+ return out
+ case map[string]any:
+ out := make(map[string]any, len(val))
+ for k, vv := range val {
+ out[k] = deepCopyValue(vv)
+ }
+ return out
+ case []any:
+ out := make([]any, len(val))
+ for i, vv := range val {
+ out[i] = deepCopyValue(vv)
+ }
+ return out
+ case []map[string]any:
+ out := make([]map[string]any, len(val))
+ for i, vv := range val {
+ out[i] = deepCopyValue(vv).(map[string]any)
+ }
+ return out
+ default:
+ return v
+ }
+}
diff --git a/internal/service/document/document.go b/internal/service/document/document.go
index 342f0721b9..d2847cde0b 100644
--- a/internal/service/document/document.go
+++ b/internal/service/document/document.go
@@ -226,8 +226,15 @@ type IngestDocumentRequest struct {
// StartParseOptions controls StartParseDocuments behavior.
type StartParseOptions struct {
- // ApplyKB merges the knowledge base's parser_config (llm_id, metadata)
- // into the document before parsing.
+ // ApplyKB scopes the dataset's parser_config (llm_id and the modular
+ // metadata object: enabled, metadata, built_in_metadata) onto the document's
+ // existing Extractor nodes before parsing. The dataset llm_id only fills
+ // Extractor/Compiler nodes whose llm_id is empty; explicit per-node values
+ // are preserved and no top-level llm_id is written. When the dataset sets
+ // modular metadata it replaces each Extractor node's metadata; when it
+ // sets none, each node's metadata is reset to the disabled default
+ // ({enabled:false, metadata:[], built_in_metadata:[]}) with no top-level
+ // metadata key left behind.
ApplyKB bool
// RerunWithDelete clears prior chunks/tasks/counters before reparsing.
RerunWithDelete bool
diff --git a/internal/service/document/document_applykb_chain_test.go b/internal/service/document/document_applykb_chain_test.go
new file mode 100644
index 0000000000..9ba903fa23
--- /dev/null
+++ b/internal/service/document/document_applykb_chain_test.go
@@ -0,0 +1,444 @@
+package document
+
+// Real writer-to-reader ApplyKB chain tests (YU-366).
+//
+// These tests never hand-craft the dataset's parser_config. The dataset side
+// always goes through a real writer entry point:
+// - DatasetService.UpdateMetadataConfig (dataset metadata API), or
+// - DatasetService.CreateDataset (dataset creation from a pipeline DSL),
+// after which the KB row is reloaded from the database and only then fed to
+// DocumentService.Ingest(ApplyKB:true). The document row is reloaded as well
+// and asserted through the pipeline's own consumer
+// (common.ExtractorMetadataConfig) — dataset service -> DB -> Ingest ->
+// DB -> consumer, no shortcuts.
+
+import (
+ "encoding/json"
+ "sort"
+ "strings"
+ "testing"
+
+ "gorm.io/gorm"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/permission"
+ "ragflow/internal/service"
+ datasetsvc "ragflow/internal/service/dataset"
+)
+
+func mustJSON(t *testing.T, v any) string {
+ t.Helper()
+ data, err := json.Marshal(v)
+ if err != nil {
+ t.Fatalf("marshal parser_config: %v", err)
+ }
+ return string(data)
+}
+
+// firstExtractorMetadata returns the modular metadata of the first (sorted)
+// Extractor node carrying one, mirroring the deterministic reader contract.
+func firstExtractorMetadata(t *testing.T, parserConfig entity.JSONMap) map[string]any {
+ t.Helper()
+ var keys []string
+ for k := range parserConfig {
+ lower := strings.ToLower(k)
+ if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
+ keys = append(keys, k)
+ }
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ node, ok := parserConfig[k].(map[string]any)
+ if !ok {
+ continue
+ }
+ if meta, ok := node["metadata"].(map[string]any); ok {
+ return meta
+ }
+ }
+ t.Fatalf("no Extractor node with modular metadata in %#v", parserConfig)
+ return nil
+}
+
+func extractorNodeLLMID(t *testing.T, parserConfig entity.JSONMap, node string) string {
+ t.Helper()
+ params, ok := parserConfig[node].(map[string]any)
+ if !ok {
+ t.Fatalf("node %s missing or not a map: %#v", node, parserConfig[node])
+ }
+ llmID, _ := params["llm_id"].(string)
+ return llmID
+}
+
+// newDatasetService builds a DatasetService the same way production code does.
+// Only the kb/tenant DAOs are exercised by these tests.
+func newDatasetService() *datasetsvc.DatasetService {
+ return datasetsvc.NewDatasetService()
+}
+
+// migrateChainModelTables adds the tables the real dataset entry points touch
+// (model resolution, connectors, canvas) beyond the document package's base
+// migration. Test scaffolding only; the dataset writes themselves stay real.
+func migrateChainModelTables(t *testing.T, db *gorm.DB) {
+ t.Helper()
+ if err := db.AutoMigrate(
+ &entity.Connector{},
+ &entity.Connector2Kb{},
+ &entity.SyncLogs{},
+ &entity.TenantModelProvider{},
+ &entity.TenantModelInstance{},
+ &entity.TenantModel{},
+ &entity.UserCanvas{},
+ &entity.PipelineOperationLog{},
+ ); err != nil {
+ t.Fatalf("migrate chain tables: %v", err)
+ }
+}
+
+// TestIngest_ApplyKB_RealDatasetAPIChain drives the real dataset metadata API
+// (UpdateMetadataConfig with enabled=true, a custom author field and a
+// built-in field), reloads the KB from the database, then Ingests with
+// ApplyKB:true and asserts the persisted document config through the
+// pipeline's own consumer.
+func TestIngest_ApplyKB_RealDatasetAPIChain(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ // Every real dataset DSL defines an Extractor node; the writer under test
+ // scopes onto it. Scaffolding the empty node shape is not faking config:
+ // no metadata values are invented here.
+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update(
+ "parser_config", entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{},
+ },
+ ).Error; err != nil {
+ t.Fatalf("seed kb extractor node: %v", err)
+ }
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "chunk_token_num": 128,
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ ctx := t.Context()
+ enabled := true
+ if _, code, err := newDatasetService().UpdateMetadataConfig(ctx, "kb-1", "user-1", &service.MetadataConfigRequest{
+ Enabled: &enabled,
+ Metadata: []service.MetadataConfigField{{Key: "author", Type: "string"}},
+ BuiltInMetadata: []service.MetadataConfigField{{Key: "document_name", Type: "string"}},
+ }); err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateMetadataConfig err=%v code=%v", err, code)
+ }
+
+ // Reload the KB: this is the real stored shape, not a fixture.
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("reload kb: %v", err)
+ }
+ t.Logf("stored kb parser_config: %s", mustJSON(t, kb.ParserConfig))
+ kbMeta := firstExtractorMetadata(t, kb.ParserConfig)
+ if enabled, _ := kbMeta["enabled"].(bool); !enabled {
+ t.Fatalf("stored dataset metadata.enabled = %v, want true", kbMeta["enabled"])
+ }
+ if !metadataFieldKeys(kbMeta)["author"] {
+ t.Fatalf("stored dataset metadata missing author: %#v", kbMeta["metadata"])
+ }
+ kbLLMID := extractorNodeLLMID(t, kb.ParserConfig, "Extractor:AutoExtractDefault")
+ if kbLLMID == "" {
+ t.Fatalf("stored dataset node llm_id is empty; tenant default was not applied")
+ }
+ t.Logf("stored dataset node llm_id: %s", kbLLMID)
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: true,
+ ApplyKB: true,
+ })
+ if err != nil {
+ t.Fatalf("Ingest: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload doc: %v", err)
+ }
+ t.Logf("persisted doc parser_config: %s", mustJSON(t, got.ParserConfig))
+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)
+ }
+ if auto["llm_id"] != kbLLMID {
+ t.Fatalf("persisted node llm_id = %v, want dataset value %v", auto["llm_id"], kbLLMID)
+ }
+ meta, ok := auto["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("persisted node metadata missing: %#v", auto)
+ }
+ if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("persisted metadata.enabled = %v, want true", meta["enabled"])
+ }
+ if !metadataFieldKeys(meta)["author"] {
+ t.Fatalf("persisted metadata missing author: %#v", meta["metadata"])
+ }
+ if _, ok := got.ParserConfig["llm_id"]; ok {
+ t.Fatalf("top-level llm_id written: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("flat enable_metadata written: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {
+ t.Fatalf("chunk_token_num dropped: %#v", got.ParserConfig)
+ }
+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))
+ if !ok {
+ t.Fatalf("consumer cannot read persisted extractor metadata: %#v", got.ParserConfig)
+ }
+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {
+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])
+ }
+}
+
+// TestIngest_ApplyKB_DisabledDatasetAPIChain pins the real disabled path: a
+// dataset explicitly configured with enabled=false must not enable document
+// auto-metadata.
+func TestIngest_ApplyKB_DisabledDatasetAPIChain(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update(
+ "parser_config", entity.JSONMap{
+ "Extractor:AutoExtractDefault": map[string]any{},
+ },
+ ).Error; err != nil {
+ t.Fatalf("seed kb extractor node: %v", err)
+ }
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "chunk_token_num": 128,
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ ctx := t.Context()
+ disabled := false
+ if _, code, err := newDatasetService().UpdateMetadataConfig(ctx, "kb-1", "user-1", &service.MetadataConfigRequest{
+ Enabled: &disabled,
+ }); err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateMetadataConfig err=%v code=%v", err, code)
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: true,
+ ApplyKB: true,
+ })
+ if err != nil {
+ t.Fatalf("Ingest: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload doc: %v", err)
+ }
+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)
+ }
+ meta, ok := auto["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("node metadata missing: %#v", auto)
+ }
+ if enabled, _ := meta["enabled"].(bool); enabled {
+ t.Fatalf("node metadata.enabled = %v, want false", meta["enabled"])
+ }
+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))
+ if !ok {
+ t.Fatalf("consumer cannot read node extractor metadata: %#v", got.ParserConfig)
+ }
+ if enabled, _ := consumerMeta["enabled"].(bool); enabled {
+ t.Fatalf("consumer metadata.enabled = %v, want false", consumerMeta["enabled"])
+ }
+}
+
+// TestIngest_ApplyKB_RealCreatedDatasetChain covers the stored shape where
+// the dataset writer persisted component-scoped config with no top-level
+// "metadata" key (dataset creation scopes the DSL config before persisting).
+// The seed goes through the real DatasetService.CreateDataset entry point.
+func TestIngest_ApplyKB_RealCreatedDatasetChain(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+ migrateChainModelTables(t, db)
+ // Mirror the dataset package's own fixture: the membership carries the
+ // resolved embedding model id, keeping CreateDataset on its real path
+ // (no embedding override).
+ if err := dao.DB.Model(&entity.UserTenant{}).Where("user_id = ? AND tenant_id = ?", "user-1", "tenant-1").Update(
+ "tenant_embd_id", "embd-1",
+ ).Error; err != nil {
+ t.Fatalf("set membership embd id: %v", err)
+ }
+
+ ctx := t.Context()
+ parserID := "general"
+ parseType := 1
+ team := "team"
+ result, code, err := newDatasetService().CreateDataset(ctx, &service.CreateDatasetRequest{
+ Name: "ds-chain",
+ ParserID: &parserID,
+ ParseType: &parseType,
+ Permission: &team,
+ ParserConfig: map[string]interface{}{
+ "Extractor:AutoExtractDefault": map[string]interface{}{
+ "llm_id": "",
+ "metadata": map[string]interface{}{
+ "enabled": true,
+ "metadata": []interface{}{
+ map[string]interface{}{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []interface{}{
+ map[string]interface{}{"key": "document_name", "type": "string"},
+ },
+ },
+ },
+ },
+ }, permission.Subject{UserID: "user-1", TenantID: "tenant-1"})
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("CreateDataset err=%v code=%v", err, code)
+ }
+ kbID, _ := result["id"].(string)
+ if kbID == "" {
+ t.Fatalf("created dataset has no id: %#v", result)
+ }
+
+ // Reload the KB: record the real stored shape the writer produced.
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, kbID)
+ if err != nil {
+ t.Fatalf("reload kb: %v", err)
+ }
+ t.Logf("stored kb parser_config: %s", mustJSON(t, kb.ParserConfig))
+ if _, ok := kb.ParserConfig["metadata"]; ok {
+ t.Fatalf("stored kb has top-level metadata, want component-scoped only: %s", mustJSON(t, kb.ParserConfig))
+ }
+ kbMeta := firstExtractorMetadata(t, kb.ParserConfig)
+ if enabled, _ := kbMeta["enabled"].(bool); !enabled {
+ t.Fatalf("stored dataset metadata.enabled = %v, want true", kbMeta["enabled"])
+ }
+ if !metadataFieldKeys(kbMeta)["author"] {
+ t.Fatalf("stored dataset metadata missing author: %#v", kbMeta["metadata"])
+ }
+ kbLLMID := ""
+ for _, k := range []string{"Extractor:AutoExtractDefault"} {
+ kbLLMID = extractorNodeLLMID(t, kb.ParserConfig, k)
+ }
+ if kbLLMID == "" {
+ t.Fatalf("stored dataset node llm_id is empty; tenant default was not applied")
+ }
+ t.Logf("stored dataset node llm_id: %s", kbLLMID)
+
+ insertTestDoc(t, "doc-1", kbID, 9, 4)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "chunk_token_num": 128,
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+
+ code, err = svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: true,
+ ApplyKB: true,
+ })
+ if err != nil {
+ t.Fatalf("Ingest: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload doc: %v", err)
+ }
+ t.Logf("persisted doc parser_config: %s", mustJSON(t, got.ParserConfig))
+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)
+ }
+ if auto["llm_id"] != kbLLMID {
+ t.Fatalf("persisted node llm_id = %v, want dataset value %v", auto["llm_id"], kbLLMID)
+ }
+ meta, ok := auto["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("persisted node metadata missing: %#v", auto)
+ }
+ if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("persisted metadata.enabled = %v, want true", meta["enabled"])
+ }
+ if !metadataFieldKeys(meta)["author"] {
+ t.Fatalf("persisted metadata missing author: %#v", meta["metadata"])
+ }
+ if _, ok := got.ParserConfig["llm_id"]; ok {
+ t.Fatalf("top-level llm_id written: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("flat enable_metadata written: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["metadata"]; ok {
+ t.Fatalf("top-level metadata written: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {
+ t.Fatalf("chunk_token_num dropped: %#v", got.ParserConfig)
+ }
+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))
+ if !ok {
+ t.Fatalf("consumer cannot read persisted extractor metadata: %#v", got.ParserConfig)
+ }
+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {
+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])
+ }
+}
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index a6469edc95..3992f8b4a3 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"ragflow/internal/service"
+ "sort"
"strconv"
"strings"
"sync"
@@ -87,6 +88,12 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.D
unlock := lockDocumentParse(doc.ID)
defer unlock()
+ if opts.ApplyKB {
+ if err := s.applyKnowledgebaseParserConfig(ctx, doc, kb); err != nil {
+ return err
+ }
+ }
+
if opts.RerunWithDelete {
if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {
return err
@@ -106,6 +113,168 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.D
return nil
}
+// applyKnowledgebaseParserConfig merges the dataset-scoped parser_config into
+// the document's existing component-scoped config and persists the merged
+// result before parsing when ApplyKB is requested.
+//
+// The dataset may contribute a top-level llm_id (used only to fill empty
+// Extractor / Compiler node llm_ids) and a top-level modular metadata object
+// (parser_config["metadata"] = {enabled, metadata, built_in_metadata}). Those
+// are scoped onto the document's existing Extractor nodes via
+// service.ApplyComponentScopedParserConfig, which also strips legacy flat
+// metadata fields. No top-level llm_id is written, and when the dataset sets
+// no metadata no top-level metadata key is left behind; the document's own
+// component nodes and unrelated keys are preserved. Nested maps/slices are
+// deep-copied so a failed persist cannot mutate the caller's document or the
+// source dataset. On a persistence failure the document's in-memory config is
+// left untouched.
+func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase) error {
+ if doc == nil || kb == nil {
+ return fmt.Errorf("apply dataset parser config: document and dataset are required")
+ }
+
+ merged := deepCopyJSONMap(doc.ParserConfig)
+ if merged == nil {
+ merged = entity.JSONMap{}
+ }
+
+ // The dataset's llm_id is used only as the helper's fill value for empty
+ // Extractor/Compiler node llm_ids. It is resolved from the same persisted
+ // source the dataset writer uses (explicit top-level value, else the
+ // tenant-resolved value already scoped into the dataset's Extractor nodes).
+ // No top-level llm_id flat key is written (the pipeline ignores it), and
+ // any legacy top-level llm_id the document already carries is dropped
+ // rather than re-written into the result.
+ llmID := resolveDatasetLLMID(kb.ParserConfig)
+ delete(merged, "llm_id")
+
+ // Carry the dataset's modular metadata object when present. Resolution
+ // follows the dataset reader contract: the top-level modular object
+ // first, else the first (sorted) Extractor node carrying modular metadata
+ // — the stored form after component scoping. When absent everywhere the
+ // dataset expresses "no metadata config": reset each existing Extractor
+ // node's metadata to the disabled default directly, mirroring the Python
+ // apply_kb defaults, without creating a top-level key the helper would
+ // otherwise have to clean up.
+ meta, metaOK, fromTopLevel := resolveDatasetModularMetadata(kb.ParserConfig)
+ if metaOK {
+ merged["metadata"] = deepCopyValue(meta)
+ } else {
+ for _, cpnID := range sortedExtractorKeys(merged) {
+ params, ok := nodeParams(merged[cpnID])
+ if !ok {
+ continue
+ }
+ params["metadata"] = map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ }
+ }
+ }
+
+ merged = service.ApplyComponentScopedParserConfig(merged, llmID)
+ if !fromTopLevel {
+ // The dataset itself carries no top-level modular object: never leave
+ // the transport key behind, mirroring the stored dataset shape.
+ delete(merged, "metadata")
+ }
+
+ if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
+ "parser_config": entity.JSONMap(merged),
+ }); err != nil {
+ return fmt.Errorf("apply dataset parser config for document %s: %w", doc.ID, err)
+ }
+ doc.ParserConfig = merged
+ return nil
+}
+
+// sortedExtractorKeys returns the component IDs in parserConfig that name
+// Extractor nodes, in sorted order so multi-Extractor configs resolve
+// deterministically instead of depending on Go map iteration order.
+func sortedExtractorKeys(parserConfig entity.JSONMap) []string {
+ var keys []string
+ for k := range parserConfig {
+ lower := strings.ToLower(k)
+ if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {
+ keys = append(keys, k)
+ }
+ }
+ sort.Strings(keys)
+ return keys
+}
+
+// nodeParams returns a writable view of a component node's params, accepting
+// both the decoded (map[string]any) and entity (entity.JSONMap) shapes.
+// The returned map shares storage with the input, so writes are visible.
+func nodeParams(raw any) (map[string]any, bool) {
+ switch params := raw.(type) {
+ case map[string]any:
+ return params, true
+ case entity.JSONMap:
+ return map[string]any(params), true
+ default:
+ return nil, false
+ }
+}
+
+// nodeModularMetadata returns the node's modular metadata object
+// ({enabled, metadata, built_in_metadata}) in either stored map shape.
+func nodeModularMetadata(params map[string]any) (map[string]any, bool) {
+ if meta, ok := params["metadata"].(map[string]any); ok {
+ return meta, true
+ }
+ if meta, ok := params["metadata"].(entity.JSONMap); ok {
+ return map[string]any(meta), true
+ }
+ return nil, false
+}
+
+// resolveDatasetModularMetadata returns the dataset's effective modular
+// metadata config, following the dataset reader contract: the top-level
+// modular object first, else the first (sorted) Extractor node carrying
+// modular metadata — the stored form after component scoping. It reports
+// whether any was found and whether it came from the top level. The returned
+// map is a live view of the dataset config; callers must copy before mutating.
+func resolveDatasetModularMetadata(parserConfig entity.JSONMap) (map[string]any, bool, bool) {
+ if mm, ok := parserConfig["metadata"].(map[string]any); ok {
+ return mm, true, true
+ }
+ if mm, ok := parserConfig["metadata"].(entity.JSONMap); ok {
+ return map[string]any(mm), true, true
+ }
+ for _, cpnID := range sortedExtractorKeys(parserConfig) {
+ params, ok := nodeParams(parserConfig[cpnID])
+ if !ok {
+ continue
+ }
+ if meta, ok := nodeModularMetadata(params); ok {
+ return meta, true, false
+ }
+ }
+ return nil, false, false
+}
+
+// resolveDatasetLLMID returns the dataset's effective LLM id: an explicit
+// non-empty top-level llm_id first, else the first (sorted) Extractor node's
+// non-empty llm_id (populated from the tenant default at dataset write time).
+// Empty means the dataset carries no llm_id and empty node llm_ids stay empty.
+func resolveDatasetLLMID(parserConfig entity.JSONMap) string {
+ if s, ok := parserConfig["llm_id"].(string); ok && strings.TrimSpace(s) != "" {
+ return s
+ }
+ for _, cpnID := range sortedExtractorKeys(parserConfig) {
+ params, ok := nodeParams(parserConfig[cpnID])
+ if !ok {
+ continue
+ }
+ if s, ok := params["llm_id"].(string); ok && strings.TrimSpace(s) != "" {
+ return s
+ }
+ }
+ return ""
+}
+
// AssertIngestionTasksTerminal verifies none of the documents has an
// in-flight (RUNNING/STOPPING) ingestion task. Used as a batch pre-check
// before re-parsing so a single non-terminal doc rejects the whole request
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 04418d7ddc..09669eafbf 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -1481,6 +1481,536 @@ func insertTestTaskWithProgress(t *testing.T, id, docID string, progress float64
}
}
+// metadataFieldKeys returns the set of "key" values in a modular metadata
+// object's "metadata" field list.
+func metadataFieldKeys(meta map[string]any) map[string]bool {
+ out := map[string]bool{}
+ list, _ := meta["metadata"].([]any)
+ for _, item := range list {
+ if m, ok := item.(map[string]any); ok {
+ if k, _ := m["key"].(string); k != "" {
+ out[k] = true
+ }
+ }
+ }
+ return out
+}
+
+func TestStartParseDocuments_ApplyKBMergesDatasetParserConfig(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
+ "llm_id": "kb-llm",
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "document_name", "type": "string"},
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("set kb parser_config: %v", err)
+ }
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+ // The document keeps its own component nodes; ApplyKB must scope the
+ // dataset config onto the existing Extractor nodes, not write flat keys
+ // the pipeline would ignore.
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "chunk_token_num": 128,
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ },
+ "Extractor:Pinned": map[string]any{
+ "llm_id": "doc-llm",
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("load kb: %v", err)
+ }
+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load doc: %v", err)
+ }
+
+ if err = svc.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{ApplyKB: true}); err != nil {
+ t.Fatalf("StartParseDocuments: %v", err)
+ }
+
+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload doc: %v", err)
+ }
+ // The empty node picks up the dataset llm_id; the explicitly pinned node
+ // keeps its own.
+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)
+ }
+ if auto["llm_id"] != "kb-llm" {
+ t.Fatalf("auto llm_id = %v, want kb-llm", auto["llm_id"])
+ }
+ pinned, ok := got.ParserConfig["Extractor:Pinned"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:Pinned missing: %#v", got.ParserConfig)
+ }
+ if pinned["llm_id"] != "doc-llm" {
+ t.Fatalf("pinned llm_id = %v, want doc-llm", pinned["llm_id"])
+ }
+ meta, ok := auto["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("auto metadata missing: %#v", auto)
+ }
+ if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("auto metadata.enabled = %v, want true", meta["enabled"])
+ }
+ if !metadataFieldKeys(meta)["author"] {
+ t.Fatalf("auto metadata missing author field: %#v", meta["metadata"])
+ }
+ // Legacy flat keys must not be written: the pipeline ignores them.
+ if _, ok := got.ParserConfig["llm_id"]; ok {
+ t.Fatalf("top-level llm_id written: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("flat enable_metadata written: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["metadata_config"]; ok {
+ t.Fatalf("flat metadata_config written: %#v", got.ParserConfig)
+ }
+ // A document-only key must survive the merge.
+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {
+ t.Fatalf("chunk_token_num dropped from parser_config: %#v", got.ParserConfig)
+ }
+ // The pipeline consumer must be able to read the merged config.
+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))
+ if !ok {
+ t.Fatalf("consumer cannot read extractor metadata: %#v", got.ParserConfig)
+ }
+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {
+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])
+ }
+}
+
+// TestIngest_ApplyKBPersistsDatasetParserConfig drives the request path the
+// future UI will use (Ingest -> StartParseDocuments ->
+// applyKnowledgebaseParserConfig) and asserts the merged dataset config is
+// what the database holds afterwards. A regression where Ingest stops
+// forwarding ApplyKB leaves the document's own llm_id in place and fails here.
+func TestIngest_ApplyKBPersistsDatasetParserConfig(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
+ "llm_id": "kb-llm",
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{
+ map[string]any{"key": "document_name", "type": "string"},
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("set kb parser_config: %v", err)
+ }
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "chunk_token_num": 128,
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: true,
+ ApplyKB: true,
+ })
+ if err != nil {
+ t.Fatalf("Ingest: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ // Read the config back from the database, not from the in-memory struct.
+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload doc: %v", err)
+ }
+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)
+ }
+ if auto["llm_id"] != "kb-llm" {
+ t.Fatalf("persisted node llm_id = %v, want kb-llm", auto["llm_id"])
+ }
+ meta, ok := auto["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("persisted node metadata missing: %#v", auto)
+ }
+ if enabled, _ := meta["enabled"].(bool); !enabled {
+ t.Fatalf("persisted metadata.enabled = %v, want true", meta["enabled"])
+ }
+ if !metadataFieldKeys(meta)["author"] {
+ t.Fatalf("persisted metadata missing author field: %#v", meta["metadata"])
+ }
+ if _, ok := got.ParserConfig["llm_id"]; ok {
+ t.Fatalf("top-level llm_id persisted: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("flat enable_metadata persisted: %#v", got.ParserConfig)
+ }
+ // A document-only key must survive the merge.
+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {
+ t.Fatalf("chunk_token_num dropped from persisted parser_config: %#v", got.ParserConfig)
+ }
+ // The pipeline consumer must read the persisted config, proving the flag
+ // travelled from Ingest all the way to a consumer-readable shape.
+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))
+ if !ok {
+ t.Fatalf("consumer cannot read persisted extractor metadata: %#v", got.ParserConfig)
+ }
+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {
+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])
+ }
+}
+
+// TestStartParseDocuments_ApplyKBUsesDatasetDefaults covers a dataset whose
+// parser_config sets neither llm_id nor modular metadata: each Extractor
+// node's metadata must reset to the disabled default
+// ({enabled:false, metadata:[], built_in_metadata:[]}) instead of keeping
+// stale fields, and no legacy flat keys may appear.
+func TestStartParseDocuments_ApplyKBUsesDatasetDefaults(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "chunk_token_num": 128,
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "stale", "type": "string"},
+ },
+ "built_in_metadata": []any{},
+ },
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("load kb: %v", err)
+ }
+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load doc: %v", err)
+ }
+
+ if err = svc.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{ApplyKB: true}); err != nil {
+ t.Fatalf("StartParseDocuments: %v", err)
+ }
+
+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload doc: %v", err)
+ }
+ if _, ok := got.ParserConfig["llm_id"]; ok {
+ t.Fatalf("top-level llm_id = %#v, want absent", got.ParserConfig["llm_id"])
+ }
+ if _, ok := got.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("flat enable_metadata = %#v, want absent", got.ParserConfig["enable_metadata"])
+ }
+ // The dataset expressed no metadata config, so no top-level metadata key
+ // may be left behind.
+ if _, ok := got.ParserConfig["metadata"]; ok {
+ t.Fatalf("top-level metadata = %#v, want absent", got.ParserConfig["metadata"])
+ }
+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)
+ }
+ meta, ok := auto["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("node metadata missing: %#v", auto)
+ }
+ if enabled, _ := meta["enabled"].(bool); enabled {
+ t.Fatalf("node metadata.enabled = %v, want false", meta["enabled"])
+ }
+ if fields, _ := meta["metadata"].([]any); len(fields) != 0 {
+ t.Fatalf("node metadata fields = %#v, want empty", meta["metadata"])
+ }
+ if builtIn, _ := meta["built_in_metadata"].([]any); len(builtIn) != 0 {
+ t.Fatalf("node built_in_metadata = %#v, want empty", meta["built_in_metadata"])
+ }
+ // A document-only key must survive the merge.
+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {
+ t.Fatalf("chunk_token_num dropped from parser_config: %#v", got.ParserConfig)
+ }
+ // The pipeline consumer must read the reset node config as disabled.
+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))
+ if !ok {
+ t.Fatalf("consumer cannot read node extractor metadata: %#v", got.ParserConfig)
+ }
+ if enabled, _ := consumerMeta["enabled"].(bool); enabled {
+ t.Fatalf("consumer metadata.enabled = %v, want false", consumerMeta["enabled"])
+ }
+}
+
+// TestIngest_ApplyKBDisabledLeavesParserConfigUnchanged pins the default
+// behaviour: without apply_kb the document keeps its own parser_config and
+// the dataset keys are not merged in.
+func TestIngest_ApplyKBDisabledLeavesParserConfigUnchanged(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
+ "llm_id": "kb-llm",
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{},
+ },
+ }).Error; err != nil {
+ t.Fatalf("set kb parser_config: %v", err)
+ }
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "llm_id": "doc-llm",
+ "chunk_token_num": 128,
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "node-llm",
+ "metadata": map[string]any{
+ "enabled": false,
+ "metadata": []any{},
+ "built_in_metadata": []any{},
+ },
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: true,
+ ApplyKB: false,
+ })
+ if err != nil {
+ t.Fatalf("Ingest: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload doc: %v", err)
+ }
+ if got.ParserConfig["llm_id"] != "doc-llm" {
+ t.Fatalf("persisted llm_id = %v, want doc-llm", got.ParserConfig["llm_id"])
+ }
+ node, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)
+ }
+ if node["llm_id"] != "node-llm" {
+ t.Fatalf("node llm_id = %v, want node-llm", node["llm_id"])
+ }
+ nodeMeta, ok := node["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("node metadata missing: %#v", node)
+ }
+ if enabled, _ := nodeMeta["enabled"].(bool); enabled {
+ t.Fatalf("node metadata.enabled merged without apply_kb: %#v", nodeMeta)
+ }
+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {
+ t.Fatalf("chunk_token_num dropped from parser_config: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("enable_metadata merged without apply_kb: %#v", got.ParserConfig)
+ }
+ if _, ok := got.ParserConfig["metadata"]; ok {
+ t.Fatalf("metadata merged without apply_kb: %#v", got.ParserConfig)
+ }
+}
+
+// TestStartParseDocuments_ApplyKBPersistFailureNamesDocument makes the
+// persistence step fail and checks the caller gets a locatable error instead
+// of a bare DAO failure.
+func TestStartParseDocuments_ApplyKBPersistFailureNamesDocument(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
+ "llm_id": "kb-llm",
+ }).Error; err != nil {
+ t.Fatalf("set kb parser_config: %v", err)
+ }
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
+ "location": "loc-1",
+ "parser_config": entity.JSONMap{
+ "llm_id": "doc-llm",
+ "Extractor:AutoExtractDefault": map[string]any{
+ "llm_id": "",
+ "metadata": map[string]any{
+ "enabled": true,
+ "metadata": []any{
+ map[string]any{"key": "author", "type": "string"},
+ },
+ "built_in_metadata": []any{},
+ },
+ },
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+ // Any write to the document table now fails, so the merge cannot be persisted.
+ if err := dao.DB.Exec("CREATE TRIGGER fail_document_update BEFORE UPDATE ON document BEGIN SELECT RAISE(ABORT, 'boom'); END;").Error; err != nil {
+ t.Fatalf("install failing trigger: %v", err)
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("load kb: %v", err)
+ }
+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load doc: %v", err)
+ }
+
+ err = svc.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{ApplyKB: true})
+ if err == nil {
+ t.Fatal("expected an error when the merged parser config cannot be persisted")
+ }
+ if !strings.Contains(err.Error(), "doc-1") {
+ t.Fatalf("error %q does not name the document", err.Error())
+ }
+ if len(publisher.messages) != 0 {
+ t.Fatalf("expected no ingestion task to be published, got %d", len(publisher.messages))
+ }
+ // The caller's original parser_config must be untouched when the persist
+ // fails, including nested component nodes (the merge works on a deep
+ // copy and is only assigned back after a successful persist).
+ if doc.ParserConfig["llm_id"] != "doc-llm" {
+ t.Fatalf("input doc.ParserConfig mutated on failure: %#v", doc.ParserConfig)
+ }
+ if _, ok := doc.ParserConfig["enable_metadata"]; ok {
+ t.Fatalf("enable_metadata leaked into input doc.ParserConfig on failure: %#v", doc.ParserConfig)
+ }
+ if _, ok := doc.ParserConfig["metadata"]; ok {
+ t.Fatalf("metadata leaked into input doc.ParserConfig on failure: %#v", doc.ParserConfig)
+ }
+ node, ok := doc.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)
+ if !ok {
+ t.Fatalf("input Extractor node missing on failure: %#v", doc.ParserConfig)
+ }
+ if node["llm_id"] != "" {
+ t.Fatalf("input node llm_id mutated on failure: %#v", node)
+ }
+ nodeMeta, ok := node["metadata"].(map[string]any)
+ if !ok {
+ t.Fatalf("input node metadata missing on failure: %#v", node)
+ }
+ if enabled, _ := nodeMeta["enabled"].(bool); !enabled {
+ t.Fatalf("input node metadata.enabled mutated on failure: %#v", nodeMeta)
+ }
+ if !metadataFieldKeys(nodeMeta)["author"] {
+ t.Fatalf("input node metadata fields mutated on failure: %#v", nodeMeta)
+ }
+}
+
+// TestApplyKnowledgebaseParserConfig_NilInputReturnsError pins the guard: a
+// nil document or nil dataset must surface an error instead of silently
+// skipping the merge.
+func TestApplyKnowledgebaseParserConfig_NilInputReturnsError(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)
+
+ svc := testDocumentService(t)
+ ctx := t.Context()
+
+ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("load kb: %v", err)
+ }
+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load doc: %v", err)
+ }
+
+ if err := svc.applyKnowledgebaseParserConfig(ctx, nil, kb); err == nil {
+ t.Fatal("expected an error for a nil document")
+ }
+ if err := svc.applyKnowledgebaseParserConfig(ctx, doc, nil); err == nil {
+ t.Fatal("expected an error for a nil dataset")
+ }
+}
+
func TestStartParseDocuments_EnqueuesIngestionTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)