RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(document): honour apply_kb so reparse applies the dataset config (#20149)
Bruce-Yii66bd0554b394
diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.goindex 7acc009553..c6d876ab01 100644--- a/internal/service/component_scoped_parser_config.go+++ b/internal/service/component_scoped_parser_config.go@@ -79,7 +79,36 @@ func ApplyComponentScopedParserConfig(func cloneJSONMap(in map[string]any) map[string]any {out := make(map[string]any, len(in))for k, v := range in {- out[k] = v+ out[k] = cloneValue(v)}return out}++func cloneValue(v any) any {+ switch val := v.(type) {+ case nil:+ return nil+ case map[string]any:+ return cloneJSONMap(val)+ case entity.JSONMap:+ out := make(map[string]any, len(val))+ for k, vv := range val {+ out[k] = cloneValue(vv)+ }+ return out+ case []any:+ out := make([]any, len(val))+ for i, vv := range val {+ out[i] = cloneValue(vv)+ }+ return out+ case []map[string]any:+ out := make([]map[string]any, len(val))+ for i, vv := range val {+ out[i] = cloneValue(vv).(map[string]any)+ }+ return out+ default:+ return v+ }+}diff --git a/internal/service/document/deep_copy.go b/internal/service/document/deep_copy.gonew file mode 100644index 0000000000..5b2aed9d73--- /dev/null+++ b/internal/service/document/deep_copy.go@@ -0,0 +1,51 @@+package document++import "ragflow/internal/entity"++// deepCopyJSONMap returns an independent copy of in, including all nested+// maps and slices. A nil input yields nil.+func deepCopyJSONMap(in entity.JSONMap) entity.JSONMap {+ if in == nil {+ return nil+ }+ out := make(entity.JSONMap, len(in))+ for k, v := range in {+ out[k] = deepCopyValue(v)+ }+ return out+}++// deepCopyValue recursively copies maps and slices; scalar values are shared+// (they are immutable from the caller's perspective).+func deepCopyValue(v any) any {+ switch val := v.(type) {+ case nil:+ return nil+ case entity.JSONMap:+ out := make(entity.JSONMap, len(val))+ for k, vv := range val {+ out[k] = deepCopyValue(vv)+ }+ return out+ case map[string]any:+ out := make(map[string]any, len(val))+ for k, vv := range val {+ out[k] = deepCopyValue(vv)+ }+ return out+ case []any:+ out := make([]any, len(val))+ for i, vv := range val {+ out[i] = deepCopyValue(vv)+ }+ return out+ case []map[string]any:+ out := make([]map[string]any, len(val))+ for i, vv := range val {+ out[i] = deepCopyValue(vv).(map[string]any)+ }+ return out+ default:+ return v+ }+}diff --git a/internal/service/document/document.go b/internal/service/document/document.goindex 342f0721b9..d2847cde0b 100644--- a/internal/service/document/document.go+++ b/internal/service/document/document.go@@ -226,8 +226,15 @@ type IngestDocumentRequest struct {// StartParseOptions controls StartParseDocuments behavior.type StartParseOptions struct {- // ApplyKB merges the knowledge base's parser_config (llm_id, metadata)- // into the document before parsing.+ // ApplyKB scopes the dataset's parser_config (llm_id and the modular+ // metadata object: enabled, metadata, built_in_metadata) onto the document's+ // existing Extractor nodes before parsing. The dataset llm_id only fills+ // Extractor/Compiler nodes whose llm_id is empty; explicit per-node values+ // are preserved and no top-level llm_id is written. When the dataset sets+ // modular metadata it replaces each Extractor node's metadata; when it+ // sets none, each node's metadata is reset to the disabled default+ // ({enabled:false, metadata:[], built_in_metadata:[]}) with no top-level+ // metadata key left behind.ApplyKB bool// RerunWithDelete clears prior chunks/tasks/counters before reparsing.RerunWithDelete booldiff --git a/internal/service/document/document_applykb_chain_test.go b/internal/service/document/document_applykb_chain_test.gonew file mode 100644index 0000000000..9ba903fa23--- /dev/null+++ b/internal/service/document/document_applykb_chain_test.go@@ -0,0 +1,444 @@+package document++// Real writer-to-reader ApplyKB chain tests (YU-366).+//+// These tests never hand-craft the dataset's parser_config. The dataset side+// always goes through a real writer entry point:+// - DatasetService.UpdateMetadataConfig (dataset metadata API), or+// - DatasetService.CreateDataset (dataset creation from a pipeline DSL),+// after which the KB row is reloaded from the database and only then fed to+// DocumentService.Ingest(ApplyKB:true). The document row is reloaded as well+// and asserted through the pipeline's own consumer+// (common.ExtractorMetadataConfig) — dataset service -> DB -> Ingest ->+// DB -> consumer, no shortcuts.++import (+ "encoding/json"+ "sort"+ "strings"+ "testing"++ "gorm.io/gorm"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/permission"+ "ragflow/internal/service"+ datasetsvc "ragflow/internal/service/dataset"+)++func mustJSON(t *testing.T, v any) string {+ t.Helper()+ data, err := json.Marshal(v)+ if err != nil {+ t.Fatalf("marshal parser_config: %v", err)+ }+ return string(data)+}++// firstExtractorMetadata returns the modular metadata of the first (sorted)+// Extractor node carrying one, mirroring the deterministic reader contract.+func firstExtractorMetadata(t *testing.T, parserConfig entity.JSONMap) map[string]any {+ t.Helper()+ var keys []string+ for k := range parserConfig {+ lower := strings.ToLower(k)+ if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {+ keys = append(keys, k)+ }+ }+ sort.Strings(keys)+ for _, k := range keys {+ node, ok := parserConfig[k].(map[string]any)+ if !ok {+ continue+ }+ if meta, ok := node["metadata"].(map[string]any); ok {+ return meta+ }+ }+ t.Fatalf("no Extractor node with modular metadata in %#v", parserConfig)+ return nil+}++func extractorNodeLLMID(t *testing.T, parserConfig entity.JSONMap, node string) string {+ t.Helper()+ params, ok := parserConfig[node].(map[string]any)+ if !ok {+ t.Fatalf("node %s missing or not a map: %#v", node, parserConfig[node])+ }+ llmID, _ := params["llm_id"].(string)+ return llmID+}++// newDatasetService builds a DatasetService the same way production code does.+// Only the kb/tenant DAOs are exercised by these tests.+func newDatasetService() *datasetsvc.DatasetService {+ return datasetsvc.NewDatasetService()+}++// migrateChainModelTables adds the tables the real dataset entry points touch+// (model resolution, connectors, canvas) beyond the document package's base+// migration. Test scaffolding only; the dataset writes themselves stay real.+func migrateChainModelTables(t *testing.T, db *gorm.DB) {+ t.Helper()+ if err := db.AutoMigrate(+ &entity.Connector{},+ &entity.Connector2Kb{},+ &entity.SyncLogs{},+ &entity.TenantModelProvider{},+ &entity.TenantModelInstance{},+ &entity.TenantModel{},+ &entity.UserCanvas{},+ &entity.PipelineOperationLog{},+ ); err != nil {+ t.Fatalf("migrate chain tables: %v", err)+ }+}++// TestIngest_ApplyKB_RealDatasetAPIChain drives the real dataset metadata API+// (UpdateMetadataConfig with enabled=true, a custom author field and a+// built-in field), reloads the KB from the database, then Ingests with+// ApplyKB:true and asserts the persisted document config through the+// pipeline's own consumer.+func TestIngest_ApplyKB_RealDatasetAPIChain(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ // Every real dataset DSL defines an Extractor node; the writer under test+ // scopes onto it. Scaffolding the empty node shape is not faking config:+ // no metadata values are invented here.+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update(+ "parser_config", entity.JSONMap{+ "Extractor:AutoExtractDefault": map[string]any{},+ },+ ).Error; err != nil {+ t.Fatalf("seed kb extractor node: %v", err)+ }+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "chunk_token_num": 128,+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "",+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ ctx := t.Context()+ enabled := true+ if _, code, err := newDatasetService().UpdateMetadataConfig(ctx, "kb-1", "user-1", &service.MetadataConfigRequest{+ Enabled: &enabled,+ Metadata: []service.MetadataConfigField{{Key: "author", Type: "string"}},+ BuiltInMetadata: []service.MetadataConfigField{{Key: "document_name", Type: "string"}},+ }); err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateMetadataConfig err=%v code=%v", err, code)+ }++ // Reload the KB: this is the real stored shape, not a fixture.+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("reload kb: %v", err)+ }+ t.Logf("stored kb parser_config: %s", mustJSON(t, kb.ParserConfig))+ kbMeta := firstExtractorMetadata(t, kb.ParserConfig)+ if enabled, _ := kbMeta["enabled"].(bool); !enabled {+ t.Fatalf("stored dataset metadata.enabled = %v, want true", kbMeta["enabled"])+ }+ if !metadataFieldKeys(kbMeta)["author"] {+ t.Fatalf("stored dataset metadata missing author: %#v", kbMeta["metadata"])+ }+ kbLLMID := extractorNodeLLMID(t, kb.ParserConfig, "Extractor:AutoExtractDefault")+ if kbLLMID == "" {+ t.Fatalf("stored dataset node llm_id is empty; tenant default was not applied")+ }+ t.Logf("stored dataset node llm_id: %s", kbLLMID)++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)++ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusRunning),+ Delete: true,+ ApplyKB: true,+ })+ if err != nil {+ t.Fatalf("Ingest: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload doc: %v", err)+ }+ t.Logf("persisted doc parser_config: %s", mustJSON(t, got.ParserConfig))+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)+ }+ if auto["llm_id"] != kbLLMID {+ t.Fatalf("persisted node llm_id = %v, want dataset value %v", auto["llm_id"], kbLLMID)+ }+ meta, ok := auto["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("persisted node metadata missing: %#v", auto)+ }+ if enabled, _ := meta["enabled"].(bool); !enabled {+ t.Fatalf("persisted metadata.enabled = %v, want true", meta["enabled"])+ }+ if !metadataFieldKeys(meta)["author"] {+ t.Fatalf("persisted metadata missing author: %#v", meta["metadata"])+ }+ if _, ok := got.ParserConfig["llm_id"]; ok {+ t.Fatalf("top-level llm_id written: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("flat enable_metadata written: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {+ t.Fatalf("chunk_token_num dropped: %#v", got.ParserConfig)+ }+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))+ if !ok {+ t.Fatalf("consumer cannot read persisted extractor metadata: %#v", got.ParserConfig)+ }+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])+ }+}++// TestIngest_ApplyKB_DisabledDatasetAPIChain pins the real disabled path: a+// dataset explicitly configured with enabled=false must not enable document+// auto-metadata.+func TestIngest_ApplyKB_DisabledDatasetAPIChain(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update(+ "parser_config", entity.JSONMap{+ "Extractor:AutoExtractDefault": map[string]any{},+ },+ ).Error; err != nil {+ t.Fatalf("seed kb extractor node: %v", err)+ }+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "chunk_token_num": 128,+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "",+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ ctx := t.Context()+ disabled := false+ if _, code, err := newDatasetService().UpdateMetadataConfig(ctx, "kb-1", "user-1", &service.MetadataConfigRequest{+ Enabled: &disabled,+ }); err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateMetadataConfig err=%v code=%v", err, code)+ }++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)++ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusRunning),+ Delete: true,+ ApplyKB: true,+ })+ if err != nil {+ t.Fatalf("Ingest: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload doc: %v", err)+ }+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)+ }+ meta, ok := auto["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("node metadata missing: %#v", auto)+ }+ if enabled, _ := meta["enabled"].(bool); enabled {+ t.Fatalf("node metadata.enabled = %v, want false", meta["enabled"])+ }+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))+ if !ok {+ t.Fatalf("consumer cannot read node extractor metadata: %#v", got.ParserConfig)+ }+ if enabled, _ := consumerMeta["enabled"].(bool); enabled {+ t.Fatalf("consumer metadata.enabled = %v, want false", consumerMeta["enabled"])+ }+}++// TestIngest_ApplyKB_RealCreatedDatasetChain covers the stored shape where+// the dataset writer persisted component-scoped config with no top-level+// "metadata" key (dataset creation scopes the DSL config before persisting).+// The seed goes through the real DatasetService.CreateDataset entry point.+func TestIngest_ApplyKB_RealCreatedDatasetChain(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")+ migrateChainModelTables(t, db)+ // Mirror the dataset package's own fixture: the membership carries the+ // resolved embedding model id, keeping CreateDataset on its real path+ // (no embedding override).+ if err := dao.DB.Model(&entity.UserTenant{}).Where("user_id = ? AND tenant_id = ?", "user-1", "tenant-1").Update(+ "tenant_embd_id", "embd-1",+ ).Error; err != nil {+ t.Fatalf("set membership embd id: %v", err)+ }++ ctx := t.Context()+ parserID := "general"+ parseType := 1+ team := "team"+ result, code, err := newDatasetService().CreateDataset(ctx, &service.CreateDatasetRequest{+ Name: "ds-chain",+ ParserID: &parserID,+ ParseType: &parseType,+ Permission: &team,+ ParserConfig: map[string]interface{}{+ "Extractor:AutoExtractDefault": map[string]interface{}{+ "llm_id": "",+ "metadata": map[string]interface{}{+ "enabled": true,+ "metadata": []interface{}{+ map[string]interface{}{"key": "author", "type": "string"},+ },+ "built_in_metadata": []interface{}{+ map[string]interface{}{"key": "document_name", "type": "string"},+ },+ },+ },+ },+ }, permission.Subject{UserID: "user-1", TenantID: "tenant-1"})+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("CreateDataset err=%v code=%v", err, code)+ }+ kbID, _ := result["id"].(string)+ if kbID == "" {+ t.Fatalf("created dataset has no id: %#v", result)+ }++ // Reload the KB: record the real stored shape the writer produced.+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, kbID)+ if err != nil {+ t.Fatalf("reload kb: %v", err)+ }+ t.Logf("stored kb parser_config: %s", mustJSON(t, kb.ParserConfig))+ if _, ok := kb.ParserConfig["metadata"]; ok {+ t.Fatalf("stored kb has top-level metadata, want component-scoped only: %s", mustJSON(t, kb.ParserConfig))+ }+ kbMeta := firstExtractorMetadata(t, kb.ParserConfig)+ if enabled, _ := kbMeta["enabled"].(bool); !enabled {+ t.Fatalf("stored dataset metadata.enabled = %v, want true", kbMeta["enabled"])+ }+ if !metadataFieldKeys(kbMeta)["author"] {+ t.Fatalf("stored dataset metadata missing author: %#v", kbMeta["metadata"])+ }+ kbLLMID := ""+ for _, k := range []string{"Extractor:AutoExtractDefault"} {+ kbLLMID = extractorNodeLLMID(t, kb.ParserConfig, k)+ }+ if kbLLMID == "" {+ t.Fatalf("stored dataset node llm_id is empty; tenant default was not applied")+ }+ t.Logf("stored dataset node llm_id: %s", kbLLMID)++ insertTestDoc(t, "doc-1", kbID, 9, 4)+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "chunk_token_num": 128,+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "",+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)++ code, err = svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusRunning),+ Delete: true,+ ApplyKB: true,+ })+ if err != nil {+ t.Fatalf("Ingest: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload doc: %v", err)+ }+ t.Logf("persisted doc parser_config: %s", mustJSON(t, got.ParserConfig))+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)+ }+ if auto["llm_id"] != kbLLMID {+ t.Fatalf("persisted node llm_id = %v, want dataset value %v", auto["llm_id"], kbLLMID)+ }+ meta, ok := auto["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("persisted node metadata missing: %#v", auto)+ }+ if enabled, _ := meta["enabled"].(bool); !enabled {+ t.Fatalf("persisted metadata.enabled = %v, want true", meta["enabled"])+ }+ if !metadataFieldKeys(meta)["author"] {+ t.Fatalf("persisted metadata missing author: %#v", meta["metadata"])+ }+ if _, ok := got.ParserConfig["llm_id"]; ok {+ t.Fatalf("top-level llm_id written: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("flat enable_metadata written: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["metadata"]; ok {+ t.Fatalf("top-level metadata written: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {+ t.Fatalf("chunk_token_num dropped: %#v", got.ParserConfig)+ }+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))+ if !ok {+ t.Fatalf("consumer cannot read persisted extractor metadata: %#v", got.ParserConfig)+ }+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])+ }+}diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex a6469edc95..3992f8b4a3 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -5,6 +5,7 @@ import ("errors""fmt""ragflow/internal/service"+ "sort""strconv""strings""sync"@@ -87,6 +88,12 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.Dunlock := lockDocumentParse(doc.ID)defer unlock()+ if opts.ApplyKB {+ if err := s.applyKnowledgebaseParserConfig(ctx, doc, kb); err != nil {+ return err+ }+ }+if opts.RerunWithDelete {if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {return err@@ -106,6 +113,168 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.Dreturn nil}+// applyKnowledgebaseParserConfig merges the dataset-scoped parser_config into+// the document's existing component-scoped config and persists the merged+// result before parsing when ApplyKB is requested.+//+// The dataset may contribute a top-level llm_id (used only to fill empty+// Extractor / Compiler node llm_ids) and a top-level modular metadata object+// (parser_config["metadata"] = {enabled, metadata, built_in_metadata}). Those+// are scoped onto the document's existing Extractor nodes via+// service.ApplyComponentScopedParserConfig, which also strips legacy flat+// metadata fields. No top-level llm_id is written, and when the dataset sets+// no metadata no top-level metadata key is left behind; the document's own+// component nodes and unrelated keys are preserved. Nested maps/slices are+// deep-copied so a failed persist cannot mutate the caller's document or the+// source dataset. On a persistence failure the document's in-memory config is+// left untouched.+func (s *DocumentService) applyKnowledgebaseParserConfig(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase) error {+ if doc == nil || kb == nil {+ return fmt.Errorf("apply dataset parser config: document and dataset are required")+ }++ merged := deepCopyJSONMap(doc.ParserConfig)+ if merged == nil {+ merged = entity.JSONMap{}+ }++ // The dataset's llm_id is used only as the helper's fill value for empty+ // Extractor/Compiler node llm_ids. It is resolved from the same persisted+ // source the dataset writer uses (explicit top-level value, else the+ // tenant-resolved value already scoped into the dataset's Extractor nodes).+ // No top-level llm_id flat key is written (the pipeline ignores it), and+ // any legacy top-level llm_id the document already carries is dropped+ // rather than re-written into the result.+ llmID := resolveDatasetLLMID(kb.ParserConfig)+ delete(merged, "llm_id")++ // Carry the dataset's modular metadata object when present. Resolution+ // follows the dataset reader contract: the top-level modular object+ // first, else the first (sorted) Extractor node carrying modular metadata+ // — the stored form after component scoping. When absent everywhere the+ // dataset expresses "no metadata config": reset each existing Extractor+ // node's metadata to the disabled default directly, mirroring the Python+ // apply_kb defaults, without creating a top-level key the helper would+ // otherwise have to clean up.+ meta, metaOK, fromTopLevel := resolveDatasetModularMetadata(kb.ParserConfig)+ if metaOK {+ merged["metadata"] = deepCopyValue(meta)+ } else {+ for _, cpnID := range sortedExtractorKeys(merged) {+ params, ok := nodeParams(merged[cpnID])+ if !ok {+ continue+ }+ params["metadata"] = map[string]any{+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},+ }+ }+ }++ merged = service.ApplyComponentScopedParserConfig(merged, llmID)+ if !fromTopLevel {+ // The dataset itself carries no top-level modular object: never leave+ // the transport key behind, mirroring the stored dataset shape.+ delete(merged, "metadata")+ }++ if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{+ "parser_config": entity.JSONMap(merged),+ }); err != nil {+ return fmt.Errorf("apply dataset parser config for document %s: %w", doc.ID, err)+ }+ doc.ParserConfig = merged+ return nil+}++// sortedExtractorKeys returns the component IDs in parserConfig that name+// Extractor nodes, in sorted order so multi-Extractor configs resolve+// deterministically instead of depending on Go map iteration order.+func sortedExtractorKeys(parserConfig entity.JSONMap) []string {+ var keys []string+ for k := range parserConfig {+ lower := strings.ToLower(k)+ if strings.HasPrefix(lower, "extractor:") || strings.HasPrefix(lower, "extractor_") {+ keys = append(keys, k)+ }+ }+ sort.Strings(keys)+ return keys+}++// nodeParams returns a writable view of a component node's params, accepting+// both the decoded (map[string]any) and entity (entity.JSONMap) shapes.+// The returned map shares storage with the input, so writes are visible.+func nodeParams(raw any) (map[string]any, bool) {+ switch params := raw.(type) {+ case map[string]any:+ return params, true+ case entity.JSONMap:+ return map[string]any(params), true+ default:+ return nil, false+ }+}++// nodeModularMetadata returns the node's modular metadata object+// ({enabled, metadata, built_in_metadata}) in either stored map shape.+func nodeModularMetadata(params map[string]any) (map[string]any, bool) {+ if meta, ok := params["metadata"].(map[string]any); ok {+ return meta, true+ }+ if meta, ok := params["metadata"].(entity.JSONMap); ok {+ return map[string]any(meta), true+ }+ return nil, false+}++// resolveDatasetModularMetadata returns the dataset's effective modular+// metadata config, following the dataset reader contract: the top-level+// modular object first, else the first (sorted) Extractor node carrying+// modular metadata — the stored form after component scoping. It reports+// whether any was found and whether it came from the top level. The returned+// map is a live view of the dataset config; callers must copy before mutating.+func resolveDatasetModularMetadata(parserConfig entity.JSONMap) (map[string]any, bool, bool) {+ if mm, ok := parserConfig["metadata"].(map[string]any); ok {+ return mm, true, true+ }+ if mm, ok := parserConfig["metadata"].(entity.JSONMap); ok {+ return map[string]any(mm), true, true+ }+ for _, cpnID := range sortedExtractorKeys(parserConfig) {+ params, ok := nodeParams(parserConfig[cpnID])+ if !ok {+ continue+ }+ if meta, ok := nodeModularMetadata(params); ok {+ return meta, true, false+ }+ }+ return nil, false, false+}++// resolveDatasetLLMID returns the dataset's effective LLM id: an explicit+// non-empty top-level llm_id first, else the first (sorted) Extractor node's+// non-empty llm_id (populated from the tenant default at dataset write time).+// Empty means the dataset carries no llm_id and empty node llm_ids stay empty.+func resolveDatasetLLMID(parserConfig entity.JSONMap) string {+ if s, ok := parserConfig["llm_id"].(string); ok && strings.TrimSpace(s) != "" {+ return s+ }+ for _, cpnID := range sortedExtractorKeys(parserConfig) {+ params, ok := nodeParams(parserConfig[cpnID])+ if !ok {+ continue+ }+ if s, ok := params["llm_id"].(string); ok && strings.TrimSpace(s) != "" {+ return s+ }+ }+ return ""+}+// AssertIngestionTasksTerminal verifies none of the documents has an// in-flight (RUNNING/STOPPING) ingestion task. Used as a batch pre-check// before re-parsing so a single non-terminal doc rejects the whole requestdiff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex 04418d7ddc..09669eafbf 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -1481,6 +1481,536 @@ func insertTestTaskWithProgress(t *testing.T, id, docID string, progress float64}}+// metadataFieldKeys returns the set of "key" values in a modular metadata+// object's "metadata" field list.+func metadataFieldKeys(meta map[string]any) map[string]bool {+ out := map[string]bool{}+ list, _ := meta["metadata"].([]any)+ for _, item := range list {+ if m, ok := item.(map[string]any); ok {+ if k, _ := m["key"].(string); k != "" {+ out[k] = true+ }+ }+ }+ return out+}++func TestStartParseDocuments_ApplyKBMergesDatasetParserConfig(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{+ "llm_id": "kb-llm",+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "document_name", "type": "string"},+ },+ },+ }).Error; err != nil {+ t.Fatalf("set kb parser_config: %v", err)+ }+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)+ // The document keeps its own component nodes; ApplyKB must scope the+ // dataset config onto the existing Extractor nodes, not write flat keys+ // the pipeline would ignore.+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "chunk_token_num": 128,+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "",+ },+ "Extractor:Pinned": map[string]any{+ "llm_id": "doc-llm",+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)+ ctx := t.Context()++ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("load kb: %v", err)+ }+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load doc: %v", err)+ }++ if err = svc.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{ApplyKB: true}); err != nil {+ t.Fatalf("StartParseDocuments: %v", err)+ }++ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload doc: %v", err)+ }+ // The empty node picks up the dataset llm_id; the explicitly pinned node+ // keeps its own.+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)+ }+ if auto["llm_id"] != "kb-llm" {+ t.Fatalf("auto llm_id = %v, want kb-llm", auto["llm_id"])+ }+ pinned, ok := got.ParserConfig["Extractor:Pinned"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:Pinned missing: %#v", got.ParserConfig)+ }+ if pinned["llm_id"] != "doc-llm" {+ t.Fatalf("pinned llm_id = %v, want doc-llm", pinned["llm_id"])+ }+ meta, ok := auto["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("auto metadata missing: %#v", auto)+ }+ if enabled, _ := meta["enabled"].(bool); !enabled {+ t.Fatalf("auto metadata.enabled = %v, want true", meta["enabled"])+ }+ if !metadataFieldKeys(meta)["author"] {+ t.Fatalf("auto metadata missing author field: %#v", meta["metadata"])+ }+ // Legacy flat keys must not be written: the pipeline ignores them.+ if _, ok := got.ParserConfig["llm_id"]; ok {+ t.Fatalf("top-level llm_id written: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("flat enable_metadata written: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["metadata_config"]; ok {+ t.Fatalf("flat metadata_config written: %#v", got.ParserConfig)+ }+ // A document-only key must survive the merge.+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {+ t.Fatalf("chunk_token_num dropped from parser_config: %#v", got.ParserConfig)+ }+ // The pipeline consumer must be able to read the merged config.+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))+ if !ok {+ t.Fatalf("consumer cannot read extractor metadata: %#v", got.ParserConfig)+ }+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])+ }+}++// TestIngest_ApplyKBPersistsDatasetParserConfig drives the request path the+// future UI will use (Ingest -> StartParseDocuments ->+// applyKnowledgebaseParserConfig) and asserts the merged dataset config is+// what the database holds afterwards. A regression where Ingest stops+// forwarding ApplyKB leaves the document's own llm_id in place and fails here.+func TestIngest_ApplyKBPersistsDatasetParserConfig(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{+ "llm_id": "kb-llm",+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "document_name", "type": "string"},+ },+ },+ }).Error; err != nil {+ t.Fatalf("set kb parser_config: %v", err)+ }+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "chunk_token_num": 128,+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "",+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)+ ctx := t.Context()++ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusRunning),+ Delete: true,+ ApplyKB: true,+ })+ if err != nil {+ t.Fatalf("Ingest: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ // Read the config back from the database, not from the in-memory struct.+ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload doc: %v", err)+ }+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)+ }+ if auto["llm_id"] != "kb-llm" {+ t.Fatalf("persisted node llm_id = %v, want kb-llm", auto["llm_id"])+ }+ meta, ok := auto["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("persisted node metadata missing: %#v", auto)+ }+ if enabled, _ := meta["enabled"].(bool); !enabled {+ t.Fatalf("persisted metadata.enabled = %v, want true", meta["enabled"])+ }+ if !metadataFieldKeys(meta)["author"] {+ t.Fatalf("persisted metadata missing author field: %#v", meta["metadata"])+ }+ if _, ok := got.ParserConfig["llm_id"]; ok {+ t.Fatalf("top-level llm_id persisted: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("flat enable_metadata persisted: %#v", got.ParserConfig)+ }+ // A document-only key must survive the merge.+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {+ t.Fatalf("chunk_token_num dropped from persisted parser_config: %#v", got.ParserConfig)+ }+ // The pipeline consumer must read the persisted config, proving the flag+ // travelled from Ingest all the way to a consumer-readable shape.+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))+ if !ok {+ t.Fatalf("consumer cannot read persisted extractor metadata: %#v", got.ParserConfig)+ }+ if enabled, _ := consumerMeta["enabled"].(bool); !enabled {+ t.Fatalf("consumer metadata.enabled = %v, want true", consumerMeta["enabled"])+ }+}++// TestStartParseDocuments_ApplyKBUsesDatasetDefaults covers a dataset whose+// parser_config sets neither llm_id nor modular metadata: each Extractor+// node's metadata must reset to the disabled default+// ({enabled:false, metadata:[], built_in_metadata:[]}) instead of keeping+// stale fields, and no legacy flat keys may appear.+func TestStartParseDocuments_ApplyKBUsesDatasetDefaults(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "chunk_token_num": 128,+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "",+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "stale", "type": "string"},+ },+ "built_in_metadata": []any{},+ },+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)+ ctx := t.Context()++ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("load kb: %v", err)+ }+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load doc: %v", err)+ }++ if err = svc.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{ApplyKB: true}); err != nil {+ t.Fatalf("StartParseDocuments: %v", err)+ }++ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload doc: %v", err)+ }+ if _, ok := got.ParserConfig["llm_id"]; ok {+ t.Fatalf("top-level llm_id = %#v, want absent", got.ParserConfig["llm_id"])+ }+ if _, ok := got.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("flat enable_metadata = %#v, want absent", got.ParserConfig["enable_metadata"])+ }+ // The dataset expressed no metadata config, so no top-level metadata key+ // may be left behind.+ if _, ok := got.ParserConfig["metadata"]; ok {+ t.Fatalf("top-level metadata = %#v, want absent", got.ParserConfig["metadata"])+ }+ auto, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)+ }+ meta, ok := auto["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("node metadata missing: %#v", auto)+ }+ if enabled, _ := meta["enabled"].(bool); enabled {+ t.Fatalf("node metadata.enabled = %v, want false", meta["enabled"])+ }+ if fields, _ := meta["metadata"].([]any); len(fields) != 0 {+ t.Fatalf("node metadata fields = %#v, want empty", meta["metadata"])+ }+ if builtIn, _ := meta["built_in_metadata"].([]any); len(builtIn) != 0 {+ t.Fatalf("node built_in_metadata = %#v, want empty", meta["built_in_metadata"])+ }+ // A document-only key must survive the merge.+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {+ t.Fatalf("chunk_token_num dropped from parser_config: %#v", got.ParserConfig)+ }+ // The pipeline consumer must read the reset node config as disabled.+ consumerMeta, ok := common.ExtractorMetadataConfig(map[string]any(got.ParserConfig))+ if !ok {+ t.Fatalf("consumer cannot read node extractor metadata: %#v", got.ParserConfig)+ }+ if enabled, _ := consumerMeta["enabled"].(bool); enabled {+ t.Fatalf("consumer metadata.enabled = %v, want false", consumerMeta["enabled"])+ }+}++// TestIngest_ApplyKBDisabledLeavesParserConfigUnchanged pins the default+// behaviour: without apply_kb the document keeps its own parser_config and+// the dataset keys are not merged in.+func TestIngest_ApplyKBDisabledLeavesParserConfigUnchanged(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{+ "llm_id": "kb-llm",+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{},+ },+ }).Error; err != nil {+ t.Fatalf("set kb parser_config: %v", err)+ }+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "llm_id": "doc-llm",+ "chunk_token_num": 128,+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "node-llm",+ "metadata": map[string]any{+ "enabled": false,+ "metadata": []any{},+ "built_in_metadata": []any{},+ },+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)+ ctx := t.Context()++ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusRunning),+ Delete: true,+ ApplyKB: false,+ })+ if err != nil {+ t.Fatalf("Ingest: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ got, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload doc: %v", err)+ }+ if got.ParserConfig["llm_id"] != "doc-llm" {+ t.Fatalf("persisted llm_id = %v, want doc-llm", got.ParserConfig["llm_id"])+ }+ node, ok := got.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("Extractor:AutoExtractDefault missing: %#v", got.ParserConfig)+ }+ if node["llm_id"] != "node-llm" {+ t.Fatalf("node llm_id = %v, want node-llm", node["llm_id"])+ }+ nodeMeta, ok := node["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("node metadata missing: %#v", node)+ }+ if enabled, _ := nodeMeta["enabled"].(bool); enabled {+ t.Fatalf("node metadata.enabled merged without apply_kb: %#v", nodeMeta)+ }+ if _, ok := got.ParserConfig["chunk_token_num"]; !ok {+ t.Fatalf("chunk_token_num dropped from parser_config: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("enable_metadata merged without apply_kb: %#v", got.ParserConfig)+ }+ if _, ok := got.ParserConfig["metadata"]; ok {+ t.Fatalf("metadata merged without apply_kb: %#v", got.ParserConfig)+ }+}++// TestStartParseDocuments_ApplyKBPersistFailureNamesDocument makes the+// persistence step fail and checks the caller gets a locatable error instead+// of a bare DAO failure.+func TestStartParseDocuments_ApplyKBPersistFailureNamesDocument(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{+ "llm_id": "kb-llm",+ }).Error; err != nil {+ t.Fatalf("set kb parser_config: %v", err)+ }+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{+ "location": "loc-1",+ "parser_config": entity.JSONMap{+ "llm_id": "doc-llm",+ "Extractor:AutoExtractDefault": map[string]any{+ "llm_id": "",+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{},+ },+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }+ // Any write to the document table now fails, so the merge cannot be persisted.+ if err := dao.DB.Exec("CREATE TRIGGER fail_document_update BEFORE UPDATE ON document BEGIN SELECT RAISE(ABORT, 'boom'); END;").Error; err != nil {+ t.Fatalf("install failing trigger: %v", err)+ }++ publisher := &recordingTaskPublisher{}+ svc := testDocumentService(t)+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)+ ctx := t.Context()++ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("load kb: %v", err)+ }+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load doc: %v", err)+ }++ err = svc.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{ApplyKB: true})+ if err == nil {+ t.Fatal("expected an error when the merged parser config cannot be persisted")+ }+ if !strings.Contains(err.Error(), "doc-1") {+ t.Fatalf("error %q does not name the document", err.Error())+ }+ if len(publisher.messages) != 0 {+ t.Fatalf("expected no ingestion task to be published, got %d", len(publisher.messages))+ }+ // The caller's original parser_config must be untouched when the persist+ // fails, including nested component nodes (the merge works on a deep+ // copy and is only assigned back after a successful persist).+ if doc.ParserConfig["llm_id"] != "doc-llm" {+ t.Fatalf("input doc.ParserConfig mutated on failure: %#v", doc.ParserConfig)+ }+ if _, ok := doc.ParserConfig["enable_metadata"]; ok {+ t.Fatalf("enable_metadata leaked into input doc.ParserConfig on failure: %#v", doc.ParserConfig)+ }+ if _, ok := doc.ParserConfig["metadata"]; ok {+ t.Fatalf("metadata leaked into input doc.ParserConfig on failure: %#v", doc.ParserConfig)+ }+ node, ok := doc.ParserConfig["Extractor:AutoExtractDefault"].(map[string]any)+ if !ok {+ t.Fatalf("input Extractor node missing on failure: %#v", doc.ParserConfig)+ }+ if node["llm_id"] != "" {+ t.Fatalf("input node llm_id mutated on failure: %#v", node)+ }+ nodeMeta, ok := node["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("input node metadata missing on failure: %#v", node)+ }+ if enabled, _ := nodeMeta["enabled"].(bool); !enabled {+ t.Fatalf("input node metadata.enabled mutated on failure: %#v", nodeMeta)+ }+ if !metadataFieldKeys(nodeMeta)["author"] {+ t.Fatalf("input node metadata fields mutated on failure: %#v", nodeMeta)+ }+}++// TestApplyKnowledgebaseParserConfig_NilInputReturnsError pins the guard: a+// nil document or nil dataset must surface an error instead of silently+// skipping the merge.+func TestApplyKnowledgebaseParserConfig_NilInputReturnsError(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 9, 4)++ svc := testDocumentService(t)+ ctx := t.Context()++ kb, err := svc.kbDAO.GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("load kb: %v", err)+ }+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load doc: %v", err)+ }++ if err := svc.applyKnowledgebaseParserConfig(ctx, nil, kb); err == nil {+ t.Fatal("expected an error for a nil document")+ }+ if err := svc.applyKnowledgebaseParserConfig(ctx, doc, nil); err == nil {+ t.Fatal("expected an error for a nil dataset")+ }+}+func TestStartParseDocuments_EnqueuesIngestionTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)