RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): fit extractor and tagger prompts to model context (#18095)
jay7772186b6f64abb8e
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 177b31f8f1..f9f1295644 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -80,6 +80,8 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common"+ "ragflow/internal/component/messagefit"+ "ragflow/internal/dao""ragflow/internal/engine/redis""ragflow/internal/entity""ragflow/internal/entity/models"@@ -1134,6 +1136,11 @@ func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extracreturn nil, err}msgs := buildExtractorMessages(in.systemPrompt, in.prompt, chunkText, in.chunks)+ fitted, fitErr := fitExtractorMessages(ctx, db, in.llmID, msgs)+ if fitErr != nil {+ return nil, fitErr+ }+ msgs = fittedinv := getExtractorChatInvoker()req := extractorChatRequest{Driver: driver,@@ -1354,6 +1361,158 @@ func isBareTenantModelID(s string) bool {return true}+// extractorContextLengthOverride is a narrow test seam mirroring+// extractorChatTargetResolverOverride: it lets unit tests supply a context+// length without a real tenant model row, so the message-fitting wiring in+// callRaw/llmTagChunk can be exercised without a DB. When set,+// extractorContextLength consults it first.+var (+ extractorContextLengthOverrideMu sync.RWMutex+ extractorContextLengthOverride func(ctx context.Context, llmID string) int+)++// SetExtractorContextLengthOverride swaps the package-level context-length+// resolver for tests. Pass nil to restore the default. Concurrent-safe.+func SetExtractorContextLengthOverride(fn func(ctx context.Context, llmID string) int) {+ extractorContextLengthOverrideMu.Lock()+ defer extractorContextLengthOverrideMu.Unlock()+ extractorContextLengthOverride = fn+}++func getExtractorContextLengthOverride() func(ctx context.Context, llmID string) int {+ extractorContextLengthOverrideMu.RLock()+ defer extractorContextLengthOverrideMu.RUnlock()+ return extractorContextLengthOverride+}++// extractorContextLength returns the chat model's context window+// (content_length) for the effective chat model used by a call, or 0 when+// unavailable. Mirrors Python's chat_mdl.max_length. Used as the token+// budget for message fitting so oversized prompts are trimmed instead of+// rejected by the provider. When llm_id is empty the call falls back to the+// tenant default chat model (see resolveExtractorChatTarget), so the same+// model is resolved here; otherwise the default-model path would never get+// message fitting. Returns 0 (skip fitting) when the model is unknown (e.g.+// unit tests with synthetic llm_id or no canvas state).+func extractorContextLength(ctx context.Context, db *gorm.DB, llmID string) int {+ if fn := getExtractorContextLengthOverride(); fn != nil {+ return fn(ctx, llmID)+ }+ if db == nil {+ db = dao.DB+ }+ state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx)+ if err != nil || state == nil {+ return 0+ }+ tidVal, _ := state.GetGlobal("tenant_id")+ tid, _ := tidVal.(string)+ if tid == "" {+ return 0+ }+ if llmID == "" {+ llmID = defaultChatModelRef(ctx, db, tid)+ }+ if llmID == "" {+ return 0+ }+ return dao.ResolveModelContentLength(ctx, db, llmID, "", "")+}++// defaultChatModelRef returns the tenant's default chat model reference —+// the tenant_model UUID when one is pinned, otherwise the composite+// "model@provider" id — or "" when the tenant has no default chat model.+func defaultChatModelRef(ctx context.Context, db *gorm.DB, tenantID string) string {+ if db == nil {+ // No database to read the tenant's default chat model from.+ return ""+ }+ tenant, err := dao.NewTenantDAO().GetByID(ctx, db, tenantID)+ if err != nil || tenant == nil {+ return ""+ }+ if tenant.TenantLLMID != nil && *tenant.TenantLLMID != "" {+ return *tenant.TenantLLMID+ }+ return tenant.LLMID+}++// extractorContextFitBudget returns 97% of the model's context window as the+// fitting budget, mirroring the agent component's contextFitBudget. The+// margin leaves headroom for the difference between the cl100k tokenizer used+// for counting and the model's own tokenizer, plus per-message formatting+// overhead, so a fitted prompt stays inside the provider's real context limit+// instead of landing exactly on it.+func extractorContextFitBudget(ctxLen int) int {+ budget := int(float64(ctxLen) * 0.97)+ if budget < 1 {+ // Never hand messagefit a <=0 budget: Fit treats <=0 as the 8192+ // default, which would stop trimming entirely for a tiny context.+ return 1+ }+ return budget+}++// fitExtractorMessages trims msgs to the chat model's context window using+// the shared messagefit fitter (mirrors Python's message_fit_in), dropping+// entries the fitter removed. It returns a clear error instead of letting a+// conversation whose final user turn was trimmed to empty reach the provider:+// the proportional trim can do that when the system prompt alone exceeds the+// context budget, and providers reject empty user turns with an obscure error+// after retries.+func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs []eschema.Message) ([]eschema.Message, error) {+ ctxLen := extractorContextLength(ctx, db, llmID)+ if ctxLen <= 0 {+ return msgs, nil+ }+ fitMsgs := make([]messagefit.Message, len(msgs))+ for i := range msgs {+ fitMsgs[i] = messagefit.Message{Role: string(msgs[i].Role), Content: msgs[i].Content}+ }+ kept, keptIdx, _ := messagefit.Fit(fitMsgs, extractorContextFitBudget(ctxLen))++ fitted := make([]eschema.Message, 0, len(kept))+ for j, i := range keptIdx {+ msgs[i].Content = kept[j].Content+ fitted = append(fitted, msgs[i])+ }+ if len(fitted) == 0 {+ return nil, errors.New("extractor: message fitting dropped every message; check the chat model context length setting")+ }+ // The system prompt carries the extraction contract (output format,+ // field definitions); sending without it would silently produce+ // garbage. The proportional trim can empty every system message when+ // the final user turn alone fills the budget, so require at least one+ // retained system message with non-empty content — a system message kept+ // but trimmed to empty is just as useless as a dropped one. The guard only+ // applies when the input actually had a system message: systemPrompt is+ // optional and a user-only prompt is a valid request.+ hadSystem := false+ for _, m := range msgs {+ if m.Role == eschema.System {+ hadSystem = true+ break+ }+ }+ if hadSystem {+ keptSystem := false+ for _, m := range fitted {+ if m.Role == eschema.System && strings.TrimSpace(m.Content) != "" {+ keptSystem = true+ break+ }+ }+ if !keptSystem {+ return nil, errors.New("extractor: message fitting emptied the system prompt; check the chat model context length setting or reduce the prompt size")+ }+ }+ last := fitted[len(fitted)-1]+ if last.Role != eschema.User || strings.TrimSpace(last.Content) == "" {+ return nil, errors.New("extractor: message fitting emptied the final user turn; check the chat model context length setting or reduce the prompt size")+ }+ return fitted, nil+}+// buildExtractorMessages assembles system + user messages for// one extraction call. The user prompt is rendered as// "<prompt>\n\n<chunkText>" so the python behavior ofdiff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.goindex 9a2b0590a7..6d4dad1235 100644--- a/internal/ingestion/component/extractor_tag.go+++ b/internal/ingestion/component/extractor_tag.go@@ -191,7 +191,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in excase <-ctx.Done():return}- llmTagChunk(ctx, inv, docsToTag[idx], indexed.allTags, examples, in.llmID, driver, model, apiKey, baseURL, topN)+ llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.llmID, driver, model, apiKey, baseURL, topN)}(i)}wg.Wait()@@ -652,6 +652,7 @@ func roundInt(f float64) int {func llmTagChunk(ctx context.Context,+ db *gorm.DB,inv extractorChatInvoker,chunk map[string]any,allTags map[string]float64,@@ -687,6 +688,17 @@ func llmTagChunk({Role: eschema.System, Content: prompt},{Role: eschema.User, Content: "Output:"},}+ // Trim the prompt to the model's context window before sending. The+ // system prompt embeds the full chunk text, the entire tag set and up+ // to two full examples, so oversized chunks or tag files would+ // otherwise be rejected by the provider (context length exceeded).+ // Mirrors Python's message_fit_in in content_tagging (generator.py:331).+ fitted, fitErr := fitExtractorMessages(ctx, db, llmID, msgs)+ if fitErr != nil {+ common.Warn("extractor tags: skipping LLM tagging, message fitting failed", zap.Error(fitErr))+ return+ }+ msgs = fittedtemperature := 0.5var result map[string]intdiff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.goindex 6161281e1a..9e7c8df374 100644--- a/internal/ingestion/component/extractor_tag_test.go+++ b/internal/ingestion/component/extractor_tag_test.go@@ -12,6 +12,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/tokenizer")type stubExtractorTagChat struct {@@ -236,6 +237,86 @@ func stubXLSXBytes(t *testing.T) []byte {return buf.Bytes()}+// capturingExtractorTagChat records the request it was given so tests can+// assert on the exact messages sent to the LLM.+type capturingExtractorTagChat struct {+ req extractorChatRequest+}++func (c *capturingExtractorTagChat) Chat(_ context.Context, req extractorChatRequest) (*extractorChatResponse, error) {+ c.req = req+ return &extractorChatResponse{Content: `{"RAG": 8, "vector database": 6}`}, nil+}++func pushCapturingTagChat(t *testing.T) *capturingExtractorTagChat {+ t.Helper()+ capt := &capturingExtractorTagChat{}+ SetExtractorChatInvoker(capt)+ t.Cleanup(func() { SetExtractorChatInvoker(nil) })+ return capt+}++func longChunkText() string {+ return strings.Repeat("RAGFlow is an open source retrieval augmented generation engine. ", 10)+}++// TestLlmtagChunk_MessageFit verifies that llmTagChunk trims the prompt to the+// model's context window before sending, mirroring Python's message_fit_in in+// content_tagging (generator.py:331). The tagger system prompt embeds the full+// chunk text plus the whole tag set, so an oversized chunk must be trimmed+// rather than rejected by the provider with "context length exceeded".+func TestLlmtagChunk_MessageFit(t *testing.T) {+ const budget = 20+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return budget })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })++ capt := pushCapturingTagChat(t)++ longText := longChunkText()+ chunk := map[string]any{"content_with_weight": longText}+ allTags := map[string]float64{"RAG": 1, "database": 1, "AI": 1}+ examples := []schema.TaggedChunk{{Content: "example one", TagWeights: map[string]int{"AI": 5}}}++ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3)++ if len(capt.req.Messages) != 2 {+ t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))+ }+ // The full chunk text must have been trimmed away from the system prompt.+ if strings.Contains(capt.req.Messages[0].Content, longText) {+ t.Fatal("system prompt was not trimmed to the context budget")+ }+ total := tokenizer.NumTokensFromString(capt.req.Messages[0].Content) ++ tokenizer.NumTokensFromString(capt.req.Messages[1].Content)+ // The budget is 97% of the resolved content_length (the override value),+ // mirroring the agent's contextFitBudget; the margin keeps a fitted+ // prompt inside the provider's real context limit.+ if total > extractorContextFitBudget(budget) {+ t.Fatalf("fitted messages total %d tokens exceeds the margin-adjusted budget %d", total, extractorContextFitBudget(budget))+ }+}++// TestLlmtagChunk_NoContextLength_SkipsFit verifies the guard: when the model's+// context length is unknown (extractorContextLength returns 0), the tagger+// passes the prompt through untrimmed instead of erroring.+func TestLlmtagChunk_NoContextLength_SkipsFit(t *testing.T) {+ capt := pushCapturingTagChat(t)++ longText := longChunkText()+ chunk := map[string]any{"content_with_weight": longText}+ allTags := map[string]float64{"RAG": 1}+ examples := []schema.TaggedChunk{{Content: "example", TagWeights: map[string]int{"AI": 5}}}++ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3)++ if len(capt.req.Messages) != 2 {+ t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))+ }+ if !strings.Contains(capt.req.Messages[0].Content, longText) {+ t.Fatal("system prompt should pass through untrimmed when context length is unknown")+ }+}+func TestParseCSVQuoteAwareReader(t *testing.T) {// A quoted content field containing a comma must not be split into extra// columns: "RAGFlow, the guide",RAG is two fields, not three.diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex d10236d2b2..0e882b23df 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -19,7 +19,6 @@ package componentimport ("context""errors"- "ragflow/internal/dao""strings""sync""sync/atomic"@@ -27,10 +26,15 @@ import ("time"eschema "github.com/cloudwego/eino/schema"+ "github.com/glebarez/sqlite"+ "gorm.io/gorm""ragflow/internal/agent/runtime""ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/tokenizer""ragflow/internal/utility")@@ -1792,3 +1796,264 @@ func TestExtractorComponent_Invoke_FieldValueContainsPlaceholderSubstring(t *test.Errorf("expected title substitution to produce literal '{text}' label: %q", userContent)}}++// TestFitExtractorMessages_RejectsEmptyUserTurn verifies that when+// messagefit's proportional trim would empty the final user turn (the system+// prompt alone exceeds the context budget), the extractor surfaces a clear+// error instead of sending [system, user:""] to the provider.+func TestFitExtractorMessages_RejectsEmptyUserTurn(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 500 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })++ msgs := []eschema.Message{+ {Role: eschema.System, Content: strings.Repeat("s ", 1000)},+ {Role: eschema.User, Content: strings.Repeat("u ", 400)},+ }+ if _, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs); err == nil {+ t.Fatal("expected an error when fitting empties the user turn")+ }+}++// TestFitExtractorMessages_KeepsUserTurn verifies the happy path: with a+// normal budget the fitter trims oversized prompts and the final user turn+// survives, so no error is returned.+func TestFitExtractorMessages_KeepsUserTurn(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })++ msgs := []eschema.Message{+ {Role: eschema.System, Content: "you are a helpful assistant"},+ {Role: eschema.User, Content: strings.Repeat("u ", 3000)},+ }+ fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if len(fitted) != 2 {+ t.Fatalf("got %d messages, want 2", len(fitted))+ }+ if strings.TrimSpace(fitted[1].Content) == "" {+ t.Fatal("user turn was emptied")+ }+}++// TestFitExtractorMessages_NoSystemPromptKeepsUserTurn verifies that a+// user-only request (no system prompt configured) is not rejected by the+// system-prompt guard: the guard only applies when a system message was+// actually present, so a valid prompt-only extractor keeps working once the+// model's content_length is resolvable.+func TestFitExtractorMessages_NoSystemPromptKeepsUserTurn(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 2000 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })++ msgs := []eschema.Message{+ {Role: eschema.User, Content: strings.Repeat("u ", 3000)},+ }+ fitted, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs)+ if err != nil {+ t.Fatalf("unexpected error for user-only prompt: %v", err)+ }+ if len(fitted) != 1 || fitted[0].Role != eschema.User {+ t.Fatalf("got %d messages, want the single user turn: %+v", len(fitted), fitted)+ }+ if strings.TrimSpace(fitted[0].Content) == "" {+ t.Fatal("user turn was emptied")+ }+}++// TestExtractorComponent_CallRaw_FitsBeforeInvoke verifies the production+// wiring end to end: callRaw resolves the model's context length, trims the+// messages to the budget, and hands the fitted messages to the invoker.+func TestExtractorComponent_CallRaw_FitsBeforeInvoke(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 200 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })++ stub := withStubChatInvoker(t, stubResponse{Content: `{"ok": true}`})+ c := &ExtractorComponent{}++ _, err := c.callText(t.Context(), nil, extractorInputs{+ systemPrompt: "extract fields",+ prompt: "summarize",+ llmID: "test@test",+ }, strings.Repeat("chunk text with lots of tokens. ", 500))+ if err != nil {+ t.Fatalf("callText: %v", err)+ }++ stub.mu.Lock()+ req := stub.lastReq+ stub.mu.Unlock()+ if len(req.Messages) == 0 {+ t.Fatal("invoker was not called")+ }+ if req.Messages[0].Role != eschema.System || strings.TrimSpace(req.Messages[0].Content) == "" {+ t.Fatalf("system prompt lost or emptied before invoke: %+v", req.Messages[0])+ }+ total := 0+ for _, m := range req.Messages {+ total += tokenizer.NumTokensFromString(m.Content)+ }+ if total > extractorContextFitBudget(200) {+ t.Fatalf("sent messages total %d exceed the fitting budget %d", total, extractorContextFitBudget(200))+ }+ if !strings.Contains(req.Messages[len(req.Messages)-1].Content, "chunk text") {+ t.Fatal("chunk text lost from the user turn")+ }+}++// openExtractorContextTestDB returns an in-memory DB with the tenant and+// tenant-model tables migrated. Tests pass the returned handle explicitly to+// extractorContextLength, defaultChatModelRef, and dao.ResolveModelContentLength,+// so no global DAO state is touched.+func openExtractorContextTestDB(t *testing.T) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err := db.AutoMigrate(&entity.Tenant{}, &entity.TenantModelProvider{}, &entity.TenantModel{}); err != nil {+ t.Fatalf("migrate: %v", err)+ }+ return db+}++// seedExtractorContextModel seeds an active OpenAI gpt-4o tenant model+// (catalog content_length 128000) plus its tenant. tenantLLMID, when+// non-empty, pins the tenant's default chat model to the tenant-model UUID;+// otherwise the tenant falls back to the composite llm_id.+func seedExtractorContextModel(t *testing.T, db *gorm.DB, tenantLLMID string) {+ t.Helper()+ status := "1"+ tenant := entity.Tenant{+ ID: "tenant-1",+ LLMID: "gpt-4o@openai",+ Status: &status,+ }+ if tenantLLMID != "" {+ tenant.TenantLLMID = &tenantLLMID+ }+ if err := db.Create(&tenant).Error; err != nil {+ t.Fatalf("create tenant: %v", err)+ }+ if err := db.Create(&entity.TenantModelProvider{+ ID: "provider-openai",+ ProviderName: "OpenAI",+ TenantID: "tenant-1",+ }).Error; err != nil {+ t.Fatalf("create provider: %v", err)+ }+ if err := db.Create(&entity.TenantModel{+ ID: "0123456789abcdef0123456789abcdef",+ ProviderID: "provider-openai",+ InstanceID: "instance-1",+ ModelName: "gpt-4o",+ ModelType: int(entity.ModelTypeChat),+ Status: "active",+ }).Error; err != nil {+ t.Fatalf("create model: %v", err)+ }+}++// extractorStateCtx returns a context carrying a canvas state with the given+// tenant_id global, as extractorContextLength expects.+func extractorStateCtx(t *testing.T, tenantID string) context.Context {+ t.Helper()+ state := runtime.NewCanvasState("run-1", "session-1")+ state.SetGlobal("tenant_id", tenantID)+ return runtime.WithState(t.Context(), state)+}++// TestExtractorContextLength_TenantModelUUID verifies extractorContextLength+// resolves content_length for a tenant_model UUID through the provider+// catalog.+func TestExtractorContextLength_TenantModelUUID(t *testing.T) {+ db := openExtractorContextTestDB(t)+ seedExtractorContextModel(t, db, "")+ ctx := extractorStateCtx(t, "tenant-1")++ if got := extractorContextLength(ctx, db, "0123456789abcdef0123456789abcdef"); got != 128000 {+ t.Fatalf("extractorContextLength(uuid) = %d, want 128000", got)+ }+}++// TestExtractorContextLength_DefaultChatModelPinned verifies the llmID==""+// fallback resolves the tenant default chat model when it is pinned to a+// tenant_model UUID.+func TestExtractorContextLength_DefaultChatModelPinned(t *testing.T) {+ db := openExtractorContextTestDB(t)+ seedExtractorContextModel(t, db, "0123456789abcdef0123456789abcdef")+ ctx := extractorStateCtx(t, "tenant-1")++ if got := extractorContextLength(ctx, db, ""); got != 128000 {+ t.Fatalf("extractorContextLength(default pinned uuid) = %d, want 128000", got)+ }+}++// TestExtractorContextLength_DefaultChatModelComposite verifies the llmID==""+// fallback resolves the tenant default chat model from the composite+// "model@provider" llm_id when no tenant_model is pinned.+func TestExtractorContextLength_DefaultChatModelComposite(t *testing.T) {+ db := openExtractorContextTestDB(t)+ seedExtractorContextModel(t, db, "")+ ctx := extractorStateCtx(t, "tenant-1")++ if got := extractorContextLength(ctx, db, ""); got != 128000 {+ t.Fatalf("extractorContextLength(default composite) = %d, want 128000", got)+ }+}++// TestExtractorContextLength_UnknownModelSkips verifies extractorContextLength+// returns 0 (skip fitting) for an unknown model reference.+func TestExtractorContextLength_UnknownModelSkips(t *testing.T) {+ db := openExtractorContextTestDB(t)+ seedExtractorContextModel(t, db, "")+ ctx := extractorStateCtx(t, "tenant-1")++ if got := extractorContextLength(ctx, db, "no-such-model@no-such-provider"); got != 0 {+ t.Fatalf("extractorContextLength(unknown) = %d, want 0", got)+ }+}++// TestExtractorContextFitBudget verifies the fitting budget is 97% of the+// resolved content_length (mirroring the agent's contextFitBudget), leaving+// headroom for tokenizer drift between cl100k and the model's own tokenizer,+// and that a tiny context never collapses to messagefit's <=0 → 8192 default.+func TestExtractorContextFitBudget(t *testing.T) {+ if got := extractorContextFitBudget(128000); got != 124160 {+ t.Fatalf("extractorContextFitBudget(128000) = %d, want 124160", got)+ }+ if got := extractorContextFitBudget(1); got != 1 {+ t.Fatalf("extractorContextFitBudget(1) = %d, want 1 (clamped to avoid the 8192 Fit default)", got)+ }+}++// TestFitExtractorMessages_RejectsSystemPromptLoss verifies the guard that a+// fitting which empties every system message is rejected instead of sending+// an instruction-less extraction request: the system prompt carries the+// extraction contract, so running with an emptied system prompt would+// silently produce garbage.+func TestFitExtractorMessages_RejectsSystemPromptLoss(t *testing.T) {+ SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return 300 })+ t.Cleanup(func() { SetExtractorContextLengthOverride(nil) })++ // System dominates (>4x the user) and the user message alone exceeds+ // the budget: the proportional trim preserves the user turn and empties+ // the system messages.+ msgs := []eschema.Message{+ {Role: eschema.System, Content: strings.Repeat("s ", 5000)},+ {Role: eschema.User, Content: strings.Repeat("u ", 400)},+ }+ if _, err := fitExtractorMessages(t.Context(), nil, "test@test", msgs); err == nil {+ t.Fatal("expected an error when fitting empties the system prompt")+ }+}++// TestExtractorContextLength_NilDBGraceful verifies that resolving the tenant+// default chat model with no database available (nil db and no override)+// degrades to 0 (skip fitting) instead of panicking in defaultChatModelRef.+func TestExtractorContextLength_NilDBGraceful(t *testing.T) {+ ctx := extractorStateCtx(t, "tenant-1")+ if got := extractorContextLength(ctx, nil, ""); got != 0 {+ t.Fatalf("extractorContextLength(nil db, default model) = %d, want 0 (skip fitting)", got)+ }+}