RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: verify decision model (#21492)
Lynnf5a7cc568291
diff --git a/internal/entity/models/systemone.go b/internal/entity/models/systemone.goindex b14543523c..91f0642fd2 100644--- a/internal/entity/models/systemone.go+++ b/internal/entity/models/systemone.go@@ -26,7 +26,8 @@ import ("ragflow/internal/common")-// systemOneDefaultSuffix is the decision endpoint declared as url_suffix.decision.+// systemOneDefaultSuffix is the decision endpoint this protocol uses, declared+// in the provider catalog as url_suffix.decision. Decide builds its URL from it.const systemOneDefaultSuffix = "systemone"// systemOneDefaultModelsSuffix lists the models the configured deployment@@ -36,8 +37,13 @@ const systemOneDefaultModelsSuffix = "models"// SystemOneModel implements ModelDriver for endpoints speaking TypeSafe System// One: one request carrying several questions, answered in a single round trip.//-// Decide supplies the shared request path for workflow classification and-// retrieval judging. Chat completions are not supported by this protocol.+// It embeds OpenAIAPICompatibleModel only to satisfy the rest of the (wide)+// ModelDriver interface. Decide is one of the two methods it implements itself:+// a probe of the decision endpoint, which is what verification calls.+//+// What this driver is for is REGISTRATION: making the provider appear in the+// picker, making its model appear once a base URL has been shown to work, and+// making that model pass the verification the model form runs before saving.type SystemOneModel struct {*OpenAIAPICompatibleModel}diff --git a/internal/entity/models/systemone_test.go b/internal/entity/models/systemone_test.goindex 8b5ec2400c..b193cbcd6d 100644--- a/internal/entity/models/systemone_test.go+++ b/internal/entity/models/systemone_test.go@@ -94,6 +94,55 @@ func TestSystemOneListModelsProbesBaseURL(t *testing.T) {}}+// A decision model that cannot answer must not be addable: verification is the+// last check before the model is saved, and it is the only one that speaks the+// decision endpoint.+//+// The request is a well-formed one on purpose. A malformed request is rejected+// before the transport call, so a test that sent one would pass here without the+// server ever being asked — and would stay green if verification's status+// handling broke. The "asked" counter below makes that impossible.+func TestSystemOneDecideFailsWhenEndpointRejects(t *testing.T) {+ withSSRFBypass(t)++ for _, tc := range []struct {+ name string+ status int+ }{+ {"rejected credential", http.StatusUnauthorized},+ {"no decision endpoint", http.StatusNotFound},+ {"server error", http.StatusInternalServerError},+ } {+ t.Run(tc.name, func(t *testing.T) {+ asked := false+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ asked = true+ w.WriteHeader(tc.status)+ _, _ = w.Write([]byte(`{"error":"no"}`))+ }))+ defer srv.Close()++ apiKey := "sk-test"+ modelName := "decision-model-preview"+ resp, err := newSystemOneForTest(srv.URL+"/v1").Decide(+ context.Background(), &modelName, &DecideRequest{+ State: map[string]string{"passage": ""},+ Questions: map[string]Question{"probe": Noul{Instructions: "Is this relevant?"}},+ }, &APIConfig{ApiKey: &apiKey}, nil,+ )+ if err == nil {+ t.Fatalf("Decide accepted status %d", tc.status)+ }+ if resp != nil {+ t.Errorf("Decide returned a response alongside the error")+ }+ if !asked {+ t.Fatal("the endpoint was never asked, so this proves nothing about its answer")+ }+ })+ }+}+// An unreachable base URL must fail the listing rather than fall back to the// static catalog, or the model would appear even though it cannot be called.func TestSystemOneListModelsFailsWhenProbeFails(t *testing.T) {diff --git a/internal/service/model_service.go b/internal/service/model_service.goindex ed4c00ae67..d1a452e05c 100644--- a/internal/service/model_service.go+++ b/internal/service/model_service.go@@ -1131,6 +1131,7 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, prvar errs []errorerrSet := make(map[string]bool)passedTypes := make(map[string]bool)+ unverifiableTypes := make(map[string]bool)for _, model := range modelsToVerify {modelName := model.Name@@ -1187,7 +1188,10 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, prerr = verifyOCRModel(ctx, driver, modelName, apiConfig)case "doc_parse":err = driver.CheckConnection(ctx, apiConfig)+ case "decision":+ err = verifyDecisionModel(ctx, driver, modelName, apiConfig)default:+ unverifiableTypes[mtLower] = truecontinue}@@ -1212,6 +1216,22 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, pr}if len(passedTypes) == 0 {+ // An empty errs must not reach %w: errors.Join over no arguments returns+ // nil, and %w renders a nil error as "%!w(<nil>)". Nothing was attempted+ // in that case, so there is no provider error to report — name the type+ // that has no verifier instead, which is what the form reports to whoever+ // is trying to add the model.+ if len(errs) == 0 {+ unverifiable := make([]string, 0, len(unverifiableTypes))+ for modelType := range unverifiableTypes {+ unverifiable = append(unverifiable, modelType)+ }+ sort.Strings(unverifiable)+ if len(unverifiable) == 0 {+ return modelVerifyResult, fmt.Errorf("model verification did not run: the requested models declare no model type")+ }+ return modelVerifyResult, fmt.Errorf("model verification did not run: %s has no verification in this build", strings.Join(unverifiable, ", "))+ }return modelVerifyResult, fmt.Errorf("model verification attempts failed: %w", errors.Join(errs...))}@@ -1328,6 +1348,36 @@ func verifyOCRModel(ctx context.Context, driver modelModule.ModelDriver, modelNareturn nil}+// decisionProbeQuestionID names the single question verification sends, whose+// answer is discarded: Decide requires at least one question, and verification+// asks nothing else.+const decisionProbeQuestionID = "verification"++// verifyDecisionModel asks the decision endpoint the verification question.+//+// It goes through Decide rather than CheckConnection: CheckConnection is the+// models list, which the picker already called to offer this model, so reusing+// it here would verify nothing that has not been verified. Decide exercises the+// endpoint the product will call.+//+// The request mirrors what the judge sends (decision_judge.go — same state key,+// same question type) so verification exercises the shape the product sends, and+// so a deployment that answers verification answers the real thing too. Only the+// instructions are synthetic, and deliberately so: no calibration depends on+// them, which keeps the request cheap and independent of the judge's phrasing.+func verifyDecisionModel(ctx context.Context, driver modelModule.ModelDriver, modelName string, apiConfig *modelModule.APIConfig) error {+ request := &modelModule.DecideRequest{+ State: map[string]string{"passage": ""},+ Questions: map[string]modelModule.Question{+ decisionProbeQuestionID: modelModule.Noul{Instructions: "Is this passage relevant to the query?"},+ },+ }+ if _, err := driver.Decide(ctx, &modelName, request, apiConfig, nil); err != nil {+ return err+ }+ return nil+}+// minimalPNG returns a 1×1 white PNG as a byte slice for OCR verification.func minimalPNG() []byte {return []byte{diff --git a/internal/service/model_service_test.go b/internal/service/model_service_test.goindex 9ca788654f..d0f31e7430 100644--- a/internal/service/model_service_test.go+++ b/internal/service/model_service_test.go@@ -2,7 +2,10 @@ package serviceimport ("context"+ "encoding/json""errors"+ "net/http"+ "net/http/httptest""strings""testing"@@ -230,6 +233,143 @@ func TestVerifyProviderModelValidatesRemoteEmbeddingMetadata(t *testing.T) {}}+// A decision model has no verification except the decision endpoint, so the+// type must be dispatched to Decide; falling through the switch leaves nothing+// passing and the model fails to add even though it works.+// decisionProbeDriver stands in for a driver whose decision endpoint is the only+// thing worth verifying. It records what Decide was asked, because the point of+// the decision case in verifyProviderModel is which call it dispatches to.+type decisionProbeDriver struct {+ *modelModule.DummyModel+ decideCalls int+ decidedModel string+ decideErr error+}++func (d *decisionProbeDriver) Decide(_ context.Context, modelName *string, _ *modelModule.DecideRequest, _ *modelModule.APIConfig, _ *common.ModelUsage) (*modelModule.DecideResponse, error) {+ d.decideCalls+++ if modelName != nil {+ d.decidedModel = *modelName+ }+ if d.decideErr != nil {+ return nil, d.decideErr+ }+ return &modelModule.DecideResponse{}, nil+}++func TestVerifyProviderModelVerifiesDecisionModels(t *testing.T) {+ driver := &decisionProbeDriver{DummyModel: modelModule.NewDummyModel(nil, modelModule.URLSuffix{})}++ result, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"), driver,+ []*modelModule.Model{{Name: "decision-model-preview", ModelTypes: []string{entity.ModelTypeDecision.String()}}},+ &modelModule.APIConfig{}, nil)+ if err != nil {+ t.Fatalf("verifyProviderModel() error = %v", err)+ }+ if driver.decideCalls != 1 {+ t.Fatalf("Decide calls = %d, want 1", driver.decideCalls)+ }+ if driver.decidedModel != "decision-model-preview" {+ t.Fatalf("Decide model = %q, want decision-model-preview", driver.decidedModel)+ }+ if result["decision-model-preview"] != entity.ModelVerifySuccess {+ t.Fatalf("verification result = %#v, want success", result)+ }+}++// A type falling through the switch collects no error, so the failure must name+// it: joining no errors returns nil, which %w renders as "%!w(<nil>)" — the+// message that hid this very case while decision had no verifier.+func TestVerifyProviderModelNamesUnverifiableTypes(t *testing.T) {+ driver := modelModule.NewDummyModel(nil, modelModule.URLSuffix{})++ _, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"), driver,+ []*modelModule.Model{{Name: "hypothetical-model", ModelTypes: []string{"quantum"}}},+ &modelModule.APIConfig{}, nil)+ if err == nil {+ t.Fatal("verifyProviderModel() error = nil, want a failure")+ }+ if strings.Contains(err.Error(), "%!w") {+ t.Fatalf("verifyProviderModel() leaked a formatting error: %v", err)+ }+ if !strings.Contains(err.Error(), "quantum") {+ t.Fatalf("verifyProviderModel() error = %v, want it to name the unverifiable type", err)+ }+}++// Verification builds the request Decide sends, so it needs exercising against+// a real driver. The stand-in used above accepts any request — including one the+// endpoint would refuse — so it cannot catch verification asking for nothing.+func TestVerifyProviderModelAsksSystemOneDecisionEndpoint(t *testing.T) {+ prev := common.AllowAnyHostForTest+ common.AllowAnyHostForTest = true+ t.Cleanup(func() { common.AllowAnyHostForTest = prev })++ asked := false+ var request struct {+ Model string `json:"model"`+ State map[string]string `json:"state"`+ Questions map[string]struct {+ Type string `json:"type"`+ Instructions string `json:"instructions"`+ } `json:"questions"`+ }+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ asked = true+ if r.URL.Path != "/v1/systemone" {+ t.Errorf("path=%s, want /v1/systemone", r.URL.Path)+ }+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Errorf("undecodable request body: %v", err)+ }+ _, _ = w.Write([]byte(`{"answers":{"verification":{"type":"noul","noul":0.5}}}`))+ }))+ defer srv.Close()++ apiKey := "sk-test"+ result, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"),+ modelModule.NewSystemOneModel(map[string]string{"default": srv.URL + "/v1"},+ modelModule.URLSuffix{Decision: "systemone", Models: "models"}),+ []*modelModule.Model{{Name: "decision-model-preview", ModelTypes: []string{entity.ModelTypeDecision.String()}}},+ &modelModule.APIConfig{ApiKey: &apiKey}, nil)+ if err != nil {+ t.Fatalf("verifyProviderModel() error = %v", err)+ }+ if !asked {+ t.Fatal("verification never reached the decision endpoint")+ }+ if result["decision-model-preview"] != entity.ModelVerifySuccess {+ t.Fatalf("verification result = %#v, want success", result)+ }+ question, ok := request.Questions["verification"]+ if !ok {+ t.Fatalf("request questions = %#v, want the verification question", request.Questions)+ }+ if question.Type != "noul" || question.Instructions == "" {+ t.Errorf("verification question = %+v, want a noul question with instructions", question)+ }+}++func TestVerifyProviderModelReportsFailingDecisionModels(t *testing.T) {+ driver := &decisionProbeDriver{+ DummyModel: modelModule.NewDummyModel(nil, modelModule.URLSuffix{}),+ decideErr: errors.New("endpoint answered 404"),+ }++ result, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"), driver,+ []*modelModule.Model{{Name: "decision-model-preview", ModelTypes: []string{entity.ModelTypeDecision.String()}}},+ &modelModule.APIConfig{}, nil)+ if err == nil {+ t.Fatal("verifyProviderModel() error = nil, want the endpoint failure")+ }+ if !strings.Contains(err.Error(), "endpoint answered 404") {+ t.Fatalf("verifyProviderModel() error = %v, want the endpoint failure", err)+ }+ if result["decision-model-preview"] != entity.ModelVerifyFail {+ t.Fatalf("verification result = %#v, want fail", result)+ }+}+func TestModelInfoWithTenantExtraAppliesEmbeddingConstraints(t *testing.T) {factoryMaxDimension := 2048factoryBatchSize := 128