Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: verify decision model (#21492)

Lynnf5a7cc568291
diff --git a/internal/entity/models/systemone.go b/internal/entity/models/systemone.go
index b14543523c..91f0642fd2 100644
--- a/internal/entity/models/systemone.go
+++ b/internal/entity/models/systemone.go
@@ -26,7 +26,8 @@ import (
"ragflow/internal/common"
)
-// systemOneDefaultSuffix is the decision endpoint declared as url_suffix.decision.
+// systemOneDefaultSuffix is the decision endpoint this protocol uses, declared
+// in the provider catalog as url_suffix.decision. Decide builds its URL from it.
const systemOneDefaultSuffix = "systemone"
// systemOneDefaultModelsSuffix lists the models the configured deployment
@@ -36,8 +37,13 @@ const systemOneDefaultModelsSuffix = "models"
// SystemOneModel implements ModelDriver for endpoints speaking TypeSafe System
// One: one request carrying several questions, answered in a single round trip.
//
-// Decide supplies the shared request path for workflow classification and
-// retrieval judging. Chat completions are not supported by this protocol.
+// It embeds OpenAIAPICompatibleModel only to satisfy the rest of the (wide)
+// ModelDriver interface. Decide is one of the two methods it implements itself:
+// a probe of the decision endpoint, which is what verification calls.
+//
+// What this driver is for is REGISTRATION: making the provider appear in the
+// picker, making its model appear once a base URL has been shown to work, and
+// making that model pass the verification the model form runs before saving.
type SystemOneModel struct {
*OpenAIAPICompatibleModel
}
diff --git a/internal/entity/models/systemone_test.go b/internal/entity/models/systemone_test.go
index 8b5ec2400c..b193cbcd6d 100644
--- a/internal/entity/models/systemone_test.go
+++ b/internal/entity/models/systemone_test.go
@@ -94,6 +94,55 @@ func TestSystemOneListModelsProbesBaseURL(t *testing.T) {
}
}
+// A decision model that cannot answer must not be addable: verification is the
+// last check before the model is saved, and it is the only one that speaks the
+// decision endpoint.
+//
+// The request is a well-formed one on purpose. A malformed request is rejected
+// before the transport call, so a test that sent one would pass here without the
+// server ever being asked — and would stay green if verification's status
+// handling broke. The "asked" counter below makes that impossible.
+func TestSystemOneDecideFailsWhenEndpointRejects(t *testing.T) {
+ withSSRFBypass(t)
+
+ for _, tc := range []struct {
+ name string
+ status int
+ }{
+ {"rejected credential", http.StatusUnauthorized},
+ {"no decision endpoint", http.StatusNotFound},
+ {"server error", http.StatusInternalServerError},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ asked := false
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ asked = true
+ w.WriteHeader(tc.status)
+ _, _ = w.Write([]byte(`{"error":"no"}`))
+ }))
+ defer srv.Close()
+
+ apiKey := "sk-test"
+ modelName := "decision-model-preview"
+ resp, err := newSystemOneForTest(srv.URL+"/v1").Decide(
+ context.Background(), &modelName, &DecideRequest{
+ State: map[string]string{"passage": ""},
+ Questions: map[string]Question{"probe": Noul{Instructions: "Is this relevant?"}},
+ }, &APIConfig{ApiKey: &apiKey}, nil,
+ )
+ if err == nil {
+ t.Fatalf("Decide accepted status %d", tc.status)
+ }
+ if resp != nil {
+ t.Errorf("Decide returned a response alongside the error")
+ }
+ if !asked {
+ t.Fatal("the endpoint was never asked, so this proves nothing about its answer")
+ }
+ })
+ }
+}
+
// An unreachable base URL must fail the listing rather than fall back to the
// static catalog, or the model would appear even though it cannot be called.
func TestSystemOneListModelsFailsWhenProbeFails(t *testing.T) {
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index ed4c00ae67..d1a452e05c 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -1131,6 +1131,7 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, pr
var errs []error
errSet := make(map[string]bool)
passedTypes := make(map[string]bool)
+ unverifiableTypes := make(map[string]bool)
for _, model := range modelsToVerify {
modelName := model.Name
@@ -1187,7 +1188,10 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, pr
err = verifyOCRModel(ctx, driver, modelName, apiConfig)
case "doc_parse":
err = driver.CheckConnection(ctx, apiConfig)
+ case "decision":
+ err = verifyDecisionModel(ctx, driver, modelName, apiConfig)
default:
+ unverifiableTypes[mtLower] = true
continue
}
@@ -1212,6 +1216,22 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, pr
}
if len(passedTypes) == 0 {
+ // An empty errs must not reach %w: errors.Join over no arguments returns
+ // nil, and %w renders a nil error as "%!w(<nil>)". Nothing was attempted
+ // in that case, so there is no provider error to report — name the type
+ // that has no verifier instead, which is what the form reports to whoever
+ // is trying to add the model.
+ if len(errs) == 0 {
+ unverifiable := make([]string, 0, len(unverifiableTypes))
+ for modelType := range unverifiableTypes {
+ unverifiable = append(unverifiable, modelType)
+ }
+ sort.Strings(unverifiable)
+ if len(unverifiable) == 0 {
+ return modelVerifyResult, fmt.Errorf("model verification did not run: the requested models declare no model type")
+ }
+ return modelVerifyResult, fmt.Errorf("model verification did not run: %s has no verification in this build", strings.Join(unverifiable, ", "))
+ }
return modelVerifyResult, fmt.Errorf("model verification attempts failed: %w", errors.Join(errs...))
}
@@ -1328,6 +1348,36 @@ func verifyOCRModel(ctx context.Context, driver modelModule.ModelDriver, modelNa
return nil
}
+// decisionProbeQuestionID names the single question verification sends, whose
+// answer is discarded: Decide requires at least one question, and verification
+// asks nothing else.
+const decisionProbeQuestionID = "verification"
+
+// verifyDecisionModel asks the decision endpoint the verification question.
+//
+// It goes through Decide rather than CheckConnection: CheckConnection is the
+// models list, which the picker already called to offer this model, so reusing
+// it here would verify nothing that has not been verified. Decide exercises the
+// endpoint the product will call.
+//
+// The request mirrors what the judge sends (decision_judge.go — same state key,
+// same question type) so verification exercises the shape the product sends, and
+// so a deployment that answers verification answers the real thing too. Only the
+// instructions are synthetic, and deliberately so: no calibration depends on
+// them, which keeps the request cheap and independent of the judge's phrasing.
+func verifyDecisionModel(ctx context.Context, driver modelModule.ModelDriver, modelName string, apiConfig *modelModule.APIConfig) error {
+ request := &modelModule.DecideRequest{
+ State: map[string]string{"passage": ""},
+ Questions: map[string]modelModule.Question{
+ decisionProbeQuestionID: modelModule.Noul{Instructions: "Is this passage relevant to the query?"},
+ },
+ }
+ if _, err := driver.Decide(ctx, &modelName, request, apiConfig, nil); err != nil {
+ return err
+ }
+ return nil
+}
+
// minimalPNG returns a 1×1 white PNG as a byte slice for OCR verification.
func minimalPNG() []byte {
return []byte{
diff --git a/internal/service/model_service_test.go b/internal/service/model_service_test.go
index 9ca788654f..d0f31e7430 100644
--- a/internal/service/model_service_test.go
+++ b/internal/service/model_service_test.go
@@ -2,7 +2,10 @@ package service
import (
"context"
+ "encoding/json"
"errors"
+ "net/http"
+ "net/http/httptest"
"strings"
"testing"
@@ -230,6 +233,143 @@ func TestVerifyProviderModelValidatesRemoteEmbeddingMetadata(t *testing.T) {
}
}
+// A decision model has no verification except the decision endpoint, so the
+// type must be dispatched to Decide; falling through the switch leaves nothing
+// passing and the model fails to add even though it works.
+// decisionProbeDriver stands in for a driver whose decision endpoint is the only
+// thing worth verifying. It records what Decide was asked, because the point of
+// the decision case in verifyProviderModel is which call it dispatches to.
+type decisionProbeDriver struct {
+ *modelModule.DummyModel
+ decideCalls int
+ decidedModel string
+ decideErr error
+}
+
+func (d *decisionProbeDriver) Decide(_ context.Context, modelName *string, _ *modelModule.DecideRequest, _ *modelModule.APIConfig, _ *common.ModelUsage) (*modelModule.DecideResponse, error) {
+ d.decideCalls++
+ if modelName != nil {
+ d.decidedModel = *modelName
+ }
+ if d.decideErr != nil {
+ return nil, d.decideErr
+ }
+ return &modelModule.DecideResponse{}, nil
+}
+
+func TestVerifyProviderModelVerifiesDecisionModels(t *testing.T) {
+ driver := &decisionProbeDriver{DummyModel: modelModule.NewDummyModel(nil, modelModule.URLSuffix{})}
+
+ result, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"), driver,
+ []*modelModule.Model{{Name: "decision-model-preview", ModelTypes: []string{entity.ModelTypeDecision.String()}}},
+ &modelModule.APIConfig{}, nil)
+ if err != nil {
+ t.Fatalf("verifyProviderModel() error = %v", err)
+ }
+ if driver.decideCalls != 1 {
+ t.Fatalf("Decide calls = %d, want 1", driver.decideCalls)
+ }
+ if driver.decidedModel != "decision-model-preview" {
+ t.Fatalf("Decide model = %q, want decision-model-preview", driver.decidedModel)
+ }
+ if result["decision-model-preview"] != entity.ModelVerifySuccess {
+ t.Fatalf("verification result = %#v, want success", result)
+ }
+}
+
+// A type falling through the switch collects no error, so the failure must name
+// it: joining no errors returns nil, which %w renders as "%!w(<nil>)" — the
+// message that hid this very case while decision had no verifier.
+func TestVerifyProviderModelNamesUnverifiableTypes(t *testing.T) {
+ driver := modelModule.NewDummyModel(nil, modelModule.URLSuffix{})
+
+ _, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"), driver,
+ []*modelModule.Model{{Name: "hypothetical-model", ModelTypes: []string{"quantum"}}},
+ &modelModule.APIConfig{}, nil)
+ if err == nil {
+ t.Fatal("verifyProviderModel() error = nil, want a failure")
+ }
+ if strings.Contains(err.Error(), "%!w") {
+ t.Fatalf("verifyProviderModel() leaked a formatting error: %v", err)
+ }
+ if !strings.Contains(err.Error(), "quantum") {
+ t.Fatalf("verifyProviderModel() error = %v, want it to name the unverifiable type", err)
+ }
+}
+
+// Verification builds the request Decide sends, so it needs exercising against
+// a real driver. The stand-in used above accepts any request — including one the
+// endpoint would refuse — so it cannot catch verification asking for nothing.
+func TestVerifyProviderModelAsksSystemOneDecisionEndpoint(t *testing.T) {
+ prev := common.AllowAnyHostForTest
+ common.AllowAnyHostForTest = true
+ t.Cleanup(func() { common.AllowAnyHostForTest = prev })
+
+ asked := false
+ var request struct {
+ Model string `json:"model"`
+ State map[string]string `json:"state"`
+ Questions map[string]struct {
+ Type string `json:"type"`
+ Instructions string `json:"instructions"`
+ } `json:"questions"`
+ }
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ asked = true
+ if r.URL.Path != "/v1/systemone" {
+ t.Errorf("path=%s, want /v1/systemone", r.URL.Path)
+ }
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Errorf("undecodable request body: %v", err)
+ }
+ _, _ = w.Write([]byte(`{"answers":{"verification":{"type":"noul","noul":0.5}}}`))
+ }))
+ defer srv.Close()
+
+ apiKey := "sk-test"
+ result, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"),
+ modelModule.NewSystemOneModel(map[string]string{"default": srv.URL + "/v1"},
+ modelModule.URLSuffix{Decision: "systemone", Models: "models"}),
+ []*modelModule.Model{{Name: "decision-model-preview", ModelTypes: []string{entity.ModelTypeDecision.String()}}},
+ &modelModule.APIConfig{ApiKey: &apiKey}, nil)
+ if err != nil {
+ t.Fatalf("verifyProviderModel() error = %v", err)
+ }
+ if !asked {
+ t.Fatal("verification never reached the decision endpoint")
+ }
+ if result["decision-model-preview"] != entity.ModelVerifySuccess {
+ t.Fatalf("verification result = %#v, want success", result)
+ }
+ question, ok := request.Questions["verification"]
+ if !ok {
+ t.Fatalf("request questions = %#v, want the verification question", request.Questions)
+ }
+ if question.Type != "noul" || question.Instructions == "" {
+ t.Errorf("verification question = %+v, want a noul question with instructions", question)
+ }
+}
+
+func TestVerifyProviderModelReportsFailingDecisionModels(t *testing.T) {
+ driver := &decisionProbeDriver{
+ DummyModel: modelModule.NewDummyModel(nil, modelModule.URLSuffix{}),
+ decideErr: errors.New("endpoint answered 404"),
+ }
+
+ result, err := verifyProviderModel(serviceTestContext(t, "user-1", "tenant-1"), driver,
+ []*modelModule.Model{{Name: "decision-model-preview", ModelTypes: []string{entity.ModelTypeDecision.String()}}},
+ &modelModule.APIConfig{}, nil)
+ if err == nil {
+ t.Fatal("verifyProviderModel() error = nil, want the endpoint failure")
+ }
+ if !strings.Contains(err.Error(), "endpoint answered 404") {
+ t.Fatalf("verifyProviderModel() error = %v, want the endpoint failure", err)
+ }
+ if result["decision-model-preview"] != entity.ModelVerifyFail {
+ t.Fatalf("verification result = %#v, want fail", result)
+ }
+}
+
func TestModelInfoWithTenantExtraAppliesEmbeddingConstraints(t *testing.T) {
factoryMaxDimension := 2048
factoryBatchSize := 128