RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Honor explicit model URL overrides for NVIDIA endpoints (#18152)
mkaaad1b02abd48772
diff --git a/conf/models/nvidia.json b/conf/models/nvidia.jsonindex a0c87b9736..5b10ec9822 100644--- a/conf/models/nvidia.json+++ b/conf/models/nvidia.json@@ -72,6 +72,7 @@"name": "meta/llama-3.2-11b-vision-instruct","content_length": 131072,"max_output": 8192,+ "url":"https://integrate.api.nvidia.com/v1/meta/llama-3.2-11b-vision-instruct","model_types": ["chat","vision"@@ -97,6 +98,7 @@"name": "meta/llama-3.2-90b-vision-instruct","content_length": 131072,"max_output": 8192,+ "url":"https://integrate.api.nvidia.com/v1/meta/llama-3.2-90b-vision-instruct","model_types": ["chat","vision"diff --git a/internal/entity/models/model.go b/internal/entity/models/model.goindex 330e0aa16e..dba48e769b 100644--- a/internal/entity/models/model.go+++ b/internal/entity/models/model.go@@ -166,6 +166,7 @@ type Model struct {Thinking *ModelThinking `json:"thinking"`Tools *ModelTools `json:"tools"`Class *string `json:"class"`+ URL string `json:"url"`MaxDimension *int `json:"max_dimension"` // used by embedding modelsMaxBatchSize *int `json:"max_batch_size"` // used by embedding modelsDimensions []int `json:"dimensions"`diff --git a/internal/entity/models/nvidia.go b/internal/entity/models/nvidia.goindex c8e6a00665..02da13f467 100644--- a/internal/entity/models/nvidia.go+++ b/internal/entity/models/nvidia.go@@ -64,6 +64,30 @@ func (n *NvidiaModel) Name() string {return "nvidia"}+// resolveEndpoint returns the endpoint URL for modelName. When the+// model's preset config in conf/models/nvidia.json carries an explicit+// url, that full endpoint is used as-is because it does not follow the+// standard baseURL + suffix assembly; otherwise the usual assembly+// applies. Every endpoint (chat, embedding, rerank, models list) goes+// through this path so the override is honored uniformly.+func (n *NvidiaModel) resolveEndpoint(modelName string, apiConfig *APIConfig, suffix string) (string, error) {+ if modelName != "" {+ if pm := GetProviderManager(); pm != nil {+ if provider := pm.FindProvider("NVIDIA"); provider != nil {+ if model := pm.FindModel(provider, modelName); model != nil && model.URL != "" {+ return strings.TrimSuffix(model.URL, "/"), nil+ }+ }+ }+ }++ resolvedBaseURL, err := n.baseModel.GetBaseURL(apiConfig)+ if err != nil {+ return "", err+ }+ return fmt.Sprintf("%s/%s", strings.TrimRight(resolvedBaseURL, "/"), strings.TrimLeft(suffix, "/")), nil+}+func (n *NvidiaModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) {if err := n.baseModel.APIConfigCheck(apiConfig); err != nil {return nil, err@@ -73,11 +97,10 @@ func (n *NvidiaModel) ChatWithMessages(ctx context.Context, modelName string, mereturn nil, fmt.Errorf("messages is empty")}- resolvedBaseURL, err := n.baseModel.GetBaseURL(apiConfig)+ baseURL, err := n.resolveEndpoint(modelName, apiConfig, n.baseModel.URLSuffix.Chat)if err != nil {return nil, err}- baseURL := fmt.Sprintf("%s/%s", resolvedBaseURL, n.baseModel.URLSuffix.Chat)reqBody := buildRequestBody(chatModelConfig, modelName, messages, false)if chatModelConfig != nil {@@ -108,11 +131,10 @@ func (n *NvidiaModel) ChatStreamlyWithSender(ctx context.Context, modelName strireturn fmt.Errorf("messages is empty")}- resolvedBaseURL, err := n.baseModel.GetBaseURL(apiConfig)+ baseURL, err := n.resolveEndpoint(modelName, apiConfig, n.baseModel.URLSuffix.Chat)if err != nil {return err}- baseURL := fmt.Sprintf("%s/%s", resolvedBaseURL, n.baseModel.URLSuffix.Chat)reqBody := buildRequestBody(modelConfig, modelName, messages, true)if modelConfig != nil {@@ -152,13 +174,11 @@ func (n *NvidiaModel) Embed(ctx context.Context, modelName *string, texts []strireturn nil, fmt.Errorf("model name is required")}- resolvedBaseURL, err := n.baseModel.GetBaseURL(apiConfig)+ baseURL, err := n.resolveEndpoint(*modelName, apiConfig, n.baseModel.URLSuffix.Embedding)if err != nil {return nil, err}- baseURL := fmt.Sprintf("%s/%s", strings.TrimSuffix(resolvedBaseURL, "/"), n.baseModel.URLSuffix.Embedding)-reqBody := map[string]interface{}{"model": *modelName,"input": texts,@@ -264,13 +284,11 @@ func (n *NvidiaModel) Rerank(ctx context.Context, modelName *string, query strinreturn nil, fmt.Errorf("model name is required")}- resolvedBaseURL, err := n.baseModel.GetBaseURL(apiConfig)+ baseURL, err := n.resolveEndpoint(*modelName, apiConfig, n.baseModel.URLSuffix.Rerank)if err != nil {return nil, err}- baseURL := fmt.Sprintf("%s/%s", strings.TrimSuffix(resolvedBaseURL, "/"), n.baseModel.URLSuffix.Rerank)-topN := len(documents)if rerankConfig != nil && rerankConfig.TopN > 0 && rerankConfig.TopN < topN {topN = rerankConfig.TopN@@ -376,11 +394,10 @@ func (n *NvidiaModel) ListModels(ctx context.Context, apiConfig *APIConfig) ([]Lreturn nil, err}- resolvedBaseURL, err := n.baseModel.GetBaseURL(apiConfig)+ baseURL, err := n.resolveEndpoint("", apiConfig, n.baseModel.URLSuffix.Models)if err != nil {return nil, err}- baseURL := fmt.Sprintf("%s/%s", strings.TrimRight(resolvedBaseURL, "/"), strings.TrimLeft(n.baseModel.URLSuffix.Models, "/"))modelListCtx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)defer cancel()@@ -422,7 +439,7 @@ func (n *NvidiaModel) ListModels(ctx context.Context, apiConfig *APIConfig) ([]Lprovider = pm.FindProvider("NVIDIA")}models := parseNvidiaModelList(modelList, provider)- if n.usesHostedCatalog(resolvedBaseURL) {+ if n.usesHostedCatalog(baseURL) {catalogCtx, catalogCancel := context.WithTimeout(ctx, nonStreamCallTimeout)catalog, catalogErr := n.fetchHostedCatalog(catalogCtx)catalogCancel()diff --git a/internal/entity/models/nvidia_test.go b/internal/entity/models/nvidia_test.goindex 61659e05a1..e02e9a969d 100644--- a/internal/entity/models/nvidia_test.go+++ b/internal/entity/models/nvidia_test.go@@ -229,6 +229,162 @@ func TestNvidiaCatalogResourceRejectsMalformedDeprecation(t *testing.T) {}}+func withNvidiaProviderManager(t *testing.T, models []*Model) {+ t.Helper()+ saved := providerManager+ providerManager = &ProviderManager{+ Providers: []Provider{{+ Name: "NVIDIA",+ URL: map[string]string{"default": "https://integrate.api.nvidia.com/v1"},+ Models: models,+ }},+ }+ t.Cleanup(func() { providerManager = saved })+}++func TestNvidiaResolveEndpointUsesModelURL(t *testing.T) {+ const modelURL = "https://integrate.api.nvidia.com/v1/meta/llama-3.2-11b-vision-instruct"+ withNvidiaProviderManager(t, []*Model{{+ Name: "meta/llama-3.2-11b-vision-instruct",+ URL: modelURL,+ }})++ driver := NewNvidiaModel(+ map[string]string{"default": "https://integrate.api.nvidia.com/v1"},+ URLSuffix{Chat: "chat/completions"},+ )+ got, err := driver.resolveEndpoint("meta/llama-3.2-11b-vision-instruct", &APIConfig{}, "chat/completions")+ if err != nil {+ t.Fatalf("resolveEndpoint() error = %v", err)+ }+ if got != modelURL {+ t.Fatalf("resolveEndpoint() = %q, want %q", got, modelURL)+ }+}++func TestNvidiaResolveEndpointFallsBackToAssembly(t *testing.T) {+ withNvidiaProviderManager(t, []*Model{{Name: "meta/llama-3.1-8b-instruct"}})++ driver := NewNvidiaModel(+ map[string]string{"default": "https://integrate.api.nvidia.com/v1/"},+ URLSuffix{Chat: "chat/completions", Embedding: "embeddings"},+ )+ got, err := driver.resolveEndpoint("meta/llama-3.1-8b-instruct", &APIConfig{}, "chat/completions")+ if err != nil {+ t.Fatalf("resolveEndpoint() error = %v", err)+ }+ if want := "https://integrate.api.nvidia.com/v1/chat/completions"; got != want {+ t.Fatalf("resolveEndpoint() = %q, want %q", got, want)+ }++ got, err = driver.resolveEndpoint("meta/llama-3.1-8b-instruct", &APIConfig{}, "embeddings")+ if err != nil {+ t.Fatalf("resolveEndpoint() error = %v", err)+ }+ if want := "https://integrate.api.nvidia.com/v1/embeddings"; got != want {+ t.Fatalf("resolveEndpoint() = %q, want %q", got, want)+ }+}++func TestNvidiaResolveEndpointIgnoresManagerWhenNil(t *testing.T) {+ saved := providerManager+ providerManager = nil+ defer func() { providerManager = saved }()++ driver := NewNvidiaModel(+ map[string]string{"default": "https://integrate.api.nvidia.com/v1"},+ URLSuffix{},+ )+ got, err := driver.resolveEndpoint("meta/llama-3.2-11b-vision-instruct", &APIConfig{}, "chat/completions")+ if err != nil {+ t.Fatalf("resolveEndpoint() error = %v", err)+ }+ if want := "https://integrate.api.nvidia.com/v1/chat/completions"; got != want {+ t.Fatalf("resolveEndpoint() = %q, want %q", got, want)+ }+}++func TestNvidiaChatUsesModelSpecificURL(t *testing.T) {+ withSSRFBypass(t)+ const apiKey = "nvapi-test"+ const modelName = "meta/llama-3.2-11b-vision-instruct"+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/meta/llama-3.2-11b-vision-instruct" {+ t.Fatalf("path = %s, want model-specific chat endpoint", r.URL.Path)+ }+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "choices": []map[string]interface{}{{+ "message": map[string]interface{}{"role": "assistant", "content": "pong"},+ "finish_reason": "stop",+ }},+ "usage": map[string]interface{}{"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},+ })+ }))+ defer server.Close()++ withNvidiaProviderManager(t, []*Model{{+ Name: modelName,+ URL: server.URL + "/v1/meta/llama-3.2-11b-vision-instruct",+ }})+ driver := NewNvidiaModel(+ map[string]string{"default": server.URL + "/v1"},+ URLSuffix{Chat: "chat/completions"},+ )+ resp, err := driver.ChatWithMessages(+ context.Background(),+ modelName,+ []Message{{Role: "user", Content: "hi"}},+ &APIConfig{ApiKey: ptr(apiKey)},+ nil,+ nil,+ )+ if err != nil {+ t.Fatalf("ChatWithMessages() error = %v", err)+ }+ if resp.Answer == nil || *resp.Answer != "pong" {+ t.Fatalf("answer = %v, want pong", resp.Answer)+ }+}++func TestNvidiaEmbedUsesModelSpecificURL(t *testing.T) {+ withSSRFBypass(t)+ const apiKey = "nvapi-test"+ const modelName = "nvidia/nv-embed-v1"+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/embeddings/nvidia/nv-embed-v1" {+ t.Fatalf("path = %s, want model-specific embed endpoint", r.URL.Path)+ }+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "data": []map[string]interface{}{{"index": 0, "embedding": []float64{0.1, 0.2}}},+ })+ }))+ defer server.Close()++ withNvidiaProviderManager(t, []*Model{{+ Name: modelName,+ URL: server.URL + "/v1/embeddings/nvidia/nv-embed-v1",+ }})+ driver := NewNvidiaModel(+ map[string]string{"default": server.URL + "/v1"},+ URLSuffix{Embedding: "embeddings"},+ )+ namePtr := modelName+ got, err := driver.Embed(+ context.Background(),+ &namePtr,+ []string{"hello"},+ &APIConfig{ApiKey: ptr(apiKey)},+ nil,+ nil,+ )+ if err != nil {+ t.Fatalf("Embed() error = %v", err)+ }+ if len(got) != 1 || len(got[0].Embedding) != 2 {+ t.Fatalf("embedding = %#v, want 1 vector of length 2", got)+ }+}+func ptr[T any](value T) *T {return &value}