RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix(go): migration cleanup and add DB version tracking (#19598)
Lynne8fcc7b15390
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 1bf95c6a58..bbfa2479a3 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -63,7 +63,7 @@ import ("ragflow/internal/agent/component""ragflow/internal/common""ragflow/internal/dao"- "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"+ infnative "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer""ragflow/internal/engine""ragflow/internal/engine/redis"et "ragflow/internal/engine/types"@@ -251,6 +251,9 @@ func parseArgs() (*serverArgs, error) {return nil, fmt.Errorf("unknown parameter: %s", arg)}}+ if args.migrateDB && args.mode != nil {+ return nil, errors.New("--migrate is a standalone action and cannot be combined with --api/--admin/--ingestor/--syncer")+ }return args, nil}@@ -262,13 +265,16 @@ func parseArgs() (*serverArgs, error) {func printHelp(args *serverArgs) {switch {case args.mode == nil:- fmt.Fprintf(os.Stderr, "Usage: %s --api|--admin|--ingestor|--syncer [OPTIONS]\n\n", os.Args[0])+ fmt.Fprintf(os.Stderr, "Usage: %s --api|--admin|--ingestor|--syncer [OPTIONS]\n", os.Args[0])+ fmt.Fprintf(os.Stderr, " %s --migrate [OPTIONS]\n\n", os.Args[0])fmt.Fprintf(os.Stderr, "RAGFlow Server - Open-source RAG engine based on deep document understanding\n\n")fmt.Fprintf(os.Stderr, "Mode selection (default: --api):\n")fmt.Fprintf(os.Stderr, " --api \tRun as API server\n")fmt.Fprintf(os.Stderr, " --admin \tRun as admin server\n")fmt.Fprintf(os.Stderr, " --ingestor \tRun as ingestion worker\n")fmt.Fprintf(os.Stderr, " --syncer \tRun as file sync service\n\n")+ fmt.Fprintf(os.Stderr, "Standalone action (mutually exclusive with a mode):\n")+ fmt.Fprintf(os.Stderr, " --migrate \tRun database migrations and exit\n\n")fmt.Fprintf(os.Stderr, "Common options:\n")fmt.Fprintf(os.Stderr, " --config string\tPath to configuration file\n")fmt.Fprintf(os.Stderr, " -v, --version \tPrint version information and exit\n")@@ -330,7 +336,7 @@ func main() {os.Exit(1)}- if arguments.helpFlag || arguments.mode == nil {+ if arguments.helpFlag || (arguments.mode == nil && !arguments.migrateDB) {printHelp(arguments)os.Exit(1)}@@ -340,6 +346,17 @@ func main() {os.Exit(1)}+ // --migrate is a standalone one-shot action: run the database migrations and+ // exit without selecting a server mode. It deliberately skips the+ // mode-specific startup (native DeepDoc, doc engine, Redis, storage, message+ // queue) so it can run independently, before any server boots.+ if arguments.migrateDB {+ if err = runMigrate(ctx, arguments); err != nil {+ common.Fatal("Failed to run database migration", zap.Error(err))+ }+ return+ }+// Initialize local variables (runtime variables from Redis)err = server.InitLocalVariables()if err != nil {@@ -457,11 +474,21 @@ func main() {common.Info(fmt.Sprintf("Starting %s server: %s, mode: %s", *arguments.mode, serverName, globalConfig.GetMode()))server.PrintAll()- // Initialize database- if err = dao.InitDB(ctx, arguments.migrateDB); err != nil {+ // Initialize database. Migrations are not run here: --migrate is a+ // standalone action, so a server-mode process only ensures the runtime+ // tables it needs (see InitDB).+ if err = dao.InitDB(ctx, false); err != nil {common.Fatal("Failed to initialize database", zap.Error(err))}+ // Refuse to start a server against a database that a newer version already+ // migrated: rolling the code back cannot roll the schema back. The+ // standalone --migrate action is exempt because advancing the database is+ // its job.+ if err = checkDatabaseVersion(ctx); err != nil {+ common.Fatal("Refusing to start: database was migrated by a newer version", zap.Error(err))+ }+// Initialize doc engineif err = engine.InitDocEngine(ctx); err != nil {common.Fatal("Failed to initialize doc engine", zap.Error(err))@@ -526,6 +553,104 @@ func main() {}}+// checkDatabaseVersion refuses to run a server when the running code is older+// than the version recorded in the system_settings migration marker. Migrating+// the database forward is a one-way operation, so an older binary would read and+// write a schema it does not understand.+//+// A missing marker, or a version on either side that cannot be parsed, never+// blocks startup: without a usable comparison there is no evidence that the+// database is ahead of the code.+func checkDatabaseVersion(ctx context.Context) error {+ databaseVersion, err := dao.GetDatabaseMigrationVersion(ctx, dao.DB)+ if err != nil {+ return fmt.Errorf("read database version marker: %w", err)+ }+ if databaseVersion == "" {+ return nil+ }++ codeVersion := common.GetRAGFlowVersion()+ older, comparable := common.IsOlderReleaseThan(codeVersion, databaseVersion)+ if !comparable {+ common.Warn("Cannot compare code version with database version, skipping the downgrade check",+ zap.String("code_version", codeVersion),+ zap.String("database_version", databaseVersion))+ return nil+ }+ if older {+ return fmt.Errorf("code version %s is older than database version %s: upgrade this deployment to %s or newer before starting",+ codeVersion, databaseVersion, databaseVersion)+ }++ common.Info("Database version check passed",+ zap.String("code_version", codeVersion),+ zap.String("database_version", databaseVersion))+ return nil+}++// runMigrate runs the database schema and data migrations and returns. It is+// the whole of the standalone --migrate action: load the configuration, run+// dao.InitDB with migrations enabled, then exit. It deliberately does not call+// registerNativeDeepDoc or initialize the doc engine, Redis, storage or the+// message queue, so it can run on its own, before any server mode boots (see+// docker/entrypoint-go.sh and docker/launch_backend_service.sh).+func runMigrate(ctx context.Context, args *serverArgs) error {+ const serverName = "migrate"++ if err := server.InitLocalVariables(); err != nil {+ return fmt.Errorf("initialize local variables: %w", err)+ }++ logLevel := "info"+ if args.debugLog {+ logLevel = "debug"+ }+ if err := common.InitLogger(logLevel, common.FileOutput{Filename: serverName + ".log", Path: "logs"}, serverName); err != nil {+ return fmt.Errorf("initialize logger: %w", err)+ }++ var configPath string+ if args.configPath != nil {+ configPath = *args.configPath+ }+ if err := server.Init(configPath); err != nil {+ return fmt.Errorf("initialize configuration: %w", err)+ }++ globalConfig := server.GetConfig()+ server.SetServerName(serverName)++ logConfig := globalConfig.GetLogConfig()+ if logConfig.Level != "" {+ logLevel = logConfig.Level+ }+ if args.debugLog {+ logLevel = "debug"+ }+ globalConfig.SetLogLevel(logLevel)++ common.SyncLog()+ if err := common.InitLogger(logLevel, common.FileOutput{+ Filename: serverName + ".log",+ Path: logConfig.Path,+ MaxSize: logConfig.MaxSize,+ MaxBackups: logConfig.MaxBackups,+ MaxAge: logConfig.MaxAge,+ Compress: logConfig.Compress,+ }, serverName); err != nil {+ common.Error("Failed to reinitialize logger with configured level", err)+ }++ common.Info("Running database migrations")+ if err := dao.InitDB(ctx, true); err != nil {+ return fmt.Errorf("initialize database: %w", err)+ }+ common.Info("Database migrations completed")++ return nil+}+func runAdmin(ctx context.Context, args *serverArgs) error {globalConfig := server.GetConfig()diff --git a/cmd/ragflow_server_test.go b/cmd/ragflow_server_test.goindex e7093fd272..1ceed649be 100644--- a/cmd/ragflow_server_test.go+++ b/cmd/ragflow_server_test.go@@ -1,8 +1,25 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//+package mainimport ("context""errors"+ "os""testing""ragflow/internal/engine/types"@@ -18,6 +35,14 @@ func kb(embdID string, tenantEmbdID string) *entity.Knowledgebase {return kb}+func parseArgsForTest(t *testing.T, argv ...string) (*serverArgs, error) {+ t.Helper()+ orig := os.Args+ os.Args = append([]string{"ragflow_server"}, argv...)+ defer func() { os.Args = orig }()+ return parseArgs()+}+// TestHasEmbedderForMirrorsPython pins dialog_service.py:362 —// `embd_mdl = LLMBundle(...) if kbs and kbs[0].embd_id else None`. The gate is// the FIRST dataset's embd_id, so a dialog whose leading dataset has no model@@ -134,6 +159,43 @@ func TestHarnessWebSearcherNilAndWrapped(t *testing.T) {}}+func TestParseArgsMigrateIsStandalone(t *testing.T) {+ args, err := parseArgsForTest(t, "--migrate")+ if err != nil {+ t.Fatalf("parseArgs(--migrate) error = %v", err)+ }+ if !args.migrateDB {+ t.Fatal("migrateDB = false, want true")+ }+ if args.mode != nil {+ t.Fatalf("mode = %q, want nil: --migrate must not select a server mode", *args.mode)+ }+}++func TestParseArgsMigrateRejectsMode(t *testing.T) {+ for _, mode := range []string{"--api", "--admin", "--ingestor", "--syncer"} {+ if _, err := parseArgsForTest(t, mode, "--migrate"); err == nil {+ t.Errorf("parseArgs(%s --migrate) error = nil, want error", mode)+ }+ if _, err := parseArgsForTest(t, "--migrate", mode); err == nil {+ t.Errorf("parseArgs(--migrate %s) error = nil, want error", mode)+ }+ }+}++func TestParseArgsModeResetsMigrate(t *testing.T) {+ args, err := parseArgsForTest(t, "--api")+ if err != nil {+ t.Fatalf("parseArgs(--api) error = %v", err)+ }+ if args.mode == nil || *args.mode != "api" {+ t.Fatalf("mode = %v, want api", args.mode)+ }+ if args.migrateDB {+ t.Fatal("migrateDB = true, want false")+ }+}+// errStubEncode stands in for a provider-side embedding failure.var errStubEncode = errors.New("stub encode failure")diff --git a/docker/docker-compose-go.yml b/docker/docker-compose-go.ymlindex 4ad60768c5..2c0e7d464f 100644--- a/docker/docker-compose-go.yml+++ b/docker/docker-compose-go.yml@@ -33,21 +33,7 @@ services:nproc:soft: 65535hard: 65535- # Example configuration to set up an MCP server:- # command:- # - --enable-mcpserver- # - --mcp-host=0.0.0.0- # - --mcp-port=9382- # - --mcp-base-url=http://127.0.0.1:9380- # - --mcp-script-path=/ragflow/mcp/server/server.py- # - --mcp-mode=self-host- # - --mcp-host-api-key=ragflow-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx- # Optional transport flags for MCP (customize if needed).- # Host mode need to combined with --no-transport-streamable-http-enabled flag, namely, host+streamable-http is not supported yet.- # The following are enabled by default unless explicitly disabled with --no-<flag>.- # - --no-transport-sse-enabled # Disable legacy SSE endpoints (/sse and /messages/)- # - --no-transport-streamable-http-enabled # Disable Streamable HTTP transport (/mcp endpoint)- # - --no-json-response # Disable JSON response mode in Streamable HTTP transport (instead of SSE over HTTP)+ # MCP is served in-process by the API server (POST /mcp on the 9380 port).# Example configuration to start Admin server:command:@@ -58,7 +44,6 @@ services:- ${SVR_WEB_HTTPS_PORT}:443- ${SVR_HTTP_PORT}:9380- ${ADMIN_SVR_HTTP_PORT}:9381- - ${SVR_MCP_PORT}:9382 # entry for MCP (host_port:docker_port). The docker_port must match the value you set for `mcp-port` above.- ${GO_HTTP_PORT}:9384- ${GO_ADMIN_PORT}:9383volumes:@@ -96,21 +81,7 @@ services:nproc:soft: 65535hard: 65535- # Example configuration to set up an MCP server:- # command:- # - --enable-mcpserver- # - --mcp-host=0.0.0.0- # - --mcp-port=9382- # - --mcp-base-url=http://127.0.0.1:9380- # - --mcp-script-path=/ragflow/mcp/server/server.py- # - --mcp-mode=self-host- # - --mcp-host-api-key=ragflow-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx- # Optional transport flags for MCP (customize if needed).- # Host mode need to combined with --no-transport-streamable-http-enabled flag, namely, host+streamable-http is not supported yet.- # The following are enabled by default unless explicitly disabled with --no-<flag>.- # - --no-transport-sse-enabled # Disable legacy SSE endpoints (/sse and /messages/)- # - --no-transport-streamable-http-enabled # Disable Streamable HTTP transport (/mcp endpoint)- # - --no-json-response # Disable JSON response mode in Streamable HTTP transport (instead of SSE over HTTP)+ # MCP is served in-process by the API server (POST /mcp on the 9380 port).# Example configuration to start Admin server:command:@@ -120,14 +91,13 @@ services:- ${SVR_WEB_HTTPS_PORT}:443- ${SVR_HTTP_PORT}:9380- ${ADMIN_SVR_HTTP_PORT}:9381- - ${SVR_MCP_PORT}:9382 # entry for MCP (host_port:docker_port). The docker_port must match the value you set for `mcp-port` above.volumes:- ./ragflow-logs:/ragflow/logs# - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf# - ./nginx/proxy.conf:/etc/nginx/proxy.conf# - ./nginx/nginx.conf:/etc/nginx/nginx.conf- ./service_conf.yaml.template:/ragflow/conf/service_conf.yaml.template- - ./entrypoint.sh:/ragflow/entrypoint.sh+ - ./entrypoint-go.sh:/ragflow/entrypoint.shenv_file: .env-go# Route the Go services' cache/queue backend to the separate Kvrocks service# (overriding REDIS_HOST=redis from .env-go). The Python services keep Valkey.diff --git a/docker/entrypoint-go.sh b/docker/entrypoint-go.shindex 7cb9c02ae5..0fe41f5b69 100755--- a/docker/entrypoint-go.sh+++ b/docker/entrypoint-go.sh@@ -14,40 +14,26 @@ function usage() {echo " --disable-webserver Disables the web server (nginx + ragflow_server)."echo " --disable-taskexecutor Disables task executor workers."echo " --disable-datasync Disables synchronization of datasource workers."- echo " --enable-mcpserver Enables the MCP server."echo " --enable-adminserver Enables the Admin server."echo " --init-model-provider-tables Run model provider table migrations and exit."- echo " --init-superuser Initializes the superuser."+ echo " --init-superuser Initializes the superuser (needs --enable-adminserver)."echo " --consumer-no-beg=<num> Start range for consumers (if using range-based)."echo " --consumer-no-end=<num> End range for consumers (if using range-based)."echo " --workers=<num> Number of task executors to run (if range is not used)."echo " --host-id=<string> Unique ID for the host (defaults to \`hostname\`)."- echo " --mcp-host=<string> Address the MCP server binds to (default: 127.0.0.1)."- echo " --mcp-port=<num> Port the MCP server listens on (default: 9382)."- echo " --mcp-base-url=<string> RAGFlow base URL the MCP server calls (default: http://127.0.0.1:9380)."- echo " --mcp-script-path=<path> MCP server entry script (default: /ragflow/mcp/server/server.py)."- echo " --mcp-mode=<self-host|host> MCP server mode (default: self-host)."- echo " --mcp-host-api-key=<string> API key required when --mcp-mode=self-host."- echo " --no-transport-sse-enabled Disables the MCP SSE transport."- echo " --no-transport-streamable-http-enabled Disables the MCP streamable HTTP transport. Disabling"- echo " both transports re-enables this one, since the server"- echo " requires at least one."- echo " --no-json-response Disables JSON responses from the MCP server."echoecho "Examples:"echo " $0 --disable-taskexecutor"echo " $0 --disable-webserver --consumer-no-beg=0 --consumer-no-end=5"echo " $0 --disable-webserver --workers=2 --host-id=myhost123"- echo " $0 --enable-mcpserver"echo " $0 --enable-adminserver"- echo " $0 --init-superuser"+ echo " $0 --enable-adminserver --init-superuser"exit 1}ENABLE_WEBSERVER=1 # Default to enable web serverENABLE_TASKEXECUTOR=1 # Default to enable task executorENABLE_DATASYNC=1-ENABLE_MCP_SERVER=0ENABLE_ADMIN_SERVER=0 # Default close admin serverINIT_SUPERUSER_ARGS="" # Default to not initialize superuserINIT_MODEL_PROVIDER_TABLES=0@@ -55,16 +41,6 @@ CONSUMER_NO_BEG=0CONSUMER_NO_END=0WORKERS=1-MCP_HOST="127.0.0.1"-MCP_PORT=9382-MCP_BASE_URL="http://127.0.0.1:9380"-MCP_SCRIPT_PATH="/ragflow/mcp/server/server.py"-MCP_MODE="self-host"-MCP_HOST_API_KEY=""-MCP_TRANSPORT_SSE_FLAG="--transport-sse-enabled"-MCP_TRANSPORT_STREAMABLE_HTTP_FLAG="--transport-streamable-http-enabled"-MCP_JSON_RESPONSE_FLAG="--json-response"-# -----------------------------------------------------------------------------# Host ID logic:# 1. By default, use the system hostname if length <= 32@@ -94,10 +70,6 @@ for arg in "$@"; doENABLE_DATASYNC=0shift;;- --enable-mcpserver)- ENABLE_MCP_SERVER=1- shift- ;;--enable-adminserver)ENABLE_ADMIN_SERVER=1shift@@ -110,42 +82,6 @@ for arg in "$@"; doINIT_SUPERUSER_ARGS="--init-superuser"shift;;- --mcp-host=*)- MCP_HOST="${arg#*=}"- shift- ;;- --mcp-port=*)- MCP_PORT="${arg#*=}"- shift- ;;- --mcp-base-url=*)- MCP_BASE_URL="${arg#*=}"- shift- ;;- --mcp-mode=*)- MCP_MODE="${arg#*=}"- shift- ;;- --mcp-host-api-key=*)- MCP_HOST_API_KEY="${arg#*=}"- shift- ;;- --mcp-script-path=*)- MCP_SCRIPT_PATH="${arg#*=}"- shift- ;;- --no-transport-sse-enabled)- MCP_TRANSPORT_SSE_FLAG="--no-transport-sse-enabled"- shift- ;;- --no-transport-streamable-http-enabled)- MCP_TRANSPORT_STREAMABLE_HTTP_FLAG="--no-transport-streamable-http-enabled"- shift- ;;- --no-json-response)- MCP_JSON_RESPONSE_FLAG="--no-json-response"- shift- ;;--consumer-no-beg=*)CONSUMER_NO_BEG="${arg#*=}"shift@@ -193,78 +129,49 @@ while IFS= read -r line || [[ -n "$line" ]]; dodone < "${TEMPLATE_FILE}"export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/"-PY=python3# ------------------------------------------------------------------------------# Select Nginx Configuration based on API_PROXY_SCHEME+# Select Nginx Configuration# -----------------------------------------------------------------------------+# This image ships the Go backend only, so the golang config is the only one+# available.NGINX_CONF_DIR="/etc/nginx/conf.d"-if [ -n "$API_PROXY_SCHEME" ]; then- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]]; then- cp -f "$NGINX_CONF_DIR/ragflow.conf.hybrid" "$NGINX_CONF_DIR/ragflow.conf"- echo "Applied nginx config: ragflow.conf.hybrid"- elif [[ "${API_PROXY_SCHEME}" == "go" ]]; then- cp -f "$NGINX_CONF_DIR/ragflow.conf.golang" "$NGINX_CONF_DIR/ragflow.conf"- echo "Applied nginx config: ragflow.conf.golang (default)"- else- cp -f "$NGINX_CONF_DIR/ragflow.conf.python" "$NGINX_CONF_DIR/ragflow.conf"- echo "Applied nginx config: ragflow.conf.python"- fi-else- # Default to python backend- cp -f "$NGINX_CONF_DIR/ragflow.conf.python" "$NGINX_CONF_DIR/ragflow.conf"- echo "Default: applied nginx config: ragflow.conf.python"-fi+cp -f "$NGINX_CONF_DIR/ragflow.conf.golang" "$NGINX_CONF_DIR/ragflow.conf"# -----------------------------------------------------------------------------# Function(s)# ------------------------------------------------------------------------------function task_exe() {- local consumer_id="$1"- local host_id="$2"-- JEMALLOC_PATH="$(pkg-config --variable=libdir jemalloc)/libjemalloc.so"- while true; do- LD_PRELOAD="$JEMALLOC_PATH" \- "$PY" rag/svr/task_executor.py -i "${host_id}_${consumer_id}" -t "common" &- wait;- sleep 1;- done-}--function start_mcp_server() {- echo "Starting MCP Server on ${MCP_HOST}:${MCP_PORT} with base URL ${MCP_BASE_URL}..."- "$PY" "${MCP_SCRIPT_PATH}" \- --host="${MCP_HOST}" \- --port="${MCP_PORT}" \- --base-url="${MCP_BASE_URL}" \- --mode="${MCP_MODE}" \- --api-key="${MCP_HOST_API_KEY}" \- "${MCP_TRANSPORT_SSE_FLAG}" \- "${MCP_TRANSPORT_STREAMABLE_HTTP_FLAG}" \- "${MCP_JSON_RESPONSE_FLAG}" &-}--function ensure_docling() {- [[ "${USE_DOCLING}" == "true" ]] || { echo "[docling] disabled by USE_DOCLING"; return 0; }- DOCLING_PIN="${DOCLING_VERSION:-==2.71.0}"- "$PY" -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('docling') else 1)" \- || uv pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --extra-index-url https://pypi.org/simple --no-cache-dir "docling${DOCLING_PIN}"+# One-shot Go migration. bin/ragflow_server --migrate is a standalone action: it+# runs the migrations and exits, independent of any server mode.+function run_go_migrations() {+ local db_type="${DB_TYPE:-mysql}"+ db_type="${db_type,,}"+ if [[ "$db_type" == "gaussdb" || "$db_type" == "gauss" ]]; then+ # The Go migrations emit MySQL-only SQL and cannot run against a GaussDB+ # metadata database.+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."+ return 0+ fi+ echo "Running model provider table migrations..."+ bin/ragflow_server --migrate}-function ensure_db_init() {- echo "Initializing database tables..."- "$PY" -c "from api.db.db_models import init_database_tables as init_web_db; init_web_db()"- echo "Database tables initialized."+# Whether any Go server mode will run. These are the processes that used to+# carry --migrate, so the standalone migration must run before them.+function go_backend_enabled() {+ if [[ "${ENABLE_DATASYNC}" -eq 1 ]]; then+ return 0+ fi+ if [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]] || [[ "${ENABLE_WEBSERVER}" -eq 1 ]]; then+ return 0+ fi+ return 1}# -----------------------------------------------------------------------------# Start components based on flags# ------------------------------------------------------------------------------#ensure_docling-#ensure_db_init-run_with_restart() {local process_name="$1"shift@@ -280,90 +187,57 @@ run_with_restart() {done}+# --init-model-provider-tables keeps its documented "run migrations and exit"+# meaning: it migrates and exits without booting any server.if [[ "${INIT_MODEL_PROVIDER_TABLES}" -eq 1 ]]; then- DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"- DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"- if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then- # This migration script contains MySQL-only SQL and cannot run against- # a GaussDB metadata database.- echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."- else- #tools/scripts/run_migrations.sh- echo ""- fi+ run_go_migrations+ echo "Model provider table migrations finished. Exiting."+ exit 0+fi++# Otherwise migrate once up front, before any Go server mode boots. --migrate is+# a standalone action, so it is no longer attached to --api/--admin/--syncer.+if go_backend_enabled; then+ run_go_migrationsfiif [[ "${ENABLE_DATASYNC}" -eq 1 ]]; thenecho "Starting data sync..."- run_with_restart "RAGFlow go server" bin/ragflow_server --syncer --migrate &+ run_with_restart "RAGFlow go server" bin/ragflow_server --syncer &fisleep 5if [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]]; then-- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then- echo "Attempt to start Admin python server..."- run_with_restart "Admin python server" "$PY" admin/server/admin_server.py &- fi-- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then- echo "Starting Admin go server..."- run_with_restart "Admin go server" bin/ragflow_server --admin --migrate &- fi+ echo "Starting Admin go server..."+ run_with_restart "Admin go server" bin/ragflow_server --admin ${INIT_SUPERUSER_ARGS} &fiif [[ "${ENABLE_WEBSERVER}" -eq 1 ]]; thenecho "Starting nginx..."/usr/sbin/nginx -c /etc/nginx/nginx.conf- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then- echo "Attempt to start RAGFlow python server..."- run_with_restart "RAGFlow python server" "$PY" api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &- fi-- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then- echo "Starting RAGFlow go server..."- run_with_restart "RAGFlow go server" bin/ragflow_server --api --migrate &- fi+ echo "Starting RAGFlow go server..."+ run_with_restart "RAGFlow go server" bin/ragflow_server --api &fi+# MCP needs no separate process: --api serves it in-process at POST /mcp on the+# main API port.--#if [[ "${ENABLE_MCP_SERVER}" -eq 1 ]]; then-# start_mcp_server-#fi--+# Task execution is the Go ingestor's job. This image ships no Python task+# executor (rag/svr/task_executor.py is not copied), so --ingestor is the only+# worker that can run here.if [[ "${ENABLE_TASKEXECUTOR}" -eq 1 ]]; thenif [[ "${CONSUMER_NO_END}" -gt "${CONSUMER_NO_BEG}" ]]; then- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then- echo "Starting python task executors on host '${HOST_ID}' for IDs in [${CONSUMER_NO_BEG}, ${CONSUMER_NO_END})..."- for (( i=CONSUMER_NO_BEG; i<CONSUMER_NO_END; i++ ))- do- task_exe "${i}" "${HOST_ID}" &- done- fi-- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then- echo "Starting go ingestor..."- run_with_restart "ingestor" bin/ragflow_server --ingestor &- fi+ echo "Starting go ingestor..."+ run_with_restart "ingestor" bin/ragflow_server --ingestor &else# Otherwise, start a fixed number of workersecho "Starting ${WORKERS} task executor(s) on host '${HOST_ID}'..."for (( i=0; i<WORKERS; i++ ))do- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then- echo "Starting python task executor..."- task_exe "${i}" "${HOST_ID}" &- sleep 1;- fi-- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then- echo "Starting go ingestor..."- run_with_restart "ingestor" bin/ragflow_server --ingestor &- fi+ echo "Starting go ingestor..."+ run_with_restart "ingestor" bin/ragflow_server --ingestor &donefifidiff --git a/docker/entrypoint.sh b/docker/entrypoint.shindex 62cba47a80..569ffd965e 100755--- a/docker/entrypoint.sh+++ b/docker/entrypoint.sh@@ -259,6 +259,27 @@ function ensure_db_init() {echo "Database tables initialized."}+# Run the model provider table migrations. The Go backend owns those tables when+# it serves the API, so it runs its own --migrate action instead of the Python+# script.+function run_model_provider_migrations() {+ DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"+ DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"+ if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then+ # These migrations contain MySQL-only SQL and cannot run against a+ # GaussDB metadata database.+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."+ return 0+ fi++ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then+ echo "Running model provider table migrations (go)..."+ bin/ragflow_server --migrate+ else+ tools/scripts/run_migrations.sh+ fi+}+# -----------------------------------------------------------------------------# Start components based on flags# -----------------------------------------------------------------------------@@ -282,18 +303,16 @@ run_with_restart() {# This used to run inside the web server block only, so hosts that ran e.g.# just the admin server, data sync, MCP server, or task executors never# initialized the schema.-ensure_db_init+#+# Under the go scheme the Go backend owns the schema: its --migrate action and+# dao.InitDB create and converge the tables. Skip the Python initialization so a+# single side owns the schema.+if [[ "${API_PROXY_SCHEME}" != "go" ]]; then+ ensure_db_init+fiif [[ "${INIT_MODEL_PROVIDER_TABLES}" -eq 1 ]]; then- DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"- DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"- if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then- # This migration script contains MySQL-only SQL and cannot run against- # a GaussDB metadata database.- echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."- else- tools/scripts/run_migrations.sh- fi+ run_model_provider_migrationsfiif [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]]; then@@ -327,11 +346,20 @@ if [[ "${ENABLE_WEBSERVER}" -eq 1 ]]; thenfiif [[ "${ENABLE_DATASYNC}" -eq 1 ]]; then- echo "Starting data sync..."- run_with_restart "Data sync" "$PY" rag/svr/sync_data_source.py &+ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then+ echo "Starting data sync (go)..."+ run_with_restart "RAGFlow go server" bin/ragflow_server --syncer &+ else+ echo "Starting data sync..."+ run_with_restart "Data sync" "$PY" rag/svr/sync_data_source.py &+ fifi-if [[ "${ENABLE_MCP_SERVER}" -eq 1 ]]; then+# The Go backend serves MCP in-process from --api (POST /mcp on the main API+# port) and exposes the same ragflow_retrieval, ragflow_list_datasets and+# ragflow_list_chats tools. The standalone Python MCP server would only+# duplicate it, so the go scheme relies on the built-in endpoint instead.+if [[ "${ENABLE_MCP_SERVER}" -eq 1 ]] && [[ "${API_PROXY_SCHEME}" != "go" ]]; thenstart_mcp_serverfidiff --git a/docker/launch_backend_service.sh b/docker/launch_backend_service.shindex 8bd643155c..5275a10aae 100755--- a/docker/launch_backend_service.sh+++ b/docker/launch_backend_service.sh@@ -15,7 +15,7 @@ usage() {echo " ragflow Start RAGFlow server based on API_PROXY_SCHEME"echo " task_executor Start task workers based on API_PROXY_SCHEME"echo " admin Start Admin server based on API_PROXY_SCHEME"- echo " data_sync Start rag/svr/sync_data_source.py"+ echo " data_sync Start the data source syncer based on API_PROXY_SCHEME"echoecho "Examples:"echo " $0"@@ -199,23 +199,33 @@ run_admin_server(){# Function to execute sync_data_source with retry logicrun_data_sync(){+ local server_name="sync_data_source.py"+ local -a sync_cmd=("$PY" "rag/svr/sync_data_source.py")+ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then+ prepare_for_go+ server_name="ragflow_server --syncer"+ sync_cmd=("bin/ragflow_server" "--syncer")+ if [[ "$DEBUG_MODE" -eq 1 ]]; then+ sync_cmd+=("--debug")+ fi+ filocal retry_count=0while ! $STOP && [ $retry_count -lt $MAX_RETRIES ]; do- echo "Starting sync_data_source.py (Attempt $((retry_count+1)))"+ echo "Starting ${sync_cmd[*]} (Attempt $((retry_count+1)))"EXIT_CODE=0- $PY rag/svr/sync_data_source.py || EXIT_CODE=$?+ "${sync_cmd[@]}" || EXIT_CODE=$?if [ $EXIT_CODE -eq 0 ]; then- echo "sync_data_source.py exited successfully."+ echo "$server_name exited successfully."breakelse- echo "sync_data_source.py failed with exit code $EXIT_CODE. Retrying..." >&2+ echo "$server_name failed with exit code $EXIT_CODE. Retrying..." >&2retry_count=$((retry_count + 1))sleep 2fidoneif [ $retry_count -ge $MAX_RETRIES ]; then- echo "sync_data_source.py failed after $MAX_RETRIES attempts. Exiting..." >&2+ echo "$server_name failed after $MAX_RETRIES attempts. Exiting..." >&2cleanupfi}@@ -226,17 +236,25 @@ ensure_db_init() {echo "Database tables initialized."}-run_mysql_migrations() {+run_migrations() {local db_type="${DB_TYPE:-mysql}"db_type="${db_type,,}"if [ "$db_type" = "gaussdb" ] || [ "$db_type" = "gauss" ]; then- # This migration script contains MySQL-only SQL and cannot run against- # a GaussDB metadata database.+ # The migrations contain MySQL-only SQL and cannot run against a GaussDB+ # metadata database.echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."return 0fi- tools/scripts/run_migrations.sh+ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then+ # The Go backend owns the model provider tables. --migrate is a+ # standalone action: it runs the migrations and exits, so it is+ # independent of any server actually starting.+ echo "Running model provider table migrations (go)..."+ bin/ragflow_server --migrate+ else+ tools/scripts/run_migrations.sh+ fi}prepare_for_go() {@@ -309,8 +327,14 @@ if [[ "$SERVICE_SELECTED" -eq 0 ]]; thenfiif [[ "$START_RAGFLOW" -eq 1 ]]; then- ensure_db_init- run_mysql_migrations+ # The Go backend owns the schema when it serves the API, so skip the+ # Python-side table creation then. A Go-only deployment may not ship the api+ # package at all, and letting both sides create tables makes them fight over+ # the same schema.+ if [[ "${API_PROXY_SCHEME}" != "go" ]]; then+ ensure_db_init+ fi+ run_migrationsfi# Start task executorsdiff --git a/internal/common/version.go b/internal/common/version.goindex 12965db071..2f9d212a89 100644--- a/internal/common/version.go+++ b/internal/common/version.go@@ -21,6 +21,8 @@ import ("path/filepath""strings""sync"++ "golang.org/x/mod/semver")var (@@ -75,3 +77,42 @@ func getClosestTagAndCount() string {}return strings.TrimSpace(string(output))}++// IsOlderReleaseThan reports whether codeVersion is an earlier release than+// targetVersion, comparing only MAJOR.MINOR.PATCH.+//+// git describe reports a build made from a commit after a tag as+// "v0.27.1-14-g<sha>". semver ranks that pre-release string below "v0.27.1"+// even though the code is newer than the tag, so pre-release and build suffixes+// are stripped before comparing.+//+// The second result is false when either version cannot be parsed, in which+// case there is no evidence of a downgrade and the caller must not block.+func IsOlderReleaseThan(codeVersion, targetVersion string) (older, comparable bool) {+ code := releaseVersion(codeVersion)+ target := releaseVersion(targetVersion)+ if code == "" || target == "" {+ return false, false+ }+ return semver.Compare(code, target) < 0, true+}++// releaseVersion normalizes a version string to the "vMAJOR.MINOR.PATCH" form+// semver accepts, dropping any pre-release or build suffix. It returns an empty+// string when the result is not a valid semver tag.+func releaseVersion(version string) string {+ normalized := strings.TrimSpace(version)+ normalized = strings.TrimPrefix(normalized, "v")+ normalized = strings.TrimPrefix(normalized, "V")+ if normalized == "" {+ return ""+ }+ if i := strings.IndexAny(normalized, "-+"); i >= 0 {+ normalized = normalized[:i]+ }+ normalized = "v" + normalized+ if !semver.IsValid(normalized) {+ return ""+ }+ return normalized+}diff --git a/internal/common/version_test.go b/internal/common/version_test.goindex 4eddc74b69..f3627d7201 100644--- a/internal/common/version_test.go+++ b/internal/common/version_test.go@@ -37,3 +37,38 @@ func TestGetClosestTagAndCount(t *testing.T) {fmt.Printf("Git Version: %s\n", version)// This test just prints the version, no strict assertion}++func TestIsOlderReleaseThan(t *testing.T) {+ cases := []struct {+ name string+ code string+ target string+ wantOlder bool+ wantCmp bool+ }{+ {name: "older patch", code: "v0.26.0", target: "v0.27.1", wantOlder: true, wantCmp: true},+ {name: "older minor", code: "v0.26.9", target: "v0.27.1", wantOlder: true, wantCmp: true},+ {name: "older major", code: "v0.27.1", target: "v1.0.0", wantOlder: true, wantCmp: true},+ {name: "equal", code: "v0.27.1", target: "v0.27.1", wantOlder: false, wantCmp: true},+ {name: "newer", code: "v0.28.0", target: "v0.27.1", wantOlder: false, wantCmp: true},+ // git describe reports commits after a tag as v0.27.1-14-g<sha>; that is+ // newer code, not an older release.+ {name: "commits after tag", code: "v0.27.1-14-g1a2b3c4", target: "v0.27.1", wantOlder: false, wantCmp: true},+ {name: "commits after older tag", code: "v0.27.0-3-g1a2b3c4", target: "v0.27.1", wantOlder: true, wantCmp: true},+ {name: "missing v prefix", code: "0.26.0", target: "v0.27.1", wantOlder: true, wantCmp: true},+ {name: "unknown code", code: "unknown", target: "v0.27.1", wantOlder: false, wantCmp: false},+ {name: "empty code", code: "", target: "v0.27.1", wantOlder: false, wantCmp: false},+ {name: "empty target", code: "v0.27.1", target: "", wantOlder: false, wantCmp: false},+ {name: "invalid code", code: "not-a-version", target: "v0.27.1", wantOlder: false, wantCmp: false},+ {name: "invalid target", code: "v0.27.1", target: "not-a-version", wantOlder: false, wantCmp: false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ older, comparable := IsOlderReleaseThan(tc.code, tc.target)+ if older != tc.wantOlder || comparable != tc.wantCmp {+ t.Fatalf("IsOlderReleaseThan(%q, %q) = (%v, %v), want (%v, %v)",+ tc.code, tc.target, older, comparable, tc.wantOlder, tc.wantCmp)+ }+ })+ }+}diff --git a/internal/dao/database.go b/internal/dao/database.goindex d187a7ca2a..a5c97f7b38 100644--- a/internal/dao/database.go+++ b/internal/dao/database.go@@ -173,20 +173,26 @@ func InitDB(ctx context.Context, migrateDB bool) error {}if migrateDB {+ // Mirror the Python flow, where tools/scripts/run_migrations.sh runs before+ // the ORM creates and converges the schema: the manual migrations have to see+ // the legacy tables as they are. Running them after AutoMigrate would let+ // AutoMigrate rewrite tenant_model.model_type from text to int before the+ // model_type_merge step can read what the Python migration wrote.+ if err = RunMigrations(ctx, DB); err != nil {+ return fmt.Errorf("failed to run manual migrations: %w", err)+ }+common.Info("Migrating database schema...")for _, m := range dataModels {if err = autoMigrateSafely(ctx, DB, m); err != nil {return fmt.Errorf("failed to migrate model %T: %w", m, err)}}-- // Run manual migrations for complex schema changes- if err = RunMigrations(ctx, DB); err != nil {- return fmt.Errorf("failed to run manual migrations: %w", err)- }common.Info("Database schema migrated successfully")} else {- // Ensure Go-exclusive runtime tables exist even if the server starts without --migrate+ // Ensure the Go-exclusive runtime tables exist. The manual migrations are+ // performed by the standalone --migrate action, so a server-mode process+ // only converges the tables it needs itself.if err = autoMigrateRuntimeModels(ctx, DB); err != nil {return fmt.Errorf("failed to auto-migrate runtime models: %w", err)}@@ -316,8 +322,9 @@ func autoMigrateSafely(ctx context.Context, db *gorm.DB, model interface{}) erroreturn err}-// autoMigrateRuntimeModels ensures Go-exclusive runtime tables exist even if-// the server starts without --migrate.+// autoMigrateRuntimeModels ensures the Go-exclusive runtime tables exist. The+// manual migrations run as the standalone --migrate action, so a server-mode+// process never runs them itself.func autoMigrateRuntimeModels(ctx context.Context, db *gorm.DB) error {goRuntimeModels := []interface{}{&entity.IngestionTask{},diff --git a/internal/dao/migration.go b/internal/dao/migration.goindex 559d4d2128..6e006c3efd 100644--- a/internal/dao/migration.go+++ b/internal/dao/migration.go@@ -21,6 +21,7 @@ import ("fmt""ragflow/internal/common""ragflow/internal/entity"+ "regexp""strings""go.uber.org/zap"@@ -72,89 +73,104 @@ func RunMigrations(ctx context.Context, db *gorm.DB) error {return fmt.Errorf("failed to migrate general chunker parser configs: %w", err)}+ // Backfill the tenant model tables from the legacy tenant_llm table.+ if err := migrateModelData(ctx, db); err != nil {+ return fmt.Errorf("failed to migrate tenant model data: %w", err)+ }+common.Info("All manual migrations completed successfully")return nil}-// migrateTenantLLMPrimaryKey migrates tenant_llm from composite primary key to ID primary key-// This corresponds to Python's update_tenant_llm_to_id_primary_key function+// migrateTenantLLMPrimaryKey migrates tenant_llm from its legacy composite+// primary key (tenant_id, llm_factory, llm_name) to a surrogate auto-increment+// "id" column.+//+// MySQL allows a single PRIMARY KEY per table, so the composite key has to be+// dropped before the new key column can be promoted. The legacy uniqueness+// guarantee is preserved by a unique index on the old key columns.func migrateTenantLLMPrimaryKey(ctx context.Context, db *gorm.DB) error {- // Check if tenant_llm table existsif !db.WithContext(ctx).Migrator().HasTable("tenant_llm") {return nil}- // Check if 'id' column already exists using raw SQL- var idColumnExists int64- err := db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS- WHERE TABLE_NAME = 'tenant_llm' AND COLUMN_NAME = 'id'- `).Scan(&idColumnExists).Error+ // Idempotency: an auto_increment "id" means the table is already migrated.+ idIsAutoIncrement, err := isAutoIncrementColumn(ctx, db, "tenant_llm", "id")if err != nil {return err}-- if idColumnExists > 0 {- // Check if id is already a primary key with auto_increment- var count int64- err = db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS- WHERE TABLE_NAME = 'tenant_llm'- AND COLUMN_NAME = 'id'- AND EXTRA LIKE '%auto_increment%'- `).Scan(&count).Error- if err != nil {- return err- }- if count > 0 {- // Already migrated- return nil- }+ if idIsAutoIncrement {+ return nil}common.Info("Migrating tenant_llm to use ID primary key...")- // Start transaction+ // MySQL commits DDL implicitly, so this transaction groups the statements+ // rather than making them atomic. Every step is therefore written to be+ // re-runnable after an interrupted attempt.return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- // Check for temp_id column and drop it if exists- var tempIdExists int64- tx.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS- WHERE TABLE_NAME = 'tenant_llm' AND COLUMN_NAME = 'temp_id'`).Scan(&tempIdExists)- if tempIdExists > 0 {- if err = tx.Exec("ALTER TABLE tenant_llm DROP COLUMN temp_id").Error; err != nil {- common.Warn("Failed to drop temp_id column", zap.Error(err))- }+ // A leftover temp_id can only come from an interrupted run.+ if err := dropColumnIfExists(ctx, tx, "tenant_llm", "temp_id"); err != nil {+ return err}- // Check if there's already an 'id' column- if idColumnExists > 0 {- // Modify existing id column to be auto_increment primary key- if err = tx.Exec(`- ALTER TABLE tenant_llm- MODIFY COLUMN id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY- `).Error; err != nil {- return fmt.Errorf("failed to modify id column: %w", err)+ idExists, err := columnExists(ctx, tx, "tenant_llm", "id")+ if err != nil {+ return err+ }++ if idExists {+ alreadyKey, err := dropLegacyPrimaryKey(ctx, tx, "tenant_llm", "id")+ if err != nil {+ return err+ }+ // Re-declaring the primary key on a column that already holds it is+ // rejected as a second primary key definition.+ definition := "BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY"+ if alreadyKey {+ definition = "BIGINT NOT NULL AUTO_INCREMENT"+ }+ if err = tx.Exec("ALTER TABLE tenant_llm MODIFY COLUMN id " + definition).Error; err != nil {+ return fmt.Errorf("failed to make tenant_llm.id auto_increment: %w", err)}} else {- // Add id column as auto_increment primary key- if err = tx.Exec(`- ALTER TABLE tenant_llm- ADD COLUMN id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST- `).Error; err != nil {- return fmt.Errorf("failed to add id column: %w", err)+ if err = tx.Exec(`ALTER TABLE tenant_llm ADD COLUMN temp_id BIGINT NULL`).Error; err != nil {+ return fmt.Errorf("failed to add temp_id column: %w", err)+ }+ // Number the rows explicitly instead of relying on assignment order,+ // so the resulting primary key is stable across runs.+ if err = tx.Exec(`SET @ragflow_tenant_llm_row = 0`).Error; err != nil {+ return fmt.Errorf("failed to initialize the row counter: %w", err)+ }+ if err = tx.Exec(`UPDATE tenant_llm+ SET temp_id = (@ragflow_tenant_llm_row := @ragflow_tenant_llm_row + 1)+ ORDER BY tenant_id, llm_factory, llm_name`).Error; err != nil {+ return fmt.Errorf("failed to number tenant_llm rows: %w", err)+ }+ if _, err = dropLegacyPrimaryKey(ctx, tx, "tenant_llm", "temp_id"); err != nil {+ return err+ }+ if err = tx.Exec(`ALTER TABLE tenant_llm+ MODIFY COLUMN temp_id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY`).Error; err != nil {+ return fmt.Errorf("failed to make temp_id the primary key: %w", err)}}- // Add unique index on (tenant_id, llm_factory, llm_name)- var idxExists int64- tx.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'tenant_llm' AND INDEX_NAME = 'idx_tenant_llm_unique'`).Scan(&idxExists)- if idxExists == 0 {- if err = tx.Exec(`- ALTER TABLE tenant_llm- ADD UNIQUE INDEX idx_tenant_llm_unique (tenant_id, llm_factory, llm_name)- `).Error; err != nil {- common.Warn("Failed to add unique index idx_tenant_llm_unique", zap.Error(err))+ // Preserve the uniqueness contract of the legacy composite primary key.+ legacyKey := []string{"tenant_id", "llm_factory", "llm_name"}+ hasUnique, err := hasUniqueIndex(ctx, tx, "tenant_llm", legacyKey)+ if err != nil {+ return err+ }+ if !hasUnique {+ if err = addUniqueIndex(ctx, tx, "tenant_llm", "uk_tenant_llm", legacyKey); err != nil {+ return err+ }+ }++ if !idExists {+ if err = tx.Exec(`ALTER TABLE tenant_llm RENAME COLUMN temp_id TO id`).Error; err != nil {+ return fmt.Errorf("failed to rename temp_id to id: %w", err)}}@@ -163,49 +179,38 @@ func migrateTenantLLMPrimaryKey(ctx context.Context, db *gorm.DB) error {})}-// migrateAddUniqueEmail adds unique index on user.email+// migrateAddUniqueEmail enforces uniqueness on user.email.+//+// Rows that already share an address are resolved before the index is created:+// the superuser - or the oldest account when the address has none - keeps it,+// and the remaining rows are renamed to "<email>_DUPLICATE_<id prefix>".+// Renaming is what makes the index creatable on installations that accumulated+// duplicates before the constraint existed.func migrateAddUniqueEmail(ctx context.Context, db *gorm.DB) error {if !db.WithContext(ctx).Migrator().HasTable("user") {return nil}- // Check if unique index already exists using raw SQL- var count int64- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'user' AND INDEX_NAME = 'idx_user_email_unique'`).Scan(&count)- if count > 0 {- return nil- }-- // Check if there's a duplicate email issue first- var duplicateCount int64- err := db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM (- SELECT email FROM user GROUP BY email HAVING COUNT(*) > 1- ) AS duplicates- `).Scan(&duplicateCount).Error+ hasUnique, err := hasUniqueIndex(ctx, db, "user", []string{"email"})if err != nil {return err}-- if duplicateCount > 0 {- common.Warn("Found duplicate emails in user table, cannot add unique index", zap.Int64("count", duplicateCount))+ if hasUnique {return nil}- common.Info("Adding unique index on user.email...")- if err = db.WithContext(ctx).Exec(`ALTER TABLE user ADD UNIQUE INDEX idx_user_email_unique (email)`).Error; err != nil {+ if err = renameDuplicateEmails(ctx, db); err != nil {+ return err+ }- // Check if error is MySQL duplicate index error (Error 1061)- errStr := err.Error()- if strings.Contains(errStr, "Error 1061") && strings.Contains(errStr, "Duplicate key name") {- common.Info("Index already exists, skipping", zap.String("error", errStr))- return nil- }- return fmt.Errorf("failed to add unique index on email: %w", err)+ // Reaching this point means any existing index on the address is not unique,+ // and a same-named non-unique index would block the unique one.+ if err = dropIndexIfExists(ctx, db, "user", "idx_user_email"); err != nil {+ return err}- return nil+ common.Info("Adding unique index on user.email...")+ return addUniqueIndex(ctx, db, "user", "idx_user_email", []string{"email"})}func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) error {@@ -214,23 +219,18 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erro}const indexName = "idx_ingestion_task_document_id"+ columns := []string{"document_id"}- var uniqueCount int64- if err := db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'ingestion_task'- AND INDEX_NAME = ?- AND COLUMN_NAME = 'document_id'- AND NON_UNIQUE = 0- `, indexName).Scan(&uniqueCount).Error; err != nil {+ hasUnique, err := hasUniqueIndex(ctx, db, "ingestion_task", columns)+ if err != nil {return err}- if uniqueCount > 0 {+ if hasUnique {return nil}var duplicateCount int64- if err := db.WithContext(ctx).Raw(`+ if err = db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM (SELECT document_id FROM ingestion_task GROUP BY document_id HAVING COUNT(*) > 1) AS duplicates@@ -242,30 +242,12 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erroreturn nil}- var existingIndexCount int64- if err := db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'ingestion_task'- AND INDEX_NAME = ?- `, indexName).Scan(&existingIndexCount).Error; err != nil {+ // A non-unique index under the same name would block the unique one.+ if err = dropIndexIfExists(ctx, db, "ingestion_task", indexName); err != nil {return err}- if existingIndexCount > 0 {- if err := db.WithContext(ctx).Exec(`ALTER TABLE ingestion_task DROP INDEX ` + indexName).Error; err != nil {- return fmt.Errorf("failed to drop existing index %s: %w", indexName, err)- }- }- if err := db.WithContext(ctx).Exec(`ALTER TABLE ingestion_task ADD UNIQUE INDEX ` + indexName + ` (document_id)`).Error; err != nil {- errStr := err.Error()- if strings.Contains(errStr, "Error 1061") && strings.Contains(errStr, "Duplicate key name") {- common.Info("Index already exists, skipping", zap.String("error", errStr))- return nil- }- return fmt.Errorf("failed to add unique index on ingestion_task.document_id: %w", err)- }-- return nil+ return addUniqueIndex(ctx, db, "ingestion_task", indexName, columns)}// migrateIngestionTaskPipelineLogID adds ingestion_task.pipeline_log_id when@@ -422,69 +404,64 @@ func migrateUserCanvasTitleUnique(ctx context.Context, db *gorm.DB) error {return nil}-// modifyColumnTypes modifies column types that need explicit ALTER statements+// modifyColumnTypes aligns columns whose stored type must match the Python+// models in api/db/db_models.py. The target types are also declared on the Go+// entities, so AutoMigrate converges on the same definition and the two paths+// cannot fight over the column.func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {- columnExists := func(table, column string) bool {- var count int64- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS- WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, table, column).Scan(&count)- return count > 0- }-- // dialog.top_k: ensure its INTEGER with default 1024- if db.WithContext(ctx).Migrator().HasTable("dialog") && columnExists("dialog", "top_k") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE dialog MODIFY COLUMN top_k BIGINT NOT NULL DEFAULT 1024`).Error; err != nil {- common.Warn("Failed to modify dialog.top_k", zap.Error(err))- }- }-- // tenant_llm.api_key: ensure it's TEXT type- if db.WithContext(ctx).Migrator().HasTable("tenant_llm") && columnExists("tenant_llm", "api_key") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE tenant_llm MODIFY COLUMN api_key VARCHAR(8192)`).Error; err != nil {- common.Warn("Failed to modify tenant_llm.api_key", zap.Error(err))- }- }-- // api_token.dialog_id: ensure it's varchar(32)- if db.WithContext(ctx).Migrator().HasTable("api_token") && columnExists("api_token", "dialog_id") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE api_token MODIFY COLUMN dialog_id VARCHAR(32)`).Error; err != nil {- common.Warn("Failed to modify api_token.dialog_id", zap.Error(err))- }- }-- // canvas_template.title and description: ensure they're LONGTEXT type (same as Python JSONField)- // Note: Python's JSONField uses null=True with application-level default, not database DEFAULT- if db.WithContext(ctx).Migrator().HasTable("canvas_template") {- if columnExists("canvas_template", "title") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE canvas_template MODIFY COLUMN title LONGTEXT NULL`).Error; err != nil {- common.Warn("Failed to modify canvas_template.title", zap.Error(err))- }- }- if columnExists("canvas_template", "description") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE canvas_template MODIFY COLUMN description LONGTEXT NULL`).Error; err != nil {- common.Warn("Failed to modify canvas_template.description", zap.Error(err))- }- }- }-- // system_settings.value: ensure it's LONGTEXT- if db.WithContext(ctx).Migrator().HasTable("system_settings") && columnExists("system_settings", "value") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE system_settings MODIFY COLUMN value LONGTEXT NOT NULL`).Error; err != nil {- common.Warn("Failed to modify system_settings.value", zap.Error(err))- }- }-- // knowledgebase.raptor_task_finish_at: ensure it's DateTime- if db.WithContext(ctx).Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "raptor_task_finish_at") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE knowledgebase MODIFY COLUMN raptor_task_finish_at DATETIME`).Error; err != nil {- common.Warn("Failed to modify knowledgebase.raptor_task_finish_at", zap.Error(err))- }- }-- // knowledgebase.mindmap_task_finish_at: ensure it's DateTime- if db.WithContext(ctx).Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "mindmap_task_finish_at") {- if err := db.WithContext(ctx).Exec(`ALTER TABLE knowledgebase MODIFY COLUMN mindmap_task_finish_at DATETIME`).Error; err != nil {- common.Warn("Failed to modify knowledgebase.mindmap_task_finish_at", zap.Error(err))+ specs := []columnSpec{+ // dialog.top_k mirrors Python's IntegerField(default=1024).+ {+ table: "dialog", column: "top_k",+ columnType: "int", nullable: false,+ definition: "int NOT NULL DEFAULT 1024",+ },+ // tenant_llm.api_key mirrors Python's TextField(null=True).+ {+ table: "tenant_llm", column: "api_key",+ columnType: "text", nullable: true,+ definition: "text",+ },+ // api_token.dialog_id mirrors Python's CharField(max_length=32, null=True).+ {+ table: "api_token", column: "dialog_id",+ columnType: "varchar(32)", nullable: true,+ definition: "varchar(32)",+ },+ // canvas_template.title/description mirror Python's JSONField, which is a+ // nullable LONGTEXT.+ {+ table: "canvas_template", column: "title",+ columnType: "longtext", nullable: true,+ definition: "longtext NULL",+ },+ {+ table: "canvas_template", column: "description",+ columnType: "longtext", nullable: true,+ definition: "longtext NULL",+ },+ // system_settings.value mirrors Python's EmptyStringTextField(null=False).+ {+ table: "system_settings", column: "value",+ columnType: "text", nullable: false,+ definition: "text NOT NULL",+ },+ // knowledgebase task timestamps mirror Python's DateTimeField.+ {+ table: "knowledgebase", column: "raptor_task_finish_at",+ columnType: "datetime", nullable: true,+ definition: "datetime",+ },+ {+ table: "knowledgebase", column: "mindmap_task_finish_at",+ columnType: "datetime", nullable: true,+ definition: "datetime",+ },+ }++ for _, spec := range specs {+ if err := applyColumnSpec(ctx, db, spec); err != nil {+ return err}}@@ -497,21 +474,19 @@ func renameColumnIfExists(ctx context.Context, db *gorm.DB, tableName, oldName,return nil}- // Helper to check if column exists- columnExists := func(column string) bool {- var count int64- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS- WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, tableName, column).Scan(&count)- return count > 0+ oldExists, err := columnExists(ctx, db, tableName, oldName)+ if err != nil {+ return err}-- // Check if old column exists- if !columnExists(oldName) {+ if !oldExists {return nil}- // Check if new column already exists- if columnExists(newName) {+ newExists, err := columnExists(ctx, db, tableName, newName)+ if err != nil {+ return err+ }+ if newExists {// Both exist, drop the old onecommon.Warn("Both old and new columns exist, dropping old one",zap.String("table", tableName),@@ -533,11 +508,11 @@ func addColumnIfNotExists(ctx context.Context, db *gorm.DB, tableName, columnNamreturn nil}- // Check if column exists using raw SQL- var count int64- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS- WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, tableName, columnName).Scan(&count)- if count > 0 {+ exists, err := columnExists(ctx, db, tableName, columnName)+ if err != nil {+ return err+ }+ if exists {return nil}@@ -548,174 +523,281 @@ func addColumnIfNotExists(ctx context.Context, db *gorm.DB, tableName, columnNamreturn db.WithContext(ctx).Exec(sql).Error}-// migrateSkillSearchTables creates skill search related tables-func migrateSkillSearchTables(ctx context.Context, db *gorm.DB) error {- // Create skill_search_configs table only- if !db.WithContext(ctx).Migrator().HasTable("skill_search_configs") {- common.Info("Creating skill_search_configs table...")- sql := `- CREATE TABLE IF NOT EXISTS skill_search_configs (- id VARCHAR(32) PRIMARY KEY,- tenant_id VARCHAR(32) NOT NULL,- space_id VARCHAR(128) NOT NULL DEFAULT 'default',- embd_id VARCHAR(128) NOT NULL,- vector_similarity_weight FLOAT DEFAULT 0.3,- similarity_threshold FLOAT DEFAULT 0.2,- field_config JSON,- rerank_id VARCHAR(128),- tenant_rerank_id BIGINT,- top_k BIGINT DEFAULT 10,- index_version VARCHAR(32) DEFAULT '1.0.0',- status VARCHAR(1) DEFAULT '1',- create_time BIGINT,- create_date DATETIME,- update_time BIGINT,- update_date DATETIME,- INDEX idx_tenant_id (tenant_id),- INDEX idx_space_id (space_id),- UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)- )- `- if err := db.WithContext(ctx).Exec(sql).Error; err != nil {- common.Warn("Failed to create skill_search_configs table with MySQL dialect, trying generic", zap.Error(err))- if err = db.WithContext(ctx).AutoMigrate(&entity.SkillSearchConfig{}); err != nil {- return err- }- // AutoMigrate doesn't create unique indexes, so create them explicitly- common.Info("Creating unique indexes for skill_search_configs...")- if err = db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs ADD UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)`).Error; err != nil {- return fmt.Errorf("failed to create unique index idx_tenant_space_embd: %w", err)- }- }- } else {- // Add space_id for existing installations.- if err := addColumnIfNotExists(ctx, db, "skill_search_configs", "space_id", "VARCHAR(128) NOT NULL DEFAULT 'default'"); err != nil {- return fmt.Errorf("failed to add space_id column to skill_search_configs: %w", err)- }- if err := addColumnIfNotExists(ctx, db, "skill_search_configs", "create_date", "DATETIME"); err != nil {- return fmt.Errorf("failed to add create_date column to skill_search_configs: %w", err)- }- if err := addColumnIfNotExists(ctx, db, "skill_search_configs", "update_date", "DATETIME"); err != nil {- return fmt.Errorf("failed to add update_date column to skill_search_configs: %w", err)- }- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs MODIFY COLUMN update_time BIGINT`).Error; err != nil {- common.Warn("Failed to modify skill_search_configs.update_time", zap.Error(err))- }+// columnExists reports whether the current database already has the column.+// TABLE_SCHEMA is pinned to DATABASE() so that a MySQL server hosting more than+// one RAGFlow database never matches another schema's tables.+//+// It replaces gorm's Migrator.HasColumn, which panics when the table is passed+// as a string: HasColumn only fills Statement.Table for strings and then+// dereferences the unparsed Statement.Schema.+func columnExists(ctx context.Context, db *gorm.DB, table, column string) (bool, error) {+ var count int64+ if err := db.WithContext(ctx).Raw(`+ SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?+ `, table, column).Scan(&count).Error; err != nil {+ return false, err+ }+ return count > 0, nil+}- // Drop legacy unique index (tenant_id, embd_id) to allow per-space configs.- var legacyIndexExists int64- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'skill_search_configs' AND INDEX_NAME = 'idx_tenant_embd'`).Scan(&legacyIndexExists)- if legacyIndexExists > 0 {- common.Info("Dropping legacy unique index idx_tenant_embd from skill_search_configs...")- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs DROP INDEX idx_tenant_embd`).Error; err != nil {- return fmt.Errorf("failed to drop legacy unique index idx_tenant_embd: %w", err)- }- }+// columnDefinition returns the stored COLUMN_TYPE and whether the column accepts+// NULL. ok is false when the column does not exist.+func columnDefinition(ctx context.Context, db *gorm.DB, table, column string) (columnType string, nullable, ok bool, err error) {+ var row struct {+ ColumnType string+ IsNullable string+ }+ if err = db.WithContext(ctx).Raw(`+ SELECT COLUMN_TYPE AS column_type, IS_NULLABLE AS is_nullable+ FROM INFORMATION_SCHEMA.COLUMNS+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?+ `, table, column).Scan(&row).Error; err != nil {+ return "", false, false, err+ }+ return row.ColumnType, strings.EqualFold(row.IsNullable, "YES"), row.ColumnType != "", nil+}- // Table exists, check if unique index exists- var indexExists int64- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'skill_search_configs' AND INDEX_NAME = 'idx_tenant_space_embd'`).Scan(&indexExists)- if indexExists == 0 {- common.Info("Adding unique index idx_tenant_space_embd to skill_search_configs...")- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs- ADD UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)`).Error; err != nil {- return fmt.Errorf("failed to add unique index idx_tenant_space_embd: %w", err)- }+// isAutoIncrementColumn reports whether the column is declared AUTO_INCREMENT.+func isAutoIncrementColumn(ctx context.Context, db *gorm.DB, table, column string) (bool, error) {+ var count int64+ if err := db.WithContext(ctx).Raw(`+ SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?+ AND EXTRA LIKE '%auto_increment%'+ `, table, column).Scan(&count).Error; err != nil {+ return false, err+ }+ return count > 0, nil+}++// primaryKeyColumns lists the current PRIMARY KEY columns in key order.+func primaryKeyColumns(ctx context.Context, db *gorm.DB, table string) ([]string, error) {+ var columns []string+ if err := db.WithContext(ctx).Raw(`+ SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'+ ORDER BY ORDINAL_POSITION+ `, table).Scan(&columns).Error; err != nil {+ return nil, err+ }+ return columns, nil+}++// dropLegacyPrimaryKey removes the table's PRIMARY KEY unless it is already the+// single-column key being migrated to. MySQL rejects a second PRIMARY KEY+// definition, so the legacy key must be dropped first. It reports whether+// keepColumn is the primary key once it returns.+func dropLegacyPrimaryKey(ctx context.Context, db *gorm.DB, table, keepColumn string) (bool, error) {+ columns, err := primaryKeyColumns(ctx, db, table)+ if err != nil {+ return false, err+ }+ if len(columns) == 1 && columns[0] == keepColumn {+ return true, nil+ }+ if len(columns) == 0 {+ return false, nil+ }+ common.Info("Dropping legacy primary key", zap.String("table", table), zap.Strings("columns", columns))+ if err = db.WithContext(ctx).Exec("ALTER TABLE " + table + " DROP PRIMARY KEY").Error; err != nil {+ return false, fmt.Errorf("failed to drop the legacy primary key on %s: %w", table, err)+ }+ return false, nil+}++// hasUniqueIndex reports whether a unique index spans exactly the given columns.+// Matching on the column set rather than on the index name keeps the migrations+// idempotent when an equivalent index already exists under another name.+func hasUniqueIndex(ctx context.Context, db *gorm.DB, table string, columns []string) (bool, error) {+ // The column names come from this file's call sites, never from input.+ quoted := "'" + strings.Join(columns, "', '") + "'"+ var count int64+ // An index only qualifies when its whole column set is the requested one: a+ // broader unique index such as (email, tenant_id) does not make email unique+ // on its own. Prefix indexes are excluded because they only constrain a+ // leading substring.+ if err := db.WithContext(ctx).Raw(fmt.Sprintf(`+ SELECT COUNT(*) FROM (+ SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?+ AND NON_UNIQUE = 0 AND INDEX_NAME <> 'PRIMARY'+ AND SUB_PART IS NULL+ GROUP BY INDEX_NAME+ HAVING COUNT(*) = %d+ AND COUNT(CASE WHEN COLUMN_NAME IN (%s) THEN 1 END) = %d+ ) AS equivalent+ `, len(columns), quoted, len(columns)), table).Scan(&count).Error; err != nil {+ return false, err+ }+ return count > 0, nil+}++// addUniqueIndex creates a unique index over the given columns.+func addUniqueIndex(ctx context.Context, db *gorm.DB, table, indexName string, columns []string) error {+ sql := fmt.Sprintf("ALTER TABLE %s ADD UNIQUE INDEX %s (%s)", table, indexName, strings.Join(columns, ", "))+ if err := db.WithContext(ctx).Exec(sql).Error; err != nil {+ if isDuplicateIndexErr(err) {+ common.Info("Index already exists, skipping", zap.String("index", indexName))+ return nil}+ return fmt.Errorf("failed to add unique index %s on %s: %w", indexName, table, err)}+ return nil+}+// dropIndexIfExists removes a named index when it is present.+func dropIndexIfExists(ctx context.Context, db *gorm.DB, table, indexName string) error {+ var count int64+ if err := db.WithContext(ctx).Raw(`+ SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?+ `, table, indexName).Scan(&count).Error; err != nil {+ return err+ }+ if count == 0 {+ return nil+ }+ if err := db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DROP INDEX %s", table, indexName)).Error; err != nil {+ return fmt.Errorf("failed to drop index %s on %s: %w", indexName, table, err)+ }return nil}-// migrateSkillSpaceTables creates skill space related tables-func migrateSkillSpaceTables(ctx context.Context, db *gorm.DB) error {- if !db.WithContext(ctx).Migrator().HasTable("skill_spaces") {- common.Info("Creating skill_spaces table...")- sql := `- CREATE TABLE IF NOT EXISTS skill_spaces (- id VARCHAR(32) PRIMARY KEY,- tenant_id VARCHAR(32) NOT NULL,- name VARCHAR(128) NOT NULL,- folder_id VARCHAR(32) NOT NULL,- description TEXT,- embd_id VARCHAR(128),- rerank_id VARCHAR(128),- top_k INT DEFAULT 10,- status VARCHAR(1) DEFAULT '1',- create_time BIGINT,- create_date DATETIME,- update_time BIGINT,- update_date DATETIME,- INDEX idx_tenant_id (tenant_id),- UNIQUE INDEX idx_tenant_name_status (tenant_id, name, status)- )- `- if err := db.WithContext(ctx).Exec(sql).Error; err != nil {- common.Warn("Failed to create skill_spaces table with MySQL dialect, trying generic", zap.Error(err))- // Try with AutoMigrate as fallback- if err = db.WithContext(ctx).AutoMigrate(&entity.SkillSpace{}); err != nil {- return err- }- // AutoMigrate doesn't create unique indexes, so create them explicitly- common.Info("Creating unique indexes for skill_spaces...")- if err = db.WithContext(ctx).Exec(`ALTER TABLE skill_spaces ADD UNIQUE INDEX idx_tenant_name_status (tenant_id, name, status)`).Error; err != nil {- return fmt.Errorf("failed to create unique index idx_tenant_name_status: %w", err)- }- }- } else {- // Migrate existing table: add status column first, then update index- if err := addColumnIfNotExists(ctx, db, "skill_spaces", "status", "VARCHAR(1) NOT NULL DEFAULT '1'"); err != nil {- return fmt.Errorf("failed to add status column to skill_spaces: %w", err)- }- if err := addColumnIfNotExists(ctx, db, "skill_spaces", "create_date", "DATETIME"); err != nil {- return fmt.Errorf("failed to add create_date column to skill_spaces: %w", err)- }- if err := addColumnIfNotExists(ctx, db, "skill_spaces", "update_date", "DATETIME"); err != nil {- return fmt.Errorf("failed to add update_date column to skill_spaces: %w", err)+// dropColumnIfExists removes a column when it is present.+func dropColumnIfExists(ctx context.Context, db *gorm.DB, table, column string) error {+ exists, err := columnExists(ctx, db, table, column)+ if err != nil {+ return err+ }+ if !exists {+ return nil+ }+ common.Info("Dropping column", zap.String("table", table), zap.String("column", column))+ if err = db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", table, column)).Error; err != nil {+ return fmt.Errorf("failed to drop column %s on %s: %w", column, table, err)+ }+ return nil+}++// isDuplicateIndexErr reports whether the error is MySQL error 1061 (duplicate+// key name), which means the index already exists.+func isDuplicateIndexErr(err error) bool {+ return strings.Contains(err.Error(), "Error 1061") || strings.Contains(err.Error(), "Duplicate key name")+}++// renameDuplicateEmails renames every user row that shares an address with+// another row, leaving the address on the superuser - or on the oldest row when+// the address has no superuser. The unique index is only creatable once no+// duplicates remain.+func renameDuplicateEmails(ctx context.Context, db *gorm.DB) error {+ var duplicates []string+ if err := db.WithContext(ctx).Raw(`+ SELECT email FROM user GROUP BY email HAVING COUNT(*) > 1+ `).Scan(&duplicates).Error; err != nil {+ return err+ }+ if len(duplicates) == 0 {+ return nil+ }++ common.Warn("Renaming duplicate user emails before adding the unique index", zap.Int("addresses", len(duplicates)))+ for _, email := range duplicates {+ var ids []string+ if err := db.WithContext(ctx).Raw(`+ SELECT id FROM user WHERE email = ?+ ORDER BY is_superuser DESC, create_time ASC+ `, email).Scan(&ids).Error; err != nil {+ return err}- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_spaces MODIFY COLUMN update_time BIGINT`).Error; err != nil {- common.Warn("Failed to modify skill_spaces.update_time", zap.Error(err))+ if len(ids) < 2 {+ continue}- // Migrate index after status column exists- if err := migrateSkillSpaceIndex(ctx, db); err != nil {- return fmt.Errorf("failed to migrate skill_space index: %w", err)+ for _, id := range ids[1:] {+ if err := db.WithContext(ctx).Exec(+ `UPDATE user SET email = ? WHERE id = ?`,+ duplicateEmailAddress(email, id), id,+ ).Error; err != nil {+ return fmt.Errorf("failed to rename duplicate email %q: %w", email, err)+ }}}-return nil}-// migrateSkillSpaceIndex migrates the unique index to include status-func migrateSkillSpaceIndex(ctx context.Context, db *gorm.DB) error {- // Check if old index exists and drop it- var oldIndexExists int64- db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'skill_spaces' AND INDEX_NAME = 'idx_tenant_name'- `).Scan(&oldIndexExists)+// duplicateEmailAddress builds the placeholder address for a shadowed duplicate+// row, clamped to the column width so the rename cannot fail on length.+func duplicateEmailAddress(email, id string) string {+ const (+ maxEmailLength = 255+ duplicateTag = "_DUPLICATE_"+ idPrefixLength = 8+ )+ if len(id) > idPrefixLength {+ id = id[:idPrefixLength]+ }+ suffix := duplicateTag + id+ runes := []rune(email)+ if keep := maxEmailLength - len(suffix); len(runes) > keep {+ runes = runes[:keep]+ }+ return string(runes) + suffix+}- if oldIndexExists > 0 {- common.Info("Dropping old idx_tenant_name index from skill_spaces...")- if err := db.WithContext(ctx).Exec(`DROP INDEX idx_tenant_name ON skill_spaces`).Error; err != nil {- return fmt.Errorf("failed to drop old index idx_tenant_name: %w", err)- }+// integerDisplayWidth matches the deprecated integer display width emitted by+// MySQL 5.7 ("int(11)"), which MySQL 8 accepts but no longer reports.+var integerDisplayWidth = regexp.MustCompile(`^(tinyint|smallint|mediumint|int|bigint)\(\d+\)$`)++// normalizeColumnType strips integer display widths so a stored type compares+// equal to its canonical name.+func normalizeColumnType(columnType string) string {+ lowered := strings.ToLower(strings.TrimSpace(columnType))+ if match := integerDisplayWidth.FindStringSubmatch(lowered); match != nil {+ return match[1]}+ return lowered+}- // Check if new index exists- var newIndexExists int64- db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS- WHERE TABLE_NAME = 'skill_spaces' AND INDEX_NAME = 'idx_tenant_name_status'- `).Scan(&newIndexExists)+// columnSpec is the definition a column is expected to have.+type columnSpec struct {+ table string+ column string+ columnType string // compared against INFORMATION_SCHEMA.COLUMNS.COLUMN_TYPE+ nullable bool+ definition string // used verbatim when an ALTER is required+}- if newIndexExists == 0 {- common.Info("Creating new idx_tenant_name_status index on skill_spaces...")- if err := db.WithContext(ctx).Exec(`CREATE UNIQUE INDEX idx_tenant_name_status ON skill_spaces(tenant_id, name, status)`).Error; err != nil {- return fmt.Errorf("failed to create unique index idx_tenant_name_status: %w", err)- }+// applyColumnSpec aligns a column with its expected definition, issuing DDL only+// when the stored definition differs so that a steady-state startup runs no+// ALTER at all.+func applyColumnSpec(ctx context.Context, db *gorm.DB, spec columnSpec) error {+ if !db.WithContext(ctx).Migrator().HasTable(spec.table) {+ return nil+ }++ currentType, nullable, exists, err := columnDefinition(ctx, db, spec.table, spec.column)+ if err != nil {+ return err+ }+ if !exists {+ return nil+ }+ if normalizeColumnType(currentType) == normalizeColumnType(spec.columnType) && nullable == spec.nullable {+ return nil}+ common.Info("Modifying column type",+ zap.String("table", spec.table),+ zap.String("column", spec.column),+ zap.String("from", currentType),+ zap.String("to", spec.columnType))+ if err = db.WithContext(ctx).Exec(+ fmt.Sprintf("ALTER TABLE %s MODIFY COLUMN %s %s", spec.table, spec.column, spec.definition),+ ).Error; err != nil {+ common.Warn("Failed to modify column",+ zap.String("table", spec.table),+ zap.String("column", spec.column),+ zap.Error(err))+ }return nil}diff --git a/internal/dao/migration_version.go b/internal/dao/migration_version.gonew file mode 100644index 0000000000..5c2fa5604b--- /dev/null+++ b/internal/dao/migration_version.go@@ -0,0 +1,125 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "database/sql"+ "errors"+ "strings"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/entity"++ "go.uber.org/zap"+ "golang.org/x/mod/semver"+ "gorm.io/gorm"+ "gorm.io/gorm/clause"+)++// migrationDBVersionMarker is the system_settings key that records how far the+// database has been migrated. It is shared with tools/scripts/mysql_migration.py+// so the Python and Go migrations can skip work the other has already done.+const migrationDBVersionMarker = "mysql_migration.database.version"++// modelMigrationBaseVersion is the version the base tenant model table+// migration brings the database to. It mirrors the first --database-version used+// by tools/scripts/run_migrations.sh, so a database already carrying it came+// from a v0.26.0 release and only needs the remaining model data stages.+const modelMigrationBaseVersion = "v0.26.0"++// modelMigrationTargetVersion is the version the remaining tenant model data+// migration brings the database to. It mirrors the final --database-version used+// by tools/scripts/run_migrations.sh.+const modelMigrationTargetVersion = "v0.27.2"++// GetDatabaseMigrationVersion reads the stored migration version. It returns an+// empty string when the marker or the system_settings table is absent.+func GetDatabaseMigrationVersion(ctx context.Context, db *gorm.DB) (string, error) {+ scoped := db.WithContext(ctx)+ if !scoped.Migrator().HasTable("system_settings") {+ common.Info("Table 'system_settings' does not exist, migration version marker is unavailable")+ return "", nil+ }++ var value string+ err := scoped.Raw("SELECT `value` FROM `system_settings` WHERE `name` = ?", migrationDBVersionMarker).Row().Scan(&value)+ if err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return "", nil+ }+ return "", err+ }+ return value, nil+}++// setDatabaseMigrationVersion upserts the migration version marker.+func setDatabaseMigrationVersion(ctx context.Context, db *gorm.DB, version string) error {+ scoped := db.WithContext(ctx)+ if !scoped.Migrator().HasTable("system_settings") {+ common.Warn("Table 'system_settings' does not exist, migration version marker was not saved")+ return nil+ }++ nowMs := time.Now().UnixMilli()+ now := time.Now()+ setting := entity.SystemSettings{+ Name: migrationDBVersionMarker,+ Source: "migration",+ DataType: "string",+ Value: version,+ BaseModel: entity.BaseModel{+ CreateTime: &nowMs,+ CreateDate: &now,+ UpdateTime: &nowMs,+ UpdateDate: &now,+ },+ }+ return scoped.Clauses(clause.OnConflict{+ Columns: []clause.Column{{Name: "name"}},+ DoUpdates: clause.AssignmentColumns([]string{"source", "data_type", "value", "update_time", "update_date"}),+ }).Create(&setting).Error+}++// shouldSkipMigration mirrors should_skip_migration in the Python migration: a+// missing or unparseable version never skips the migration.+func shouldSkipMigration(currentDBVersion, targetVersion string) bool {+ current := parseMigrationVersion(currentDBVersion)+ target := parseMigrationVersion(targetVersion)+ if current == "" || target == "" {+ return false+ }+ return semver.Compare(current, target) >= 0+}++// parseMigrationVersion normalizes a version to the "vMAJOR.MINOR.PATCH" form+// understood by semver, returning an empty string when it cannot be parsed.+func parseMigrationVersion(version string) string {+ normalized := strings.TrimSpace(version)+ normalized = strings.TrimPrefix(normalized, "v")+ normalized = strings.TrimPrefix(normalized, "V")+ if normalized == "" {+ return ""+ }+ normalized = "v" + normalized+ if !semver.IsValid(normalized) {+ common.Warn("Invalid migration version format", zap.String("version", version))+ return ""+ }+ return normalized+}diff --git a/internal/dao/migration_version_test.go b/internal/dao/migration_version_test.gonew file mode 100644index 0000000000..1842ad22d4--- /dev/null+++ b/internal/dao/migration_version_test.go@@ -0,0 +1,158 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/entity"+)++func setupMigrationVersionTestDB(t *testing.T, migrate bool) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }+ if migrate {+ if err = db.AutoMigrate(&entity.SystemSettings{}); err != nil {+ t.Fatalf("failed to migrate system_settings: %v", err)+ }+ }+ return db+}++func TestShouldSkipMigration(t *testing.T) {+ cases := []struct {+ name string+ current string+ target string+ want bool+ }{+ {name: "missing current", current: "", target: "v0.27.1", want: false},+ {name: "older current", current: "v0.26.0", target: "v0.27.1", want: false},+ {name: "equal", current: "v0.27.1", target: "v0.27.1", want: true},+ {name: "newer current", current: "v0.28.0", target: "v0.27.1", want: true},+ {name: "missing v prefix", current: "0.27.1", target: "v0.27.1", want: true},+ {name: "invalid current", current: "not-a-version", target: "v0.27.1", want: false},+ {name: "invalid target", current: "v0.27.1", target: "not-a-version", want: false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := shouldSkipMigration(tc.current, tc.target); got != tc.want {+ t.Fatalf("shouldSkipMigration(%q, %q) = %v, want %v", tc.current, tc.target, got, tc.want)+ }+ })+ }+}++func TestGetAndSetDatabaseMigrationVersion(t *testing.T) {+ db := setupMigrationVersionTestDB(t, true)+ ctx := t.Context()++ version, err := GetDatabaseMigrationVersion(ctx, db)+ if err != nil {+ t.Fatalf("GetDatabaseMigrationVersion() error = %v", err)+ }+ if version != "" {+ t.Fatalf("initial version = %q, want empty", version)+ }++ if err = setDatabaseMigrationVersion(ctx, db, "v0.27.1"); err != nil {+ t.Fatalf("setDatabaseMigrationVersion() error = %v", err)+ }+ version, err = GetDatabaseMigrationVersion(ctx, db)+ if err != nil {+ t.Fatalf("GetDatabaseMigrationVersion() after set error = %v", err)+ }+ if version != "v0.27.1" {+ t.Fatalf("version = %q, want v0.27.1", version)+ }++ // Upsert must update the existing marker instead of inserting a second row.+ if err = setDatabaseMigrationVersion(ctx, db, "v0.28.0"); err != nil {+ t.Fatalf("setDatabaseMigrationVersion() second error = %v", err)+ }+ version, err = GetDatabaseMigrationVersion(ctx, db)+ if err != nil {+ t.Fatalf("GetDatabaseMigrationVersion() second error = %v", err)+ }+ if version != "v0.28.0" {+ t.Fatalf("version = %q, want v0.28.0", version)+ }++ var count int64+ if err = db.Model(&entity.SystemSettings{}).Where("name = ?", migrationDBVersionMarker).Count(&count).Error; err != nil {+ t.Fatalf("count markers: %v", err)+ }+ if count != 1 {+ t.Fatalf("marker count = %d, want 1", count)+ }+}++func TestGetDatabaseMigrationVersionWithoutTable(t *testing.T) {+ db := setupMigrationVersionTestDB(t, false)+ ctx := t.Context()++ version, err := GetDatabaseMigrationVersion(ctx, db)+ if err != nil {+ t.Fatalf("GetDatabaseMigrationVersion() error = %v", err)+ }+ if version != "" {+ t.Fatalf("version = %q, want empty", version)+ }+ if err = setDatabaseMigrationVersion(ctx, db, "v0.27.1"); err != nil {+ t.Fatalf("setDatabaseMigrationVersion() without table error = %v", err)+ }+}++func TestEnsureMigrationVersionTable(t *testing.T) {+ db := setupMigrationVersionTestDB(t, false)+ ctx := t.Context()++ if err := ensureMigrationVersionTable(ctx, db); err != nil {+ t.Fatalf("ensureMigrationVersionTable() error = %v", err)+ }+ if !db.Migrator().HasTable("system_settings") {+ t.Fatal("system_settings was not created")+ }++ // Idempotent: the existing table is left alone.+ if err := ensureMigrationVersionTable(ctx, db); err != nil {+ t.Fatalf("ensureMigrationVersionTable() second error = %v", err)+ }+}++func TestMigrateModelDataPersistsVersionOnFirstRun(t *testing.T) {+ db := setupMigrationVersionTestDB(t, false)+ ctx := t.Context()++ if err := migrateModelData(ctx, db); err != nil {+ t.Fatalf("migrateModelData() error = %v", err)+ }+ version, err := GetDatabaseMigrationVersion(ctx, db)+ if err != nil {+ t.Fatalf("GetDatabaseMigrationVersion() error = %v", err)+ }+ if version != modelMigrationTargetVersion {+ t.Fatalf("version = %q, want %q: the marker must be persisted on the first run so a populated database is never migrated twice",+ version, modelMigrationTargetVersion)+ }+}diff --git a/internal/dao/model_merge_test.go b/internal/dao/model_merge_test.gonew file mode 100644index 0000000000..0a4c66e36f--- /dev/null+++ b/internal/dao/model_merge_test.go@@ -0,0 +1,266 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "fmt"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"+)++func setupTenantModelMergeDB(t *testing.T, modelTypeDefinition string) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }+ ddl := fmt.Sprintf(`CREATE TABLE tenant_model (+ id VARCHAR(32) NOT NULL PRIMARY KEY,+ model_name VARCHAR(128),+ provider_id VARCHAR(32) NOT NULL,+ instance_id VARCHAR(32) NOT NULL,+ model_type %s NOT NULL,+ status VARCHAR(32) DEFAULT 'active',+ extra VARCHAR(1024) DEFAULT '{}',+ create_time BIGINT,+ create_date DATETIME,+ update_time BIGINT,+ update_date DATETIME+ )`, modelTypeDefinition)+ if err = db.Exec(ddl).Error; err != nil {+ t.Fatalf("failed to create tenant_model: %v", err)+ }+ return db+}++func insertTenantModelMergeRow(t *testing.T, db *gorm.DB, id, model, provider, instance, modelType, status string) {+ t.Helper()+ if err := db.Exec(+ `INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, '{}', 1000, NULL, 1000, NULL)`,+ id, model, provider, instance, modelType, status,+ ).Error; err != nil {+ t.Fatalf("failed to seed tenant_model row: %v", err)+ }+}++func TestMergeTenantModelTypes(t *testing.T) {+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")+ ctx := t.Context()++ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "active")+ insertTenantModelMergeRow(t, db, "2", "gpt-4o", "p1", "i1", "vision", "active")+ insertTenantModelMergeRow(t, db, "3", "m1", "p2", "i2", "embedding", "unsupported")+ insertTenantModelMergeRow(t, db, "4", "m1", "p2", "i2", "chat", "active")+ insertTenantModelMergeRow(t, db, "5", "m2", "p3", "i3", "chat", "inactive")+ // The seeding stage writes the merged integer into the text column.+ insertTenantModelMergeRow(t, db, "6", "m3", "p4", "i4", "1", "active")++ if err := mergeTenantModelTypes(ctx, db); err != nil {+ t.Fatalf("mergeTenantModelTypes() error = %v", err)+ }++ type mergedRow struct {+ ModelName string `gorm:"column:model_name"`+ ModelType int `gorm:"column:model_type"`+ Status string `gorm:"column:status"`+ }+ var rows []mergedRow+ if err := db.Raw("SELECT model_name, model_type, status FROM tenant_model ORDER BY model_name").Scan(&rows).Error; err != nil {+ t.Fatalf("failed to read merged rows: %v", err)+ }++ want := []mergedRow{+ {ModelName: "gpt-4o", ModelType: 9, Status: "active"}, // chat | vision+ {ModelName: "m1", ModelType: 1, Status: "active"}, // chat, embedding dropped as unsupported+ {ModelName: "m3", ModelType: 1, Status: "active"}, // numeric text preserved+ }+ if len(rows) != len(want) {+ t.Fatalf("merged row count = %d, want %d (%+v)", len(rows), len(want), rows)+ }+ for i := range want {+ if rows[i] != want[i] {+ t.Fatalf("merged row %d = %+v, want %+v", i, rows[i], want[i])+ }+ }+}++func TestMergeTenantModelTypesSkipsIntegerColumn(t *testing.T) {+ db := setupTenantModelMergeDB(t, "INT")+ ctx := t.Context()++ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "1", "active")+ insertTenantModelMergeRow(t, db, "2", "gpt-4o", "p1", "i1", "1", "active")++ if err := mergeTenantModelTypes(ctx, db); err != nil {+ t.Fatalf("mergeTenantModelTypes() error = %v", err)+ }++ var count int64+ if err := db.Raw("SELECT COUNT(*) FROM tenant_model").Scan(&count).Error; err != nil {+ t.Fatalf("failed to count rows: %v", err)+ }+ if count != 2 {+ t.Fatalf("row count = %d, want 2 (integer column must be left untouched)", count)+ }+}++func TestMergeTenantModelTypesPreservesTimestamps(t *testing.T) {+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")+ ctx := t.Context()++ if err := db.Exec(+ `INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, '{}', ?, ?, ?, ?)`,+ "1", "gpt-4o", "p1", "i1", "chat", "active", 1700000000000, "2023-11-14 22:13:20", 1700000000000, "2023-11-14 22:13:20",+ ).Error; err != nil {+ t.Fatalf("failed to seed tenant_model row: %v", err)+ }++ if err := mergeTenantModelTypes(ctx, db); err != nil {+ t.Fatalf("mergeTenantModelTypes() error = %v", err)+ }++ var row struct {+ CreateTime int64 `gorm:"column:create_time"`+ UpdateTime int64 `gorm:"column:update_time"`+ }+ if err := db.Raw("SELECT create_time, update_time FROM tenant_model").Scan(&row).Error; err != nil {+ t.Fatalf("failed to read merged row: %v", err)+ }+ if row.CreateTime != 1700000000000 || row.UpdateTime != 1700000000000 {+ t.Fatalf("merged timestamps = (%d, %d), want (1700000000000, 1700000000000)", row.CreateTime, row.UpdateTime)+ }+}++func TestMergeTenantModelTypesClearsDroppedRows(t *testing.T) {+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")+ ctx := t.Context()++ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "inactive")++ if err := mergeTenantModelTypes(ctx, db); err != nil {+ t.Fatalf("mergeTenantModelTypes() error = %v", err)+ }++ var count int64+ if err := db.Raw("SELECT COUNT(*) FROM tenant_model").Scan(&count).Error; err != nil {+ t.Fatalf("failed to count rows: %v", err)+ }+ if count != 0 {+ t.Fatalf("row count = %d, want 0 (dropped groups must not stay in their text shape)", count)+ }+}++func TestEnsureTenantModelTables(t *testing.T) {+ t.Run("creates the missing tables without touching a text tenant_model", func(t *testing.T) {+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")+ if err := db.Exec("CREATE TABLE tenant_llm (id VARCHAR(32) NOT NULL PRIMARY KEY)").Error; err != nil {+ t.Fatalf("failed to create tenant_llm: %v", err)+ }+ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "active")++ if err := ensureTenantModelTables(t.Context(), db); err != nil {+ t.Fatalf("ensureTenantModelTables() error = %v", err)+ }++ for _, table := range []string{"tenant_model_provider", "tenant_model_instance"} {+ if !db.Migrator().HasTable(table) {+ t.Fatalf("%s was not created", table)+ }+ }+ var modelType string+ if err := db.Raw("SELECT model_type FROM tenant_model").Scan(&modelType).Error; err != nil {+ t.Fatalf("failed to read model_type: %v", err)+ }+ if modelType != "chat" {+ t.Fatalf("model_type = %q, want the untouched text value \"chat\"", modelType)+ }+ })++ t.Run("leaves the database alone without tenant_llm", func(t *testing.T) {+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")++ if err := ensureTenantModelTables(t.Context(), db); err != nil {+ t.Fatalf("ensureTenantModelTables() error = %v", err)+ }+ if db.Migrator().HasTable("tenant_model_provider") {+ t.Fatal("tenant_model_provider must not be created without tenant_llm")+ }+ })+}++func TestLoadTenantModelRowMapParsesTextModelType(t *testing.T) {+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")+ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "active")+ insertTenantModelMergeRow(t, db, "2", "m1", "p2", "i2", "9", "active")++ rows, err := loadTenantModelRowMap(db)+ if err != nil {+ t.Fatalf("loadTenantModelRowMap() error = %v", err)+ }+ if got := rows["p1\x00i1\x00gpt-4o"].modelType; got != 1 {+ t.Fatalf("gpt-4o modelType = %d, want 1 (text name)", got)+ }+ if got := rows["p2\x00i2\x00m1"].modelType; got != 9 {+ t.Fatalf("m1 modelType = %d, want 9 (numeric text)", got)+ }+}++func TestTenantModelModelTypeIsInteger(t *testing.T) {+ cases := []struct {+ name string+ definition string+ want bool+ }{+ {name: "text", definition: "VARCHAR(32)", want: false},+ {name: "integer", definition: "INT", want: true},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ db := setupTenantModelMergeDB(t, tc.definition)+ got, err := tenantModelModelTypeIsInteger(db)+ if err != nil {+ t.Fatalf("tenantModelModelTypeIsInteger() error = %v", err)+ }+ if got != tc.want {+ t.Fatalf("tenantModelModelTypeIsInteger() = %v, want %v", got, tc.want)+ }+ })+ }+}++func TestMergeTenantModelTypeBits(t *testing.T) {+ cases := []struct {+ raw string+ want int+ }{+ {raw: "chat", want: 1},+ {raw: "Vision", want: 8},+ {raw: "speech2text", want: 4},+ {raw: " 7 ", want: 7},+ {raw: "unknown", want: 0},+ {raw: "", want: 0},+ }+ for _, tc := range cases {+ t.Run(tc.raw, func(t *testing.T) {+ if got := mergeTenantModelTypeBits(tc.raw); got != tc.want {+ t.Fatalf("mergeTenantModelTypeBits(%q) = %d, want %d", tc.raw, got, tc.want)+ }+ })+ }+}diff --git a/internal/dao/model_migration.go b/internal/dao/model_migration.gonew file mode 100644index 0000000000..822438d4c3--- /dev/null+++ b/internal/dao/model_migration.go@@ -0,0 +1,1556 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "crypto/rand"+ "database/sql"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "sort"+ "strconv"+ "strings"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/entity"++ "go.uber.org/zap"+ "gorm.io/gorm"+)++// This file ports tools/scripts/mysql_migration.py. It backfills the tenant+// model tables (tenant_model_provider, tenant_model_instance, tenant_model)+// from the legacy tenant_llm table, rewrites stored model ids from+// "<model>@<provider>" to "<model>@default@<provider>", and populates the+// tenant_*_id columns. Every step is idempotent, so re-running a stage is safe,+// and the two steps are gated by the database version in system_settings,+// mirroring the two --database-version calls in tools/scripts/run_migrations.sh.++const modelMigrationInstanceName = "default"++// modelTypeStringToBit mirrors MODEL_TYPE_TO_INT in the Python migration and+// entity.ModelType's bit layout.+var modelTypeStringToBit = map[string]int{+ "chat": 1,+ "embedding": 2,+ "asr": 4,+ "speech2text": 4,+ "vision": 8,+ "image2text": 8,+ "rerank": 16,+ "tts": 32,+ "ocr": 64,+}++// ensureTenantModelTables creates the tenant model tables the base step backfills+// when they do not exist yet and the legacy tenant_llm table does. It mirrors the+// CREATE TABLE IF NOT EXISTS that the Python stages run before inserting.+//+// Existing tables are never altered: tenant_model.model_type must keep the text+// shape the Python stage left behind so mergeTenantModelTypes can still read the+// legacy model names instead of the integer AutoMigrate would converge it to.+func ensureTenantModelTables(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ if !scoped.Migrator().HasTable("tenant_llm") {+ return nil+ }+ for _, model := range []any{&entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{}} {+ if scoped.Migrator().HasTable(model) {+ continue+ }+ if err := scoped.Migrator().CreateTable(model); err != nil {+ return err+ }+ }+ return nil+}++// ensureMigrationVersionTable creates system_settings when it is missing. The+// model data migration records its progress there and runs before AutoMigrate+// (see InitDB), so on a database that predates the table the version marker could+// not be written. Every later startup would then replay the migration against+// tenant_model rows that already hold data.+func ensureMigrationVersionTable(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ if scoped.Migrator().HasTable("system_settings") {+ return nil+ }+ return scoped.Migrator().CreateTable(&entity.SystemSettings{})+}++// migrateModelData runs the model related data migrations in the two versioned+// steps used by tools/scripts/run_migrations.sh. Each step records its own+// version in migrationDBVersionMarker, so work already performed by the Python+// script, or by a previous Go run, is skipped.+//+// - The base step (modelMigrationBaseVersion) rebuilds+// tenant_model_provider/tenant_model_instance/tenant_model from tenant_llm+// and normalizes the stored model ids. It is only needed on a database older+// than v0.26.0.+// - The follow-up step (modelMigrationTargetVersion) seeds the factory+// declared models, merges the tenant_model rows into the integer model_type+// representation and populates the tenant_*_id columns. It is needed on+// every database older than v0.27.2, including one already at v0.26.0.+//+// The step runs before AutoMigrate (see InitDB), so it creates the tables it+// depends on itself: system_settings for the version marker, and, on a database+// older than v0.26.0, the tenant model tables it backfills from tenant_llm.+func migrateModelData(ctx context.Context, db *gorm.DB) error {+ if err := ensureMigrationVersionTable(ctx, db); err != nil {+ return fmt.Errorf("ensure migration version table: %w", err)+ }+ if err := ensureTenantModelTables(ctx, db); err != nil {+ return fmt.Errorf("ensure tenant model tables: %w", err)+ }++ currentVersion, err := GetDatabaseMigrationVersion(ctx, db)+ if err != nil {+ return fmt.Errorf("read database migration version: %w", err)+ }++ if shouldSkipMigration(currentVersion, modelMigrationBaseVersion) {+ common.Info("Tenant model tables already migrated, skipping base step",+ zap.String("current_version", currentVersion),+ zap.String("step_version", modelMigrationBaseVersion))+ } else {+ common.Info("Running base tenant model table migration",+ zap.String("current_version", currentVersion),+ zap.String("step_version", modelMigrationBaseVersion))++ if err := migrateTenantModelProviders(ctx, db); err != nil {+ return fmt.Errorf("migrate tenant_model_provider: %w", err)+ }+ if err := migrateTenantModelInstances(ctx, db); err != nil {+ return fmt.Errorf("migrate tenant_model_instance: %w", err)+ }+ if err := migrateTenantModels(ctx, db); err != nil {+ return fmt.Errorf("migrate tenant_model: %w", err)+ }+ if err := normalizeStoredModelIDs(ctx, db); err != nil {+ return fmt.Errorf("normalize model ids: %w", err)+ }+ if err := setDatabaseMigrationVersion(ctx, db, modelMigrationBaseVersion); err != nil {+ return fmt.Errorf("mark database migration version %s: %w", modelMigrationBaseVersion, err)+ }+ currentVersion = modelMigrationBaseVersion+ }++ if shouldSkipMigration(currentVersion, modelMigrationTargetVersion) {+ common.Info("Tenant model data already migrated, skipping",+ zap.String("current_version", currentVersion),+ zap.String("target_version", modelMigrationTargetVersion))+ return nil+ }+ common.Info("Running tenant model data migration",+ zap.String("current_version", currentVersion),+ zap.String("target_version", modelMigrationTargetVersion))++ if err := seedTenantModels(ctx, db); err != nil {+ return fmt.Errorf("seed tenant_model: %w", err)+ }+ if err := mergeTenantModelTypes(ctx, db); err != nil {+ return fmt.Errorf("merge tenant_model model types: %w", err)+ }+ if err := populateTenantModelIDColumns(ctx, db); err != nil {+ return fmt.Errorf("populate tenant model id columns: %w", err)+ }++ if err := setDatabaseMigrationVersion(ctx, db, modelMigrationTargetVersion); err != nil {+ return fmt.Errorf("mark database migration version: %w", err)+ }+ common.Info("Tenant model data migration completed", zap.String("version", modelMigrationTargetVersion))+ return nil+}++// modelMigrationNewID returns a 32 character hex id, matching uuid.uuid1().hex.+func modelMigrationNewID() string {+ buffer := make([]byte, 16)+ if _, err := rand.Read(buffer); err != nil {+ return fmt.Sprintf("%032x", time.Now().UnixNano())+ }+ return hex.EncodeToString(buffer)+}++// modelStatusIsActive mirrors the status mapping used by the Python stages:+// "active" if status in ("1", "active", "enable") else "inactive".+func modelStatusIsActive(status string) bool {+ switch strings.ToLower(strings.TrimSpace(status)) {+ case "1", "active", "enable":+ return true+ default:+ return false+ }+}++func modelStatusValue(status string) string {+ if modelStatusIsActive(status) {+ return "active"+ }+ return "inactive"+}++func isAPIKeyFieldSubset(parsed map[string]any) bool {+ for key := range parsed {+ if key != "api_key" && key != "is_tools" {+ return false+ }+ }+ return true+}++// stripIsToolsFromAPIKey mirrors MigrationStage._strip_is_tools_from_api_key.+// It is the canonical form used to match an api_key against stored instances.+func stripIsToolsFromAPIKey(apiKey string) string {+ if !strings.HasPrefix(apiKey, "{") {+ return apiKey+ }+ var parsed map[string]any+ if err := json.Unmarshal([]byte(apiKey), &parsed); err != nil {+ return apiKey+ }+ if isAPIKeyFieldSubset(parsed) {+ if inner, ok := parsed["api_key"].(string); ok {+ return inner+ }+ return apiKey+ }+ if _, ok := parsed["is_tools"]; !ok {+ return apiKey+ }+ delete(parsed, "is_tools")+ encoded, err := json.Marshal(parsed)+ if err != nil {+ return apiKey+ }+ return string(encoded)+}++// extractExtraFromAPIKey mirrors TenantModelStage._extract_extra_from_api_key.+func extractExtraFromAPIKey(apiKey string) string {+ if !strings.HasPrefix(apiKey, "{") {+ return "{}"+ }+ var parsed map[string]any+ if err := json.Unmarshal([]byte(apiKey), &parsed); err != nil {+ return "{}"+ }+ if isTools, ok := parsed["is_tools"].(bool); ok && isTools {+ return `{"is_tools":true}`+ }+ return "{}"+}++// tenantLLMSourceRecord is a projected row of the legacy tenant_llm table.+type tenantLLMSourceRecord struct {+ TenantID string `gorm:"column:tenant_id"`+ Factory string `gorm:"column:llm_factory"`+ ModelName sql.NullString `gorm:"column:model_name"`+ ModelType sql.NullString `gorm:"column:model_type"`+ APIKey sql.NullString `gorm:"column:api_key"`+ Status sql.NullString `gorm:"column:status"`+ ProviderID string `gorm:"column:provider_id"`+}++// migrateTenantModelProviders ports TenantModelProviderStage: one provider row+// per distinct (tenant_id, llm_factory) found in tenant_llm.+func migrateTenantModelProviders(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ if !scoped.Migrator().HasTable("tenant_llm") || !scoped.Migrator().HasTable("tenant_model_provider") {+ return nil+ }++ var rows []struct {+ TenantID string `gorm:"column:tenant_id"`+ Factory string `gorm:"column:llm_factory"`+ }+ if err := scoped.Raw(`+ SELECT DISTINCT t1.tenant_id AS tenant_id, t1.llm_factory AS llm_factory+ FROM tenant_llm t1+ WHERE NOT EXISTS (+ SELECT 1 FROM tenant_model_provider t2+ WHERE t2.tenant_id = t1.tenant_id AND t2.provider_name = t1.llm_factory+ )`).Scan(&rows).Error; err != nil {+ return err+ }+ if len(rows) == 0 {+ return nil+ }++ nowMs := time.Now().UnixMilli()+ now := time.Now()+ if err := scoped.Transaction(func(tx *gorm.DB) error {+ for _, row := range rows {+ if err := tx.Exec(+ "INSERT INTO tenant_model_provider (id, provider_name, tenant_id, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?)",+ modelMigrationNewID(), row.Factory, row.TenantID, nowMs, now, nowMs, now,+ ).Error; err != nil {+ return err+ }+ }+ return nil+ }); err != nil {+ return err+ }++ common.Info("Migrated tenant_model_provider records from tenant_llm", zap.Int("count", len(rows)))+ return nil+}++// migrateTenantModelInstances ports TenantModelInstanceStage: one instance row+// per (tenant_id, llm_factory) taking the first api_key value after dedup.+func migrateTenantModelInstances(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ for _, table := range []string{"tenant_llm", "tenant_model_provider", "tenant_model_instance"} {+ if !scoped.Migrator().HasTable(table) {+ return nil+ }+ }++ existing, err := buildTenantModelInstanceCanonicalSet(ctx, scoped)+ if err != nil {+ return err+ }++ var rows []tenantLLMSourceRecord+ if err := scoped.Raw(`+ SELECT tl.tenant_id AS tenant_id, tl.llm_factory AS llm_factory, tl.api_key AS api_key,+ MAX(tl.status) AS status, tmp.id AS provider_id+ FROM tenant_llm tl+ INNER JOIN tenant_model_provider tmp+ ON tmp.tenant_id = tl.tenant_id AND tmp.provider_name = tl.llm_factory+ WHERE NOT EXISTS (+ SELECT 1 FROM tenant_model_instance tmi+ WHERE tmi.provider_id = tmp.id AND tmi.api_key = tl.api_key+ )+ GROUP BY tl.tenant_id, tl.llm_factory, tl.api_key, tmp.id`).Scan(&rows).Error; err != nil {+ return err+ }++ rows = filterExistingTenantModelInstances(rows, existing)+ rows = dedupTenantModelInstances(rows)+ if len(rows) == 0 {+ return nil+ }++ nowMs := time.Now().UnixMilli()+ now := time.Now()+ if err := scoped.Transaction(func(tx *gorm.DB) error {+ for _, row := range rows {+ if err := tx.Exec(+ "INSERT INTO tenant_model_instance (id, instance_name, provider_id, api_key, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",+ modelMigrationNewID(), modelMigrationInstanceName, row.ProviderID, row.APIKey.String,+ modelStatusValue(row.Status.String), "{}", nowMs, now, nowMs, now,+ ).Error; err != nil {+ return err+ }+ }+ return nil+ }); err != nil {+ return err+ }++ common.Info("Migrated tenant_model_instance records from tenant_llm", zap.Int("count", len(rows)))+ return nil+}++// buildTenantModelInstanceCanonicalSet collects the canonical api_key already+// stored for each provider so repeated runs do not insert duplicate instances.+func buildTenantModelInstanceCanonicalSet(ctx context.Context, db *gorm.DB) (map[string]bool, error) {+ var rows []struct {+ ProviderID string `gorm:"column:provider_id"`+ APIKey sql.NullString `gorm:"column:api_key"`+ }+ if err := db.Raw("SELECT provider_id, api_key FROM tenant_model_instance").Scan(&rows).Error; err != nil {+ return nil, err+ }+ existing := make(map[string]bool, len(rows))+ for _, row := range rows {+ existing[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)] = true+ }+ return existing, nil+}++func filterExistingTenantModelInstances(rows []tenantLLMSourceRecord, existing map[string]bool) []tenantLLMSourceRecord {+ if len(existing) == 0 {+ return rows+ }+ filtered := make([]tenantLLMSourceRecord, 0, len(rows))+ for _, row := range rows {+ if existing[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)] {+ continue+ }+ filtered = append(filtered, row)+ }+ return filtered+}++// dedupTenantModelInstances keeps a single instance per provider, dropping rows+// whose canonical api_key repeats within the same (tenant, factory, provider).+func dedupTenantModelInstances(rows []tenantLLMSourceRecord) []tenantLLMSourceRecord {+ groups := make(map[string][]tenantLLMSourceRecord)+ order := make([]string, 0, len(rows))+ for _, row := range rows {+ key := row.TenantID + "\x00" + row.Factory + "\x00" + row.ProviderID+ if _, ok := groups[key]; !ok {+ order = append(order, key)+ }+ groups[key] = append(groups[key], row)+ }++ result := make([]tenantLLMSourceRecord, 0, len(rows))+ for _, key := range order {+ group := groups[key]+ seen := make(map[string]bool, len(group))+ for _, row := range group {+ canonical := stripIsToolsFromAPIKey(row.APIKey.String)+ if seen[canonical] {+ continue+ }+ seen[canonical] = true+ result = append(result, row)+ }+ }+ return result+}++// migrateTenantModels ports TenantModelStage combined with ModelTypeMergeStage.+// tenant_model.model_type is an integer column, so the merged representation is+// written directly: only groups whose merged status is "active" are kept and+// model_type is the OR of the contributing model bits.+func migrateTenantModels(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ for _, table := range []string{"tenant_llm", "tenant_model_provider", "tenant_model_instance", "tenant_model"} {+ if !scoped.Migrator().HasTable(table) {+ return nil+ }+ }++ instanceLookup, err := buildTenantModelInstanceLookup(ctx, scoped)+ if err != nil {+ return err+ }+ if len(instanceLookup) == 0 {+ return nil+ }++ query := fmt.Sprintf(`+ SELECT tl.llm_name AS model_name, tmp.id AS provider_id, tl.model_type AS model_type,+ tl.status AS status, tl.api_key AS api_key+ FROM tenant_llm tl+ INNER JOIN tenant_model_provider tmp+ ON tmp.tenant_id = tl.tenant_id AND tmp.provider_name = tl.llm_factory+ WHERE %s+ AND NOT EXISTS (+ SELECT 1 FROM tenant_model tm+ WHERE tm.provider_id = tmp.id AND tm.model_name = tl.llm_name+ )`, buildTenantModelStatusCondition())++ var rows []tenantLLMSourceRecord+ if err := scoped.Raw(query).Scan(&rows).Error; err != nil {+ return err+ }+ if len(rows) == 0 {+ return nil+ }++ type modelGroup struct {+ providerID string+ instanceID string+ modelName string+ modelType int+ status string+ extra string+ }++ groups := make(map[string]*modelGroup)+ order := make([]string, 0, len(rows))+ for _, row := range rows {+ instanceID := instanceLookup[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)]+ if instanceID == "" {+ continue+ }+ key := row.ProviderID + "\x00" + instanceID + "\x00" + row.ModelName.String+ group, ok := groups[key]+ if !ok {+ group = &modelGroup{+ providerID: row.ProviderID,+ instanceID: instanceID,+ modelName: row.ModelName.String,+ status: modelStatusValue(row.Status.String),+ extra: extractExtraFromAPIKey(row.APIKey.String),+ }+ groups[key] = group+ order = append(order, key)+ }+ group.modelType |= modelTypeStringToBit[strings.ToLower(strings.TrimSpace(row.ModelType.String))]+ }++ nowMs := time.Now().UnixMilli()+ now := time.Now()+ inserted := 0+ if err := scoped.Transaction(func(tx *gorm.DB) error {+ for _, key := range order {+ group := groups[key]+ if group.status != "active" {+ continue+ }+ if err := tx.Exec(+ "INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",+ modelMigrationNewID(), group.modelName, group.providerID, group.instanceID,+ group.modelType, "active", group.extra, nowMs, now, nowMs, now,+ ).Error; err != nil {+ return err+ }+ inserted+++ }+ return nil+ }); err != nil {+ return err+ }++ if inserted > 0 {+ common.Info("Migrated tenant_model records from tenant_llm", zap.Int("count", inserted))+ }+ return nil+}++// buildTenantModelInstanceLookup maps provider_id + canonical api_key to the+// instance id, mirroring TenantModelStage._get_instance_lookup.+func buildTenantModelInstanceLookup(ctx context.Context, db *gorm.DB) (map[string]string, error) {+ var rows []struct {+ ID string `gorm:"column:id"`+ ProviderID string `gorm:"column:provider_id"`+ APIKey sql.NullString `gorm:"column:api_key"`+ }+ if err := db.Raw("SELECT id, provider_id, api_key FROM tenant_model_instance").Scan(&rows).Error; err != nil {+ return nil, err+ }+ lookup := make(map[string]string, len(rows))+ for _, row := range rows {+ lookup[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)] = row.ID+ }+ return lookup, nil+}++// buildTenantModelStatusCondition mirrors TenantModelStage._build_status_condition.+func buildTenantModelStatusCondition() string {+ emptyFactories := emptyLLMFactories()+ if len(emptyFactories) == 0 {+ return "tl.status = '0'"+ }+ sorted := append([]string(nil), emptyFactories...)+ sort.Strings(sorted)+ quoted := make([]string, 0, len(sorted))+ for _, name := range sorted {+ quoted = append(quoted, "'"+strings.ReplaceAll(name, "'", "''")+"'")+ }+ return fmt.Sprintf("(tl.status = '0' OR (tl.status = '1' AND tl.llm_factory IN (%s)))", strings.Join(quoted, ", "))+}++// emptyLLMFactories returns the factory names whose "llm" list is empty in+// conf/llm_factories.json, matching TenantModelStage._get_empty_llm_factories.+func emptyLLMFactories() []string {+ var empty []string+ for _, factory := range loadFactoryLLMInfos() {+ if len(factory.LLM) == 0 {+ empty = append(empty, factory.Name)+ }+ }+ return empty+}++// factoryLLMInfo is one entry of "factory_llm_infos" in conf/llm_factories.json.+type factoryLLMInfo struct {+ Name string `json:"name"`+ LLM []factoryLLM `json:"llm"`+}++// factoryLLM is one model declared by a factory. ModelType is either a string or+// an array of strings, so it is decoded lazily.+type factoryLLM struct {+ LLMName string `json:"llm_name"`+ ModelType json.RawMessage `json:"model_type"`+ IsTools bool `json:"is_tools"`+ MaxTokens json.RawMessage `json:"max_tokens"`+}++// loadFactoryLLMInfos reads and parses conf/llm_factories.json, returning nil when+// the file is missing or unreadable.+func loadFactoryLLMInfos() []factoryLLMInfo {+ path := locateLLMFactoriesFile()+ if path == "" {+ return nil+ }+ raw, err := os.ReadFile(path)+ if err != nil {+ common.Warn("Failed to read llm_factories.json for model migration", zap.String("path", path), zap.Error(err))+ return nil+ }+ var payload struct {+ FactoryLLMInfos []factoryLLMInfo `json:"factory_llm_infos"`+ }+ if err := json.Unmarshal(raw, &payload); err != nil {+ common.Warn("Failed to parse llm_factories.json for model migration", zap.String("path", path), zap.Error(err))+ return nil+ }+ return payload.FactoryLLMInfos+}++// locateLLMFactoriesFile finds conf/llm_factories.json by walking up from the+// working directory and falling back to the executable's directory.+func locateLLMFactoriesFile() string {+ candidates := []string{filepath.Join("conf", "llm_factories.json")}+ if wd, err := os.Getwd(); err == nil {+ dir := wd+ for depth := 0; depth < 5; depth++ {+ candidates = append(candidates, filepath.Join(dir, "conf", "llm_factories.json"))+ parent := filepath.Dir(dir)+ if parent == dir {+ break+ }+ dir = parent+ }+ }+ if exe, err := os.Executable(); err == nil {+ candidates = append(candidates, filepath.Join(filepath.Dir(exe), "conf", "llm_factories.json"))+ }+ for _, candidate := range candidates {+ if info, err := os.Stat(candidate); err == nil && !info.IsDir() {+ return candidate+ }+ }+ return ""+}++// Tenant model seeding ports TenantModelSeedingStage.++// tenantModelSeedRow tracks an existing tenant_model row so factory-declared+// model types can be merged into it without a second table rewrite.+type tenantModelSeedRow struct {+ id string+ modelType int+}++// seedTenantModels fills tenant_model with the models declared by+// conf/llm_factories.json for every provider/instance created by the previous+// steps. model_type is stored as the merged integer, so the declared factory+// bits are OR-ed into the matching (provider_id, instance_id, model_name) row+// and a fresh row is inserted when none exists.+func seedTenantModels(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ for _, table := range []string{"tenant_model_provider", "tenant_model_instance", "tenant_model"} {+ if !scoped.Migrator().HasTable(table) {+ return nil+ }+ }++ factories := loadFactoryLLMInfos()+ if len(factories) == 0 {+ return nil+ }++ existingExtras, err := loadTenantModelExtraMap(scoped)+ if err != nil {+ return err+ }+ existingStatuses, err := loadTenantModelStatusMap(scoped)+ if err != nil {+ return err+ }+ existing, err := loadTenantModelRowMap(scoped)+ if err != nil {+ return err+ }++ nowMs := time.Now().UnixMilli()+ now := time.Now()+ inserted, updated := 0, 0+ err = scoped.Transaction(func(tx *gorm.DB) error {+ for _, factory := range factories {+ if len(factory.LLM) == 0 {+ continue+ }+ providerName, extraInclude, extraExclude := factoryProviderFilter(factory.Name)+ providerIDs, err := loadProviderIDsByName(scoped, providerName)+ if err != nil {+ return err+ }+ for _, providerID := range providerIDs {+ instanceIDs, err := loadInstanceIDsByProvider(scoped, providerID, extraInclude, extraExclude)+ if err != nil {+ return err+ }+ for _, instanceID := range instanceIDs {+ for _, llm := range factory.LLM {+ if llm.LLMName == "" {+ continue+ }+ bits := factoryModelTypeBits(llm)+ if bits == 0 {+ continue+ }+ key := providerID + "\x00" + instanceID + "\x00" + llm.LLMName+ if row, ok := existing[key]; ok {+ if row.modelType|bits == row.modelType {+ continue+ }+ merged := row.modelType | bits+ if err := tx.Exec(+ "UPDATE tenant_model SET model_type = ?, update_time = ?, update_date = ? WHERE id = ?",+ merged, nowMs, now, row.id,+ ).Error; err != nil {+ return err+ }+ existing[key] = tenantModelSeedRow{id: row.id, modelType: merged}+ updated+++ continue+ }+ extra := existingExtras[llm.LLMName]+ if extra == "" {+ extra = buildFactoryModelExtra(llm)+ }+ status := existingStatuses[llm.LLMName]+ if status == "" {+ status = "active"+ }+ id := modelMigrationNewID()+ if err := tx.Exec(+ "INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",+ id, llm.LLMName, providerID, instanceID, bits, status, extra, nowMs, now, nowMs, now,+ ).Error; err != nil {+ return err+ }+ existing[key] = tenantModelSeedRow{id: id, modelType: bits}+ inserted+++ }+ }+ }+ }+ return nil+ })+ if err != nil {+ return err+ }++ if inserted > 0 || updated > 0 {+ common.Info("Seeded tenant_model from llm_factories.json", zap.Int("inserted", inserted), zap.Int("merged", updated))+ }+ return nil+}++// factoryProviderFilter mirrors the siliconflow special case in+// TenantModelSeedingStage.execute: the intl factory targets SILICONFLOW instances+// whose extra contains "region": "intl", while the global factory excludes them.+func factoryProviderFilter(factoryName string) (providerName, extraInclude, extraExclude string) {+ switch factoryName {+ case "siliconflow_intl":+ return "SILICONFLOW", `%"region": "intl"%`, ""+ case "SILICONFLOW":+ return "SILICONFLOW", "", `%"region": "intl"%`+ default:+ return factoryName, "", ""+ }+}++// factoryModelTypeBits merges every model_type declared for a factory model.+func factoryModelTypeBits(llm factoryLLM) int {+ bits := 0+ for _, name := range factoryLLMModelTypes(llm.ModelType) {+ bits |= modelTypeStringToBit[strings.ToLower(strings.TrimSpace(name))]+ }+ return bits+}++// factoryLLMModelTypes decodes a model_type that is either a string or an array+// of strings, defaulting to "chat" like the Python stage.+func factoryLLMModelTypes(raw json.RawMessage) []string {+ var single string+ if err := json.Unmarshal(raw, &single); err == nil && single != "" {+ return []string{single}+ }+ var list []string+ if err := json.Unmarshal(raw, &list); err == nil && len(list) > 0 {+ return list+ }+ return []string{"chat"}+}++// buildFactoryModelExtra builds the extra JSON stored for a newly seeded model,+// mirroring TenantModelSeedingStage._build_extra.+func buildFactoryModelExtra(llm factoryLLM) string {+ extra := make(map[string]any)+ if llm.IsTools {+ extra["is_tools"] = true+ }+ if len(llm.MaxTokens) > 0 && string(llm.MaxTokens) != "null" {+ var tokens any+ if err := json.Unmarshal(llm.MaxTokens, &tokens); err == nil {+ extra["max_tokens"] = tokens+ }+ }+ if len(extra) == 0 {+ return "{}"+ }+ encoded, err := json.Marshal(extra)+ if err != nil {+ return "{}"+ }+ return string(encoded)+}++func loadTenantModelRowMap(db *gorm.DB) (map[string]tenantModelSeedRow, error) {+ var rows []struct {+ ID string `gorm:"column:id"`+ ProviderID string `gorm:"column:provider_id"`+ InstanceID string `gorm:"column:instance_id"`+ ModelName string `gorm:"column:model_name"`+ ModelType sql.NullString `gorm:"column:model_type"`+ }+ if err := db.Raw("SELECT id, provider_id, instance_id, model_name, model_type FROM tenant_model").Scan(&rows).Error; err != nil {+ return nil, err+ }+ result := make(map[string]tenantModelSeedRow, len(rows))+ for _, row := range rows {+ result[row.ProviderID+"\x00"+row.InstanceID+"\x00"+row.ModelName] = tenantModelSeedRow{+ id: row.ID,+ modelType: mergeTenantModelTypeBits(row.ModelType.String),+ }+ }+ return result, nil+}++func loadTenantModelExtraMap(db *gorm.DB) (map[string]string, error) {+ var rows []struct {+ ModelName string `gorm:"column:model_name"`+ Extra string `gorm:"column:extra"`+ }+ if err := db.Raw("SELECT model_name, extra FROM tenant_model WHERE extra IS NOT NULL AND extra != '{}'").Scan(&rows).Error; err != nil {+ return nil, err+ }+ result := make(map[string]string, len(rows))+ for _, row := range rows {+ if row.ModelName != "" && row.Extra != "" && row.Extra != "{}" {+ result[row.ModelName] = row.Extra+ }+ }+ return result, nil+}++func loadTenantModelStatusMap(db *gorm.DB) (map[string]string, error) {+ var rows []struct {+ ModelName string `gorm:"column:model_name"`+ Status string `gorm:"column:status"`+ }+ if err := db.Raw("SELECT model_name, status FROM tenant_model WHERE status IS NOT NULL AND status != 'unsupported'").Scan(&rows).Error; err != nil {+ return nil, err+ }+ result := make(map[string]string, len(rows))+ for _, row := range rows {+ if row.ModelName != "" && row.Status != "" {+ result[row.ModelName] = row.Status+ }+ }+ return result, nil+}++func loadProviderIDsByName(db *gorm.DB, providerName string) ([]string, error) {+ var rows []struct {+ ID string `gorm:"column:id"`+ }+ if err := db.Raw("SELECT id FROM tenant_model_provider WHERE provider_name = ?", providerName).Scan(&rows).Error; err != nil {+ return nil, err+ }+ ids := make([]string, 0, len(rows))+ for _, row := range rows {+ ids = append(ids, row.ID)+ }+ return ids, nil+}++func loadInstanceIDsByProvider(db *gorm.DB, providerID, extraInclude, extraExclude string) ([]string, error) {+ var (+ rows []struct {+ ID string `gorm:"column:id"`+ }+ query string+ args []any+ )+ switch {+ case extraInclude != "" && extraExclude != "":+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ? AND extra LIKE ? AND extra NOT LIKE ?"+ args = []any{providerID, extraInclude, extraExclude}+ case extraInclude != "":+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ? AND extra LIKE ?"+ args = []any{providerID, extraInclude}+ case extraExclude != "":+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ? AND extra NOT LIKE ?"+ args = []any{providerID, extraExclude}+ default:+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ?"+ args = []any{providerID}+ }+ if err := db.Raw(query, args...).Scan(&rows).Error; err != nil {+ return nil, err+ }+ ids := make([]string, 0, len(rows))+ for _, row := range rows {+ ids = append(ids, row.ID)+ }+ return ids, nil+}++// Merge tenant model types ports ModelTypeMergeStage.++// tenantModelMergeSource is one tenant_model row read by mergeTenantModelTypes.+// model_type is read as text because the stage only runs on tables that have not+// been converted to the integer representation yet.+type tenantModelMergeSource struct {+ ID string `gorm:"column:id"`+ ModelName sql.NullString `gorm:"column:model_name"`+ ProviderID string `gorm:"column:provider_id"`+ InstanceID string `gorm:"column:instance_id"`+ ModelType sql.NullString `gorm:"column:model_type"`+ Status sql.NullString `gorm:"column:status"`+ Extra sql.NullString `gorm:"column:extra"`+ CreateTime sql.NullInt64 `gorm:"column:create_time"`+ CreateDate sql.NullTime `gorm:"column:create_date"`+ UpdateTime sql.NullInt64 `gorm:"column:update_time"`+ UpdateDate sql.NullTime `gorm:"column:update_date"`+}++// mergedTenantModelRecord is one merged tenant_model row written back by+// mergeTenantModelTypes.+type mergedTenantModelRecord struct {+ modelName sql.NullString+ providerID string+ instanceID string+ modelType int+ status string+ extra sql.NullString+ createTime sql.NullInt64+ createDate sql.NullTime+ updateTime sql.NullInt64+ updateDate sql.NullTime+}++// mergeTenantModelTypes ports ModelTypeMergeStage: it collapses the tenant_model+// rows sharing (provider_id, instance_id, model_name) into a single row whose+// model_type is the integer OR of the contributing model bits, dropping the bits+// that only ever appeared on "unsupported" rows. Groups whose merged status is+// not "active" are dropped.+//+// The stage only rewrites a table whose model_type is still stored as text. When+// the column is already an integer type the merge has been applied and the stage+// is a no-op, mirroring ModelTypeMergeStage.check.+func mergeTenantModelTypes(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ if !scoped.Migrator().HasTable("tenant_model") {+ return nil+ }++ isInteger, err := tenantModelModelTypeIsInteger(scoped)+ if err != nil {+ return err+ }+ if isInteger {+ common.Info("tenant_model.model_type is already an integer, merge stage skipped")+ return nil+ }++ var rows []tenantModelMergeSource+ if err := scoped.Raw(+ "SELECT id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date FROM tenant_model ORDER BY id",+ ).Scan(&rows).Error; err != nil {+ return err+ }+ if len(rows) == 0 {+ return nil+ }++ type mergeGroup struct {+ first tenantModelMergeSource+ supported int+ unsupported int+ status string+ }++ groups := make(map[string]*mergeGroup, len(rows))+ order := make([]string, 0, len(rows))+ for _, row := range rows {+ key := row.ProviderID + "\x00" + row.InstanceID + "\x00" + row.ModelName.String+ group, ok := groups[key]+ if !ok {+ group = &mergeGroup{first: row}+ groups[key] = group+ order = append(order, key)+ }+ bits := mergeTenantModelTypeBits(row.ModelType.String)+ if row.Status.String == "unsupported" {+ group.unsupported |= bits+ continue+ }+ group.supported |= bits+ if group.status == "" {+ group.status = row.Status.String+ }+ }++ nowMs := time.Now().UnixMilli()+ now := time.Now()+ merged := make([]mergedTenantModelRecord, 0, len(order))+ for _, key := range order {+ group := groups[key]+ status := group.status+ if status == "" {+ status = "active"+ }+ if status != "active" {+ continue+ }+ record := mergedTenantModelRecord{+ modelName: group.first.ModelName,+ providerID: group.first.ProviderID,+ instanceID: group.first.InstanceID,+ modelType: group.supported &^ group.unsupported,+ status: status,+ extra: group.first.Extra,+ createTime: group.first.CreateTime,+ createDate: group.first.CreateDate,+ updateTime: group.first.UpdateTime,+ updateDate: group.first.UpdateDate,+ }+ if !record.createTime.Valid {+ record.createTime = sql.NullInt64{Int64: nowMs, Valid: true}+ }+ if !record.updateTime.Valid {+ record.updateTime = sql.NullInt64{Int64: nowMs, Valid: true}+ }+ if !record.createDate.Valid {+ record.createDate = sql.NullTime{Time: now, Valid: true}+ }+ if !record.updateDate.Valid {+ record.updateDate = sql.NullTime{Time: now, Valid: true}+ }+ merged = append(merged, record)+ }++ // The whole table is replaced even when every group was dropped, mirroring the+ // Python stage that always swaps in the merged table. Leaving the source rows+ // behind would keep text model_type values in place for the integer typed reads+ // that follow.+ if err := scoped.Transaction(func(tx *gorm.DB) error {+ if err := tx.Exec("DELETE FROM tenant_model").Error; err != nil {+ return err+ }+ for _, record := range merged {+ if err := tx.Exec(+ "INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",+ modelMigrationNewID(), record.modelName, record.providerID, record.instanceID,+ record.modelType, record.status, record.extra,+ record.createTime, record.createDate, record.updateTime, record.updateDate,+ ).Error; err != nil {+ return err+ }+ }+ return nil+ }); err != nil {+ return err+ }++ common.Info("Merged tenant_model model types", zap.Int("rows", len(merged)))+ return nil+}++// mergeTenantModelTypeBits maps a stored model_type to its bit. Legacy rows hold+// the LLMType name, while rows written by the seeding stage already hold the+// merged integer, so numeric text is accepted as well.+func mergeTenantModelTypeBits(raw string) int {+ trimmed := strings.TrimSpace(raw)+ if bit, ok := modelTypeStringToBit[strings.ToLower(trimmed)]; ok {+ return bit+ }+ if value, err := strconv.Atoi(trimmed); err == nil {+ return value+ }+ return 0+}++// tenantModelModelTypeIsInteger reports whether tenant_model.model_type is stored+// as an integer column, in which case the merge has already been applied.+func tenantModelModelTypeIsInteger(db *gorm.DB) (bool, error) {+ columnTypes, err := db.Migrator().ColumnTypes("tenant_model")+ if err != nil {+ return false, err+ }+ for _, columnType := range columnTypes {+ if columnType.Name() != "model_type" {+ continue+ }+ return strings.Contains(strings.ToLower(columnType.DatabaseTypeName()), "int"), nil+ }+ return false, nil+}++// Model id normalization ports ModelIdConfigStage.++var modelIDStringColumns = map[string][]string{+ "tenant": {"llm_id", "embd_id", "asr_id", "img2txt_id", "rerank_id", "tts_id", "ocr_id"},+ "knowledgebase": {"embd_id"},+ "dialog": {"llm_id", "rerank_id"},+ "memory": {"embd_id", "llm_id"},+}++var modelIDJSONColumns = map[string][]string{+ "knowledgebase": {"parser_config"},+ "document": {"parser_config"},+ "search": {"search_config"},+ "user_canvas": {"dsl"},+ "canvas_template": {"dsl"},+ "user_canvas_version": {"dsl"},+ "api_4_conversation": {"dsl"},+ "pipeline_operation_log": {"dsl"},+ "connector": {"config"},+ "evaluation_runs": {"config_snapshot"},+}++var modelIDFieldSet = map[string]bool{+ "llm_id": true,+ "embd_id": true,+ "embedding_model": true,+ "rerank_id": true,+ "asr_id": true,+ "img2txt_id": true,+ "tts_id": true,+ "ocr_id": true,+}++var searchConfigModelIDFieldSet = map[string]bool{"chat_id": true}++// normalizeModelID rewrites "<model>@<provider>" into "<model>@default@<provider>".+// Values that are not exactly two non-empty parts are returned unchanged.+func normalizeModelID(value string) (string, bool) {+ parts := strings.Split(value, "@")+ if len(parts) != 2 {+ return value, false+ }+ modelName, providerName := parts[0], parts[1]+ if modelName == "" || providerName == "" {+ return value, false+ }+ return modelName + "@default@" + providerName, true+}++func containsString(values []string, target string) bool {+ for _, value := range values {+ if value == target {+ return true+ }+ }+ return false+}++// normalizeModelIDConfig recursively normalizes model id fields inside a decoded+// JSON document, mirroring ModelIdConfigStage.normalize_config.+func normalizeModelIDConfig(value any, path []string) (any, bool) {+ switch typed := value.(type) {+ case map[string]any:+ changed := false+ normalized := make(map[string]any, len(typed))+ for key, item := range typed {+ keyPath := append(append([]string{}, path...), key)+ shouldNormalize := modelIDFieldSet[key] || (searchConfigModelIDFieldSet[key] && containsString(path, "search_config"))+ if shouldNormalize {+ if str, ok := item.(string); ok {+ normalizedValue, itemChanged := normalizeModelID(str)+ normalized[key] = normalizedValue+ changed = changed || itemChanged+ continue+ }+ normalized[key] = item+ continue+ }+ normalizedItem, itemChanged := normalizeModelIDConfig(item, keyPath)+ normalized[key] = normalizedItem+ changed = changed || itemChanged+ }+ return normalized, changed+ case []any:+ changed := false+ normalized := make([]any, len(typed))+ for index, item := range typed {+ indexPath := append(append([]string{}, path...), strconv.Itoa(index))+ normalizedItem, itemChanged := normalizeModelIDConfig(item, indexPath)+ normalized[index] = normalizedItem+ changed = changed || itemChanged+ }+ return normalized, changed+ default:+ return value, false+ }+}++// normalizeStoredModelIDs rewrites stored model ids across the tables listed by+// ModelIdConfigStage.+func normalizeStoredModelIDs(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ for _, table := range sortedTableNames(modelIDStringColumns) {+ if !scoped.Migrator().HasTable(table) {+ continue+ }+ for _, column := range modelIDStringColumns[table] {+ exists, err := columnExists(ctx, scoped, table, column)+ if err != nil {+ return err+ }+ if !exists {+ continue+ }+ if err := normalizeModelIDStringColumn(ctx, scoped, table, column); err != nil {+ return err+ }+ }+ }+ for _, table := range sortedTableNames(modelIDJSONColumns) {+ if !scoped.Migrator().HasTable(table) {+ continue+ }+ for _, column := range modelIDJSONColumns[table] {+ exists, err := columnExists(ctx, scoped, table, column)+ if err != nil {+ return err+ }+ if !exists {+ continue+ }+ if err := normalizeModelIDJSONColumn(ctx, scoped, table, column); err != nil {+ return err+ }+ }+ }+ return nil+}++// sortedTableNames returns a stable ordering so the migration is deterministic.+func sortedTableNames(columns map[string][]string) []string {+ names := make([]string, 0, len(columns))+ for name := range columns {+ names = append(names, name)+ }+ sort.Strings(names)+ return names+}++func normalizeModelIDStringColumn(ctx context.Context, db *gorm.DB, table, column string) error {+ var rows []struct {+ ID string `gorm:"column:id"`+ Value sql.NullString `gorm:"column:value"`+ }+ query := fmt.Sprintf("SELECT id AS id, `%s` AS value FROM `%s` WHERE `%s` IS NOT NULL AND `%s` LIKE '%%@%%'", column, table, column, column)+ if err := db.Raw(query).Scan(&rows).Error; err != nil {+ return err+ }++ updates := make(map[string]string)+ for _, row := range rows {+ if !row.Value.Valid || row.Value.String == "" {+ continue+ }+ if normalized, changed := normalizeModelID(row.Value.String); changed {+ updates[row.ID] = normalized+ }+ }+ return applyModelIDUpdates(ctx, db, table, column, updates)+}++func normalizeModelIDJSONColumn(ctx context.Context, db *gorm.DB, table, column string) error {+ var rows []struct {+ ID string `gorm:"column:id"`+ Value sql.NullString `gorm:"column:value"`+ }+ query := fmt.Sprintf("SELECT id AS id, `%s` AS value FROM `%s` WHERE `%s` IS NOT NULL AND `%s` != ''", column, table, column, column)+ if err := db.Raw(query).Scan(&rows).Error; err != nil {+ return err+ }++ updates := make(map[string]string)+ for _, row := range rows {+ var decoded any+ if err := json.Unmarshal([]byte(row.Value.String), &decoded); err != nil {+ continue+ }+ normalized, changed := normalizeModelIDConfig(decoded, []string{column})+ if !changed {+ continue+ }+ encoded, err := json.Marshal(normalized)+ if err != nil {+ continue+ }+ updates[row.ID] = string(encoded)+ }+ return applyModelIDUpdates(ctx, db, table, column, updates)+}++func applyModelIDUpdates(ctx context.Context, db *gorm.DB, table, column string, updates map[string]string) error {+ if len(updates) == 0 {+ return nil+ }+ statement := fmt.Sprintf("UPDATE `%s` SET `%s` = ? WHERE id = ?", table, column)+ return db.Transaction(func(tx *gorm.DB) error {+ for id, value := range updates {+ if err := tx.Exec(statement, value, id).Error; err != nil {+ return err+ }+ }+ return nil+ })+}++// Tenant model id population ports TenantModelIdMigrationStage.++type tenantModelIDField struct {+ tenantIDColumn string+ legacyColumn string+ modelType string+}++var tenantModelIDFields = map[string][]tenantModelIDField{+ "tenant": {+ {"tenant_llm_id", "llm_id", "chat"},+ {"tenant_embd_id", "embd_id", "embedding"},+ {"tenant_asr_id", "asr_id", "speech2text"},+ {"tenant_img2txt_id", "img2txt_id", "image2text"},+ {"tenant_rerank_id", "rerank_id", "rerank"},+ {"tenant_tts_id", "tts_id", "tts"},+ {"tenant_ocr_id", "ocr_id", "ocr"},+ },+ "knowledgebase": {+ {"tenant_embd_id", "embd_id", "embedding"},+ },+ "dialog": {+ {"tenant_llm_id", "llm_id", "chat"},+ {"tenant_rerank_id", "rerank_id", "rerank"},+ },+ "memory": {+ {"tenant_embd_id", "embd_id", "embedding"},+ {"tenant_llm_id", "llm_id", "chat"},+ },+}++type tenantModelLookupKey struct {+ tenantID string+ modelName string+ provider string+ modelType string+}++// populateTenantModelIDColumns ports TenantModelIdMigrationStage: it ensures the+// tenant_*_id columns exist and fills them from the legacy *_id columns by+// resolving the referenced tenant_model row.+func populateTenantModelIDColumns(ctx context.Context, db *gorm.DB) error {+ scoped := db.WithContext(ctx)+ for _, table := range []string{"tenant_model", "tenant_model_provider", "tenant_model_instance"} {+ if !scoped.Migrator().HasTable(table) {+ return nil+ }+ }++ for _, table := range sortedTenantModelIDTables() {+ if !scoped.Migrator().HasTable(table) {+ continue+ }+ for _, field := range tenantModelIDFields[table] {+ exists, err := columnExists(ctx, scoped, table, field.tenantIDColumn)+ if err != nil {+ return err+ }+ if !exists {+ if err := ensureTenantModelIDColumn(ctx, scoped, table, field.tenantIDColumn); err != nil {+ return err+ }+ }+ }+ }++ lookup, err := buildTenantModelLookup(ctx, scoped)+ if err != nil {+ return err+ }+ if len(lookup) == 0 {+ return nil+ }++ total := 0+ for _, table := range sortedTenantModelIDTables() {+ if !scoped.Migrator().HasTable(table) {+ continue+ }+ for _, field := range tenantModelIDFields[table] {+ tenantColumnExists, err := columnExists(ctx, scoped, table, field.tenantIDColumn)+ if err != nil {+ return err+ }+ legacyColumnExists, err := columnExists(ctx, scoped, table, field.legacyColumn)+ if err != nil {+ return err+ }+ if !tenantColumnExists || !legacyColumnExists {+ continue+ }+ updated, err := populateTenantModelIDColumn(ctx, scoped, table, field, lookup)+ if err != nil {+ return err+ }+ total += updated+ }+ }++ if total > 0 {+ common.Info("Populated tenant model id columns", zap.Int("rows", total))+ }+ return nil+}++func sortedTenantModelIDTables() []string {+ names := make([]string, 0, len(tenantModelIDFields))+ for name := range tenantModelIDFields {+ names = append(names, name)+ }+ sort.Strings(names)+ return names+}++func ensureTenantModelIDColumn(ctx context.Context, db *gorm.DB, table, column string) error {+ common.Info("Adding tenant model id column", zap.String("table", table), zap.String("column", column))+ statement := fmt.Sprintf("ALTER TABLE `%s` ADD COLUMN `%s` VARCHAR(32) NULL", table, column)+ return db.Exec(statement).Error+}++// buildTenantModelLookup maps (tenant_id, model_name, provider, model_type) to+// tenant_model.id for every active model, expanding the model_type bitmask.+func buildTenantModelLookup(ctx context.Context, db *gorm.DB) (map[tenantModelLookupKey]string, error) {+ var rows []struct {+ ID string `gorm:"column:id"`+ ModelName string `gorm:"column:model_name"`+ ModelType int `gorm:"column:model_type"`+ TenantID string `gorm:"column:tenant_id"`+ ProviderName string `gorm:"column:provider_name"`+ }+ if err := db.Raw(`+ SELECT tm.id AS id, tm.model_name AS model_name, tm.model_type AS model_type,+ tmp.tenant_id AS tenant_id, tmp.provider_name AS provider_name+ FROM tenant_model tm+ INNER JOIN tenant_model_provider tmp ON tm.provider_id = tmp.id+ WHERE tm.status = 'active'`).Scan(&rows).Error; err != nil {+ return nil, err+ }++ lookup := make(map[tenantModelLookupKey]string)+ for _, row := range rows {+ for typeName, typeBit := range modelTypeStringToBit {+ if row.ModelType&typeBit != 0 {+ lookup[tenantModelLookupKey{row.TenantID, row.ModelName, row.ProviderName, typeName}] = row.ID+ }+ }+ }+ return lookup, nil+}++func populateTenantModelIDColumn(ctx context.Context, db *gorm.DB, table string, field tenantModelIDField, lookup map[tenantModelLookupKey]string) (int, error) {+ tenantIDExpr := "tenant_id"+ if table == "tenant" {+ // The tenant table's primary key is the tenant id itself.+ tenantIDExpr = "id"+ }+ query := fmt.Sprintf(+ "SELECT id AS id, %s AS tenant_id, `%s` AS model_name FROM `%s` WHERE (`%s` IS NULL OR `%s` = '' OR LENGTH(`%s`) <> 32) AND `%s` IS NOT NULL AND `%s` != ''",+ tenantIDExpr, field.legacyColumn, table,+ field.tenantIDColumn, field.tenantIDColumn, field.tenantIDColumn,+ field.legacyColumn, field.legacyColumn,+ )++ var rows []struct {+ ID string `gorm:"column:id"`+ TenantID string `gorm:"column:tenant_id"`+ ModelName string `gorm:"column:model_name"`+ }+ if err := db.Raw(query).Scan(&rows).Error; err != nil {+ return 0, err+ }+ if len(rows) == 0 {+ return 0, nil+ }++ updateStatement := fmt.Sprintf("UPDATE `%s` SET `%s` = ? WHERE id = ?", table, field.tenantIDColumn)+ updated := 0+ if err := db.Transaction(func(tx *gorm.DB) error {+ for _, row := range rows {+ tenantID := row.TenantID+ if table == "tenant" {+ tenantID = row.ID+ }+ resolved := resolveTenantModelID(lookup, tenantID, row.ModelName, field.modelType)+ if err := tx.Exec(updateStatement, resolved, row.ID).Error; err != nil {+ return err+ }+ updated+++ }+ return nil+ }); err != nil {+ return 0, err+ }+ return updated, nil+}++// resolveTenantModelID ports TenantModelIdMigrationStage._resolve_model_id.+func resolveTenantModelID(lookup map[tenantModelLookupKey]string, tenantID, modelName, modelType string) string {+ pureName, _, providerName := splitModelName(modelName)+ if pureName == "" || providerName == "" {+ return ""+ }+ if id, ok := lookup[tenantModelLookupKey{tenantID, pureName, providerName, modelType}]; ok {+ return id+ }+ for key, id := range lookup {+ if key.tenantID == tenantID && key.modelName == pureName && key.modelType == modelType {+ return id+ }+ }+ return ""+}++// splitModelName ports TenantModelIdMigrationStage._split_model_name:+// "{model}@{factory}" or "{model}@{instance}@{factory}".+func splitModelName(modelName string) (string, string, string) {+ if modelName == "" {+ return "", "", ""+ }+ parts := strings.Split(modelName, "@")+ switch len(parts) {+ case 1:+ return parts[0], modelMigrationInstanceName, ""+ case 2:+ return parts[0], modelMigrationInstanceName, parts[1]+ default:+ return parts[0], parts[1], parts[2]+ }+}diff --git a/internal/development.md b/internal/development.mdindex 4fcce80c0a..17321c84af 100644--- a/internal/development.md+++ b/internal/development.md@@ -211,8 +211,8 @@ Note: admin server must be started first; otherwise, api server will encounter e``````bash-# Start admin server and migrate database-./bin/ragflow_server --admin --migrate+# Run database migrations (standalone action; does not start a server)+./bin/ragflow_server --migrate``````bashdiff --git a/internal/entity/chat.go b/internal/entity/chat.goindex ff2bd278c0..cf544057f6 100644--- a/internal/entity/chat.go+++ b/internal/entity/chat.go@@ -42,7 +42,7 @@ type Chat struct {VectorSimilarityWeight float64 `gorm:"column:vector_similarity_weight" json:"vector_similarity_weight"`TopN int64 `gorm:"column:top_n" json:"top_n"`RerankCandidatesCount int64 `gorm:"column:rerank_candidates_count" json:"rerank_candidates_count"`- TopK int64 `gorm:"column:top_k" json:"top_k"`+ TopK int64 `gorm:"column:top_k;type:int" json:"top_k"`DoRefer string `gorm:"column:do_refer;size:1;not null" json:"do_refer"`RerankID string `gorm:"column:rerank_id;size:128;not null;default:''" json:"rerank_id"`TenantRerankID *string `gorm:"column:tenant_rerank_id;size:32;index" json:"tenant_rerank_id,omitempty"`diff --git a/internal/entity/dataset.go b/internal/entity/dataset.goindex 09d9a3d7f6..c8cc953897 100644--- a/internal/entity/dataset.go+++ b/internal/entity/dataset.go@@ -119,9 +119,9 @@ type Knowledgebase struct {GraphragTaskID *string `gorm:"column:graphrag_task_id;size:32;index" json:"graphrag_task_id,omitempty"`GraphragTaskFinishAt *time.Time `gorm:"column:graphrag_task_finish_at" json:"graphrag_task_finish_at,omitempty"`RaptorTaskID *string `gorm:"column:raptor_task_id;size:32;index" json:"raptor_task_id,omitempty"`- RaptorTaskFinishAt *time.Time `gorm:"column:raptor_task_finish_at" json:"raptor_task_finish_at,omitempty"`+ RaptorTaskFinishAt *time.Time `gorm:"column:raptor_task_finish_at;type:datetime" json:"raptor_task_finish_at,omitempty"`MindmapTaskID *string `gorm:"column:mindmap_task_id;size:32;index" json:"mindmap_task_id,omitempty"`- MindmapTaskFinishAt *time.Time `gorm:"column:mindmap_task_finish_at" json:"mindmap_task_finish_at,omitempty"`+ MindmapTaskFinishAt *time.Time `gorm:"column:mindmap_task_finish_at;type:datetime" json:"mindmap_task_finish_at,omitempty"`WikiTaskID *string `gorm:"column:wiki_task_id;size:32;index" json:"wiki_task_id,omitempty"`WikiTaskFinishAt *time.Time `gorm:"column:wiki_task_finish_at" json:"wiki_task_finish_at,omitempty"`SkillTaskID *string `gorm:"column:skill_task_id;size:32;index" json:"skill_task_id,omitempty"`diff --git a/internal/entity/system.go b/internal/entity/system.goindex 49541ec3ef..94e720cb65 100644--- a/internal/entity/system.go+++ b/internal/entity/system.go@@ -21,7 +21,7 @@ type SystemSettings struct {Name string `gorm:"column:name;primaryKey;size:128" json:"name"`Source string `gorm:"column:source;size:32;not null" json:"source"`DataType string `gorm:"column:data_type;size:32;not null" json:"data_type"`- Value string `gorm:"column:value;type:longtext;not null" json:"value"`+ Value string `gorm:"column:value;type:text;not null" json:"value"`BaseModel}diff --git a/internal/entity/tenant_llm.go b/internal/entity/tenant_llm.goindex 9b3829c6b9..1c8c7f3cf2 100644--- a/internal/entity/tenant_llm.go+++ b/internal/entity/tenant_llm.go@@ -24,7 +24,7 @@ type TenantLLM struct {LLMFactory string `gorm:"column:llm_factory;size:128;not null" json:"llm_factory"`ModelType *string `gorm:"column:model_type;size:128;index" json:"model_type,omitempty"`LLMName *string `gorm:"column:llm_name;size:128;index;default:''" json:"llm_name,omitempty"`- APIKey *string `gorm:"column:api_key;size:8192" json:"api_key,omitempty"`+ APIKey *string `gorm:"column:api_key;type:text" json:"api_key,omitempty"`APIBase *string `gorm:"column:api_base;size:255" json:"api_base,omitempty"`MaxTokens int64 `gorm:"column:max_tokens;default:8192" json:"max_tokens"`UsedTokens int64 `gorm:"column:used_tokens;default:0" json:"used_tokens"`diff --git a/internal/entity/user.go b/internal/entity/user.goindex 7d70f9197b..4746ff0dac 100644--- a/internal/entity/user.go+++ b/internal/entity/user.go@@ -24,7 +24,7 @@ type User struct {AccessToken *string `gorm:"column:access_token;size:255;index" json:"access_token,omitempty"`Nickname string `gorm:"column:nickname;size:100;not null;index" json:"nickname"`Password *string `gorm:"column:password;size:255;index" json:"-"`- Email string `gorm:"column:email;size:255;not null;unique" json:"email"`+ Email string `gorm:"column:email;size:255;not null" json:"email"`Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"`Language *string `gorm:"column:language;size:32;index;default:'English'" json:"language,omitempty"`ColorSchema *string `gorm:"column:color_schema;size:32;index;default:'Bright'" json:"color_schema,omitempty"`