Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix(go): migration cleanup and add DB version tracking (#19598)

Lynne8fcc7b15390
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 1bf95c6a58..bbfa2479a3 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -63,7 +63,7 @@ import (
"ragflow/internal/agent/component"
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"
+ infnative "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"
"ragflow/internal/engine"
"ragflow/internal/engine/redis"
et "ragflow/internal/engine/types"
@@ -251,6 +251,9 @@ func parseArgs() (*serverArgs, error) {
return nil, fmt.Errorf("unknown parameter: %s", arg)
}
}
+ if args.migrateDB && args.mode != nil {
+ return nil, errors.New("--migrate is a standalone action and cannot be combined with --api/--admin/--ingestor/--syncer")
+ }
return args, nil
}
@@ -262,13 +265,16 @@ func parseArgs() (*serverArgs, error) {
func printHelp(args *serverArgs) {
switch {
case args.mode == nil:
- fmt.Fprintf(os.Stderr, "Usage: %s --api|--admin|--ingestor|--syncer [OPTIONS]\n\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, "Usage: %s --api|--admin|--ingestor|--syncer [OPTIONS]\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, " %s --migrate [OPTIONS]\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "RAGFlow Server - Open-source RAG engine based on deep document understanding\n\n")
fmt.Fprintf(os.Stderr, "Mode selection (default: --api):\n")
fmt.Fprintf(os.Stderr, " --api \tRun as API server\n")
fmt.Fprintf(os.Stderr, " --admin \tRun as admin server\n")
fmt.Fprintf(os.Stderr, " --ingestor \tRun as ingestion worker\n")
fmt.Fprintf(os.Stderr, " --syncer \tRun as file sync service\n\n")
+ fmt.Fprintf(os.Stderr, "Standalone action (mutually exclusive with a mode):\n")
+ fmt.Fprintf(os.Stderr, " --migrate \tRun database migrations and exit\n\n")
fmt.Fprintf(os.Stderr, "Common options:\n")
fmt.Fprintf(os.Stderr, " --config string\tPath to configuration file\n")
fmt.Fprintf(os.Stderr, " -v, --version \tPrint version information and exit\n")
@@ -330,7 +336,7 @@ func main() {
os.Exit(1)
}
- if arguments.helpFlag || arguments.mode == nil {
+ if arguments.helpFlag || (arguments.mode == nil && !arguments.migrateDB) {
printHelp(arguments)
os.Exit(1)
}
@@ -340,6 +346,17 @@ func main() {
os.Exit(1)
}
+ // --migrate is a standalone one-shot action: run the database migrations and
+ // exit without selecting a server mode. It deliberately skips the
+ // mode-specific startup (native DeepDoc, doc engine, Redis, storage, message
+ // queue) so it can run independently, before any server boots.
+ if arguments.migrateDB {
+ if err = runMigrate(ctx, arguments); err != nil {
+ common.Fatal("Failed to run database migration", zap.Error(err))
+ }
+ return
+ }
+
// Initialize local variables (runtime variables from Redis)
err = server.InitLocalVariables()
if err != nil {
@@ -457,11 +474,21 @@ func main() {
common.Info(fmt.Sprintf("Starting %s server: %s, mode: %s", *arguments.mode, serverName, globalConfig.GetMode()))
server.PrintAll()
- // Initialize database
- if err = dao.InitDB(ctx, arguments.migrateDB); err != nil {
+ // Initialize database. Migrations are not run here: --migrate is a
+ // standalone action, so a server-mode process only ensures the runtime
+ // tables it needs (see InitDB).
+ if err = dao.InitDB(ctx, false); err != nil {
common.Fatal("Failed to initialize database", zap.Error(err))
}
+ // Refuse to start a server against a database that a newer version already
+ // migrated: rolling the code back cannot roll the schema back. The
+ // standalone --migrate action is exempt because advancing the database is
+ // its job.
+ if err = checkDatabaseVersion(ctx); err != nil {
+ common.Fatal("Refusing to start: database was migrated by a newer version", zap.Error(err))
+ }
+
// Initialize doc engine
if err = engine.InitDocEngine(ctx); err != nil {
common.Fatal("Failed to initialize doc engine", zap.Error(err))
@@ -526,6 +553,104 @@ func main() {
}
}
+// checkDatabaseVersion refuses to run a server when the running code is older
+// than the version recorded in the system_settings migration marker. Migrating
+// the database forward is a one-way operation, so an older binary would read and
+// write a schema it does not understand.
+//
+// A missing marker, or a version on either side that cannot be parsed, never
+// blocks startup: without a usable comparison there is no evidence that the
+// database is ahead of the code.
+func checkDatabaseVersion(ctx context.Context) error {
+ databaseVersion, err := dao.GetDatabaseMigrationVersion(ctx, dao.DB)
+ if err != nil {
+ return fmt.Errorf("read database version marker: %w", err)
+ }
+ if databaseVersion == "" {
+ return nil
+ }
+
+ codeVersion := common.GetRAGFlowVersion()
+ older, comparable := common.IsOlderReleaseThan(codeVersion, databaseVersion)
+ if !comparable {
+ common.Warn("Cannot compare code version with database version, skipping the downgrade check",
+ zap.String("code_version", codeVersion),
+ zap.String("database_version", databaseVersion))
+ return nil
+ }
+ if older {
+ return fmt.Errorf("code version %s is older than database version %s: upgrade this deployment to %s or newer before starting",
+ codeVersion, databaseVersion, databaseVersion)
+ }
+
+ common.Info("Database version check passed",
+ zap.String("code_version", codeVersion),
+ zap.String("database_version", databaseVersion))
+ return nil
+}
+
+// runMigrate runs the database schema and data migrations and returns. It is
+// the whole of the standalone --migrate action: load the configuration, run
+// dao.InitDB with migrations enabled, then exit. It deliberately does not call
+// registerNativeDeepDoc or initialize the doc engine, Redis, storage or the
+// message queue, so it can run on its own, before any server mode boots (see
+// docker/entrypoint-go.sh and docker/launch_backend_service.sh).
+func runMigrate(ctx context.Context, args *serverArgs) error {
+ const serverName = "migrate"
+
+ if err := server.InitLocalVariables(); err != nil {
+ return fmt.Errorf("initialize local variables: %w", err)
+ }
+
+ logLevel := "info"
+ if args.debugLog {
+ logLevel = "debug"
+ }
+ if err := common.InitLogger(logLevel, common.FileOutput{Filename: serverName + ".log", Path: "logs"}, serverName); err != nil {
+ return fmt.Errorf("initialize logger: %w", err)
+ }
+
+ var configPath string
+ if args.configPath != nil {
+ configPath = *args.configPath
+ }
+ if err := server.Init(configPath); err != nil {
+ return fmt.Errorf("initialize configuration: %w", err)
+ }
+
+ globalConfig := server.GetConfig()
+ server.SetServerName(serverName)
+
+ logConfig := globalConfig.GetLogConfig()
+ if logConfig.Level != "" {
+ logLevel = logConfig.Level
+ }
+ if args.debugLog {
+ logLevel = "debug"
+ }
+ globalConfig.SetLogLevel(logLevel)
+
+ common.SyncLog()
+ if err := common.InitLogger(logLevel, common.FileOutput{
+ Filename: serverName + ".log",
+ Path: logConfig.Path,
+ MaxSize: logConfig.MaxSize,
+ MaxBackups: logConfig.MaxBackups,
+ MaxAge: logConfig.MaxAge,
+ Compress: logConfig.Compress,
+ }, serverName); err != nil {
+ common.Error("Failed to reinitialize logger with configured level", err)
+ }
+
+ common.Info("Running database migrations")
+ if err := dao.InitDB(ctx, true); err != nil {
+ return fmt.Errorf("initialize database: %w", err)
+ }
+ common.Info("Database migrations completed")
+
+ return nil
+}
+
func runAdmin(ctx context.Context, args *serverArgs) error {
globalConfig := server.GetConfig()
diff --git a/cmd/ragflow_server_test.go b/cmd/ragflow_server_test.go
index e7093fd272..1ceed649be 100644
--- a/cmd/ragflow_server_test.go
+++ b/cmd/ragflow_server_test.go
@@ -1,8 +1,25 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
package main
import (
"context"
"errors"
+ "os"
"testing"
"ragflow/internal/engine/types"
@@ -18,6 +35,14 @@ func kb(embdID string, tenantEmbdID string) *entity.Knowledgebase {
return kb
}
+func parseArgsForTest(t *testing.T, argv ...string) (*serverArgs, error) {
+ t.Helper()
+ orig := os.Args
+ os.Args = append([]string{"ragflow_server"}, argv...)
+ defer func() { os.Args = orig }()
+ return parseArgs()
+}
+
// TestHasEmbedderForMirrorsPython pins dialog_service.py:362 —
// `embd_mdl = LLMBundle(...) if kbs and kbs[0].embd_id else None`. The gate is
// the FIRST dataset's embd_id, so a dialog whose leading dataset has no model
@@ -134,6 +159,43 @@ func TestHarnessWebSearcherNilAndWrapped(t *testing.T) {
}
}
+func TestParseArgsMigrateIsStandalone(t *testing.T) {
+ args, err := parseArgsForTest(t, "--migrate")
+ if err != nil {
+ t.Fatalf("parseArgs(--migrate) error = %v", err)
+ }
+ if !args.migrateDB {
+ t.Fatal("migrateDB = false, want true")
+ }
+ if args.mode != nil {
+ t.Fatalf("mode = %q, want nil: --migrate must not select a server mode", *args.mode)
+ }
+}
+
+func TestParseArgsMigrateRejectsMode(t *testing.T) {
+ for _, mode := range []string{"--api", "--admin", "--ingestor", "--syncer"} {
+ if _, err := parseArgsForTest(t, mode, "--migrate"); err == nil {
+ t.Errorf("parseArgs(%s --migrate) error = nil, want error", mode)
+ }
+ if _, err := parseArgsForTest(t, "--migrate", mode); err == nil {
+ t.Errorf("parseArgs(--migrate %s) error = nil, want error", mode)
+ }
+ }
+}
+
+func TestParseArgsModeResetsMigrate(t *testing.T) {
+ args, err := parseArgsForTest(t, "--api")
+ if err != nil {
+ t.Fatalf("parseArgs(--api) error = %v", err)
+ }
+ if args.mode == nil || *args.mode != "api" {
+ t.Fatalf("mode = %v, want api", args.mode)
+ }
+ if args.migrateDB {
+ t.Fatal("migrateDB = true, want false")
+ }
+}
+
// errStubEncode stands in for a provider-side embedding failure.
var errStubEncode = errors.New("stub encode failure")
diff --git a/docker/docker-compose-go.yml b/docker/docker-compose-go.yml
index 4ad60768c5..2c0e7d464f 100644
--- a/docker/docker-compose-go.yml
+++ b/docker/docker-compose-go.yml
@@ -33,21 +33,7 @@ services:
nproc:
soft: 65535
hard: 65535
- # Example configuration to set up an MCP server:
- # command:
- # - --enable-mcpserver
- # - --mcp-host=0.0.0.0
- # - --mcp-port=9382
- # - --mcp-base-url=http://127.0.0.1:9380
- # - --mcp-script-path=/ragflow/mcp/server/server.py
- # - --mcp-mode=self-host
- # - --mcp-host-api-key=ragflow-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
- # Optional transport flags for MCP (customize if needed).
- # Host mode need to combined with --no-transport-streamable-http-enabled flag, namely, host+streamable-http is not supported yet.
- # The following are enabled by default unless explicitly disabled with --no-<flag>.
- # - --no-transport-sse-enabled # Disable legacy SSE endpoints (/sse and /messages/)
- # - --no-transport-streamable-http-enabled # Disable Streamable HTTP transport (/mcp endpoint)
- # - --no-json-response # Disable JSON response mode in Streamable HTTP transport (instead of SSE over HTTP)
+ # MCP is served in-process by the API server (POST /mcp on the 9380 port).
# Example configuration to start Admin server:
command:
@@ -58,7 +44,6 @@ services:
- ${SVR_WEB_HTTPS_PORT}:443
- ${SVR_HTTP_PORT}:9380
- ${ADMIN_SVR_HTTP_PORT}:9381
- - ${SVR_MCP_PORT}:9382 # entry for MCP (host_port:docker_port). The docker_port must match the value you set for `mcp-port` above.
- ${GO_HTTP_PORT}:9384
- ${GO_ADMIN_PORT}:9383
volumes:
@@ -96,21 +81,7 @@ services:
nproc:
soft: 65535
hard: 65535
- # Example configuration to set up an MCP server:
- # command:
- # - --enable-mcpserver
- # - --mcp-host=0.0.0.0
- # - --mcp-port=9382
- # - --mcp-base-url=http://127.0.0.1:9380
- # - --mcp-script-path=/ragflow/mcp/server/server.py
- # - --mcp-mode=self-host
- # - --mcp-host-api-key=ragflow-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
- # Optional transport flags for MCP (customize if needed).
- # Host mode need to combined with --no-transport-streamable-http-enabled flag, namely, host+streamable-http is not supported yet.
- # The following are enabled by default unless explicitly disabled with --no-<flag>.
- # - --no-transport-sse-enabled # Disable legacy SSE endpoints (/sse and /messages/)
- # - --no-transport-streamable-http-enabled # Disable Streamable HTTP transport (/mcp endpoint)
- # - --no-json-response # Disable JSON response mode in Streamable HTTP transport (instead of SSE over HTTP)
+ # MCP is served in-process by the API server (POST /mcp on the 9380 port).
# Example configuration to start Admin server:
command:
@@ -120,14 +91,13 @@ services:
- ${SVR_WEB_HTTPS_PORT}:443
- ${SVR_HTTP_PORT}:9380
- ${ADMIN_SVR_HTTP_PORT}:9381
- - ${SVR_MCP_PORT}:9382 # entry for MCP (host_port:docker_port). The docker_port must match the value you set for `mcp-port` above.
volumes:
- ./ragflow-logs:/ragflow/logs
# - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf
# - ./nginx/proxy.conf:/etc/nginx/proxy.conf
# - ./nginx/nginx.conf:/etc/nginx/nginx.conf
- ./service_conf.yaml.template:/ragflow/conf/service_conf.yaml.template
- - ./entrypoint.sh:/ragflow/entrypoint.sh
+ - ./entrypoint-go.sh:/ragflow/entrypoint.sh
env_file: .env-go
# Route the Go services' cache/queue backend to the separate Kvrocks service
# (overriding REDIS_HOST=redis from .env-go). The Python services keep Valkey.
diff --git a/docker/entrypoint-go.sh b/docker/entrypoint-go.sh
index 7cb9c02ae5..0fe41f5b69 100755
--- a/docker/entrypoint-go.sh
+++ b/docker/entrypoint-go.sh
@@ -14,40 +14,26 @@ function usage() {
echo " --disable-webserver Disables the web server (nginx + ragflow_server)."
echo " --disable-taskexecutor Disables task executor workers."
echo " --disable-datasync Disables synchronization of datasource workers."
- echo " --enable-mcpserver Enables the MCP server."
echo " --enable-adminserver Enables the Admin server."
echo " --init-model-provider-tables Run model provider table migrations and exit."
- echo " --init-superuser Initializes the superuser."
+ echo " --init-superuser Initializes the superuser (needs --enable-adminserver)."
echo " --consumer-no-beg=<num> Start range for consumers (if using range-based)."
echo " --consumer-no-end=<num> End range for consumers (if using range-based)."
echo " --workers=<num> Number of task executors to run (if range is not used)."
echo " --host-id=<string> Unique ID for the host (defaults to \`hostname\`)."
- echo " --mcp-host=<string> Address the MCP server binds to (default: 127.0.0.1)."
- echo " --mcp-port=<num> Port the MCP server listens on (default: 9382)."
- echo " --mcp-base-url=<string> RAGFlow base URL the MCP server calls (default: http://127.0.0.1:9380)."
- echo " --mcp-script-path=<path> MCP server entry script (default: /ragflow/mcp/server/server.py)."
- echo " --mcp-mode=<self-host|host> MCP server mode (default: self-host)."
- echo " --mcp-host-api-key=<string> API key required when --mcp-mode=self-host."
- echo " --no-transport-sse-enabled Disables the MCP SSE transport."
- echo " --no-transport-streamable-http-enabled Disables the MCP streamable HTTP transport. Disabling"
- echo " both transports re-enables this one, since the server"
- echo " requires at least one."
- echo " --no-json-response Disables JSON responses from the MCP server."
echo
echo "Examples:"
echo " $0 --disable-taskexecutor"
echo " $0 --disable-webserver --consumer-no-beg=0 --consumer-no-end=5"
echo " $0 --disable-webserver --workers=2 --host-id=myhost123"
- echo " $0 --enable-mcpserver"
echo " $0 --enable-adminserver"
- echo " $0 --init-superuser"
+ echo " $0 --enable-adminserver --init-superuser"
exit 1
}
ENABLE_WEBSERVER=1 # Default to enable web server
ENABLE_TASKEXECUTOR=1 # Default to enable task executor
ENABLE_DATASYNC=1
-ENABLE_MCP_SERVER=0
ENABLE_ADMIN_SERVER=0 # Default close admin server
INIT_SUPERUSER_ARGS="" # Default to not initialize superuser
INIT_MODEL_PROVIDER_TABLES=0
@@ -55,16 +41,6 @@ CONSUMER_NO_BEG=0
CONSUMER_NO_END=0
WORKERS=1
-MCP_HOST="127.0.0.1"
-MCP_PORT=9382
-MCP_BASE_URL="http://127.0.0.1:9380"
-MCP_SCRIPT_PATH="/ragflow/mcp/server/server.py"
-MCP_MODE="self-host"
-MCP_HOST_API_KEY=""
-MCP_TRANSPORT_SSE_FLAG="--transport-sse-enabled"
-MCP_TRANSPORT_STREAMABLE_HTTP_FLAG="--transport-streamable-http-enabled"
-MCP_JSON_RESPONSE_FLAG="--json-response"
-
# -----------------------------------------------------------------------------
# Host ID logic:
# 1. By default, use the system hostname if length <= 32
@@ -94,10 +70,6 @@ for arg in "$@"; do
ENABLE_DATASYNC=0
shift
;;
- --enable-mcpserver)
- ENABLE_MCP_SERVER=1
- shift
- ;;
--enable-adminserver)
ENABLE_ADMIN_SERVER=1
shift
@@ -110,42 +82,6 @@ for arg in "$@"; do
INIT_SUPERUSER_ARGS="--init-superuser"
shift
;;
- --mcp-host=*)
- MCP_HOST="${arg#*=}"
- shift
- ;;
- --mcp-port=*)
- MCP_PORT="${arg#*=}"
- shift
- ;;
- --mcp-base-url=*)
- MCP_BASE_URL="${arg#*=}"
- shift
- ;;
- --mcp-mode=*)
- MCP_MODE="${arg#*=}"
- shift
- ;;
- --mcp-host-api-key=*)
- MCP_HOST_API_KEY="${arg#*=}"
- shift
- ;;
- --mcp-script-path=*)
- MCP_SCRIPT_PATH="${arg#*=}"
- shift
- ;;
- --no-transport-sse-enabled)
- MCP_TRANSPORT_SSE_FLAG="--no-transport-sse-enabled"
- shift
- ;;
- --no-transport-streamable-http-enabled)
- MCP_TRANSPORT_STREAMABLE_HTTP_FLAG="--no-transport-streamable-http-enabled"
- shift
- ;;
- --no-json-response)
- MCP_JSON_RESPONSE_FLAG="--no-json-response"
- shift
- ;;
--consumer-no-beg=*)
CONSUMER_NO_BEG="${arg#*=}"
shift
@@ -193,78 +129,49 @@ while IFS= read -r line || [[ -n "$line" ]]; do
done < "${TEMPLATE_FILE}"
export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/"
-PY=python3
# -----------------------------------------------------------------------------
-# Select Nginx Configuration based on API_PROXY_SCHEME
+# Select Nginx Configuration
# -----------------------------------------------------------------------------
+# This image ships the Go backend only, so the golang config is the only one
+# available.
NGINX_CONF_DIR="/etc/nginx/conf.d"
-if [ -n "$API_PROXY_SCHEME" ]; then
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]]; then
- cp -f "$NGINX_CONF_DIR/ragflow.conf.hybrid" "$NGINX_CONF_DIR/ragflow.conf"
- echo "Applied nginx config: ragflow.conf.hybrid"
- elif [[ "${API_PROXY_SCHEME}" == "go" ]]; then
- cp -f "$NGINX_CONF_DIR/ragflow.conf.golang" "$NGINX_CONF_DIR/ragflow.conf"
- echo "Applied nginx config: ragflow.conf.golang (default)"
- else
- cp -f "$NGINX_CONF_DIR/ragflow.conf.python" "$NGINX_CONF_DIR/ragflow.conf"
- echo "Applied nginx config: ragflow.conf.python"
- fi
-else
- # Default to python backend
- cp -f "$NGINX_CONF_DIR/ragflow.conf.python" "$NGINX_CONF_DIR/ragflow.conf"
- echo "Default: applied nginx config: ragflow.conf.python"
-fi
+cp -f "$NGINX_CONF_DIR/ragflow.conf.golang" "$NGINX_CONF_DIR/ragflow.conf"
# -----------------------------------------------------------------------------
# Function(s)
# -----------------------------------------------------------------------------
-function task_exe() {
- local consumer_id="$1"
- local host_id="$2"
-
- JEMALLOC_PATH="$(pkg-config --variable=libdir jemalloc)/libjemalloc.so"
- while true; do
- LD_PRELOAD="$JEMALLOC_PATH" \
- "$PY" rag/svr/task_executor.py -i "${host_id}_${consumer_id}" -t "common" &
- wait;
- sleep 1;
- done
-}
-
-function start_mcp_server() {
- echo "Starting MCP Server on ${MCP_HOST}:${MCP_PORT} with base URL ${MCP_BASE_URL}..."
- "$PY" "${MCP_SCRIPT_PATH}" \
- --host="${MCP_HOST}" \
- --port="${MCP_PORT}" \
- --base-url="${MCP_BASE_URL}" \
- --mode="${MCP_MODE}" \
- --api-key="${MCP_HOST_API_KEY}" \
- "${MCP_TRANSPORT_SSE_FLAG}" \
- "${MCP_TRANSPORT_STREAMABLE_HTTP_FLAG}" \
- "${MCP_JSON_RESPONSE_FLAG}" &
-}
-
-function ensure_docling() {
- [[ "${USE_DOCLING}" == "true" ]] || { echo "[docling] disabled by USE_DOCLING"; return 0; }
- DOCLING_PIN="${DOCLING_VERSION:-==2.71.0}"
- "$PY" -c "import importlib.util,sys; sys.exit(0 if importlib.util.find_spec('docling') else 1)" \
- || uv pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --extra-index-url https://pypi.org/simple --no-cache-dir "docling${DOCLING_PIN}"
+# One-shot Go migration. bin/ragflow_server --migrate is a standalone action: it
+# runs the migrations and exits, independent of any server mode.
+function run_go_migrations() {
+ local db_type="${DB_TYPE:-mysql}"
+ db_type="${db_type,,}"
+ if [[ "$db_type" == "gaussdb" || "$db_type" == "gauss" ]]; then
+ # The Go migrations emit MySQL-only SQL and cannot run against a GaussDB
+ # metadata database.
+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."
+ return 0
+ fi
+ echo "Running model provider table migrations..."
+ bin/ragflow_server --migrate
}
-function ensure_db_init() {
- echo "Initializing database tables..."
- "$PY" -c "from api.db.db_models import init_database_tables as init_web_db; init_web_db()"
- echo "Database tables initialized."
+# Whether any Go server mode will run. These are the processes that used to
+# carry --migrate, so the standalone migration must run before them.
+function go_backend_enabled() {
+ if [[ "${ENABLE_DATASYNC}" -eq 1 ]]; then
+ return 0
+ fi
+ if [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]] || [[ "${ENABLE_WEBSERVER}" -eq 1 ]]; then
+ return 0
+ fi
+ return 1
}
# -----------------------------------------------------------------------------
# Start components based on flags
# -----------------------------------------------------------------------------
-#ensure_docling
-#ensure_db_init
-
run_with_restart() {
local process_name="$1"
shift
@@ -280,90 +187,57 @@ run_with_restart() {
done
}
+# --init-model-provider-tables keeps its documented "run migrations and exit"
+# meaning: it migrates and exits without booting any server.
if [[ "${INIT_MODEL_PROVIDER_TABLES}" -eq 1 ]]; then
- DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"
- DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"
- if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then
- # This migration script contains MySQL-only SQL and cannot run against
- # a GaussDB metadata database.
- echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."
- else
- #tools/scripts/run_migrations.sh
- echo ""
- fi
+ run_go_migrations
+ echo "Model provider table migrations finished. Exiting."
+ exit 0
+fi
+
+# Otherwise migrate once up front, before any Go server mode boots. --migrate is
+# a standalone action, so it is no longer attached to --api/--admin/--syncer.
+if go_backend_enabled; then
+ run_go_migrations
fi
if [[ "${ENABLE_DATASYNC}" -eq 1 ]]; then
echo "Starting data sync..."
- run_with_restart "RAGFlow go server" bin/ragflow_server --syncer --migrate &
+ run_with_restart "RAGFlow go server" bin/ragflow_server --syncer &
fi
sleep 5
if [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]]; then
-
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then
- echo "Attempt to start Admin python server..."
- run_with_restart "Admin python server" "$PY" admin/server/admin_server.py &
- fi
-
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then
- echo "Starting Admin go server..."
- run_with_restart "Admin go server" bin/ragflow_server --admin --migrate &
- fi
+ echo "Starting Admin go server..."
+ run_with_restart "Admin go server" bin/ragflow_server --admin ${INIT_SUPERUSER_ARGS} &
fi
if [[ "${ENABLE_WEBSERVER}" -eq 1 ]]; then
echo "Starting nginx..."
/usr/sbin/nginx -c /etc/nginx/nginx.conf
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then
- echo "Attempt to start RAGFlow python server..."
- run_with_restart "RAGFlow python server" "$PY" api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &
- fi
-
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then
- echo "Starting RAGFlow go server..."
- run_with_restart "RAGFlow go server" bin/ragflow_server --api --migrate &
- fi
+ echo "Starting RAGFlow go server..."
+ run_with_restart "RAGFlow go server" bin/ragflow_server --api &
fi
+# MCP needs no separate process: --api serves it in-process at POST /mcp on the
+# main API port.
-
-#if [[ "${ENABLE_MCP_SERVER}" -eq 1 ]]; then
-# start_mcp_server
-#fi
-
-
+# Task execution is the Go ingestor's job. This image ships no Python task
+# executor (rag/svr/task_executor.py is not copied), so --ingestor is the only
+# worker that can run here.
if [[ "${ENABLE_TASKEXECUTOR}" -eq 1 ]]; then
if [[ "${CONSUMER_NO_END}" -gt "${CONSUMER_NO_BEG}" ]]; then
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then
- echo "Starting python task executors on host '${HOST_ID}' for IDs in [${CONSUMER_NO_BEG}, ${CONSUMER_NO_END})..."
- for (( i=CONSUMER_NO_BEG; i<CONSUMER_NO_END; i++ ))
- do
- task_exe "${i}" "${HOST_ID}" &
- done
- fi
-
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then
- echo "Starting go ingestor..."
- run_with_restart "ingestor" bin/ragflow_server --ingestor &
- fi
+ echo "Starting go ingestor..."
+ run_with_restart "ingestor" bin/ragflow_server --ingestor &
else
# Otherwise, start a fixed number of workers
echo "Starting ${WORKERS} task executor(s) on host '${HOST_ID}'..."
for (( i=0; i<WORKERS; i++ ))
do
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "python" ]]; then
- echo "Starting python task executor..."
- task_exe "${i}" "${HOST_ID}" &
- sleep 1;
- fi
-
- if [[ "${API_PROXY_SCHEME}" == "hybrid" ]] || [[ "${API_PROXY_SCHEME}" == "go" ]]; then
- echo "Starting go ingestor..."
- run_with_restart "ingestor" bin/ragflow_server --ingestor &
- fi
+ echo "Starting go ingestor..."
+ run_with_restart "ingestor" bin/ragflow_server --ingestor &
done
fi
fi
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 62cba47a80..569ffd965e 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -259,6 +259,27 @@ function ensure_db_init() {
echo "Database tables initialized."
}
+# Run the model provider table migrations. The Go backend owns those tables when
+# it serves the API, so it runs its own --migrate action instead of the Python
+# script.
+function run_model_provider_migrations() {
+ DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"
+ DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"
+ if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then
+ # These migrations contain MySQL-only SQL and cannot run against a
+ # GaussDB metadata database.
+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."
+ return 0
+ fi
+
+ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
+ echo "Running model provider table migrations (go)..."
+ bin/ragflow_server --migrate
+ else
+ tools/scripts/run_migrations.sh
+ fi
+}
+
# -----------------------------------------------------------------------------
# Start components based on flags
# -----------------------------------------------------------------------------
@@ -282,18 +303,16 @@ run_with_restart() {
# This used to run inside the web server block only, so hosts that ran e.g.
# just the admin server, data sync, MCP server, or task executors never
# initialized the schema.
-ensure_db_init
+#
+# Under the go scheme the Go backend owns the schema: its --migrate action and
+# dao.InitDB create and converge the tables. Skip the Python initialization so a
+# single side owns the schema.
+if [[ "${API_PROXY_SCHEME}" != "go" ]]; then
+ ensure_db_init
+fi
if [[ "${INIT_MODEL_PROVIDER_TABLES}" -eq 1 ]]; then
- DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"
- DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"
- if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then
- # This migration script contains MySQL-only SQL and cannot run against
- # a GaussDB metadata database.
- echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."
- else
- tools/scripts/run_migrations.sh
- fi
+ run_model_provider_migrations
fi
if [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]]; then
@@ -327,11 +346,20 @@ if [[ "${ENABLE_WEBSERVER}" -eq 1 ]]; then
fi
if [[ "${ENABLE_DATASYNC}" -eq 1 ]]; then
- echo "Starting data sync..."
- run_with_restart "Data sync" "$PY" rag/svr/sync_data_source.py &
+ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
+ echo "Starting data sync (go)..."
+ run_with_restart "RAGFlow go server" bin/ragflow_server --syncer &
+ else
+ echo "Starting data sync..."
+ run_with_restart "Data sync" "$PY" rag/svr/sync_data_source.py &
+ fi
fi
-if [[ "${ENABLE_MCP_SERVER}" -eq 1 ]]; then
+# The Go backend serves MCP in-process from --api (POST /mcp on the main API
+# port) and exposes the same ragflow_retrieval, ragflow_list_datasets and
+# ragflow_list_chats tools. The standalone Python MCP server would only
+# duplicate it, so the go scheme relies on the built-in endpoint instead.
+if [[ "${ENABLE_MCP_SERVER}" -eq 1 ]] && [[ "${API_PROXY_SCHEME}" != "go" ]]; then
start_mcp_server
fi
diff --git a/docker/launch_backend_service.sh b/docker/launch_backend_service.sh
index 8bd643155c..5275a10aae 100755
--- a/docker/launch_backend_service.sh
+++ b/docker/launch_backend_service.sh
@@ -15,7 +15,7 @@ usage() {
echo " ragflow Start RAGFlow server based on API_PROXY_SCHEME"
echo " task_executor Start task workers based on API_PROXY_SCHEME"
echo " admin Start Admin server based on API_PROXY_SCHEME"
- echo " data_sync Start rag/svr/sync_data_source.py"
+ echo " data_sync Start the data source syncer based on API_PROXY_SCHEME"
echo
echo "Examples:"
echo " $0"
@@ -199,23 +199,33 @@ run_admin_server(){
# Function to execute sync_data_source with retry logic
run_data_sync(){
+ local server_name="sync_data_source.py"
+ local -a sync_cmd=("$PY" "rag/svr/sync_data_source.py")
+ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
+ prepare_for_go
+ server_name="ragflow_server --syncer"
+ sync_cmd=("bin/ragflow_server" "--syncer")
+ if [[ "$DEBUG_MODE" -eq 1 ]]; then
+ sync_cmd+=("--debug")
+ fi
+ fi
local retry_count=0
while ! $STOP && [ $retry_count -lt $MAX_RETRIES ]; do
- echo "Starting sync_data_source.py (Attempt $((retry_count+1)))"
+ echo "Starting ${sync_cmd[*]} (Attempt $((retry_count+1)))"
EXIT_CODE=0
- $PY rag/svr/sync_data_source.py || EXIT_CODE=$?
+ "${sync_cmd[@]}" || EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
- echo "sync_data_source.py exited successfully."
+ echo "$server_name exited successfully."
break
else
- echo "sync_data_source.py failed with exit code $EXIT_CODE. Retrying..." >&2
+ echo "$server_name failed with exit code $EXIT_CODE. Retrying..." >&2
retry_count=$((retry_count + 1))
sleep 2
fi
done
if [ $retry_count -ge $MAX_RETRIES ]; then
- echo "sync_data_source.py failed after $MAX_RETRIES attempts. Exiting..." >&2
+ echo "$server_name failed after $MAX_RETRIES attempts. Exiting..." >&2
cleanup
fi
}
@@ -226,17 +236,25 @@ ensure_db_init() {
echo "Database tables initialized."
}
-run_mysql_migrations() {
+run_migrations() {
local db_type="${DB_TYPE:-mysql}"
db_type="${db_type,,}"
if [ "$db_type" = "gaussdb" ] || [ "$db_type" = "gauss" ]; then
- # This migration script contains MySQL-only SQL and cannot run against
- # a GaussDB metadata database.
+ # The migrations contain MySQL-only SQL and cannot run against a GaussDB
+ # metadata database.
echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."
return 0
fi
- tools/scripts/run_migrations.sh
+ if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
+ # The Go backend owns the model provider tables. --migrate is a
+ # standalone action: it runs the migrations and exits, so it is
+ # independent of any server actually starting.
+ echo "Running model provider table migrations (go)..."
+ bin/ragflow_server --migrate
+ else
+ tools/scripts/run_migrations.sh
+ fi
}
prepare_for_go() {
@@ -309,8 +327,14 @@ if [[ "$SERVICE_SELECTED" -eq 0 ]]; then
fi
if [[ "$START_RAGFLOW" -eq 1 ]]; then
- ensure_db_init
- run_mysql_migrations
+ # The Go backend owns the schema when it serves the API, so skip the
+ # Python-side table creation then. A Go-only deployment may not ship the api
+ # package at all, and letting both sides create tables makes them fight over
+ # the same schema.
+ if [[ "${API_PROXY_SCHEME}" != "go" ]]; then
+ ensure_db_init
+ fi
+ run_migrations
fi
# Start task executors
diff --git a/internal/common/version.go b/internal/common/version.go
index 12965db071..2f9d212a89 100644
--- a/internal/common/version.go
+++ b/internal/common/version.go
@@ -21,6 +21,8 @@ import (
"path/filepath"
"strings"
"sync"
+
+ "golang.org/x/mod/semver"
)
var (
@@ -75,3 +77,42 @@ func getClosestTagAndCount() string {
}
return strings.TrimSpace(string(output))
}
+
+// IsOlderReleaseThan reports whether codeVersion is an earlier release than
+// targetVersion, comparing only MAJOR.MINOR.PATCH.
+//
+// git describe reports a build made from a commit after a tag as
+// "v0.27.1-14-g<sha>". semver ranks that pre-release string below "v0.27.1"
+// even though the code is newer than the tag, so pre-release and build suffixes
+// are stripped before comparing.
+//
+// The second result is false when either version cannot be parsed, in which
+// case there is no evidence of a downgrade and the caller must not block.
+func IsOlderReleaseThan(codeVersion, targetVersion string) (older, comparable bool) {
+ code := releaseVersion(codeVersion)
+ target := releaseVersion(targetVersion)
+ if code == "" || target == "" {
+ return false, false
+ }
+ return semver.Compare(code, target) < 0, true
+}
+
+// releaseVersion normalizes a version string to the "vMAJOR.MINOR.PATCH" form
+// semver accepts, dropping any pre-release or build suffix. It returns an empty
+// string when the result is not a valid semver tag.
+func releaseVersion(version string) string {
+ normalized := strings.TrimSpace(version)
+ normalized = strings.TrimPrefix(normalized, "v")
+ normalized = strings.TrimPrefix(normalized, "V")
+ if normalized == "" {
+ return ""
+ }
+ if i := strings.IndexAny(normalized, "-+"); i >= 0 {
+ normalized = normalized[:i]
+ }
+ normalized = "v" + normalized
+ if !semver.IsValid(normalized) {
+ return ""
+ }
+ return normalized
+}
diff --git a/internal/common/version_test.go b/internal/common/version_test.go
index 4eddc74b69..f3627d7201 100644
--- a/internal/common/version_test.go
+++ b/internal/common/version_test.go
@@ -37,3 +37,38 @@ func TestGetClosestTagAndCount(t *testing.T) {
fmt.Printf("Git Version: %s\n", version)
// This test just prints the version, no strict assertion
}
+
+func TestIsOlderReleaseThan(t *testing.T) {
+ cases := []struct {
+ name string
+ code string
+ target string
+ wantOlder bool
+ wantCmp bool
+ }{
+ {name: "older patch", code: "v0.26.0", target: "v0.27.1", wantOlder: true, wantCmp: true},
+ {name: "older minor", code: "v0.26.9", target: "v0.27.1", wantOlder: true, wantCmp: true},
+ {name: "older major", code: "v0.27.1", target: "v1.0.0", wantOlder: true, wantCmp: true},
+ {name: "equal", code: "v0.27.1", target: "v0.27.1", wantOlder: false, wantCmp: true},
+ {name: "newer", code: "v0.28.0", target: "v0.27.1", wantOlder: false, wantCmp: true},
+ // git describe reports commits after a tag as v0.27.1-14-g<sha>; that is
+ // newer code, not an older release.
+ {name: "commits after tag", code: "v0.27.1-14-g1a2b3c4", target: "v0.27.1", wantOlder: false, wantCmp: true},
+ {name: "commits after older tag", code: "v0.27.0-3-g1a2b3c4", target: "v0.27.1", wantOlder: true, wantCmp: true},
+ {name: "missing v prefix", code: "0.26.0", target: "v0.27.1", wantOlder: true, wantCmp: true},
+ {name: "unknown code", code: "unknown", target: "v0.27.1", wantOlder: false, wantCmp: false},
+ {name: "empty code", code: "", target: "v0.27.1", wantOlder: false, wantCmp: false},
+ {name: "empty target", code: "v0.27.1", target: "", wantOlder: false, wantCmp: false},
+ {name: "invalid code", code: "not-a-version", target: "v0.27.1", wantOlder: false, wantCmp: false},
+ {name: "invalid target", code: "v0.27.1", target: "not-a-version", wantOlder: false, wantCmp: false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ older, comparable := IsOlderReleaseThan(tc.code, tc.target)
+ if older != tc.wantOlder || comparable != tc.wantCmp {
+ t.Fatalf("IsOlderReleaseThan(%q, %q) = (%v, %v), want (%v, %v)",
+ tc.code, tc.target, older, comparable, tc.wantOlder, tc.wantCmp)
+ }
+ })
+ }
+}
diff --git a/internal/dao/database.go b/internal/dao/database.go
index d187a7ca2a..a5c97f7b38 100644
--- a/internal/dao/database.go
+++ b/internal/dao/database.go
@@ -173,20 +173,26 @@ func InitDB(ctx context.Context, migrateDB bool) error {
}
if migrateDB {
+ // Mirror the Python flow, where tools/scripts/run_migrations.sh runs before
+ // the ORM creates and converges the schema: the manual migrations have to see
+ // the legacy tables as they are. Running them after AutoMigrate would let
+ // AutoMigrate rewrite tenant_model.model_type from text to int before the
+ // model_type_merge step can read what the Python migration wrote.
+ if err = RunMigrations(ctx, DB); err != nil {
+ return fmt.Errorf("failed to run manual migrations: %w", err)
+ }
+
common.Info("Migrating database schema...")
for _, m := range dataModels {
if err = autoMigrateSafely(ctx, DB, m); err != nil {
return fmt.Errorf("failed to migrate model %T: %w", m, err)
}
}
-
- // Run manual migrations for complex schema changes
- if err = RunMigrations(ctx, DB); err != nil {
- return fmt.Errorf("failed to run manual migrations: %w", err)
- }
common.Info("Database schema migrated successfully")
} else {
- // Ensure Go-exclusive runtime tables exist even if the server starts without --migrate
+ // Ensure the Go-exclusive runtime tables exist. The manual migrations are
+ // performed by the standalone --migrate action, so a server-mode process
+ // only converges the tables it needs itself.
if err = autoMigrateRuntimeModels(ctx, DB); err != nil {
return fmt.Errorf("failed to auto-migrate runtime models: %w", err)
}
@@ -316,8 +322,9 @@ func autoMigrateSafely(ctx context.Context, db *gorm.DB, model interface{}) erro
return err
}
-// autoMigrateRuntimeModels ensures Go-exclusive runtime tables exist even if
-// the server starts without --migrate.
+// autoMigrateRuntimeModels ensures the Go-exclusive runtime tables exist. The
+// manual migrations run as the standalone --migrate action, so a server-mode
+// process never runs them itself.
func autoMigrateRuntimeModels(ctx context.Context, db *gorm.DB) error {
goRuntimeModels := []interface{}{
&entity.IngestionTask{},
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index 559d4d2128..6e006c3efd 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -21,6 +21,7 @@ import (
"fmt"
"ragflow/internal/common"
"ragflow/internal/entity"
+ "regexp"
"strings"
"go.uber.org/zap"
@@ -72,89 +73,104 @@ func RunMigrations(ctx context.Context, db *gorm.DB) error {
return fmt.Errorf("failed to migrate general chunker parser configs: %w", err)
}
+ // Backfill the tenant model tables from the legacy tenant_llm table.
+ if err := migrateModelData(ctx, db); err != nil {
+ return fmt.Errorf("failed to migrate tenant model data: %w", err)
+ }
+
common.Info("All manual migrations completed successfully")
return nil
}
-// migrateTenantLLMPrimaryKey migrates tenant_llm from composite primary key to ID primary key
-// This corresponds to Python's update_tenant_llm_to_id_primary_key function
+// migrateTenantLLMPrimaryKey migrates tenant_llm from its legacy composite
+// primary key (tenant_id, llm_factory, llm_name) to a surrogate auto-increment
+// "id" column.
+//
+// MySQL allows a single PRIMARY KEY per table, so the composite key has to be
+// dropped before the new key column can be promoted. The legacy uniqueness
+// guarantee is preserved by a unique index on the old key columns.
func migrateTenantLLMPrimaryKey(ctx context.Context, db *gorm.DB) error {
- // Check if tenant_llm table exists
if !db.WithContext(ctx).Migrator().HasTable("tenant_llm") {
return nil
}
- // Check if 'id' column already exists using raw SQL
- var idColumnExists int64
- err := db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
- WHERE TABLE_NAME = 'tenant_llm' AND COLUMN_NAME = 'id'
- `).Scan(&idColumnExists).Error
+ // Idempotency: an auto_increment "id" means the table is already migrated.
+ idIsAutoIncrement, err := isAutoIncrementColumn(ctx, db, "tenant_llm", "id")
if err != nil {
return err
}
-
- if idColumnExists > 0 {
- // Check if id is already a primary key with auto_increment
- var count int64
- err = db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
- WHERE TABLE_NAME = 'tenant_llm'
- AND COLUMN_NAME = 'id'
- AND EXTRA LIKE '%auto_increment%'
- `).Scan(&count).Error
- if err != nil {
- return err
- }
- if count > 0 {
- // Already migrated
- return nil
- }
+ if idIsAutoIncrement {
+ return nil
}
common.Info("Migrating tenant_llm to use ID primary key...")
- // Start transaction
+ // MySQL commits DDL implicitly, so this transaction groups the statements
+ // rather than making them atomic. Every step is therefore written to be
+ // re-runnable after an interrupted attempt.
return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- // Check for temp_id column and drop it if exists
- var tempIdExists int64
- tx.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
- WHERE TABLE_NAME = 'tenant_llm' AND COLUMN_NAME = 'temp_id'`).Scan(&tempIdExists)
- if tempIdExists > 0 {
- if err = tx.Exec("ALTER TABLE tenant_llm DROP COLUMN temp_id").Error; err != nil {
- common.Warn("Failed to drop temp_id column", zap.Error(err))
- }
+ // A leftover temp_id can only come from an interrupted run.
+ if err := dropColumnIfExists(ctx, tx, "tenant_llm", "temp_id"); err != nil {
+ return err
}
- // Check if there's already an 'id' column
- if idColumnExists > 0 {
- // Modify existing id column to be auto_increment primary key
- if err = tx.Exec(`
- ALTER TABLE tenant_llm
- MODIFY COLUMN id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY
- `).Error; err != nil {
- return fmt.Errorf("failed to modify id column: %w", err)
+ idExists, err := columnExists(ctx, tx, "tenant_llm", "id")
+ if err != nil {
+ return err
+ }
+
+ if idExists {
+ alreadyKey, err := dropLegacyPrimaryKey(ctx, tx, "tenant_llm", "id")
+ if err != nil {
+ return err
+ }
+ // Re-declaring the primary key on a column that already holds it is
+ // rejected as a second primary key definition.
+ definition := "BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY"
+ if alreadyKey {
+ definition = "BIGINT NOT NULL AUTO_INCREMENT"
+ }
+ if err = tx.Exec("ALTER TABLE tenant_llm MODIFY COLUMN id " + definition).Error; err != nil {
+ return fmt.Errorf("failed to make tenant_llm.id auto_increment: %w", err)
}
} else {
- // Add id column as auto_increment primary key
- if err = tx.Exec(`
- ALTER TABLE tenant_llm
- ADD COLUMN id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST
- `).Error; err != nil {
- return fmt.Errorf("failed to add id column: %w", err)
+ if err = tx.Exec(`ALTER TABLE tenant_llm ADD COLUMN temp_id BIGINT NULL`).Error; err != nil {
+ return fmt.Errorf("failed to add temp_id column: %w", err)
+ }
+ // Number the rows explicitly instead of relying on assignment order,
+ // so the resulting primary key is stable across runs.
+ if err = tx.Exec(`SET @ragflow_tenant_llm_row = 0`).Error; err != nil {
+ return fmt.Errorf("failed to initialize the row counter: %w", err)
+ }
+ if err = tx.Exec(`UPDATE tenant_llm
+ SET temp_id = (@ragflow_tenant_llm_row := @ragflow_tenant_llm_row + 1)
+ ORDER BY tenant_id, llm_factory, llm_name`).Error; err != nil {
+ return fmt.Errorf("failed to number tenant_llm rows: %w", err)
+ }
+ if _, err = dropLegacyPrimaryKey(ctx, tx, "tenant_llm", "temp_id"); err != nil {
+ return err
+ }
+ if err = tx.Exec(`ALTER TABLE tenant_llm
+ MODIFY COLUMN temp_id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY`).Error; err != nil {
+ return fmt.Errorf("failed to make temp_id the primary key: %w", err)
}
}
- // Add unique index on (tenant_id, llm_factory, llm_name)
- var idxExists int64
- tx.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'tenant_llm' AND INDEX_NAME = 'idx_tenant_llm_unique'`).Scan(&idxExists)
- if idxExists == 0 {
- if err = tx.Exec(`
- ALTER TABLE tenant_llm
- ADD UNIQUE INDEX idx_tenant_llm_unique (tenant_id, llm_factory, llm_name)
- `).Error; err != nil {
- common.Warn("Failed to add unique index idx_tenant_llm_unique", zap.Error(err))
+ // Preserve the uniqueness contract of the legacy composite primary key.
+ legacyKey := []string{"tenant_id", "llm_factory", "llm_name"}
+ hasUnique, err := hasUniqueIndex(ctx, tx, "tenant_llm", legacyKey)
+ if err != nil {
+ return err
+ }
+ if !hasUnique {
+ if err = addUniqueIndex(ctx, tx, "tenant_llm", "uk_tenant_llm", legacyKey); err != nil {
+ return err
+ }
+ }
+
+ if !idExists {
+ if err = tx.Exec(`ALTER TABLE tenant_llm RENAME COLUMN temp_id TO id`).Error; err != nil {
+ return fmt.Errorf("failed to rename temp_id to id: %w", err)
}
}
@@ -163,49 +179,38 @@ func migrateTenantLLMPrimaryKey(ctx context.Context, db *gorm.DB) error {
})
}
-// migrateAddUniqueEmail adds unique index on user.email
+// migrateAddUniqueEmail enforces uniqueness on user.email.
+//
+// Rows that already share an address are resolved before the index is created:
+// the superuser - or the oldest account when the address has none - keeps it,
+// and the remaining rows are renamed to "<email>_DUPLICATE_<id prefix>".
+// Renaming is what makes the index creatable on installations that accumulated
+// duplicates before the constraint existed.
func migrateAddUniqueEmail(ctx context.Context, db *gorm.DB) error {
if !db.WithContext(ctx).Migrator().HasTable("user") {
return nil
}
- // Check if unique index already exists using raw SQL
- var count int64
- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'user' AND INDEX_NAME = 'idx_user_email_unique'`).Scan(&count)
- if count > 0 {
- return nil
- }
-
- // Check if there's a duplicate email issue first
- var duplicateCount int64
- err := db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM (
- SELECT email FROM user GROUP BY email HAVING COUNT(*) > 1
- ) AS duplicates
- `).Scan(&duplicateCount).Error
+ hasUnique, err := hasUniqueIndex(ctx, db, "user", []string{"email"})
if err != nil {
return err
}
-
- if duplicateCount > 0 {
- common.Warn("Found duplicate emails in user table, cannot add unique index", zap.Int64("count", duplicateCount))
+ if hasUnique {
return nil
}
- common.Info("Adding unique index on user.email...")
- if err = db.WithContext(ctx).Exec(`ALTER TABLE user ADD UNIQUE INDEX idx_user_email_unique (email)`).Error; err != nil {
+ if err = renameDuplicateEmails(ctx, db); err != nil {
+ return err
+ }
- // Check if error is MySQL duplicate index error (Error 1061)
- errStr := err.Error()
- if strings.Contains(errStr, "Error 1061") && strings.Contains(errStr, "Duplicate key name") {
- common.Info("Index already exists, skipping", zap.String("error", errStr))
- return nil
- }
- return fmt.Errorf("failed to add unique index on email: %w", err)
+ // Reaching this point means any existing index on the address is not unique,
+ // and a same-named non-unique index would block the unique one.
+ if err = dropIndexIfExists(ctx, db, "user", "idx_user_email"); err != nil {
+ return err
}
- return nil
+ common.Info("Adding unique index on user.email...")
+ return addUniqueIndex(ctx, db, "user", "idx_user_email", []string{"email"})
}
func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) error {
@@ -214,23 +219,18 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erro
}
const indexName = "idx_ingestion_task_document_id"
+ columns := []string{"document_id"}
- var uniqueCount int64
- if err := db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'ingestion_task'
- AND INDEX_NAME = ?
- AND COLUMN_NAME = 'document_id'
- AND NON_UNIQUE = 0
- `, indexName).Scan(&uniqueCount).Error; err != nil {
+ hasUnique, err := hasUniqueIndex(ctx, db, "ingestion_task", columns)
+ if err != nil {
return err
}
- if uniqueCount > 0 {
+ if hasUnique {
return nil
}
var duplicateCount int64
- if err := db.WithContext(ctx).Raw(`
+ if err = db.WithContext(ctx).Raw(`
SELECT COUNT(*) FROM (
SELECT document_id FROM ingestion_task GROUP BY document_id HAVING COUNT(*) > 1
) AS duplicates
@@ -242,30 +242,12 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erro
return nil
}
- var existingIndexCount int64
- if err := db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'ingestion_task'
- AND INDEX_NAME = ?
- `, indexName).Scan(&existingIndexCount).Error; err != nil {
+ // A non-unique index under the same name would block the unique one.
+ if err = dropIndexIfExists(ctx, db, "ingestion_task", indexName); err != nil {
return err
}
- if existingIndexCount > 0 {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE ingestion_task DROP INDEX ` + indexName).Error; err != nil {
- return fmt.Errorf("failed to drop existing index %s: %w", indexName, err)
- }
- }
- if err := db.WithContext(ctx).Exec(`ALTER TABLE ingestion_task ADD UNIQUE INDEX ` + indexName + ` (document_id)`).Error; err != nil {
- errStr := err.Error()
- if strings.Contains(errStr, "Error 1061") && strings.Contains(errStr, "Duplicate key name") {
- common.Info("Index already exists, skipping", zap.String("error", errStr))
- return nil
- }
- return fmt.Errorf("failed to add unique index on ingestion_task.document_id: %w", err)
- }
-
- return nil
+ return addUniqueIndex(ctx, db, "ingestion_task", indexName, columns)
}
// migrateIngestionTaskPipelineLogID adds ingestion_task.pipeline_log_id when
@@ -422,69 +404,64 @@ func migrateUserCanvasTitleUnique(ctx context.Context, db *gorm.DB) error {
return nil
}
-// modifyColumnTypes modifies column types that need explicit ALTER statements
+// modifyColumnTypes aligns columns whose stored type must match the Python
+// models in api/db/db_models.py. The target types are also declared on the Go
+// entities, so AutoMigrate converges on the same definition and the two paths
+// cannot fight over the column.
func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {
- columnExists := func(table, column string) bool {
- var count int64
- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
- WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, table, column).Scan(&count)
- return count > 0
- }
-
- // dialog.top_k: ensure its INTEGER with default 1024
- if db.WithContext(ctx).Migrator().HasTable("dialog") && columnExists("dialog", "top_k") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE dialog MODIFY COLUMN top_k BIGINT NOT NULL DEFAULT 1024`).Error; err != nil {
- common.Warn("Failed to modify dialog.top_k", zap.Error(err))
- }
- }
-
- // tenant_llm.api_key: ensure it's TEXT type
- if db.WithContext(ctx).Migrator().HasTable("tenant_llm") && columnExists("tenant_llm", "api_key") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE tenant_llm MODIFY COLUMN api_key VARCHAR(8192)`).Error; err != nil {
- common.Warn("Failed to modify tenant_llm.api_key", zap.Error(err))
- }
- }
-
- // api_token.dialog_id: ensure it's varchar(32)
- if db.WithContext(ctx).Migrator().HasTable("api_token") && columnExists("api_token", "dialog_id") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE api_token MODIFY COLUMN dialog_id VARCHAR(32)`).Error; err != nil {
- common.Warn("Failed to modify api_token.dialog_id", zap.Error(err))
- }
- }
-
- // canvas_template.title and description: ensure they're LONGTEXT type (same as Python JSONField)
- // Note: Python's JSONField uses null=True with application-level default, not database DEFAULT
- if db.WithContext(ctx).Migrator().HasTable("canvas_template") {
- if columnExists("canvas_template", "title") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE canvas_template MODIFY COLUMN title LONGTEXT NULL`).Error; err != nil {
- common.Warn("Failed to modify canvas_template.title", zap.Error(err))
- }
- }
- if columnExists("canvas_template", "description") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE canvas_template MODIFY COLUMN description LONGTEXT NULL`).Error; err != nil {
- common.Warn("Failed to modify canvas_template.description", zap.Error(err))
- }
- }
- }
-
- // system_settings.value: ensure it's LONGTEXT
- if db.WithContext(ctx).Migrator().HasTable("system_settings") && columnExists("system_settings", "value") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE system_settings MODIFY COLUMN value LONGTEXT NOT NULL`).Error; err != nil {
- common.Warn("Failed to modify system_settings.value", zap.Error(err))
- }
- }
-
- // knowledgebase.raptor_task_finish_at: ensure it's DateTime
- if db.WithContext(ctx).Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "raptor_task_finish_at") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE knowledgebase MODIFY COLUMN raptor_task_finish_at DATETIME`).Error; err != nil {
- common.Warn("Failed to modify knowledgebase.raptor_task_finish_at", zap.Error(err))
- }
- }
-
- // knowledgebase.mindmap_task_finish_at: ensure it's DateTime
- if db.WithContext(ctx).Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "mindmap_task_finish_at") {
- if err := db.WithContext(ctx).Exec(`ALTER TABLE knowledgebase MODIFY COLUMN mindmap_task_finish_at DATETIME`).Error; err != nil {
- common.Warn("Failed to modify knowledgebase.mindmap_task_finish_at", zap.Error(err))
+ specs := []columnSpec{
+ // dialog.top_k mirrors Python's IntegerField(default=1024).
+ {
+ table: "dialog", column: "top_k",
+ columnType: "int", nullable: false,
+ definition: "int NOT NULL DEFAULT 1024",
+ },
+ // tenant_llm.api_key mirrors Python's TextField(null=True).
+ {
+ table: "tenant_llm", column: "api_key",
+ columnType: "text", nullable: true,
+ definition: "text",
+ },
+ // api_token.dialog_id mirrors Python's CharField(max_length=32, null=True).
+ {
+ table: "api_token", column: "dialog_id",
+ columnType: "varchar(32)", nullable: true,
+ definition: "varchar(32)",
+ },
+ // canvas_template.title/description mirror Python's JSONField, which is a
+ // nullable LONGTEXT.
+ {
+ table: "canvas_template", column: "title",
+ columnType: "longtext", nullable: true,
+ definition: "longtext NULL",
+ },
+ {
+ table: "canvas_template", column: "description",
+ columnType: "longtext", nullable: true,
+ definition: "longtext NULL",
+ },
+ // system_settings.value mirrors Python's EmptyStringTextField(null=False).
+ {
+ table: "system_settings", column: "value",
+ columnType: "text", nullable: false,
+ definition: "text NOT NULL",
+ },
+ // knowledgebase task timestamps mirror Python's DateTimeField.
+ {
+ table: "knowledgebase", column: "raptor_task_finish_at",
+ columnType: "datetime", nullable: true,
+ definition: "datetime",
+ },
+ {
+ table: "knowledgebase", column: "mindmap_task_finish_at",
+ columnType: "datetime", nullable: true,
+ definition: "datetime",
+ },
+ }
+
+ for _, spec := range specs {
+ if err := applyColumnSpec(ctx, db, spec); err != nil {
+ return err
}
}
@@ -497,21 +474,19 @@ func renameColumnIfExists(ctx context.Context, db *gorm.DB, tableName, oldName,
return nil
}
- // Helper to check if column exists
- columnExists := func(column string) bool {
- var count int64
- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
- WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, tableName, column).Scan(&count)
- return count > 0
+ oldExists, err := columnExists(ctx, db, tableName, oldName)
+ if err != nil {
+ return err
}
-
- // Check if old column exists
- if !columnExists(oldName) {
+ if !oldExists {
return nil
}
- // Check if new column already exists
- if columnExists(newName) {
+ newExists, err := columnExists(ctx, db, tableName, newName)
+ if err != nil {
+ return err
+ }
+ if newExists {
// Both exist, drop the old one
common.Warn("Both old and new columns exist, dropping old one",
zap.String("table", tableName),
@@ -533,11 +508,11 @@ func addColumnIfNotExists(ctx context.Context, db *gorm.DB, tableName, columnNam
return nil
}
- // Check if column exists using raw SQL
- var count int64
- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
- WHERE TABLE_NAME = ? AND COLUMN_NAME = ?`, tableName, columnName).Scan(&count)
- if count > 0 {
+ exists, err := columnExists(ctx, db, tableName, columnName)
+ if err != nil {
+ return err
+ }
+ if exists {
return nil
}
@@ -548,174 +523,281 @@ func addColumnIfNotExists(ctx context.Context, db *gorm.DB, tableName, columnNam
return db.WithContext(ctx).Exec(sql).Error
}
-// migrateSkillSearchTables creates skill search related tables
-func migrateSkillSearchTables(ctx context.Context, db *gorm.DB) error {
- // Create skill_search_configs table only
- if !db.WithContext(ctx).Migrator().HasTable("skill_search_configs") {
- common.Info("Creating skill_search_configs table...")
- sql := `
- CREATE TABLE IF NOT EXISTS skill_search_configs (
- id VARCHAR(32) PRIMARY KEY,
- tenant_id VARCHAR(32) NOT NULL,
- space_id VARCHAR(128) NOT NULL DEFAULT 'default',
- embd_id VARCHAR(128) NOT NULL,
- vector_similarity_weight FLOAT DEFAULT 0.3,
- similarity_threshold FLOAT DEFAULT 0.2,
- field_config JSON,
- rerank_id VARCHAR(128),
- tenant_rerank_id BIGINT,
- top_k BIGINT DEFAULT 10,
- index_version VARCHAR(32) DEFAULT '1.0.0',
- status VARCHAR(1) DEFAULT '1',
- create_time BIGINT,
- create_date DATETIME,
- update_time BIGINT,
- update_date DATETIME,
- INDEX idx_tenant_id (tenant_id),
- INDEX idx_space_id (space_id),
- UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)
- )
- `
- if err := db.WithContext(ctx).Exec(sql).Error; err != nil {
- common.Warn("Failed to create skill_search_configs table with MySQL dialect, trying generic", zap.Error(err))
- if err = db.WithContext(ctx).AutoMigrate(&entity.SkillSearchConfig{}); err != nil {
- return err
- }
- // AutoMigrate doesn't create unique indexes, so create them explicitly
- common.Info("Creating unique indexes for skill_search_configs...")
- if err = db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs ADD UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)`).Error; err != nil {
- return fmt.Errorf("failed to create unique index idx_tenant_space_embd: %w", err)
- }
- }
- } else {
- // Add space_id for existing installations.
- if err := addColumnIfNotExists(ctx, db, "skill_search_configs", "space_id", "VARCHAR(128) NOT NULL DEFAULT 'default'"); err != nil {
- return fmt.Errorf("failed to add space_id column to skill_search_configs: %w", err)
- }
- if err := addColumnIfNotExists(ctx, db, "skill_search_configs", "create_date", "DATETIME"); err != nil {
- return fmt.Errorf("failed to add create_date column to skill_search_configs: %w", err)
- }
- if err := addColumnIfNotExists(ctx, db, "skill_search_configs", "update_date", "DATETIME"); err != nil {
- return fmt.Errorf("failed to add update_date column to skill_search_configs: %w", err)
- }
- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs MODIFY COLUMN update_time BIGINT`).Error; err != nil {
- common.Warn("Failed to modify skill_search_configs.update_time", zap.Error(err))
- }
+// columnExists reports whether the current database already has the column.
+// TABLE_SCHEMA is pinned to DATABASE() so that a MySQL server hosting more than
+// one RAGFlow database never matches another schema's tables.
+//
+// It replaces gorm's Migrator.HasColumn, which panics when the table is passed
+// as a string: HasColumn only fills Statement.Table for strings and then
+// dereferences the unparsed Statement.Schema.
+func columnExists(ctx context.Context, db *gorm.DB, table, column string) (bool, error) {
+ var count int64
+ if err := db.WithContext(ctx).Raw(`
+ SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?
+ `, table, column).Scan(&count).Error; err != nil {
+ return false, err
+ }
+ return count > 0, nil
+}
- // Drop legacy unique index (tenant_id, embd_id) to allow per-space configs.
- var legacyIndexExists int64
- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'skill_search_configs' AND INDEX_NAME = 'idx_tenant_embd'`).Scan(&legacyIndexExists)
- if legacyIndexExists > 0 {
- common.Info("Dropping legacy unique index idx_tenant_embd from skill_search_configs...")
- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs DROP INDEX idx_tenant_embd`).Error; err != nil {
- return fmt.Errorf("failed to drop legacy unique index idx_tenant_embd: %w", err)
- }
- }
+// columnDefinition returns the stored COLUMN_TYPE and whether the column accepts
+// NULL. ok is false when the column does not exist.
+func columnDefinition(ctx context.Context, db *gorm.DB, table, column string) (columnType string, nullable, ok bool, err error) {
+ var row struct {
+ ColumnType string
+ IsNullable string
+ }
+ if err = db.WithContext(ctx).Raw(`
+ SELECT COLUMN_TYPE AS column_type, IS_NULLABLE AS is_nullable
+ FROM INFORMATION_SCHEMA.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?
+ `, table, column).Scan(&row).Error; err != nil {
+ return "", false, false, err
+ }
+ return row.ColumnType, strings.EqualFold(row.IsNullable, "YES"), row.ColumnType != "", nil
+}
- // Table exists, check if unique index exists
- var indexExists int64
- db.WithContext(ctx).Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'skill_search_configs' AND INDEX_NAME = 'idx_tenant_space_embd'`).Scan(&indexExists)
- if indexExists == 0 {
- common.Info("Adding unique index idx_tenant_space_embd to skill_search_configs...")
- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_search_configs
- ADD UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)`).Error; err != nil {
- return fmt.Errorf("failed to add unique index idx_tenant_space_embd: %w", err)
- }
+// isAutoIncrementColumn reports whether the column is declared AUTO_INCREMENT.
+func isAutoIncrementColumn(ctx context.Context, db *gorm.DB, table, column string) (bool, error) {
+ var count int64
+ if err := db.WithContext(ctx).Raw(`
+ SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?
+ AND EXTRA LIKE '%auto_increment%'
+ `, table, column).Scan(&count).Error; err != nil {
+ return false, err
+ }
+ return count > 0, nil
+}
+
+// primaryKeyColumns lists the current PRIMARY KEY columns in key order.
+func primaryKeyColumns(ctx context.Context, db *gorm.DB, table string) ([]string, error) {
+ var columns []string
+ if err := db.WithContext(ctx).Raw(`
+ SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'
+ ORDER BY ORDINAL_POSITION
+ `, table).Scan(&columns).Error; err != nil {
+ return nil, err
+ }
+ return columns, nil
+}
+
+// dropLegacyPrimaryKey removes the table's PRIMARY KEY unless it is already the
+// single-column key being migrated to. MySQL rejects a second PRIMARY KEY
+// definition, so the legacy key must be dropped first. It reports whether
+// keepColumn is the primary key once it returns.
+func dropLegacyPrimaryKey(ctx context.Context, db *gorm.DB, table, keepColumn string) (bool, error) {
+ columns, err := primaryKeyColumns(ctx, db, table)
+ if err != nil {
+ return false, err
+ }
+ if len(columns) == 1 && columns[0] == keepColumn {
+ return true, nil
+ }
+ if len(columns) == 0 {
+ return false, nil
+ }
+ common.Info("Dropping legacy primary key", zap.String("table", table), zap.Strings("columns", columns))
+ if err = db.WithContext(ctx).Exec("ALTER TABLE " + table + " DROP PRIMARY KEY").Error; err != nil {
+ return false, fmt.Errorf("failed to drop the legacy primary key on %s: %w", table, err)
+ }
+ return false, nil
+}
+
+// hasUniqueIndex reports whether a unique index spans exactly the given columns.
+// Matching on the column set rather than on the index name keeps the migrations
+// idempotent when an equivalent index already exists under another name.
+func hasUniqueIndex(ctx context.Context, db *gorm.DB, table string, columns []string) (bool, error) {
+ // The column names come from this file's call sites, never from input.
+ quoted := "'" + strings.Join(columns, "', '") + "'"
+ var count int64
+ // An index only qualifies when its whole column set is the requested one: a
+ // broader unique index such as (email, tenant_id) does not make email unique
+ // on its own. Prefix indexes are excluded because they only constrain a
+ // leading substring.
+ if err := db.WithContext(ctx).Raw(fmt.Sprintf(`
+ SELECT COUNT(*) FROM (
+ SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?
+ AND NON_UNIQUE = 0 AND INDEX_NAME <> 'PRIMARY'
+ AND SUB_PART IS NULL
+ GROUP BY INDEX_NAME
+ HAVING COUNT(*) = %d
+ AND COUNT(CASE WHEN COLUMN_NAME IN (%s) THEN 1 END) = %d
+ ) AS equivalent
+ `, len(columns), quoted, len(columns)), table).Scan(&count).Error; err != nil {
+ return false, err
+ }
+ return count > 0, nil
+}
+
+// addUniqueIndex creates a unique index over the given columns.
+func addUniqueIndex(ctx context.Context, db *gorm.DB, table, indexName string, columns []string) error {
+ sql := fmt.Sprintf("ALTER TABLE %s ADD UNIQUE INDEX %s (%s)", table, indexName, strings.Join(columns, ", "))
+ if err := db.WithContext(ctx).Exec(sql).Error; err != nil {
+ if isDuplicateIndexErr(err) {
+ common.Info("Index already exists, skipping", zap.String("index", indexName))
+ return nil
}
+ return fmt.Errorf("failed to add unique index %s on %s: %w", indexName, table, err)
}
+ return nil
+}
+// dropIndexIfExists removes a named index when it is present.
+func dropIndexIfExists(ctx context.Context, db *gorm.DB, table, indexName string) error {
+ var count int64
+ if err := db.WithContext(ctx).Raw(`
+ SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?
+ `, table, indexName).Scan(&count).Error; err != nil {
+ return err
+ }
+ if count == 0 {
+ return nil
+ }
+ if err := db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DROP INDEX %s", table, indexName)).Error; err != nil {
+ return fmt.Errorf("failed to drop index %s on %s: %w", indexName, table, err)
+ }
return nil
}
-// migrateSkillSpaceTables creates skill space related tables
-func migrateSkillSpaceTables(ctx context.Context, db *gorm.DB) error {
- if !db.WithContext(ctx).Migrator().HasTable("skill_spaces") {
- common.Info("Creating skill_spaces table...")
- sql := `
- CREATE TABLE IF NOT EXISTS skill_spaces (
- id VARCHAR(32) PRIMARY KEY,
- tenant_id VARCHAR(32) NOT NULL,
- name VARCHAR(128) NOT NULL,
- folder_id VARCHAR(32) NOT NULL,
- description TEXT,
- embd_id VARCHAR(128),
- rerank_id VARCHAR(128),
- top_k INT DEFAULT 10,
- status VARCHAR(1) DEFAULT '1',
- create_time BIGINT,
- create_date DATETIME,
- update_time BIGINT,
- update_date DATETIME,
- INDEX idx_tenant_id (tenant_id),
- UNIQUE INDEX idx_tenant_name_status (tenant_id, name, status)
- )
- `
- if err := db.WithContext(ctx).Exec(sql).Error; err != nil {
- common.Warn("Failed to create skill_spaces table with MySQL dialect, trying generic", zap.Error(err))
- // Try with AutoMigrate as fallback
- if err = db.WithContext(ctx).AutoMigrate(&entity.SkillSpace{}); err != nil {
- return err
- }
- // AutoMigrate doesn't create unique indexes, so create them explicitly
- common.Info("Creating unique indexes for skill_spaces...")
- if err = db.WithContext(ctx).Exec(`ALTER TABLE skill_spaces ADD UNIQUE INDEX idx_tenant_name_status (tenant_id, name, status)`).Error; err != nil {
- return fmt.Errorf("failed to create unique index idx_tenant_name_status: %w", err)
- }
- }
- } else {
- // Migrate existing table: add status column first, then update index
- if err := addColumnIfNotExists(ctx, db, "skill_spaces", "status", "VARCHAR(1) NOT NULL DEFAULT '1'"); err != nil {
- return fmt.Errorf("failed to add status column to skill_spaces: %w", err)
- }
- if err := addColumnIfNotExists(ctx, db, "skill_spaces", "create_date", "DATETIME"); err != nil {
- return fmt.Errorf("failed to add create_date column to skill_spaces: %w", err)
- }
- if err := addColumnIfNotExists(ctx, db, "skill_spaces", "update_date", "DATETIME"); err != nil {
- return fmt.Errorf("failed to add update_date column to skill_spaces: %w", err)
+// dropColumnIfExists removes a column when it is present.
+func dropColumnIfExists(ctx context.Context, db *gorm.DB, table, column string) error {
+ exists, err := columnExists(ctx, db, table, column)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ return nil
+ }
+ common.Info("Dropping column", zap.String("table", table), zap.String("column", column))
+ if err = db.WithContext(ctx).Exec(fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", table, column)).Error; err != nil {
+ return fmt.Errorf("failed to drop column %s on %s: %w", column, table, err)
+ }
+ return nil
+}
+
+// isDuplicateIndexErr reports whether the error is MySQL error 1061 (duplicate
+// key name), which means the index already exists.
+func isDuplicateIndexErr(err error) bool {
+ return strings.Contains(err.Error(), "Error 1061") || strings.Contains(err.Error(), "Duplicate key name")
+}
+
+// renameDuplicateEmails renames every user row that shares an address with
+// another row, leaving the address on the superuser - or on the oldest row when
+// the address has no superuser. The unique index is only creatable once no
+// duplicates remain.
+func renameDuplicateEmails(ctx context.Context, db *gorm.DB) error {
+ var duplicates []string
+ if err := db.WithContext(ctx).Raw(`
+ SELECT email FROM user GROUP BY email HAVING COUNT(*) > 1
+ `).Scan(&duplicates).Error; err != nil {
+ return err
+ }
+ if len(duplicates) == 0 {
+ return nil
+ }
+
+ common.Warn("Renaming duplicate user emails before adding the unique index", zap.Int("addresses", len(duplicates)))
+ for _, email := range duplicates {
+ var ids []string
+ if err := db.WithContext(ctx).Raw(`
+ SELECT id FROM user WHERE email = ?
+ ORDER BY is_superuser DESC, create_time ASC
+ `, email).Scan(&ids).Error; err != nil {
+ return err
}
- if err := db.WithContext(ctx).Exec(`ALTER TABLE skill_spaces MODIFY COLUMN update_time BIGINT`).Error; err != nil {
- common.Warn("Failed to modify skill_spaces.update_time", zap.Error(err))
+ if len(ids) < 2 {
+ continue
}
- // Migrate index after status column exists
- if err := migrateSkillSpaceIndex(ctx, db); err != nil {
- return fmt.Errorf("failed to migrate skill_space index: %w", err)
+ for _, id := range ids[1:] {
+ if err := db.WithContext(ctx).Exec(
+ `UPDATE user SET email = ? WHERE id = ?`,
+ duplicateEmailAddress(email, id), id,
+ ).Error; err != nil {
+ return fmt.Errorf("failed to rename duplicate email %q: %w", email, err)
+ }
}
}
-
return nil
}
-// migrateSkillSpaceIndex migrates the unique index to include status
-func migrateSkillSpaceIndex(ctx context.Context, db *gorm.DB) error {
- // Check if old index exists and drop it
- var oldIndexExists int64
- db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'skill_spaces' AND INDEX_NAME = 'idx_tenant_name'
- `).Scan(&oldIndexExists)
+// duplicateEmailAddress builds the placeholder address for a shadowed duplicate
+// row, clamped to the column width so the rename cannot fail on length.
+func duplicateEmailAddress(email, id string) string {
+ const (
+ maxEmailLength = 255
+ duplicateTag = "_DUPLICATE_"
+ idPrefixLength = 8
+ )
+ if len(id) > idPrefixLength {
+ id = id[:idPrefixLength]
+ }
+ suffix := duplicateTag + id
+ runes := []rune(email)
+ if keep := maxEmailLength - len(suffix); len(runes) > keep {
+ runes = runes[:keep]
+ }
+ return string(runes) + suffix
+}
- if oldIndexExists > 0 {
- common.Info("Dropping old idx_tenant_name index from skill_spaces...")
- if err := db.WithContext(ctx).Exec(`DROP INDEX idx_tenant_name ON skill_spaces`).Error; err != nil {
- return fmt.Errorf("failed to drop old index idx_tenant_name: %w", err)
- }
+// integerDisplayWidth matches the deprecated integer display width emitted by
+// MySQL 5.7 ("int(11)"), which MySQL 8 accepts but no longer reports.
+var integerDisplayWidth = regexp.MustCompile(`^(tinyint|smallint|mediumint|int|bigint)\(\d+\)$`)
+
+// normalizeColumnType strips integer display widths so a stored type compares
+// equal to its canonical name.
+func normalizeColumnType(columnType string) string {
+ lowered := strings.ToLower(strings.TrimSpace(columnType))
+ if match := integerDisplayWidth.FindStringSubmatch(lowered); match != nil {
+ return match[1]
}
+ return lowered
+}
- // Check if new index exists
- var newIndexExists int64
- db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
- WHERE TABLE_NAME = 'skill_spaces' AND INDEX_NAME = 'idx_tenant_name_status'
- `).Scan(&newIndexExists)
+// columnSpec is the definition a column is expected to have.
+type columnSpec struct {
+ table string
+ column string
+ columnType string // compared against INFORMATION_SCHEMA.COLUMNS.COLUMN_TYPE
+ nullable bool
+ definition string // used verbatim when an ALTER is required
+}
- if newIndexExists == 0 {
- common.Info("Creating new idx_tenant_name_status index on skill_spaces...")
- if err := db.WithContext(ctx).Exec(`CREATE UNIQUE INDEX idx_tenant_name_status ON skill_spaces(tenant_id, name, status)`).Error; err != nil {
- return fmt.Errorf("failed to create unique index idx_tenant_name_status: %w", err)
- }
+// applyColumnSpec aligns a column with its expected definition, issuing DDL only
+// when the stored definition differs so that a steady-state startup runs no
+// ALTER at all.
+func applyColumnSpec(ctx context.Context, db *gorm.DB, spec columnSpec) error {
+ if !db.WithContext(ctx).Migrator().HasTable(spec.table) {
+ return nil
+ }
+
+ currentType, nullable, exists, err := columnDefinition(ctx, db, spec.table, spec.column)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ return nil
+ }
+ if normalizeColumnType(currentType) == normalizeColumnType(spec.columnType) && nullable == spec.nullable {
+ return nil
}
+ common.Info("Modifying column type",
+ zap.String("table", spec.table),
+ zap.String("column", spec.column),
+ zap.String("from", currentType),
+ zap.String("to", spec.columnType))
+ if err = db.WithContext(ctx).Exec(
+ fmt.Sprintf("ALTER TABLE %s MODIFY COLUMN %s %s", spec.table, spec.column, spec.definition),
+ ).Error; err != nil {
+ common.Warn("Failed to modify column",
+ zap.String("table", spec.table),
+ zap.String("column", spec.column),
+ zap.Error(err))
+ }
return nil
}
diff --git a/internal/dao/migration_version.go b/internal/dao/migration_version.go
new file mode 100644
index 0000000000..5c2fa5604b
--- /dev/null
+++ b/internal/dao/migration_version.go
@@ -0,0 +1,125 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "strings"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+
+ "go.uber.org/zap"
+ "golang.org/x/mod/semver"
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+)
+
+// migrationDBVersionMarker is the system_settings key that records how far the
+// database has been migrated. It is shared with tools/scripts/mysql_migration.py
+// so the Python and Go migrations can skip work the other has already done.
+const migrationDBVersionMarker = "mysql_migration.database.version"
+
+// modelMigrationBaseVersion is the version the base tenant model table
+// migration brings the database to. It mirrors the first --database-version used
+// by tools/scripts/run_migrations.sh, so a database already carrying it came
+// from a v0.26.0 release and only needs the remaining model data stages.
+const modelMigrationBaseVersion = "v0.26.0"
+
+// modelMigrationTargetVersion is the version the remaining tenant model data
+// migration brings the database to. It mirrors the final --database-version used
+// by tools/scripts/run_migrations.sh.
+const modelMigrationTargetVersion = "v0.27.2"
+
+// GetDatabaseMigrationVersion reads the stored migration version. It returns an
+// empty string when the marker or the system_settings table is absent.
+func GetDatabaseMigrationVersion(ctx context.Context, db *gorm.DB) (string, error) {
+ scoped := db.WithContext(ctx)
+ if !scoped.Migrator().HasTable("system_settings") {
+ common.Info("Table 'system_settings' does not exist, migration version marker is unavailable")
+ return "", nil
+ }
+
+ var value string
+ err := scoped.Raw("SELECT `value` FROM `system_settings` WHERE `name` = ?", migrationDBVersionMarker).Row().Scan(&value)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return "", nil
+ }
+ return "", err
+ }
+ return value, nil
+}
+
+// setDatabaseMigrationVersion upserts the migration version marker.
+func setDatabaseMigrationVersion(ctx context.Context, db *gorm.DB, version string) error {
+ scoped := db.WithContext(ctx)
+ if !scoped.Migrator().HasTable("system_settings") {
+ common.Warn("Table 'system_settings' does not exist, migration version marker was not saved")
+ return nil
+ }
+
+ nowMs := time.Now().UnixMilli()
+ now := time.Now()
+ setting := entity.SystemSettings{
+ Name: migrationDBVersionMarker,
+ Source: "migration",
+ DataType: "string",
+ Value: version,
+ BaseModel: entity.BaseModel{
+ CreateTime: &nowMs,
+ CreateDate: &now,
+ UpdateTime: &nowMs,
+ UpdateDate: &now,
+ },
+ }
+ return scoped.Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "name"}},
+ DoUpdates: clause.AssignmentColumns([]string{"source", "data_type", "value", "update_time", "update_date"}),
+ }).Create(&setting).Error
+}
+
+// shouldSkipMigration mirrors should_skip_migration in the Python migration: a
+// missing or unparseable version never skips the migration.
+func shouldSkipMigration(currentDBVersion, targetVersion string) bool {
+ current := parseMigrationVersion(currentDBVersion)
+ target := parseMigrationVersion(targetVersion)
+ if current == "" || target == "" {
+ return false
+ }
+ return semver.Compare(current, target) >= 0
+}
+
+// parseMigrationVersion normalizes a version to the "vMAJOR.MINOR.PATCH" form
+// understood by semver, returning an empty string when it cannot be parsed.
+func parseMigrationVersion(version string) string {
+ normalized := strings.TrimSpace(version)
+ normalized = strings.TrimPrefix(normalized, "v")
+ normalized = strings.TrimPrefix(normalized, "V")
+ if normalized == "" {
+ return ""
+ }
+ normalized = "v" + normalized
+ if !semver.IsValid(normalized) {
+ common.Warn("Invalid migration version format", zap.String("version", version))
+ return ""
+ }
+ return normalized
+}
diff --git a/internal/dao/migration_version_test.go b/internal/dao/migration_version_test.go
new file mode 100644
index 0000000000..1842ad22d4
--- /dev/null
+++ b/internal/dao/migration_version_test.go
@@ -0,0 +1,158 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+)
+
+func setupMigrationVersionTestDB(t *testing.T, migrate bool) *gorm.DB {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("failed to open sqlite: %v", err)
+ }
+ if migrate {
+ if err = db.AutoMigrate(&entity.SystemSettings{}); err != nil {
+ t.Fatalf("failed to migrate system_settings: %v", err)
+ }
+ }
+ return db
+}
+
+func TestShouldSkipMigration(t *testing.T) {
+ cases := []struct {
+ name string
+ current string
+ target string
+ want bool
+ }{
+ {name: "missing current", current: "", target: "v0.27.1", want: false},
+ {name: "older current", current: "v0.26.0", target: "v0.27.1", want: false},
+ {name: "equal", current: "v0.27.1", target: "v0.27.1", want: true},
+ {name: "newer current", current: "v0.28.0", target: "v0.27.1", want: true},
+ {name: "missing v prefix", current: "0.27.1", target: "v0.27.1", want: true},
+ {name: "invalid current", current: "not-a-version", target: "v0.27.1", want: false},
+ {name: "invalid target", current: "v0.27.1", target: "not-a-version", want: false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := shouldSkipMigration(tc.current, tc.target); got != tc.want {
+ t.Fatalf("shouldSkipMigration(%q, %q) = %v, want %v", tc.current, tc.target, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestGetAndSetDatabaseMigrationVersion(t *testing.T) {
+ db := setupMigrationVersionTestDB(t, true)
+ ctx := t.Context()
+
+ version, err := GetDatabaseMigrationVersion(ctx, db)
+ if err != nil {
+ t.Fatalf("GetDatabaseMigrationVersion() error = %v", err)
+ }
+ if version != "" {
+ t.Fatalf("initial version = %q, want empty", version)
+ }
+
+ if err = setDatabaseMigrationVersion(ctx, db, "v0.27.1"); err != nil {
+ t.Fatalf("setDatabaseMigrationVersion() error = %v", err)
+ }
+ version, err = GetDatabaseMigrationVersion(ctx, db)
+ if err != nil {
+ t.Fatalf("GetDatabaseMigrationVersion() after set error = %v", err)
+ }
+ if version != "v0.27.1" {
+ t.Fatalf("version = %q, want v0.27.1", version)
+ }
+
+ // Upsert must update the existing marker instead of inserting a second row.
+ if err = setDatabaseMigrationVersion(ctx, db, "v0.28.0"); err != nil {
+ t.Fatalf("setDatabaseMigrationVersion() second error = %v", err)
+ }
+ version, err = GetDatabaseMigrationVersion(ctx, db)
+ if err != nil {
+ t.Fatalf("GetDatabaseMigrationVersion() second error = %v", err)
+ }
+ if version != "v0.28.0" {
+ t.Fatalf("version = %q, want v0.28.0", version)
+ }
+
+ var count int64
+ if err = db.Model(&entity.SystemSettings{}).Where("name = ?", migrationDBVersionMarker).Count(&count).Error; err != nil {
+ t.Fatalf("count markers: %v", err)
+ }
+ if count != 1 {
+ t.Fatalf("marker count = %d, want 1", count)
+ }
+}
+
+func TestGetDatabaseMigrationVersionWithoutTable(t *testing.T) {
+ db := setupMigrationVersionTestDB(t, false)
+ ctx := t.Context()
+
+ version, err := GetDatabaseMigrationVersion(ctx, db)
+ if err != nil {
+ t.Fatalf("GetDatabaseMigrationVersion() error = %v", err)
+ }
+ if version != "" {
+ t.Fatalf("version = %q, want empty", version)
+ }
+ if err = setDatabaseMigrationVersion(ctx, db, "v0.27.1"); err != nil {
+ t.Fatalf("setDatabaseMigrationVersion() without table error = %v", err)
+ }
+}
+
+func TestEnsureMigrationVersionTable(t *testing.T) {
+ db := setupMigrationVersionTestDB(t, false)
+ ctx := t.Context()
+
+ if err := ensureMigrationVersionTable(ctx, db); err != nil {
+ t.Fatalf("ensureMigrationVersionTable() error = %v", err)
+ }
+ if !db.Migrator().HasTable("system_settings") {
+ t.Fatal("system_settings was not created")
+ }
+
+ // Idempotent: the existing table is left alone.
+ if err := ensureMigrationVersionTable(ctx, db); err != nil {
+ t.Fatalf("ensureMigrationVersionTable() second error = %v", err)
+ }
+}
+
+func TestMigrateModelDataPersistsVersionOnFirstRun(t *testing.T) {
+ db := setupMigrationVersionTestDB(t, false)
+ ctx := t.Context()
+
+ if err := migrateModelData(ctx, db); err != nil {
+ t.Fatalf("migrateModelData() error = %v", err)
+ }
+ version, err := GetDatabaseMigrationVersion(ctx, db)
+ if err != nil {
+ t.Fatalf("GetDatabaseMigrationVersion() error = %v", err)
+ }
+ if version != modelMigrationTargetVersion {
+ t.Fatalf("version = %q, want %q: the marker must be persisted on the first run so a populated database is never migrated twice",
+ version, modelMigrationTargetVersion)
+ }
+}
diff --git a/internal/dao/model_merge_test.go b/internal/dao/model_merge_test.go
new file mode 100644
index 0000000000..0a4c66e36f
--- /dev/null
+++ b/internal/dao/model_merge_test.go
@@ -0,0 +1,266 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+func setupTenantModelMergeDB(t *testing.T, modelTypeDefinition string) *gorm.DB {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("failed to open sqlite: %v", err)
+ }
+ ddl := fmt.Sprintf(`CREATE TABLE tenant_model (
+ id VARCHAR(32) NOT NULL PRIMARY KEY,
+ model_name VARCHAR(128),
+ provider_id VARCHAR(32) NOT NULL,
+ instance_id VARCHAR(32) NOT NULL,
+ model_type %s NOT NULL,
+ status VARCHAR(32) DEFAULT 'active',
+ extra VARCHAR(1024) DEFAULT '{}',
+ create_time BIGINT,
+ create_date DATETIME,
+ update_time BIGINT,
+ update_date DATETIME
+ )`, modelTypeDefinition)
+ if err = db.Exec(ddl).Error; err != nil {
+ t.Fatalf("failed to create tenant_model: %v", err)
+ }
+ return db
+}
+
+func insertTenantModelMergeRow(t *testing.T, db *gorm.DB, id, model, provider, instance, modelType, status string) {
+ t.Helper()
+ if err := db.Exec(
+ `INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, '{}', 1000, NULL, 1000, NULL)`,
+ id, model, provider, instance, modelType, status,
+ ).Error; err != nil {
+ t.Fatalf("failed to seed tenant_model row: %v", err)
+ }
+}
+
+func TestMergeTenantModelTypes(t *testing.T) {
+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")
+ ctx := t.Context()
+
+ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "active")
+ insertTenantModelMergeRow(t, db, "2", "gpt-4o", "p1", "i1", "vision", "active")
+ insertTenantModelMergeRow(t, db, "3", "m1", "p2", "i2", "embedding", "unsupported")
+ insertTenantModelMergeRow(t, db, "4", "m1", "p2", "i2", "chat", "active")
+ insertTenantModelMergeRow(t, db, "5", "m2", "p3", "i3", "chat", "inactive")
+ // The seeding stage writes the merged integer into the text column.
+ insertTenantModelMergeRow(t, db, "6", "m3", "p4", "i4", "1", "active")
+
+ if err := mergeTenantModelTypes(ctx, db); err != nil {
+ t.Fatalf("mergeTenantModelTypes() error = %v", err)
+ }
+
+ type mergedRow struct {
+ ModelName string `gorm:"column:model_name"`
+ ModelType int `gorm:"column:model_type"`
+ Status string `gorm:"column:status"`
+ }
+ var rows []mergedRow
+ if err := db.Raw("SELECT model_name, model_type, status FROM tenant_model ORDER BY model_name").Scan(&rows).Error; err != nil {
+ t.Fatalf("failed to read merged rows: %v", err)
+ }
+
+ want := []mergedRow{
+ {ModelName: "gpt-4o", ModelType: 9, Status: "active"}, // chat | vision
+ {ModelName: "m1", ModelType: 1, Status: "active"}, // chat, embedding dropped as unsupported
+ {ModelName: "m3", ModelType: 1, Status: "active"}, // numeric text preserved
+ }
+ if len(rows) != len(want) {
+ t.Fatalf("merged row count = %d, want %d (%+v)", len(rows), len(want), rows)
+ }
+ for i := range want {
+ if rows[i] != want[i] {
+ t.Fatalf("merged row %d = %+v, want %+v", i, rows[i], want[i])
+ }
+ }
+}
+
+func TestMergeTenantModelTypesSkipsIntegerColumn(t *testing.T) {
+ db := setupTenantModelMergeDB(t, "INT")
+ ctx := t.Context()
+
+ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "1", "active")
+ insertTenantModelMergeRow(t, db, "2", "gpt-4o", "p1", "i1", "1", "active")
+
+ if err := mergeTenantModelTypes(ctx, db); err != nil {
+ t.Fatalf("mergeTenantModelTypes() error = %v", err)
+ }
+
+ var count int64
+ if err := db.Raw("SELECT COUNT(*) FROM tenant_model").Scan(&count).Error; err != nil {
+ t.Fatalf("failed to count rows: %v", err)
+ }
+ if count != 2 {
+ t.Fatalf("row count = %d, want 2 (integer column must be left untouched)", count)
+ }
+}
+
+func TestMergeTenantModelTypesPreservesTimestamps(t *testing.T) {
+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")
+ ctx := t.Context()
+
+ if err := db.Exec(
+ `INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, '{}', ?, ?, ?, ?)`,
+ "1", "gpt-4o", "p1", "i1", "chat", "active", 1700000000000, "2023-11-14 22:13:20", 1700000000000, "2023-11-14 22:13:20",
+ ).Error; err != nil {
+ t.Fatalf("failed to seed tenant_model row: %v", err)
+ }
+
+ if err := mergeTenantModelTypes(ctx, db); err != nil {
+ t.Fatalf("mergeTenantModelTypes() error = %v", err)
+ }
+
+ var row struct {
+ CreateTime int64 `gorm:"column:create_time"`
+ UpdateTime int64 `gorm:"column:update_time"`
+ }
+ if err := db.Raw("SELECT create_time, update_time FROM tenant_model").Scan(&row).Error; err != nil {
+ t.Fatalf("failed to read merged row: %v", err)
+ }
+ if row.CreateTime != 1700000000000 || row.UpdateTime != 1700000000000 {
+ t.Fatalf("merged timestamps = (%d, %d), want (1700000000000, 1700000000000)", row.CreateTime, row.UpdateTime)
+ }
+}
+
+func TestMergeTenantModelTypesClearsDroppedRows(t *testing.T) {
+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")
+ ctx := t.Context()
+
+ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "inactive")
+
+ if err := mergeTenantModelTypes(ctx, db); err != nil {
+ t.Fatalf("mergeTenantModelTypes() error = %v", err)
+ }
+
+ var count int64
+ if err := db.Raw("SELECT COUNT(*) FROM tenant_model").Scan(&count).Error; err != nil {
+ t.Fatalf("failed to count rows: %v", err)
+ }
+ if count != 0 {
+ t.Fatalf("row count = %d, want 0 (dropped groups must not stay in their text shape)", count)
+ }
+}
+
+func TestEnsureTenantModelTables(t *testing.T) {
+ t.Run("creates the missing tables without touching a text tenant_model", func(t *testing.T) {
+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")
+ if err := db.Exec("CREATE TABLE tenant_llm (id VARCHAR(32) NOT NULL PRIMARY KEY)").Error; err != nil {
+ t.Fatalf("failed to create tenant_llm: %v", err)
+ }
+ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "active")
+
+ if err := ensureTenantModelTables(t.Context(), db); err != nil {
+ t.Fatalf("ensureTenantModelTables() error = %v", err)
+ }
+
+ for _, table := range []string{"tenant_model_provider", "tenant_model_instance"} {
+ if !db.Migrator().HasTable(table) {
+ t.Fatalf("%s was not created", table)
+ }
+ }
+ var modelType string
+ if err := db.Raw("SELECT model_type FROM tenant_model").Scan(&modelType).Error; err != nil {
+ t.Fatalf("failed to read model_type: %v", err)
+ }
+ if modelType != "chat" {
+ t.Fatalf("model_type = %q, want the untouched text value \"chat\"", modelType)
+ }
+ })
+
+ t.Run("leaves the database alone without tenant_llm", func(t *testing.T) {
+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")
+
+ if err := ensureTenantModelTables(t.Context(), db); err != nil {
+ t.Fatalf("ensureTenantModelTables() error = %v", err)
+ }
+ if db.Migrator().HasTable("tenant_model_provider") {
+ t.Fatal("tenant_model_provider must not be created without tenant_llm")
+ }
+ })
+}
+
+func TestLoadTenantModelRowMapParsesTextModelType(t *testing.T) {
+ db := setupTenantModelMergeDB(t, "VARCHAR(32)")
+ insertTenantModelMergeRow(t, db, "1", "gpt-4o", "p1", "i1", "chat", "active")
+ insertTenantModelMergeRow(t, db, "2", "m1", "p2", "i2", "9", "active")
+
+ rows, err := loadTenantModelRowMap(db)
+ if err != nil {
+ t.Fatalf("loadTenantModelRowMap() error = %v", err)
+ }
+ if got := rows["p1\x00i1\x00gpt-4o"].modelType; got != 1 {
+ t.Fatalf("gpt-4o modelType = %d, want 1 (text name)", got)
+ }
+ if got := rows["p2\x00i2\x00m1"].modelType; got != 9 {
+ t.Fatalf("m1 modelType = %d, want 9 (numeric text)", got)
+ }
+}
+
+func TestTenantModelModelTypeIsInteger(t *testing.T) {
+ cases := []struct {
+ name string
+ definition string
+ want bool
+ }{
+ {name: "text", definition: "VARCHAR(32)", want: false},
+ {name: "integer", definition: "INT", want: true},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ db := setupTenantModelMergeDB(t, tc.definition)
+ got, err := tenantModelModelTypeIsInteger(db)
+ if err != nil {
+ t.Fatalf("tenantModelModelTypeIsInteger() error = %v", err)
+ }
+ if got != tc.want {
+ t.Fatalf("tenantModelModelTypeIsInteger() = %v, want %v", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestMergeTenantModelTypeBits(t *testing.T) {
+ cases := []struct {
+ raw string
+ want int
+ }{
+ {raw: "chat", want: 1},
+ {raw: "Vision", want: 8},
+ {raw: "speech2text", want: 4},
+ {raw: " 7 ", want: 7},
+ {raw: "unknown", want: 0},
+ {raw: "", want: 0},
+ }
+ for _, tc := range cases {
+ t.Run(tc.raw, func(t *testing.T) {
+ if got := mergeTenantModelTypeBits(tc.raw); got != tc.want {
+ t.Fatalf("mergeTenantModelTypeBits(%q) = %d, want %d", tc.raw, got, tc.want)
+ }
+ })
+ }
+}
diff --git a/internal/dao/model_migration.go b/internal/dao/model_migration.go
new file mode 100644
index 0000000000..822438d4c3
--- /dev/null
+++ b/internal/dao/model_migration.go
@@ -0,0 +1,1556 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "crypto/rand"
+ "database/sql"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+
+ "go.uber.org/zap"
+ "gorm.io/gorm"
+)
+
+// This file ports tools/scripts/mysql_migration.py. It backfills the tenant
+// model tables (tenant_model_provider, tenant_model_instance, tenant_model)
+// from the legacy tenant_llm table, rewrites stored model ids from
+// "<model>@<provider>" to "<model>@default@<provider>", and populates the
+// tenant_*_id columns. Every step is idempotent, so re-running a stage is safe,
+// and the two steps are gated by the database version in system_settings,
+// mirroring the two --database-version calls in tools/scripts/run_migrations.sh.
+
+const modelMigrationInstanceName = "default"
+
+// modelTypeStringToBit mirrors MODEL_TYPE_TO_INT in the Python migration and
+// entity.ModelType's bit layout.
+var modelTypeStringToBit = map[string]int{
+ "chat": 1,
+ "embedding": 2,
+ "asr": 4,
+ "speech2text": 4,
+ "vision": 8,
+ "image2text": 8,
+ "rerank": 16,
+ "tts": 32,
+ "ocr": 64,
+}
+
+// ensureTenantModelTables creates the tenant model tables the base step backfills
+// when they do not exist yet and the legacy tenant_llm table does. It mirrors the
+// CREATE TABLE IF NOT EXISTS that the Python stages run before inserting.
+//
+// Existing tables are never altered: tenant_model.model_type must keep the text
+// shape the Python stage left behind so mergeTenantModelTypes can still read the
+// legacy model names instead of the integer AutoMigrate would converge it to.
+func ensureTenantModelTables(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ if !scoped.Migrator().HasTable("tenant_llm") {
+ return nil
+ }
+ for _, model := range []any{&entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{}} {
+ if scoped.Migrator().HasTable(model) {
+ continue
+ }
+ if err := scoped.Migrator().CreateTable(model); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// ensureMigrationVersionTable creates system_settings when it is missing. The
+// model data migration records its progress there and runs before AutoMigrate
+// (see InitDB), so on a database that predates the table the version marker could
+// not be written. Every later startup would then replay the migration against
+// tenant_model rows that already hold data.
+func ensureMigrationVersionTable(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ if scoped.Migrator().HasTable("system_settings") {
+ return nil
+ }
+ return scoped.Migrator().CreateTable(&entity.SystemSettings{})
+}
+
+// migrateModelData runs the model related data migrations in the two versioned
+// steps used by tools/scripts/run_migrations.sh. Each step records its own
+// version in migrationDBVersionMarker, so work already performed by the Python
+// script, or by a previous Go run, is skipped.
+//
+// - The base step (modelMigrationBaseVersion) rebuilds
+// tenant_model_provider/tenant_model_instance/tenant_model from tenant_llm
+// and normalizes the stored model ids. It is only needed on a database older
+// than v0.26.0.
+// - The follow-up step (modelMigrationTargetVersion) seeds the factory
+// declared models, merges the tenant_model rows into the integer model_type
+// representation and populates the tenant_*_id columns. It is needed on
+// every database older than v0.27.2, including one already at v0.26.0.
+//
+// The step runs before AutoMigrate (see InitDB), so it creates the tables it
+// depends on itself: system_settings for the version marker, and, on a database
+// older than v0.26.0, the tenant model tables it backfills from tenant_llm.
+func migrateModelData(ctx context.Context, db *gorm.DB) error {
+ if err := ensureMigrationVersionTable(ctx, db); err != nil {
+ return fmt.Errorf("ensure migration version table: %w", err)
+ }
+ if err := ensureTenantModelTables(ctx, db); err != nil {
+ return fmt.Errorf("ensure tenant model tables: %w", err)
+ }
+
+ currentVersion, err := GetDatabaseMigrationVersion(ctx, db)
+ if err != nil {
+ return fmt.Errorf("read database migration version: %w", err)
+ }
+
+ if shouldSkipMigration(currentVersion, modelMigrationBaseVersion) {
+ common.Info("Tenant model tables already migrated, skipping base step",
+ zap.String("current_version", currentVersion),
+ zap.String("step_version", modelMigrationBaseVersion))
+ } else {
+ common.Info("Running base tenant model table migration",
+ zap.String("current_version", currentVersion),
+ zap.String("step_version", modelMigrationBaseVersion))
+
+ if err := migrateTenantModelProviders(ctx, db); err != nil {
+ return fmt.Errorf("migrate tenant_model_provider: %w", err)
+ }
+ if err := migrateTenantModelInstances(ctx, db); err != nil {
+ return fmt.Errorf("migrate tenant_model_instance: %w", err)
+ }
+ if err := migrateTenantModels(ctx, db); err != nil {
+ return fmt.Errorf("migrate tenant_model: %w", err)
+ }
+ if err := normalizeStoredModelIDs(ctx, db); err != nil {
+ return fmt.Errorf("normalize model ids: %w", err)
+ }
+ if err := setDatabaseMigrationVersion(ctx, db, modelMigrationBaseVersion); err != nil {
+ return fmt.Errorf("mark database migration version %s: %w", modelMigrationBaseVersion, err)
+ }
+ currentVersion = modelMigrationBaseVersion
+ }
+
+ if shouldSkipMigration(currentVersion, modelMigrationTargetVersion) {
+ common.Info("Tenant model data already migrated, skipping",
+ zap.String("current_version", currentVersion),
+ zap.String("target_version", modelMigrationTargetVersion))
+ return nil
+ }
+ common.Info("Running tenant model data migration",
+ zap.String("current_version", currentVersion),
+ zap.String("target_version", modelMigrationTargetVersion))
+
+ if err := seedTenantModels(ctx, db); err != nil {
+ return fmt.Errorf("seed tenant_model: %w", err)
+ }
+ if err := mergeTenantModelTypes(ctx, db); err != nil {
+ return fmt.Errorf("merge tenant_model model types: %w", err)
+ }
+ if err := populateTenantModelIDColumns(ctx, db); err != nil {
+ return fmt.Errorf("populate tenant model id columns: %w", err)
+ }
+
+ if err := setDatabaseMigrationVersion(ctx, db, modelMigrationTargetVersion); err != nil {
+ return fmt.Errorf("mark database migration version: %w", err)
+ }
+ common.Info("Tenant model data migration completed", zap.String("version", modelMigrationTargetVersion))
+ return nil
+}
+
+// modelMigrationNewID returns a 32 character hex id, matching uuid.uuid1().hex.
+func modelMigrationNewID() string {
+ buffer := make([]byte, 16)
+ if _, err := rand.Read(buffer); err != nil {
+ return fmt.Sprintf("%032x", time.Now().UnixNano())
+ }
+ return hex.EncodeToString(buffer)
+}
+
+// modelStatusIsActive mirrors the status mapping used by the Python stages:
+// "active" if status in ("1", "active", "enable") else "inactive".
+func modelStatusIsActive(status string) bool {
+ switch strings.ToLower(strings.TrimSpace(status)) {
+ case "1", "active", "enable":
+ return true
+ default:
+ return false
+ }
+}
+
+func modelStatusValue(status string) string {
+ if modelStatusIsActive(status) {
+ return "active"
+ }
+ return "inactive"
+}
+
+func isAPIKeyFieldSubset(parsed map[string]any) bool {
+ for key := range parsed {
+ if key != "api_key" && key != "is_tools" {
+ return false
+ }
+ }
+ return true
+}
+
+// stripIsToolsFromAPIKey mirrors MigrationStage._strip_is_tools_from_api_key.
+// It is the canonical form used to match an api_key against stored instances.
+func stripIsToolsFromAPIKey(apiKey string) string {
+ if !strings.HasPrefix(apiKey, "{") {
+ return apiKey
+ }
+ var parsed map[string]any
+ if err := json.Unmarshal([]byte(apiKey), &parsed); err != nil {
+ return apiKey
+ }
+ if isAPIKeyFieldSubset(parsed) {
+ if inner, ok := parsed["api_key"].(string); ok {
+ return inner
+ }
+ return apiKey
+ }
+ if _, ok := parsed["is_tools"]; !ok {
+ return apiKey
+ }
+ delete(parsed, "is_tools")
+ encoded, err := json.Marshal(parsed)
+ if err != nil {
+ return apiKey
+ }
+ return string(encoded)
+}
+
+// extractExtraFromAPIKey mirrors TenantModelStage._extract_extra_from_api_key.
+func extractExtraFromAPIKey(apiKey string) string {
+ if !strings.HasPrefix(apiKey, "{") {
+ return "{}"
+ }
+ var parsed map[string]any
+ if err := json.Unmarshal([]byte(apiKey), &parsed); err != nil {
+ return "{}"
+ }
+ if isTools, ok := parsed["is_tools"].(bool); ok && isTools {
+ return `{"is_tools":true}`
+ }
+ return "{}"
+}
+
+// tenantLLMSourceRecord is a projected row of the legacy tenant_llm table.
+type tenantLLMSourceRecord struct {
+ TenantID string `gorm:"column:tenant_id"`
+ Factory string `gorm:"column:llm_factory"`
+ ModelName sql.NullString `gorm:"column:model_name"`
+ ModelType sql.NullString `gorm:"column:model_type"`
+ APIKey sql.NullString `gorm:"column:api_key"`
+ Status sql.NullString `gorm:"column:status"`
+ ProviderID string `gorm:"column:provider_id"`
+}
+
+// migrateTenantModelProviders ports TenantModelProviderStage: one provider row
+// per distinct (tenant_id, llm_factory) found in tenant_llm.
+func migrateTenantModelProviders(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ if !scoped.Migrator().HasTable("tenant_llm") || !scoped.Migrator().HasTable("tenant_model_provider") {
+ return nil
+ }
+
+ var rows []struct {
+ TenantID string `gorm:"column:tenant_id"`
+ Factory string `gorm:"column:llm_factory"`
+ }
+ if err := scoped.Raw(`
+ SELECT DISTINCT t1.tenant_id AS tenant_id, t1.llm_factory AS llm_factory
+ FROM tenant_llm t1
+ WHERE NOT EXISTS (
+ SELECT 1 FROM tenant_model_provider t2
+ WHERE t2.tenant_id = t1.tenant_id AND t2.provider_name = t1.llm_factory
+ )`).Scan(&rows).Error; err != nil {
+ return err
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+
+ nowMs := time.Now().UnixMilli()
+ now := time.Now()
+ if err := scoped.Transaction(func(tx *gorm.DB) error {
+ for _, row := range rows {
+ if err := tx.Exec(
+ "INSERT INTO tenant_model_provider (id, provider_name, tenant_id, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?)",
+ modelMigrationNewID(), row.Factory, row.TenantID, nowMs, now, nowMs, now,
+ ).Error; err != nil {
+ return err
+ }
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ common.Info("Migrated tenant_model_provider records from tenant_llm", zap.Int("count", len(rows)))
+ return nil
+}
+
+// migrateTenantModelInstances ports TenantModelInstanceStage: one instance row
+// per (tenant_id, llm_factory) taking the first api_key value after dedup.
+func migrateTenantModelInstances(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ for _, table := range []string{"tenant_llm", "tenant_model_provider", "tenant_model_instance"} {
+ if !scoped.Migrator().HasTable(table) {
+ return nil
+ }
+ }
+
+ existing, err := buildTenantModelInstanceCanonicalSet(ctx, scoped)
+ if err != nil {
+ return err
+ }
+
+ var rows []tenantLLMSourceRecord
+ if err := scoped.Raw(`
+ SELECT tl.tenant_id AS tenant_id, tl.llm_factory AS llm_factory, tl.api_key AS api_key,
+ MAX(tl.status) AS status, tmp.id AS provider_id
+ FROM tenant_llm tl
+ INNER JOIN tenant_model_provider tmp
+ ON tmp.tenant_id = tl.tenant_id AND tmp.provider_name = tl.llm_factory
+ WHERE NOT EXISTS (
+ SELECT 1 FROM tenant_model_instance tmi
+ WHERE tmi.provider_id = tmp.id AND tmi.api_key = tl.api_key
+ )
+ GROUP BY tl.tenant_id, tl.llm_factory, tl.api_key, tmp.id`).Scan(&rows).Error; err != nil {
+ return err
+ }
+
+ rows = filterExistingTenantModelInstances(rows, existing)
+ rows = dedupTenantModelInstances(rows)
+ if len(rows) == 0 {
+ return nil
+ }
+
+ nowMs := time.Now().UnixMilli()
+ now := time.Now()
+ if err := scoped.Transaction(func(tx *gorm.DB) error {
+ for _, row := range rows {
+ if err := tx.Exec(
+ "INSERT INTO tenant_model_instance (id, instance_name, provider_id, api_key, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ modelMigrationNewID(), modelMigrationInstanceName, row.ProviderID, row.APIKey.String,
+ modelStatusValue(row.Status.String), "{}", nowMs, now, nowMs, now,
+ ).Error; err != nil {
+ return err
+ }
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ common.Info("Migrated tenant_model_instance records from tenant_llm", zap.Int("count", len(rows)))
+ return nil
+}
+
+// buildTenantModelInstanceCanonicalSet collects the canonical api_key already
+// stored for each provider so repeated runs do not insert duplicate instances.
+func buildTenantModelInstanceCanonicalSet(ctx context.Context, db *gorm.DB) (map[string]bool, error) {
+ var rows []struct {
+ ProviderID string `gorm:"column:provider_id"`
+ APIKey sql.NullString `gorm:"column:api_key"`
+ }
+ if err := db.Raw("SELECT provider_id, api_key FROM tenant_model_instance").Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+ existing := make(map[string]bool, len(rows))
+ for _, row := range rows {
+ existing[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)] = true
+ }
+ return existing, nil
+}
+
+func filterExistingTenantModelInstances(rows []tenantLLMSourceRecord, existing map[string]bool) []tenantLLMSourceRecord {
+ if len(existing) == 0 {
+ return rows
+ }
+ filtered := make([]tenantLLMSourceRecord, 0, len(rows))
+ for _, row := range rows {
+ if existing[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)] {
+ continue
+ }
+ filtered = append(filtered, row)
+ }
+ return filtered
+}
+
+// dedupTenantModelInstances keeps a single instance per provider, dropping rows
+// whose canonical api_key repeats within the same (tenant, factory, provider).
+func dedupTenantModelInstances(rows []tenantLLMSourceRecord) []tenantLLMSourceRecord {
+ groups := make(map[string][]tenantLLMSourceRecord)
+ order := make([]string, 0, len(rows))
+ for _, row := range rows {
+ key := row.TenantID + "\x00" + row.Factory + "\x00" + row.ProviderID
+ if _, ok := groups[key]; !ok {
+ order = append(order, key)
+ }
+ groups[key] = append(groups[key], row)
+ }
+
+ result := make([]tenantLLMSourceRecord, 0, len(rows))
+ for _, key := range order {
+ group := groups[key]
+ seen := make(map[string]bool, len(group))
+ for _, row := range group {
+ canonical := stripIsToolsFromAPIKey(row.APIKey.String)
+ if seen[canonical] {
+ continue
+ }
+ seen[canonical] = true
+ result = append(result, row)
+ }
+ }
+ return result
+}
+
+// migrateTenantModels ports TenantModelStage combined with ModelTypeMergeStage.
+// tenant_model.model_type is an integer column, so the merged representation is
+// written directly: only groups whose merged status is "active" are kept and
+// model_type is the OR of the contributing model bits.
+func migrateTenantModels(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ for _, table := range []string{"tenant_llm", "tenant_model_provider", "tenant_model_instance", "tenant_model"} {
+ if !scoped.Migrator().HasTable(table) {
+ return nil
+ }
+ }
+
+ instanceLookup, err := buildTenantModelInstanceLookup(ctx, scoped)
+ if err != nil {
+ return err
+ }
+ if len(instanceLookup) == 0 {
+ return nil
+ }
+
+ query := fmt.Sprintf(`
+ SELECT tl.llm_name AS model_name, tmp.id AS provider_id, tl.model_type AS model_type,
+ tl.status AS status, tl.api_key AS api_key
+ FROM tenant_llm tl
+ INNER JOIN tenant_model_provider tmp
+ ON tmp.tenant_id = tl.tenant_id AND tmp.provider_name = tl.llm_factory
+ WHERE %s
+ AND NOT EXISTS (
+ SELECT 1 FROM tenant_model tm
+ WHERE tm.provider_id = tmp.id AND tm.model_name = tl.llm_name
+ )`, buildTenantModelStatusCondition())
+
+ var rows []tenantLLMSourceRecord
+ if err := scoped.Raw(query).Scan(&rows).Error; err != nil {
+ return err
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+
+ type modelGroup struct {
+ providerID string
+ instanceID string
+ modelName string
+ modelType int
+ status string
+ extra string
+ }
+
+ groups := make(map[string]*modelGroup)
+ order := make([]string, 0, len(rows))
+ for _, row := range rows {
+ instanceID := instanceLookup[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)]
+ if instanceID == "" {
+ continue
+ }
+ key := row.ProviderID + "\x00" + instanceID + "\x00" + row.ModelName.String
+ group, ok := groups[key]
+ if !ok {
+ group = &modelGroup{
+ providerID: row.ProviderID,
+ instanceID: instanceID,
+ modelName: row.ModelName.String,
+ status: modelStatusValue(row.Status.String),
+ extra: extractExtraFromAPIKey(row.APIKey.String),
+ }
+ groups[key] = group
+ order = append(order, key)
+ }
+ group.modelType |= modelTypeStringToBit[strings.ToLower(strings.TrimSpace(row.ModelType.String))]
+ }
+
+ nowMs := time.Now().UnixMilli()
+ now := time.Now()
+ inserted := 0
+ if err := scoped.Transaction(func(tx *gorm.DB) error {
+ for _, key := range order {
+ group := groups[key]
+ if group.status != "active" {
+ continue
+ }
+ if err := tx.Exec(
+ "INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ modelMigrationNewID(), group.modelName, group.providerID, group.instanceID,
+ group.modelType, "active", group.extra, nowMs, now, nowMs, now,
+ ).Error; err != nil {
+ return err
+ }
+ inserted++
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ if inserted > 0 {
+ common.Info("Migrated tenant_model records from tenant_llm", zap.Int("count", inserted))
+ }
+ return nil
+}
+
+// buildTenantModelInstanceLookup maps provider_id + canonical api_key to the
+// instance id, mirroring TenantModelStage._get_instance_lookup.
+func buildTenantModelInstanceLookup(ctx context.Context, db *gorm.DB) (map[string]string, error) {
+ var rows []struct {
+ ID string `gorm:"column:id"`
+ ProviderID string `gorm:"column:provider_id"`
+ APIKey sql.NullString `gorm:"column:api_key"`
+ }
+ if err := db.Raw("SELECT id, provider_id, api_key FROM tenant_model_instance").Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+ lookup := make(map[string]string, len(rows))
+ for _, row := range rows {
+ lookup[row.ProviderID+"\x00"+stripIsToolsFromAPIKey(row.APIKey.String)] = row.ID
+ }
+ return lookup, nil
+}
+
+// buildTenantModelStatusCondition mirrors TenantModelStage._build_status_condition.
+func buildTenantModelStatusCondition() string {
+ emptyFactories := emptyLLMFactories()
+ if len(emptyFactories) == 0 {
+ return "tl.status = '0'"
+ }
+ sorted := append([]string(nil), emptyFactories...)
+ sort.Strings(sorted)
+ quoted := make([]string, 0, len(sorted))
+ for _, name := range sorted {
+ quoted = append(quoted, "'"+strings.ReplaceAll(name, "'", "''")+"'")
+ }
+ return fmt.Sprintf("(tl.status = '0' OR (tl.status = '1' AND tl.llm_factory IN (%s)))", strings.Join(quoted, ", "))
+}
+
+// emptyLLMFactories returns the factory names whose "llm" list is empty in
+// conf/llm_factories.json, matching TenantModelStage._get_empty_llm_factories.
+func emptyLLMFactories() []string {
+ var empty []string
+ for _, factory := range loadFactoryLLMInfos() {
+ if len(factory.LLM) == 0 {
+ empty = append(empty, factory.Name)
+ }
+ }
+ return empty
+}
+
+// factoryLLMInfo is one entry of "factory_llm_infos" in conf/llm_factories.json.
+type factoryLLMInfo struct {
+ Name string `json:"name"`
+ LLM []factoryLLM `json:"llm"`
+}
+
+// factoryLLM is one model declared by a factory. ModelType is either a string or
+// an array of strings, so it is decoded lazily.
+type factoryLLM struct {
+ LLMName string `json:"llm_name"`
+ ModelType json.RawMessage `json:"model_type"`
+ IsTools bool `json:"is_tools"`
+ MaxTokens json.RawMessage `json:"max_tokens"`
+}
+
+// loadFactoryLLMInfos reads and parses conf/llm_factories.json, returning nil when
+// the file is missing or unreadable.
+func loadFactoryLLMInfos() []factoryLLMInfo {
+ path := locateLLMFactoriesFile()
+ if path == "" {
+ return nil
+ }
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ common.Warn("Failed to read llm_factories.json for model migration", zap.String("path", path), zap.Error(err))
+ return nil
+ }
+ var payload struct {
+ FactoryLLMInfos []factoryLLMInfo `json:"factory_llm_infos"`
+ }
+ if err := json.Unmarshal(raw, &payload); err != nil {
+ common.Warn("Failed to parse llm_factories.json for model migration", zap.String("path", path), zap.Error(err))
+ return nil
+ }
+ return payload.FactoryLLMInfos
+}
+
+// locateLLMFactoriesFile finds conf/llm_factories.json by walking up from the
+// working directory and falling back to the executable's directory.
+func locateLLMFactoriesFile() string {
+ candidates := []string{filepath.Join("conf", "llm_factories.json")}
+ if wd, err := os.Getwd(); err == nil {
+ dir := wd
+ for depth := 0; depth < 5; depth++ {
+ candidates = append(candidates, filepath.Join(dir, "conf", "llm_factories.json"))
+ parent := filepath.Dir(dir)
+ if parent == dir {
+ break
+ }
+ dir = parent
+ }
+ }
+ if exe, err := os.Executable(); err == nil {
+ candidates = append(candidates, filepath.Join(filepath.Dir(exe), "conf", "llm_factories.json"))
+ }
+ for _, candidate := range candidates {
+ if info, err := os.Stat(candidate); err == nil && !info.IsDir() {
+ return candidate
+ }
+ }
+ return ""
+}
+
+// Tenant model seeding ports TenantModelSeedingStage.
+
+// tenantModelSeedRow tracks an existing tenant_model row so factory-declared
+// model types can be merged into it without a second table rewrite.
+type tenantModelSeedRow struct {
+ id string
+ modelType int
+}
+
+// seedTenantModels fills tenant_model with the models declared by
+// conf/llm_factories.json for every provider/instance created by the previous
+// steps. model_type is stored as the merged integer, so the declared factory
+// bits are OR-ed into the matching (provider_id, instance_id, model_name) row
+// and a fresh row is inserted when none exists.
+func seedTenantModels(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ for _, table := range []string{"tenant_model_provider", "tenant_model_instance", "tenant_model"} {
+ if !scoped.Migrator().HasTable(table) {
+ return nil
+ }
+ }
+
+ factories := loadFactoryLLMInfos()
+ if len(factories) == 0 {
+ return nil
+ }
+
+ existingExtras, err := loadTenantModelExtraMap(scoped)
+ if err != nil {
+ return err
+ }
+ existingStatuses, err := loadTenantModelStatusMap(scoped)
+ if err != nil {
+ return err
+ }
+ existing, err := loadTenantModelRowMap(scoped)
+ if err != nil {
+ return err
+ }
+
+ nowMs := time.Now().UnixMilli()
+ now := time.Now()
+ inserted, updated := 0, 0
+ err = scoped.Transaction(func(tx *gorm.DB) error {
+ for _, factory := range factories {
+ if len(factory.LLM) == 0 {
+ continue
+ }
+ providerName, extraInclude, extraExclude := factoryProviderFilter(factory.Name)
+ providerIDs, err := loadProviderIDsByName(scoped, providerName)
+ if err != nil {
+ return err
+ }
+ for _, providerID := range providerIDs {
+ instanceIDs, err := loadInstanceIDsByProvider(scoped, providerID, extraInclude, extraExclude)
+ if err != nil {
+ return err
+ }
+ for _, instanceID := range instanceIDs {
+ for _, llm := range factory.LLM {
+ if llm.LLMName == "" {
+ continue
+ }
+ bits := factoryModelTypeBits(llm)
+ if bits == 0 {
+ continue
+ }
+ key := providerID + "\x00" + instanceID + "\x00" + llm.LLMName
+ if row, ok := existing[key]; ok {
+ if row.modelType|bits == row.modelType {
+ continue
+ }
+ merged := row.modelType | bits
+ if err := tx.Exec(
+ "UPDATE tenant_model SET model_type = ?, update_time = ?, update_date = ? WHERE id = ?",
+ merged, nowMs, now, row.id,
+ ).Error; err != nil {
+ return err
+ }
+ existing[key] = tenantModelSeedRow{id: row.id, modelType: merged}
+ updated++
+ continue
+ }
+ extra := existingExtras[llm.LLMName]
+ if extra == "" {
+ extra = buildFactoryModelExtra(llm)
+ }
+ status := existingStatuses[llm.LLMName]
+ if status == "" {
+ status = "active"
+ }
+ id := modelMigrationNewID()
+ if err := tx.Exec(
+ "INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ id, llm.LLMName, providerID, instanceID, bits, status, extra, nowMs, now, nowMs, now,
+ ).Error; err != nil {
+ return err
+ }
+ existing[key] = tenantModelSeedRow{id: id, modelType: bits}
+ inserted++
+ }
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+
+ if inserted > 0 || updated > 0 {
+ common.Info("Seeded tenant_model from llm_factories.json", zap.Int("inserted", inserted), zap.Int("merged", updated))
+ }
+ return nil
+}
+
+// factoryProviderFilter mirrors the siliconflow special case in
+// TenantModelSeedingStage.execute: the intl factory targets SILICONFLOW instances
+// whose extra contains "region": "intl", while the global factory excludes them.
+func factoryProviderFilter(factoryName string) (providerName, extraInclude, extraExclude string) {
+ switch factoryName {
+ case "siliconflow_intl":
+ return "SILICONFLOW", `%"region": "intl"%`, ""
+ case "SILICONFLOW":
+ return "SILICONFLOW", "", `%"region": "intl"%`
+ default:
+ return factoryName, "", ""
+ }
+}
+
+// factoryModelTypeBits merges every model_type declared for a factory model.
+func factoryModelTypeBits(llm factoryLLM) int {
+ bits := 0
+ for _, name := range factoryLLMModelTypes(llm.ModelType) {
+ bits |= modelTypeStringToBit[strings.ToLower(strings.TrimSpace(name))]
+ }
+ return bits
+}
+
+// factoryLLMModelTypes decodes a model_type that is either a string or an array
+// of strings, defaulting to "chat" like the Python stage.
+func factoryLLMModelTypes(raw json.RawMessage) []string {
+ var single string
+ if err := json.Unmarshal(raw, &single); err == nil && single != "" {
+ return []string{single}
+ }
+ var list []string
+ if err := json.Unmarshal(raw, &list); err == nil && len(list) > 0 {
+ return list
+ }
+ return []string{"chat"}
+}
+
+// buildFactoryModelExtra builds the extra JSON stored for a newly seeded model,
+// mirroring TenantModelSeedingStage._build_extra.
+func buildFactoryModelExtra(llm factoryLLM) string {
+ extra := make(map[string]any)
+ if llm.IsTools {
+ extra["is_tools"] = true
+ }
+ if len(llm.MaxTokens) > 0 && string(llm.MaxTokens) != "null" {
+ var tokens any
+ if err := json.Unmarshal(llm.MaxTokens, &tokens); err == nil {
+ extra["max_tokens"] = tokens
+ }
+ }
+ if len(extra) == 0 {
+ return "{}"
+ }
+ encoded, err := json.Marshal(extra)
+ if err != nil {
+ return "{}"
+ }
+ return string(encoded)
+}
+
+func loadTenantModelRowMap(db *gorm.DB) (map[string]tenantModelSeedRow, error) {
+ var rows []struct {
+ ID string `gorm:"column:id"`
+ ProviderID string `gorm:"column:provider_id"`
+ InstanceID string `gorm:"column:instance_id"`
+ ModelName string `gorm:"column:model_name"`
+ ModelType sql.NullString `gorm:"column:model_type"`
+ }
+ if err := db.Raw("SELECT id, provider_id, instance_id, model_name, model_type FROM tenant_model").Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+ result := make(map[string]tenantModelSeedRow, len(rows))
+ for _, row := range rows {
+ result[row.ProviderID+"\x00"+row.InstanceID+"\x00"+row.ModelName] = tenantModelSeedRow{
+ id: row.ID,
+ modelType: mergeTenantModelTypeBits(row.ModelType.String),
+ }
+ }
+ return result, nil
+}
+
+func loadTenantModelExtraMap(db *gorm.DB) (map[string]string, error) {
+ var rows []struct {
+ ModelName string `gorm:"column:model_name"`
+ Extra string `gorm:"column:extra"`
+ }
+ if err := db.Raw("SELECT model_name, extra FROM tenant_model WHERE extra IS NOT NULL AND extra != '{}'").Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+ result := make(map[string]string, len(rows))
+ for _, row := range rows {
+ if row.ModelName != "" && row.Extra != "" && row.Extra != "{}" {
+ result[row.ModelName] = row.Extra
+ }
+ }
+ return result, nil
+}
+
+func loadTenantModelStatusMap(db *gorm.DB) (map[string]string, error) {
+ var rows []struct {
+ ModelName string `gorm:"column:model_name"`
+ Status string `gorm:"column:status"`
+ }
+ if err := db.Raw("SELECT model_name, status FROM tenant_model WHERE status IS NOT NULL AND status != 'unsupported'").Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+ result := make(map[string]string, len(rows))
+ for _, row := range rows {
+ if row.ModelName != "" && row.Status != "" {
+ result[row.ModelName] = row.Status
+ }
+ }
+ return result, nil
+}
+
+func loadProviderIDsByName(db *gorm.DB, providerName string) ([]string, error) {
+ var rows []struct {
+ ID string `gorm:"column:id"`
+ }
+ if err := db.Raw("SELECT id FROM tenant_model_provider WHERE provider_name = ?", providerName).Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+ ids := make([]string, 0, len(rows))
+ for _, row := range rows {
+ ids = append(ids, row.ID)
+ }
+ return ids, nil
+}
+
+func loadInstanceIDsByProvider(db *gorm.DB, providerID, extraInclude, extraExclude string) ([]string, error) {
+ var (
+ rows []struct {
+ ID string `gorm:"column:id"`
+ }
+ query string
+ args []any
+ )
+ switch {
+ case extraInclude != "" && extraExclude != "":
+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ? AND extra LIKE ? AND extra NOT LIKE ?"
+ args = []any{providerID, extraInclude, extraExclude}
+ case extraInclude != "":
+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ? AND extra LIKE ?"
+ args = []any{providerID, extraInclude}
+ case extraExclude != "":
+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ? AND extra NOT LIKE ?"
+ args = []any{providerID, extraExclude}
+ default:
+ query = "SELECT id FROM tenant_model_instance WHERE provider_id = ?"
+ args = []any{providerID}
+ }
+ if err := db.Raw(query, args...).Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+ ids := make([]string, 0, len(rows))
+ for _, row := range rows {
+ ids = append(ids, row.ID)
+ }
+ return ids, nil
+}
+
+// Merge tenant model types ports ModelTypeMergeStage.
+
+// tenantModelMergeSource is one tenant_model row read by mergeTenantModelTypes.
+// model_type is read as text because the stage only runs on tables that have not
+// been converted to the integer representation yet.
+type tenantModelMergeSource struct {
+ ID string `gorm:"column:id"`
+ ModelName sql.NullString `gorm:"column:model_name"`
+ ProviderID string `gorm:"column:provider_id"`
+ InstanceID string `gorm:"column:instance_id"`
+ ModelType sql.NullString `gorm:"column:model_type"`
+ Status sql.NullString `gorm:"column:status"`
+ Extra sql.NullString `gorm:"column:extra"`
+ CreateTime sql.NullInt64 `gorm:"column:create_time"`
+ CreateDate sql.NullTime `gorm:"column:create_date"`
+ UpdateTime sql.NullInt64 `gorm:"column:update_time"`
+ UpdateDate sql.NullTime `gorm:"column:update_date"`
+}
+
+// mergedTenantModelRecord is one merged tenant_model row written back by
+// mergeTenantModelTypes.
+type mergedTenantModelRecord struct {
+ modelName sql.NullString
+ providerID string
+ instanceID string
+ modelType int
+ status string
+ extra sql.NullString
+ createTime sql.NullInt64
+ createDate sql.NullTime
+ updateTime sql.NullInt64
+ updateDate sql.NullTime
+}
+
+// mergeTenantModelTypes ports ModelTypeMergeStage: it collapses the tenant_model
+// rows sharing (provider_id, instance_id, model_name) into a single row whose
+// model_type is the integer OR of the contributing model bits, dropping the bits
+// that only ever appeared on "unsupported" rows. Groups whose merged status is
+// not "active" are dropped.
+//
+// The stage only rewrites a table whose model_type is still stored as text. When
+// the column is already an integer type the merge has been applied and the stage
+// is a no-op, mirroring ModelTypeMergeStage.check.
+func mergeTenantModelTypes(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ if !scoped.Migrator().HasTable("tenant_model") {
+ return nil
+ }
+
+ isInteger, err := tenantModelModelTypeIsInteger(scoped)
+ if err != nil {
+ return err
+ }
+ if isInteger {
+ common.Info("tenant_model.model_type is already an integer, merge stage skipped")
+ return nil
+ }
+
+ var rows []tenantModelMergeSource
+ if err := scoped.Raw(
+ "SELECT id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date FROM tenant_model ORDER BY id",
+ ).Scan(&rows).Error; err != nil {
+ return err
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+
+ type mergeGroup struct {
+ first tenantModelMergeSource
+ supported int
+ unsupported int
+ status string
+ }
+
+ groups := make(map[string]*mergeGroup, len(rows))
+ order := make([]string, 0, len(rows))
+ for _, row := range rows {
+ key := row.ProviderID + "\x00" + row.InstanceID + "\x00" + row.ModelName.String
+ group, ok := groups[key]
+ if !ok {
+ group = &mergeGroup{first: row}
+ groups[key] = group
+ order = append(order, key)
+ }
+ bits := mergeTenantModelTypeBits(row.ModelType.String)
+ if row.Status.String == "unsupported" {
+ group.unsupported |= bits
+ continue
+ }
+ group.supported |= bits
+ if group.status == "" {
+ group.status = row.Status.String
+ }
+ }
+
+ nowMs := time.Now().UnixMilli()
+ now := time.Now()
+ merged := make([]mergedTenantModelRecord, 0, len(order))
+ for _, key := range order {
+ group := groups[key]
+ status := group.status
+ if status == "" {
+ status = "active"
+ }
+ if status != "active" {
+ continue
+ }
+ record := mergedTenantModelRecord{
+ modelName: group.first.ModelName,
+ providerID: group.first.ProviderID,
+ instanceID: group.first.InstanceID,
+ modelType: group.supported &^ group.unsupported,
+ status: status,
+ extra: group.first.Extra,
+ createTime: group.first.CreateTime,
+ createDate: group.first.CreateDate,
+ updateTime: group.first.UpdateTime,
+ updateDate: group.first.UpdateDate,
+ }
+ if !record.createTime.Valid {
+ record.createTime = sql.NullInt64{Int64: nowMs, Valid: true}
+ }
+ if !record.updateTime.Valid {
+ record.updateTime = sql.NullInt64{Int64: nowMs, Valid: true}
+ }
+ if !record.createDate.Valid {
+ record.createDate = sql.NullTime{Time: now, Valid: true}
+ }
+ if !record.updateDate.Valid {
+ record.updateDate = sql.NullTime{Time: now, Valid: true}
+ }
+ merged = append(merged, record)
+ }
+
+ // The whole table is replaced even when every group was dropped, mirroring the
+ // Python stage that always swaps in the merged table. Leaving the source rows
+ // behind would keep text model_type values in place for the integer typed reads
+ // that follow.
+ if err := scoped.Transaction(func(tx *gorm.DB) error {
+ if err := tx.Exec("DELETE FROM tenant_model").Error; err != nil {
+ return err
+ }
+ for _, record := range merged {
+ if err := tx.Exec(
+ "INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type, status, extra, create_time, create_date, update_time, update_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ modelMigrationNewID(), record.modelName, record.providerID, record.instanceID,
+ record.modelType, record.status, record.extra,
+ record.createTime, record.createDate, record.updateTime, record.updateDate,
+ ).Error; err != nil {
+ return err
+ }
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ common.Info("Merged tenant_model model types", zap.Int("rows", len(merged)))
+ return nil
+}
+
+// mergeTenantModelTypeBits maps a stored model_type to its bit. Legacy rows hold
+// the LLMType name, while rows written by the seeding stage already hold the
+// merged integer, so numeric text is accepted as well.
+func mergeTenantModelTypeBits(raw string) int {
+ trimmed := strings.TrimSpace(raw)
+ if bit, ok := modelTypeStringToBit[strings.ToLower(trimmed)]; ok {
+ return bit
+ }
+ if value, err := strconv.Atoi(trimmed); err == nil {
+ return value
+ }
+ return 0
+}
+
+// tenantModelModelTypeIsInteger reports whether tenant_model.model_type is stored
+// as an integer column, in which case the merge has already been applied.
+func tenantModelModelTypeIsInteger(db *gorm.DB) (bool, error) {
+ columnTypes, err := db.Migrator().ColumnTypes("tenant_model")
+ if err != nil {
+ return false, err
+ }
+ for _, columnType := range columnTypes {
+ if columnType.Name() != "model_type" {
+ continue
+ }
+ return strings.Contains(strings.ToLower(columnType.DatabaseTypeName()), "int"), nil
+ }
+ return false, nil
+}
+
+// Model id normalization ports ModelIdConfigStage.
+
+var modelIDStringColumns = map[string][]string{
+ "tenant": {"llm_id", "embd_id", "asr_id", "img2txt_id", "rerank_id", "tts_id", "ocr_id"},
+ "knowledgebase": {"embd_id"},
+ "dialog": {"llm_id", "rerank_id"},
+ "memory": {"embd_id", "llm_id"},
+}
+
+var modelIDJSONColumns = map[string][]string{
+ "knowledgebase": {"parser_config"},
+ "document": {"parser_config"},
+ "search": {"search_config"},
+ "user_canvas": {"dsl"},
+ "canvas_template": {"dsl"},
+ "user_canvas_version": {"dsl"},
+ "api_4_conversation": {"dsl"},
+ "pipeline_operation_log": {"dsl"},
+ "connector": {"config"},
+ "evaluation_runs": {"config_snapshot"},
+}
+
+var modelIDFieldSet = map[string]bool{
+ "llm_id": true,
+ "embd_id": true,
+ "embedding_model": true,
+ "rerank_id": true,
+ "asr_id": true,
+ "img2txt_id": true,
+ "tts_id": true,
+ "ocr_id": true,
+}
+
+var searchConfigModelIDFieldSet = map[string]bool{"chat_id": true}
+
+// normalizeModelID rewrites "<model>@<provider>" into "<model>@default@<provider>".
+// Values that are not exactly two non-empty parts are returned unchanged.
+func normalizeModelID(value string) (string, bool) {
+ parts := strings.Split(value, "@")
+ if len(parts) != 2 {
+ return value, false
+ }
+ modelName, providerName := parts[0], parts[1]
+ if modelName == "" || providerName == "" {
+ return value, false
+ }
+ return modelName + "@default@" + providerName, true
+}
+
+func containsString(values []string, target string) bool {
+ for _, value := range values {
+ if value == target {
+ return true
+ }
+ }
+ return false
+}
+
+// normalizeModelIDConfig recursively normalizes model id fields inside a decoded
+// JSON document, mirroring ModelIdConfigStage.normalize_config.
+func normalizeModelIDConfig(value any, path []string) (any, bool) {
+ switch typed := value.(type) {
+ case map[string]any:
+ changed := false
+ normalized := make(map[string]any, len(typed))
+ for key, item := range typed {
+ keyPath := append(append([]string{}, path...), key)
+ shouldNormalize := modelIDFieldSet[key] || (searchConfigModelIDFieldSet[key] && containsString(path, "search_config"))
+ if shouldNormalize {
+ if str, ok := item.(string); ok {
+ normalizedValue, itemChanged := normalizeModelID(str)
+ normalized[key] = normalizedValue
+ changed = changed || itemChanged
+ continue
+ }
+ normalized[key] = item
+ continue
+ }
+ normalizedItem, itemChanged := normalizeModelIDConfig(item, keyPath)
+ normalized[key] = normalizedItem
+ changed = changed || itemChanged
+ }
+ return normalized, changed
+ case []any:
+ changed := false
+ normalized := make([]any, len(typed))
+ for index, item := range typed {
+ indexPath := append(append([]string{}, path...), strconv.Itoa(index))
+ normalizedItem, itemChanged := normalizeModelIDConfig(item, indexPath)
+ normalized[index] = normalizedItem
+ changed = changed || itemChanged
+ }
+ return normalized, changed
+ default:
+ return value, false
+ }
+}
+
+// normalizeStoredModelIDs rewrites stored model ids across the tables listed by
+// ModelIdConfigStage.
+func normalizeStoredModelIDs(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ for _, table := range sortedTableNames(modelIDStringColumns) {
+ if !scoped.Migrator().HasTable(table) {
+ continue
+ }
+ for _, column := range modelIDStringColumns[table] {
+ exists, err := columnExists(ctx, scoped, table, column)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ continue
+ }
+ if err := normalizeModelIDStringColumn(ctx, scoped, table, column); err != nil {
+ return err
+ }
+ }
+ }
+ for _, table := range sortedTableNames(modelIDJSONColumns) {
+ if !scoped.Migrator().HasTable(table) {
+ continue
+ }
+ for _, column := range modelIDJSONColumns[table] {
+ exists, err := columnExists(ctx, scoped, table, column)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ continue
+ }
+ if err := normalizeModelIDJSONColumn(ctx, scoped, table, column); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// sortedTableNames returns a stable ordering so the migration is deterministic.
+func sortedTableNames(columns map[string][]string) []string {
+ names := make([]string, 0, len(columns))
+ for name := range columns {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ return names
+}
+
+func normalizeModelIDStringColumn(ctx context.Context, db *gorm.DB, table, column string) error {
+ var rows []struct {
+ ID string `gorm:"column:id"`
+ Value sql.NullString `gorm:"column:value"`
+ }
+ query := fmt.Sprintf("SELECT id AS id, `%s` AS value FROM `%s` WHERE `%s` IS NOT NULL AND `%s` LIKE '%%@%%'", column, table, column, column)
+ if err := db.Raw(query).Scan(&rows).Error; err != nil {
+ return err
+ }
+
+ updates := make(map[string]string)
+ for _, row := range rows {
+ if !row.Value.Valid || row.Value.String == "" {
+ continue
+ }
+ if normalized, changed := normalizeModelID(row.Value.String); changed {
+ updates[row.ID] = normalized
+ }
+ }
+ return applyModelIDUpdates(ctx, db, table, column, updates)
+}
+
+func normalizeModelIDJSONColumn(ctx context.Context, db *gorm.DB, table, column string) error {
+ var rows []struct {
+ ID string `gorm:"column:id"`
+ Value sql.NullString `gorm:"column:value"`
+ }
+ query := fmt.Sprintf("SELECT id AS id, `%s` AS value FROM `%s` WHERE `%s` IS NOT NULL AND `%s` != ''", column, table, column, column)
+ if err := db.Raw(query).Scan(&rows).Error; err != nil {
+ return err
+ }
+
+ updates := make(map[string]string)
+ for _, row := range rows {
+ var decoded any
+ if err := json.Unmarshal([]byte(row.Value.String), &decoded); err != nil {
+ continue
+ }
+ normalized, changed := normalizeModelIDConfig(decoded, []string{column})
+ if !changed {
+ continue
+ }
+ encoded, err := json.Marshal(normalized)
+ if err != nil {
+ continue
+ }
+ updates[row.ID] = string(encoded)
+ }
+ return applyModelIDUpdates(ctx, db, table, column, updates)
+}
+
+func applyModelIDUpdates(ctx context.Context, db *gorm.DB, table, column string, updates map[string]string) error {
+ if len(updates) == 0 {
+ return nil
+ }
+ statement := fmt.Sprintf("UPDATE `%s` SET `%s` = ? WHERE id = ?", table, column)
+ return db.Transaction(func(tx *gorm.DB) error {
+ for id, value := range updates {
+ if err := tx.Exec(statement, value, id).Error; err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
+
+// Tenant model id population ports TenantModelIdMigrationStage.
+
+type tenantModelIDField struct {
+ tenantIDColumn string
+ legacyColumn string
+ modelType string
+}
+
+var tenantModelIDFields = map[string][]tenantModelIDField{
+ "tenant": {
+ {"tenant_llm_id", "llm_id", "chat"},
+ {"tenant_embd_id", "embd_id", "embedding"},
+ {"tenant_asr_id", "asr_id", "speech2text"},
+ {"tenant_img2txt_id", "img2txt_id", "image2text"},
+ {"tenant_rerank_id", "rerank_id", "rerank"},
+ {"tenant_tts_id", "tts_id", "tts"},
+ {"tenant_ocr_id", "ocr_id", "ocr"},
+ },
+ "knowledgebase": {
+ {"tenant_embd_id", "embd_id", "embedding"},
+ },
+ "dialog": {
+ {"tenant_llm_id", "llm_id", "chat"},
+ {"tenant_rerank_id", "rerank_id", "rerank"},
+ },
+ "memory": {
+ {"tenant_embd_id", "embd_id", "embedding"},
+ {"tenant_llm_id", "llm_id", "chat"},
+ },
+}
+
+type tenantModelLookupKey struct {
+ tenantID string
+ modelName string
+ provider string
+ modelType string
+}
+
+// populateTenantModelIDColumns ports TenantModelIdMigrationStage: it ensures the
+// tenant_*_id columns exist and fills them from the legacy *_id columns by
+// resolving the referenced tenant_model row.
+func populateTenantModelIDColumns(ctx context.Context, db *gorm.DB) error {
+ scoped := db.WithContext(ctx)
+ for _, table := range []string{"tenant_model", "tenant_model_provider", "tenant_model_instance"} {
+ if !scoped.Migrator().HasTable(table) {
+ return nil
+ }
+ }
+
+ for _, table := range sortedTenantModelIDTables() {
+ if !scoped.Migrator().HasTable(table) {
+ continue
+ }
+ for _, field := range tenantModelIDFields[table] {
+ exists, err := columnExists(ctx, scoped, table, field.tenantIDColumn)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ if err := ensureTenantModelIDColumn(ctx, scoped, table, field.tenantIDColumn); err != nil {
+ return err
+ }
+ }
+ }
+ }
+
+ lookup, err := buildTenantModelLookup(ctx, scoped)
+ if err != nil {
+ return err
+ }
+ if len(lookup) == 0 {
+ return nil
+ }
+
+ total := 0
+ for _, table := range sortedTenantModelIDTables() {
+ if !scoped.Migrator().HasTable(table) {
+ continue
+ }
+ for _, field := range tenantModelIDFields[table] {
+ tenantColumnExists, err := columnExists(ctx, scoped, table, field.tenantIDColumn)
+ if err != nil {
+ return err
+ }
+ legacyColumnExists, err := columnExists(ctx, scoped, table, field.legacyColumn)
+ if err != nil {
+ return err
+ }
+ if !tenantColumnExists || !legacyColumnExists {
+ continue
+ }
+ updated, err := populateTenantModelIDColumn(ctx, scoped, table, field, lookup)
+ if err != nil {
+ return err
+ }
+ total += updated
+ }
+ }
+
+ if total > 0 {
+ common.Info("Populated tenant model id columns", zap.Int("rows", total))
+ }
+ return nil
+}
+
+func sortedTenantModelIDTables() []string {
+ names := make([]string, 0, len(tenantModelIDFields))
+ for name := range tenantModelIDFields {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ return names
+}
+
+func ensureTenantModelIDColumn(ctx context.Context, db *gorm.DB, table, column string) error {
+ common.Info("Adding tenant model id column", zap.String("table", table), zap.String("column", column))
+ statement := fmt.Sprintf("ALTER TABLE `%s` ADD COLUMN `%s` VARCHAR(32) NULL", table, column)
+ return db.Exec(statement).Error
+}
+
+// buildTenantModelLookup maps (tenant_id, model_name, provider, model_type) to
+// tenant_model.id for every active model, expanding the model_type bitmask.
+func buildTenantModelLookup(ctx context.Context, db *gorm.DB) (map[tenantModelLookupKey]string, error) {
+ var rows []struct {
+ ID string `gorm:"column:id"`
+ ModelName string `gorm:"column:model_name"`
+ ModelType int `gorm:"column:model_type"`
+ TenantID string `gorm:"column:tenant_id"`
+ ProviderName string `gorm:"column:provider_name"`
+ }
+ if err := db.Raw(`
+ SELECT tm.id AS id, tm.model_name AS model_name, tm.model_type AS model_type,
+ tmp.tenant_id AS tenant_id, tmp.provider_name AS provider_name
+ FROM tenant_model tm
+ INNER JOIN tenant_model_provider tmp ON tm.provider_id = tmp.id
+ WHERE tm.status = 'active'`).Scan(&rows).Error; err != nil {
+ return nil, err
+ }
+
+ lookup := make(map[tenantModelLookupKey]string)
+ for _, row := range rows {
+ for typeName, typeBit := range modelTypeStringToBit {
+ if row.ModelType&typeBit != 0 {
+ lookup[tenantModelLookupKey{row.TenantID, row.ModelName, row.ProviderName, typeName}] = row.ID
+ }
+ }
+ }
+ return lookup, nil
+}
+
+func populateTenantModelIDColumn(ctx context.Context, db *gorm.DB, table string, field tenantModelIDField, lookup map[tenantModelLookupKey]string) (int, error) {
+ tenantIDExpr := "tenant_id"
+ if table == "tenant" {
+ // The tenant table's primary key is the tenant id itself.
+ tenantIDExpr = "id"
+ }
+ query := fmt.Sprintf(
+ "SELECT id AS id, %s AS tenant_id, `%s` AS model_name FROM `%s` WHERE (`%s` IS NULL OR `%s` = '' OR LENGTH(`%s`) <> 32) AND `%s` IS NOT NULL AND `%s` != ''",
+ tenantIDExpr, field.legacyColumn, table,
+ field.tenantIDColumn, field.tenantIDColumn, field.tenantIDColumn,
+ field.legacyColumn, field.legacyColumn,
+ )
+
+ var rows []struct {
+ ID string `gorm:"column:id"`
+ TenantID string `gorm:"column:tenant_id"`
+ ModelName string `gorm:"column:model_name"`
+ }
+ if err := db.Raw(query).Scan(&rows).Error; err != nil {
+ return 0, err
+ }
+ if len(rows) == 0 {
+ return 0, nil
+ }
+
+ updateStatement := fmt.Sprintf("UPDATE `%s` SET `%s` = ? WHERE id = ?", table, field.tenantIDColumn)
+ updated := 0
+ if err := db.Transaction(func(tx *gorm.DB) error {
+ for _, row := range rows {
+ tenantID := row.TenantID
+ if table == "tenant" {
+ tenantID = row.ID
+ }
+ resolved := resolveTenantModelID(lookup, tenantID, row.ModelName, field.modelType)
+ if err := tx.Exec(updateStatement, resolved, row.ID).Error; err != nil {
+ return err
+ }
+ updated++
+ }
+ return nil
+ }); err != nil {
+ return 0, err
+ }
+ return updated, nil
+}
+
+// resolveTenantModelID ports TenantModelIdMigrationStage._resolve_model_id.
+func resolveTenantModelID(lookup map[tenantModelLookupKey]string, tenantID, modelName, modelType string) string {
+ pureName, _, providerName := splitModelName(modelName)
+ if pureName == "" || providerName == "" {
+ return ""
+ }
+ if id, ok := lookup[tenantModelLookupKey{tenantID, pureName, providerName, modelType}]; ok {
+ return id
+ }
+ for key, id := range lookup {
+ if key.tenantID == tenantID && key.modelName == pureName && key.modelType == modelType {
+ return id
+ }
+ }
+ return ""
+}
+
+// splitModelName ports TenantModelIdMigrationStage._split_model_name:
+// "{model}@{factory}" or "{model}@{instance}@{factory}".
+func splitModelName(modelName string) (string, string, string) {
+ if modelName == "" {
+ return "", "", ""
+ }
+ parts := strings.Split(modelName, "@")
+ switch len(parts) {
+ case 1:
+ return parts[0], modelMigrationInstanceName, ""
+ case 2:
+ return parts[0], modelMigrationInstanceName, parts[1]
+ default:
+ return parts[0], parts[1], parts[2]
+ }
+}
diff --git a/internal/development.md b/internal/development.md
index 4fcce80c0a..17321c84af 100644
--- a/internal/development.md
+++ b/internal/development.md
@@ -211,8 +211,8 @@ Note: admin server must be started first; otherwise, api server will encounter e
```
```bash
-# Start admin server and migrate database
-./bin/ragflow_server --admin --migrate
+# Run database migrations (standalone action; does not start a server)
+./bin/ragflow_server --migrate
```
```bash
diff --git a/internal/entity/chat.go b/internal/entity/chat.go
index ff2bd278c0..cf544057f6 100644
--- a/internal/entity/chat.go
+++ b/internal/entity/chat.go
@@ -42,7 +42,7 @@ type Chat struct {
VectorSimilarityWeight float64 `gorm:"column:vector_similarity_weight" json:"vector_similarity_weight"`
TopN int64 `gorm:"column:top_n" json:"top_n"`
RerankCandidatesCount int64 `gorm:"column:rerank_candidates_count" json:"rerank_candidates_count"`
- TopK int64 `gorm:"column:top_k" json:"top_k"`
+ TopK int64 `gorm:"column:top_k;type:int" json:"top_k"`
DoRefer string `gorm:"column:do_refer;size:1;not null" json:"do_refer"`
RerankID string `gorm:"column:rerank_id;size:128;not null;default:''" json:"rerank_id"`
TenantRerankID *string `gorm:"column:tenant_rerank_id;size:32;index" json:"tenant_rerank_id,omitempty"`
diff --git a/internal/entity/dataset.go b/internal/entity/dataset.go
index 09d9a3d7f6..c8cc953897 100644
--- a/internal/entity/dataset.go
+++ b/internal/entity/dataset.go
@@ -119,9 +119,9 @@ type Knowledgebase struct {
GraphragTaskID *string `gorm:"column:graphrag_task_id;size:32;index" json:"graphrag_task_id,omitempty"`
GraphragTaskFinishAt *time.Time `gorm:"column:graphrag_task_finish_at" json:"graphrag_task_finish_at,omitempty"`
RaptorTaskID *string `gorm:"column:raptor_task_id;size:32;index" json:"raptor_task_id,omitempty"`
- RaptorTaskFinishAt *time.Time `gorm:"column:raptor_task_finish_at" json:"raptor_task_finish_at,omitempty"`
+ RaptorTaskFinishAt *time.Time `gorm:"column:raptor_task_finish_at;type:datetime" json:"raptor_task_finish_at,omitempty"`
MindmapTaskID *string `gorm:"column:mindmap_task_id;size:32;index" json:"mindmap_task_id,omitempty"`
- MindmapTaskFinishAt *time.Time `gorm:"column:mindmap_task_finish_at" json:"mindmap_task_finish_at,omitempty"`
+ MindmapTaskFinishAt *time.Time `gorm:"column:mindmap_task_finish_at;type:datetime" json:"mindmap_task_finish_at,omitempty"`
WikiTaskID *string `gorm:"column:wiki_task_id;size:32;index" json:"wiki_task_id,omitempty"`
WikiTaskFinishAt *time.Time `gorm:"column:wiki_task_finish_at" json:"wiki_task_finish_at,omitempty"`
SkillTaskID *string `gorm:"column:skill_task_id;size:32;index" json:"skill_task_id,omitempty"`
diff --git a/internal/entity/system.go b/internal/entity/system.go
index 49541ec3ef..94e720cb65 100644
--- a/internal/entity/system.go
+++ b/internal/entity/system.go
@@ -21,7 +21,7 @@ type SystemSettings struct {
Name string `gorm:"column:name;primaryKey;size:128" json:"name"`
Source string `gorm:"column:source;size:32;not null" json:"source"`
DataType string `gorm:"column:data_type;size:32;not null" json:"data_type"`
- Value string `gorm:"column:value;type:longtext;not null" json:"value"`
+ Value string `gorm:"column:value;type:text;not null" json:"value"`
BaseModel
}
diff --git a/internal/entity/tenant_llm.go b/internal/entity/tenant_llm.go
index 9b3829c6b9..1c8c7f3cf2 100644
--- a/internal/entity/tenant_llm.go
+++ b/internal/entity/tenant_llm.go
@@ -24,7 +24,7 @@ type TenantLLM struct {
LLMFactory string `gorm:"column:llm_factory;size:128;not null" json:"llm_factory"`
ModelType *string `gorm:"column:model_type;size:128;index" json:"model_type,omitempty"`
LLMName *string `gorm:"column:llm_name;size:128;index;default:''" json:"llm_name,omitempty"`
- APIKey *string `gorm:"column:api_key;size:8192" json:"api_key,omitempty"`
+ APIKey *string `gorm:"column:api_key;type:text" json:"api_key,omitempty"`
APIBase *string `gorm:"column:api_base;size:255" json:"api_base,omitempty"`
MaxTokens int64 `gorm:"column:max_tokens;default:8192" json:"max_tokens"`
UsedTokens int64 `gorm:"column:used_tokens;default:0" json:"used_tokens"`
diff --git a/internal/entity/user.go b/internal/entity/user.go
index 7d70f9197b..4746ff0dac 100644
--- a/internal/entity/user.go
+++ b/internal/entity/user.go
@@ -24,7 +24,7 @@ type User struct {
AccessToken *string `gorm:"column:access_token;size:255;index" json:"access_token,omitempty"`
Nickname string `gorm:"column:nickname;size:100;not null;index" json:"nickname"`
Password *string `gorm:"column:password;size:255;index" json:"-"`
- Email string `gorm:"column:email;size:255;not null;unique" json:"email"`
+ Email string `gorm:"column:email;size:255;not null" json:"email"`
Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"`
Language *string `gorm:"column:language;size:32;index;default:'English'" json:"language,omitempty"`
ColorSchema *string `gorm:"column:color_schema;size:32;index;default:'Bright'" json:"color_schema,omitempty"`