Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: handle int tenant_*_id (#20830)

Lynn79c24d8286cb
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index da661a0e5c..1aa4c9bea3 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -23,6 +23,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
"regexp"
+ "sort"
"strings"
"go.uber.org/zap"
@@ -661,6 +662,7 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {
definition: "datetime",
},
}
+ specs = append(specs, tenantModelIDColumnSpecs()...)
for _, spec := range specs {
if err := applyColumnSpec(ctx, db, spec); err != nil {
@@ -671,6 +673,45 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {
return nil
}
+// tenantModelIDColumnSpecs projects tenantModelIDFields into column specs for
+// the tenant model id columns. These columns were once declared *int64, so a
+// database built by one of those releases holds them as bigint and the daily
+// cast back to varchar(32) is what lets AutoMigrate succeed at all. Sharing
+// tenantModelIDFields keeps this list and the backfill reading one inventory.
+func tenantModelIDColumnSpecs() []columnSpec {
+ tables := tenantModelIDFieldTables()
+
+ total := 0
+ for _, table := range tables {
+ total += len(tenantModelIDFields[table])
+ }
+
+ specs := make([]columnSpec, 0, total)
+ for _, table := range tables {
+ for _, field := range tenantModelIDFields[table] {
+ specs = append(specs, columnSpec{
+ table: table,
+ column: field.tenantIDColumn,
+ columnType: "varchar(32)", nullable: true,
+ definition: "varchar(32)",
+ keepLength: 32,
+ })
+ }
+ }
+ return specs
+}
+
+// tenantModelIDFieldTables lists the tables carrying tenant model id columns in
+// a stable order, so both the spec builder and anything reporting on them agree.
+func tenantModelIDFieldTables() []string {
+ tables := make([]string, 0, len(tenantModelIDFields))
+ for table := range tenantModelIDFields {
+ tables = append(tables, table)
+ }
+ sort.Strings(tables)
+ return tables
+}
+
// renameColumnIfExists renames a column if it exists and the new column doesn't exist
func renameColumnIfExists(ctx context.Context, db *gorm.DB, tableName, oldName, newName string) error {
if !db.WithContext(ctx).Migrator().HasTable(tableName) {
@@ -968,6 +1009,7 @@ type columnSpec struct {
columnType string // compared against INFORMATION_SCHEMA.COLUMNS.COLUMN_TYPE
nullable bool
definition string // used verbatim when an ALTER is required
+ keepLength int // when non-zero, values wider or narrower than this are orphaned by the type change and reset to NULL
}
// applyColumnSpec aligns a column with its expected definition, issuing DDL only
@@ -1001,6 +1043,49 @@ func applyColumnSpec(ctx context.Context, db *gorm.DB, spec columnSpec) error {
zap.String("table", spec.table),
zap.String("column", spec.column),
zap.Error(err))
+ return nil
}
+ discardOrphanedColumnValues(ctx, db, spec)
return nil
}
+
+// discardOrphanedColumnValues clears values that the type change in spec leaves
+// behind. This exists because a successful MODIFY says nothing about whether
+// the surviving values are still meaningful: an int column converted to a
+// hex-id column keeps "3", which every reader treats as an unresolved model
+// reference rather than as the unset value it has become.
+//
+// LENGTH() is NULL on NULL rows and NULL <> any number is not true, so rows
+// already unset are never counted or rewritten.
+func discardOrphanedColumnValues(ctx context.Context, db *gorm.DB, spec columnSpec) {
+ if spec.keepLength <= 0 {
+ return
+ }
+
+ stale := fmt.Sprintf("SELECT COUNT(1) FROM %s WHERE LENGTH(%s) <> %d", spec.table, spec.column, spec.keepLength)
+ var count int64
+ if err := db.WithContext(ctx).Raw(stale).Scan(&count).Error; err != nil {
+ common.Warn("Failed to count orphaned values",
+ zap.String("table", spec.table),
+ zap.String("column", spec.column),
+ zap.Error(err))
+ return
+ }
+ if count == 0 {
+ return
+ }
+
+ common.Info("Discarding orphaned values from converted column",
+ zap.String("table", spec.table),
+ zap.String("column", spec.column),
+ zap.Int64("count", count))
+ result := db.WithContext(ctx).Exec(
+ fmt.Sprintf("UPDATE %s SET %s = NULL WHERE LENGTH(%s) <> %d", spec.table, spec.column, spec.column, spec.keepLength),
+ )
+ if result.Error != nil {
+ common.Warn("Failed to discard orphaned values",
+ zap.String("table", spec.table),
+ zap.String("column", spec.column),
+ zap.Error(result.Error))
+ }
+}
diff --git a/internal/dao/migration_integration_test.go b/internal/dao/migration_integration_test.go
new file mode 100644
index 0000000000..4ab7207424
--- /dev/null
+++ b/internal/dao/migration_integration_test.go
@@ -0,0 +1,97 @@
+//go:build integration
+// +build integration
+
+package dao
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "gorm.io/gorm"
+)
+
+// TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration reproduces
+// the schema that blocks startup: tenant model id columns created while the Go
+// entities declared them *int64, which leaves MySQL holding bigint. No
+// migration step converts them back, so AutoMigrate ends up being the one that
+// tries and fails.
+//
+// The specs name real tables, so this needs a throwaway database rather than a
+// shared one. Finding any of those tables already present aborts before any DDL
+// lands, which is what keeps the DROP in t.Cleanup from ever touching a live
+// schema.
+//
+// Run with:
+//
+// RAGFLOW_TEST_MYSQL_DSN='root:infini_rag_flow@tcp(127.0.0.1:3306)/ragflow_test?charset=utf8mb4&parseTime=True&loc=Local' \
+// bash build.sh --test-integration ./internal/dao/...
+func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testing.T) {
+ dsn := os.Getenv("RAGFLOW_TEST_MYSQL_DSN")
+ if dsn == "" {
+ t.Skip("RAGFLOW_TEST_MYSQL_DSN unset: this test needs a throwaway MySQL database")
+ }
+
+ db, err := gorm.Open(newMigrationAwareDialector(dsn), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("failed to connect: %v", err)
+ }
+
+ ctx := context.Background()
+ for _, table := range tenantModelIDFieldTables() {
+ if db.WithContext(ctx).Migrator().HasTable(table) {
+ t.Fatalf("%s already exists in this database; refusing to run against a live schema", table)
+ }
+ }
+
+ create := `CREATE TABLE dialog (
+ id varchar(32) NOT NULL,
+ llm_id varchar(128) NOT NULL DEFAULT '',
+ rerank_id varchar(128) NOT NULL DEFAULT '',
+ tenant_llm_id bigint DEFAULT NULL,
+ tenant_rerank_id bigint DEFAULT NULL,
+ PRIMARY KEY (id)
+ )`
+ if err = db.WithContext(ctx).Exec(create).Error; err != nil {
+ t.Fatalf("failed to create legacy dialog: %v", err)
+ }
+ t.Cleanup(func() {
+ _ = db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS dialog").Error
+ })
+
+ // llm_id is empty on purpose: this row would not be backfilled even if the
+ // version gate let populateTenantModelIDColumns through, so the type repair
+ // is its only chance to stop holding a dead reference.
+ if err = db.WithContext(ctx).Exec(
+ "INSERT INTO dialog (id, llm_id, rerank_id, tenant_llm_id, tenant_rerank_id) VALUES ('d1', '', '', 3, 7)",
+ ).Error; err != nil {
+ t.Fatalf("failed to seed dialog: %v", err)
+ }
+
+ if err = modifyColumnTypes(ctx, db); err != nil {
+ t.Fatalf("modifyColumnTypes failed: %v", err)
+ }
+
+ for _, column := range []string{"tenant_llm_id", "tenant_rerank_id"} {
+ columnType, _, ok, err := columnDefinition(ctx, db, "dialog", column)
+ if err != nil {
+ t.Fatalf("failed to read %s definition: %v", column, err)
+ }
+ if !ok {
+ t.Fatalf("%s vanished from dialog", column)
+ }
+ if columnType != "varchar(32)" {
+ t.Errorf("%s type = %q, want varchar(32)", column, columnType)
+ }
+ }
+
+ var remaining int64
+ if err = db.WithContext(ctx).Raw(
+ "SELECT COUNT(1) FROM dialog WHERE tenant_llm_id IS NOT NULL OR tenant_rerank_id IS NOT NULL",
+ ).Scan(&remaining).Error; err != nil {
+ t.Fatalf("failed to count remaining references: %v", err)
+ }
+ if remaining != 0 {
+ t.Errorf("%d rows still carry an orphaned integer model id", remaining)
+ }
+}
diff --git a/internal/dao/migration_test.go b/internal/dao/migration_test.go
new file mode 100644
index 0000000000..4591a11087
--- /dev/null
+++ b/internal/dao/migration_test.go
@@ -0,0 +1,162 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+const tenantModelIDWidth = 32
+
+// setupOrphanColumnTestDB builds the narrow shape discardOrphanedColumnValues
+// needs: a table holding values left over from an int column widened to hold
+// hex model ids.
+func setupOrphanColumnTestDB(t *testing.T) *gorm.DB {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("failed to open sqlite: %v", err)
+ }
+ if err = db.Exec("CREATE TABLE legacy_dialog (id TEXT PRIMARY KEY, tenant_llm_id TEXT)").Error; err != nil {
+ t.Fatalf("failed to create legacy_dialog: %v", err)
+ }
+ return db
+}
+
+// TestTenantModelIDColumnSpecsMirrorsBackfillFields guards the shared inventory:
+// the type repair and the backfill must cover exactly the same columns, and the
+// order must be stable so consecutive startups log identically.
+func TestTenantModelIDColumnSpecsMirrorsBackfillFields(t *testing.T) {
+ want := 0
+ for _, fields := range tenantModelIDFields {
+ want += len(fields)
+ }
+
+ specs := tenantModelIDColumnSpecs()
+ if len(specs) != want {
+ t.Fatalf("spec count = %d, want %d", len(specs), want)
+ }
+
+ got := map[string]columnSpec{}
+ for _, spec := range specs {
+ got[spec.table+"."+spec.column] = spec
+ }
+
+ for table, fields := range tenantModelIDFields {
+ for _, field := range fields {
+ spec, ok := got[table+"."+field.tenantIDColumn]
+ if !ok {
+ t.Fatalf("missing spec for %s.%s", table, field.tenantIDColumn)
+ }
+ if spec.columnType != "varchar(32)" || spec.definition != "varchar(32)" {
+ t.Errorf("%s.%s types = %q/%q, want varchar(32)/varchar(32)",
+ table, field.tenantIDColumn, spec.columnType, spec.definition)
+ }
+ if !spec.nullable {
+ t.Errorf("%s.%s nullable = false, want true", table, field.tenantIDColumn)
+ }
+ if spec.keepLength != tenantModelIDWidth {
+ t.Errorf("%s.%s keepLength = %d, want %d", table, field.tenantIDColumn, spec.keepLength, tenantModelIDWidth)
+ }
+ if strings.Contains(spec.column, "`") {
+ t.Errorf("%s.%s column carries a backtick: %q", table, field.tenantIDColumn, spec.column)
+ }
+ }
+ }
+
+ again := tenantModelIDColumnSpecs()
+ for i := range specs {
+ if specs[i] != again[i] {
+ t.Fatalf("spec order is not deterministic at index %d: %v then %v", i, specs[i], again[i])
+ }
+ }
+}
+
+// TestDiscardOrphanedColumnValuesResetsOnlyOrphans covers the values a widened
+// int column carries into its new life as a hex-id column. "3" is what matters:
+// it looks like a model id to every reader, so leaving it behind turns a visible
+// migration failure into silent bad lookups.
+func TestDiscardOrphanedColumnValuesResetsOnlyOrphans(t *testing.T) {
+ db := setupOrphanColumnTestDB(t)
+ ctx := context.Background()
+
+ validID := strings.Repeat("a", tenantModelIDWidth)
+ orphan := "3"
+ blank := ""
+ tooShort := strings.Repeat("b", tenantModelIDWidth-1)
+ rows := []struct {
+ id string
+ val *string
+ }{
+ {"d1", &orphan}, // leftover int key
+ {"d2", nil}, // already unset
+ {"d3", &validID}, // real model id
+ {"d4", &blank}, // empty string
+ {"d5", &tooShort}, // too short
+ }
+ for _, row := range rows {
+ if err := db.Exec("INSERT INTO legacy_dialog (id, tenant_llm_id) VALUES (?, ?)", row.id, row.val).Error; err != nil {
+ t.Fatalf("insert %s: %v", row.id, err)
+ }
+ }
+
+ discardOrphanedColumnValues(ctx, db, columnSpec{
+ table: "legacy_dialog", column: "tenant_llm_id", keepLength: tenantModelIDWidth,
+ })
+
+ var cleared int64
+ if err := db.Raw("SELECT COUNT(1) FROM legacy_dialog WHERE tenant_llm_id IS NULL").Scan(&cleared).Error; err != nil {
+ t.Fatalf("count cleared: %v", err)
+ }
+ if cleared != 4 {
+ t.Errorf("cleared rows = %d, want 4", cleared)
+ }
+
+ var survivor string
+ if err := db.Raw("SELECT tenant_llm_id FROM legacy_dialog WHERE id = 'd3'").Scan(&survivor).Error; err != nil {
+ t.Fatalf("read d3: %v", err)
+ }
+ if survivor != validID {
+ t.Errorf("valid model id was rewritten to %q", survivor)
+ }
+}
+
+// TestDiscardOrphanedColumnValuesIsNoopWithoutKeepLength pins the opt-in: a
+// spec that declares no width is a plain type change and must not touch data.
+func TestDiscardOrphanedColumnValuesIsNoopWithoutKeepLength(t *testing.T) {
+ db := setupOrphanColumnTestDB(t)
+ ctx := context.Background()
+
+ if err := db.Exec("INSERT INTO legacy_dialog (id, tenant_llm_id) VALUES ('d1', '3')").Error; err != nil {
+ t.Fatalf("insert: %v", err)
+ }
+
+ discardOrphanedColumnValues(ctx, db, columnSpec{table: "legacy_dialog", column: "tenant_llm_id"})
+
+ var survivor string
+ if err := db.Raw("SELECT tenant_llm_id FROM legacy_dialog WHERE id = 'd1'").Scan(&survivor).Error; err != nil {
+ t.Fatalf("read d1: %v", err)
+ }
+ if survivor != "3" {
+ t.Errorf("tenant_llm_id = %q, want untouched 3", survivor)
+ }
+}