RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: handle int tenant_*_id (#20830)
Lynn79c24d8286cb
diff --git a/internal/dao/migration.go b/internal/dao/migration.goindex da661a0e5c..1aa4c9bea3 100644--- a/internal/dao/migration.go+++ b/internal/dao/migration.go@@ -23,6 +23,7 @@ import ("ragflow/internal/common""ragflow/internal/entity""regexp"+ "sort""strings""go.uber.org/zap"@@ -661,6 +662,7 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {definition: "datetime",},}+ specs = append(specs, tenantModelIDColumnSpecs()...)for _, spec := range specs {if err := applyColumnSpec(ctx, db, spec); err != nil {@@ -671,6 +673,45 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {return nil}+// tenantModelIDColumnSpecs projects tenantModelIDFields into column specs for+// the tenant model id columns. These columns were once declared *int64, so a+// database built by one of those releases holds them as bigint and the daily+// cast back to varchar(32) is what lets AutoMigrate succeed at all. Sharing+// tenantModelIDFields keeps this list and the backfill reading one inventory.+func tenantModelIDColumnSpecs() []columnSpec {+ tables := tenantModelIDFieldTables()++ total := 0+ for _, table := range tables {+ total += len(tenantModelIDFields[table])+ }++ specs := make([]columnSpec, 0, total)+ for _, table := range tables {+ for _, field := range tenantModelIDFields[table] {+ specs = append(specs, columnSpec{+ table: table,+ column: field.tenantIDColumn,+ columnType: "varchar(32)", nullable: true,+ definition: "varchar(32)",+ keepLength: 32,+ })+ }+ }+ return specs+}++// tenantModelIDFieldTables lists the tables carrying tenant model id columns in+// a stable order, so both the spec builder and anything reporting on them agree.+func tenantModelIDFieldTables() []string {+ tables := make([]string, 0, len(tenantModelIDFields))+ for table := range tenantModelIDFields {+ tables = append(tables, table)+ }+ sort.Strings(tables)+ return tables+}+// renameColumnIfExists renames a column if it exists and the new column doesn't existfunc renameColumnIfExists(ctx context.Context, db *gorm.DB, tableName, oldName, newName string) error {if !db.WithContext(ctx).Migrator().HasTable(tableName) {@@ -968,6 +1009,7 @@ type columnSpec struct {columnType string // compared against INFORMATION_SCHEMA.COLUMNS.COLUMN_TYPEnullable booldefinition string // used verbatim when an ALTER is required+ keepLength int // when non-zero, values wider or narrower than this are orphaned by the type change and reset to NULL}// applyColumnSpec aligns a column with its expected definition, issuing DDL only@@ -1001,6 +1043,49 @@ func applyColumnSpec(ctx context.Context, db *gorm.DB, spec columnSpec) error {zap.String("table", spec.table),zap.String("column", spec.column),zap.Error(err))+ return nil}+ discardOrphanedColumnValues(ctx, db, spec)return nil}++// discardOrphanedColumnValues clears values that the type change in spec leaves+// behind. This exists because a successful MODIFY says nothing about whether+// the surviving values are still meaningful: an int column converted to a+// hex-id column keeps "3", which every reader treats as an unresolved model+// reference rather than as the unset value it has become.+//+// LENGTH() is NULL on NULL rows and NULL <> any number is not true, so rows+// already unset are never counted or rewritten.+func discardOrphanedColumnValues(ctx context.Context, db *gorm.DB, spec columnSpec) {+ if spec.keepLength <= 0 {+ return+ }++ stale := fmt.Sprintf("SELECT COUNT(1) FROM %s WHERE LENGTH(%s) <> %d", spec.table, spec.column, spec.keepLength)+ var count int64+ if err := db.WithContext(ctx).Raw(stale).Scan(&count).Error; err != nil {+ common.Warn("Failed to count orphaned values",+ zap.String("table", spec.table),+ zap.String("column", spec.column),+ zap.Error(err))+ return+ }+ if count == 0 {+ return+ }++ common.Info("Discarding orphaned values from converted column",+ zap.String("table", spec.table),+ zap.String("column", spec.column),+ zap.Int64("count", count))+ result := db.WithContext(ctx).Exec(+ fmt.Sprintf("UPDATE %s SET %s = NULL WHERE LENGTH(%s) <> %d", spec.table, spec.column, spec.column, spec.keepLength),+ )+ if result.Error != nil {+ common.Warn("Failed to discard orphaned values",+ zap.String("table", spec.table),+ zap.String("column", spec.column),+ zap.Error(result.Error))+ }+}diff --git a/internal/dao/migration_integration_test.go b/internal/dao/migration_integration_test.gonew file mode 100644index 0000000000..4ab7207424--- /dev/null+++ b/internal/dao/migration_integration_test.go@@ -0,0 +1,97 @@+//go:build integration+// +build integration++package dao++import (+ "context"+ "os"+ "testing"++ "gorm.io/gorm"+)++// TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration reproduces+// the schema that blocks startup: tenant model id columns created while the Go+// entities declared them *int64, which leaves MySQL holding bigint. No+// migration step converts them back, so AutoMigrate ends up being the one that+// tries and fails.+//+// The specs name real tables, so this needs a throwaway database rather than a+// shared one. Finding any of those tables already present aborts before any DDL+// lands, which is what keeps the DROP in t.Cleanup from ever touching a live+// schema.+//+// Run with:+//+// RAGFLOW_TEST_MYSQL_DSN='root:infini_rag_flow@tcp(127.0.0.1:3306)/ragflow_test?charset=utf8mb4&parseTime=True&loc=Local' \+// bash build.sh --test-integration ./internal/dao/...+func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testing.T) {+ dsn := os.Getenv("RAGFLOW_TEST_MYSQL_DSN")+ if dsn == "" {+ t.Skip("RAGFLOW_TEST_MYSQL_DSN unset: this test needs a throwaway MySQL database")+ }++ db, err := gorm.Open(newMigrationAwareDialector(dsn), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("failed to connect: %v", err)+ }++ ctx := context.Background()+ for _, table := range tenantModelIDFieldTables() {+ if db.WithContext(ctx).Migrator().HasTable(table) {+ t.Fatalf("%s already exists in this database; refusing to run against a live schema", table)+ }+ }++ create := `CREATE TABLE dialog (+ id varchar(32) NOT NULL,+ llm_id varchar(128) NOT NULL DEFAULT '',+ rerank_id varchar(128) NOT NULL DEFAULT '',+ tenant_llm_id bigint DEFAULT NULL,+ tenant_rerank_id bigint DEFAULT NULL,+ PRIMARY KEY (id)+ )`+ if err = db.WithContext(ctx).Exec(create).Error; err != nil {+ t.Fatalf("failed to create legacy dialog: %v", err)+ }+ t.Cleanup(func() {+ _ = db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS dialog").Error+ })++ // llm_id is empty on purpose: this row would not be backfilled even if the+ // version gate let populateTenantModelIDColumns through, so the type repair+ // is its only chance to stop holding a dead reference.+ if err = db.WithContext(ctx).Exec(+ "INSERT INTO dialog (id, llm_id, rerank_id, tenant_llm_id, tenant_rerank_id) VALUES ('d1', '', '', 3, 7)",+ ).Error; err != nil {+ t.Fatalf("failed to seed dialog: %v", err)+ }++ if err = modifyColumnTypes(ctx, db); err != nil {+ t.Fatalf("modifyColumnTypes failed: %v", err)+ }++ for _, column := range []string{"tenant_llm_id", "tenant_rerank_id"} {+ columnType, _, ok, err := columnDefinition(ctx, db, "dialog", column)+ if err != nil {+ t.Fatalf("failed to read %s definition: %v", column, err)+ }+ if !ok {+ t.Fatalf("%s vanished from dialog", column)+ }+ if columnType != "varchar(32)" {+ t.Errorf("%s type = %q, want varchar(32)", column, columnType)+ }+ }++ var remaining int64+ if err = db.WithContext(ctx).Raw(+ "SELECT COUNT(1) FROM dialog WHERE tenant_llm_id IS NOT NULL OR tenant_rerank_id IS NOT NULL",+ ).Scan(&remaining).Error; err != nil {+ t.Fatalf("failed to count remaining references: %v", err)+ }+ if remaining != 0 {+ t.Errorf("%d rows still carry an orphaned integer model id", remaining)+ }+}diff --git a/internal/dao/migration_test.go b/internal/dao/migration_test.gonew file mode 100644index 0000000000..4591a11087--- /dev/null+++ b/internal/dao/migration_test.go@@ -0,0 +1,162 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "strings"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"+)++const tenantModelIDWidth = 32++// setupOrphanColumnTestDB builds the narrow shape discardOrphanedColumnValues+// needs: a table holding values left over from an int column widened to hold+// hex model ids.+func setupOrphanColumnTestDB(t *testing.T) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }+ if err = db.Exec("CREATE TABLE legacy_dialog (id TEXT PRIMARY KEY, tenant_llm_id TEXT)").Error; err != nil {+ t.Fatalf("failed to create legacy_dialog: %v", err)+ }+ return db+}++// TestTenantModelIDColumnSpecsMirrorsBackfillFields guards the shared inventory:+// the type repair and the backfill must cover exactly the same columns, and the+// order must be stable so consecutive startups log identically.+func TestTenantModelIDColumnSpecsMirrorsBackfillFields(t *testing.T) {+ want := 0+ for _, fields := range tenantModelIDFields {+ want += len(fields)+ }++ specs := tenantModelIDColumnSpecs()+ if len(specs) != want {+ t.Fatalf("spec count = %d, want %d", len(specs), want)+ }++ got := map[string]columnSpec{}+ for _, spec := range specs {+ got[spec.table+"."+spec.column] = spec+ }++ for table, fields := range tenantModelIDFields {+ for _, field := range fields {+ spec, ok := got[table+"."+field.tenantIDColumn]+ if !ok {+ t.Fatalf("missing spec for %s.%s", table, field.tenantIDColumn)+ }+ if spec.columnType != "varchar(32)" || spec.definition != "varchar(32)" {+ t.Errorf("%s.%s types = %q/%q, want varchar(32)/varchar(32)",+ table, field.tenantIDColumn, spec.columnType, spec.definition)+ }+ if !spec.nullable {+ t.Errorf("%s.%s nullable = false, want true", table, field.tenantIDColumn)+ }+ if spec.keepLength != tenantModelIDWidth {+ t.Errorf("%s.%s keepLength = %d, want %d", table, field.tenantIDColumn, spec.keepLength, tenantModelIDWidth)+ }+ if strings.Contains(spec.column, "`") {+ t.Errorf("%s.%s column carries a backtick: %q", table, field.tenantIDColumn, spec.column)+ }+ }+ }++ again := tenantModelIDColumnSpecs()+ for i := range specs {+ if specs[i] != again[i] {+ t.Fatalf("spec order is not deterministic at index %d: %v then %v", i, specs[i], again[i])+ }+ }+}++// TestDiscardOrphanedColumnValuesResetsOnlyOrphans covers the values a widened+// int column carries into its new life as a hex-id column. "3" is what matters:+// it looks like a model id to every reader, so leaving it behind turns a visible+// migration failure into silent bad lookups.+func TestDiscardOrphanedColumnValuesResetsOnlyOrphans(t *testing.T) {+ db := setupOrphanColumnTestDB(t)+ ctx := context.Background()++ validID := strings.Repeat("a", tenantModelIDWidth)+ orphan := "3"+ blank := ""+ tooShort := strings.Repeat("b", tenantModelIDWidth-1)+ rows := []struct {+ id string+ val *string+ }{+ {"d1", &orphan}, // leftover int key+ {"d2", nil}, // already unset+ {"d3", &validID}, // real model id+ {"d4", &blank}, // empty string+ {"d5", &tooShort}, // too short+ }+ for _, row := range rows {+ if err := db.Exec("INSERT INTO legacy_dialog (id, tenant_llm_id) VALUES (?, ?)", row.id, row.val).Error; err != nil {+ t.Fatalf("insert %s: %v", row.id, err)+ }+ }++ discardOrphanedColumnValues(ctx, db, columnSpec{+ table: "legacy_dialog", column: "tenant_llm_id", keepLength: tenantModelIDWidth,+ })++ var cleared int64+ if err := db.Raw("SELECT COUNT(1) FROM legacy_dialog WHERE tenant_llm_id IS NULL").Scan(&cleared).Error; err != nil {+ t.Fatalf("count cleared: %v", err)+ }+ if cleared != 4 {+ t.Errorf("cleared rows = %d, want 4", cleared)+ }++ var survivor string+ if err := db.Raw("SELECT tenant_llm_id FROM legacy_dialog WHERE id = 'd3'").Scan(&survivor).Error; err != nil {+ t.Fatalf("read d3: %v", err)+ }+ if survivor != validID {+ t.Errorf("valid model id was rewritten to %q", survivor)+ }+}++// TestDiscardOrphanedColumnValuesIsNoopWithoutKeepLength pins the opt-in: a+// spec that declares no width is a plain type change and must not touch data.+func TestDiscardOrphanedColumnValuesIsNoopWithoutKeepLength(t *testing.T) {+ db := setupOrphanColumnTestDB(t)+ ctx := context.Background()++ if err := db.Exec("INSERT INTO legacy_dialog (id, tenant_llm_id) VALUES ('d1', '3')").Error; err != nil {+ t.Fatalf("insert: %v", err)+ }++ discardOrphanedColumnValues(ctx, db, columnSpec{table: "legacy_dialog", column: "tenant_llm_id"})++ var survivor string+ if err := db.Raw("SELECT tenant_llm_id FROM legacy_dialog WHERE id = 'd1'").Scan(&survivor).Error; err != nil {+ t.Fatalf("read d1: %v", err)+ }+ if survivor != "3" {+ t.Errorf("tenant_llm_id = %q, want untouched 3", survivor)+ }+}