RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feat: migrate chat message and reference (#19724)
Lynn81a5fdd56e8a
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex 99d3025936..daf7d5004d 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -739,7 +739,7 @@ jobs:# because .env-go's COMPOSE_PROFILES lacks "tei-", causing# PATCH /models/default to fail for the Builtin/Local embedding# model and datasets to be created with an empty embedding_model.- sed -i '/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d' .env-go+ sed -i '/^COMPOSE_PROFILES=/d;/^RAGFLOW_DEV_MODE=/d;/^TEI_MODEL=/d' .env-go{echo ""echo "ES_PORT=${ES_PORT}"@@ -780,6 +780,7 @@ jobs:# config as the Python container. Only these two affect the# in-process checkModelAvailable logic; other .env keys are# either yml-interpolation-only or already correct in .env-go.+ # RAGFLOW_DEV_MODE rides along for the same reason.{echo ""if [ "${API_PROXY_SCHEME}" = "go" ]; then@@ -788,6 +789,11 @@ jobs:echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"fiecho "TEI_MODEL=BAAI/bge-small-en-v1.5"+ # The image stamps its git-describe output into /ragflow/VERSION,+ # so this checkout reports v0.27.x while --migrate records the+ # v1.0.0-rc1.dev1 database marker. Without the override the Go+ # server refuses to boot against the marker it just wrote.+ echo "RAGFLOW_DEV_MODE=true"} >> .env-goif [[ "${API_PROXY_SCHEME}" == "go" ]]; thenRAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}@@ -1417,7 +1423,7 @@ jobs:# because .env-go's COMPOSE_PROFILES lacks "tei-", causing# PATCH /models/default to fail for the Builtin/Local embedding# model and datasets to be created with an empty embedding_model.- sed -i '/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d' .env-go+ sed -i '/^COMPOSE_PROFILES=/d;/^RAGFLOW_DEV_MODE=/d;/^TEI_MODEL=/d' .env-go{echo ""echo "ES_PORT=${ES_PORT}"@@ -1458,6 +1464,7 @@ jobs:# config as the Python container. Only these two affect the# in-process checkModelAvailable logic; other .env keys are# either yml-interpolation-only or already correct in .env-go.+ # RAGFLOW_DEV_MODE rides along for the same reason.{echo ""if [ "${API_PROXY_SCHEME}" = "go" ]; then@@ -1466,6 +1473,11 @@ jobs:echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"fiecho "TEI_MODEL=BAAI/bge-small-en-v1.5"+ # The image stamps its git-describe output into /ragflow/VERSION,+ # so this checkout reports v0.27.x while --migrate records the+ # v1.0.0-rc1.dev1 database marker. Without the override the Go+ # server refuses to boot against the marker it just wrote.+ echo "RAGFLOW_DEV_MODE=true"} >> .env-goif [[ "${API_PROXY_SCHEME}" == "go" ]]; thenRAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 2248b9cc9c..5137fc7a8c 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -564,7 +564,17 @@ func main() {// A missing marker, or a version on either side that cannot be parsed, never// blocks startup: without a usable comparison there is no evidence that the// database is ahead of the code.+//+// RAGFLOW_DEV_MODE turns the check off entirely. A development build can carry+// a marker for a release that is not tagged yet, in which case the comparison+// would reject the build that wrote the marker.func checkDatabaseVersion(ctx context.Context) error {+ if common.DevModeEnabled() {+ common.Warn("Development mode is enabled, skipping the database downgrade check",+ zap.String("env", common.EnvRAGFlowDevMode))+ return nil+ }+databaseVersion, err := dao.GetDatabaseMigrationVersion(ctx, dao.DB)if err != nil {return fmt.Errorf("read database version marker: %w", err)diff --git a/docker/.env b/docker/.envindex 147d1b6c13..5bb2d7a12d 100644--- a/docker/.env+++ b/docker/.env@@ -5,6 +5,15 @@# You can generate a random string using: openssl rand -hex 32# -----------------------------------------------------------------------------+# ------------------------------+# Development mode+# ------------------------------+# Set to `true` only while developing: the server then starts even when the+# running code reports a version older than the database migration marker,+# which any branch targeting a not-yet-tagged release does. Leave it off in+# production, where an older binary must not touch a newer database.+RAGFLOW_DEV_MODE=false+# ------------------------------# docker env var for specifying vector db type at startup# (based on the vector db type, the corresponding dockerdiff --git a/docker/.env-go b/docker/.env-goindex 606e6ad11d..3c9707587e 100644--- a/docker/.env-go+++ b/docker/.env-go@@ -5,6 +5,20 @@# You can generate a random string using: openssl rand -hex 32# -----------------------------------------------------------------------------+# ------------------------------+# Development mode+# ------------------------------+# Set to `true` only while developing: the server then starts even when the+# running code reports a version older than the database migration marker,+# which any branch targeting a not-yet-tagged release does. Leave it off in+# production, where an older binary must not touch a newer database.+#+# CI (.github/workflows/sep-tests.yml) appends `true` here on every run: the+# image stamps its own git-describe output into /ragflow/VERSION, so a CI build+# reports v0.27.x against the v1.0.0-rc1.dev1 marker it migrated to moments+# earlier. Do not treat that generated value as the default.+RAGFLOW_DEV_MODE=false+# ------------------------------# docker env var for specifying vector db type at startup# (based on the vector db type, the corresponding dockerdiff --git a/internal/common/environments.go b/internal/common/environments.goindex cc526ae10c..24387050d5 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -39,6 +39,7 @@ func IsLLMDebugEnabled() bool {// environment variablesconst (EnvTensorrtDLAServer = "TENSORRT_DLA_SVR"+ EnvRAGFlowDevMode = "RAGFLOW_DEV_MODE"EnvRAGFlowTTSCacheTTLSeconds = "RAGFLOW_TTS_CACHE_TTL_SECONDS"EnvRerankTokenLimitMode = "RERANK_TOKEN_LIMIT_MODE"EnvComponentExecTimeout = "COMPONENT_EXEC_TIMEOUT"diff --git a/internal/common/version.go b/internal/common/version.goindex 2f9d212a89..f8b5412b9e 100644--- a/internal/common/version.go+++ b/internal/common/version.go@@ -19,6 +19,7 @@ import ("os""os/exec""path/filepath"+ "strconv""strings""sync"@@ -97,6 +98,20 @@ func IsOlderReleaseThan(codeVersion, targetVersion string) (older, comparable boreturn semver.Compare(code, target) < 0, true}+// DevModeEnabled reports whether this process runs in development mode, which+// relaxes the guard that refuses to start when the code is older than the+// database. Development builds routinely carry a marker for a release that has+// not been tagged yet — the migration that splits conversation history records+// v1.0.0-rc1.dev1 while the branch still reports v0.27.x — so the guard would+// otherwise block the very deployment that wrote the marker.+//+// It is off unless the variable is explicitly set to a true value, so a+// production deployment keeps the guard even when the variable is absent.+func DevModeEnabled() bool {+ enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(EnvRAGFlowDevMode)))+ return err == nil && enabled+}+// releaseVersion normalizes a version string to the "vMAJOR.MINOR.PATCH" form// semver accepts, dropping any pre-release or build suffix. It returns an empty// string when the result is not a valid semver tag.diff --git a/internal/common/version_test.go b/internal/common/version_test.goindex f3627d7201..5b1c064caf 100644--- a/internal/common/version_test.go+++ b/internal/common/version_test.go@@ -21,6 +21,34 @@ import ("testing")+func TestDevModeEnabled(t *testing.T) {+ cases := []struct {+ value string+ set bool+ want bool+ }{+ {value: "", set: false, want: false},+ {value: "", set: true, want: false},+ {value: "false", set: true, want: false},+ {value: "0", set: true, want: false},+ {value: "nonsense", set: true, want: false},+ {value: "true", set: true, want: true},+ {value: "True", set: true, want: true},+ {value: "1", set: true, want: true},+ {value: " true ", set: true, want: true},+ }+ for _, tc := range cases {+ t.Run(fmt.Sprintf("%q/set=%v", tc.value, tc.set), func(t *testing.T) {+ if tc.set {+ t.Setenv(EnvRAGFlowDevMode, tc.value)+ }+ if got := DevModeEnabled(); got != tc.want {+ t.Fatalf("DevModeEnabled() = %v, want %v", got, tc.want)+ }+ })+ }+}+func TestGetRAGFlowVersion(t *testing.T) {version := GetRAGFlowVersion()fmt.Printf("RAGFlow Version: %s\n", version)diff --git a/internal/dao/conversation_history.go b/internal/dao/conversation_history.goindex 11382ffcdf..287f995447 100644--- a/internal/dao/conversation_history.go+++ b/internal/dao/conversation_history.go@@ -331,9 +331,22 @@ func messageValues(fields entity.ConversationMessageFields) map[string]interface}func createHistory(ctx context.Context, db *gorm.DB, table, kind, conversationID string, raw json.RawMessage) error {+ rows, err := historyRows(kind, conversationID, raw)+ if err != nil {+ return err+ }+ if len(rows) == 0 {+ return nil+ }+ return db.WithContext(ctx).Table(table).CreateInBatches(rows, 100).Error+}++// historyRows flattens a message or reference payload into child rows. Its+// positions are the payload indexes, starting from zero.+func historyRows(kind, conversationID string, raw json.RawMessage) ([]map[string]interface{}, error) {items, err := splitHistory(raw, kind)if err != nil {- return fmt.Errorf("split %s history: %w", kind, err)+ return nil, fmt.Errorf("split %s history: %w", kind, err)}rows := make([]map[string]interface{}, 0, len(items))for position, item := range items {@@ -341,17 +354,14 @@ func createHistory(ctx context.Context, db *gorm.DB, table, kind, conversationIDif kind == "message" {fields, err := flattenMessage(item)if err != nil {- return fmt.Errorf("flatten message %d: %w", position, err)+ return nil, fmt.Errorf("flatten message %d: %w", position, err)}row = messageValues(fields)row["conversation_id"], row["position"] = conversationID, position}rows = append(rows, row)}- if len(rows) == 0 {- return nil- }- return db.WithContext(ctx).Table(table).CreateInBatches(rows, 100).Error+ return rows, nil}func loadHistory(ctx context.Context, db *gorm.DB, table, payloadColumn string, conversationIDs []string) (map[string]json.RawMessage, error) {diff --git a/internal/dao/conversation_history_migration.go b/internal/dao/conversation_history_migration.gonew file mode 100644index 0000000000..a82b31e5a7--- /dev/null+++ b/internal/dao/conversation_history_migration.go@@ -0,0 +1,232 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "encoding/json"+ "fmt"++ "go.uber.org/zap"+ "gorm.io/gorm"++ "ragflow/internal/common"+)++// conversationHistoryTargetVersion is the database version a completed split+// brings the database to. It is stored in system_settings under+// mysql_migration.database.version, the marker shared with the tenant model+// migration, so both steps read the same progress record instead of keeping one+// each. The dev prerelease keeps it below v1.0.0-rc1, which owns the schema+// this step only prepares.+const conversationHistoryTargetVersion = "v1.0.0-rc1.dev1"++// conversationHistoryBatchSize bounds how many conversations are read and+// written per transaction, so an interrupted run resumes from a committed+// batch instead of reprocessing the whole table.+const conversationHistoryBatchSize = 100++// legacyConversationHistory addresses conversation.message and+// conversation.reference. ChatSession drops both columns with gorm:"-", so a+// probe model is the only way to reach them.+type legacyConversationHistory struct {+ ID string `gorm:"column:id;primaryKey;size:32"`+ Message *string `gorm:"column:message"`+ Reference *string `gorm:"column:reference"`+}++func (legacyConversationHistory) TableName() string {+ return "conversation"+}++// legacyAPI4ConversationHistory is the api_4_conversation counterpart of+// legacyConversationHistory.+type legacyAPI4ConversationHistory struct {+ ID string `gorm:"column:id;primaryKey;size:32"`+ Message *string `gorm:"column:message"`+ Reference *string `gorm:"column:reference"`+}++func (legacyAPI4ConversationHistory) TableName() string {+ return "api_4_conversation"+}++// legacyHistoryRow is one parent row read for backfill. Columns that the+// parent table does not carry stay nil.+type legacyHistoryRow struct {+ ID string `gorm:"column:id"`+ Message *string `gorm:"column:message"`+ Reference *string `gorm:"column:reference"`+}++// conversationHistorySplit maps one parent table onto its two child tables.+type conversationHistorySplit struct {+ parentTable string+ probe interface{}+ messageTable string+ referenceTable string+}++func (s conversationHistorySplit) childTables() []string {+ return []string{s.messageTable, s.referenceTable}+}++// migrateConversationHistory backfills conversation_message,+// conversation_reference and their api_4_conversation counterparts from the+// message and reference payload columns the parent tables still hold.+//+// It runs after AutoMigrate: unlike RunMigrations, which has to observe the+// legacy schema, it writes to tables AutoMigrate creates.+func migrateConversationHistory(ctx context.Context, db *gorm.DB) error {+ currentVersion, err := GetDatabaseMigrationVersion(ctx, db)+ if err != nil {+ return fmt.Errorf("read database migration version: %w", err)+ }+ if shouldSkipMigration(currentVersion, conversationHistoryTargetVersion) {+ common.Info("Conversation history already split, skipping",+ zap.String("current_version", currentVersion),+ zap.String("target_version", conversationHistoryTargetVersion))+ return nil+ }+ splits := []conversationHistorySplit{+ {+ parentTable: "conversation",+ probe: &legacyConversationHistory{},+ messageTable: conversationMessageTable,+ referenceTable: conversationReferenceTable,+ },+ {+ parentTable: "api_4_conversation",+ probe: &legacyAPI4ConversationHistory{},+ messageTable: apiConversationMessageTable,+ referenceTable: apiConversationReferenceTable,+ },+ }+ for _, split := range splits {+ if err := migrateConversationHistorySplit(ctx, db, split); err != nil {+ return err+ }+ }+ if err := setDatabaseMigrationVersion(ctx, db, conversationHistoryTargetVersion); err != nil {+ return fmt.Errorf("mark database migration version %s: %w", conversationHistoryTargetVersion, err)+ }+ common.Info("Conversation history migration completed", zap.String("version", conversationHistoryTargetVersion))+ return nil+}++func migrateConversationHistorySplit(ctx context.Context, db *gorm.DB, split conversationHistorySplit) error {+ scoped := db.WithContext(ctx)+ if !scoped.Migrator().HasTable(split.parentTable) {+ return nil+ }+ hasMessage := scoped.Migrator().HasColumn(split.probe, "message")+ hasReference := scoped.Migrator().HasColumn(split.probe, "reference")+ if !hasMessage && !hasReference {+ // The Go schema never carried these columns, so there is nothing to split.+ return nil+ }+ columns := []string{"id"}+ if hasMessage {+ columns = append(columns, "message")+ }+ if hasReference {+ columns = append(columns, "reference")+ }++ lastID := ""+ migrated := 0+ for {+ var batch []legacyHistoryRow+ err := scoped.Table(split.parentTable).Select(columns).+ Where("id > ?", lastID).Order("id").Limit(conversationHistoryBatchSize).Find(&batch).Error+ if err != nil {+ return fmt.Errorf("read %s history payloads: %w", split.parentTable, err)+ }+ if len(batch) == 0 {+ break+ }+ err = scoped.Transaction(func(tx *gorm.DB) error {+ for _, row := range batch {+ written, err := splitConversationHistory(ctx, tx, split, row)+ if err != nil {+ return err+ }+ if written {+ migrated+++ }+ }+ return nil+ })+ if err != nil {+ return err+ }+ lastID = batch[len(batch)-1].ID+ }+ common.Info("Split conversation history",+ zap.String("table", split.parentTable), zap.Int("conversations", migrated))+ return nil+}++// splitConversationHistory writes one parent row's payloads into the child+// tables. Rows written after the split took effect win over the payload, so a+// rerun never duplicates or clobbers them.+func splitConversationHistory(ctx context.Context, db *gorm.DB, split conversationHistorySplit, row legacyHistoryRow) (bool, error) {+ for _, table := range split.childTables() {+ hasRows, err := conversationHistoryHasRows(ctx, db, table, row.ID)+ if err != nil {+ return false, err+ }+ if hasRows {+ return false, nil+ }+ }+ written := false+ if row.Message != nil {+ rows, err := historyRows("message", row.ID, json.RawMessage(*row.Message))+ if err != nil {+ return false, fmt.Errorf("split %s %s message payload: %w", split.parentTable, row.ID, err)+ }+ if len(rows) > 0 {+ if err := db.WithContext(ctx).Table(split.messageTable).CreateInBatches(rows, 100).Error; err != nil {+ return false, err+ }+ written = true+ }+ }+ if row.Reference != nil {+ rows, err := historyRows("reference", row.ID, json.RawMessage(*row.Reference))+ if err != nil {+ return false, fmt.Errorf("split %s %s reference payload: %w", split.parentTable, row.ID, err)+ }+ if len(rows) > 0 {+ if err := db.WithContext(ctx).Table(split.referenceTable).CreateInBatches(rows, 100).Error; err != nil {+ return false, err+ }+ written = true+ }+ }+ return written, nil+}++func conversationHistoryHasRows(ctx context.Context, db *gorm.DB, table, conversationID string) (bool, error) {+ var count int64+ err := db.WithContext(ctx).Table(table).Where("conversation_id = ?", conversationID).Count(&count).Error+ if err != nil {+ return false, err+ }+ return count > 0, nil+}diff --git a/internal/dao/conversation_history_migration_test.go b/internal/dao/conversation_history_migration_test.gonew file mode 100644index 0000000000..10d5fff024--- /dev/null+++ b/internal/dao/conversation_history_migration_test.go@@ -0,0 +1,278 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "fmt"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/entity"+)++// setupConversationHistoryMigrationDB builds a schema that mirrors a database+// created before the split: the Go entities drop the message and reference+// columns, so add them back the way the previous schema had them.+func setupConversationHistoryMigrationDB(t *testing.T, legacyColumns bool) *gorm.DB {+ t.Helper()++ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{+ TranslateError: true,+ })+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }++ models := []interface{}{+ &entity.ChatSession{},+ &entity.ConversationMessage{},+ &entity.ConversationReference{},+ &entity.API4Conversation{},+ &entity.API4ConversationMessage{},+ &entity.API4ConversationReference{},+ &entity.SystemSettings{},+ }+ if err := db.AutoMigrate(models...); err != nil {+ t.Fatalf("failed to migrate: %v", err)+ }++ if legacyColumns {+ for _, table := range []string{"conversation", "api_4_conversation"} {+ for _, column := range []string{"message", "reference"} {+ if err := db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s longtext", table, column)).Error; err != nil {+ t.Fatalf("failed to add %s.%s: %v", table, column, err)+ }+ }+ }+ }++ return db+}++func insertLegacyConversation(t *testing.T, db *gorm.DB, table, id string, message, reference string) {+ t.Helper()+ if err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, dialog_id, user_id, message, reference) VALUES (?, ?, ?, ?, ?)", table),+ id, "dialog-"+id, "user-"+id, message, reference).Error; err != nil {+ t.Fatalf("failed to insert %s %s: %v", table, id, err)+ }+}++// insertConversation adds a parent row with no payload columns at all, the way+// the Go schema creates them.+func insertConversation(t *testing.T, db *gorm.DB, table, id string) {+ t.Helper()+ if err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, dialog_id, user_id) VALUES (?, ?, ?)", table),+ id, "dialog-"+id, "user-"+id).Error; err != nil {+ t.Fatalf("failed to insert %s %s: %v", table, id, err)+ }+}++func countHistoryRows(t *testing.T, db *gorm.DB, table string) int64 {+ t.Helper()+ var count int64+ if err := db.Table(table).Count(&count).Error; err != nil {+ t.Fatalf("failed to count %s: %v", table, err)+ }+ return count+}++func databaseVersion(t *testing.T, db *gorm.DB) string {+ t.Helper()+ version, err := GetDatabaseMigrationVersion(t.Context(), db)+ if err != nil {+ t.Fatalf("failed to read database version: %v", err)+ }+ return version+}++func writeDatabaseVersion(t *testing.T, db *gorm.DB, version string) {+ t.Helper()+ if err := setDatabaseMigrationVersion(t.Context(), db, version); err != nil {+ t.Fatalf("failed to write database version %s: %v", version, err)+ }+}++func TestMigrateConversationHistorySplitsPayloads(t *testing.T) {+ message := `[{"role":"user","content":"hi","id":"m1"},{"role":"assistant","content":"hello","id":"m2","reference":[{"chunk-0":{"content":"chunk"}}]}]`+ reference := `[{"chunks":["a"],"doc_aggs":[]},{"chunks":["b"],"doc_aggs":[]}]`+ db := setupConversationHistoryMigrationDB(t, true)+ insertLegacyConversation(t, db, "conversation", "c1", message, reference)+ insertLegacyConversation(t, db, "api_4_conversation", "a1", message, reference)++ if err := migrateConversationHistory(t.Context(), db); err != nil {+ t.Fatalf("migrateConversationHistory: %v", err)+ }++ for _, tc := range []struct{ messageTable, referenceTable string }{+ {conversationMessageTable, conversationReferenceTable},+ {apiConversationMessageTable, apiConversationReferenceTable},+ } {+ if got := countHistoryRows(t, db, tc.messageTable); got != 2 {+ t.Fatalf("%s: expected 2 messages, got %d", tc.messageTable, got)+ }+ if got := countHistoryRows(t, db, tc.referenceTable); got != 2 {+ t.Fatalf("%s: expected 2 references, got %d", tc.referenceTable, got)+ }+ }++ var msg entity.ConversationMessage+ if err := db.Table(conversationMessageTable).Where("conversation_id = ? AND position = ?", "c1", 1).Take(&msg).Error; err != nil {+ t.Fatalf("load message: %v", err)+ }+ if derefString(msg.Role) != "assistant" || derefString(msg.Content) != "hello" || derefString(msg.MessageID) != "m2" {+ t.Fatalf("unexpected assistant message: %+v", msg)+ }++ var storedReference string+ err := db.Table(conversationReferenceTable).Select("reference").+ Where("conversation_id = ? AND position = ?", "c1", 1).Row().Scan(&storedReference)+ if err != nil {+ t.Fatalf("load reference: %v", err)+ }+ if storedReference != `{"chunks":["b"],"doc_aggs":[]}` {+ t.Fatalf("unexpected reference: %s", storedReference)+ }++ // Rows backfilled from a payload carry no explicit message binding.+ var messagePos *int+ err = db.Table(conversationReferenceTable).Select("message_position").+ Where("conversation_id = ? AND position = ?", "c1", 1).Row().Scan(&messagePos)+ if err != nil {+ t.Fatalf("load message_position: %v", err)+ }+ if messagePos != nil {+ t.Fatalf("expected nil message_position, got %d", *messagePos)+ }++ if got := databaseVersion(t, db); got != conversationHistoryTargetVersion {+ t.Fatalf("expected database version %s, got %s", conversationHistoryTargetVersion, got)+ }+}++func TestMigrateConversationHistoryIsIdempotent(t *testing.T) {+ message := `[{"role":"user","content":"hi","id":"m1"},{"role":"assistant","content":"hello","id":"m2"}]`+ db := setupConversationHistoryMigrationDB(t, true)+ insertLegacyConversation(t, db, "conversation", "c1", message, `[{"chunks":["a"]}]`)+ insertLegacyConversation(t, db, "conversation", "c2", `[]`, `[]`)++ for range 2 {+ // Rewind the marker so the second pass really reruns the backfill+ // instead of short-circuiting on the recorded version.+ writeDatabaseVersion(t, db, modelMigrationTargetVersion)+ if err := migrateConversationHistory(t.Context(), db); err != nil {+ t.Fatalf("migrateConversationHistory: %v", err)+ }+ }++ if got := countHistoryRows(t, db, conversationMessageTable); got != 2 {+ t.Fatalf("expected 2 messages after rerun, got %d", got)+ }+ if got := countHistoryRows(t, db, conversationReferenceTable); got != 1 {+ t.Fatalf("expected 1 reference after rerun, got %d", got)+ }+}++func TestMigrateConversationHistoryKeepsRowsWrittenAfterSplit(t *testing.T) {+ db := setupConversationHistoryMigrationDB(t, true)+ insertLegacyConversation(t, db, "conversation", "c1", `[{"role":"user","content":"hi","id":"m1"}]`, `[]`)+ if err := db.Table(conversationMessageTable).Create(map[string]interface{}{+ "conversation_id": "c1", "position": 0, "role": "assistant", "content": "written by the runtime",+ }).Error; err != nil {+ t.Fatalf("seed message: %v", err)+ }++ if err := migrateConversationHistory(t.Context(), db); err != nil {+ t.Fatalf("migrateConversationHistory: %v", err)+ }++ var msg entity.ConversationMessage+ if err := db.Table(conversationMessageTable).Where("conversation_id = ? AND position = ?", "c1", 0).Take(&msg).Error; err != nil {+ t.Fatalf("load message: %v", err)+ }+ if derefString(msg.Content) != "written by the runtime" {+ t.Fatalf("expected the runtime row to win, got %q", derefString(msg.Content))+ }+}++func derefString(value *string) string {+ if value == nil {+ return ""+ }+ return *value+}++func TestMigrateConversationHistorySkipsTablesWithoutPayloadColumns(t *testing.T) {+ db := setupConversationHistoryMigrationDB(t, false)+ insertConversation(t, db, "conversation", "c1")++ if err := migrateConversationHistory(t.Context(), db); err != nil {+ t.Fatalf("migrateConversationHistory: %v", err)+ }++ if got := countHistoryRows(t, db, conversationMessageTable); got != 0 {+ t.Fatalf("expected no backfill without payload columns, got %d", got)+ }+ // The step still advances the marker: a schema without the payload columns+ // has nothing left to split, ever.+ if got := databaseVersion(t, db); got != conversationHistoryTargetVersion {+ t.Fatalf("expected database version %s, got %s", conversationHistoryTargetVersion, got)+ }+}++func TestMigrateConversationHistoryHonoursRecordedVersion(t *testing.T) {+ for _, tc := range []struct {+ name string+ current string+ migrate bool+ }{+ {"no recorded version", "", true},+ {"release line below the split", "v0.27.2", true},+ {"earlier dev build", "v1.0.0-rc1.dev0", true},+ {"the split version itself", conversationHistoryTargetVersion, false},+ {"release the dev build leads to", "v1.0.0-rc1", false},+ {"later dev build", "v1.0.0-rc1.dev2", false},+ {"later release candidate", "v1.0.0-rc2", false},+ {"final release", "v1.0.0", false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ db := setupConversationHistoryMigrationDB(t, true)+ insertLegacyConversation(t, db, "conversation", "c1", `[{"role":"user","content":"hi"}]`, `[]`)+ if tc.current != "" {+ writeDatabaseVersion(t, db, tc.current)+ }++ if err := migrateConversationHistory(t.Context(), db); err != nil {+ t.Fatalf("migrateConversationHistory: %v", err)+ }++ var wantMessages int64+ wantVersion := tc.current+ if tc.migrate {+ wantMessages = 1+ wantVersion = conversationHistoryTargetVersion+ }+ if got := countHistoryRows(t, db, conversationMessageTable); got != wantMessages {+ t.Fatalf("expected %d messages, got %d", wantMessages, got)+ }+ if got := databaseVersion(t, db); got != wantVersion {+ t.Fatalf("expected database version %s, got %s", wantVersion, got)+ }+ })+ }+}diff --git a/internal/dao/database.go b/internal/dao/database.goindex a5c97f7b38..1543d286b2 100644--- a/internal/dao/database.go+++ b/internal/dao/database.go@@ -189,6 +189,13 @@ func InitDB(ctx context.Context, migrateDB bool) error {}}common.Info("Database schema migrated successfully")++ // Split the conversation message and reference payloads out of their+ // parent tables. It has to run after AutoMigrate, which unlike+ // RunMigrations creates the child tables this backfill writes to.+ if err = migrateConversationHistory(ctx, DB); err != nil {+ return fmt.Errorf("failed to migrate conversation history: %w", err)+ }} else {// Ensure the Go-exclusive runtime tables exist. The manual migrations are// performed by the standalone --migrate action, so a server-mode processdiff --git a/internal/dao/migration_version.go b/internal/dao/migration_version.goindex 5c2fa5604b..e9fdf6761f 100644--- a/internal/dao/migration_version.go+++ b/internal/dao/migration_version.go@@ -20,6 +20,7 @@ import ("context""database/sql""errors"+ "strconv""strings""time"@@ -48,6 +49,10 @@ const modelMigrationBaseVersion = "v0.26.0"// by tools/scripts/run_migrations.sh.const modelMigrationTargetVersion = "v0.27.2"+// devPrereleaseIdentifier is the prerelease identifier that marks a version as+// a development build of the version it is appended to.+const devPrereleaseIdentifier = "dev"+// GetDatabaseMigrationVersion reads the stored migration version. It returns an// empty string when the marker or the system_settings table is absent.func GetDatabaseMigrationVersion(ctx context.Context, db *gorm.DB) (string, error) {@@ -97,14 +102,90 @@ func setDatabaseMigrationVersion(ctx context.Context, db *gorm.DB, version strin}// shouldSkipMigration mirrors should_skip_migration in the Python migration: a-// missing or unparseable version never skips the migration.+// missing or unparseable version never skips the migration. Both sides share+// the mysql_migration.database.version marker, so they have to order versions+// the same way, which for Go means compareMigrationVersion instead of raw+// semver.func shouldSkipMigration(currentDBVersion, targetVersion string) bool {current := parseMigrationVersion(currentDBVersion)target := parseMigrationVersion(targetVersion)if current == "" || target == "" {return false}- return semver.Compare(current, target) >= 0+ return compareMigrationVersion(current, target) >= 0+}++// compareMigrationVersion orders two canonical migration versions, returning -1,+// 0 or 1 as current sorts before, equal to, or after target.+//+// It follows semver except for development prereleases, where it matches PEP+// 440, the ordering the Python migration gets from packaging.version: a+// trailing ".devN" identifier marks a build leading the version it is attached+// to, so v1.0.0-rc1.dev1 sorts below v1.0.0-rc1 and below v1.0.0-rc1.dev2.+// semver orders a longer prerelease above its prefix, which would place a dev+// build ahead of the very release it precedes.+func compareMigrationVersion(current, target string) int {+ currentCore := migrationVersionCore(current)+ targetCore := migrationVersionCore(target)+ if cmp := semver.Compare(currentCore, targetCore); cmp != 0 {+ return cmp+ }+ currentBase, currentDev := splitDevPrerelease(semver.Prerelease(current))+ targetBase, targetDev := splitDevPrerelease(semver.Prerelease(target))+ if cmp := semver.Compare(currentCore+currentBase, targetCore+targetBase); cmp != 0 {+ return cmp+ }+ switch {+ case currentDev == targetDev:+ return 0+ case currentDev < 0:+ // The base version outranks every dev build leading up to it.+ return 1+ case targetDev < 0:+ return -1+ case currentDev < targetDev:+ return -1+ }+ return 1+}++// migrationVersionCore strips prerelease and build metadata, leaving the+// "vMAJOR.MINOR.PATCH" core the two versions are first compared on.+func migrationVersionCore(version string) string {+ core := strings.TrimSuffix(version, semver.Build(version))+ return strings.TrimSuffix(core, semver.Prerelease(version))+}++// splitDevPrerelease splits a "-rc1.dev1" prerelease into its "-rc1" base and+// the trailing development number. dev is -1 when there is no dev identifier.+func splitDevPrerelease(prerelease string) (base string, dev int) {+ trimmed := strings.TrimPrefix(prerelease, "-")+ if trimmed == "" {+ return "", -1+ }+ identifiers := strings.Split(trimmed, ".")+ number, ok := devIdentifierNumber(identifiers[len(identifiers)-1])+ if !ok {+ return prerelease, -1+ }+ base = strings.Join(identifiers[:len(identifiers)-1], ".")+ if base != "" {+ base = "-" + base+ }+ return base, number+}++// devIdentifierNumber reads the N out of a "devN" prerelease identifier.+func devIdentifierNumber(identifier string) (int, bool) {+ number, ok := strings.CutPrefix(identifier, devPrereleaseIdentifier)+ if !ok {+ return 0, false+ }+ value, err := strconv.Atoi(number)+ if err != nil {+ return 0, false+ }+ return value, true}// parseMigrationVersion normalizes a version to the "vMAJOR.MINOR.PATCH" formdiff --git a/internal/dao/migration_version_test.go b/internal/dao/migration_version_test.goindex 1842ad22d4..20ea95fe08 100644--- a/internal/dao/migration_version_test.go+++ b/internal/dao/migration_version_test.go@@ -63,6 +63,33 @@ func TestShouldSkipMigration(t *testing.T) {}}+func TestCompareMigrationVersionDevPrerelease(t *testing.T) {+ cases := []struct {+ name string+ current string+ target string+ want int+ }{+ {name: "dev build is newer than the previous release line", current: "v1.0.0-rc1.dev1", target: "v0.27.2", want: 1},+ {name: "dev build is older than the release it leads to", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc1", want: -1},+ {name: "release is newer than its dev build", current: "v1.0.0-rc1", target: "v1.0.0-rc1.dev1", want: 1},+ {name: "dev builds order by number", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc1.dev2", want: -1},+ {name: "equal dev builds", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc1.dev1", want: 0},+ {name: "dev build is older than the next release candidate", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc2", want: -1},+ {name: "dev build is older than the final release", current: "v1.0.0-rc1.dev1", target: "v1.0.0", want: -1},+ {name: "release candidates still order lexically", current: "v1.0.0-rc1", target: "v1.0.0-rc2", want: -1},+ {name: "dev builds of different bases follow their base", current: "v1.0.0-rc1.dev9", target: "v1.0.0-rc2.dev1", want: -1},+ {name: "patch releases stay comparable", current: "v0.27.1", target: "v0.27.2", want: -1},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := compareMigrationVersion(tc.current, tc.target); got != tc.want {+ t.Fatalf("compareMigrationVersion(%q, %q) = %d, want %d", tc.current, tc.target, got, tc.want)+ }+ })+ }+}+func TestGetAndSetDatabaseMigrationVersion(t *testing.T) {db := setupMigrationVersionTestDB(t, true)ctx := t.Context()diff --git a/internal/development.md b/internal/development.mdindex 17321c84af..a618ea8436 100644--- a/internal/development.md+++ b/internal/development.md@@ -215,6 +215,14 @@ Note: admin server must be started first; otherwise, api server will encounter e./bin/ragflow_server --migrate```+`--migrate` writes the database version marker that server modes check on+startup, and a development branch regularly records a version for a release+that has not been tagged yet — a build from a `v0.27.x` commit that writes+`v1.0.0-rc1.dev1` refuses to start afterwards, because the recorded version+looks newer than the code. Set `RAGFLOW_DEV_MODE=true` (see `docker/.env`) for+such a checkout: it turns the "code version must not be older than the database+version" guard off. Leave it off in production.+```bash# Start RAGFlow server./bin/ragflow_server --api