Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feat: migrate chat message and reference (#19724)

Lynn81a5fdd56e8a
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index 99d3025936..daf7d5004d 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -739,7 +739,7 @@ jobs:
# because .env-go's COMPOSE_PROFILES lacks "tei-", causing
# PATCH /models/default to fail for the Builtin/Local embedding
# model and datasets to be created with an empty embedding_model.
- sed -i '/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d' .env-go
+ sed -i '/^COMPOSE_PROFILES=/d;/^RAGFLOW_DEV_MODE=/d;/^TEI_MODEL=/d' .env-go
{
echo ""
echo "ES_PORT=${ES_PORT}"
@@ -780,6 +780,7 @@ jobs:
# config as the Python container. Only these two affect the
# in-process checkModelAvailable logic; other .env keys are
# either yml-interpolation-only or already correct in .env-go.
+ # RAGFLOW_DEV_MODE rides along for the same reason.
{
echo ""
if [ "${API_PROXY_SCHEME}" = "go" ]; then
@@ -788,6 +789,11 @@ jobs:
echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
fi
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
+ # The image stamps its git-describe output into /ragflow/VERSION,
+ # so this checkout reports v0.27.x while --migrate records the
+ # v1.0.0-rc1.dev1 database marker. Without the override the Go
+ # server refuses to boot against the marker it just wrote.
+ echo "RAGFLOW_DEV_MODE=true"
} >> .env-go
if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}
@@ -1417,7 +1423,7 @@ jobs:
# because .env-go's COMPOSE_PROFILES lacks "tei-", causing
# PATCH /models/default to fail for the Builtin/Local embedding
# model and datasets to be created with an empty embedding_model.
- sed -i '/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d' .env-go
+ sed -i '/^COMPOSE_PROFILES=/d;/^RAGFLOW_DEV_MODE=/d;/^TEI_MODEL=/d' .env-go
{
echo ""
echo "ES_PORT=${ES_PORT}"
@@ -1458,6 +1464,7 @@ jobs:
# config as the Python container. Only these two affect the
# in-process checkModelAvailable logic; other .env keys are
# either yml-interpolation-only or already correct in .env-go.
+ # RAGFLOW_DEV_MODE rides along for the same reason.
{
echo ""
if [ "${API_PROXY_SCHEME}" = "go" ]; then
@@ -1466,6 +1473,11 @@ jobs:
echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
fi
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
+ # The image stamps its git-describe output into /ragflow/VERSION,
+ # so this checkout reports v0.27.x while --migrate records the
+ # v1.0.0-rc1.dev1 database marker. Without the override the Go
+ # server refuses to boot against the marker it just wrote.
+ echo "RAGFLOW_DEV_MODE=true"
} >> .env-go
if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 2248b9cc9c..5137fc7a8c 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -564,7 +564,17 @@ func main() {
// A missing marker, or a version on either side that cannot be parsed, never
// blocks startup: without a usable comparison there is no evidence that the
// database is ahead of the code.
+//
+// RAGFLOW_DEV_MODE turns the check off entirely. A development build can carry
+// a marker for a release that is not tagged yet, in which case the comparison
+// would reject the build that wrote the marker.
func checkDatabaseVersion(ctx context.Context) error {
+ if common.DevModeEnabled() {
+ common.Warn("Development mode is enabled, skipping the database downgrade check",
+ zap.String("env", common.EnvRAGFlowDevMode))
+ return nil
+ }
+
databaseVersion, err := dao.GetDatabaseMigrationVersion(ctx, dao.DB)
if err != nil {
return fmt.Errorf("read database version marker: %w", err)
diff --git a/docker/.env b/docker/.env
index 147d1b6c13..5bb2d7a12d 100644
--- a/docker/.env
+++ b/docker/.env
@@ -5,6 +5,15 @@
# You can generate a random string using: openssl rand -hex 32
# -----------------------------------------------------------------------------
+# ------------------------------
+# Development mode
+# ------------------------------
+# Set to `true` only while developing: the server then starts even when the
+# running code reports a version older than the database migration marker,
+# which any branch targeting a not-yet-tagged release does. Leave it off in
+# production, where an older binary must not touch a newer database.
+RAGFLOW_DEV_MODE=false
+
# ------------------------------
# docker env var for specifying vector db type at startup
# (based on the vector db type, the corresponding docker
diff --git a/docker/.env-go b/docker/.env-go
index 606e6ad11d..3c9707587e 100644
--- a/docker/.env-go
+++ b/docker/.env-go
@@ -5,6 +5,20 @@
# You can generate a random string using: openssl rand -hex 32
# -----------------------------------------------------------------------------
+# ------------------------------
+# Development mode
+# ------------------------------
+# Set to `true` only while developing: the server then starts even when the
+# running code reports a version older than the database migration marker,
+# which any branch targeting a not-yet-tagged release does. Leave it off in
+# production, where an older binary must not touch a newer database.
+#
+# CI (.github/workflows/sep-tests.yml) appends `true` here on every run: the
+# image stamps its own git-describe output into /ragflow/VERSION, so a CI build
+# reports v0.27.x against the v1.0.0-rc1.dev1 marker it migrated to moments
+# earlier. Do not treat that generated value as the default.
+RAGFLOW_DEV_MODE=false
+
# ------------------------------
# docker env var for specifying vector db type at startup
# (based on the vector db type, the corresponding docker
diff --git a/internal/common/environments.go b/internal/common/environments.go
index cc526ae10c..24387050d5 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -39,6 +39,7 @@ func IsLLMDebugEnabled() bool {
// environment variables
const (
EnvTensorrtDLAServer = "TENSORRT_DLA_SVR"
+ EnvRAGFlowDevMode = "RAGFLOW_DEV_MODE"
EnvRAGFlowTTSCacheTTLSeconds = "RAGFLOW_TTS_CACHE_TTL_SECONDS"
EnvRerankTokenLimitMode = "RERANK_TOKEN_LIMIT_MODE"
EnvComponentExecTimeout = "COMPONENT_EXEC_TIMEOUT"
diff --git a/internal/common/version.go b/internal/common/version.go
index 2f9d212a89..f8b5412b9e 100644
--- a/internal/common/version.go
+++ b/internal/common/version.go
@@ -19,6 +19,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "strconv"
"strings"
"sync"
@@ -97,6 +98,20 @@ func IsOlderReleaseThan(codeVersion, targetVersion string) (older, comparable bo
return semver.Compare(code, target) < 0, true
}
+// DevModeEnabled reports whether this process runs in development mode, which
+// relaxes the guard that refuses to start when the code is older than the
+// database. Development builds routinely carry a marker for a release that has
+// not been tagged yet — the migration that splits conversation history records
+// v1.0.0-rc1.dev1 while the branch still reports v0.27.x — so the guard would
+// otherwise block the very deployment that wrote the marker.
+//
+// It is off unless the variable is explicitly set to a true value, so a
+// production deployment keeps the guard even when the variable is absent.
+func DevModeEnabled() bool {
+ enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(EnvRAGFlowDevMode)))
+ return err == nil && enabled
+}
+
// releaseVersion normalizes a version string to the "vMAJOR.MINOR.PATCH" form
// semver accepts, dropping any pre-release or build suffix. It returns an empty
// string when the result is not a valid semver tag.
diff --git a/internal/common/version_test.go b/internal/common/version_test.go
index f3627d7201..5b1c064caf 100644
--- a/internal/common/version_test.go
+++ b/internal/common/version_test.go
@@ -21,6 +21,34 @@ import (
"testing"
)
+func TestDevModeEnabled(t *testing.T) {
+ cases := []struct {
+ value string
+ set bool
+ want bool
+ }{
+ {value: "", set: false, want: false},
+ {value: "", set: true, want: false},
+ {value: "false", set: true, want: false},
+ {value: "0", set: true, want: false},
+ {value: "nonsense", set: true, want: false},
+ {value: "true", set: true, want: true},
+ {value: "True", set: true, want: true},
+ {value: "1", set: true, want: true},
+ {value: " true ", set: true, want: true},
+ }
+ for _, tc := range cases {
+ t.Run(fmt.Sprintf("%q/set=%v", tc.value, tc.set), func(t *testing.T) {
+ if tc.set {
+ t.Setenv(EnvRAGFlowDevMode, tc.value)
+ }
+ if got := DevModeEnabled(); got != tc.want {
+ t.Fatalf("DevModeEnabled() = %v, want %v", got, tc.want)
+ }
+ })
+ }
+}
+
func TestGetRAGFlowVersion(t *testing.T) {
version := GetRAGFlowVersion()
fmt.Printf("RAGFlow Version: %s\n", version)
diff --git a/internal/dao/conversation_history.go b/internal/dao/conversation_history.go
index 11382ffcdf..287f995447 100644
--- a/internal/dao/conversation_history.go
+++ b/internal/dao/conversation_history.go
@@ -331,9 +331,22 @@ func messageValues(fields entity.ConversationMessageFields) map[string]interface
}
func createHistory(ctx context.Context, db *gorm.DB, table, kind, conversationID string, raw json.RawMessage) error {
+ rows, err := historyRows(kind, conversationID, raw)
+ if err != nil {
+ return err
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+ return db.WithContext(ctx).Table(table).CreateInBatches(rows, 100).Error
+}
+
+// historyRows flattens a message or reference payload into child rows. Its
+// positions are the payload indexes, starting from zero.
+func historyRows(kind, conversationID string, raw json.RawMessage) ([]map[string]interface{}, error) {
items, err := splitHistory(raw, kind)
if err != nil {
- return fmt.Errorf("split %s history: %w", kind, err)
+ return nil, fmt.Errorf("split %s history: %w", kind, err)
}
rows := make([]map[string]interface{}, 0, len(items))
for position, item := range items {
@@ -341,17 +354,14 @@ func createHistory(ctx context.Context, db *gorm.DB, table, kind, conversationID
if kind == "message" {
fields, err := flattenMessage(item)
if err != nil {
- return fmt.Errorf("flatten message %d: %w", position, err)
+ return nil, fmt.Errorf("flatten message %d: %w", position, err)
}
row = messageValues(fields)
row["conversation_id"], row["position"] = conversationID, position
}
rows = append(rows, row)
}
- if len(rows) == 0 {
- return nil
- }
- return db.WithContext(ctx).Table(table).CreateInBatches(rows, 100).Error
+ return rows, nil
}
func loadHistory(ctx context.Context, db *gorm.DB, table, payloadColumn string, conversationIDs []string) (map[string]json.RawMessage, error) {
diff --git a/internal/dao/conversation_history_migration.go b/internal/dao/conversation_history_migration.go
new file mode 100644
index 0000000000..a82b31e5a7
--- /dev/null
+++ b/internal/dao/conversation_history_migration.go
@@ -0,0 +1,232 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+
+ "go.uber.org/zap"
+ "gorm.io/gorm"
+
+ "ragflow/internal/common"
+)
+
+// conversationHistoryTargetVersion is the database version a completed split
+// brings the database to. It is stored in system_settings under
+// mysql_migration.database.version, the marker shared with the tenant model
+// migration, so both steps read the same progress record instead of keeping one
+// each. The dev prerelease keeps it below v1.0.0-rc1, which owns the schema
+// this step only prepares.
+const conversationHistoryTargetVersion = "v1.0.0-rc1.dev1"
+
+// conversationHistoryBatchSize bounds how many conversations are read and
+// written per transaction, so an interrupted run resumes from a committed
+// batch instead of reprocessing the whole table.
+const conversationHistoryBatchSize = 100
+
+// legacyConversationHistory addresses conversation.message and
+// conversation.reference. ChatSession drops both columns with gorm:"-", so a
+// probe model is the only way to reach them.
+type legacyConversationHistory struct {
+ ID string `gorm:"column:id;primaryKey;size:32"`
+ Message *string `gorm:"column:message"`
+ Reference *string `gorm:"column:reference"`
+}
+
+func (legacyConversationHistory) TableName() string {
+ return "conversation"
+}
+
+// legacyAPI4ConversationHistory is the api_4_conversation counterpart of
+// legacyConversationHistory.
+type legacyAPI4ConversationHistory struct {
+ ID string `gorm:"column:id;primaryKey;size:32"`
+ Message *string `gorm:"column:message"`
+ Reference *string `gorm:"column:reference"`
+}
+
+func (legacyAPI4ConversationHistory) TableName() string {
+ return "api_4_conversation"
+}
+
+// legacyHistoryRow is one parent row read for backfill. Columns that the
+// parent table does not carry stay nil.
+type legacyHistoryRow struct {
+ ID string `gorm:"column:id"`
+ Message *string `gorm:"column:message"`
+ Reference *string `gorm:"column:reference"`
+}
+
+// conversationHistorySplit maps one parent table onto its two child tables.
+type conversationHistorySplit struct {
+ parentTable string
+ probe interface{}
+ messageTable string
+ referenceTable string
+}
+
+func (s conversationHistorySplit) childTables() []string {
+ return []string{s.messageTable, s.referenceTable}
+}
+
+// migrateConversationHistory backfills conversation_message,
+// conversation_reference and their api_4_conversation counterparts from the
+// message and reference payload columns the parent tables still hold.
+//
+// It runs after AutoMigrate: unlike RunMigrations, which has to observe the
+// legacy schema, it writes to tables AutoMigrate creates.
+func migrateConversationHistory(ctx context.Context, db *gorm.DB) error {
+ currentVersion, err := GetDatabaseMigrationVersion(ctx, db)
+ if err != nil {
+ return fmt.Errorf("read database migration version: %w", err)
+ }
+ if shouldSkipMigration(currentVersion, conversationHistoryTargetVersion) {
+ common.Info("Conversation history already split, skipping",
+ zap.String("current_version", currentVersion),
+ zap.String("target_version", conversationHistoryTargetVersion))
+ return nil
+ }
+ splits := []conversationHistorySplit{
+ {
+ parentTable: "conversation",
+ probe: &legacyConversationHistory{},
+ messageTable: conversationMessageTable,
+ referenceTable: conversationReferenceTable,
+ },
+ {
+ parentTable: "api_4_conversation",
+ probe: &legacyAPI4ConversationHistory{},
+ messageTable: apiConversationMessageTable,
+ referenceTable: apiConversationReferenceTable,
+ },
+ }
+ for _, split := range splits {
+ if err := migrateConversationHistorySplit(ctx, db, split); err != nil {
+ return err
+ }
+ }
+ if err := setDatabaseMigrationVersion(ctx, db, conversationHistoryTargetVersion); err != nil {
+ return fmt.Errorf("mark database migration version %s: %w", conversationHistoryTargetVersion, err)
+ }
+ common.Info("Conversation history migration completed", zap.String("version", conversationHistoryTargetVersion))
+ return nil
+}
+
+func migrateConversationHistorySplit(ctx context.Context, db *gorm.DB, split conversationHistorySplit) error {
+ scoped := db.WithContext(ctx)
+ if !scoped.Migrator().HasTable(split.parentTable) {
+ return nil
+ }
+ hasMessage := scoped.Migrator().HasColumn(split.probe, "message")
+ hasReference := scoped.Migrator().HasColumn(split.probe, "reference")
+ if !hasMessage && !hasReference {
+ // The Go schema never carried these columns, so there is nothing to split.
+ return nil
+ }
+ columns := []string{"id"}
+ if hasMessage {
+ columns = append(columns, "message")
+ }
+ if hasReference {
+ columns = append(columns, "reference")
+ }
+
+ lastID := ""
+ migrated := 0
+ for {
+ var batch []legacyHistoryRow
+ err := scoped.Table(split.parentTable).Select(columns).
+ Where("id > ?", lastID).Order("id").Limit(conversationHistoryBatchSize).Find(&batch).Error
+ if err != nil {
+ return fmt.Errorf("read %s history payloads: %w", split.parentTable, err)
+ }
+ if len(batch) == 0 {
+ break
+ }
+ err = scoped.Transaction(func(tx *gorm.DB) error {
+ for _, row := range batch {
+ written, err := splitConversationHistory(ctx, tx, split, row)
+ if err != nil {
+ return err
+ }
+ if written {
+ migrated++
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ lastID = batch[len(batch)-1].ID
+ }
+ common.Info("Split conversation history",
+ zap.String("table", split.parentTable), zap.Int("conversations", migrated))
+ return nil
+}
+
+// splitConversationHistory writes one parent row's payloads into the child
+// tables. Rows written after the split took effect win over the payload, so a
+// rerun never duplicates or clobbers them.
+func splitConversationHistory(ctx context.Context, db *gorm.DB, split conversationHistorySplit, row legacyHistoryRow) (bool, error) {
+ for _, table := range split.childTables() {
+ hasRows, err := conversationHistoryHasRows(ctx, db, table, row.ID)
+ if err != nil {
+ return false, err
+ }
+ if hasRows {
+ return false, nil
+ }
+ }
+ written := false
+ if row.Message != nil {
+ rows, err := historyRows("message", row.ID, json.RawMessage(*row.Message))
+ if err != nil {
+ return false, fmt.Errorf("split %s %s message payload: %w", split.parentTable, row.ID, err)
+ }
+ if len(rows) > 0 {
+ if err := db.WithContext(ctx).Table(split.messageTable).CreateInBatches(rows, 100).Error; err != nil {
+ return false, err
+ }
+ written = true
+ }
+ }
+ if row.Reference != nil {
+ rows, err := historyRows("reference", row.ID, json.RawMessage(*row.Reference))
+ if err != nil {
+ return false, fmt.Errorf("split %s %s reference payload: %w", split.parentTable, row.ID, err)
+ }
+ if len(rows) > 0 {
+ if err := db.WithContext(ctx).Table(split.referenceTable).CreateInBatches(rows, 100).Error; err != nil {
+ return false, err
+ }
+ written = true
+ }
+ }
+ return written, nil
+}
+
+func conversationHistoryHasRows(ctx context.Context, db *gorm.DB, table, conversationID string) (bool, error) {
+ var count int64
+ err := db.WithContext(ctx).Table(table).Where("conversation_id = ?", conversationID).Count(&count).Error
+ if err != nil {
+ return false, err
+ }
+ return count > 0, nil
+}
diff --git a/internal/dao/conversation_history_migration_test.go b/internal/dao/conversation_history_migration_test.go
new file mode 100644
index 0000000000..10d5fff024
--- /dev/null
+++ b/internal/dao/conversation_history_migration_test.go
@@ -0,0 +1,278 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+)
+
+// setupConversationHistoryMigrationDB builds a schema that mirrors a database
+// created before the split: the Go entities drop the message and reference
+// columns, so add them back the way the previous schema had them.
+func setupConversationHistoryMigrationDB(t *testing.T, legacyColumns bool) *gorm.DB {
+ t.Helper()
+
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
+ TranslateError: true,
+ })
+ if err != nil {
+ t.Fatalf("failed to open sqlite: %v", err)
+ }
+
+ models := []interface{}{
+ &entity.ChatSession{},
+ &entity.ConversationMessage{},
+ &entity.ConversationReference{},
+ &entity.API4Conversation{},
+ &entity.API4ConversationMessage{},
+ &entity.API4ConversationReference{},
+ &entity.SystemSettings{},
+ }
+ if err := db.AutoMigrate(models...); err != nil {
+ t.Fatalf("failed to migrate: %v", err)
+ }
+
+ if legacyColumns {
+ for _, table := range []string{"conversation", "api_4_conversation"} {
+ for _, column := range []string{"message", "reference"} {
+ if err := db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s longtext", table, column)).Error; err != nil {
+ t.Fatalf("failed to add %s.%s: %v", table, column, err)
+ }
+ }
+ }
+ }
+
+ return db
+}
+
+func insertLegacyConversation(t *testing.T, db *gorm.DB, table, id string, message, reference string) {
+ t.Helper()
+ if err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, dialog_id, user_id, message, reference) VALUES (?, ?, ?, ?, ?)", table),
+ id, "dialog-"+id, "user-"+id, message, reference).Error; err != nil {
+ t.Fatalf("failed to insert %s %s: %v", table, id, err)
+ }
+}
+
+// insertConversation adds a parent row with no payload columns at all, the way
+// the Go schema creates them.
+func insertConversation(t *testing.T, db *gorm.DB, table, id string) {
+ t.Helper()
+ if err := db.Exec(fmt.Sprintf("INSERT INTO %s (id, dialog_id, user_id) VALUES (?, ?, ?)", table),
+ id, "dialog-"+id, "user-"+id).Error; err != nil {
+ t.Fatalf("failed to insert %s %s: %v", table, id, err)
+ }
+}
+
+func countHistoryRows(t *testing.T, db *gorm.DB, table string) int64 {
+ t.Helper()
+ var count int64
+ if err := db.Table(table).Count(&count).Error; err != nil {
+ t.Fatalf("failed to count %s: %v", table, err)
+ }
+ return count
+}
+
+func databaseVersion(t *testing.T, db *gorm.DB) string {
+ t.Helper()
+ version, err := GetDatabaseMigrationVersion(t.Context(), db)
+ if err != nil {
+ t.Fatalf("failed to read database version: %v", err)
+ }
+ return version
+}
+
+func writeDatabaseVersion(t *testing.T, db *gorm.DB, version string) {
+ t.Helper()
+ if err := setDatabaseMigrationVersion(t.Context(), db, version); err != nil {
+ t.Fatalf("failed to write database version %s: %v", version, err)
+ }
+}
+
+func TestMigrateConversationHistorySplitsPayloads(t *testing.T) {
+ message := `[{"role":"user","content":"hi","id":"m1"},{"role":"assistant","content":"hello","id":"m2","reference":[{"chunk-0":{"content":"chunk"}}]}]`
+ reference := `[{"chunks":["a"],"doc_aggs":[]},{"chunks":["b"],"doc_aggs":[]}]`
+ db := setupConversationHistoryMigrationDB(t, true)
+ insertLegacyConversation(t, db, "conversation", "c1", message, reference)
+ insertLegacyConversation(t, db, "api_4_conversation", "a1", message, reference)
+
+ if err := migrateConversationHistory(t.Context(), db); err != nil {
+ t.Fatalf("migrateConversationHistory: %v", err)
+ }
+
+ for _, tc := range []struct{ messageTable, referenceTable string }{
+ {conversationMessageTable, conversationReferenceTable},
+ {apiConversationMessageTable, apiConversationReferenceTable},
+ } {
+ if got := countHistoryRows(t, db, tc.messageTable); got != 2 {
+ t.Fatalf("%s: expected 2 messages, got %d", tc.messageTable, got)
+ }
+ if got := countHistoryRows(t, db, tc.referenceTable); got != 2 {
+ t.Fatalf("%s: expected 2 references, got %d", tc.referenceTable, got)
+ }
+ }
+
+ var msg entity.ConversationMessage
+ if err := db.Table(conversationMessageTable).Where("conversation_id = ? AND position = ?", "c1", 1).Take(&msg).Error; err != nil {
+ t.Fatalf("load message: %v", err)
+ }
+ if derefString(msg.Role) != "assistant" || derefString(msg.Content) != "hello" || derefString(msg.MessageID) != "m2" {
+ t.Fatalf("unexpected assistant message: %+v", msg)
+ }
+
+ var storedReference string
+ err := db.Table(conversationReferenceTable).Select("reference").
+ Where("conversation_id = ? AND position = ?", "c1", 1).Row().Scan(&storedReference)
+ if err != nil {
+ t.Fatalf("load reference: %v", err)
+ }
+ if storedReference != `{"chunks":["b"],"doc_aggs":[]}` {
+ t.Fatalf("unexpected reference: %s", storedReference)
+ }
+
+ // Rows backfilled from a payload carry no explicit message binding.
+ var messagePos *int
+ err = db.Table(conversationReferenceTable).Select("message_position").
+ Where("conversation_id = ? AND position = ?", "c1", 1).Row().Scan(&messagePos)
+ if err != nil {
+ t.Fatalf("load message_position: %v", err)
+ }
+ if messagePos != nil {
+ t.Fatalf("expected nil message_position, got %d", *messagePos)
+ }
+
+ if got := databaseVersion(t, db); got != conversationHistoryTargetVersion {
+ t.Fatalf("expected database version %s, got %s", conversationHistoryTargetVersion, got)
+ }
+}
+
+func TestMigrateConversationHistoryIsIdempotent(t *testing.T) {
+ message := `[{"role":"user","content":"hi","id":"m1"},{"role":"assistant","content":"hello","id":"m2"}]`
+ db := setupConversationHistoryMigrationDB(t, true)
+ insertLegacyConversation(t, db, "conversation", "c1", message, `[{"chunks":["a"]}]`)
+ insertLegacyConversation(t, db, "conversation", "c2", `[]`, `[]`)
+
+ for range 2 {
+ // Rewind the marker so the second pass really reruns the backfill
+ // instead of short-circuiting on the recorded version.
+ writeDatabaseVersion(t, db, modelMigrationTargetVersion)
+ if err := migrateConversationHistory(t.Context(), db); err != nil {
+ t.Fatalf("migrateConversationHistory: %v", err)
+ }
+ }
+
+ if got := countHistoryRows(t, db, conversationMessageTable); got != 2 {
+ t.Fatalf("expected 2 messages after rerun, got %d", got)
+ }
+ if got := countHistoryRows(t, db, conversationReferenceTable); got != 1 {
+ t.Fatalf("expected 1 reference after rerun, got %d", got)
+ }
+}
+
+func TestMigrateConversationHistoryKeepsRowsWrittenAfterSplit(t *testing.T) {
+ db := setupConversationHistoryMigrationDB(t, true)
+ insertLegacyConversation(t, db, "conversation", "c1", `[{"role":"user","content":"hi","id":"m1"}]`, `[]`)
+ if err := db.Table(conversationMessageTable).Create(map[string]interface{}{
+ "conversation_id": "c1", "position": 0, "role": "assistant", "content": "written by the runtime",
+ }).Error; err != nil {
+ t.Fatalf("seed message: %v", err)
+ }
+
+ if err := migrateConversationHistory(t.Context(), db); err != nil {
+ t.Fatalf("migrateConversationHistory: %v", err)
+ }
+
+ var msg entity.ConversationMessage
+ if err := db.Table(conversationMessageTable).Where("conversation_id = ? AND position = ?", "c1", 0).Take(&msg).Error; err != nil {
+ t.Fatalf("load message: %v", err)
+ }
+ if derefString(msg.Content) != "written by the runtime" {
+ t.Fatalf("expected the runtime row to win, got %q", derefString(msg.Content))
+ }
+}
+
+func derefString(value *string) string {
+ if value == nil {
+ return ""
+ }
+ return *value
+}
+
+func TestMigrateConversationHistorySkipsTablesWithoutPayloadColumns(t *testing.T) {
+ db := setupConversationHistoryMigrationDB(t, false)
+ insertConversation(t, db, "conversation", "c1")
+
+ if err := migrateConversationHistory(t.Context(), db); err != nil {
+ t.Fatalf("migrateConversationHistory: %v", err)
+ }
+
+ if got := countHistoryRows(t, db, conversationMessageTable); got != 0 {
+ t.Fatalf("expected no backfill without payload columns, got %d", got)
+ }
+ // The step still advances the marker: a schema without the payload columns
+ // has nothing left to split, ever.
+ if got := databaseVersion(t, db); got != conversationHistoryTargetVersion {
+ t.Fatalf("expected database version %s, got %s", conversationHistoryTargetVersion, got)
+ }
+}
+
+func TestMigrateConversationHistoryHonoursRecordedVersion(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ current string
+ migrate bool
+ }{
+ {"no recorded version", "", true},
+ {"release line below the split", "v0.27.2", true},
+ {"earlier dev build", "v1.0.0-rc1.dev0", true},
+ {"the split version itself", conversationHistoryTargetVersion, false},
+ {"release the dev build leads to", "v1.0.0-rc1", false},
+ {"later dev build", "v1.0.0-rc1.dev2", false},
+ {"later release candidate", "v1.0.0-rc2", false},
+ {"final release", "v1.0.0", false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ db := setupConversationHistoryMigrationDB(t, true)
+ insertLegacyConversation(t, db, "conversation", "c1", `[{"role":"user","content":"hi"}]`, `[]`)
+ if tc.current != "" {
+ writeDatabaseVersion(t, db, tc.current)
+ }
+
+ if err := migrateConversationHistory(t.Context(), db); err != nil {
+ t.Fatalf("migrateConversationHistory: %v", err)
+ }
+
+ var wantMessages int64
+ wantVersion := tc.current
+ if tc.migrate {
+ wantMessages = 1
+ wantVersion = conversationHistoryTargetVersion
+ }
+ if got := countHistoryRows(t, db, conversationMessageTable); got != wantMessages {
+ t.Fatalf("expected %d messages, got %d", wantMessages, got)
+ }
+ if got := databaseVersion(t, db); got != wantVersion {
+ t.Fatalf("expected database version %s, got %s", wantVersion, got)
+ }
+ })
+ }
+}
diff --git a/internal/dao/database.go b/internal/dao/database.go
index a5c97f7b38..1543d286b2 100644
--- a/internal/dao/database.go
+++ b/internal/dao/database.go
@@ -189,6 +189,13 @@ func InitDB(ctx context.Context, migrateDB bool) error {
}
}
common.Info("Database schema migrated successfully")
+
+ // Split the conversation message and reference payloads out of their
+ // parent tables. It has to run after AutoMigrate, which unlike
+ // RunMigrations creates the child tables this backfill writes to.
+ if err = migrateConversationHistory(ctx, DB); err != nil {
+ return fmt.Errorf("failed to migrate conversation history: %w", err)
+ }
} else {
// Ensure the Go-exclusive runtime tables exist. The manual migrations are
// performed by the standalone --migrate action, so a server-mode process
diff --git a/internal/dao/migration_version.go b/internal/dao/migration_version.go
index 5c2fa5604b..e9fdf6761f 100644
--- a/internal/dao/migration_version.go
+++ b/internal/dao/migration_version.go
@@ -20,6 +20,7 @@ import (
"context"
"database/sql"
"errors"
+ "strconv"
"strings"
"time"
@@ -48,6 +49,10 @@ const modelMigrationBaseVersion = "v0.26.0"
// by tools/scripts/run_migrations.sh.
const modelMigrationTargetVersion = "v0.27.2"
+// devPrereleaseIdentifier is the prerelease identifier that marks a version as
+// a development build of the version it is appended to.
+const devPrereleaseIdentifier = "dev"
+
// GetDatabaseMigrationVersion reads the stored migration version. It returns an
// empty string when the marker or the system_settings table is absent.
func GetDatabaseMigrationVersion(ctx context.Context, db *gorm.DB) (string, error) {
@@ -97,14 +102,90 @@ func setDatabaseMigrationVersion(ctx context.Context, db *gorm.DB, version strin
}
// shouldSkipMigration mirrors should_skip_migration in the Python migration: a
-// missing or unparseable version never skips the migration.
+// missing or unparseable version never skips the migration. Both sides share
+// the mysql_migration.database.version marker, so they have to order versions
+// the same way, which for Go means compareMigrationVersion instead of raw
+// semver.
func shouldSkipMigration(currentDBVersion, targetVersion string) bool {
current := parseMigrationVersion(currentDBVersion)
target := parseMigrationVersion(targetVersion)
if current == "" || target == "" {
return false
}
- return semver.Compare(current, target) >= 0
+ return compareMigrationVersion(current, target) >= 0
+}
+
+// compareMigrationVersion orders two canonical migration versions, returning -1,
+// 0 or 1 as current sorts before, equal to, or after target.
+//
+// It follows semver except for development prereleases, where it matches PEP
+// 440, the ordering the Python migration gets from packaging.version: a
+// trailing ".devN" identifier marks a build leading the version it is attached
+// to, so v1.0.0-rc1.dev1 sorts below v1.0.0-rc1 and below v1.0.0-rc1.dev2.
+// semver orders a longer prerelease above its prefix, which would place a dev
+// build ahead of the very release it precedes.
+func compareMigrationVersion(current, target string) int {
+ currentCore := migrationVersionCore(current)
+ targetCore := migrationVersionCore(target)
+ if cmp := semver.Compare(currentCore, targetCore); cmp != 0 {
+ return cmp
+ }
+ currentBase, currentDev := splitDevPrerelease(semver.Prerelease(current))
+ targetBase, targetDev := splitDevPrerelease(semver.Prerelease(target))
+ if cmp := semver.Compare(currentCore+currentBase, targetCore+targetBase); cmp != 0 {
+ return cmp
+ }
+ switch {
+ case currentDev == targetDev:
+ return 0
+ case currentDev < 0:
+ // The base version outranks every dev build leading up to it.
+ return 1
+ case targetDev < 0:
+ return -1
+ case currentDev < targetDev:
+ return -1
+ }
+ return 1
+}
+
+// migrationVersionCore strips prerelease and build metadata, leaving the
+// "vMAJOR.MINOR.PATCH" core the two versions are first compared on.
+func migrationVersionCore(version string) string {
+ core := strings.TrimSuffix(version, semver.Build(version))
+ return strings.TrimSuffix(core, semver.Prerelease(version))
+}
+
+// splitDevPrerelease splits a "-rc1.dev1" prerelease into its "-rc1" base and
+// the trailing development number. dev is -1 when there is no dev identifier.
+func splitDevPrerelease(prerelease string) (base string, dev int) {
+ trimmed := strings.TrimPrefix(prerelease, "-")
+ if trimmed == "" {
+ return "", -1
+ }
+ identifiers := strings.Split(trimmed, ".")
+ number, ok := devIdentifierNumber(identifiers[len(identifiers)-1])
+ if !ok {
+ return prerelease, -1
+ }
+ base = strings.Join(identifiers[:len(identifiers)-1], ".")
+ if base != "" {
+ base = "-" + base
+ }
+ return base, number
+}
+
+// devIdentifierNumber reads the N out of a "devN" prerelease identifier.
+func devIdentifierNumber(identifier string) (int, bool) {
+ number, ok := strings.CutPrefix(identifier, devPrereleaseIdentifier)
+ if !ok {
+ return 0, false
+ }
+ value, err := strconv.Atoi(number)
+ if err != nil {
+ return 0, false
+ }
+ return value, true
}
// parseMigrationVersion normalizes a version to the "vMAJOR.MINOR.PATCH" form
diff --git a/internal/dao/migration_version_test.go b/internal/dao/migration_version_test.go
index 1842ad22d4..20ea95fe08 100644
--- a/internal/dao/migration_version_test.go
+++ b/internal/dao/migration_version_test.go
@@ -63,6 +63,33 @@ func TestShouldSkipMigration(t *testing.T) {
}
}
+func TestCompareMigrationVersionDevPrerelease(t *testing.T) {
+ cases := []struct {
+ name string
+ current string
+ target string
+ want int
+ }{
+ {name: "dev build is newer than the previous release line", current: "v1.0.0-rc1.dev1", target: "v0.27.2", want: 1},
+ {name: "dev build is older than the release it leads to", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc1", want: -1},
+ {name: "release is newer than its dev build", current: "v1.0.0-rc1", target: "v1.0.0-rc1.dev1", want: 1},
+ {name: "dev builds order by number", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc1.dev2", want: -1},
+ {name: "equal dev builds", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc1.dev1", want: 0},
+ {name: "dev build is older than the next release candidate", current: "v1.0.0-rc1.dev1", target: "v1.0.0-rc2", want: -1},
+ {name: "dev build is older than the final release", current: "v1.0.0-rc1.dev1", target: "v1.0.0", want: -1},
+ {name: "release candidates still order lexically", current: "v1.0.0-rc1", target: "v1.0.0-rc2", want: -1},
+ {name: "dev builds of different bases follow their base", current: "v1.0.0-rc1.dev9", target: "v1.0.0-rc2.dev1", want: -1},
+ {name: "patch releases stay comparable", current: "v0.27.1", target: "v0.27.2", want: -1},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := compareMigrationVersion(tc.current, tc.target); got != tc.want {
+ t.Fatalf("compareMigrationVersion(%q, %q) = %d, want %d", tc.current, tc.target, got, tc.want)
+ }
+ })
+ }
+}
+
func TestGetAndSetDatabaseMigrationVersion(t *testing.T) {
db := setupMigrationVersionTestDB(t, true)
ctx := t.Context()
diff --git a/internal/development.md b/internal/development.md
index 17321c84af..a618ea8436 100644
--- a/internal/development.md
+++ b/internal/development.md
@@ -215,6 +215,14 @@ Note: admin server must be started first; otherwise, api server will encounter e
./bin/ragflow_server --migrate
```
+`--migrate` writes the database version marker that server modes check on
+startup, and a development branch regularly records a version for a release
+that has not been tagged yet — a build from a `v0.27.x` commit that writes
+`v1.0.0-rc1.dev1` refuses to start afterwards, because the recorded version
+looks newer than the code. Set `RAGFLOW_DEV_MODE=true` (see `docker/.env`) for
+such a checkout: it turns the "code version must not be older than the database
+version" guard off. Leave it off in production.
+
```bash
# Start RAGFlow server
./bin/ragflow_server --api