Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(knowledge_compile): widen backlog columns to LONGTEXT (#20855)

lorenzozane5ea453c2a45f
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index 214830559e..67d64258fd 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -644,6 +644,17 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {
columnType: "longtext", nullable: true,
definition: "longtext NULL",
},
+ // Knowledge-compile queues can exceed MySQL TEXT's 64 KiB limit.
+ {
+ table: "knowledge_compile_docs", column: "backlog_doc_ids",
+ columnType: "longtext", nullable: false,
+ definition: "longtext NOT NULL",
+ },
+ {
+ table: "knowledge_compile_docs", column: "inflight_doc_ids",
+ columnType: "longtext", nullable: false,
+ definition: "longtext NOT NULL",
+ },
// system_settings.value mirrors Python's EmptyStringTextField(null=False).
{
table: "system_settings", column: "value",
@@ -669,6 +680,17 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {
return err
}
}
+ if db.WithContext(ctx).Migrator().HasTable("knowledge_compile_docs") {
+ for _, column := range []string{"backlog_doc_ids", "inflight_doc_ids"} {
+ columnType, nullable, exists, err := columnDefinition(ctx, db, "knowledge_compile_docs", column)
+ if err != nil {
+ return fmt.Errorf("failed to verify knowledge_compile_docs.%s after migration: %w", column, err)
+ }
+ if exists && (normalizeColumnType(columnType) != "longtext" || nullable) {
+ return fmt.Errorf("knowledge_compile_docs.%s is %s nullable=%v after migration, want longtext NOT NULL", column, columnType, nullable)
+ }
+ }
+ }
return nil
}
diff --git a/internal/dao/migration_integration_test.go b/internal/dao/migration_integration_test.go
index 4ab7207424..13dc7db0ef 100644
--- a/internal/dao/migration_integration_test.go
+++ b/internal/dao/migration_integration_test.go
@@ -5,12 +5,43 @@ package dao
import (
"context"
+ "fmt"
"os"
+ "strings"
"testing"
"gorm.io/gorm"
)
+func assertModifyColumnTypesTablesAbsent(t *testing.T, db *gorm.DB, allowedTable string) {
+ t.Helper()
+ tables := append([]string{
+ "dialog",
+ "tenant_llm",
+ "api_token",
+ "canvas_template",
+ "system_settings",
+ "knowledgebase",
+ "knowledge_compile_docs",
+ }, tenantModelIDFieldTables()...)
+ for _, table := range tables {
+ if table == allowedTable {
+ continue
+ }
+ if db.Migrator().HasTable(table) {
+ t.Fatalf("%s already exists in this database; refusing to run modifyColumnTypes against a populated schema", table)
+ }
+ }
+}
+
+func knowledgeCompileBacklogFixture(count int) string {
+ entries := make([]string, count)
+ for i := range entries {
+ entries[i] = fmt.Sprintf(`{"doc_id":"%032d","event_type":"doc_completed","variants":["structure"],"task_types":["Graph"]}`, i)
+ }
+ return "[" + strings.Join(entries, ",") + "]"
+}
+
// TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration reproduces
// the schema that blocks startup: tenant model id columns created while the Go
// entities declared them *int64, which leaves MySQL holding bigint. No
@@ -36,13 +67,18 @@ func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testing
if err != nil {
t.Fatalf("failed to connect: %v", err)
}
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("failed to get database connection pool: %v", err)
+ }
+ t.Cleanup(func() {
+ if err := sqlDB.Close(); err != nil {
+ t.Errorf("failed to close database connection pool: %v", err)
+ }
+ })
ctx := context.Background()
- for _, table := range tenantModelIDFieldTables() {
- if db.WithContext(ctx).Migrator().HasTable(table) {
- t.Fatalf("%s already exists in this database; refusing to run against a live schema", table)
- }
- }
+ assertModifyColumnTypesTablesAbsent(t, db.WithContext(ctx), "dialog")
create := `CREATE TABLE dialog (
id varchar(32) NOT NULL,
@@ -56,7 +92,9 @@ func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testing
t.Fatalf("failed to create legacy dialog: %v", err)
}
t.Cleanup(func() {
- _ = db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS dialog").Error
+ if err := db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS dialog").Error; err != nil {
+ t.Errorf("failed to drop dialog test table: %v", err)
+ }
})
// llm_id is empty on purpose: this row would not be backfilled even if the
@@ -95,3 +133,91 @@ func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testing
t.Errorf("%d rows still carry an orphaned integer model id", remaining)
}
}
+
+func TestModifyColumnTypesWidensKnowledgeCompileBacklogs_Integration(t *testing.T) {
+ dsn := os.Getenv("RAGFLOW_TEST_MYSQL_DSN")
+ if dsn == "" {
+ t.Skip("RAGFLOW_TEST_MYSQL_DSN unset: this test needs a throwaway MySQL database")
+ }
+
+ db, err := gorm.Open(newMigrationAwareDialector(dsn), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("failed to connect: %v", err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("failed to get database connection pool: %v", err)
+ }
+ t.Cleanup(func() {
+ if err := sqlDB.Close(); err != nil {
+ t.Errorf("failed to close database connection pool: %v", err)
+ }
+ })
+ ctx := context.Background()
+ assertModifyColumnTypesTablesAbsent(t, db.WithContext(ctx), "knowledge_compile_docs")
+ if err = db.WithContext(ctx).Exec(`CREATE TABLE knowledge_compile_docs (
+ dataset_id varchar(64) NOT NULL PRIMARY KEY,
+ backlog_doc_ids text NOT NULL,
+ inflight_doc_ids text NOT NULL
+ )`).Error; err != nil {
+ t.Fatalf("failed to create legacy knowledge_compile_docs: %v", err)
+ }
+ t.Cleanup(func() {
+ if err := db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS knowledge_compile_docs").Error; err != nil {
+ t.Errorf("failed to drop knowledge_compile_docs test table: %v", err)
+ }
+ })
+
+ legacyBacklog := knowledgeCompileBacklogFixture(480)
+ if len(legacyBacklog) >= 65535 {
+ t.Fatalf("legacy fixture is %d bytes, want less than TEXT's 65,535-byte limit", len(legacyBacklog))
+ }
+ if err = db.WithContext(ctx).Exec(
+ "INSERT INTO knowledge_compile_docs (dataset_id, backlog_doc_ids, inflight_doc_ids) VALUES (?, ?, '[]')",
+ "kb1", legacyBacklog,
+ ).Error; err != nil {
+ t.Fatalf("failed to seed legacy backlog: %v", err)
+ }
+ if err = modifyColumnTypes(ctx, db); err != nil {
+ t.Fatalf("modifyColumnTypes failed: %v", err)
+ }
+
+ for _, column := range []string{"backlog_doc_ids", "inflight_doc_ids"} {
+ columnType, nullable, ok, err := columnDefinition(ctx, db, "knowledge_compile_docs", column)
+ if err != nil {
+ t.Fatalf("failed to read %s definition: %v", column, err)
+ }
+ if !ok || columnType != "longtext" || nullable {
+ t.Errorf("%s definition = %q nullable=%v exists=%v, want longtext NOT NULL", column, columnType, nullable, ok)
+ }
+ }
+
+ var preserved string
+ if err = db.WithContext(ctx).Raw(
+ "SELECT backlog_doc_ids FROM knowledge_compile_docs WHERE dataset_id = ?", "kb1",
+ ).Scan(&preserved).Error; err != nil {
+ t.Fatalf("failed to read migrated backlog: %v", err)
+ }
+ if preserved != legacyBacklog {
+ t.Fatal("migration changed the existing backlog")
+ }
+
+ oversizedBacklog := knowledgeCompileBacklogFixture(600)
+ if len(oversizedBacklog) <= 65535 {
+ t.Fatalf("oversized fixture is %d bytes, want more than TEXT's 65,535-byte limit", len(oversizedBacklog))
+ }
+ if err = db.WithContext(ctx).Exec(
+ "UPDATE knowledge_compile_docs SET backlog_doc_ids = ? WHERE dataset_id = ?",
+ oversizedBacklog, "kb1",
+ ).Error; err != nil {
+ t.Fatalf("failed to store backlog larger than MySQL TEXT: %v", err)
+ }
+ if err = db.WithContext(ctx).Raw(
+ "SELECT backlog_doc_ids FROM knowledge_compile_docs WHERE dataset_id = ?", "kb1",
+ ).Scan(&preserved).Error; err != nil {
+ t.Fatalf("failed to read oversized backlog: %v", err)
+ }
+ if preserved != oversizedBacklog {
+ t.Fatal("oversized backlog was truncated")
+ }
+}
diff --git a/internal/dao/migration_test.go b/internal/dao/migration_test.go
index 4591a11087..6cf7feedda 100644
--- a/internal/dao/migration_test.go
+++ b/internal/dao/migration_test.go
@@ -19,14 +19,34 @@ package dao
import (
"context"
"strings"
+ "sync"
"testing"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
+ "gorm.io/gorm/schema"
+ "ragflow/internal/entity"
)
const tenantModelIDWidth = 32
+func TestKnowledgeCompileBacklogColumnsUseLongText(t *testing.T) {
+ parsed, err := schema.Parse(&entity.KnowledgeCompileDataset{}, &sync.Map{}, schema.NamingStrategy{})
+ if err != nil {
+ t.Fatalf("parse KnowledgeCompileDataset schema: %v", err)
+ }
+
+ for _, name := range []string{"BacklogDocIDs", "InflightDocIDs"} {
+ field := parsed.FieldsByName[name]
+ if field == nil {
+ t.Fatalf("missing field %s", name)
+ }
+ if field.DataType != "longtext" {
+ t.Errorf("%s database type = %q, want longtext", name, field.DataType)
+ }
+ }
+}
+
// setupOrphanColumnTestDB builds the narrow shape discardOrphanedColumnValues
// needs: a table holding values left over from an int column widened to hold
// hex model ids.
diff --git a/internal/entity/knowledge_compile_doc.go b/internal/entity/knowledge_compile_doc.go
index 099c25d579..fe5469586d 100644
--- a/internal/entity/knowledge_compile_doc.go
+++ b/internal/entity/knowledge_compile_doc.go
@@ -35,19 +35,16 @@ const (
// recovery. NATS notify is only a wake-up; same-KB serialization comes from
// these rows, not from the broker.
//
-// The *_doc_ids columns store a JSON array of knowledge_compile.BacklogEntry
-// (doc_id + event_type + seq) as TEXT so the consumer can re-apply the same
+// The *_doc_ids columns store JSON arrays of knowledge_compile.BacklogEntry
+// (doc_id + event_type + seq) as LONGTEXT so the consumer can re-apply the same
// out-of-order / tombstone guards as the broker-based design without re-reading
-// the queue.
+// the queue, even when a backlog exceeds MySQL TEXT's 64 KiB limit.
type KnowledgeCompileDataset struct {
DatasetID string `gorm:"primaryKey;column:dataset_id;size:64" json:"dataset_id"`
TenantID string `gorm:"column:tenant_id;size:64;not null;default:''" json:"tenant_id"`
- // The *_doc_ids columns store a JSON array as TEXT. No DDL default is set:
- // MySQL (8.0.13+) rejects a literal DEFAULT on TEXT/BLOB columns (Error
- // 1101), and the application always writes "[]" explicitly on insert/update
- // (scheduler.go FirstOrCreate / release paths), so the default is redundant.
- BacklogDocIDs string `gorm:"column:backlog_doc_ids;type:text;not null" json:"backlog_doc_ids"`
- InflightDocIDs string `gorm:"column:inflight_doc_ids;type:text;not null" json:"inflight_doc_ids"`
+ // No DDL default is set because the application writes "[]" explicitly.
+ BacklogDocIDs string `gorm:"column:backlog_doc_ids;type:longtext;not null" json:"backlog_doc_ids"`
+ InflightDocIDs string `gorm:"column:inflight_doc_ids;type:longtext;not null" json:"inflight_doc_ids"`
ClaimOwner string `gorm:"column:claim_owner;size:64;not null;default:''" json:"claim_owner"`
ClaimToken string `gorm:"column:claim_token;size:64;not null;default:''" json:"claim_token"`
ClaimExpiresAt *time.Time `gorm:"column:claim_expires_at;default:null" json:"claim_expires_at"`