RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(knowledge_compile): widen backlog columns to LONGTEXT (#20855)
lorenzozane5ea453c2a45f
diff --git a/internal/dao/migration.go b/internal/dao/migration.goindex 214830559e..67d64258fd 100644--- a/internal/dao/migration.go+++ b/internal/dao/migration.go@@ -644,6 +644,17 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {columnType: "longtext", nullable: true,definition: "longtext NULL",},+ // Knowledge-compile queues can exceed MySQL TEXT's 64 KiB limit.+ {+ table: "knowledge_compile_docs", column: "backlog_doc_ids",+ columnType: "longtext", nullable: false,+ definition: "longtext NOT NULL",+ },+ {+ table: "knowledge_compile_docs", column: "inflight_doc_ids",+ columnType: "longtext", nullable: false,+ definition: "longtext NOT NULL",+ },// system_settings.value mirrors Python's EmptyStringTextField(null=False).{table: "system_settings", column: "value",@@ -669,6 +680,17 @@ func modifyColumnTypes(ctx context.Context, db *gorm.DB) error {return err}}+ if db.WithContext(ctx).Migrator().HasTable("knowledge_compile_docs") {+ for _, column := range []string{"backlog_doc_ids", "inflight_doc_ids"} {+ columnType, nullable, exists, err := columnDefinition(ctx, db, "knowledge_compile_docs", column)+ if err != nil {+ return fmt.Errorf("failed to verify knowledge_compile_docs.%s after migration: %w", column, err)+ }+ if exists && (normalizeColumnType(columnType) != "longtext" || nullable) {+ return fmt.Errorf("knowledge_compile_docs.%s is %s nullable=%v after migration, want longtext NOT NULL", column, columnType, nullable)+ }+ }+ }return nil}diff --git a/internal/dao/migration_integration_test.go b/internal/dao/migration_integration_test.goindex 4ab7207424..13dc7db0ef 100644--- a/internal/dao/migration_integration_test.go+++ b/internal/dao/migration_integration_test.go@@ -5,12 +5,43 @@ package daoimport ("context"+ "fmt""os"+ "strings""testing""gorm.io/gorm")+func assertModifyColumnTypesTablesAbsent(t *testing.T, db *gorm.DB, allowedTable string) {+ t.Helper()+ tables := append([]string{+ "dialog",+ "tenant_llm",+ "api_token",+ "canvas_template",+ "system_settings",+ "knowledgebase",+ "knowledge_compile_docs",+ }, tenantModelIDFieldTables()...)+ for _, table := range tables {+ if table == allowedTable {+ continue+ }+ if db.Migrator().HasTable(table) {+ t.Fatalf("%s already exists in this database; refusing to run modifyColumnTypes against a populated schema", table)+ }+ }+}++func knowledgeCompileBacklogFixture(count int) string {+ entries := make([]string, count)+ for i := range entries {+ entries[i] = fmt.Sprintf(`{"doc_id":"%032d","event_type":"doc_completed","variants":["structure"],"task_types":["Graph"]}`, i)+ }+ return "[" + strings.Join(entries, ",") + "]"+}+// TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration reproduces// the schema that blocks startup: tenant model id columns created while the Go// entities declared them *int64, which leaves MySQL holding bigint. No@@ -36,13 +67,18 @@ func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testingif err != nil {t.Fatalf("failed to connect: %v", err)}+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("failed to get database connection pool: %v", err)+ }+ t.Cleanup(func() {+ if err := sqlDB.Close(); err != nil {+ t.Errorf("failed to close database connection pool: %v", err)+ }+ })ctx := context.Background()- for _, table := range tenantModelIDFieldTables() {- if db.WithContext(ctx).Migrator().HasTable(table) {- t.Fatalf("%s already exists in this database; refusing to run against a live schema", table)- }- }+ assertModifyColumnTypesTablesAbsent(t, db.WithContext(ctx), "dialog")create := `CREATE TABLE dialog (id varchar(32) NOT NULL,@@ -56,7 +92,9 @@ func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testingt.Fatalf("failed to create legacy dialog: %v", err)}t.Cleanup(func() {- _ = db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS dialog").Error+ if err := db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS dialog").Error; err != nil {+ t.Errorf("failed to drop dialog test table: %v", err)+ }})// llm_id is empty on purpose: this row would not be backfilled even if the@@ -95,3 +133,91 @@ func TestModifyColumnTypesHealsLegacyTenantModelIDColumns_Integration(t *testingt.Errorf("%d rows still carry an orphaned integer model id", remaining)}}++func TestModifyColumnTypesWidensKnowledgeCompileBacklogs_Integration(t *testing.T) {+ dsn := os.Getenv("RAGFLOW_TEST_MYSQL_DSN")+ if dsn == "" {+ t.Skip("RAGFLOW_TEST_MYSQL_DSN unset: this test needs a throwaway MySQL database")+ }++ db, err := gorm.Open(newMigrationAwareDialector(dsn), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("failed to connect: %v", err)+ }+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("failed to get database connection pool: %v", err)+ }+ t.Cleanup(func() {+ if err := sqlDB.Close(); err != nil {+ t.Errorf("failed to close database connection pool: %v", err)+ }+ })+ ctx := context.Background()+ assertModifyColumnTypesTablesAbsent(t, db.WithContext(ctx), "knowledge_compile_docs")+ if err = db.WithContext(ctx).Exec(`CREATE TABLE knowledge_compile_docs (+ dataset_id varchar(64) NOT NULL PRIMARY KEY,+ backlog_doc_ids text NOT NULL,+ inflight_doc_ids text NOT NULL+ )`).Error; err != nil {+ t.Fatalf("failed to create legacy knowledge_compile_docs: %v", err)+ }+ t.Cleanup(func() {+ if err := db.WithContext(context.Background()).Exec("DROP TABLE IF EXISTS knowledge_compile_docs").Error; err != nil {+ t.Errorf("failed to drop knowledge_compile_docs test table: %v", err)+ }+ })++ legacyBacklog := knowledgeCompileBacklogFixture(480)+ if len(legacyBacklog) >= 65535 {+ t.Fatalf("legacy fixture is %d bytes, want less than TEXT's 65,535-byte limit", len(legacyBacklog))+ }+ if err = db.WithContext(ctx).Exec(+ "INSERT INTO knowledge_compile_docs (dataset_id, backlog_doc_ids, inflight_doc_ids) VALUES (?, ?, '[]')",+ "kb1", legacyBacklog,+ ).Error; err != nil {+ t.Fatalf("failed to seed legacy backlog: %v", err)+ }+ if err = modifyColumnTypes(ctx, db); err != nil {+ t.Fatalf("modifyColumnTypes failed: %v", err)+ }++ for _, column := range []string{"backlog_doc_ids", "inflight_doc_ids"} {+ columnType, nullable, ok, err := columnDefinition(ctx, db, "knowledge_compile_docs", column)+ if err != nil {+ t.Fatalf("failed to read %s definition: %v", column, err)+ }+ if !ok || columnType != "longtext" || nullable {+ t.Errorf("%s definition = %q nullable=%v exists=%v, want longtext NOT NULL", column, columnType, nullable, ok)+ }+ }++ var preserved string+ if err = db.WithContext(ctx).Raw(+ "SELECT backlog_doc_ids FROM knowledge_compile_docs WHERE dataset_id = ?", "kb1",+ ).Scan(&preserved).Error; err != nil {+ t.Fatalf("failed to read migrated backlog: %v", err)+ }+ if preserved != legacyBacklog {+ t.Fatal("migration changed the existing backlog")+ }++ oversizedBacklog := knowledgeCompileBacklogFixture(600)+ if len(oversizedBacklog) <= 65535 {+ t.Fatalf("oversized fixture is %d bytes, want more than TEXT's 65,535-byte limit", len(oversizedBacklog))+ }+ if err = db.WithContext(ctx).Exec(+ "UPDATE knowledge_compile_docs SET backlog_doc_ids = ? WHERE dataset_id = ?",+ oversizedBacklog, "kb1",+ ).Error; err != nil {+ t.Fatalf("failed to store backlog larger than MySQL TEXT: %v", err)+ }+ if err = db.WithContext(ctx).Raw(+ "SELECT backlog_doc_ids FROM knowledge_compile_docs WHERE dataset_id = ?", "kb1",+ ).Scan(&preserved).Error; err != nil {+ t.Fatalf("failed to read oversized backlog: %v", err)+ }+ if preserved != oversizedBacklog {+ t.Fatal("oversized backlog was truncated")+ }+}diff --git a/internal/dao/migration_test.go b/internal/dao/migration_test.goindex 4591a11087..6cf7feedda 100644--- a/internal/dao/migration_test.go+++ b/internal/dao/migration_test.go@@ -19,14 +19,34 @@ package daoimport ("context""strings"+ "sync""testing""github.com/glebarez/sqlite""gorm.io/gorm"+ "gorm.io/gorm/schema"+ "ragflow/internal/entity")const tenantModelIDWidth = 32+func TestKnowledgeCompileBacklogColumnsUseLongText(t *testing.T) {+ parsed, err := schema.Parse(&entity.KnowledgeCompileDataset{}, &sync.Map{}, schema.NamingStrategy{})+ if err != nil {+ t.Fatalf("parse KnowledgeCompileDataset schema: %v", err)+ }++ for _, name := range []string{"BacklogDocIDs", "InflightDocIDs"} {+ field := parsed.FieldsByName[name]+ if field == nil {+ t.Fatalf("missing field %s", name)+ }+ if field.DataType != "longtext" {+ t.Errorf("%s database type = %q, want longtext", name, field.DataType)+ }+ }+}+// setupOrphanColumnTestDB builds the narrow shape discardOrphanedColumnValues// needs: a table holding values left over from an int column widened to hold// hex model ids.diff --git a/internal/entity/knowledge_compile_doc.go b/internal/entity/knowledge_compile_doc.goindex 099c25d579..fe5469586d 100644--- a/internal/entity/knowledge_compile_doc.go+++ b/internal/entity/knowledge_compile_doc.go@@ -35,19 +35,16 @@ const (// recovery. NATS notify is only a wake-up; same-KB serialization comes from// these rows, not from the broker.//-// The *_doc_ids columns store a JSON array of knowledge_compile.BacklogEntry-// (doc_id + event_type + seq) as TEXT so the consumer can re-apply the same+// The *_doc_ids columns store JSON arrays of knowledge_compile.BacklogEntry+// (doc_id + event_type + seq) as LONGTEXT so the consumer can re-apply the same// out-of-order / tombstone guards as the broker-based design without re-reading-// the queue.+// the queue, even when a backlog exceeds MySQL TEXT's 64 KiB limit.type KnowledgeCompileDataset struct {DatasetID string `gorm:"primaryKey;column:dataset_id;size:64" json:"dataset_id"`TenantID string `gorm:"column:tenant_id;size:64;not null;default:''" json:"tenant_id"`- // The *_doc_ids columns store a JSON array as TEXT. No DDL default is set:- // MySQL (8.0.13+) rejects a literal DEFAULT on TEXT/BLOB columns (Error- // 1101), and the application always writes "[]" explicitly on insert/update- // (scheduler.go FirstOrCreate / release paths), so the default is redundant.- BacklogDocIDs string `gorm:"column:backlog_doc_ids;type:text;not null" json:"backlog_doc_ids"`- InflightDocIDs string `gorm:"column:inflight_doc_ids;type:text;not null" json:"inflight_doc_ids"`+ // No DDL default is set because the application writes "[]" explicitly.+ BacklogDocIDs string `gorm:"column:backlog_doc_ids;type:longtext;not null" json:"backlog_doc_ids"`+ InflightDocIDs string `gorm:"column:inflight_doc_ids;type:longtext;not null" json:"inflight_doc_ids"`ClaimOwner string `gorm:"column:claim_owner;size:64;not null;default:''" json:"claim_owner"`ClaimToken string `gorm:"column:claim_token;size:64;not null;default:''" json:"claim_token"`ClaimExpiresAt *time.Time `gorm:"column:claim_expires_at;default:null" json:"claim_expires_at"`