Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
perf(chunk): speed up chunk ingestion and batch updates (#3369)
lyingbug68d99d7d2111
diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.goindex d06c83cf7..78eb77471 100644--- a/internal/application/repository/chunk.go+++ b/internal/application/repository/chunk.go@@ -33,40 +33,56 @@ func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository {return &chunkRepository{db: db}}-// CreateChunks creates multiple chunks in batches.-// Uses Omit("SeqID") so GORM won't include the auto-increment column in the-// INSERT, which avoids MySQL generating ON DUPLICATE KEY UPDATE and the-// resulting gap-lock deadlocks under concurrent writes.-// A deadlock retry wrapper is kept as defense-in-depth for any remaining-// edge cases on secondary unique indexes.+// createChunksBatchSize is the number of rows per INSERT statement. Chunk rows+// carry long text columns, so the batch is kept well below the bind-parameter+// limits of the supported drivers while still amortizing round trips.+const createChunksBatchSize = 500++// updateChunksBatchSize bounds the rows per batch UPDATE so the bind-parameter+// count (6 per row) stays far below the PostgreSQL limit of 65535.+const updateChunksBatchSize = 1000++// updateByIDsBatchSize bounds the IN list for UPDATE ... WHERE id IN (...).+const updateByIDsBatchSize = 5000++// CreateChunks creates multiple chunks in batches inside a single transaction.+//+// SourceContent is intentionally left empty on create. It records the parser+// output only once a user edits the chunk (see chunkService.UpdateDocumentChunk,+// which backfills it from Content on the first edit). Writing a copy of Content+// for every chunk doubled the insert volume and the TOAST footprint for no+// benefit.func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {+ if len(chunks) == 0 {+ return nil+ }for _, chunk := range chunks {chunk.Content = common.CleanInvalidUTF8(chunk.Content)chunk.ContextHeader = common.CleanInvalidUTF8(chunk.ContextHeader)- if chunk.SourceContent == "" {- chunk.SourceContent = chunk.Content+ if chunk.SourceContent != "" {+ chunk.SourceContent = common.CleanInvalidUTF8(chunk.SourceContent)}if chunk.IndexStatus == "" {chunk.IndexStatus = "ready"}}- db := r.db.WithContext(ctx)-- // SQLite doesn't support autoIncrement on non-PK columns,- // so we must pre-assign SeqIDs manually (safe: single connection).- // PostgreSQL / MySQL use DB sequences — skip to avoid duplicate key- // races under concurrent inserts.- if db.Dialector.Name() == "sqlite" {- if err := types.AssignChunkSeqIDs(db, chunks); err != nil {- return fmt.Errorf("failed to assign chunk seq_ids: %w", err)+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ // SQLite doesn't support autoIncrement on non-PK columns, so SeqIDs are+ // pre-assigned from MAX(seq_id). Doing it inside the write transaction+ // keeps the read and the insert on the same connection.+ // PostgreSQL uses a DB sequence — skip to avoid duplicate key races.+ if tx.Name() == "sqlite" {+ if err := types.AssignChunkSeqIDs(tx, chunks); err != nil {+ return fmt.Errorf("failed to assign chunk seq_ids: %w", err)+ }}- }- // Select("*") ensures zero-value fields (IsEnabled=false, Flags=0) are- // explicitly inserted, bypassing GORM's default value behavior.- // SeqID=0 is skipped by GORM automatically (autoIncrement tag).- return db.Select("*").CreateInBatches(chunks, 100).Error+ // Select("*") ensures zero-value fields (IsEnabled=false, Flags=0) are+ // explicitly inserted, bypassing GORM's default value behavior.+ // SeqID=0 is skipped by GORM automatically (autoIncrement tag).+ return tx.Select("*").CreateInBatches(chunks, createChunksBatchSize).Error+ })}// GetChunkByID retrieves a chunk by its ID and tenant ID@@ -414,109 +430,127 @@ func (r *chunkRepository) SaveChunks(ctx context.Context, chunks []*types.Chunk)// - other fields not listed above//// If you need to update metadata or content_hash, use UpdateChunk (single) instead.+//+// On PostgreSQL the rows are joined against a VALUES list, so the statement+// costs O(N) instead of the O(N²) of a CASE-per-column chain. Other dialects+// run one small UPDATE per row inside a single transaction, which is also+// O(N) and keeps the statement well within their bind-parameter limits.func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error {if len(chunks) == 0 {return nil}- // Build batch update SQL with CASE expressions- var ids []string- contentCases := make([]string, 0, len(chunks))- isEnabledCases := make([]string, 0, len(chunks))- tagIDCases := make([]string, 0, len(chunks))- flagsCases := make([]string, 0, len(chunks))- statusCases := make([]string, 0, len(chunks))+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ for start := 0; start < len(chunks); start += updateChunksBatchSize {+ end := start + updateChunksBatchSize+ if end > len(chunks) {+ end = len(chunks)+ }+ batch := chunks[start:end]- var contentArgs []interface{}- var isEnabledArgs []interface{}- var tagIDArgs []interface{}- var flagsArgs []interface{}- var statusArgs []interface{}+ var err error+ if tx.Name() == "postgres" {+ err = updateChunksPostgres(tx, batch)+ } else {+ err = updateChunksRowByRow(tx, batch)+ }+ if err != nil {+ return err+ }+ }+ return nil+ })+}+// updateChunksPostgres issues a single UPDATE ... FROM (VALUES ...) statement.+// Every value is cast explicitly so PostgreSQL can type the VALUES columns+// without inspecting the bind parameters.+func updateChunksPostgres(tx *gorm.DB, chunks []*types.Chunk) error {+ rows := make([]string, 0, len(chunks))+ args := make([]interface{}, 0, len(chunks)*6)for _, chunk := range chunks {- ids = append(ids, chunk.ID)- content := common.CleanInvalidUTF8(chunk.Content)+ rows = append(rows, "(?::varchar, ?::text, ?::boolean, ?::varchar, ?::integer, ?::integer)")+ args = append(args,+ chunk.ID,+ common.CleanInvalidUTF8(chunk.Content),+ chunk.IsEnabled,+ chunk.TagID,+ int(chunk.Flags),+ chunk.Status,+ )+ }- contentCases = append(contentCases, "WHEN id = ? THEN ?")- contentArgs = append(contentArgs, chunk.ID, content)+ sql := fmt.Sprintf(`+ UPDATE chunks AS c SET+ content = v.content,+ is_enabled = v.is_enabled,+ tag_id = v.tag_id,+ flags = v.flags,+ status = v.status,+ updated_at = NOW()+ FROM (VALUES %s) AS v(id, content, is_enabled, tag_id, flags, status)+ WHERE c.id = v.id+ `, strings.Join(rows, ",\n"))++ return tx.Exec(sql, args...).Error+}- // Convert bool to string for PostgreSQL compatibility- isEnabledStr := "false"- if chunk.IsEnabled {- isEnabledStr = "true"+// updateChunksRowByRow updates each chunk with its own statement. GORM sets+// updated_at automatically for map-based Updates, which also sidesteps the+// NOW() vs datetime('now') dialect difference.+func updateChunksRowByRow(tx *gorm.DB, chunks []*types.Chunk) error {+ for _, chunk := range chunks {+ err := tx.Unscoped().Model(&types.Chunk{}).+ Where("id = ?", chunk.ID).+ Updates(map[string]interface{}{+ "content": common.CleanInvalidUTF8(chunk.Content),+ "is_enabled": chunk.IsEnabled,+ "tag_id": chunk.TagID,+ "flags": int(chunk.Flags),+ "status": chunk.Status,+ }).Error+ if err != nil {+ return err}- isEnabledCases = append(isEnabledCases, "WHEN id = ? THEN ?")- isEnabledArgs = append(isEnabledArgs, chunk.ID, isEnabledStr)-- tagIDCases = append(tagIDCases, "WHEN id = ? THEN ?")- tagIDArgs = append(tagIDArgs, chunk.ID, chunk.TagID)-- flagsCases = append(flagsCases, "WHEN id = ? THEN ?")- flagsArgs = append(flagsArgs, chunk.ID, fmt.Sprintf("%d", chunk.Flags))-- statusCases = append(statusCases, "WHEN id = ? THEN ?")- statusArgs = append(statusArgs, chunk.ID, fmt.Sprintf("%d", chunk.Status))}+ return nil+}- // Build IN clause placeholders- inPlaceholders := make([]string, len(ids))- for i := range ids {- inPlaceholders[i] = "?"+// UpdateChunkFieldsByIDs sets the same column values on every listed chunk+// with one UPDATE per batch of IDs. Use it for uniform state transitions+// (for example flipping status after indexing) instead of UpdateChunks, which+// has to ship every row's content back to the database.+//+// fields maps column names to values. updated_at is set automatically.+func (r *chunkRepository) UpdateChunkFieldsByIDs(+ ctx context.Context, tenantID uint64, ids []string, fields map[string]interface{},+) error {+ if len(ids) == 0 || len(fields) == 0 {+ return nil}-- // Combine args in correct order: content, is_enabled, tag_id, flags, status, then IN clause- var args []interface{}- args = append(args, contentArgs...)- args = append(args, isEnabledArgs...)- args = append(args, tagIDArgs...)- args = append(args, flagsArgs...)- args = append(args, statusArgs...)- for _, id := range ids {- args = append(args, id)+ updates := make(map[string]interface{}, len(fields)+1)+ for column, value := range fields {+ updates[column] = value}-- isPostgres := r.db.Dialector.Name() == "postgres"-- var sql string- if isPostgres {- sql = fmt.Sprintf(`- UPDATE chunks SET- content = CASE %s END,- is_enabled = (CASE %s END)::boolean,- tag_id = CASE %s END,- flags = (CASE %s END)::integer,- status = (CASE %s END)::integer,- updated_at = NOW()- WHERE id IN (%s)- `,- strings.Join(contentCases, " "),- strings.Join(isEnabledCases, " "),- strings.Join(tagIDCases, " "),- strings.Join(flagsCases, " "),- strings.Join(statusCases, " "),- strings.Join(inPlaceholders, ","),- )- } else {- sql = fmt.Sprintf(`- UPDATE chunks SET- content = CASE %s END,- is_enabled = CASE %s END,- tag_id = CASE %s END,- flags = CASE %s END,- status = CASE %s END,- updated_at = datetime('now')- WHERE id IN (%s)- `,- strings.Join(contentCases, " "),- strings.Join(isEnabledCases, " "),- strings.Join(tagIDCases, " "),- strings.Join(flagsCases, " "),- strings.Join(statusCases, " "),- strings.Join(inPlaceholders, ","),- )+ if _, ok := updates["updated_at"]; !ok {+ updates["updated_at"] = time.Now()}- return r.db.WithContext(ctx).Exec(sql, args...).Error+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ for start := 0; start < len(ids); start += updateByIDsBatchSize {+ end := start + updateByIDsBatchSize+ if end > len(ids) {+ end = len(ids)+ }+ err := tx.Model(&types.Chunk{}).+ Where("tenant_id = ? AND id IN ?", tenantID, ids[start:end]).+ Updates(updates).Error+ if err != nil {+ return err+ }+ }+ return nil+ })}// DeleteChunk deletes a chunk by its IDdiff --git a/internal/application/repository/chunk_sqlite_test.go b/internal/application/repository/chunk_sqlite_test.goindex 89587055a..93c2f0a63 100644--- a/internal/application/repository/chunk_sqlite_test.go+++ b/internal/application/repository/chunk_sqlite_test.go@@ -361,3 +361,118 @@ func TestListRecentDocumentChunksWithQuestions_UnionsExplicitKBAndKnowledge(t *trequire.Len(t, got, 2)assert.ElementsMatch(t, []string{fromExplicitKB.ID, fromExplicitDocument.ID}, []string{got[0].ID, got[1].ID})}++func TestCreateChunks_LeavesSourceContentEmpty(t *testing.T) {+ db := setupChunkTestDB(t)+ repo := NewChunkRepository(db)++ chunk := makeChunk("kb-src", "k-src", types.ChunkTypeText)+ require.NoError(t, repo.CreateChunks(context.Background(), []*types.Chunk{chunk}))++ var saved types.Chunk+ require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error)+ assert.Equal(t, "test content", saved.Content)+ assert.Empty(t, saved.SourceContent, "source_content is backfilled lazily on first edit, not on create")+ assert.Equal(t, "ready", saved.IndexStatus)+}++func TestUpdateChunks_UpdatesOnlyDocumentedColumns(t *testing.T) {+ db := setupChunkTestDB(t)+ repo := NewChunkRepository(db)+ ctx := context.Background()++ chunks := make([]*types.Chunk, 0, 3)+ for i := 0; i < 3; i++ {+ c := makeChunk("kb-upd", "k-upd", types.ChunkTypeFAQ)+ c.Metadata = types.JSON(`{"keep":"me"}`)+ c.ContentHash = "hash-original"+ chunks = append(chunks, c)+ }+ require.NoError(t, repo.CreateChunks(ctx, chunks))++ var before types.Chunk+ require.NoError(t, db.First(&before, "id = ?", chunks[0].ID).Error)++ chunks[0].Content = "edited-0"+ chunks[0].IsEnabled = false+ chunks[0].Flags = 0+ chunks[0].Status = int(types.ChunkStatusIndexed)+ chunks[0].TagID = "tag-a"+ chunks[0].ContentHash = "hash-should-not-persist"+ chunks[1].Content = "edited-1"+ chunks[1].Status = int(types.ChunkStatusStored)+ // chunks[2] is deliberately not passed to UpdateChunks.+ require.NoError(t, repo.UpdateChunks(ctx, chunks[:2]))++ var saved []types.Chunk+ require.NoError(t, db.Order("chunk_index").Find(&saved, "knowledge_id = ?", "k-upd").Error)+ byID := make(map[string]types.Chunk, len(saved))+ for _, s := range saved {+ byID[s.ID] = s+ }++ c0 := byID[chunks[0].ID]+ assert.Equal(t, "edited-0", c0.Content)+ assert.False(t, c0.IsEnabled, "zero-value bool must be written")+ assert.Equal(t, types.ChunkFlags(0), c0.Flags, "zero-value flags must be written")+ assert.Equal(t, int(types.ChunkStatusIndexed), c0.Status)+ assert.Equal(t, "tag-a", c0.TagID)+ assert.Equal(t, "hash-original", c0.ContentHash, "content_hash is outside the UpdateChunks contract")+ assert.JSONEq(t, `{"keep":"me"}`, string(c0.Metadata), "metadata is outside the UpdateChunks contract")+ assert.Equal(t, before.SeqID, c0.SeqID)+ assert.False(t, c0.UpdatedAt.Before(before.UpdatedAt))++ c1 := byID[chunks[1].ID]+ assert.Equal(t, "edited-1", c1.Content)+ assert.True(t, c1.IsEnabled)+ assert.Equal(t, int(types.ChunkStatusStored), c1.Status)++ c2 := byID[chunks[2].ID]+ assert.Equal(t, "test content", c2.Content, "rows not in the batch stay untouched")+}++func TestUpdateChunks_CleansInvalidUTF8(t *testing.T) {+ db := setupChunkTestDB(t)+ repo := NewChunkRepository(db)+ ctx := context.Background()++ chunk := makeChunk("kb-utf8", "k-utf8", types.ChunkTypeText)+ require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{chunk}))++ chunk.Content = "ok\xffbad"+ require.NoError(t, repo.UpdateChunks(ctx, []*types.Chunk{chunk}))++ var saved types.Chunk+ require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error)+ assert.True(t, utf8.ValidString(saved.Content))+}++func TestUpdateChunkFieldsByIDs_ScopesByTenantAndSetsUpdatedAt(t *testing.T) {+ db := setupChunkTestDB(t)+ repo := NewChunkRepository(db)+ ctx := context.Background()++ mine := makeChunk("kb-fields", "k-fields", types.ChunkTypeFAQ)+ other := makeChunk("kb-fields", "k-fields", types.ChunkTypeFAQ)+ other.TenantID = 2+ require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{mine, other}))++ var before types.Chunk+ require.NoError(t, db.First(&before, "id = ?", mine.ID).Error)++ require.NoError(t, repo.UpdateChunkFieldsByIDs(ctx, 1, []string{mine.ID, other.ID}, map[string]interface{}{+ "status": int(types.ChunkStatusIndexed),+ }))++ var savedMine, savedOther types.Chunk+ require.NoError(t, db.First(&savedMine, "id = ?", mine.ID).Error)+ require.NoError(t, db.First(&savedOther, "id = ?", other.ID).Error)+ assert.Equal(t, int(types.ChunkStatusIndexed), savedMine.Status)+ assert.Equal(t, "test content", savedMine.Content, "other columns are untouched")+ assert.False(t, savedMine.UpdatedAt.Before(before.UpdatedAt))+ assert.Equal(t, int(types.ChunkStatusDefault), savedOther.Status, "rows of another tenant must not change")++ // Empty inputs are no-ops.+ require.NoError(t, repo.UpdateChunkFieldsByIDs(ctx, 1, nil, map[string]interface{}{"status": 0}))+ require.NoError(t, repo.UpdateChunkFieldsByIDs(ctx, 1, []string{mine.ID}, nil))+}diff --git a/internal/application/service/knowledge_faq_import.go b/internal/application/service/knowledge_faq_import.goindex c944b846d..ec4b2c1d1 100644--- a/internal/application/service/knowledge_faq_import.go+++ b/internal/application/service/knowledge_faq_import.go@@ -1593,17 +1593,19 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,indexDuration,)- // 更新chunks的Status为已索引- chunksToUpdate := make([]*types.Chunk, 0, len(chunks))+ // 更新chunks的Status为已索引:所有行写同一个值,一条 UPDATE ... WHERE id IN 即可,+ // 不需要把 content 等字段再回传一遍。for _, chunk := range chunks {chunk.Status = int(types.ChunkStatusIndexed) // indexed- chunksToUpdate = append(chunksToUpdate, chunk)}- if err := s.chunkService.UpdateChunks(ctx, chunksToUpdate); err != nil {+ if err := s.chunkRepo.UpdateChunkFieldsByIDs(ctx, tenantID, chunkIds, map[string]interface{}{+ "status": int(types.ChunkStatusIndexed),+ }); err != nil {return fmt.Errorf("failed to update chunks status: %w", err)}- // 收集成功条目信息+ // 收集成功条目信息(tag 信息按批一次查出,避免每条一次查询)+ tagsByID := s.loadFAQTagsForChunks(ctx, tenantID, chunks)for idx, chunk := range chunks {entryIdx := i + idx + processedCount // 原始条目索引meta, _ := chunk.FAQMetadata()@@ -1611,15 +1613,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,if meta != nil {standardQ = meta.StandardQuestion}- // 获取 tag info- var tagID int64- tagName := ""- if chunk.TagID != "" {- if tag, err := s.tagRepo.GetByID(ctx, tenantID, chunk.TagID); err == nil && tag != nil {- tagID = tag.SeqID- tagName = tag.Name- }- }+ tagID, tagName := faqTagInfo(tagsByID, chunk.TagID)progress.SuccessEntries = append(progress.SuccessEntries, types.FAQSuccessEntry{Index: entryIdx,SeqID: chunk.SeqID,@@ -2687,17 +2681,11 @@ func (s *knowledgeService) executeFAQMergeOperations(}// 5. 收集成功条目信息+ tagsByID := s.loadFAQTagsForChunks(ctx, tenantID, mergedChunks)for i, op := range batch {chunk := mergedChunks[i]meta := op.MergedMeta- var tagID int64- tagName := ""- if chunk.TagID != "" {- if tag, tErr := s.tagRepo.GetByID(ctx, tenantID, chunk.TagID); tErr == nil && tag != nil {- tagID = tag.SeqID- tagName = tag.Name- }- }+ tagID, tagName := faqTagInfo(tagsByID, chunk.TagID)progress.SuccessEntries = append(progress.SuccessEntries, types.FAQSuccessEntry{Index: op.Detail.Index,SeqID: chunk.SeqID,@@ -2715,6 +2703,53 @@ func (s *knowledgeService) executeFAQMergeOperations(return mergedCount, nil}+// loadFAQTagsForChunks resolves every distinct tag referenced by chunks with a+// single query. Lookup failures are logged and yield an empty map so the+// import result degrades to "no tag info" instead of aborting the batch.+func (s *knowledgeService) loadFAQTagsForChunks(+ ctx context.Context, tenantID uint64, chunks []*types.Chunk,+) map[string]*types.KnowledgeTag {+ tagsByID := make(map[string]*types.KnowledgeTag)+ seen := make(map[string]struct{})+ ids := make([]string, 0)+ for _, chunk := range chunks {+ if chunk == nil || chunk.TagID == "" {+ continue+ }+ if _, ok := seen[chunk.TagID]; ok {+ continue+ }+ seen[chunk.TagID] = struct{}{}+ ids = append(ids, chunk.TagID)+ }+ if len(ids) == 0 {+ return tagsByID+ }+ tags, err := s.tagRepo.GetByIDs(ctx, tenantID, ids)+ if err != nil {+ logger.Warnf(ctx, "Failed to load FAQ tags for import result: %v", err)+ return tagsByID+ }+ for _, tag := range tags {+ if tag != nil {+ tagsByID[tag.ID] = tag+ }+ }+ return tagsByID+}++// faqTagInfo returns the external (seq_id, name) pair for tagID, or zero values+// when the chunk has no tag or the tag could not be loaded.+func faqTagInfo(tagsByID map[string]*types.KnowledgeTag, tagID string) (int64, string) {+ if tagID == "" {+ return 0, ""+ }+ if tag, ok := tagsByID[tagID]; ok && tag != nil {+ return tag.SeqID, tag.Name+ }+ return 0, ""+}+// buildFAQImportResultMessage 构建 FAQ 导入 / 验证最终结果的人类可读消息。// 前端会把它直接展示在 toast / 任务列表里,所以保持简单清晰:// - 默认形态:"导入完成 / 上传 N 条 / 成功 X 条 [/ 失败 Y 条] [/ 部分失败 Z 条]"diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.goindex ae3e68eb1..f02d26181 100644--- a/internal/database/migration_sqlite_versioned_schema_test.go+++ b/internal/database/migration_sqlite_versioned_schema_test.go@@ -44,7 +44,7 @@ var versionedSQLiteColumns = map[string][]string{"mcp_tool_approvals": {"enabled"}, // 000091}-const expectedSQLiteMigrationVersion = 19+const expectedSQLiteMigrationVersion = 20func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {repoRoot := sqliteRepoRoot(t)diff --git a/internal/types/chunk.go b/internal/types/chunk.goindex 33a4d2366..72196d2ef 100644--- a/internal/types/chunk.go+++ b/internal/types/chunk.go@@ -163,7 +163,7 @@ type Chunk struct {// Metadata 存储 chunk 级别的扩展信息,例如 FAQ 元数据Metadata JSON `json:"metadata" gorm:"type:json"`// ContentHash 存储内容的 hash 值,用于快速匹配(主要用于 FAQ)- ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`+ ContentHash string `json:"content_hash" gorm:"type:varchar(64)"`// 图片信息,存储为 JSONImageInfo string `json:"image_info" gorm:"type:text"`// Chunk creation timediff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.goindex c8f7db2bd..90675ad86 100644--- a/internal/types/interfaces/chunk.go+++ b/internal/types/interfaces/chunk.go@@ -81,6 +81,10 @@ type ChunkRepository interface {UpdateChunks(ctx context.Context, chunks []*types.Chunk) error// SaveChunks persists full chunk objects in a single transaction using GORM Save (UPDATE).SaveChunks(ctx context.Context, chunks []*types.Chunk) error+ // UpdateChunkFieldsByIDs sets the same column values (e.g. {"status": 2})+ // on every listed chunk of the tenant with one UPDATE per batch of IDs.+ // updated_at is set automatically.+ UpdateChunkFieldsByIDs(ctx context.Context, tenantID uint64, ids []string, fields map[string]interface{}) error// DeleteChunk deletes a chunkDeleteChunk(ctx context.Context, tenantID uint64, id string) error// DeleteChunks deletes chunks by IDs in batchdiff --git a/migrations/sqlite/000020_chunks_index_diet.down.sql b/migrations/sqlite/000020_chunks_index_diet.down.sqlnew file mode 100644index 000000000..9b5f7f749--- /dev/null+++ b/migrations/sqlite/000020_chunks_index_diet.down.sql@@ -0,0 +1,3 @@+CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type);+CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);+CREATE INDEX IF NOT EXISTS idx_chunks_tenant_kg ON chunks(tenant_id, knowledge_id);diff --git a/migrations/sqlite/000020_chunks_index_diet.up.sql b/migrations/sqlite/000020_chunks_index_diet.up.sqlnew file mode 100644index 000000000..36a7f54ad--- /dev/null+++ b/migrations/sqlite/000020_chunks_index_diet.up.sql@@ -0,0 +1,4 @@+-- Drop chunks indexes that only slow down writes (see versioned/000100).+DROP INDEX IF EXISTS idx_chunks_chunk_type;+DROP INDEX IF EXISTS idx_chunks_content_hash;+DROP INDEX IF EXISTS idx_chunks_tenant_kg;diff --git a/migrations/versioned/000100_chunks_index_diet.down.sql b/migrations/versioned/000100_chunks_index_diet.down.sqlnew file mode 100644index 000000000..78393f807--- /dev/null+++ b/migrations/versioned/000100_chunks_index_diet.down.sql@@ -0,0 +1,4 @@+-- Migration 000100 rollback: recreate the dropped chunks indexes.+CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type);+CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);+CREATE INDEX IF NOT EXISTS idx_chunks_tenant_kg ON chunks(tenant_id, knowledge_id);diff --git a/migrations/versioned/000100_chunks_index_diet.up.sql b/migrations/versioned/000100_chunks_index_diet.up.sqlnew file mode 100644index 000000000..0773155ae--- /dev/null+++ b/migrations/versioned/000100_chunks_index_diet.up.sql@@ -0,0 +1,22 @@+-- ============================================================================+-- Migration 000100: Drop chunks indexes that only slow down writes+-- ============================================================================+-- Every INSERT/UPDATE on chunks maintains all of its indexes. Three of them+-- never help a query:+-- * idx_chunks_chunk_type — chunk_type has a handful of distinct values+-- and is always combined with knowledge_id /+-- knowledge_base_id, which already have indexes.+-- * idx_chunks_content_hash — no query filters on content_hash; FAQ diffing+-- reads the column but scans by knowledge_base_id.+-- * idx_chunks_tenant_kg — (tenant_id, knowledge_id) is covered by+-- idx_chunks_knowledge_enabled, which leads with+-- the high-cardinality knowledge_id.+-- ============================================================================++DO $$ BEGIN RAISE NOTICE '[Migration 000100] Dropping redundant chunks indexes'; END $$;++DROP INDEX IF EXISTS idx_chunks_chunk_type;+DROP INDEX IF EXISTS idx_chunks_content_hash;+DROP INDEX IF EXISTS idx_chunks_tenant_kg;++DO $$ BEGIN RAISE NOTICE '[Migration 000100] Redundant chunks indexes dropped'; END $$;