Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

perf(chunk): speed up chunk ingestion and batch updates (#3369)

lyingbug68d99d7d2111
diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go
index d06c83cf7..78eb77471 100644
--- a/internal/application/repository/chunk.go
+++ b/internal/application/repository/chunk.go
@@ -33,40 +33,56 @@ func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository {
return &chunkRepository{db: db}
}
-// CreateChunks creates multiple chunks in batches.
-// Uses Omit("SeqID") so GORM won't include the auto-increment column in the
-// INSERT, which avoids MySQL generating ON DUPLICATE KEY UPDATE and the
-// resulting gap-lock deadlocks under concurrent writes.
-// A deadlock retry wrapper is kept as defense-in-depth for any remaining
-// edge cases on secondary unique indexes.
+// createChunksBatchSize is the number of rows per INSERT statement. Chunk rows
+// carry long text columns, so the batch is kept well below the bind-parameter
+// limits of the supported drivers while still amortizing round trips.
+const createChunksBatchSize = 500
+
+// updateChunksBatchSize bounds the rows per batch UPDATE so the bind-parameter
+// count (6 per row) stays far below the PostgreSQL limit of 65535.
+const updateChunksBatchSize = 1000
+
+// updateByIDsBatchSize bounds the IN list for UPDATE ... WHERE id IN (...).
+const updateByIDsBatchSize = 5000
+
+// CreateChunks creates multiple chunks in batches inside a single transaction.
+//
+// SourceContent is intentionally left empty on create. It records the parser
+// output only once a user edits the chunk (see chunkService.UpdateDocumentChunk,
+// which backfills it from Content on the first edit). Writing a copy of Content
+// for every chunk doubled the insert volume and the TOAST footprint for no
+// benefit.
func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
+ if len(chunks) == 0 {
+ return nil
+ }
for _, chunk := range chunks {
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
chunk.ContextHeader = common.CleanInvalidUTF8(chunk.ContextHeader)
- if chunk.SourceContent == "" {
- chunk.SourceContent = chunk.Content
+ if chunk.SourceContent != "" {
+ chunk.SourceContent = common.CleanInvalidUTF8(chunk.SourceContent)
}
if chunk.IndexStatus == "" {
chunk.IndexStatus = "ready"
}
}
- db := r.db.WithContext(ctx)
-
- // SQLite doesn't support autoIncrement on non-PK columns,
- // so we must pre-assign SeqIDs manually (safe: single connection).
- // PostgreSQL / MySQL use DB sequences — skip to avoid duplicate key
- // races under concurrent inserts.
- if db.Dialector.Name() == "sqlite" {
- if err := types.AssignChunkSeqIDs(db, chunks); err != nil {
- return fmt.Errorf("failed to assign chunk seq_ids: %w", err)
+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ // SQLite doesn't support autoIncrement on non-PK columns, so SeqIDs are
+ // pre-assigned from MAX(seq_id). Doing it inside the write transaction
+ // keeps the read and the insert on the same connection.
+ // PostgreSQL uses a DB sequence — skip to avoid duplicate key races.
+ if tx.Name() == "sqlite" {
+ if err := types.AssignChunkSeqIDs(tx, chunks); err != nil {
+ return fmt.Errorf("failed to assign chunk seq_ids: %w", err)
+ }
}
- }
- // Select("*") ensures zero-value fields (IsEnabled=false, Flags=0) are
- // explicitly inserted, bypassing GORM's default value behavior.
- // SeqID=0 is skipped by GORM automatically (autoIncrement tag).
- return db.Select("*").CreateInBatches(chunks, 100).Error
+ // Select("*") ensures zero-value fields (IsEnabled=false, Flags=0) are
+ // explicitly inserted, bypassing GORM's default value behavior.
+ // SeqID=0 is skipped by GORM automatically (autoIncrement tag).
+ return tx.Select("*").CreateInBatches(chunks, createChunksBatchSize).Error
+ })
}
// GetChunkByID retrieves a chunk by its ID and tenant ID
@@ -414,109 +430,127 @@ func (r *chunkRepository) SaveChunks(ctx context.Context, chunks []*types.Chunk)
// - other fields not listed above
//
// If you need to update metadata or content_hash, use UpdateChunk (single) instead.
+//
+// On PostgreSQL the rows are joined against a VALUES list, so the statement
+// costs O(N) instead of the O(N²) of a CASE-per-column chain. Other dialects
+// run one small UPDATE per row inside a single transaction, which is also
+// O(N) and keeps the statement well within their bind-parameter limits.
func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error {
if len(chunks) == 0 {
return nil
}
- // Build batch update SQL with CASE expressions
- var ids []string
- contentCases := make([]string, 0, len(chunks))
- isEnabledCases := make([]string, 0, len(chunks))
- tagIDCases := make([]string, 0, len(chunks))
- flagsCases := make([]string, 0, len(chunks))
- statusCases := make([]string, 0, len(chunks))
+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ for start := 0; start < len(chunks); start += updateChunksBatchSize {
+ end := start + updateChunksBatchSize
+ if end > len(chunks) {
+ end = len(chunks)
+ }
+ batch := chunks[start:end]
- var contentArgs []interface{}
- var isEnabledArgs []interface{}
- var tagIDArgs []interface{}
- var flagsArgs []interface{}
- var statusArgs []interface{}
+ var err error
+ if tx.Name() == "postgres" {
+ err = updateChunksPostgres(tx, batch)
+ } else {
+ err = updateChunksRowByRow(tx, batch)
+ }
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
+// updateChunksPostgres issues a single UPDATE ... FROM (VALUES ...) statement.
+// Every value is cast explicitly so PostgreSQL can type the VALUES columns
+// without inspecting the bind parameters.
+func updateChunksPostgres(tx *gorm.DB, chunks []*types.Chunk) error {
+ rows := make([]string, 0, len(chunks))
+ args := make([]interface{}, 0, len(chunks)*6)
for _, chunk := range chunks {
- ids = append(ids, chunk.ID)
- content := common.CleanInvalidUTF8(chunk.Content)
+ rows = append(rows, "(?::varchar, ?::text, ?::boolean, ?::varchar, ?::integer, ?::integer)")
+ args = append(args,
+ chunk.ID,
+ common.CleanInvalidUTF8(chunk.Content),
+ chunk.IsEnabled,
+ chunk.TagID,
+ int(chunk.Flags),
+ chunk.Status,
+ )
+ }
- contentCases = append(contentCases, "WHEN id = ? THEN ?")
- contentArgs = append(contentArgs, chunk.ID, content)
+ sql := fmt.Sprintf(`
+ UPDATE chunks AS c SET
+ content = v.content,
+ is_enabled = v.is_enabled,
+ tag_id = v.tag_id,
+ flags = v.flags,
+ status = v.status,
+ updated_at = NOW()
+ FROM (VALUES %s) AS v(id, content, is_enabled, tag_id, flags, status)
+ WHERE c.id = v.id
+ `, strings.Join(rows, ",\n"))
+
+ return tx.Exec(sql, args...).Error
+}
- // Convert bool to string for PostgreSQL compatibility
- isEnabledStr := "false"
- if chunk.IsEnabled {
- isEnabledStr = "true"
+// updateChunksRowByRow updates each chunk with its own statement. GORM sets
+// updated_at automatically for map-based Updates, which also sidesteps the
+// NOW() vs datetime('now') dialect difference.
+func updateChunksRowByRow(tx *gorm.DB, chunks []*types.Chunk) error {
+ for _, chunk := range chunks {
+ err := tx.Unscoped().Model(&types.Chunk{}).
+ Where("id = ?", chunk.ID).
+ Updates(map[string]interface{}{
+ "content": common.CleanInvalidUTF8(chunk.Content),
+ "is_enabled": chunk.IsEnabled,
+ "tag_id": chunk.TagID,
+ "flags": int(chunk.Flags),
+ "status": chunk.Status,
+ }).Error
+ if err != nil {
+ return err
}
- isEnabledCases = append(isEnabledCases, "WHEN id = ? THEN ?")
- isEnabledArgs = append(isEnabledArgs, chunk.ID, isEnabledStr)
-
- tagIDCases = append(tagIDCases, "WHEN id = ? THEN ?")
- tagIDArgs = append(tagIDArgs, chunk.ID, chunk.TagID)
-
- flagsCases = append(flagsCases, "WHEN id = ? THEN ?")
- flagsArgs = append(flagsArgs, chunk.ID, fmt.Sprintf("%d", chunk.Flags))
-
- statusCases = append(statusCases, "WHEN id = ? THEN ?")
- statusArgs = append(statusArgs, chunk.ID, fmt.Sprintf("%d", chunk.Status))
}
+ return nil
+}
- // Build IN clause placeholders
- inPlaceholders := make([]string, len(ids))
- for i := range ids {
- inPlaceholders[i] = "?"
+// UpdateChunkFieldsByIDs sets the same column values on every listed chunk
+// with one UPDATE per batch of IDs. Use it for uniform state transitions
+// (for example flipping status after indexing) instead of UpdateChunks, which
+// has to ship every row's content back to the database.
+//
+// fields maps column names to values. updated_at is set automatically.
+func (r *chunkRepository) UpdateChunkFieldsByIDs(
+ ctx context.Context, tenantID uint64, ids []string, fields map[string]interface{},
+) error {
+ if len(ids) == 0 || len(fields) == 0 {
+ return nil
}
-
- // Combine args in correct order: content, is_enabled, tag_id, flags, status, then IN clause
- var args []interface{}
- args = append(args, contentArgs...)
- args = append(args, isEnabledArgs...)
- args = append(args, tagIDArgs...)
- args = append(args, flagsArgs...)
- args = append(args, statusArgs...)
- for _, id := range ids {
- args = append(args, id)
+ updates := make(map[string]interface{}, len(fields)+1)
+ for column, value := range fields {
+ updates[column] = value
}
-
- isPostgres := r.db.Dialector.Name() == "postgres"
-
- var sql string
- if isPostgres {
- sql = fmt.Sprintf(`
- UPDATE chunks SET
- content = CASE %s END,
- is_enabled = (CASE %s END)::boolean,
- tag_id = CASE %s END,
- flags = (CASE %s END)::integer,
- status = (CASE %s END)::integer,
- updated_at = NOW()
- WHERE id IN (%s)
- `,
- strings.Join(contentCases, " "),
- strings.Join(isEnabledCases, " "),
- strings.Join(tagIDCases, " "),
- strings.Join(flagsCases, " "),
- strings.Join(statusCases, " "),
- strings.Join(inPlaceholders, ","),
- )
- } else {
- sql = fmt.Sprintf(`
- UPDATE chunks SET
- content = CASE %s END,
- is_enabled = CASE %s END,
- tag_id = CASE %s END,
- flags = CASE %s END,
- status = CASE %s END,
- updated_at = datetime('now')
- WHERE id IN (%s)
- `,
- strings.Join(contentCases, " "),
- strings.Join(isEnabledCases, " "),
- strings.Join(tagIDCases, " "),
- strings.Join(flagsCases, " "),
- strings.Join(statusCases, " "),
- strings.Join(inPlaceholders, ","),
- )
+ if _, ok := updates["updated_at"]; !ok {
+ updates["updated_at"] = time.Now()
}
- return r.db.WithContext(ctx).Exec(sql, args...).Error
+ return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ for start := 0; start < len(ids); start += updateByIDsBatchSize {
+ end := start + updateByIDsBatchSize
+ if end > len(ids) {
+ end = len(ids)
+ }
+ err := tx.Model(&types.Chunk{}).
+ Where("tenant_id = ? AND id IN ?", tenantID, ids[start:end]).
+ Updates(updates).Error
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ })
}
// DeleteChunk deletes a chunk by its ID
diff --git a/internal/application/repository/chunk_sqlite_test.go b/internal/application/repository/chunk_sqlite_test.go
index 89587055a..93c2f0a63 100644
--- a/internal/application/repository/chunk_sqlite_test.go
+++ b/internal/application/repository/chunk_sqlite_test.go
@@ -361,3 +361,118 @@ func TestListRecentDocumentChunksWithQuestions_UnionsExplicitKBAndKnowledge(t *t
require.Len(t, got, 2)
assert.ElementsMatch(t, []string{fromExplicitKB.ID, fromExplicitDocument.ID}, []string{got[0].ID, got[1].ID})
}
+
+func TestCreateChunks_LeavesSourceContentEmpty(t *testing.T) {
+ db := setupChunkTestDB(t)
+ repo := NewChunkRepository(db)
+
+ chunk := makeChunk("kb-src", "k-src", types.ChunkTypeText)
+ require.NoError(t, repo.CreateChunks(context.Background(), []*types.Chunk{chunk}))
+
+ var saved types.Chunk
+ require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error)
+ assert.Equal(t, "test content", saved.Content)
+ assert.Empty(t, saved.SourceContent, "source_content is backfilled lazily on first edit, not on create")
+ assert.Equal(t, "ready", saved.IndexStatus)
+}
+
+func TestUpdateChunks_UpdatesOnlyDocumentedColumns(t *testing.T) {
+ db := setupChunkTestDB(t)
+ repo := NewChunkRepository(db)
+ ctx := context.Background()
+
+ chunks := make([]*types.Chunk, 0, 3)
+ for i := 0; i < 3; i++ {
+ c := makeChunk("kb-upd", "k-upd", types.ChunkTypeFAQ)
+ c.Metadata = types.JSON(`{"keep":"me"}`)
+ c.ContentHash = "hash-original"
+ chunks = append(chunks, c)
+ }
+ require.NoError(t, repo.CreateChunks(ctx, chunks))
+
+ var before types.Chunk
+ require.NoError(t, db.First(&before, "id = ?", chunks[0].ID).Error)
+
+ chunks[0].Content = "edited-0"
+ chunks[0].IsEnabled = false
+ chunks[0].Flags = 0
+ chunks[0].Status = int(types.ChunkStatusIndexed)
+ chunks[0].TagID = "tag-a"
+ chunks[0].ContentHash = "hash-should-not-persist"
+ chunks[1].Content = "edited-1"
+ chunks[1].Status = int(types.ChunkStatusStored)
+ // chunks[2] is deliberately not passed to UpdateChunks.
+ require.NoError(t, repo.UpdateChunks(ctx, chunks[:2]))
+
+ var saved []types.Chunk
+ require.NoError(t, db.Order("chunk_index").Find(&saved, "knowledge_id = ?", "k-upd").Error)
+ byID := make(map[string]types.Chunk, len(saved))
+ for _, s := range saved {
+ byID[s.ID] = s
+ }
+
+ c0 := byID[chunks[0].ID]
+ assert.Equal(t, "edited-0", c0.Content)
+ assert.False(t, c0.IsEnabled, "zero-value bool must be written")
+ assert.Equal(t, types.ChunkFlags(0), c0.Flags, "zero-value flags must be written")
+ assert.Equal(t, int(types.ChunkStatusIndexed), c0.Status)
+ assert.Equal(t, "tag-a", c0.TagID)
+ assert.Equal(t, "hash-original", c0.ContentHash, "content_hash is outside the UpdateChunks contract")
+ assert.JSONEq(t, `{"keep":"me"}`, string(c0.Metadata), "metadata is outside the UpdateChunks contract")
+ assert.Equal(t, before.SeqID, c0.SeqID)
+ assert.False(t, c0.UpdatedAt.Before(before.UpdatedAt))
+
+ c1 := byID[chunks[1].ID]
+ assert.Equal(t, "edited-1", c1.Content)
+ assert.True(t, c1.IsEnabled)
+ assert.Equal(t, int(types.ChunkStatusStored), c1.Status)
+
+ c2 := byID[chunks[2].ID]
+ assert.Equal(t, "test content", c2.Content, "rows not in the batch stay untouched")
+}
+
+func TestUpdateChunks_CleansInvalidUTF8(t *testing.T) {
+ db := setupChunkTestDB(t)
+ repo := NewChunkRepository(db)
+ ctx := context.Background()
+
+ chunk := makeChunk("kb-utf8", "k-utf8", types.ChunkTypeText)
+ require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{chunk}))
+
+ chunk.Content = "ok\xffbad"
+ require.NoError(t, repo.UpdateChunks(ctx, []*types.Chunk{chunk}))
+
+ var saved types.Chunk
+ require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error)
+ assert.True(t, utf8.ValidString(saved.Content))
+}
+
+func TestUpdateChunkFieldsByIDs_ScopesByTenantAndSetsUpdatedAt(t *testing.T) {
+ db := setupChunkTestDB(t)
+ repo := NewChunkRepository(db)
+ ctx := context.Background()
+
+ mine := makeChunk("kb-fields", "k-fields", types.ChunkTypeFAQ)
+ other := makeChunk("kb-fields", "k-fields", types.ChunkTypeFAQ)
+ other.TenantID = 2
+ require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{mine, other}))
+
+ var before types.Chunk
+ require.NoError(t, db.First(&before, "id = ?", mine.ID).Error)
+
+ require.NoError(t, repo.UpdateChunkFieldsByIDs(ctx, 1, []string{mine.ID, other.ID}, map[string]interface{}{
+ "status": int(types.ChunkStatusIndexed),
+ }))
+
+ var savedMine, savedOther types.Chunk
+ require.NoError(t, db.First(&savedMine, "id = ?", mine.ID).Error)
+ require.NoError(t, db.First(&savedOther, "id = ?", other.ID).Error)
+ assert.Equal(t, int(types.ChunkStatusIndexed), savedMine.Status)
+ assert.Equal(t, "test content", savedMine.Content, "other columns are untouched")
+ assert.False(t, savedMine.UpdatedAt.Before(before.UpdatedAt))
+ assert.Equal(t, int(types.ChunkStatusDefault), savedOther.Status, "rows of another tenant must not change")
+
+ // Empty inputs are no-ops.
+ require.NoError(t, repo.UpdateChunkFieldsByIDs(ctx, 1, nil, map[string]interface{}{"status": 0}))
+ require.NoError(t, repo.UpdateChunkFieldsByIDs(ctx, 1, []string{mine.ID}, nil))
+}
diff --git a/internal/application/service/knowledge_faq_import.go b/internal/application/service/knowledge_faq_import.go
index c944b846d..ec4b2c1d1 100644
--- a/internal/application/service/knowledge_faq_import.go
+++ b/internal/application/service/knowledge_faq_import.go
@@ -1593,17 +1593,19 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
indexDuration,
)
- // 更新chunks的Status为已索引
- chunksToUpdate := make([]*types.Chunk, 0, len(chunks))
+ // 更新chunks的Status为已索引:所有行写同一个值,一条 UPDATE ... WHERE id IN 即可,
+ // 不需要把 content 等字段再回传一遍。
for _, chunk := range chunks {
chunk.Status = int(types.ChunkStatusIndexed) // indexed
- chunksToUpdate = append(chunksToUpdate, chunk)
}
- if err := s.chunkService.UpdateChunks(ctx, chunksToUpdate); err != nil {
+ if err := s.chunkRepo.UpdateChunkFieldsByIDs(ctx, tenantID, chunkIds, map[string]interface{}{
+ "status": int(types.ChunkStatusIndexed),
+ }); err != nil {
return fmt.Errorf("failed to update chunks status: %w", err)
}
- // 收集成功条目信息
+ // 收集成功条目信息(tag 信息按批一次查出,避免每条一次查询)
+ tagsByID := s.loadFAQTagsForChunks(ctx, tenantID, chunks)
for idx, chunk := range chunks {
entryIdx := i + idx + processedCount // 原始条目索引
meta, _ := chunk.FAQMetadata()
@@ -1611,15 +1613,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
if meta != nil {
standardQ = meta.StandardQuestion
}
- // 获取 tag info
- var tagID int64
- tagName := ""
- if chunk.TagID != "" {
- if tag, err := s.tagRepo.GetByID(ctx, tenantID, chunk.TagID); err == nil && tag != nil {
- tagID = tag.SeqID
- tagName = tag.Name
- }
- }
+ tagID, tagName := faqTagInfo(tagsByID, chunk.TagID)
progress.SuccessEntries = append(progress.SuccessEntries, types.FAQSuccessEntry{
Index: entryIdx,
SeqID: chunk.SeqID,
@@ -2687,17 +2681,11 @@ func (s *knowledgeService) executeFAQMergeOperations(
}
// 5. 收集成功条目信息
+ tagsByID := s.loadFAQTagsForChunks(ctx, tenantID, mergedChunks)
for i, op := range batch {
chunk := mergedChunks[i]
meta := op.MergedMeta
- var tagID int64
- tagName := ""
- if chunk.TagID != "" {
- if tag, tErr := s.tagRepo.GetByID(ctx, tenantID, chunk.TagID); tErr == nil && tag != nil {
- tagID = tag.SeqID
- tagName = tag.Name
- }
- }
+ tagID, tagName := faqTagInfo(tagsByID, chunk.TagID)
progress.SuccessEntries = append(progress.SuccessEntries, types.FAQSuccessEntry{
Index: op.Detail.Index,
SeqID: chunk.SeqID,
@@ -2715,6 +2703,53 @@ func (s *knowledgeService) executeFAQMergeOperations(
return mergedCount, nil
}
+// loadFAQTagsForChunks resolves every distinct tag referenced by chunks with a
+// single query. Lookup failures are logged and yield an empty map so the
+// import result degrades to "no tag info" instead of aborting the batch.
+func (s *knowledgeService) loadFAQTagsForChunks(
+ ctx context.Context, tenantID uint64, chunks []*types.Chunk,
+) map[string]*types.KnowledgeTag {
+ tagsByID := make(map[string]*types.KnowledgeTag)
+ seen := make(map[string]struct{})
+ ids := make([]string, 0)
+ for _, chunk := range chunks {
+ if chunk == nil || chunk.TagID == "" {
+ continue
+ }
+ if _, ok := seen[chunk.TagID]; ok {
+ continue
+ }
+ seen[chunk.TagID] = struct{}{}
+ ids = append(ids, chunk.TagID)
+ }
+ if len(ids) == 0 {
+ return tagsByID
+ }
+ tags, err := s.tagRepo.GetByIDs(ctx, tenantID, ids)
+ if err != nil {
+ logger.Warnf(ctx, "Failed to load FAQ tags for import result: %v", err)
+ return tagsByID
+ }
+ for _, tag := range tags {
+ if tag != nil {
+ tagsByID[tag.ID] = tag
+ }
+ }
+ return tagsByID
+}
+
+// faqTagInfo returns the external (seq_id, name) pair for tagID, or zero values
+// when the chunk has no tag or the tag could not be loaded.
+func faqTagInfo(tagsByID map[string]*types.KnowledgeTag, tagID string) (int64, string) {
+ if tagID == "" {
+ return 0, ""
+ }
+ if tag, ok := tagsByID[tagID]; ok && tag != nil {
+ return tag.SeqID, tag.Name
+ }
+ return 0, ""
+}
+
// buildFAQImportResultMessage 构建 FAQ 导入 / 验证最终结果的人类可读消息。
// 前端会把它直接展示在 toast / 任务列表里,所以保持简单清晰:
// - 默认形态:"导入完成 / 上传 N 条 / 成功 X 条 [/ 失败 Y 条] [/ 部分失败 Z 条]"
diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.go
index ae3e68eb1..f02d26181 100644
--- a/internal/database/migration_sqlite_versioned_schema_test.go
+++ b/internal/database/migration_sqlite_versioned_schema_test.go
@@ -44,7 +44,7 @@ var versionedSQLiteColumns = map[string][]string{
"mcp_tool_approvals": {"enabled"}, // 000091
}
-const expectedSQLiteMigrationVersion = 19
+const expectedSQLiteMigrationVersion = 20
func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {
repoRoot := sqliteRepoRoot(t)
diff --git a/internal/types/chunk.go b/internal/types/chunk.go
index 33a4d2366..72196d2ef 100644
--- a/internal/types/chunk.go
+++ b/internal/types/chunk.go
@@ -163,7 +163,7 @@ type Chunk struct {
// Metadata 存储 chunk 级别的扩展信息,例如 FAQ 元数据
Metadata JSON `json:"metadata" gorm:"type:json"`
// ContentHash 存储内容的 hash 值,用于快速匹配(主要用于 FAQ)
- ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`
+ ContentHash string `json:"content_hash" gorm:"type:varchar(64)"`
// 图片信息,存储为 JSON
ImageInfo string `json:"image_info" gorm:"type:text"`
// Chunk creation time
diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go
index c8f7db2bd..90675ad86 100644
--- a/internal/types/interfaces/chunk.go
+++ b/internal/types/interfaces/chunk.go
@@ -81,6 +81,10 @@ type ChunkRepository interface {
UpdateChunks(ctx context.Context, chunks []*types.Chunk) error
// SaveChunks persists full chunk objects in a single transaction using GORM Save (UPDATE).
SaveChunks(ctx context.Context, chunks []*types.Chunk) error
+ // UpdateChunkFieldsByIDs sets the same column values (e.g. {"status": 2})
+ // on every listed chunk of the tenant with one UPDATE per batch of IDs.
+ // updated_at is set automatically.
+ UpdateChunkFieldsByIDs(ctx context.Context, tenantID uint64, ids []string, fields map[string]interface{}) error
// DeleteChunk deletes a chunk
DeleteChunk(ctx context.Context, tenantID uint64, id string) error
// DeleteChunks deletes chunks by IDs in batch
diff --git a/migrations/sqlite/000020_chunks_index_diet.down.sql b/migrations/sqlite/000020_chunks_index_diet.down.sql
new file mode 100644
index 000000000..9b5f7f749
--- /dev/null
+++ b/migrations/sqlite/000020_chunks_index_diet.down.sql
@@ -0,0 +1,3 @@
+CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type);
+CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
+CREATE INDEX IF NOT EXISTS idx_chunks_tenant_kg ON chunks(tenant_id, knowledge_id);
diff --git a/migrations/sqlite/000020_chunks_index_diet.up.sql b/migrations/sqlite/000020_chunks_index_diet.up.sql
new file mode 100644
index 000000000..36a7f54ad
--- /dev/null
+++ b/migrations/sqlite/000020_chunks_index_diet.up.sql
@@ -0,0 +1,4 @@
+-- Drop chunks indexes that only slow down writes (see versioned/000100).
+DROP INDEX IF EXISTS idx_chunks_chunk_type;
+DROP INDEX IF EXISTS idx_chunks_content_hash;
+DROP INDEX IF EXISTS idx_chunks_tenant_kg;
diff --git a/migrations/versioned/000100_chunks_index_diet.down.sql b/migrations/versioned/000100_chunks_index_diet.down.sql
new file mode 100644
index 000000000..78393f807
--- /dev/null
+++ b/migrations/versioned/000100_chunks_index_diet.down.sql
@@ -0,0 +1,4 @@
+-- Migration 000100 rollback: recreate the dropped chunks indexes.
+CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type);
+CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
+CREATE INDEX IF NOT EXISTS idx_chunks_tenant_kg ON chunks(tenant_id, knowledge_id);
diff --git a/migrations/versioned/000100_chunks_index_diet.up.sql b/migrations/versioned/000100_chunks_index_diet.up.sql
new file mode 100644
index 000000000..0773155ae
--- /dev/null
+++ b/migrations/versioned/000100_chunks_index_diet.up.sql
@@ -0,0 +1,22 @@
+-- ============================================================================
+-- Migration 000100: Drop chunks indexes that only slow down writes
+-- ============================================================================
+-- Every INSERT/UPDATE on chunks maintains all of its indexes. Three of them
+-- never help a query:
+-- * idx_chunks_chunk_type — chunk_type has a handful of distinct values
+-- and is always combined with knowledge_id /
+-- knowledge_base_id, which already have indexes.
+-- * idx_chunks_content_hash — no query filters on content_hash; FAQ diffing
+-- reads the column but scans by knowledge_base_id.
+-- * idx_chunks_tenant_kg — (tenant_id, knowledge_id) is covered by
+-- idx_chunks_knowledge_enabled, which leads with
+-- the high-cardinality knowledge_id.
+-- ============================================================================
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000100] Dropping redundant chunks indexes'; END $$;
+
+DROP INDEX IF EXISTS idx_chunks_chunk_type;
+DROP INDEX IF EXISTS idx_chunks_content_hash;
+DROP INDEX IF EXISTS idx_chunks_tenant_kg;
+
+DO $$ BEGIN RAISE NOTICE '[Migration 000100] Redundant chunks indexes dropped'; END $$;