Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Bound embedding scheduling and honor configured batches https://github.com/siyuan-note/siyuan/pull/20609

Daniel93ee4b0cef9f
diff --git a/app/src/types/api/index.d.ts b/app/src/types/api/index.d.ts
index f9285d1f93..07a4777465 100644
--- a/app/src/types/api/index.d.ts
+++ b/app/src/types/api/index.d.ts
@@ -6667,6 +6667,7 @@ export interface APIPOSTRoutes {
/**
* 使用共享规范化规则;智能体确认超时负数回退 600 秒,0 保留无限等待语义。
* 工具调用轮数和输出 token 上限的负数归零;温度超出 0–2 回退 1。
+ * 嵌入并发数和批大小省略、null 或小于 1 时分别使用 8 和 10,超过上限时分别限制为 64 和 128。
* ChatGPT 账户绑定变化时与主配置一起可恢复提交,失败时回滚且不发布新配置;未变化时不访问绑定存储。
*/
"/api/setting/setAI": {
diff --git a/app/src/types/config.d.ts b/app/src/types/config.d.ts
index b239288eaa..250bb7652c 100644
--- a/app/src/types/config.d.ts
+++ b/app/src/types/config.d.ts
@@ -233,7 +233,9 @@ declare namespace Config {
name: string;
timeout: number;
dimensions: number;
+ /** 嵌入索引并发请求数,默认 8,范围 1–64。 */
concurrency: number;
+ /** 每次嵌入索引请求的文本块数,默认 10,范围 1–128。 */
batchSize: number;
}
diff --git a/kernel/api/contract_setting_ranges_test.go b/kernel/api/contract_setting_ranges_test.go
index 1dd1034a94..c43c94c1e1 100644
--- a/kernel/api/contract_setting_ranges_test.go
+++ b/kernel/api/contract_setting_ranges_test.go
@@ -44,6 +44,12 @@ func TestAPIContractSettingValueRanges(t *testing.T) {
{"flashcard", "setFlashcard", setFlashcard, `{"reviewMode":7}`, map[string]float64{"reviewMode": 0}},
{"ai", "setAI", setAI, `{"agent":{"maxToolCallRounds":-7,"confirmTimeout":-9,"temperature":-3,"maxCompletionTokens":-2}}`,
map[string]float64{"agent.maxToolCallRounds": 0, "agent.confirmTimeout": 600, "agent.temperature": 1, "agent.maxCompletionTokens": 0}},
+ {"ai", "setAI", setAI, `{"embedding":{"concurrency":2147483647,"batchSize":2147483647}}`,
+ map[string]float64{"embedding.concurrency": 64, "embedding.batchSize": 128}},
+ {"ai", "setAI", setAI, `{"embedding":{"concurrency":-1,"batchSize":0}}`,
+ map[string]float64{"embedding.concurrency": 8, "embedding.batchSize": 10}},
+ {"ai", "setAI", setAI, `{"embedding":{"concurrency":null,"batchSize":null}}`,
+ map[string]float64{"embedding.concurrency": 8, "embedding.batchSize": 10}},
{"system", "", nil, `{"autoLaunch2":9,"lockScreenMode":3}`, map[string]float64{"autoLaunch2": 0, "lockScreenMode": 0}},
{"repo", "", nil, `{"indexRetentionDays":100000}`, map[string]float64{"indexRetentionDays": 100000}},
} {
diff --git a/kernel/apicontract/contracts.go b/kernel/apicontract/contracts.go
index b19785ee4e..0799e0ecb5 100644
--- a/kernel/apicontract/contracts.go
+++ b/kernel/apicontract/contracts.go
@@ -1095,6 +1095,7 @@ var SetBazaar = define[SetBazaarRequest, *SettingBazaar]("setBazaar", "/api/sett
// SetAI 使用共享规范化规则;智能体确认超时负数回退 600 秒,0 保留无限等待语义。
// 工具调用轮数和输出 token 上限的负数归零;温度超出 0–2 回退 1。
+// 嵌入并发数和批大小省略、null 或小于 1 时分别使用 8 和 10,超过上限时分别限制为 64 和 128。
// ChatGPT 账户绑定变化时与主配置一起可恢复提交,失败时回滚且不发布新配置;未变化时不访问绑定存储。
var SetAI = define[SetAIRequest, *SettingAI]("setAI", "/api/setting/setAI", AuthenticatedAccess|AdminAccess|WritableAccess, JSONBody, ResponseOptions{Setting: SettingWrite{Namespace: "ai"}}, "POST")
var SetSecrets = define[SetSecretsRequest, *SettingSecrets]("setSecrets", "/api/setting/setSecrets", AuthenticatedAccess|AdminAccess|WritableAccess, JSONBody, ResponseOptions{Setting: SettingWrite{Namespace: "secrets"}}, "POST")
diff --git a/kernel/apicontract/setting.go b/kernel/apicontract/setting.go
index 466431751c..f7e0de02ed 100644
--- a/kernel/apicontract/setting.go
+++ b/kernel/apicontract/setting.go
@@ -48,11 +48,11 @@ type SettingCapabilityPolicy struct {
}
type SettingEmbedding struct {
- ID string `json:"id" api:"optional,nullable"`
- Enabled bool `json:"enabled" api:"optional,nullable"`
- APIKey string `json:"apiKey" api:"optional,nullable"`
- BaseURL string `json:"baseURL" api:"optional,nullable"`
- Name string `json:"name" api:"optional,nullable"`
+ ID string `json:"id" api:"optional,nullable"`
+ Enabled bool `json:"enabled" api:"optional,nullable"`
+ APIKey string `json:"apiKey" api:"optional,nullable"`
+ BaseURL string `json:"baseURL" api:"optional,nullable"`
+ Name string `json:"name" api:"optional,nullable"`
Timeout int `json:"timeout" api:"optional,nullable"`
Dimensions int `json:"dimensions" api:"optional,nullable"`
Concurrency int `json:"concurrency" api:"optional,nullable"`
diff --git a/kernel/conf/ai.go b/kernel/conf/ai.go
index 7d5ae2e4a7..b417570f52 100644
--- a/kernel/conf/ai.go
+++ b/kernel/conf/ai.go
@@ -110,12 +110,20 @@ type ImageGeneration struct {
OutputFormat string `json:"outputFormat"`
}
+// 嵌入索引调度的默认值和资源上限,与设置控件范围保持一致。
+const (
+ DefaultEmbeddingConcurrency = 8
+ MaxEmbeddingConcurrency = 64
+ DefaultEmbeddingBatchSize = 10
+ MaxEmbeddingBatchSize = 128
+)
+
type Embedding struct {
- ID string `json:"id"`
- Enabled bool `json:"enabled"`
- APIKey string `json:"apiKey"`
- BaseURL string `json:"baseURL"`
- Name string `json:"name"`
+ ID string `json:"id"`
+ Enabled bool `json:"enabled"`
+ APIKey string `json:"apiKey"`
+ BaseURL string `json:"baseURL"`
+ Name string `json:"name"`
Timeout int `json:"timeout"`
Dimensions int `json:"dimensions"` // 输出向量维度,仅 text-embedding-3 及以上模型支持;0 表示用模型默认值(不传该参数)
Concurrency int `json:"concurrency"`
@@ -325,7 +333,7 @@ type MCPServer struct {
}
func defaultEmbedding() *Embedding {
- return &Embedding{Timeout: 30, Concurrency: 8, BatchSize: 10}
+ return &Embedding{Timeout: 30, Concurrency: DefaultEmbeddingConcurrency, BatchSize: DefaultEmbeddingBatchSize}
}
func defaultRerank() *Rerank {
@@ -846,11 +854,13 @@ func (ai *AI) Normalize() {
ai.Embedding.Dimensions = 0 // 负值非法,归零表示用模型默认维度
}
if ai.Embedding.Concurrency < 1 {
- ai.Embedding.Concurrency = 8
+ ai.Embedding.Concurrency = DefaultEmbeddingConcurrency
}
+ ai.Embedding.Concurrency = min(ai.Embedding.Concurrency, MaxEmbeddingConcurrency)
if ai.Embedding.BatchSize < 1 {
- ai.Embedding.BatchSize = 10
+ ai.Embedding.BatchSize = DefaultEmbeddingBatchSize
}
+ ai.Embedding.BatchSize = min(ai.Embedding.BatchSize, MaxEmbeddingBatchSize)
if !ast.IsNodeIDPattern(ai.Embedding.ID) {
ai.Embedding.ID = normalizedAIID("embedding")
}
diff --git a/kernel/conf/settings_ranges_test.go b/kernel/conf/settings_ranges_test.go
index 320c76765c..1615da3d42 100644
--- a/kernel/conf/settings_ranges_test.go
+++ b/kernel/conf/settings_ranges_test.go
@@ -99,3 +99,26 @@ func TestNormalizeSettingsAgentRanges(t *testing.T) {
})
}
}
+
+func TestNormalizeSettingsEmbeddingRanges(t *testing.T) {
+ constructor := func() *Embedding { return NewAI().Embedding }
+ normalize := func(embedding *Embedding) *Embedding {
+ ai := NewAI()
+ ai.Embedding = embedding
+ return NormalizeAI(ai).Embedding
+ }
+ checkNormalizedIntValues(t, constructor, normalize, "Concurrency",
+ [][2]int{{-1, 8}, {0, 8}, {1, 1}, {8, 8}, {64, 64}, {65, 64}, {math.MaxInt, 64}})
+ checkNormalizedIntValues(t, constructor, normalize, "BatchSize",
+ [][2]int{{-1, 10}, {0, 10}, {1, 1}, {10, 10}, {128, 128}, {129, 128}, {math.MaxInt, 128}})
+ for _, body := range []string{`{}`, `{"embedding":null}`, `{"embedding":{"enabled":true,"apiKey":"existing"}}`} {
+ var ai AI
+ if err := json.Unmarshal([]byte(body), &ai); err != nil {
+ t.Fatal(err)
+ }
+ embedding := NormalizeAI(&ai).Embedding
+ if embedding.Concurrency != 8 || embedding.BatchSize != 10 {
+ t.Fatalf("legacy configuration %s has unexpected defaults: %+v", body, embedding)
+ }
+ }
+}
diff --git a/kernel/model/embedding.go b/kernel/model/embedding.go
index 7b86de3bf7..b110b0149a 100644
--- a/kernel/model/embedding.go
+++ b/kernel/model/embedding.go
@@ -34,17 +34,16 @@ import (
ignore "github.com/sabhiram/go-gitignore"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
+ "github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
)
const (
- defaultEmbeddingBatchSize = 10
- defaultEmbeddingConcurrency = 8
- embeddingMinTextLen = 7
- embeddingMaxContentLen = 12000
- embeddingVectorDim = 4 // float32 = 4 bytes
+ embeddingMinTextLen = 7
+ embeddingMaxContentLen = 12000
+ embeddingVectorDim = 4 // float32 = 4 bytes
// 嵌入失败重试的退避参数:API 不可用时避免连接风暴
// delay = min(embeddingBackoffBase << (failCount-1), embeddingBackoffMax)
@@ -162,6 +161,8 @@ func processPendingEmbeddingRows(query func(string, ...any) ([]map[string]any, e
concurrency := embeddingConcurrency()
batchSize := embeddingBatchSize()
+ // 每页容纳所有 worker 的完整批次,同时保留小批量配置下的查询下限。
+ pageSize := max(concurrency, (100+batchSize-1)/batchSize) * batchSize
workCh := make(chan embeddingJob, concurrency*2)
var jobsWg sync.WaitGroup
@@ -193,7 +194,7 @@ func processPendingEmbeddingRows(query func(string, ...any) ([]map[string]any, e
cutoff := now - int64(embeddingBackoffBase) // embeddingBackoffBase 单位为秒
embeddingResultMu.Lock()
generation := embeddingGeneration
- results, err := query(stmtPendingBlocks, embeddingMaxFailCount, cutoff)
+ results, err := query(stmtPendingBlocks, embeddingMaxFailCount, cutoff, pageSize)
embeddingResultMu.Unlock()
if err != nil {
logging.LogErrorf("query pending embedding blocks failed: %s", err)
@@ -272,14 +273,14 @@ func processPendingEmbeddingRows(query func(string, ...any) ([]map[string]any, e
// 1. 从未尝试过(e.id IS NULL);
// 2. 失败过、未达永久失败阈值、且距上次尝试已超过退避间隔(e.last_tried < ?)。
//
-// 参数顺序:?1=maxFailCount,?2=now-backoff(仅作 fail_count>0 块的粗筛下界,精确退避由 Go 侧按每块 fail_count 计算)。
+// 参数顺序:?1=maxFailCount,?2=now-backoff(失败块退避的粗筛下界),?3=每页块数。
const stmtPendingBlocks = "SELECT b.id, b.root_id, b.box, b.path, b.content, b.updated, " +
"COALESCE(e.fail_count, 0) AS fail_count, COALESCE(e.last_tried, 0) AS last_tried " +
"FROM blocks b " +
"LEFT JOIN block_embeddings e ON b.id = e.id " +
"WHERE e.id IS NULL " +
"OR (e.fail_count > 0 AND e.fail_count < ? AND e.last_tried < ?) " +
- "ORDER BY fail_count ASC, b.updated DESC LIMIT 100"
+ "ORDER BY fail_count ASC, b.updated DESC LIMIT ?"
// embeddingBackoffFor 返回给定失败次数对应的退避间隔(首次失败 fail_count=1 对应 base)。
func embeddingBackoffFor(failCount int) time.Duration {
@@ -881,16 +882,16 @@ func embeddingConcurrency() int {
confAISnapshot := Conf.LoadAI()
if nil != confAISnapshot.Embedding && confAISnapshot.Embedding.Enabled && 0 < confAISnapshot.Embedding.Concurrency {
- return confAISnapshot.Embedding.Concurrency
+ return min(confAISnapshot.Embedding.Concurrency, conf.MaxEmbeddingConcurrency)
}
- return defaultEmbeddingConcurrency
+ return conf.DefaultEmbeddingConcurrency
}
func embeddingBatchSize() int {
confAISnapshot := Conf.LoadAI()
if nil != confAISnapshot.Embedding && confAISnapshot.Embedding.Enabled && 0 < confAISnapshot.Embedding.BatchSize {
- return confAISnapshot.Embedding.BatchSize
+ return min(confAISnapshot.Embedding.BatchSize, conf.MaxEmbeddingBatchSize)
}
- return defaultEmbeddingBatchSize
+ return conf.DefaultEmbeddingBatchSize
}
diff --git a/kernel/model/embedding_queue_test.go b/kernel/model/embedding_queue_test.go
index 51f4435de9..a8d72bb7c2 100644
--- a/kernel/model/embedding_queue_test.go
+++ b/kernel/model/embedding_queue_test.go
@@ -2,6 +2,7 @@ package model
import (
"fmt"
+ "math"
"net/http"
"net/http/httptest"
"sync"
@@ -90,7 +91,7 @@ func TestEmbeddingQueueStopsUnsentJobs(t *testing.T) {
embeddingStop.Store(true)
calls.Add(1)
})
- if queries.Load() != 1 || calls.Load() < 1 || calls.Load() > defaultEmbeddingConcurrency {
+ if queries.Load() != 1 || calls.Load() < 1 || calls.Load() > conf.DefaultEmbeddingConcurrency {
t.Fatalf("queries=%d, calls=%d", queries.Load(), calls.Load())
}
if embeddingErrNotified.Load() {
@@ -157,3 +158,94 @@ func TestEmbeddingRebuildDiscardsInflightFailure(t *testing.T) {
t.Fatal("a request from before rebuilding must not stop the new index")
}
}
+
+func TestEmbeddingQueueConfiguredBatches(t *testing.T) {
+ for _, entry := range []struct {
+ concurrency, batchSize, total int
+ }{
+ {8, 10, 205},
+ {1, 64, 200},
+ {2, 3, 205},
+ {1, 128, 200},
+ {64, 10, 640},
+ {64, 128, 8192},
+ } {
+ t.Run(fmt.Sprintf("%d/%d", entry.concurrency, entry.batchSize), func(t *testing.T) {
+ prepareEmbeddingQueueTest(t)
+ Conf.AI.Load().Embedding.Concurrency = entry.concurrency
+ Conf.AI.Load().Embedding.BatchSize = entry.batchSize
+ var written atomic.Int64
+ var batchesMu sync.Mutex
+ var sizes []int
+ started := make(chan struct{}, entry.concurrency)
+ release, done := make(chan struct{}), make(chan struct{})
+ query := func(_ string, args ...any) ([]map[string]any, error) {
+ limit := args[2].(int)
+ offset := int(written.Load())
+ rows := make([]map[string]any, min(limit, entry.total-offset))
+ for i := range rows {
+ rows[i] = map[string]any{"id": fmt.Sprint(offset + i), "content": "pending content"}
+ }
+ return rows, nil
+ }
+ go func() {
+ defer close(done)
+ processPendingEmbeddingRows(query, func(texts []string, _ []map[string]any, _ uint64) {
+ batchesMu.Lock()
+ sizes = append(sizes, len(texts))
+ batchesMu.Unlock()
+ select {
+ case started <- struct{}{}:
+ default:
+ }
+ <-release
+ written.Add(int64(len(texts)))
+ })
+ }()
+ for range entry.concurrency {
+ select {
+ case <-started:
+ case <-time.After(5 * time.Second):
+ close(release)
+ <-done
+ t.Fatal("configured workers could not start a full batch concurrently")
+ }
+ }
+ close(release)
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("queue did not finish")
+ }
+ wantBatches := (entry.total + entry.batchSize - 1) / entry.batchSize
+ if written.Load() != int64(entry.total) || len(sizes) != wantBatches {
+ t.Fatalf("written=%d, batch sizes=%v", written.Load(), sizes)
+ }
+ for i, size := range sizes {
+ want := min(entry.batchSize, entry.total-i*entry.batchSize)
+ if size != want {
+ t.Fatalf("batch %d: got %d blocks, want %d", i, size, want)
+ }
+ }
+ })
+ }
+}
+
+func TestEmbeddingQueueBoundsUnnormalizedConfiguration(t *testing.T) {
+ prepareEmbeddingQueueTest(t)
+ Conf.AI.Load().Embedding.Concurrency = math.MaxInt
+ Conf.AI.Load().Embedding.BatchSize = math.MaxInt
+ queried := false
+ processPendingEmbeddingRows(func(_ string, args ...any) ([]map[string]any, error) {
+ queried = true
+ if limit := args[2].(int); limit != 64*128 {
+ t.Errorf("unbounded page size: %d", limit)
+ }
+ return nil, nil
+ }, func([]string, []map[string]any, uint64) {
+ t.Error("empty queue must not submit a request")
+ })
+ if !queried {
+ t.Fatal("queue did not query pending blocks")
+ }
+}
diff --git a/kernel/model/embedding_store_test.go b/kernel/model/embedding_store_test.go
index b74652b735..18e78589a2 100644
--- a/kernel/model/embedding_store_test.go
+++ b/kernel/model/embedding_store_test.go
@@ -69,3 +69,35 @@ func TestEmbeddingResultWrites(t *testing.T) {
exec(stmtFailedEmbedding, []byte{}, "model", 5, "id", "1", "edited")
check(0, 0, 0)
}
+
+func TestEmbeddingPendingQueryPageSize(t *testing.T) {
+ db, err := sql.Open("sqlite3", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ for _, stmt := range []string{
+ "CREATE TABLE blocks (id INTEGER PRIMARY KEY, root_id TEXT, box TEXT, path TEXT, content TEXT, updated TEXT)",
+ "CREATE TABLE block_embeddings (id INTEGER PRIMARY KEY, fail_count INTEGER, last_tried INTEGER)",
+ "WITH RECURSIVE ids(id) AS (SELECT 1 UNION ALL SELECT id + 1 FROM ids WHERE id < 200) INSERT INTO blocks (id) SELECT id FROM ids",
+ } {
+ if _, err = db.Exec(stmt); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for _, limit := range []int{64, 128, 200, 8192} {
+ rows, queryErr := db.Query(stmtPendingBlocks, embeddingMaxFailCount, 0, limit)
+ if queryErr != nil {
+ t.Fatal(queryErr)
+ }
+ count := 0
+ for rows.Next() {
+ count++
+ }
+ err = rows.Err()
+ rows.Close()
+ if err != nil || count != min(limit, 200) {
+ t.Fatalf("limit=%d, rows=%d, error=%v", limit, count, err)
+ }
+ }
+}