An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:bug: Bound embedding scheduling and honor configured batches https://github.com/siyuan-note/siyuan/pull/20609
Daniel93ee4b0cef9f
diff --git a/app/src/types/api/index.d.ts b/app/src/types/api/index.d.tsindex f9285d1f93..07a4777465 100644--- a/app/src/types/api/index.d.ts+++ b/app/src/types/api/index.d.ts@@ -6667,6 +6667,7 @@ export interface APIPOSTRoutes {/*** 使用共享规范化规则;智能体确认超时负数回退 600 秒,0 保留无限等待语义。* 工具调用轮数和输出 token 上限的负数归零;温度超出 0–2 回退 1。+ * 嵌入并发数和批大小省略、null 或小于 1 时分别使用 8 和 10,超过上限时分别限制为 64 和 128。* ChatGPT 账户绑定变化时与主配置一起可恢复提交,失败时回滚且不发布新配置;未变化时不访问绑定存储。*/"/api/setting/setAI": {diff --git a/app/src/types/config.d.ts b/app/src/types/config.d.tsindex b239288eaa..250bb7652c 100644--- a/app/src/types/config.d.ts+++ b/app/src/types/config.d.ts@@ -233,7 +233,9 @@ declare namespace Config {name: string;timeout: number;dimensions: number;+ /** 嵌入索引并发请求数,默认 8,范围 1–64。 */concurrency: number;+ /** 每次嵌入索引请求的文本块数,默认 10,范围 1–128。 */batchSize: number;}diff --git a/kernel/api/contract_setting_ranges_test.go b/kernel/api/contract_setting_ranges_test.goindex 1dd1034a94..c43c94c1e1 100644--- a/kernel/api/contract_setting_ranges_test.go+++ b/kernel/api/contract_setting_ranges_test.go@@ -44,6 +44,12 @@ func TestAPIContractSettingValueRanges(t *testing.T) {{"flashcard", "setFlashcard", setFlashcard, `{"reviewMode":7}`, map[string]float64{"reviewMode": 0}},{"ai", "setAI", setAI, `{"agent":{"maxToolCallRounds":-7,"confirmTimeout":-9,"temperature":-3,"maxCompletionTokens":-2}}`,map[string]float64{"agent.maxToolCallRounds": 0, "agent.confirmTimeout": 600, "agent.temperature": 1, "agent.maxCompletionTokens": 0}},+ {"ai", "setAI", setAI, `{"embedding":{"concurrency":2147483647,"batchSize":2147483647}}`,+ map[string]float64{"embedding.concurrency": 64, "embedding.batchSize": 128}},+ {"ai", "setAI", setAI, `{"embedding":{"concurrency":-1,"batchSize":0}}`,+ map[string]float64{"embedding.concurrency": 8, "embedding.batchSize": 10}},+ {"ai", "setAI", setAI, `{"embedding":{"concurrency":null,"batchSize":null}}`,+ map[string]float64{"embedding.concurrency": 8, "embedding.batchSize": 10}},{"system", "", nil, `{"autoLaunch2":9,"lockScreenMode":3}`, map[string]float64{"autoLaunch2": 0, "lockScreenMode": 0}},{"repo", "", nil, `{"indexRetentionDays":100000}`, map[string]float64{"indexRetentionDays": 100000}},} {diff --git a/kernel/apicontract/contracts.go b/kernel/apicontract/contracts.goindex b19785ee4e..0799e0ecb5 100644--- a/kernel/apicontract/contracts.go+++ b/kernel/apicontract/contracts.go@@ -1095,6 +1095,7 @@ var SetBazaar = define[SetBazaarRequest, *SettingBazaar]("setBazaar", "/api/sett// SetAI 使用共享规范化规则;智能体确认超时负数回退 600 秒,0 保留无限等待语义。// 工具调用轮数和输出 token 上限的负数归零;温度超出 0–2 回退 1。+// 嵌入并发数和批大小省略、null 或小于 1 时分别使用 8 和 10,超过上限时分别限制为 64 和 128。// ChatGPT 账户绑定变化时与主配置一起可恢复提交,失败时回滚且不发布新配置;未变化时不访问绑定存储。var SetAI = define[SetAIRequest, *SettingAI]("setAI", "/api/setting/setAI", AuthenticatedAccess|AdminAccess|WritableAccess, JSONBody, ResponseOptions{Setting: SettingWrite{Namespace: "ai"}}, "POST")var SetSecrets = define[SetSecretsRequest, *SettingSecrets]("setSecrets", "/api/setting/setSecrets", AuthenticatedAccess|AdminAccess|WritableAccess, JSONBody, ResponseOptions{Setting: SettingWrite{Namespace: "secrets"}}, "POST")diff --git a/kernel/apicontract/setting.go b/kernel/apicontract/setting.goindex 466431751c..f7e0de02ed 100644--- a/kernel/apicontract/setting.go+++ b/kernel/apicontract/setting.go@@ -48,11 +48,11 @@ type SettingCapabilityPolicy struct {}type SettingEmbedding struct {- ID string `json:"id" api:"optional,nullable"`- Enabled bool `json:"enabled" api:"optional,nullable"`- APIKey string `json:"apiKey" api:"optional,nullable"`- BaseURL string `json:"baseURL" api:"optional,nullable"`- Name string `json:"name" api:"optional,nullable"`+ ID string `json:"id" api:"optional,nullable"`+ Enabled bool `json:"enabled" api:"optional,nullable"`+ APIKey string `json:"apiKey" api:"optional,nullable"`+ BaseURL string `json:"baseURL" api:"optional,nullable"`+ Name string `json:"name" api:"optional,nullable"`Timeout int `json:"timeout" api:"optional,nullable"`Dimensions int `json:"dimensions" api:"optional,nullable"`Concurrency int `json:"concurrency" api:"optional,nullable"`diff --git a/kernel/conf/ai.go b/kernel/conf/ai.goindex 7d5ae2e4a7..b417570f52 100644--- a/kernel/conf/ai.go+++ b/kernel/conf/ai.go@@ -110,12 +110,20 @@ type ImageGeneration struct {OutputFormat string `json:"outputFormat"`}+// 嵌入索引调度的默认值和资源上限,与设置控件范围保持一致。+const (+ DefaultEmbeddingConcurrency = 8+ MaxEmbeddingConcurrency = 64+ DefaultEmbeddingBatchSize = 10+ MaxEmbeddingBatchSize = 128+)+type Embedding struct {- ID string `json:"id"`- Enabled bool `json:"enabled"`- APIKey string `json:"apiKey"`- BaseURL string `json:"baseURL"`- Name string `json:"name"`+ ID string `json:"id"`+ Enabled bool `json:"enabled"`+ APIKey string `json:"apiKey"`+ BaseURL string `json:"baseURL"`+ Name string `json:"name"`Timeout int `json:"timeout"`Dimensions int `json:"dimensions"` // 输出向量维度,仅 text-embedding-3 及以上模型支持;0 表示用模型默认值(不传该参数)Concurrency int `json:"concurrency"`@@ -325,7 +333,7 @@ type MCPServer struct {}func defaultEmbedding() *Embedding {- return &Embedding{Timeout: 30, Concurrency: 8, BatchSize: 10}+ return &Embedding{Timeout: 30, Concurrency: DefaultEmbeddingConcurrency, BatchSize: DefaultEmbeddingBatchSize}}func defaultRerank() *Rerank {@@ -846,11 +854,13 @@ func (ai *AI) Normalize() {ai.Embedding.Dimensions = 0 // 负值非法,归零表示用模型默认维度}if ai.Embedding.Concurrency < 1 {- ai.Embedding.Concurrency = 8+ ai.Embedding.Concurrency = DefaultEmbeddingConcurrency}+ ai.Embedding.Concurrency = min(ai.Embedding.Concurrency, MaxEmbeddingConcurrency)if ai.Embedding.BatchSize < 1 {- ai.Embedding.BatchSize = 10+ ai.Embedding.BatchSize = DefaultEmbeddingBatchSize}+ ai.Embedding.BatchSize = min(ai.Embedding.BatchSize, MaxEmbeddingBatchSize)if !ast.IsNodeIDPattern(ai.Embedding.ID) {ai.Embedding.ID = normalizedAIID("embedding")}diff --git a/kernel/conf/settings_ranges_test.go b/kernel/conf/settings_ranges_test.goindex 320c76765c..1615da3d42 100644--- a/kernel/conf/settings_ranges_test.go+++ b/kernel/conf/settings_ranges_test.go@@ -99,3 +99,26 @@ func TestNormalizeSettingsAgentRanges(t *testing.T) {})}}++func TestNormalizeSettingsEmbeddingRanges(t *testing.T) {+ constructor := func() *Embedding { return NewAI().Embedding }+ normalize := func(embedding *Embedding) *Embedding {+ ai := NewAI()+ ai.Embedding = embedding+ return NormalizeAI(ai).Embedding+ }+ checkNormalizedIntValues(t, constructor, normalize, "Concurrency",+ [][2]int{{-1, 8}, {0, 8}, {1, 1}, {8, 8}, {64, 64}, {65, 64}, {math.MaxInt, 64}})+ checkNormalizedIntValues(t, constructor, normalize, "BatchSize",+ [][2]int{{-1, 10}, {0, 10}, {1, 1}, {10, 10}, {128, 128}, {129, 128}, {math.MaxInt, 128}})+ for _, body := range []string{`{}`, `{"embedding":null}`, `{"embedding":{"enabled":true,"apiKey":"existing"}}`} {+ var ai AI+ if err := json.Unmarshal([]byte(body), &ai); err != nil {+ t.Fatal(err)+ }+ embedding := NormalizeAI(&ai).Embedding+ if embedding.Concurrency != 8 || embedding.BatchSize != 10 {+ t.Fatalf("legacy configuration %s has unexpected defaults: %+v", body, embedding)+ }+ }+}diff --git a/kernel/model/embedding.go b/kernel/model/embedding.goindex 7b86de3bf7..b110b0149a 100644--- a/kernel/model/embedding.go+++ b/kernel/model/embedding.go@@ -34,17 +34,16 @@ import (ignore "github.com/sabhiram/go-gitignore""github.com/siyuan-note/eventbus""github.com/siyuan-note/logging"+ "github.com/siyuan-note/siyuan/kernel/conf""github.com/siyuan-note/siyuan/kernel/sql""github.com/siyuan-note/siyuan/kernel/task""github.com/siyuan-note/siyuan/kernel/util")const (- defaultEmbeddingBatchSize = 10- defaultEmbeddingConcurrency = 8- embeddingMinTextLen = 7- embeddingMaxContentLen = 12000- embeddingVectorDim = 4 // float32 = 4 bytes+ embeddingMinTextLen = 7+ embeddingMaxContentLen = 12000+ embeddingVectorDim = 4 // float32 = 4 bytes// 嵌入失败重试的退避参数:API 不可用时避免连接风暴// delay = min(embeddingBackoffBase << (failCount-1), embeddingBackoffMax)@@ -162,6 +161,8 @@ func processPendingEmbeddingRows(query func(string, ...any) ([]map[string]any, econcurrency := embeddingConcurrency()batchSize := embeddingBatchSize()+ // 每页容纳所有 worker 的完整批次,同时保留小批量配置下的查询下限。+ pageSize := max(concurrency, (100+batchSize-1)/batchSize) * batchSizeworkCh := make(chan embeddingJob, concurrency*2)var jobsWg sync.WaitGroup@@ -193,7 +194,7 @@ func processPendingEmbeddingRows(query func(string, ...any) ([]map[string]any, ecutoff := now - int64(embeddingBackoffBase) // embeddingBackoffBase 单位为秒embeddingResultMu.Lock()generation := embeddingGeneration- results, err := query(stmtPendingBlocks, embeddingMaxFailCount, cutoff)+ results, err := query(stmtPendingBlocks, embeddingMaxFailCount, cutoff, pageSize)embeddingResultMu.Unlock()if err != nil {logging.LogErrorf("query pending embedding blocks failed: %s", err)@@ -272,14 +273,14 @@ func processPendingEmbeddingRows(query func(string, ...any) ([]map[string]any, e// 1. 从未尝试过(e.id IS NULL);// 2. 失败过、未达永久失败阈值、且距上次尝试已超过退避间隔(e.last_tried < ?)。//-// 参数顺序:?1=maxFailCount,?2=now-backoff(仅作 fail_count>0 块的粗筛下界,精确退避由 Go 侧按每块 fail_count 计算)。+// 参数顺序:?1=maxFailCount,?2=now-backoff(失败块退避的粗筛下界),?3=每页块数。const stmtPendingBlocks = "SELECT b.id, b.root_id, b.box, b.path, b.content, b.updated, " +"COALESCE(e.fail_count, 0) AS fail_count, COALESCE(e.last_tried, 0) AS last_tried " +"FROM blocks b " +"LEFT JOIN block_embeddings e ON b.id = e.id " +"WHERE e.id IS NULL " +"OR (e.fail_count > 0 AND e.fail_count < ? AND e.last_tried < ?) " +- "ORDER BY fail_count ASC, b.updated DESC LIMIT 100"+ "ORDER BY fail_count ASC, b.updated DESC LIMIT ?"// embeddingBackoffFor 返回给定失败次数对应的退避间隔(首次失败 fail_count=1 对应 base)。func embeddingBackoffFor(failCount int) time.Duration {@@ -881,16 +882,16 @@ func embeddingConcurrency() int {confAISnapshot := Conf.LoadAI()if nil != confAISnapshot.Embedding && confAISnapshot.Embedding.Enabled && 0 < confAISnapshot.Embedding.Concurrency {- return confAISnapshot.Embedding.Concurrency+ return min(confAISnapshot.Embedding.Concurrency, conf.MaxEmbeddingConcurrency)}- return defaultEmbeddingConcurrency+ return conf.DefaultEmbeddingConcurrency}func embeddingBatchSize() int {confAISnapshot := Conf.LoadAI()if nil != confAISnapshot.Embedding && confAISnapshot.Embedding.Enabled && 0 < confAISnapshot.Embedding.BatchSize {- return confAISnapshot.Embedding.BatchSize+ return min(confAISnapshot.Embedding.BatchSize, conf.MaxEmbeddingBatchSize)}- return defaultEmbeddingBatchSize+ return conf.DefaultEmbeddingBatchSize}diff --git a/kernel/model/embedding_queue_test.go b/kernel/model/embedding_queue_test.goindex 51f4435de9..a8d72bb7c2 100644--- a/kernel/model/embedding_queue_test.go+++ b/kernel/model/embedding_queue_test.go@@ -2,6 +2,7 @@ package modelimport ("fmt"+ "math""net/http""net/http/httptest""sync"@@ -90,7 +91,7 @@ func TestEmbeddingQueueStopsUnsentJobs(t *testing.T) {embeddingStop.Store(true)calls.Add(1)})- if queries.Load() != 1 || calls.Load() < 1 || calls.Load() > defaultEmbeddingConcurrency {+ if queries.Load() != 1 || calls.Load() < 1 || calls.Load() > conf.DefaultEmbeddingConcurrency {t.Fatalf("queries=%d, calls=%d", queries.Load(), calls.Load())}if embeddingErrNotified.Load() {@@ -157,3 +158,94 @@ func TestEmbeddingRebuildDiscardsInflightFailure(t *testing.T) {t.Fatal("a request from before rebuilding must not stop the new index")}}++func TestEmbeddingQueueConfiguredBatches(t *testing.T) {+ for _, entry := range []struct {+ concurrency, batchSize, total int+ }{+ {8, 10, 205},+ {1, 64, 200},+ {2, 3, 205},+ {1, 128, 200},+ {64, 10, 640},+ {64, 128, 8192},+ } {+ t.Run(fmt.Sprintf("%d/%d", entry.concurrency, entry.batchSize), func(t *testing.T) {+ prepareEmbeddingQueueTest(t)+ Conf.AI.Load().Embedding.Concurrency = entry.concurrency+ Conf.AI.Load().Embedding.BatchSize = entry.batchSize+ var written atomic.Int64+ var batchesMu sync.Mutex+ var sizes []int+ started := make(chan struct{}, entry.concurrency)+ release, done := make(chan struct{}), make(chan struct{})+ query := func(_ string, args ...any) ([]map[string]any, error) {+ limit := args[2].(int)+ offset := int(written.Load())+ rows := make([]map[string]any, min(limit, entry.total-offset))+ for i := range rows {+ rows[i] = map[string]any{"id": fmt.Sprint(offset + i), "content": "pending content"}+ }+ return rows, nil+ }+ go func() {+ defer close(done)+ processPendingEmbeddingRows(query, func(texts []string, _ []map[string]any, _ uint64) {+ batchesMu.Lock()+ sizes = append(sizes, len(texts))+ batchesMu.Unlock()+ select {+ case started <- struct{}{}:+ default:+ }+ <-release+ written.Add(int64(len(texts)))+ })+ }()+ for range entry.concurrency {+ select {+ case <-started:+ case <-time.After(5 * time.Second):+ close(release)+ <-done+ t.Fatal("configured workers could not start a full batch concurrently")+ }+ }+ close(release)+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("queue did not finish")+ }+ wantBatches := (entry.total + entry.batchSize - 1) / entry.batchSize+ if written.Load() != int64(entry.total) || len(sizes) != wantBatches {+ t.Fatalf("written=%d, batch sizes=%v", written.Load(), sizes)+ }+ for i, size := range sizes {+ want := min(entry.batchSize, entry.total-i*entry.batchSize)+ if size != want {+ t.Fatalf("batch %d: got %d blocks, want %d", i, size, want)+ }+ }+ })+ }+}++func TestEmbeddingQueueBoundsUnnormalizedConfiguration(t *testing.T) {+ prepareEmbeddingQueueTest(t)+ Conf.AI.Load().Embedding.Concurrency = math.MaxInt+ Conf.AI.Load().Embedding.BatchSize = math.MaxInt+ queried := false+ processPendingEmbeddingRows(func(_ string, args ...any) ([]map[string]any, error) {+ queried = true+ if limit := args[2].(int); limit != 64*128 {+ t.Errorf("unbounded page size: %d", limit)+ }+ return nil, nil+ }, func([]string, []map[string]any, uint64) {+ t.Error("empty queue must not submit a request")+ })+ if !queried {+ t.Fatal("queue did not query pending blocks")+ }+}diff --git a/kernel/model/embedding_store_test.go b/kernel/model/embedding_store_test.goindex b74652b735..18e78589a2 100644--- a/kernel/model/embedding_store_test.go+++ b/kernel/model/embedding_store_test.go@@ -69,3 +69,35 @@ func TestEmbeddingResultWrites(t *testing.T) {exec(stmtFailedEmbedding, []byte{}, "model", 5, "id", "1", "edited")check(0, 0, 0)}++func TestEmbeddingPendingQueryPageSize(t *testing.T) {+ db, err := sql.Open("sqlite3", ":memory:")+ if err != nil {+ t.Fatal(err)+ }+ defer db.Close()+ for _, stmt := range []string{+ "CREATE TABLE blocks (id INTEGER PRIMARY KEY, root_id TEXT, box TEXT, path TEXT, content TEXT, updated TEXT)",+ "CREATE TABLE block_embeddings (id INTEGER PRIMARY KEY, fail_count INTEGER, last_tried INTEGER)",+ "WITH RECURSIVE ids(id) AS (SELECT 1 UNION ALL SELECT id + 1 FROM ids WHERE id < 200) INSERT INTO blocks (id) SELECT id FROM ids",+ } {+ if _, err = db.Exec(stmt); err != nil {+ t.Fatal(err)+ }+ }+ for _, limit := range []int{64, 128, 200, 8192} {+ rows, queryErr := db.Query(stmtPendingBlocks, embeddingMaxFailCount, 0, limit)+ if queryErr != nil {+ t.Fatal(queryErr)+ }+ count := 0+ for rows.Next() {+ count+++ }+ err = rows.Err()+ rows.Close()+ if err != nil || count != min(limit, 200) {+ t.Fatalf("limit=%d, rows=%d, error=%v", limit, count, err)+ }+ }+}