Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(embedding): read the pool's first-error guard under its own mutex (#4014)
ordinary being6dd0a6a3d628
diff --git a/internal/models/embedding/batch.go b/internal/models/embedding/batch.goindex 2959e8a23..e4017b551 100644--- a/internal/models/embedding/batch.go+++ b/internal/models/embedding/batch.go@@ -45,8 +45,16 @@ func (e *batchEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder,processChunk := func(texts []*textEmbedding) func() {return func() {defer wg.Done()- // If an error has already occurred, don't continue processing- if firstErr != nil {+ // If an error has already occurred, don't continue processing. The+ // read takes mu for the same reason the writes below do: every pool+ // worker evaluates this guard, so an unsynchronised read races with+ // another worker's write. Beyond the detector, a worker that misses+ // the write bills another provider request after the run has already+ // failed, which is exactly what this guard exists to prevent.+ mu.Lock()+ aborted := firstErr != nil+ mu.Unlock()+ if aborted {return}// Embed textdiff --git a/internal/models/embedding/batch_race_test.go b/internal/models/embedding/batch_race_test.gonew file mode 100644index 000000000..d0dbd47d2--- /dev/null+++ b/internal/models/embedding/batch_race_test.go@@ -0,0 +1,145 @@+package embedding++import (+ "context"+ "errors"+ "fmt"+ "sync/atomic"+ "testing"++ "github.com/panjf2000/ants/v2"+)++// flakyEmbedder fails the first sub-batch it is given and succeeds afterwards,+// so the pool has a failure to record while other sub-batches are still queued.+type flakyEmbedder struct {+ calls atomic.Int32+}++func (f *flakyEmbedder) Embed(context.Context, string) ([]float32, error) {+ return nil, errProviderDown+}++func (f *flakyEmbedder) BatchEmbed(_ context.Context, texts []string) ([][]float32, error) {+ if f.calls.Add(1) == 1 {+ return nil, errProviderDown+ }+ out := make([][]float32, len(texts))+ for i := range out {+ out[i] = []float32{1}+ }+ return out, nil+}++func (f *flakyEmbedder) GetModelName() string { return "flaky" }+func (f *flakyEmbedder) GetDimensions() int { return 1 }+func (f *flakyEmbedder) GetModelID() string { return "flaky" }++// BatchEmbedWithPool satisfies EmbedderPooler; the pool under test is the one+// passed explicitly to it, so this is never the path the tests exercise.+func (f *flakyEmbedder) BatchEmbedWithPool(+ ctx context.Context, model Embedder, texts []string,+) ([][]float32, error) {+ return model.BatchEmbed(ctx, texts)+}++var errProviderDown = errors.New("provider down")++// BatchEmbedWithPool fans a batch out over the ants pool, so with+// BATCH_EMBED_SIZE=1 every text becomes its own sub-batch and several workers+// run at once. When one of them fails, the remaining queued sub-batches read+// firstErr to decide whether to skip — a decision every worker makes+// concurrently with other workers' writes to that same variable.+//+// That read is only observable under the race detector: whether a particular+// worker happens to observe the write depends on scheduling, so no assertion on+// call counts can pin it without being flaky. Run this with -race:+//+// go test -race ./internal/models/embedding/ -run TestBatchEmbedWithPoolErrorPath+//+// Before the read was taken under mu, the detector reported a race between+// batch.go's guard and the write in the failure branch.+func TestBatchEmbedWithPoolErrorPath(t *testing.T) {+ t.Setenv("BATCH_EMBED_SIZE", "1")++ pool, err := ants.NewPool(8)+ if err != nil {+ t.Fatalf("new pool: %v", err)+ }+ t.Cleanup(pool.Release)+ pooler := NewBatchEmbedder(pool)++ // Repeat so the workers overlap in many different interleavings rather than+ // relying on one lucky schedule.+ for round := range 20 {+ texts := make([]string, 64)+ for i := range texts {+ texts[i] = fmt.Sprintf("round-%d-chunk-%d", round, i)+ }++ model := &flakyEmbedder{}+ got, err := pooler.BatchEmbedWithPool(context.Background(), model, texts)+ if !errors.Is(err, errProviderDown) {+ t.Fatalf("round %d: err = %v, want %v", round, err, errProviderDown)+ }+ if got != nil {+ t.Fatalf("round %d: results = %v, want none alongside an error", round, got)+ }+ }+}++// A provider that returns fewer vectors than inputs is the second failure mode+// the pool records. It writes firstErr from the same branch, so it guards+// against the same regression.+func TestBatchEmbedWithPoolLengthMismatchPath(t *testing.T) {+ t.Setenv("BATCH_EMBED_SIZE", "1")++ pool, err := ants.NewPool(8)+ if err != nil {+ t.Fatalf("new pool: %v", err)+ }+ t.Cleanup(pool.Release)+ pooler := NewBatchEmbedder(pool)++ for round := range 20 {+ texts := make([]string, 64)+ for i := range texts {+ texts[i] = fmt.Sprintf("round-%d-chunk-%d", round, i)+ }++ model := &shortEmbedder{}+ got, err := pooler.BatchEmbedWithPool(context.Background(), model, texts)+ if err == nil {+ t.Fatalf("round %d: err = nil, want a length-mismatch error", round)+ }+ if got != nil {+ t.Fatalf("round %d: results = %v, want none alongside an error", round, got)+ }+ }+}++// shortEmbedder always answers with one vector fewer than it was given.+type shortEmbedder struct{}++func (shortEmbedder) Embed(context.Context, string) ([]float32, error) {+ return nil, fmt.Errorf("unexpected Embed call")+}++func (shortEmbedder) BatchEmbed(_ context.Context, texts []string) ([][]float32, error) {+ out := make([][]float32, len(texts))+ for i := range out {+ out[i] = []float32{1}+ }+ return out[:len(out)-1], nil+}++func (shortEmbedder) GetModelName() string { return "short" }+func (shortEmbedder) GetDimensions() int { return 1 }+func (shortEmbedder) GetModelID() string { return "short" }++// BatchEmbedWithPool satisfies EmbedderPooler; see flakyEmbedder.+func (shortEmbedder) BatchEmbedWithPool(+ ctx context.Context, model Embedder, texts []string,+) ([][]float32, error) {+ return model.BatchEmbed(ctx, texts)+}