RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(test): remove two races that reddened the ingestion packages (#21716)
CCLUCKYf80c4c13693e
diff --git a/internal/ingestion/component/vision_enhancement_cgo_test.go b/internal/ingestion/component/vision_enhancement_cgo_test.goindex 8e9e729eba..2a3353dcb0 100644--- a/internal/ingestion/component/vision_enhancement_cgo_test.go+++ b/internal/ingestion/component/vision_enhancement_cgo_test.go@@ -5,6 +5,8 @@ package componentimport ("context""image"+ "sync"+ "sync/atomic""testing""time"@@ -57,6 +59,41 @@ func cgoPositions() map[string]any {}}+// expiringContext expires on demand instead of on a wall clock, so a test can+// expire an item budget at a chosen point in the sequence. Done closes on that+// expiry or when the parent is done, keeping Done and Err consistent.+type expiringContext struct {+ context.Context+ done chan struct{}+ once sync.Once+ expired atomic.Bool+}++func newExpiringContext(parent context.Context) (*expiringContext, func()) {+ ctx := &expiringContext{Context: parent, done: make(chan struct{})}+ stopParent := context.AfterFunc(parent, func() { ctx.close() })+ expire := func() {+ ctx.expired.Store(true)+ ctx.close()+ stopParent()+ }+ return ctx, expire+}++func (c *expiringContext) close() { c.once.Do(func() { close(c.done) }) }++func (c *expiringContext) Done() <-chan struct{} { return c.done }++func (c *expiringContext) Err() error {+ if err := c.Context.Err(); err != nil {+ return err+ }+ if c.expired.Load() {+ return context.DeadlineExceeded+ }+ return nil+}+// TestVisionCropImage_OnDemandFromPositions verifies the core fix: a parsed// item that carries PDF positions but no inlined image is cropped on demand// by re-acquiring the source PDF from storage and rendering the region.@@ -343,7 +380,10 @@ func TestVisionCropImage_ReusesSourceAfterItemContextExpires(t *testing.T) {visionEngineOpener = oldOpener})- fetchStarted := make(chan struct{})+ // Buffered: the fetcher signals with a non-blocking send, so an unbuffered+ // channel drops the signal whenever this goroutine is not parked on the+ // receive yet, and the test then blocks there forever.+ fetchStarted := make(chan struct{}, 1)finishFetch := make(chan struct{})fetchCalls := 0visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {@@ -371,8 +411,11 @@ func TestVisionCropImage_ReusesSourceAfterItemContextExpires(t *testing.T) {}t.Cleanup(func() { _ = cropper.Close() })- firstItemCtx, cancelFirstItem := context.WithTimeout(invocationCtx, 20*time.Millisecond)- defer cancelFirstItem()+ // Expire the item context exactly when the shared fetch is in flight. A+ // wall-clock deadline races Crop's entry check on a loaded runner: when the+ // 20ms elapsed first, Crop returned before fetching and the test waited for+ // a fetch that never started.+ firstItemCtx, expireFirstItem := newExpiringContext(invocationCtx)firstResult := make(chan struct {img *visionImageerr error@@ -384,8 +427,14 @@ func TestVisionCropImage_ReusesSourceAfterItemContextExpires(t *testing.T) {err error}{img: img, err: err}}()- <-fetchStarted- <-firstItemCtx.Done()+ // Bound the wait: an unbounded receive turns a missing fetch into a+ // package-wide 10m timeout instead of a local failure.+ select {+ case <-fetchStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("source fetch did not start")+ }+ expireFirstItem()close(finishFetch)first := <-firstResultif first.err != nil {diff --git a/internal/ingestion/service/worker_dispatcher_integration_test.go b/internal/ingestion/service/worker_dispatcher_integration_test.goindex 010d973331..169f9eaaa9 100644--- a/internal/ingestion/service/worker_dispatcher_integration_test.go+++ b/internal/ingestion/service/worker_dispatcher_integration_test.go@@ -22,6 +22,7 @@ import ("context""encoding/json""net"+ "os""sync""sync/atomic""testing"@@ -38,10 +39,20 @@ import (func setupRealNatsCluster(t *testing.T) (host string, port int) {t.Helper()+ // JetStream can still write consumer state while the store is being removed,+ // which surfaced as "directory not empty" from t.TempDir's cleanup and+ // failed the test after it had passed. Own the directory and retry the+ // removal instead.+ storeDir, err := os.MkdirTemp("", "ragflow-nats-store-")+ if err != nil {+ t.Fatalf("create JetStream store dir: %v", err)+ }+ // Registered before the server cleanup so it runs after it.+ t.Cleanup(func() { removeStoreDir(t, storeDir) })opts := &server.Options{Port: -1,JetStream: true,- StoreDir: t.TempDir(),+ StoreDir: storeDir,NoLog: true,NoSigs: true,}@@ -52,6 +63,7 @@ func setupRealNatsCluster(t *testing.T) (host string, port int) {ns.Start()if !ns.ReadyForConnections(10 * time.Second) {ns.Shutdown()+ ns.WaitForShutdown()t.Fatal("embedded NATS server did not become ready within 10s")}t.Cleanup(func() {@@ -62,6 +74,18 @@ func setupRealNatsCluster(t *testing.T) (host string, port int) {return "127.0.0.1", addr.Port}+func removeStoreDir(t *testing.T, dir string) {+ t.Helper()+ var err error+ for attempt := 0; attempt < 20; attempt++ {+ if err = os.RemoveAll(dir); err == nil {+ return+ }+ time.Sleep(50 * time.Millisecond)+ }+ t.Errorf("remove JetStream store dir %s: %v", dir, err)+}+// TestIntegration_MultiInstanceSharedConsumerNoOverExecution verifies TaskRP.md §6.2:// Multiple ingestor instances sharing the same JetStream durable consumer (RAGFLOW_CONSUMER)// compete for tasks without duplicate or overlapping execution while the broker is healthy.