Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix(mcp): serialize concurrent engine cache publication (#1215)

Ilia Alshanetsky2f9bfe831725
diff --git a/changelog.d/+cr-010-mcp-cache-race.fixed.md b/changelog.d/+cr-010-mcp-cache-race.fixed.md
new file mode 100644
index 00000000..d5b8c992
--- /dev/null
+++ b/changelog.d/+cr-010-mcp-cache-race.fixed.md
@@ -0,0 +1 @@
+Prevent concurrent MCP server startups from replacing a complete engine cache with a partial extraction. Each process now stages its own files, bounds publication-lock waits to five seconds, and safely reclaims staging files left by terminated extractors.
diff --git a/mcp/internal/engine/cache_lock.go b/mcp/internal/engine/cache_lock.go
new file mode 100644
index 00000000..7ebbcbc4
--- /dev/null
+++ b/mcp/internal/engine/cache_lock.go
@@ -0,0 +1,49 @@
+package engine
+
+import (
+ "fmt"
+ "os"
+ "time"
+)
+
+const cacheLockTimeout = 5 * time.Second
+
+func openPublicationLock(cacheDir string, wait bool) (*os.File, error) {
+ file, err := os.OpenFile(cacheDir+".lock", os.O_CREATE|os.O_RDWR, 0o600)
+ if err != nil {
+ return nil, err
+ }
+ if wait {
+ err = lockCacheFile(file)
+ } else {
+ var locked bool
+ locked, err = tryLockCacheFile(file)
+ if !locked && err == nil {
+ _ = file.Close()
+ return nil, nil
+ }
+ }
+ if err != nil {
+ _ = file.Close()
+ return nil, err
+ }
+ return file, nil
+}
+
+func lockCacheFile(file *os.File) error {
+ deadline := time.Now().Add(cacheLockTimeout)
+ for {
+ locked, err := tryLockCacheFile(file)
+ if err != nil {
+ return err
+ }
+ if locked {
+ return nil
+ }
+ remaining := time.Until(deadline)
+ if remaining <= 0 {
+ return fmt.Errorf("cache lock unavailable after %s: %w", cacheLockTimeout, os.ErrDeadlineExceeded)
+ }
+ time.Sleep(min(25*time.Millisecond, remaining))
+ }
+}
diff --git a/mcp/internal/engine/cache_lock_other.go b/mcp/internal/engine/cache_lock_other.go
new file mode 100644
index 00000000..0a4a3a93
--- /dev/null
+++ b/mcp/internal/engine/cache_lock_other.go
@@ -0,0 +1,13 @@
+//go:build !darwin && !dragonfly && !freebsd && !illumos && !linux && !netbsd && !openbsd && !windows
+
+package engine
+
+import (
+ "fmt"
+ "os"
+ "runtime"
+)
+
+func tryLockCacheFile(file *os.File) (bool, error) {
+ return false, fmt.Errorf("cache locking is unsupported on %s", runtime.GOOS)
+}
diff --git a/mcp/internal/engine/cache_lock_test.go b/mcp/internal/engine/cache_lock_test.go
new file mode 100644
index 00000000..9e78994c
--- /dev/null
+++ b/mcp/internal/engine/cache_lock_test.go
@@ -0,0 +1,310 @@
+package engine
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "testing/fstest"
+ "time"
+)
+
+func TestCacheLockOwnerProcess(t *testing.T) {
+ path := os.Getenv("LAST30DAYS_TEST_CACHE_LOCK")
+ if path == "" {
+ return
+ }
+ file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer file.Close()
+ if err := lockCacheFile(file); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path+".ready", nil, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ time.Sleep(time.Minute)
+}
+
+func startCacheLockOwner(t *testing.T, path string) func() {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return startCacheTestProcess(t, "TestCacheLockOwnerProcess", []string{"LAST30DAYS_TEST_CACHE_LOCK=" + path}, path+".ready")
+}
+
+func startCacheTestProcess(t *testing.T, test string, env []string, ready string) func() {
+ t.Helper()
+ executable, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ t.Cleanup(cancel)
+ cmd := exec.CommandContext(ctx, executable, "-test.run=^"+test+"$")
+ cmd.Env = append(os.Environ(), env...)
+ var output bytes.Buffer
+ cmd.Stdout = &output
+ cmd.Stderr = &output
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
+ }
+ var stopped sync.Once
+ stop := func() {
+ stopped.Do(func() {
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ })
+ }
+ t.Cleanup(stop)
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ if _, err := os.Stat(ready); err == nil {
+ return stop
+ }
+ if time.Now().After(deadline) {
+ stop()
+ t.Fatalf("child process did not become ready: %s", &output)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+}
+
+func cacheStagePaths(t *testing.T, base string) []string {
+ t.Helper()
+ paths, err := filepath.Glob(filepath.Join(base, cacheSubdir, "v1.tmp-*"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var stages []string
+ for _, path := range paths {
+ if !strings.HasSuffix(path, ".lock") {
+ stages = append(stages, path)
+ }
+ }
+ return stages
+}
+
+func TestEnsureReclaimsStagesAfterProcessDeath(t *testing.T) {
+ base := t.TempDir()
+ t.Setenv(CacheEnvOverride, base)
+ stopOwner := startCacheTestProcess(t, "TestEnsureConcurrentProcesses",
+ []string{"LAST30DAYS_TEST_EXTRACT_ROLE=paused"}, filepath.Join(base, "ready"))
+ stages := cacheStagePaths(t, base)
+ if len(stages) != 1 {
+ t.Fatalf("paused extractor stages = %v, want one", stages)
+ }
+ stopOwner()
+ cacheDir, err := EnsureUserCache(newTestFS(), "v1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
+ if _, err := os.Stat(stages[0]); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("dead extractor's stage was not reclaimed: %v", err)
+ }
+ if _, err := os.Stat(stages[0] + ".lock"); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("dead extractor's ownership file was not reclaimed: %v", err)
+ }
+}
+
+func TestEnsurePreservesActiveStagesAndReclaimsThemAfterOwnerDeath(t *testing.T) {
+ base := t.TempDir()
+ t.Setenv(CacheEnvOverride, base)
+ stopOwner := startCacheTestProcess(t, "TestEnsureConcurrentProcesses",
+ []string{"LAST30DAYS_TEST_EXTRACT_ROLE=paused"}, filepath.Join(base, "ready"))
+ stages := cacheStagePaths(t, base)
+ if len(stages) != 1 {
+ t.Fatalf("paused extractor stages = %v, want one", stages)
+ }
+ cacheDir, err := EnsureUserCache(newTestFS(), "v1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ mustReadFile(t, filepath.Join(stages[0], "alpha.txt"), "alpha")
+ stopOwner()
+ resetOnce(cacheDir)
+ if _, err := EnsureUserCache(newTestFS(), "v1"); err != nil {
+ t.Fatalf("reuse completed cache: %v", err)
+ }
+ if _, err := os.Stat(stages[0]); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("cached reuse did not reclaim dead extractor's stage: %v", err)
+ }
+ mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
+}
+
+func TestEnsurePreservesUnmarkedStagingDirectories(t *testing.T) {
+ base := t.TempDir()
+ stage := filepath.Join(base, cacheSubdir, "v1.tmp-unmarked")
+ if err := os.MkdirAll(stage, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(stage, "legacy.txt"), []byte(base), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Ensure(newTestFS(), base, "v1"); err != nil {
+ t.Fatal(err)
+ }
+ mustReadFile(t, filepath.Join(stage, "legacy.txt"), base)
+}
+
+func TestEnsureReclaimsUnownedStageRegistration(t *testing.T) {
+ base := t.TempDir()
+ parent := filepath.Join(base, cacheSubdir)
+ if err := os.MkdirAll(parent, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ owner := filepath.Join(parent, "v1.tmp-abandoned.lock")
+ if err := os.WriteFile(owner, nil, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Ensure(newTestFS(), base, "v1"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(owner); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("abandoned registration was not reclaimed: %v", err)
+ }
+}
+
+func TestEnsureHeldPublicationLockTimesOut(t *testing.T) {
+ base := t.TempDir()
+ t.Setenv(CacheEnvOverride, base)
+ stopOwner := startCacheLockOwner(t, filepath.Join(base, cacheSubdir, "v1.lock"))
+ const callers = 4
+ done := make(chan error, callers)
+ started := time.Now()
+ for range callers {
+ go func() {
+ _, err := EnsureUserCache(newTestFS(), "v1")
+ done <- err
+ }()
+ }
+ deadline := time.After(7 * time.Second)
+ var results []error
+ for range callers {
+ select {
+ case err := <-done:
+ results = append(results, err)
+ case <-deadline:
+ stopOwner()
+ for len(results) < callers {
+ select {
+ case err := <-done:
+ results = append(results, err)
+ case <-time.After(5 * time.Second):
+ t.Fatal("extraction did not finish after lock owner was killed")
+ }
+ }
+ t.Fatal("held publication lock exceeded the seven-second test deadline")
+ }
+ }
+ for _, err := range results {
+ if !errors.Is(err, os.ErrDeadlineExceeded) {
+ t.Fatalf("held publication lock: got %v, want deadline error", err)
+ }
+ }
+ t.Logf("all %d held-lock callers returned deadline errors after %s", callers, time.Since(started))
+ if _, err := os.Stat(filepath.Join(base, cacheSubdir, "v1")); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("timed-out contender published a cache: %v", err)
+ }
+ staged, err := filepath.Glob(filepath.Join(base, cacheSubdir, "v1.tmp-*"))
+ if err != nil || len(staged) != 0 {
+ t.Fatalf("timed-out contender left staging directories: %v, %v", staged, err)
+ }
+ stopOwner()
+ cacheDir, err := EnsureUserCache(newTestFS(), "v1")
+ if err != nil {
+ t.Fatalf("retry after lock owner death: %v", err)
+ }
+ mustReadFile(t, filepath.Join(cacheDir, "last30days.py"), "# last30days entry\n")
+}
+
+func TestCacheLockExcludesOtherProcessesAndReleasesOnOwnerDeath(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "cache.lock")
+ stopOwner := startCacheLockOwner(t, path)
+ file, err := os.OpenFile(path, os.O_RDWR, 0o600)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer file.Close()
+ done := make(chan error, 1)
+ go func() { done <- lockCacheFile(file) }()
+ select {
+ case err := <-done:
+ t.Fatalf("contender returned while another process held the lock: %v", err)
+ case <-time.After(250 * time.Millisecond):
+ }
+ stopOwner()
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatalf("acquire after owner death: %v", err)
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("lock was not released after owner death")
+ }
+}
+
+func TestEnsurePublicationTimeoutCleansOwnedStage(t *testing.T) {
+ base := t.TempDir()
+ t.Setenv(CacheEnvOverride, base)
+ src := newTestFS()
+ src["zeta.txt"] = &fstest.MapFile{Data: []byte("zeta")}
+ done := make(chan error, 1)
+ finished := false
+ t.Cleanup(func() {
+ _ = os.WriteFile(filepath.Join(base, "release"), nil, 0o600)
+ if !finished {
+ select {
+ case <-done:
+ case <-time.After(7 * time.Second):
+ t.Error("extractor did not finish during cleanup")
+ }
+ }
+ })
+ go func() {
+ _, err := EnsureUserCache(pausedExtractionFS{FS: src, base: base}, "v1")
+ done <- err
+ }()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ if _, err := os.Stat(filepath.Join(base, "ready")); err == nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("extractor did not pause before publication")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ stages := cacheStagePaths(t, base)
+ if len(stages) != 1 {
+ t.Fatalf("paused extractor stages = %v, want one", stages)
+ }
+ startCacheLockOwner(t, filepath.Join(base, cacheSubdir, "v1.lock"))
+ if err := os.WriteFile(filepath.Join(base, "release"), nil, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case err := <-done:
+ finished = true
+ if !errors.Is(err, os.ErrDeadlineExceeded) {
+ t.Fatalf("publication contention: got %v, want deadline error", err)
+ }
+ case <-time.After(7 * time.Second):
+ t.Fatal("publication contention exceeded the seven-second test deadline")
+ }
+ if _, err := os.Stat(stages[0]); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("timed-out extractor retained its stage: %v", err)
+ }
+ if _, err := os.Stat(stages[0] + ".lock"); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("timed-out extractor retained its ownership file: %v", err)
+ }
+}
diff --git a/mcp/internal/engine/cache_lock_unix.go b/mcp/internal/engine/cache_lock_unix.go
new file mode 100644
index 00000000..fb5a8769
--- /dev/null
+++ b/mcp/internal/engine/cache_lock_unix.go
@@ -0,0 +1,17 @@
+//go:build darwin || dragonfly || freebsd || illumos || linux || netbsd || openbsd
+
+package engine
+
+import (
+ "errors"
+ "os"
+ "syscall"
+)
+
+func tryLockCacheFile(file *os.File) (bool, error) {
+ err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+ if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
+ return false, nil
+ }
+ return err == nil, err
+}
diff --git a/mcp/internal/engine/cache_lock_windows.go b/mcp/internal/engine/cache_lock_windows.go
new file mode 100644
index 00000000..008d0471
--- /dev/null
+++ b/mcp/internal/engine/cache_lock_windows.go
@@ -0,0 +1,24 @@
+package engine
+
+import (
+ "os"
+ "syscall"
+ "unsafe"
+)
+
+var lockFileEx = syscall.NewLazyDLL("kernel32.dll").NewProc("LockFileEx")
+
+func tryLockCacheFile(file *os.File) (bool, error) {
+ const exclusiveLock = 0x00000002
+ const failImmediately = 0x00000001
+ const lockViolation = syscall.Errno(33)
+ var overlapped syscall.Overlapped
+ result, _, err := lockFileEx.Call(file.Fd(), exclusiveLock|failImmediately, 0, 1, 0, uintptr(unsafe.Pointer(&overlapped)))
+ if result == 0 {
+ if err == lockViolation {
+ return false, nil
+ }
+ return false, err
+ }
+ return true, nil
+}
diff --git a/mcp/internal/engine/cache_staging.go b/mcp/internal/engine/cache_staging.go
new file mode 100644
index 00000000..6f6a98cc
--- /dev/null
+++ b/mcp/internal/engine/cache_staging.go
@@ -0,0 +1,82 @@
+package engine
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+func prepareCacheStage(cacheDir, version string) (string, *os.File, error) {
+ publication, err := openPublicationLock(cacheDir, true)
+ if err != nil {
+ return "", nil, fmt.Errorf("engine: lock cache staging: %w", err)
+ }
+ defer publication.Close()
+ if err := reclaimCacheStages(cacheDir); err != nil {
+ return "", nil, fmt.Errorf("engine: reclaim cache staging: %w", err)
+ }
+ if sentinelMatches(cacheDir, version) {
+ return "", nil, nil
+ }
+ owner, err := os.CreateTemp(filepath.Dir(cacheDir), filepath.Base(cacheDir)+".tmp-*.lock")
+ if err != nil {
+ return "", nil, fmt.Errorf("engine: create staging ownership file: %w", err)
+ }
+ stage := strings.TrimSuffix(owner.Name(), ".lock")
+ locked, err := tryLockCacheFile(owner)
+ if !locked && err == nil {
+ err = errors.New("new staging ownership file is already locked")
+ }
+ if err == nil {
+ err = os.Mkdir(stage, 0o700)
+ }
+ if err != nil {
+ _ = owner.Close()
+ _ = os.Remove(owner.Name())
+ return "", nil, fmt.Errorf("engine: create cache staging: %w", err)
+ }
+ return stage, owner, nil
+}
+
+// Registration and reclamation both hold the publication lock. A stage's
+// separate ownership lock stays held until extraction and publication finish.
+func reclaimCacheStages(cacheDir string) error {
+ parent := filepath.Dir(cacheDir)
+ entries, err := os.ReadDir(parent)
+ if err != nil {
+ return err
+ }
+ prefix := filepath.Base(cacheDir) + ".tmp-"
+ for _, entry := range entries {
+ if !entry.Type().IsRegular() || !strings.HasPrefix(entry.Name(), prefix) || !strings.HasSuffix(entry.Name(), ".lock") {
+ continue
+ }
+ path := filepath.Join(parent, entry.Name())
+ owner, err := os.OpenFile(path, os.O_RDWR, 0o600)
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ if err != nil {
+ return err
+ }
+ locked, err := tryLockCacheFile(owner)
+ if err != nil || !locked {
+ _ = owner.Close()
+ if err != nil {
+ return err
+ }
+ continue
+ }
+ err = os.RemoveAll(strings.TrimSuffix(path, ".lock"))
+ _ = owner.Close()
+ if err != nil {
+ return err
+ }
+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/mcp/internal/engine/extract.go b/mcp/internal/engine/extract.go
index 6342dbba..bb5d30ad 100644
--- a/mcp/internal/engine/extract.go
+++ b/mcp/internal/engine/extract.go
@@ -29,10 +29,9 @@ const CacheEnvOverride = "LAST30DAYS_CACHE_DIR"
// directory is reused without rewriting. version must be non-empty so the
// cache layout always namespaces by version.
//
-// Extraction writes to a sibling .tmp directory and renames it on success
-// so a partial extraction can never be mistaken for a complete one. Concurrent
-// callers within the same process serialize behind a per-cache-dir sync.Once
-// so the rename happens exactly once.
+// Extraction writes to a unique sibling directory and publishes it under a
+// filesystem lock so another process cannot remove a completed cache. Concurrent
+// callers within the same process serialize behind a per-cache-dir sync.Once.
func Ensure(src fs.FS, baseDir, version string) (string, error) {
if version == "" {
return "", errors.New("engine: version is required")
@@ -40,16 +39,16 @@ func Ensure(src fs.FS, baseDir, version string) (string, error) {
cacheDir := filepath.Join(baseDir, cacheSubdir, version)
once := getOnce(cacheDir)
- var extractErr error
once.Do(func() {
- extractErr = ensureLocked(src, cacheDir, version)
+ once.err = ensureLocked(src, cacheDir, version)
+ if once.err != nil {
+ // Waiting callers share this error; a later call can retry with
+ // a new entry after the failed extraction has finished.
+ resetOnce(cacheDir)
+ }
})
- if extractErr != nil {
- // Reset the sync.Once so a follow-up call can retry rather than
- // permanently caching the error. Retry is the right default when
- // the failure is transient (e.g., disk full, parent dir restored).
- resetOnce(cacheDir)
- return "", extractErr
+ if once.err != nil {
+ return "", once.err
}
return cacheDir, nil
}
@@ -70,30 +69,49 @@ func EnsureUserCache(src fs.FS, version string) (string, error) {
func ensureLocked(src fs.FS, cacheDir, version string) error {
if sentinelMatches(cacheDir, version) {
+ if lock, err := openPublicationLock(cacheDir, false); err == nil && lock != nil {
+ _ = reclaimCacheStages(cacheDir)
+ _ = lock.Close()
+ }
return nil
}
- tmpDir := cacheDir + ".tmp"
- if err := os.RemoveAll(tmpDir); err != nil {
- return fmt.Errorf("engine: clean tmp cache: %w", err)
+ parent := filepath.Dir(cacheDir)
+ if err := os.MkdirAll(parent, 0o755); err != nil {
+ return fmt.Errorf("engine: create cache parent (%s, set %s to override): %w", parent, CacheEnvOverride, err)
}
- if err := os.MkdirAll(tmpDir, 0o755); err != nil {
- return fmt.Errorf("engine: create tmp cache (%s, set %s to override): %w", tmpDir, CacheEnvOverride, err)
+ tmpDir, owner, err := prepareCacheStage(cacheDir, version)
+ if err != nil {
+ return err
}
- if err := extractAll(src, tmpDir); err != nil {
+ if tmpDir == "" {
+ return nil
+ }
+ defer func() {
+ _ = owner.Close()
_ = os.RemoveAll(tmpDir)
+ _ = os.Remove(owner.Name())
+ }()
+ if err := extractAll(src, tmpDir); err != nil {
return err
}
sentinel := filepath.Join(tmpDir, SentinelFilename)
if err := os.WriteFile(sentinel, []byte(version), 0o644); err != nil {
- _ = os.RemoveAll(tmpDir)
return fmt.Errorf("engine: write sentinel: %w", err)
}
+ lock, err := openPublicationLock(cacheDir, true)
+ if err != nil {
+ return fmt.Errorf("engine: lock cache publication: %w", err)
+ }
+ defer func() { _ = lock.Close() }()
+ // Keep the lock file in place: unlinking it can give concurrent publishers
+ // different lock inodes. Closing the descriptor releases the lock.
+ if sentinelMatches(cacheDir, version) {
+ return nil
+ }
if err := os.RemoveAll(cacheDir); err != nil {
- _ = os.RemoveAll(tmpDir)
return fmt.Errorf("engine: clean old cache: %w", err)
}
if err := os.Rename(tmpDir, cacheDir); err != nil {
- _ = os.RemoveAll(tmpDir)
return fmt.Errorf("engine: promote tmp cache: %w", err)
}
return nil
@@ -145,18 +163,23 @@ func copyEmbeddedFile(src fs.FS, srcPath, dst string) error {
// onceRegistry serializes first-call extraction per cache directory so the
// rename in ensureLocked happens exactly once across goroutines.
+type extractionOnce struct {
+ sync.Once
+ err error
+}
+
var (
onceMu sync.Mutex
- onceRegistry = map[string]*sync.Once{}
+ onceRegistry = map[string]*extractionOnce{}
)
-func getOnce(cacheDir string) *sync.Once {
+func getOnce(cacheDir string) *extractionOnce {
onceMu.Lock()
defer onceMu.Unlock()
if o, ok := onceRegistry[cacheDir]; ok {
return o
}
- o := &sync.Once{}
+ o := &extractionOnce{}
onceRegistry[cacheDir] = o
return o
}
diff --git a/mcp/internal/engine/extract_test.go b/mcp/internal/engine/extract_test.go
index c4c0a3aa..76e2fcc6 100644
--- a/mcp/internal/engine/extract_test.go
+++ b/mcp/internal/engine/extract_test.go
@@ -1,16 +1,22 @@
package engine
import (
+ "bytes"
+ "context"
+ "fmt"
+ "io/fs"
"os"
+ "os/exec"
"path/filepath"
"sync"
"testing"
"testing/fstest"
+ "time"
)
func newTestFS() fstest.MapFS {
return fstest.MapFS{
- "last30days.py": &fstest.MapFile{Data: []byte("# last30days entry\n"), Mode: 0o644},
+ "last30days.py": &fstest.MapFile{Data: []byte("# last30days entry\n"), Mode: 0o644},
"lib/__init__.py": &fstest.MapFile{Data: []byte(""), Mode: 0o644},
"lib/env.py": &fstest.MapFile{Data: []byte("# env helpers\n"), Mode: 0o644},
}
@@ -118,6 +124,124 @@ func TestEnsureConcurrentFirstCall(t *testing.T) {
mustReadFile(t, filepath.Join(results[0], "last30days.py"), "# last30days entry\n")
}
+type pausedExtractionFS struct {
+ fs.FS
+ base string
+}
+
+func (src pausedExtractionFS) Open(name string) (fs.File, error) {
+ if name == "zeta.txt" {
+ if err := os.WriteFile(filepath.Join(src.base, "ready"), nil, 0o600); err != nil {
+ return nil, err
+ }
+ deadline := time.Now().Add(20 * time.Second)
+ for {
+ if _, err := os.Stat(filepath.Join(src.base, "release")); err == nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ return nil, fmt.Errorf("timed out waiting to resume extraction")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ }
+ return src.FS.Open(name)
+}
+
+func TestEnsureConcurrentProcesses(t *testing.T) {
+ src := fstest.MapFS{
+ "alpha.txt": &fstest.MapFile{Data: []byte("alpha")},
+ "zeta.txt": &fstest.MapFile{Data: []byte("zeta")},
+ }
+ if role := os.Getenv("LAST30DAYS_TEST_EXTRACT_ROLE"); role != "" {
+ var source fs.FS = src
+ if role == "paused" {
+ source = pausedExtractionFS{FS: src, base: os.Getenv(CacheEnvOverride)}
+ }
+ if _, err := EnsureUserCache(source, "v1"); err != nil {
+ t.Fatal(err)
+ }
+ return
+ }
+
+ for _, initialState := range []string{"missing", "incomplete", "stale"} {
+ t.Run(initialState, func(t *testing.T) {
+ base := t.TempDir()
+ cacheDir := filepath.Join(base, cacheSubdir, "v1")
+ if initialState != "missing" {
+ if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(cacheDir, "alpha.txt"), []byte("partial"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if initialState == "stale" {
+ if err := os.WriteFile(filepath.Join(cacheDir, SentinelFilename), []byte("old-version"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ executable, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv(CacheEnvOverride, base)
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ child := func(role string) *exec.Cmd {
+ cmd := exec.CommandContext(ctx, executable, "-test.run=^TestEnsureConcurrentProcesses$")
+ cmd.Env = append(os.Environ(), "LAST30DAYS_TEST_EXTRACT_ROLE="+role)
+ return cmd
+ }
+ paused := child("paused")
+ var output bytes.Buffer
+ paused.Stdout = &output
+ paused.Stderr = &output
+ if err := paused.Start(); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ _ = paused.Process.Kill()
+ if paused.ProcessState == nil {
+ _ = paused.Wait()
+ }
+ })
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ if _, err := os.Stat(filepath.Join(base, "ready")); err == nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("first process did not pause during extraction")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ if out, err := child("winner").CombinedOutput(); err != nil {
+ t.Fatalf("second process: %v\n%s", err, out)
+ }
+ mustReadFile(t, filepath.Join(cacheDir, "alpha.txt"), "alpha")
+ mustReadFile(t, filepath.Join(cacheDir, "zeta.txt"), "zeta")
+ canary := filepath.Join(cacheDir, "winner-only.txt")
+ if err := os.WriteFile(canary, []byte(base), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(base, "release"), nil, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := paused.Wait(); err != nil {
+ t.Fatalf("first process: %v\n%s", err, &output)
+ }
+ if _, err := EnsureUserCache(src, "v1"); err != nil {
+ t.Fatalf("reuse completed cache: %v", err)
+ }
+ mustReadFile(t, filepath.Join(cacheDir, "alpha.txt"), "alpha")
+ mustReadFile(t, filepath.Join(cacheDir, "zeta.txt"), "zeta")
+ mustReadFile(t, filepath.Join(cacheDir, SentinelFilename), "v1")
+ mustReadFile(t, canary, base)
+ })
+ }
+}
+
func TestEnsureRejectsEmptyVersion(t *testing.T) {
if _, err := Ensure(newTestFS(), t.TempDir(), ""); err == nil {
t.Fatal("expected error for empty version")