Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

filememo: another process's writes to magpie's files are watched for, not polled every 5s (#3185)

yetonede49433ff816
diff --git a/go.mod b/go.mod
index b54469913..20b2c58f0 100644
--- a/go.mod
+++ b/go.mod
@@ -39,6 +39,7 @@ require (
github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+ github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
diff --git a/go.sum b/go.sum
index 6befac56c..a4ed75dfb 100644
--- a/go.sum
+++ b/go.sum
@@ -34,6 +34,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e h1:Lf/gRkoycfOBPa42vU2bbgPurFong6zXeFtPoxholzU=
github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e/go.mod h1:uNVvRXArCGbZ508SxYYTC5v1JWoz2voff5pm25jU1Ok=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
diff --git a/internal/filememo/filememo.go b/internal/filememo/filememo.go
index a636d4a30..94531e462 100644
--- a/internal/filememo/filememo.go
+++ b/internal/filememo/filememo.go
@@ -93,24 +93,31 @@ func Forget() {
mu.Unlock()
}
+// wake has every loop waiting on Written look at its files, with nothing
+// read forgotten: another process wrote one (Watch), which a look at the
+// disk tells.
+func wake() {
+ mu.Lock()
+ close(written)
+ written = make(chan struct{})
+ mu.Unlock()
+}
+
// written is closed, and made anew, by every Forget.
var written = make(chan struct{})
-// Written is closed by the next write magpie makes (Forget): a loop that
-// follows magpie's files waits on it, and looks at them only every so
-// often for what another process wrote, rather than polling each second
-// for its own writes too (#3185: those polls woke an idle magpie ~12
-// times a second).
+// Written is closed by the next write magpie makes (Forget), or by
+// another process's that Watch is told of: a loop that follows magpie's
+// files waits on it, and looks at them only every so often (OthersEvery)
+// for what another process wrote, rather than polling each second for its
+// own writes too (#3185: those polls woke an idle magpie ~12 times a
+// second).
func Written() <-chan struct{} {
mu.Lock()
defer mu.Unlock()
return written
}
-// OthersEvery is how often such a loop looks for what another process
-// (the CLI, a second magpie) wrote.
-const OthersEvery = 5 * time.Second
-
// gens is gen, read without mu.
var gens atomic.Uint64
diff --git a/internal/filememo/watch.go b/internal/filememo/watch.go
new file mode 100644
index 000000000..ce4a36dc7
--- /dev/null
+++ b/internal/filememo/watch.go
@@ -0,0 +1,83 @@
+package filememo
+
+import (
+ "context"
+ "log"
+ "path/filepath"
+ "sync/atomic"
+ "time"
+
+ "github.com/fsnotify/fsnotify"
+)
+
+// OthersEvery is how often a loop that follows magpie's files (Written)
+// looks for what another process (the CLI, a second magpie) wrote: every
+// few seconds, or, while Watch is told of their changes, only now and then
+// for one it missed (#3185: the looks woke an idle magpie on each beat,
+// a dozen times every five seconds).
+func OthersEvery() time.Duration {
+ if watching.Load() {
+ return othersWatched
+ }
+ return othersPolled
+}
+
+const (
+ othersPolled = 5 * time.Second
+ othersWatched = time.Minute
+)
+
+// watching is whether a Watch is told of the files' changes.
+var watching atomic.Bool
+
+// Watch has a change another process makes to one of paths seen at once,
+// as a write of magpie's own is (Written), until ctx ends. Their folders
+// are watched, not the files: a file written whole is a new one put in
+// its place. Where the folders can't be watched, the loops go on looking
+// every few seconds.
+func Watch(ctx context.Context, paths ...string) {
+ w, err := fsnotify.NewWatcher()
+ if err != nil {
+ log.Printf("filememo: can't watch for changes: %v", err)
+ return
+ }
+ defer w.Close()
+ ours := map[string]bool{}
+ dirs := map[string]bool{}
+ for _, p := range paths {
+ p = filepath.Clean(p)
+ ours[p] = true
+ dirs[filepath.Dir(p)] = true
+ }
+ for d := range dirs {
+ if err := w.Add(d); err != nil {
+ log.Printf("filememo: can't watch %s for changes: %v", d, err)
+ return
+ }
+ }
+ watching.Store(true)
+ defer watching.Store(false)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case e, ok := <-w.Events:
+ if !ok {
+ return
+ }
+ if p := filepath.Clean(e.Name); ours[p] {
+ Look(p)
+ wake()
+ }
+ case err, ok := <-w.Errors:
+ if !ok {
+ return
+ }
+ // a change may be lost (the queue overflowed): looked for at
+ // once, and every few seconds again from here
+ log.Printf("filememo: watching for changes: %v", err)
+ wake()
+ return
+ }
+ }
+}
diff --git a/internal/filememo/watch_test.go b/internal/filememo/watch_test.go
new file mode 100644
index 000000000..71dd48b7c
--- /dev/null
+++ b/internal/filememo/watch_test.go
@@ -0,0 +1,55 @@
+package filememo
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+// What another process writes to a followed file wakes the loops waiting
+// on Written at once, written whole (a new file put in its place), in
+// place, or made where there was none; and while that holds the loops
+// look for it themselves only now and then (#3185).
+func TestWatchWakesOnOthersWrites(t *testing.T) {
+ dir := t.TempDir()
+ whole, inPlace, made := filepath.Join(dir, "settings.json"), filepath.Join(dir, "providers.json"), filepath.Join(dir, "plugins.yml")
+ os.WriteFile(whole, []byte("{}"), 0o600)
+ os.WriteFile(inPlace, []byte("{}"), 0o600)
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() { Watch(ctx, whole, inPlace, made); close(done) }()
+ defer func() { cancel(); <-done }()
+ for deadline := time.Now().Add(10 * time.Second); OthersEvery() != othersWatched; time.Sleep(5 * time.Millisecond) {
+ if time.Now().After(deadline) {
+ t.Fatal("the watch never started")
+ }
+ }
+ woken := func(name string, write func()) {
+ t.Helper()
+ w := Written()
+ write()
+ select {
+ case <-w:
+ case <-time.After(10 * time.Second):
+ t.Fatalf("%s written by another process woke nothing", name)
+ }
+ }
+ woken("a file written whole", func() {
+ tmp := whole + ".tmp"
+ os.WriteFile(tmp, []byte(`{"a":1}`), 0o600)
+ os.Rename(tmp, whole)
+ })
+ woken("a file written in place", func() {
+ f, _ := os.OpenFile(inPlace, os.O_WRONLY|os.O_APPEND, 0)
+ f.WriteString(" ")
+ f.Close()
+ })
+ woken("a file made", func() { os.WriteFile(made, []byte("on: []"), 0o600) })
+ cancel()
+ <-done
+ if OthersEvery() != othersPolled {
+ t.Fatalf("with the watch ended the loops look every %v, want %v", OthersEvery(), othersPolled)
+ }
+}
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index 14b677c34..35e684ffa 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -614,6 +614,9 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
defer stopLimits()
freshCtx, stopFresh := context.WithCancel(ctx)
defer stopFresh()
+ // what another process writes to them is seen at once, not on the
+ // loops' next look (#3185)
+ go filememo.Watch(freshCtx, pluginrt.Path(), settings.Path(), provider.Path(), plugin.ListPath(), plugin.ToolsPath(), access.Path())
go pluginrt.KeepFresh(freshCtx, filememo.OthersEvery, settings.Path(), provider.Path(), plugin.ListPath(), plugin.ToolsPath(), access.Path())
srv := &http.Server{Handler: s.Front(), ReadHeaderTimeout: 30 * time.Second, IdleTimeout: 5 * time.Minute}
// the magpie serving the gateway, and only it, keeps the saved accounts
diff --git a/internal/middleware/points.go b/internal/middleware/points.go
index f32edcdc2..d9e7b02a1 100644
--- a/internal/middleware/points.go
+++ b/internal/middleware/points.go
@@ -419,7 +419,7 @@ func (o *observer) Serve() (stop func()) {
case <-done:
return
case <-written:
- case <-beat.Next(filememo.OthersEvery):
+ case <-beat.Next(filememo.OthersEvery()):
}
written = filememo.Written()
o.refresh()
diff --git a/internal/otlp/otel_sessions.go b/internal/otlp/otel_sessions.go
index 17de70919..e94901ccf 100644
--- a/internal/otlp/otel_sessions.go
+++ b/internal/otlp/otel_sessions.go
@@ -135,7 +135,7 @@ func (e *otelExporter) watchSessions(ctx context.Context, started time.Time, eve
if err != nil || !config.Enabled || !config.Sessions {
reader = nil
e.sessions.Store(nil)
- wait = filememo.OthersEvery
+ wait = filememo.OthersEvery()
continue
}
wait = every
diff --git a/internal/pluginrt/config.go b/internal/pluginrt/config.go
index 7e863eea3..c252e4b60 100644
--- a/internal/pluginrt/config.go
+++ b/internal/pluginrt/config.go
@@ -360,7 +360,7 @@ func Refresh() ([]Composed, error) {
// `magpie plugin builtin` in another process may turn a plugin on or off,
// and what a plugin registers with the gateway (an observer) follows
// without a restart.
-func KeepFresh(ctx context.Context, every time.Duration, also ...string) {
+func KeepFresh(ctx context.Context, every func() time.Duration, also ...string) {
files := append([]string{Path()}, also...)
stamp := func() string {
var b strings.Builder
@@ -388,7 +388,7 @@ func KeepFresh(ctx context.Context, every time.Duration, also ...string) {
select {
case <-ctx.Done():
return
- case <-beat.Next(every):
+ case <-beat.Next(every()):
case <-written:
}
}
diff --git a/internal/pluginrt/config_test.go b/internal/pluginrt/config_test.go
index d8661e215..4fb64a42c 100644
--- a/internal/pluginrt/config_test.go
+++ b/internal/pluginrt/config_test.go
@@ -180,7 +180,7 @@ func TestKeepFreshFollowsPluginsYml(t *testing.T) {
keep := func(every time.Duration) (stop func()) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
- go func() { KeepFresh(ctx, every); close(done) }()
+ go func() { KeepFresh(ctx, func() time.Duration { return every }); close(done) }()
return func() { cancel(); <-done }
}
stop := keep(10 * time.Millisecond)