Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

idle: the loops following magpie's files wake on its own writes, not a 1-2s poll (#3185)

yetone091153bf71c6
diff --git a/internal/filememo/filememo.go b/internal/filememo/filememo.go
index 3623f57d3..a636d4a30 100644
--- a/internal/filememo/filememo.go
+++ b/internal/filememo/filememo.go
@@ -88,9 +88,29 @@ func Forget() {
mu.Lock()
gen++
gens.Store(gen)
+ close(written)
+ written = make(chan struct{})
mu.Unlock()
}
+// written is closed, and made anew, by every Forget.
+var written = make(chan struct{})
+
+// Written is closed by the next write magpie makes (Forget): a loop that
+// follows magpie's files waits on it, and looks at them only every so
+// often for what another process wrote, rather than polling each second
+// for its own writes too (#3185: those polls woke an idle magpie ~12
+// times a second).
+func Written() <-chan struct{} {
+ mu.Lock()
+ defer mu.Unlock()
+ return written
+}
+
+// OthersEvery is how often such a loop looks for what another process
+// (the CLI, a second magpie) wrote.
+const OthersEvery = 5 * time.Second
+
// gens is gen, read without mu.
var gens atomic.Uint64
diff --git a/internal/filememo/filememo_test.go b/internal/filememo/filememo_test.go
index f539abf41..021eb3ab5 100644
--- a/internal/filememo/filememo_test.go
+++ b/internal/filememo/filememo_test.go
@@ -151,3 +151,25 @@ func TestHoldMissing(t *testing.T) {
t.Fatalf("unheld, %d looks for 3 reads", n)
}
}
+
+// Written is closed by the next Forget, and a wait begun after it waits for
+// the one after.
+func TestWritten(t *testing.T) {
+ w := Written()
+ select {
+ case <-w:
+ t.Fatal("closed before any write")
+ default:
+ }
+ Forget()
+ select {
+ case <-w:
+ case <-time.After(5 * time.Second):
+ t.Fatal("not closed by Forget")
+ }
+ select {
+ case <-Written():
+ t.Fatal("the next wait is closed already")
+ default:
+ }
+}
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index d71481c24..66acc2814 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -32,6 +32,7 @@ import (
"github.com/yetone/magpie/internal/access"
"github.com/yetone/magpie/internal/awake"
"github.com/yetone/magpie/internal/catalog"
+ "github.com/yetone/magpie/internal/filememo"
"github.com/yetone/magpie/internal/gwbudget"
"github.com/yetone/magpie/internal/gwmw"
"github.com/yetone/magpie/internal/gwtool"
@@ -613,7 +614,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
defer stopLimits()
freshCtx, stopFresh := context.WithCancel(ctx)
defer stopFresh()
- go pluginrt.KeepFresh(freshCtx, 2*time.Second, settings.Path(), provider.Path(), plugin.ListPath(), plugin.ToolsPath(), access.Path())
+ go pluginrt.KeepFresh(freshCtx, filememo.OthersEvery, settings.Path(), provider.Path(), plugin.ListPath(), plugin.ToolsPath(), access.Path())
srv := &http.Server{Handler: s.Front(), ReadHeaderTimeout: 30 * time.Second, IdleTimeout: 5 * time.Minute}
// the magpie serving the gateway, and only it, keeps the saved accounts
// signed in, so two never refresh one sign-in at once
diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go
index cab7bc670..2830edc81 100644
--- a/internal/middleware/middleware.go
+++ b/internal/middleware/middleware.go
@@ -25,6 +25,7 @@ import (
"time"
"github.com/Calcium-Ion/moejs"
+ "github.com/yetone/magpie/internal/filememo"
"github.com/yetone/magpie/internal/jsmod"
"github.com/yetone/magpie/internal/plugin"
)
@@ -84,17 +85,22 @@ var (
loadMu sync.Mutex
cur atomic.Pointer[set]
nextCheck atomic.Int64
+ // checkedGen is filememo's generation when current last looked
+ checkedGen atomic.Uint64
)
// current is the middleware in force, loaded again when plugins.json or a
-// middleware's file changed, which is looked at once a second.
+// middleware's file changed, which is looked at once a second, and at
+// once after magpie wrote a file of its own.
func current() *set {
s := cur.Load()
now := time.Now().UnixNano()
n := nextCheck.Load()
- if s != nil && (now < n || !nextCheck.CompareAndSwap(n, now+int64(time.Second))) {
+ g := filememo.Gen()
+ if s != nil && g == checkedGen.Load() && (now < n || !nextCheck.CompareAndSwap(n, now+int64(time.Second))) {
return s
}
+ checkedGen.Store(g)
st := stamp()
if s != nil && s.stamp == st {
return s
diff --git a/internal/middleware/points.go b/internal/middleware/points.go
index c2aea094e..9d781fce4 100644
--- a/internal/middleware/points.go
+++ b/internal/middleware/points.go
@@ -35,6 +35,7 @@ import (
"github.com/Calcium-Ion/moejs"
+ "github.com/yetone/magpie/internal/filememo"
"github.com/yetone/magpie/internal/plugin"
"github.com/yetone/magpie/internal/routes"
"github.com/yetone/magpie/internal/usage"
@@ -404,19 +405,25 @@ func (o *observer) Serve() (stop func()) {
o.mu.Lock()
o.queue = q
o.mu.Unlock()
+ written := filememo.Written() // before the look, so a write made during it isn't missed
o.refresh()
wg.Add(2)
go func() {
defer wg.Done()
- t := time.NewTicker(time.Second)
+ // a plugin turned on here is seen once saved (Written); every
+ // OthersEvery only catches one the CLI or a second magpie turned on
+ // (#3185: a 1s tick woke an idle magpie every second)
+ t := time.NewTicker(filememo.OthersEvery)
defer t.Stop()
for {
select {
case <-done:
return
+ case <-written:
case <-t.C:
- o.refresh()
}
+ written = filememo.Written()
+ o.refresh()
}
}()
go func() {
diff --git a/internal/middleware/points_test.go b/internal/middleware/points_test.go
index 6c968b4a9..723b0a26c 100644
--- a/internal/middleware/points_test.go
+++ b/internal/middleware/points_test.go
@@ -119,6 +119,27 @@ func TestStrategiesServedWhileOn(t *testing.T) {
}
}
+// A module turned off here (the Plugins page) is no longer served at once,
+// without waiting for the look for other processes' writes, which is
+// seconds apart (#3185).
+func TestStrategiesFollowOwnWrite(t *testing.T) {
+ e := file(t, `export const strategies = { a: (p, s) => s.map((_, i) => i) };`)
+ use(t, e)
+ serve(t)
+ if !routes.Served("x/a") {
+ t.Fatal("not served")
+ }
+ if err := plugin.SetOff(plugin.Name(e.Spec), true); err != nil {
+ t.Fatal(err)
+ }
+ for end := time.Now().Add(2 * time.Second); routes.Served("x/a"); {
+ if time.Now().After(end) {
+ t.Fatal("still served 2s after it was turned off here")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+}
+
// Not served (plugins/middleware off), nothing is registered.
func TestStrategiesStopWithServe(t *testing.T) {
use(t, file(t, `export const strategies = { a: (p, s) => s.map((_, i) => i) };`))
diff --git a/internal/otlp/otel.go b/internal/otlp/otel.go
index 7334abe52..959ea77ce 100644
--- a/internal/otlp/otel.go
+++ b/internal/otlp/otel.go
@@ -217,10 +217,20 @@ func (e *otelExporter) run(every time.Duration) {
defer close(e.done)
defer e.cancel()
defer e.client.CloseIdleConnections()
- ticker := time.NewTicker(every)
- defer ticker.Stop()
+ // the timer runs only while the batch holds something, so an idle
+ // exporter doesn't wake every few seconds (#3185)
+ timer := time.NewTimer(every)
+ timer.Stop()
+ defer timer.Stop()
batch := make([]otelItem, 0, otelBatchSize)
+ add := func(it otelItem) {
+ if len(batch) == 0 {
+ timer.Reset(every)
+ }
+ batch = append(batch, it)
+ }
flush := func() {
+ timer.Stop()
e.flush(batch)
for _, it := range batch {
if n := int64(recordBytes(it.record)); n > 0 {
@@ -232,11 +242,11 @@ func (e *otelExporter) run(every time.Duration) {
for {
select {
case it := <-e.queue:
- batch = append(batch, it)
+ add(it)
if len(batch) == otelBatchSize {
flush()
}
- case <-ticker.C:
+ case <-timer.C:
flush()
case <-e.stop:
for {
diff --git a/internal/otlp/otel_sessions.go b/internal/otlp/otel_sessions.go
index 5920cd610..6113ba2a0 100644
--- a/internal/otlp/otel_sessions.go
+++ b/internal/otlp/otel_sessions.go
@@ -5,6 +5,7 @@ import (
"reflect"
"time"
+ "github.com/yetone/magpie/internal/filememo"
"github.com/yetone/magpie/internal/redact"
"github.com/yetone/magpie/internal/sessions"
"github.com/yetone/magpie/internal/settings"
@@ -113,23 +114,38 @@ func (e *otelExporter) watchSessions(ctx context.Context, started time.Time, eve
defer ticker.Stop()
var reader *sessions.TraceReader
var prior settings.OTel
+ written := filememo.Written() // before each look, so a save during it isn't missed
if config, err := settings.OTelExport(); err == nil && config.Enabled && config.Sessions {
reader = sessions.NewTraceReader(started)
reader.SetSessionIndex(&e.identities)
prior = config
}
+ // while sessions aren't exported, a save here turns them on at once
+ // (Written), and OthersEvery only catches the CLI's (#3185: a 2s tick
+ // woke an idle magpie that exports nothing)
+ wait := every
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
+ case <-written:
}
+ written = filememo.Written()
config, err := settings.OTelExport()
if err != nil || !config.Enabled || !config.Sessions {
reader = nil
e.sessions.Store(nil)
+ if wait != filememo.OthersEvery {
+ wait = filememo.OthersEvery
+ ticker.Reset(wait)
+ }
continue
}
+ if wait != every {
+ wait = every
+ ticker.Reset(wait)
+ }
if reader == nil || !reflect.DeepEqual(config, prior) {
reader = sessions.NewTraceReader(time.Now())
reader.SetSessionIndex(&e.identities)
diff --git a/internal/plugin/store.go b/internal/plugin/store.go
index 7ff970b9f..0c4f0b261 100644
--- a/internal/plugin/store.go
+++ b/internal/plugin/store.go
@@ -34,6 +34,7 @@ import (
"sync/atomic"
"time"
+ "github.com/yetone/magpie/internal/filememo"
"github.com/yetone/magpie/internal/lastgood"
"github.com/yetone/magpie/internal/netproxy"
"github.com/yetone/magpie/internal/proc"
@@ -228,6 +229,7 @@ func save(l List) error {
if err := lastgood.Keep(listPath(), lastgood.JSON, 0o600); err != nil {
return err
}
+ defer filememo.Forget() // what follows plugins.json looks again (Written)
if err := writeWhole(listPath(), append(b, '\n')); err != nil {
return err
}
diff --git a/internal/plugin/tools.go b/internal/plugin/tools.go
index d70544f24..cf9108111 100644
--- a/internal/plugin/tools.go
+++ b/internal/plugin/tools.go
@@ -13,6 +13,7 @@ import (
"sync"
"time"
+ "github.com/yetone/magpie/internal/filememo"
"github.com/yetone/magpie/internal/settings"
"github.com/yetone/magpie/internal/steady"
)
@@ -107,6 +108,7 @@ func noteTools(loaded []Loaded) {
if len(m) == 0 && errors.Is(err, fs.ErrNotExist) {
return
}
+ defer filememo.Forget() // the gateway follows this note (Written)
if err := writeWhole(toolsPath(), append(b, '\n')); err != nil {
log.Printf("plugin tools: noting them: %v", err)
}
diff --git a/internal/pluginrt/config.go b/internal/pluginrt/config.go
index 16b59d5b2..291a4cd6f 100644
--- a/internal/pluginrt/config.go
+++ b/internal/pluginrt/config.go
@@ -17,6 +17,7 @@ import (
"github.com/yetone/magpie/internal/appdir"
"github.com/yetone/magpie/internal/filelock"
+ "github.com/yetone/magpie/internal/filememo"
)
// plugins.yml says which built-in plugins are on, and each one's config:
@@ -292,6 +293,7 @@ func write(f File) error {
}
b = append([]byte(header), b...)
path := Path()
+ defer filememo.Forget() // KeepFresh looks again (Written)
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
@@ -374,16 +376,21 @@ func KeepFresh(ctx context.Context, every time.Duration, also ...string) {
tick := time.NewTicker(every)
defer tick.Stop()
for {
+ // taken before the look, so a write made during it isn't missed
+ written := filememo.Written()
if now := stamp(); now != last {
last = now
if _, err := Refresh(); err != nil {
log.Printf("plugins: %v", err)
}
}
+ // a write of this process's is looked at once made; every only
+ // catches another process's
select {
case <-ctx.Done():
return
case <-tick.C:
+ case <-written:
}
}
}
diff --git a/internal/pluginrt/config_test.go b/internal/pluginrt/config_test.go
index baa02dae4..d8661e215 100644
--- a/internal/pluginrt/config_test.go
+++ b/internal/pluginrt/config_test.go
@@ -167,7 +167,8 @@ func TestBrokenFileIsKept(t *testing.T) {
}
// A plugin turned off or on in plugins.yml by another process is disposed
-// or applied in the one keeping fresh, without a restart.
+// or applied in the one keeping fresh, without a restart; turned off or on
+// in that process, at once.
func TestKeepFreshFollowsPluginsYml(t *testing.T) {
home(t)
var applied atomic.Int32
@@ -176,10 +177,14 @@ func TestKeepFreshFollowsPluginsYml(t *testing.T) {
c.OnDispose(func() { applied.Add(-1) })
return nil
}})
- ctx, cancel := context.WithCancel(context.Background())
- done := make(chan struct{})
- go func() { KeepFresh(ctx, 10*time.Millisecond); close(done) }()
- defer func() { cancel(); <-done; Set("fresh-test", false); Refresh() }()
+ keep := func(every time.Duration) (stop func()) {
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() { KeepFresh(ctx, every); close(done) }()
+ return func() { cancel(); <-done }
+ }
+ stop := keep(10 * time.Millisecond)
+ defer func() { stop(); Set("fresh-test", false); Refresh() }()
wait := func(want int32) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
@@ -199,4 +204,19 @@ func TestKeepFreshFollowsPluginsYml(t *testing.T) {
t.Fatal(err)
}
wait(1)
+
+ // turned off or on in this process, it follows at once, not on the
+ // next look for another process's writes (#3185: those looks woke an
+ // idle magpie, so they are seconds apart now)
+ stop()
+ stop = keep(time.Hour)
+ wait(1)
+ if err := Set("fresh-test", false); err != nil {
+ t.Fatal(err)
+ }
+ wait(0)
+ if err := Set("fresh-test", true); err != nil {
+ t.Fatal(err)
+ }
+ wait(1)
}