Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

usage: a deleted provider's tests stay listed under it, not its host (cybercaius)

yetone94c2aaecbaed
diff --git a/docs/subsystems/gateway-sessions.md b/docs/subsystems/gateway-sessions.md
index 329b9aa14..ff8dcdfd0 100644
--- a/docs/subsystems/gateway-sessions.md
+++ b/docs/subsystems/gateway-sessions.md
@@ -537,6 +537,14 @@ for: `provider_test`, `api_detect` or `model_list`
request page lists them as a chunk of their own (`probeChunk`) and adds
them to no total, cell, group, chart or filter count; `?via=` other than
all leaves them out.
+- **Whose:** a row recorded under its host, sent before its provider had
+ an id, is listed under the provider saved on that host now
+ (`Renames.ProbeOf`), and with none on it, a deleted one's, under the id
+ the log's calls to that host were made under (`logSnapshot.hosts`,
+ `probeProvider`). A host two providers are or were on stays the host,
+ lowercased (yyddbds and cybercaius on Discord).
+ `TestProbesOfAProviderBeingAddedListItOnce` and
+ `TestProbesOfADeletedProviderStayWithIt` cover it.
- **Showing:** the row's badge is the kind's name ("Test", "Detect APIs",
"Model list", from `purposeKinds`), and its tooltip and the "magpie's
own" badge's say it counts in no total (`probeWhy` in
diff --git a/internal/provider/rename.go b/internal/provider/rename.go
index 80f274b0f..959c1c093 100644
--- a/internal/provider/rename.go
+++ b/internal/provider/rename.go
@@ -207,6 +207,12 @@ func (r Renames) ProbeOf(id string, at time.Time) string {
return r.Of(id, at)
}
+// OnHost is whether a provider saved now is on host, one or more.
+func (r Renames) OnHost(host string) bool {
+ _, ok := r.hosts[strings.ToLower(host)]
+ return ok
+}
+
// Timed is whether the id's calls go to more than one provider by when
// they were made, so Of needs each call's time.
func (r Renames) Timed(id string) bool { return len(r.gave[id]) > 0 }
diff --git a/internal/usage/log_index.go b/internal/usage/log_index.go
index d07f5444a..129c6053d 100644
--- a/internal/usage/log_index.go
+++ b/internal/usage/log_index.go
@@ -14,6 +14,7 @@ import (
"maps"
"os"
"slices"
+ "strings"
"sync"
"time"
)
@@ -52,6 +53,11 @@ type logSnapshot struct {
// out of blocks: every reader of those counts what it reads, and the
// request log alone lists these (queryPage)
probes []probeRow
+ // hosts are the providers by the host their calls went to, as the log
+ // recorded them: "" for a host two providers were on. A probe made
+ // while a provider was being added, under its host, is that
+ // provider's after it is deleted too (probeChunk; cybercaius).
+ hosts map[string]string
}
var logIndex struct {
@@ -71,6 +77,24 @@ var logAppends struct {
base, last os.FileInfo
}
+// sawHost notes the provider r's call went to by its host. A probe made
+// before its provider had an id names the host for both, and says nothing.
+func (s *logSnapshot) sawHost(r Record) {
+ h, _, _ := strings.Cut(r.Host, " as ") // a subscription's "host as user"
+ h = strings.ToLower(strings.TrimSpace(h))
+ if h == "" || r.Provider == "" || strings.EqualFold(r.Provider, h) || r.IsRejected() {
+ return
+ }
+ if s.hosts == nil {
+ s.hosts = map[string]string{}
+ }
+ if to, ok := s.hosts[h]; !ok {
+ s.hosts[h] = r.Provider
+ } else if to != r.Provider {
+ s.hosts[h] = ""
+ }
+}
+
func readLogSnapshot() *logSnapshot {
// the clients the user counts as agents, read once for the pass the
// snapshot is read for (#1958)
@@ -103,7 +127,7 @@ func readLogSnapshot() *logSnapshot {
logIndex.version++
}
- next := &logSnapshot{path: path, info: info, settled: settled, version: logIndex.version, keyProviders: map[string]bool{}}
+ next := &logSnapshot{path: path, info: info, settled: settled, version: logIndex.version, keyProviders: map[string]bool{}, hosts: map[string]string{}}
if err != nil {
logIndex.snapshot = next
return next
@@ -121,6 +145,7 @@ func readLogSnapshot() *logSnapshot {
next.blocks = slices.Clone(old.blocks)
next.keyProviders = maps.Clone(old.keyProviders)
next.probes = slices.Clip(old.probes) // appended to, never written in
+ next.hosts = maps.Clone(old.hosts)
}
logIndex.reads++
f, err := os.Open(path)
@@ -152,6 +177,7 @@ func readLogSnapshot() *logSnapshot {
if json.Unmarshal(b, &r) != nil {
continue
}
+ next.sawHost(r)
if r.IsProbe() {
next.probes = append(next.probes, probeRow{r, next.off})
continue
diff --git a/internal/usage/probes.go b/internal/usage/probes.go
index 9f13842e7..d0fa6834e 100644
--- a/internal/usage/probes.go
+++ b/internal/usage/probes.go
@@ -1,6 +1,7 @@
package usage
import (
+ "strings"
"time"
"github.com/yetone/magpie/internal/probelog"
@@ -58,8 +59,11 @@ func clipProbeError(s string) string {
// A provider renamed since is listed by the id it has now (Renames): the
// Requests page's provider list named a renamed one twice (cybercaius).
// One asked while a provider was being added, under its host, is listed
-// by the id of the provider on that host (Renames.ProbeOf).
-func probeChunk(probes []probeRow, since time.Time, renames provider.Renames) *rowChunk {
+// by the id of the provider on that host (Renames.ProbeOf), and once no
+// provider is on it, a deleted one's, by the id the log's calls to that
+// host were made under (hosts, logSnapshot.hosts): cybercaius on Discord,
+// a deleted provider's tests were listed under its host beside its id.
+func probeChunk(probes []probeRow, since time.Time, renames provider.Renames, hosts map[string]string) *rowChunk {
var c *rowChunk
for _, p := range probes {
if !since.IsZero() && p.Time.Before(since) {
@@ -69,7 +73,7 @@ func probeChunk(probes []probeRow, since time.Time, renames provider.Renames) *r
c = &rowChunk{}
}
r := p.Record
- r.Provider = renames.ProbeOf(r.Provider, r.Time)
+ r.Provider = probeProvider(r, renames, hosts)
c.add(Row{Record: r, Source: SourceProbe}, "", p.off, false)
}
if c == nil {
@@ -78,3 +82,19 @@ func probeChunk(probes []probeRow, since time.Time, renames provider.Renames) *r
c.dict, c.shapeIDs = nil, nil
return c
}
+
+// probeProvider is the provider a probe row is listed under: its id as it
+// is now, and for one recorded under its host before the provider had an
+// id, the provider saved on that host, or, with none on it now, the one
+// the log's calls to that host went to. A host two providers were on, or
+// none, stays the host.
+func probeProvider(r Record, renames provider.Renames, hosts map[string]string) string {
+ id := renames.ProbeOf(r.Provider, r.Time)
+ if id != r.Provider || !strings.EqualFold(r.Provider, r.Host) || renames.OnHost(r.Host) {
+ return id
+ }
+ if to := hosts[strings.ToLower(r.Host)]; to != "" {
+ return renames.Of(to, r.Time)
+ }
+ return strings.ToLower(id) // one host, however it was typed
+}
diff --git a/internal/usage/rename_usage_test.go b/internal/usage/rename_usage_test.go
index da2ec1ef6..a6e5f09b7 100644
--- a/internal/usage/rename_usage_test.go
+++ b/internal/usage/rename_usage_test.go
@@ -119,3 +119,48 @@ func TestProbesOfAProviderBeingAddedListItOnce(t *testing.T) {
t.Fatalf("%s's rows: total %d, sum %+v", id, p.Total, p.Sum)
}
}
+
+// cybercaius on Discord: a provider tested while it was being added was
+// listed on the Requests page under its id and its host, and after it
+// was deleted the host stayed beside the id. Its probes are listed under
+// the id its calls to that host were made under, deleted or not; a host
+// two deleted providers were on stays as it was recorded.
+func TestProbesOfADeletedProviderStayWithIt(t *testing.T) {
+ reportedHome(t)
+ then := time.Now().Add(-time.Hour)
+ adding := probelog.With(context.Background(), probelog.KindDetect, "", "")
+ probelog.Note(adding, "https://api.example.me/v1/models", http.MethodGet, then, 200, nil)
+ probelog.Note(adding, "https://API.example.me/v1/chat/completions", http.MethodPost, then, 200, nil)
+ probelog.Note(adding, "https://twice.example/v1/models", http.MethodGet, then, 200, nil)
+ probelog.Note(adding, "https://Twice.Example/v1/models", http.MethodGet, then, 200, nil)
+ id, err := provider.Add(provider.Provider{Name: "Example", Key: "k", Chat: "https://api.example.me/v1", Models: []string{"m"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ saved, err := provider.Find(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ tested := probelog.With(context.Background(), probelog.KindTest, id, "")
+ probelog.Note(tested, "https://api.example.me/v1/chat/completions", http.MethodPost, then.Add(time.Minute), 200, nil)
+ Append(Record{Time: then.Add(2 * time.Minute), Agent: "codex", Session: "s1", Provider: id, Host: saved.Where(), Model: "m", Input: 100, Output: 10, Status: 200})
+ for _, other := range []string{"one", "two"} {
+ Append(Record{Time: then.Add(2 * time.Minute), Agent: "codex", Session: "s2", Provider: other, Host: "twice.example", Model: "m", Input: 1, Output: 1, Status: 200})
+ }
+ list := func() []string { return QueryPage(All, Filter{}, 0, 50).Providers }
+ if want := []string{id, "one", "twice.example", "two"}; !slices.Equal(list(), want) {
+ t.Fatalf("saved: providers %v, want %v", list(), want)
+ }
+ if err := provider.Delete(id); err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{id, "one", "twice.example", "two"}; !slices.Equal(list(), want) {
+ t.Fatalf("deleted: providers %v, want %v", list(), want)
+ }
+ if p := QueryPage(All, Filter{Provider: id}, 0, 50); p.Total != 4 || p.Sum.Calls != 1 {
+ t.Fatalf("deleted %s's rows: total %d, sum %+v", id, p.Total, p.Sum)
+ }
+ if p := QueryPage(All, Filter{Provider: "twice.example"}, 0, 50); p.Total != 2 {
+ t.Fatalf("twice.example's rows: total %d", p.Total)
+ }
+}
diff --git a/internal/usage/request_page.go b/internal/usage/request_page.go
index fa256e980..433a7df8c 100644
--- a/internal/usage/request_page.go
+++ b/internal/usage/request_page.go
@@ -905,7 +905,7 @@ func readRequestBlocks(idx *requestIndex, st requestState, p Period, now time.Ti
gateways = append(gateways, c)
}
}
- if c := probeChunk(snapshot.probes, gatewaySince, renames); c != nil {
+ if c := probeChunk(snapshot.probes, gatewaySince, renames, snapshot.hosts); c != nil {
gateways = append(gateways, c) // listed, never counted (SourceProbe)
}
on := map[string]bool{}