Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
pii: of the requests waiting for the local model, the one with the least left to read goes next
yetone43a9d044003a
diff --git a/internal/pii/find.go b/internal/pii/find.goindex 5153fe10b..c1793324a 100644--- a/internal/pii/find.go+++ b/internal/pii/find.go@@ -60,6 +60,9 @@ type backend struct {// sent are the texts and pieces a Bounded request went without the// model's reading (by hash): what a request sent unread before.sent sentSet+ // queue are the requests waiting for turn, the least left to read+ // first+ queue queue}func (b *backend) size() int {@@ -139,13 +142,25 @@ func (b *backend) find(ctx context.Context, texts []string) ([][]redact.Found, e}backlog = b.sent.mark(ks)}+ // how much of the texts this request has left for the model to read+ left := func(got map[[32]byte][]redact.Found, miss []int, read int) int {+ n, old := 0, 0+ for _, u := range b.missing(texts, miss, got) {+ if backlog[sha256.Sum256([]byte(u))] {+ old += len(u)+ } else {+ n += len(u)+ }+ }+ return n + min(old, max(0, backlogCap-read))+ }if ctx.Err() != nil {return b.partial(ctx, texts)}if err := b.heldBack(); err != nil {return nil, err}- if !b.takeTurn(ctx) {+ if !b.takeTurn(ctx, left(nil, miss, 0)) {return b.partial(ctx, texts)}type result struct {@@ -176,11 +191,13 @@ func (b *backend) find(ctx context.Context, texts []string) ([][]redact.Found, edone <- result{nil, errWaitEnded}return}- // give way to a request waiting for the turn: it goes- // before the next chunk+ // give way to a request waiting for the turn with less+ // to read than this one has left: it goes before the+ // next chunk+ n := left(got, miss, read)<-b.turnheld = false- if !b.takeTurn(ctx) {+ if !b.takeTurn(ctx, n) {done <- result{nil, errWaitEnded}return}@@ -289,16 +306,98 @@ func (b *backend) partial(ctx context.Context, texts []string) ([][]redact.Found}// takeTurn waits for the model's turn while ctx lives, counted as-// waiting; false when ctx ended first.-func (b *backend) takeTurn(ctx context.Context) bool {+// waiting, for a request with left bytes still to read; false when ctx+// ended first. Of the requests waiting, the one with the least left takes+// the turn next (the first to come of those with as little): a request of+// a few lines goes before the next chunk of a long one, and of every+// other long one waiting. Before, they took it in the order they came,+// and a short request waited a chunk of each long one ahead of it: on a+// 4-core CPU, a second or more apiece, past masking's default wait of 2s+// with two ahead (wangwu25685333 on X, 10-29: requests of 1.4K tokens+// waited 8.5s behind a fresh session's 31.5K, and their client gave up at+// 20s).+func (b *backend) takeTurn(ctx context.Context, left int) bool {b.waiting.Add(1)defer b.waiting.Add(-1)- select {- case b.turn <- struct{}{}:- return true- case <-ctx.Done():- return false+ w := b.queue.add(left)+ defer b.queue.remove(w)+ for {+ next, changed := b.queue.next(w)+ if !next {+ select {+ case <-changed:+ continue+ case <-ctx.Done():+ return false+ }+ }+ select {+ case b.turn <- struct{}{}:+ return true+ case <-changed:+ case <-ctx.Done():+ return false+ }+ }+}++// queue are the requests waiting for a backend's turn. Its zero value is+// empty.+type queue struct {+ mu sync.Mutex+ seq uint64+ waiters map[*waiter]bool+ // changed is closed when a request comes or goes, and made again+ changed chan struct{}+}++// waiter is a request in a queue: left bytes to read, seq-th to come.+type waiter struct {+ left int+ seq uint64+}++// add puts a request with left bytes to read in q.+func (q *queue) add(left int) *waiter {+ q.mu.Lock()+ defer q.mu.Unlock()+ q.seq+++ w := &waiter{left, q.seq}+ if q.waiters == nil {+ q.waiters = map[*waiter]bool{}+ }+ q.waiters[w] = true+ q.changedLocked()+ return w+}++// remove takes w out of q.+func (q *queue) remove(w *waiter) {+ q.mu.Lock()+ defer q.mu.Unlock()+ delete(q.waiters, w)+ q.changedLocked()+}++// changedLocked wakes those waiting on q to look again; q.mu is held.+func (q *queue) changedLocked() {+ if q.changed != nil {+ close(q.changed)+ }+ q.changed = make(chan struct{})+}++// next says whether w is the one to take the turn next, and gives what+// is closed when that may have changed.+func (q *queue) next(w *waiter) (bool, <-chan struct{}) {+ q.mu.Lock()+ defer q.mu.Unlock()+ for o := range q.waiters {+ if o.left < w.left || o.left == w.left && o.seq < w.seq {+ return false, q.changed+ }}+ return true, q.changed}// waitErr is a request's that stopped waiting for the model.diff --git a/internal/pii/turn_test.go b/internal/pii/turn_test.goindex 014e9cb5b..bb4dee9aa 100644--- a/internal/pii/turn_test.go+++ b/internal/pii/turn_test.go@@ -378,3 +378,54 @@ func TestStoppedRequestKeepsWhatWasRead(t *testing.T) {t.Fatalf("got %v, want the history's Alice and nothing for the unread text", got)}}++// wangwu25685333 on X (10-29): requests of a few lines waited 8.5s for+// the model behind a fresh session's long system prompt, one chunk of+// each long request ahead of them. Now the one with the least left to+// read goes next: a short request that comes while one long request is+// on the model and another waits goes before either's next chunk.+func TestShortRequestGoesBeforeLongOnes(t *testing.T) {+ r := newReader(t, guardChunk)+ gate := make(chan struct{})+ r.during = func(n int) { <-gate }+ // four texts of a chunk each+ long := func(cs string) []string {+ var out []string+ for _, c := range cs {+ out = append(out, strings.Repeat(string(c)+" ", guardChunk/2-8))+ }+ return out+ }+ first, second, short := long("abcd"), long("efgh"), "hi Alice, the build is green"+ done := make(chan string, 3)+ run := func(name string, texts []string) {+ go func() {+ if _, err := r.find(context.Background(), texts); err != nil {+ t.Error(name, err)+ }+ done <- name+ }()+ }+ asked := func() int { r.mu.Lock(); defer r.mu.Unlock(); return len(r.asked) }+ run("first", first)+ waitFor(t, "the first long request on the model", func() bool { return asked() == 1 })+ run("second", second)+ waitFor(t, "the second long request waiting", func() bool { return r.waiting.Load() == 1 })+ run("short", []string{short})+ waitFor(t, "the short request waiting", func() bool { return r.waiting.Load() == 2 })+ for range 9 {+ gate <- struct{}{}+ }+ for range 3 {+ <-done+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ if len(r.asked) != 9 || r.asked[1] != short {+ var order []string+ for _, s := range r.asked {+ order = append(order, s[:min(8, len(s))])+ }+ t.Fatalf("read in the order %q: want the short request second", order)+ }+}