Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
plugin: GitHub's search gets 60s a page, and a mirror is asked without the token (#3923)
yetonee4ccf67b112d
diff --git a/internal/ghtoken/ghtoken.go b/internal/ghtoken/ghtoken.goindex a91da5182..505b94d3c 100644--- a/internal/ghtoken/ghtoken.go+++ b/internal/ghtoken/ghtoken.go@@ -117,6 +117,11 @@ func SettingsRefused() (time.Time, bool) {func Do(req *http.Request, api string, send func(*http.Request) (*http.Response, error)) (*http.Response, bool, error) {t, from := authorize(req, api)resp, err := send(req)+ // a mirror answers a request sent it without the token+ // (source.Do), and its 401 isn't GitHub refusing the token+ if t != "" && err == nil && resp.Request != nil && resp.Request.Header.Get("Authorization") == "" {+ t = ""+ }if err != nil || t == "" || resp.StatusCode != http.StatusUnauthorized {return resp, t != "" && err == nil, err}diff --git a/internal/ghtoken/ghtoken_test.go b/internal/ghtoken/ghtoken_test.goindex 9203e5c8c..32e726804 100644--- a/internal/ghtoken/ghtoken_test.go+++ b/internal/ghtoken/ghtoken_test.go@@ -233,3 +233,34 @@ func TestSettingsTokenRefusedGoesWithoutEnvToken(t *testing.T) {t.Errorf("Token() = %q from %q, want none", tok, from)}}++// A mirror answers a request sent it without the token (source.Do,+// #3923): that answer didn't carry the token, and a 401 from it isn't+// GitHub refusing the token, which goes on being sent.+func TestMirrorAnswerIsNotTheToken(t *testing.T) {+ setToken(t, "ghp_settings_good", "", "")+ for _, status := range []int{200, 401} {+ var asked []string+ send := func(r *http.Request) (*http.Response, error) {+ asked = append(asked, r.Header.Get("Authorization"))+ bare := r.Clone(r.Context())+ bare.Header.Del("Authorization")+ return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("{}")), Request: bare}, nil+ }+ req, _ := http.NewRequest("GET", "https://api.github.com/repos/o/r", nil)+ resp, token, err := Do(req, "https://api.github.com", send)+ if err != nil {+ t.Fatal(err)+ }+ resp.Body.Close()+ if token || resp.StatusCode != status || len(asked) != 1 {+ t.Fatalf("%d from a mirror: token=%v, status %d, asked %q", status, token, resp.StatusCode, asked)+ }+ if _, ok := SettingsRefused(); ok {+ t.Fatalf("%d from a mirror: the Settings token counted refused", status)+ }+ if tok, _ := Token(); tok != "ghp_settings_good" {+ t.Fatalf("%d from a mirror: Token() = %q", status, tok)+ }+ }+}diff --git a/internal/plugin/github.go b/internal/plugin/github.goindex 1d8d07a02..cbe447289 100644--- a/internal/plugin/github.go+++ b/internal/plugin/github.go@@ -58,14 +58,18 @@ var (// kept while its last push stays the same, so asking again costs no more.const taggedTTL = 10 * time.Minute-// taggedWait is how long the page waits for GitHub; taggedEach bounds a-// repository's reads (package.json, npm, its entry file), which go on past+// taggedSearch bounds each page of GitHub's search: from mainland China+// a page straight from api.github.com takes about 30s (MYCF711, #3923),+// and with the search under taggedWait the list was empty there.+// taggedWait is how long the page then waits for the repositories' reads;+// taggedEach bounds a repository's reads (package.json, npm, its entry file), which go on past// that wait and are kept for the next ask; taggedAtOnce repositories are// read at once; and a list that left a repository out for want of an// answer is asked again after taggedRetry. Through a proxy far away each// read takes a few hundred milliseconds (larchsis on X: a list cut short// at the wait was kept ten minutes).var (+ taggedSearch = 60 * time.SecondtaggedWait = 8 * time.SecondtaggedEach = 20 * time.SecondtaggedAtOnce = 12@@ -200,8 +204,6 @@ func TaggedAskAgain() {// left out for want of an answer in taggedWait: it is then as last had// it, if it had it, and missing counts those last didn't have.func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, missing *TaggedMissing, err error) {- c, cancel := context.WithTimeout(ctx, taggedWait)- defer cancel()type item struct {FullName string `json:"full_name"`HTMLURL string `json:"html_url"`@@ -226,7 +228,9 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, missfor page := 1; page*taggedPage <= 1000; page++ {v := url.Values{"q": {"topic:" + Topic + " archived:false fork:false"}, "sort": {"stars"}, "order": {"desc"},"per_page": {strconv.Itoa(taggedPage)}, "page": {strconv.Itoa(page)}}- b, err := fetchJSON(c, githubAPI+"/search/repositories?"+v.Encode(), 4<<20)+ sc, cancel := context.WithTimeout(ctx, taggedSearch)+ b, err := fetchJSON(sc, githubAPI+"/search/repositories?"+v.Encode(), 4<<20)+ cancel()if err != nil {return nil, false, nil, err}@@ -248,6 +252,8 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, missbreak}}+ c, cancel := context.WithTimeout(ctx, taggedWait)+ defer cancel()before := map[string]Tagged{}for _, t := range last {before[strings.ToLower(t.Repo)] = tdiff --git a/internal/plugin/github_test.go b/internal/plugin/github_test.goindex f88667693..c857ec639 100644--- a/internal/plugin/github_test.go+++ b/internal/plugin/github_test.go@@ -512,6 +512,51 @@ func TestTaggedReposStaleEnvTokenDropped(t *testing.T) {}}+// A search page slower than taggedWait still lists (MYCF711, #3923: from+// mainland China a page from api.github.com takes about 30s, and the+// page's 8s wait left the list empty). taggedWait bounds the+// repositories' reads only; a page has taggedSearch, and one that never+// answers in it is still a failure.+func TestTaggedReposSlowSearch(t *testing.T) {+ var up atomic.Bool+ var asked atomic.Int32+ up.Store(true)+ fakeGitHub(t, &up, &asked)+ wait, search := taggedWait, taggedSearch+ taggedWait = time.Second+ t.Cleanup(func() { taggedWait, taggedSearch = wait, search })+ h := &held{path: "/search/repositories", go_: make(chan struct{})}+ holding.Store(h)+ let := sync.OnceFunc(func() { close(h.go_) })+ t.Cleanup(func() { let(); holding.Store(nil) })++ // the search answers three taggedWaits on+ time.AfterFunc(3*taggedWait, let)+ l, err := TaggedRepos(context.Background())+ var m *TaggedMissing+ if err != nil && !errors.As(err, &m) {+ t.Fatalf("a search slower than taggedWait: %v", err)+ }+ if asked.Load() != 1 || len(l) == 0 {+ t.Fatalf("asked %d, %d repos", asked.Load(), len(l))+ }++ // a search that never answers fails at taggedSearch+ let2 := make(chan struct{})+ holding.Store(&held{path: "/search/repositories", go_: let2})+ t.Cleanup(func() { close(let2) })+ taggedSearch = 300 * time.Millisecond+ taggedMu.Lock()+ taggedL = nil+ taggedMu.Unlock()+ if err := os.Remove(taggedCache()); err != nil {+ t.Fatal(err)+ }+ if _, err := TaggedRepos(context.Background()); err == nil || errors.As(err, &m) {+ t.Fatalf("a search that never answers: %v", err)+ }+}+func TestTaggedReposUnreadable(t *testing.T) {var up atomic.Boolvar asked atomic.Int32diff --git a/internal/source/source.go b/internal/source/source.goindex 3e2123fae..4ba394582 100644--- a/internal/source/source.go+++ b/internal/source/source.go@@ -160,14 +160,34 @@ func distinct(urls ...string) []string {// Do sends req as usual and, when the official answer fails in a way a// mirror could answer, tries its mirrors in order. A request that carries-// credentials is never sent to a mirror.+// credentials is never sent to a mirror with them: GitHub's, whose only+// one is the user's token, goes to a mirror without it (bare).func Do(c *http.Client, req *http.Request) (*http.Response, error) {- if !mirrorable(req) {+ if !mirrorable(req) && !bare(req) {return do(c, req, []string{req.URL.String()})}return do(c, req, URLs(req.URL.String()))}+// bare is whether req's one credential is an Authorization header to+// GitHub, whose answers come without it too, at the anonymous rate: a+// mirror is then asked without the header, and never sees the token. A+// GitHub token in Settings turned the mirror off for every request to+// GitHub's API (MYCF711, #3923).+func bare(req *http.Request) bool {+ if req.URL == nil || req.Header.Get("Authorization") == "" {+ return false+ }+ switch strings.ToLower(req.URL.Hostname()) {+ case "github.com", "api.github.com", "raw.githubusercontent.com", "codeload.github.com":+ default:+ return false+ }+ r := req.Clone(req.Context())+ r.Header.Del("Authorization")+ return mirrorable(r)+}+// mirrorable is whether req may be sent to a mirror. Credentials stay with// the official source: a mirror must never see a token, cookie or key.func mirrorable(req *http.Request) bool {@@ -207,7 +227,7 @@ func DoOfficial(c *http.Client, req *http.Request) (*http.Response, error) {// when the 「国内镜像」 switch is on, but never to a fallback mirror: for a// file nothing checks, which only a copy of the very file may stand in for.func DoFaithful(c *http.Client, req *http.Request) (*http.Response, error) {- if !mirrorable(req) {+ if !mirrorable(req) && !bare(req) {return DoOfficial(c, req)}return do(c, req, faithful(req.URL.String()))@@ -229,6 +249,8 @@ func do(c *http.Client, req *http.Request, urls []string) (*http.Response, error}r.URL = ur.Host = ""+ // only a request bare allows has one here (Do)+ r.Header.Del("Authorization")if r.Body != nil {if r.GetBody == nil {breakdiff --git a/internal/source/source_test.go b/internal/source/source_test.goindex aad018a8a..6874b339e 100644--- a/internal/source/source_test.go+++ b/internal/source/source_test.go@@ -234,10 +234,12 @@ func TestDoBodyOutlivesCandidateTimeout(t *testing.T) {type recordedTransport struct {requests []string+ auth []string // each request's Authorization}func (rt *recordedTransport) RoundTrip(req *http.Request) (*http.Response, error) {rt.requests = append(rt.requests, req.URL.String())+ rt.auth = append(rt.auth, req.Header.Get("Authorization"))status := http.StatusBadGatewayif strings.HasPrefix(req.URL.String(), "https://gh.example/") {status = http.StatusOK@@ -251,20 +253,55 @@ func (rt *recordedTransport) RoundTrip(req *http.Request) (*http.Response, error}, nil}+// A request to GitHub whose one credential is the user's token reaches a+// mirror without it (#3923): the official address is asked with it, the+// mirror never sees it. Any other credential, or a host other than+// GitHub's, keeps the request official.func TestDoKeepsCredentialsOfficial(t *testing.T) {t.Setenv("MAGPIE_MIRRORS", "on")t.Setenv("MAGPIE_GITHUB_MIRROR", "https://gh.example")- rt := &recordedTransport{}- req, _ := http.NewRequest("GET", "https://api.github.com/repos/o/r", nil)- req.Header.Set("Authorization", "Bearer secret")- resp, err := Do(&http.Client{Transport: rt}, req)- if err != nil {- t.Fatal(err)+ t.Setenv("MAGPIE_NPM_REGISTRY", "https://gh.example/npm")+ send := func(raw string, h map[string]string, do func(*http.Client, *http.Request) (*http.Response, error)) (*recordedTransport, *http.Response) {+ t.Helper()+ rt := &recordedTransport{}+ req, _ := http.NewRequest("GET", raw, nil)+ for k, v := range h {+ req.Header.Set(k, v)+ }+ resp, err := do(&http.Client{Transport: rt}, req)+ if err != nil {+ t.Fatal(err)+ }+ resp.Body.Close()+ if req.Header.Get("Authorization") != h["Authorization"] {+ t.Fatalf("the caller's request lost its token: %q", req.Header.Get("Authorization"))+ }+ return rt, resp}- resp.Body.Close()- if len(rt.requests) != 1 || rt.requests[0] != "https://api.github.com/repos/o/r" {++ rt, resp := send("https://api.github.com/repos/o/r", map[string]string{"Authorization": "Bearer secret"}, Do)+ if fmt.Sprint(rt.requests) != "[https://api.github.com/repos/o/r https://gh.example/https://api.github.com/repos/o/r]" {t.Fatalf("requests = %q", rt.requests)}+ if rt.auth[0] != "Bearer secret" || rt.auth[1] != "" {+ t.Fatalf("Authorization = %q, want the token official only", rt.auth)+ }+ if resp.StatusCode != http.StatusOK || resp.Request.Header.Get("Authorization") != "" {+ t.Fatalf("answer %d, sent with %q", resp.StatusCode, resp.Request.Header.Get("Authorization"))+ }++ for _, c := range []struct {+ raw string+ h map[string]string+ }{+ {"https://api.github.com/repos/o/r", map[string]string{"Authorization": "Bearer secret", "Cookie": "s=1"}},+ {"https://api.github.com/repos/o/r?access_token=x", map[string]string{"Authorization": "Bearer secret"}},+ {"https://registry.npmjs.org/pkg", map[string]string{"Authorization": "Bearer secret"}},+ } {+ if rt, _ := send(c.raw, c.h, Do); len(rt.requests) != 1 || rt.requests[0] != c.raw {+ t.Fatalf("%s %v: requests = %q", c.raw, c.h, rt.requests)+ }+ }}// budgetTransport answers the first (official) address only when its