Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

plugin: GitHub's search gets 60s a page, and a mirror is asked without the token (#3923)

yetonee4ccf67b112d
diff --git a/internal/ghtoken/ghtoken.go b/internal/ghtoken/ghtoken.go
index a91da5182..505b94d3c 100644
--- a/internal/ghtoken/ghtoken.go
+++ b/internal/ghtoken/ghtoken.go
@@ -117,6 +117,11 @@ func SettingsRefused() (time.Time, bool) {
func Do(req *http.Request, api string, send func(*http.Request) (*http.Response, error)) (*http.Response, bool, error) {
t, from := authorize(req, api)
resp, err := send(req)
+ // a mirror answers a request sent it without the token
+ // (source.Do), and its 401 isn't GitHub refusing the token
+ if t != "" && err == nil && resp.Request != nil && resp.Request.Header.Get("Authorization") == "" {
+ t = ""
+ }
if err != nil || t == "" || resp.StatusCode != http.StatusUnauthorized {
return resp, t != "" && err == nil, err
}
diff --git a/internal/ghtoken/ghtoken_test.go b/internal/ghtoken/ghtoken_test.go
index 9203e5c8c..32e726804 100644
--- a/internal/ghtoken/ghtoken_test.go
+++ b/internal/ghtoken/ghtoken_test.go
@@ -233,3 +233,34 @@ func TestSettingsTokenRefusedGoesWithoutEnvToken(t *testing.T) {
t.Errorf("Token() = %q from %q, want none", tok, from)
}
}
+
+// A mirror answers a request sent it without the token (source.Do,
+// #3923): that answer didn't carry the token, and a 401 from it isn't
+// GitHub refusing the token, which goes on being sent.
+func TestMirrorAnswerIsNotTheToken(t *testing.T) {
+ setToken(t, "ghp_settings_good", "", "")
+ for _, status := range []int{200, 401} {
+ var asked []string
+ send := func(r *http.Request) (*http.Response, error) {
+ asked = append(asked, r.Header.Get("Authorization"))
+ bare := r.Clone(r.Context())
+ bare.Header.Del("Authorization")
+ return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("{}")), Request: bare}, nil
+ }
+ req, _ := http.NewRequest("GET", "https://api.github.com/repos/o/r", nil)
+ resp, token, err := Do(req, "https://api.github.com", send)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp.Body.Close()
+ if token || resp.StatusCode != status || len(asked) != 1 {
+ t.Fatalf("%d from a mirror: token=%v, status %d, asked %q", status, token, resp.StatusCode, asked)
+ }
+ if _, ok := SettingsRefused(); ok {
+ t.Fatalf("%d from a mirror: the Settings token counted refused", status)
+ }
+ if tok, _ := Token(); tok != "ghp_settings_good" {
+ t.Fatalf("%d from a mirror: Token() = %q", status, tok)
+ }
+ }
+}
diff --git a/internal/plugin/github.go b/internal/plugin/github.go
index 1d8d07a02..cbe447289 100644
--- a/internal/plugin/github.go
+++ b/internal/plugin/github.go
@@ -58,14 +58,18 @@ var (
// kept while its last push stays the same, so asking again costs no more.
const taggedTTL = 10 * time.Minute
-// taggedWait is how long the page waits for GitHub; taggedEach bounds a
-// repository's reads (package.json, npm, its entry file), which go on past
+// taggedSearch bounds each page of GitHub's search: from mainland China
+// a page straight from api.github.com takes about 30s (MYCF711, #3923),
+// and with the search under taggedWait the list was empty there.
+// taggedWait is how long the page then waits for the repositories' reads;
+// taggedEach bounds a repository's reads (package.json, npm, its entry file), which go on past
// that wait and are kept for the next ask; taggedAtOnce repositories are
// read at once; and a list that left a repository out for want of an
// answer is asked again after taggedRetry. Through a proxy far away each
// read takes a few hundred milliseconds (larchsis on X: a list cut short
// at the wait was kept ten minutes).
var (
+ taggedSearch = 60 * time.Second
taggedWait = 8 * time.Second
taggedEach = 20 * time.Second
taggedAtOnce = 12
@@ -200,8 +204,6 @@ func TaggedAskAgain() {
// left out for want of an answer in taggedWait: it is then as last had
// it, if it had it, and missing counts those last didn't have.
func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, missing *TaggedMissing, err error) {
- c, cancel := context.WithTimeout(ctx, taggedWait)
- defer cancel()
type item struct {
FullName string `json:"full_name"`
HTMLURL string `json:"html_url"`
@@ -226,7 +228,9 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, miss
for page := 1; page*taggedPage <= 1000; page++ {
v := url.Values{"q": {"topic:" + Topic + " archived:false fork:false"}, "sort": {"stars"}, "order": {"desc"},
"per_page": {strconv.Itoa(taggedPage)}, "page": {strconv.Itoa(page)}}
- b, err := fetchJSON(c, githubAPI+"/search/repositories?"+v.Encode(), 4<<20)
+ sc, cancel := context.WithTimeout(ctx, taggedSearch)
+ b, err := fetchJSON(sc, githubAPI+"/search/repositories?"+v.Encode(), 4<<20)
+ cancel()
if err != nil {
return nil, false, nil, err
}
@@ -248,6 +252,8 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, miss
break
}
}
+ c, cancel := context.WithTimeout(ctx, taggedWait)
+ defer cancel()
before := map[string]Tagged{}
for _, t := range last {
before[strings.ToLower(t.Repo)] = t
diff --git a/internal/plugin/github_test.go b/internal/plugin/github_test.go
index f88667693..c857ec639 100644
--- a/internal/plugin/github_test.go
+++ b/internal/plugin/github_test.go
@@ -512,6 +512,51 @@ func TestTaggedReposStaleEnvTokenDropped(t *testing.T) {
}
}
+// A search page slower than taggedWait still lists (MYCF711, #3923: from
+// mainland China a page from api.github.com takes about 30s, and the
+// page's 8s wait left the list empty). taggedWait bounds the
+// repositories' reads only; a page has taggedSearch, and one that never
+// answers in it is still a failure.
+func TestTaggedReposSlowSearch(t *testing.T) {
+ var up atomic.Bool
+ var asked atomic.Int32
+ up.Store(true)
+ fakeGitHub(t, &up, &asked)
+ wait, search := taggedWait, taggedSearch
+ taggedWait = time.Second
+ t.Cleanup(func() { taggedWait, taggedSearch = wait, search })
+ h := &held{path: "/search/repositories", go_: make(chan struct{})}
+ holding.Store(h)
+ let := sync.OnceFunc(func() { close(h.go_) })
+ t.Cleanup(func() { let(); holding.Store(nil) })
+
+ // the search answers three taggedWaits on
+ time.AfterFunc(3*taggedWait, let)
+ l, err := TaggedRepos(context.Background())
+ var m *TaggedMissing
+ if err != nil && !errors.As(err, &m) {
+ t.Fatalf("a search slower than taggedWait: %v", err)
+ }
+ if asked.Load() != 1 || len(l) == 0 {
+ t.Fatalf("asked %d, %d repos", asked.Load(), len(l))
+ }
+
+ // a search that never answers fails at taggedSearch
+ let2 := make(chan struct{})
+ holding.Store(&held{path: "/search/repositories", go_: let2})
+ t.Cleanup(func() { close(let2) })
+ taggedSearch = 300 * time.Millisecond
+ taggedMu.Lock()
+ taggedL = nil
+ taggedMu.Unlock()
+ if err := os.Remove(taggedCache()); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := TaggedRepos(context.Background()); err == nil || errors.As(err, &m) {
+ t.Fatalf("a search that never answers: %v", err)
+ }
+}
+
func TestTaggedReposUnreadable(t *testing.T) {
var up atomic.Bool
var asked atomic.Int32
diff --git a/internal/source/source.go b/internal/source/source.go
index 3e2123fae..4ba394582 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -160,14 +160,34 @@ func distinct(urls ...string) []string {
// Do sends req as usual and, when the official answer fails in a way a
// mirror could answer, tries its mirrors in order. A request that carries
-// credentials is never sent to a mirror.
+// credentials is never sent to a mirror with them: GitHub's, whose only
+// one is the user's token, goes to a mirror without it (bare).
func Do(c *http.Client, req *http.Request) (*http.Response, error) {
- if !mirrorable(req) {
+ if !mirrorable(req) && !bare(req) {
return do(c, req, []string{req.URL.String()})
}
return do(c, req, URLs(req.URL.String()))
}
+// bare is whether req's one credential is an Authorization header to
+// GitHub, whose answers come without it too, at the anonymous rate: a
+// mirror is then asked without the header, and never sees the token. A
+// GitHub token in Settings turned the mirror off for every request to
+// GitHub's API (MYCF711, #3923).
+func bare(req *http.Request) bool {
+ if req.URL == nil || req.Header.Get("Authorization") == "" {
+ return false
+ }
+ switch strings.ToLower(req.URL.Hostname()) {
+ case "github.com", "api.github.com", "raw.githubusercontent.com", "codeload.github.com":
+ default:
+ return false
+ }
+ r := req.Clone(req.Context())
+ r.Header.Del("Authorization")
+ return mirrorable(r)
+}
+
// mirrorable is whether req may be sent to a mirror. Credentials stay with
// the official source: a mirror must never see a token, cookie or key.
func mirrorable(req *http.Request) bool {
@@ -207,7 +227,7 @@ func DoOfficial(c *http.Client, req *http.Request) (*http.Response, error) {
// when the 「国内镜像」 switch is on, but never to a fallback mirror: for a
// file nothing checks, which only a copy of the very file may stand in for.
func DoFaithful(c *http.Client, req *http.Request) (*http.Response, error) {
- if !mirrorable(req) {
+ if !mirrorable(req) && !bare(req) {
return DoOfficial(c, req)
}
return do(c, req, faithful(req.URL.String()))
@@ -229,6 +249,8 @@ func do(c *http.Client, req *http.Request, urls []string) (*http.Response, error
}
r.URL = u
r.Host = ""
+ // only a request bare allows has one here (Do)
+ r.Header.Del("Authorization")
if r.Body != nil {
if r.GetBody == nil {
break
diff --git a/internal/source/source_test.go b/internal/source/source_test.go
index aad018a8a..6874b339e 100644
--- a/internal/source/source_test.go
+++ b/internal/source/source_test.go
@@ -234,10 +234,12 @@ func TestDoBodyOutlivesCandidateTimeout(t *testing.T) {
type recordedTransport struct {
requests []string
+ auth []string // each request's Authorization
}
func (rt *recordedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
rt.requests = append(rt.requests, req.URL.String())
+ rt.auth = append(rt.auth, req.Header.Get("Authorization"))
status := http.StatusBadGateway
if strings.HasPrefix(req.URL.String(), "https://gh.example/") {
status = http.StatusOK
@@ -251,20 +253,55 @@ func (rt *recordedTransport) RoundTrip(req *http.Request) (*http.Response, error
}, nil
}
+// A request to GitHub whose one credential is the user's token reaches a
+// mirror without it (#3923): the official address is asked with it, the
+// mirror never sees it. Any other credential, or a host other than
+// GitHub's, keeps the request official.
func TestDoKeepsCredentialsOfficial(t *testing.T) {
t.Setenv("MAGPIE_MIRRORS", "on")
t.Setenv("MAGPIE_GITHUB_MIRROR", "https://gh.example")
- rt := &recordedTransport{}
- req, _ := http.NewRequest("GET", "https://api.github.com/repos/o/r", nil)
- req.Header.Set("Authorization", "Bearer secret")
- resp, err := Do(&http.Client{Transport: rt}, req)
- if err != nil {
- t.Fatal(err)
+ t.Setenv("MAGPIE_NPM_REGISTRY", "https://gh.example/npm")
+ send := func(raw string, h map[string]string, do func(*http.Client, *http.Request) (*http.Response, error)) (*recordedTransport, *http.Response) {
+ t.Helper()
+ rt := &recordedTransport{}
+ req, _ := http.NewRequest("GET", raw, nil)
+ for k, v := range h {
+ req.Header.Set(k, v)
+ }
+ resp, err := do(&http.Client{Transport: rt}, req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp.Body.Close()
+ if req.Header.Get("Authorization") != h["Authorization"] {
+ t.Fatalf("the caller's request lost its token: %q", req.Header.Get("Authorization"))
+ }
+ return rt, resp
}
- resp.Body.Close()
- if len(rt.requests) != 1 || rt.requests[0] != "https://api.github.com/repos/o/r" {
+
+ rt, resp := send("https://api.github.com/repos/o/r", map[string]string{"Authorization": "Bearer secret"}, Do)
+ if fmt.Sprint(rt.requests) != "[https://api.github.com/repos/o/r https://gh.example/https://api.github.com/repos/o/r]" {
t.Fatalf("requests = %q", rt.requests)
}
+ if rt.auth[0] != "Bearer secret" || rt.auth[1] != "" {
+ t.Fatalf("Authorization = %q, want the token official only", rt.auth)
+ }
+ if resp.StatusCode != http.StatusOK || resp.Request.Header.Get("Authorization") != "" {
+ t.Fatalf("answer %d, sent with %q", resp.StatusCode, resp.Request.Header.Get("Authorization"))
+ }
+
+ for _, c := range []struct {
+ raw string
+ h map[string]string
+ }{
+ {"https://api.github.com/repos/o/r", map[string]string{"Authorization": "Bearer secret", "Cookie": "s=1"}},
+ {"https://api.github.com/repos/o/r?access_token=x", map[string]string{"Authorization": "Bearer secret"}},
+ {"https://registry.npmjs.org/pkg", map[string]string{"Authorization": "Bearer secret"}},
+ } {
+ if rt, _ := send(c.raw, c.h, Do); len(rt.requests) != 1 || rt.requests[0] != c.raw {
+ t.Fatalf("%s %v: requests = %q", c.raw, c.h, rt.requests)
+ }
+ }
}
// budgetTransport answers the first (official) address only when its