Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

plugins: a GitHub list that couldn't be read says why with Try again, instead of the section vanishing (till on Discord)

yetonefb101b0d573e
diff --git a/internal/gui/assets/i18n.js b/internal/gui/assets/i18n.js
index ebb23ea5e..a72a13a2e 100644
--- a/internal/gui/assets/i18n.js
+++ b/internal/gui/assets/i18n.js
@@ -3497,6 +3497,9 @@ const I18N = {
"Unofficial, on GitHub": "非官方插件 · GitHub",
"Looking on GitHub": "正在 GitHub 上查找",
"Couldn't ask GitHub: {error}": "无法连接 GitHub:{error}",
+ "{n} more on GitHub couldn't be read: {error}": "GitHub 上还有 {n} 个插件没能读取:{error}",
+ "GitHub found {n} repositories, but none of them could be read: {error}": "GitHub 找到了 {n} 个仓库,但一个都没能读取:{error}",
+ "Couldn't ask GitHub, so these are from before: {error}": "无法连接 GitHub,下面是之前获取的列表:{error}",
"Kind of plugin": "插件类型",
"Installed or not": "是否已安装",
"Not installed": "未安装",
@@ -7193,6 +7196,9 @@ const I18N = {
"Unofficial, on GitHub": "非官方外掛 · GitHub",
"Looking on GitHub": "正在 GitHub 上尋找",
"Couldn't ask GitHub: {error}": "無法連線 GitHub:{error}",
+ "{n} more on GitHub couldn't be read: {error}": "GitHub 上還有 {n} 個外掛未能讀取:{error}",
+ "GitHub found {n} repositories, but none of them could be read: {error}": "GitHub 找到了 {n} 個儲存庫,但一個都未能讀取:{error}",
+ "Couldn't ask GitHub, so these are from before: {error}": "無法連線 GitHub,以下是先前取得的列表:{error}",
"Kind of plugin": "外掛類型",
"Installed or not": "是否已安裝",
"Not installed": "未安裝",
@@ -10911,6 +10917,9 @@ const I18N = {
"Unofficial, on GitHub": "非公式 · GitHub",
"Looking on GitHub": "GitHub を検索中",
"Couldn't ask GitHub: {error}": "GitHub に接続できませんでした:{error}",
+ "{n} more on GitHub couldn't be read: {error}": "GitHub 上のほか {n} 件を読み込めませんでした:{error}",
+ "GitHub found {n} repositories, but none of them could be read: {error}": "GitHub で {n} 件のリポジトリが見つかりましたが、どれも読み込めませんでした:{error}",
+ "Couldn't ask GitHub, so these are from before: {error}": "GitHub に接続できないため、以前の一覧を表示しています:{error}",
"Kind of plugin": "プラグインの種類",
"Installed or not": "インストール済みかどうか",
"Not installed": "未インストール",
@@ -14642,6 +14651,9 @@ const I18N = {
"Unofficial, on GitHub": "Inoffiziell, auf GitHub",
"Looking on GitHub": "Suche auf GitHub",
"Couldn't ask GitHub: {error}": "GitHub nicht erreichbar: {error}",
+ "{n} more on GitHub couldn't be read: {error}": "{n} weitere auf GitHub konnten nicht gelesen werden: {error}",
+ "GitHub found {n} repositories, but none of them could be read: {error}": "GitHub hat {n} Repositories gefunden, aber keines konnte gelesen werden: {error}",
+ "Couldn't ask GitHub, so these are from before: {error}": "GitHub nicht erreichbar, dies ist die Liste von zuvor: {error}",
"Kind of plugin": "Art des Plugins",
"Installed or not": "Installiert oder nicht",
"Not installed": "Nicht installiert",
diff --git a/internal/gui/assets/plugins.css b/internal/gui/assets/plugins.css
index 389afc265..4b5c6cd93 100644
--- a/internal/gui/assets/plugins.css
+++ b/internal/gui/assets/plugins.css
@@ -50,6 +50,10 @@
.pm-ghbar .segs { flex: none; max-width: 100%; }
.pm-ghfind { width: min(240px, 100%); }
.pm-ghnone { display: flex; flex-direction: column; align-items: center; gap: 6px; padding: 22px 12px; border-radius: 12px; border: 1px dashed var(--line); font-size: 12px; color: var(--muted); text-align: center; }
+/* what GitHub couldn't give, over the cards it could: a quiet line, its
+ reason wrapping anywhere (a URL in it is one long word) */
+.pm-ghwarn { display: flex; align-items: baseline; flex-wrap: wrap; gap: 2px 10px; font-size: 12px; color: var(--muted); }
+.pm-ghwarn > span, .pm-ghnone > span { overflow-wrap: anywhere; }
.pm-logo { flex: none; width: 36px; height: 36px; border-radius: 10px; display: grid; place-items: center; background: var(--pill); color: var(--fg-2); }
.pm-logo .ic { width: 20px; height: 20px; }
diff --git a/internal/gui/assets/plugins.js b/internal/gui/assets/plugins.js
index 36f627569..0abb82bdb 100644
--- a/internal/gui/assets/plugins.js
+++ b/internal/gui/assets/plugins.js
@@ -14,7 +14,8 @@
let mine = null; // /api/plugins: what's installed
let listings = null; // /api/plugins/listings: the plugins suggested
let tagged = null; // /api/plugins/github: repositories their authors tagged magpie-plugin, as listings
- let taggedErr = ""; // why they couldn't be asked for, while none were had
+ let taggedErr = ""; // why the list isn't GitHub's whole answer: it couldn't be asked, or...
+ let taggedMissing = 0; // ...this many repositories it named couldn't be read
let topic = "magpie-plugin";
const npm = {}; // /api/plugins/npm: package → what npm says of it
const heads = {}; // /api/plugins/git: a git plugin's spec → the commit its repository is at
@@ -170,14 +171,22 @@
summary: r.description ? { en: r.description } : undefined,
npm: { version: r.version || "", publisher: r.owner, repository: r.url, license: r.license, weekly: 0 },
});
- async function loadTagged() {
+ // A failed ask is said, with why, whether or not a list was had before:
+ // an empty list was read as there being no unofficial plugins (till on
+ // Discord: GitHub unreachable from their desktop, the section gone).
+ // A list had before stands through a failed ask that brings none.
+ // again: Try again, which has magpie ask GitHub at once.
+ async function loadTagged(again) {
try {
- const r = await api("plugins/github");
- tagged = (r.repos || []).map(ghListing);
- taggedErr = "";
+ const r = await api("plugins/github" + (again ? "?again=1" : ""));
+ const got = (r.repos || []).map(ghListing);
+ if (!(r.error && !got.length && tagged?.length)) tagged = got;
+ taggedErr = r.error || "";
+ taggedMissing = r.missing || 0;
if (r.topic) topic = r.topic;
} catch (e) {
- if (!tagged) taggedErr = e.message || String(e);
+ taggedErr = e.message || String(e);
+ taggedMissing = 0;
tagged = tagged || [];
}
}
@@ -644,6 +653,10 @@
replaceKeeping(ghSec, [ghHead, skeleton(3)]);
return ghSec;
}
+ // why the list isn't GitHub's whole answer; kept: cards are shown
+ const ghWhy = (kept) => taggedMissing
+ ? t(kept ? "{n} more on GitHub couldn't be read: {error}" : "GitHub found {n} repositories, but none of them could be read: {error}", { n: taggedMissing, error: taggedErr })
+ : t(kept ? "Couldn't ask GitHub, so these are from before: {error}" : "Couldn't ask GitHub: {error}", { error: taggedErr });
// and one GitHub couldn't be asked of says so, with another go
function ghFailed() {
ghHead.replaceChildren(...ghTitle());
@@ -654,13 +667,28 @@
again.onclick = () => {
tagged = null;
taggedErr = "";
+ taggedMissing = 0;
drawBody();
- loadTagged().then(drawBody);
+ loadTagged(true).then(drawBody);
};
- box.append(el("span", "", t("Couldn't ask GitHub: {error}", { error: taggedErr })), again);
+ box.append(el("span", "", ghWhy(false)), again);
replaceKeeping(ghSec, [ghHead, box]);
return ghSec;
}
+ // the cards GitHub could give, and a line saying what it couldn't, with
+ // another go that keeps them until it answers
+ function ghNote() {
+ const note = el("div", "pm-ghwarn");
+ note.setAttribute("role", "status");
+ const again = el("button", "text", t("Try again"));
+ again.type = "button";
+ again.onclick = () => {
+ again.disabled = true;
+ loadTagged(true).then(drawBody);
+ };
+ note.append(el("span", "", ghWhy(true)), again);
+ return note;
+ }
function ghSection(list) {
ghList = list;
ghHead.replaceChildren(...ghTitle());
@@ -677,9 +705,10 @@
return ghSec;
}
function drawGh() {
+ const head = taggedErr ? [ghHead, ghNote()] : [ghHead];
if ((tagged || []).length <= 6) {
ghGrid.replaceChildren(...ghList.map(card));
- replaceKeeping(ghSec, [ghHead, ghGrid]);
+ replaceKeeping(ghSec, [...head, ghGrid]);
return;
}
// the kinds there are, in the market's order, each with how many the
@@ -724,7 +753,7 @@
const shown = found.filter((l) => (!ghKind || kindOf(l) === ghKind) && (!ghHave || have(l) === (ghHave === "have")));
if (shown.length) {
ghGrid.replaceChildren(...shown.map(card));
- replaceKeeping(ghSec, [ghHead, ghBar, ghGrid]);
+ replaceKeeping(ghSec, [...head, ghBar, ghGrid]);
return;
}
const none = el("div", "pm-ghnone");
@@ -732,7 +761,7 @@
all.type = "button";
all.onclick = () => { ghKind = ""; ghHave = ""; ghQuery = ""; drawGh(); ghQ.focus(); };
none.append(el("span", "", t("No plugin on GitHub matches")), all);
- replaceKeeping(ghSec, [ghHead, ghBar, none]);
+ replaceKeeping(ghSec, [...head, ghBar, none]);
}
// the head and its search field are made once and drawn again in
@@ -886,7 +915,7 @@
if (pgs.length) out.push(section(t("Pages"), t("pages of their own in magpie's window, each shut off from the rest and reading only what it lists"), pgs));
if (gh.length) out.push(ghSection(gh));
else if (!tagged) out.push(ghWaiting());
- else if (taggedErr) out.push(ghFailed());
+ else if (taggedErr && !tagged.length) out.push(ghFailed());
out.push(manual());
replaceKeeping(body, out);
return;
@@ -894,6 +923,7 @@
if (ls.length) out.push(section(t("Suggested"), "", ls));
if (gh.length) out.push(ghSection(gh));
else if (!tagged) out.push(ghWaiting());
+ else if (taggedErr && !tagged.length) out.push(ghFailed());
const known = new Set([...listings.map((l) => l.package), ...(tagged || []).map((l) => l.github.package).filter(Boolean)]);
const box = el("section", "pm-sec");
const h = el("div", "pm-sechead");
diff --git a/internal/gui/plugins.go b/internal/gui/plugins.go
index e10c18db6..d36829d47 100644
--- a/internal/gui/plugins.go
+++ b/internal/gui/plugins.go
@@ -3,6 +3,7 @@ package gui
import (
"context"
"encoding/json"
+ "errors"
"io"
"net/http"
"regexp"
@@ -495,9 +496,15 @@ func pluginRoutes(mux *http.ServeMux, w Windows) {
// repositories on GitHub tagged magpie-plugin: nobody's list, shown
// apart as not reviewed, installed from the repository
mux.HandleFunc("GET /api/plugins/github", func(rw http.ResponseWriter, r *http.Request) {
+ // Try again asks GitHub at once when the list standing isn't its
+ // whole answer
+ if r.URL.Query().Get("again") != "" {
+ plugin.TaggedAskAgain()
+ }
+ got, err := plugin.TaggedRepos(r.Context())
// each one's own picture as the page can show it, kept here as an
// installed plugin's is (a data URI isn't sent on to the page)
- repos := append([]plugin.Tagged(nil), plugin.TaggedRepos(r.Context())...)
+ repos := append([]plugin.Tagged(nil), got...)
said := make([]string, len(repos))
for i := range repos {
said[i] = repos[i].Icon
@@ -505,7 +512,17 @@ func pluginRoutes(mux *http.ServeMux, w Windows) {
for i, ic := range provider.RepoIcons(said, 3*time.Second) {
repos[i].Icon = ic
}
- writeJSON(rw, map[string]any{"repos": repos, "topic": plugin.Topic})
+ out := map[string]any{"repos": repos, "topic": plugin.Topic}
+ // why the list isn't GitHub's whole answer: the page says so with
+ // Try again, rather than the section reading as there being none
+ if err != nil {
+ out["error"] = err.Error()
+ var m *plugin.TaggedMissing
+ if errors.As(err, &m) {
+ out["error"], out["missing"] = m.Why.Error(), m.Missing
+ }
+ }
+ writeJSON(rw, out)
})
// the commit each installed git plugin's repository is at now (asked
// at most every ten minutes), by spec; one that didn't answer is left
diff --git a/internal/gui/tests/plugin-github-wait.test.cjs b/internal/gui/tests/plugin-github-wait.test.cjs
index a64eee21f..927d30e03 100644
--- a/internal/gui/tests/plugin-github-wait.test.cjs
+++ b/internal/gui/tests/plugin-github-wait.test.cjs
@@ -7,7 +7,12 @@
// place, before the field, with its head, a word that it is looking and
// cards still to come; let go, its cards come in where those were and the
// word goes. A search while it waits shows the same. When the ask fails,
-// the section says so with Try again, which asks again. English, Chinese,
+// the section says so with Try again, which asks again. magpie answering
+// with an error beside the list (GitHub unreachable from the reader's
+// desktop: till on Discord, where the section just wasn't there) is said
+// too: with no list, in the section's place with why and Try again, which
+// has magpie ask GitHub at once (?again=1); with the cards it could give,
+// a line over them saying how many more couldn't be read, and why. English, Chinese,
// Japanese and German, wide and 440px, in Chromium and WebKit; nothing
// scrolls on a click. MAGPIE_WAIT_ASSETS names an older assets directory,
// to see this fail on the code before it.
@@ -28,8 +33,9 @@ const LISTINGS = [
];
// gh: how /api/plugins/github answers each time it is asked, in turn: a
-// promise held until let go, or "fail"
-function server(lang, gh) {
+// promise held until let go, "fail", or { json } as magpie's answer;
+// urls: each ask's URL
+function server(lang, gh, urls = []) {
let asked = 0;
return async (route) => {
const url = new URL(route.request().url());
@@ -43,6 +49,8 @@ function server(lang, gh) {
if (url.pathname === "/api/plugins/listings") return json({ listings: LISTINGS });
if (url.pathname === "/api/plugins/github") {
const next = gh[asked++] ?? gh.at(-1);
+ urls.push(url.pathname + url.search);
+ if (next?.json) return json(next.json);
if (next === "fail") return route.fulfill({ status: 502, contentType: "text/plain", body: "GitHub didn't answer" });
await next;
return json({ repos: REPOS, topic: "magpie-plugin" }).catch(() => {});
@@ -57,10 +65,10 @@ function server(lang, gh) {
}
const L = {
- en: { section: "Unofficial, on GitHub", look: "Looking on GitHub", failed: /Couldn't ask GitHub/, again: "Try again" },
- zh: { section: "非官方插件 · GitHub", look: "正在 GitHub 上查找", failed: /无法连接 GitHub/, again: "重试" },
- ja: { section: "非公式 · GitHub", look: "GitHub を検索中", failed: /GitHub に接続できませんでした/, again: "再試行" },
- de: { section: "Inoffiziell, auf GitHub", look: "Suche auf GitHub", failed: /GitHub nicht erreichbar/, again: "Erneut versuchen" },
+ en: { section: "Unofficial, on GitHub", look: "Looking on GitHub", failed: /Couldn't ask GitHub/, again: "Try again", none: /GitHub found 12 repositories, but none of them could be read/, more: /12 more on GitHub couldn't be read/ },
+ zh: { section: "非官方插件 · GitHub", look: "正在 GitHub 上查找", failed: /无法连接 GitHub/, again: "重试", none: /GitHub 找到了 12 个仓库,但一个都没能读取/, more: /GitHub 上还有 12 个插件没能读取/ },
+ ja: { section: "非公式 · GitHub", look: "GitHub を検索中", failed: /GitHub に接続できませんでした/, again: "再試行", none: /GitHub で 12 件のリポジトリが見つかりましたが/, more: /GitHub 上のほか 12 件を読み込めませんでした/ },
+ de: { section: "Inoffiziell, auf GitHub", look: "Suche auf GitHub", failed: /GitHub nicht erreichbar/, again: "Erneut versuchen", none: /GitHub hat 12 Repositories gefunden/, more: /12 weitere auf GitHub konnten nicht gelesen werden/ },
};
const shot = async (loc, name) => {
@@ -78,13 +86,13 @@ for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium",
for (const width of [980, 440]) {
await t.test(lang + " " + width, async () => {
const w = L[lang];
- const open = async (gh) => {
+ const open = async (gh, urls) => {
const ctx = await browser.newContext({ viewport: { width, height: 860 } });
const page = await ctx.newPage();
page.setDefaultTimeout(5000);
const errors = [];
page.on("pageerror", (e) => errors.push(e.message));
- await page.route("**/*", server(lang, gh));
+ await page.route("**/*", server(lang, gh, urls));
await page.addInitScript(() => { try { localStorage.setItem("magpie.pluginTab", "discover"); } catch {} });
await page.goto("http://magpie.test/?view=plugins");
const view = page.locator("#view-plugins");
@@ -152,6 +160,72 @@ for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium",
await f.sec.locator(".pm-card:not(.ghost)").first().waitFor();
assert.equal(await f.sec.locator(".pm-ghnone").count(), 0);
assert.deepEqual(f.errors, []);
+
+ // magpie answers, but GitHub couldn't give it the list: said in
+ // the section's place with why, never the section gone
+ const why = 'http://raw.githubusercontent.com/a/b/main/package.json: Get "https://raw.githubusercontent.com/a/b/main/package.json": dial tcp: i/o timeout';
+ const urls = [];
+ const later = held();
+ const e = await open([{ json: { repos: [], topic: "magpie-plugin", error: why, missing: 12 } }, later], urls);
+ t.after(() => e.ctx.close());
+ await e.sec.locator(".pm-ghnone").waitFor();
+ const said = await e.sec.locator(".pm-ghnone").innerText();
+ assert.match(said, w.none);
+ assert.ok(said.includes("i/o timeout"), said);
+ // its reason wraps: nothing spills at this width
+ assert.ok(await e.sec.evaluate((s) => s.scrollWidth <= s.clientWidth + 1), "the reason fits");
+ await shot(e.sec, `unread-${engine}-${lang}-${width}`);
+ // the search field's results say it too
+ const eq = e.view.locator(".pm-head .pm-find input");
+ await eq.fill("claude");
+ await e.sec.locator(".pm-ghnone").waitFor();
+ await eq.fill("");
+ await e.sec.locator(".pm-ghnone").waitFor();
+ // Try again has magpie ask GitHub at once (wheeled into sight
+ // first, clear of the window's footer, as a reader would)
+ const eAgain = e.sec.getByRole("button", { name: w.again });
+ await e.page.mouse.move(width / 2, 300);
+ for (let i = 0; i < 4; i++) await e.page.mouse.wheel(0, 400);
+ await eAgain.click();
+ await e.sec.locator(".pm-ghlook").waitFor();
+ later.go();
+ await e.sec.locator(".pm-card:not(.ghost)").first().waitFor();
+ assert.deepEqual(urls, ["/api/plugins/github", "/api/plugins/github?again=1"]);
+ assert.equal(await e.sec.locator(".pm-ghwarn, .pm-ghnone").count(), 0);
+ assert.deepEqual(e.errors, []);
+
+ // some came, some couldn't be read: the cards, and a line over
+ // them saying so with Try again, which keeps the cards meanwhile
+ const urls2 = [];
+ const later2 = held();
+ const m = await open([{ json: { repos: REPOS, topic: "magpie-plugin", error: why, missing: 12 } }, later2], urls2);
+ t.after(() => m.ctx.close());
+ await m.sec.locator(".pm-ghwarn").waitFor();
+ assert.match(await m.sec.locator(".pm-ghwarn").innerText(), w.more);
+ assert.equal(await m.sec.locator(".pm-card:not(.ghost)").count(), 2);
+ assert.ok(await m.sec.evaluate((s) => s.scrollWidth <= s.clientWidth + 1), "the line fits");
+ await shot(m.sec, `partial-${engine}-${lang}-${width}`);
+ const mAgain = m.sec.locator(".pm-ghwarn").getByRole("button", { name: w.again });
+ await m.page.mouse.move(width / 2, 300);
+ for (let i = 0; i < 4; i++) await m.page.mouse.wheel(0, 400);
+ await mAgain.click();
+ assert.equal(await m.sec.locator(".pm-card:not(.ghost)").count(), 2, "cards kept while asking");
+ later2.go();
+ await m.sec.locator(".pm-ghwarn").waitFor({ state: "detached" });
+ assert.equal(await m.sec.locator(".pm-card:not(.ghost)").count(), 2);
+ assert.deepEqual(urls2, ["/api/plugins/github", "/api/plugins/github?again=1"]);
+ assert.deepEqual(m.errors, []);
+
+ // a failed ask after a list was had: the list stands, and the
+ // line says it is from before
+ const k = await open([{ json: { repos: REPOS, topic: "magpie-plugin" } }, { json: { repos: [], topic: "magpie-plugin", error: "GitHub's search: 403 rate limit" } }]);
+ t.after(() => k.ctx.close());
+ await k.sec.locator(".pm-card:not(.ghost)").first().waitFor();
+ await k.page.evaluate(() => window.loadPlugins());
+ await k.sec.locator(".pm-ghwarn").waitFor();
+ assert.match(await k.sec.locator(".pm-ghwarn").innerText(), /403 rate limit/);
+ assert.equal(await k.sec.locator(".pm-card:not(.ghost)").count(), 2);
+ assert.deepEqual(k.errors, []);
});
}
}
diff --git a/internal/plugin/github.go b/internal/plugin/github.go
index 6d73d97ea..5d8dace88 100644
--- a/internal/plugin/github.go
+++ b/internal/plugin/github.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"net/url"
"os"
"path"
@@ -78,6 +79,10 @@ var (
taggedMu sync.Mutex
taggedL []Tagged
taggedAt time.Time
+ // taggedWhy is why taggedL isn't GitHub's whole answer: the search
+ // failed (taggedL is what it said before), or repositories it named
+ // couldn't be read. nil when it is.
+ taggedWhy error
// owner/repo@pushed → what its package.json and npm made of it, nil
// when it was left out
readMu sync.Mutex
@@ -96,6 +101,20 @@ func taggedOnDisk() []Tagged {
return l
}
+// TaggedMissing is the error TaggedRepos gives when GitHub's search
+// answered but repositories it named couldn't be read: Missing of them
+// aren't in the list, for Why.
+type TaggedMissing struct {
+ Missing int
+ Why error
+}
+
+func (e *TaggedMissing) Error() string {
+ return fmt.Sprintf("%d repositories tagged %s couldn't be read from GitHub: %v", e.Missing, Topic, e.Why)
+}
+
+func (e *TaggedMissing) Unwrap() error { return e.Why }
+
// TaggedRepos are the repositories tagged with Topic, the most starred
// first: GitHub asked at most every ten minutes (its search lets 10 a minute
// through without a token), else what it said last, kept on disk. Archived
@@ -104,22 +123,29 @@ func taggedOnDisk() []Tagged {
// couldn't install either. A repository whose package.json didn't come in
// time is as the list had it last, and the list is asked again in
// taggedRetry. MAGPIE_PLUGIN_MARKET=off keeps to none.
-func TaggedRepos(ctx context.Context) []Tagged {
+//
+// The error says the list isn't GitHub's whole answer, and why: the search
+// failed (the list is what GitHub said before, empty when it never
+// answered), or, a *TaggedMissing, repositories it named couldn't be read.
+// An empty list is never a failed read passed off as there being none
+// (till on Discord: GitHub unreachable from their desktop, and the
+// section just wasn't there).
+func TaggedRepos(ctx context.Context) ([]Tagged, error) {
taggedMu.Lock()
defer taggedMu.Unlock()
if taggedL != nil && time.Since(taggedAt) < taggedTTL {
- return taggedL
+ return taggedL, taggedWhy
}
if os.Getenv("MAGPIE_PLUGIN_MARKET") == "off" {
- return []Tagged{}
+ return []Tagged{}, nil
}
// a magpie started again within the ten minutes: what GitHub said then,
// without the wait of asking (a package.json read for each)
if taggedL == nil {
if fi, err := os.Stat(taggedCache()); err == nil && time.Since(fi.ModTime()) < taggedTTL {
if l := taggedOnDisk(); l != nil {
- taggedL, taggedAt = l, fi.ModTime()
- return l
+ taggedL, taggedAt, taggedWhy = l, fi.ModTime(), nil
+ return l, nil
}
}
}
@@ -127,33 +153,49 @@ func TaggedRepos(ctx context.Context) []Tagged {
if last == nil {
last = taggedOnDisk()
}
- if l, whole, err := askTagged(ctx, last); err == nil {
- taggedL, taggedAt = l, time.Now()
+ if l, whole, missing, err := askTagged(ctx, last); err == nil {
+ taggedL, taggedAt, taggedWhy = l, time.Now(), nil
if !whole {
taggedAt = time.Now().Add(-taggedTTL + taggedRetry)
}
+ if missing != nil {
+ taggedWhy = missing
+ }
if b, err := json.Marshal(l); err == nil {
_ = os.MkdirAll(filepath.Dir(taggedCache()), 0o755)
_ = os.WriteFile(taggedCache(), b, 0o644)
// a magpie started again meanwhile asks again too
_ = os.Chtimes(taggedCache(), taggedAt, taggedAt)
}
- return l
+ return l, taggedWhy
+ } else {
+ // GitHub didn't answer: what it said last, asked again in a
+ // minute, and why it is that
+ l := last
+ if l == nil {
+ l = []Tagged{}
+ }
+ taggedL, taggedAt, taggedWhy = l, time.Now().Add(-taggedTTL+time.Minute), err
+ return l, err
}
- // GitHub didn't answer: what it said last, asked again in a minute
- l := last
- if l == nil {
- l = []Tagged{}
+}
+
+// TaggedAskAgain has the next TaggedRepos ask GitHub at once, when what
+// stands now isn't its whole answer: the Plugins page's Try again. A list
+// GitHub gave whole stands its ten minutes all the same.
+func TaggedAskAgain() {
+ taggedMu.Lock()
+ defer taggedMu.Unlock()
+ if taggedWhy != nil {
+ taggedAt = time.Time{}
}
- taggedL, taggedAt = l, time.Now().Add(-taggedTTL+time.Minute)
- return l
}
// askTagged asks GitHub's search, and reads each repository it names
// that wasn't read at its last push. whole is false when a repository was
// left out for want of an answer in taggedWait: it is then as last had
-// it, if it had it.
-func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, err error) {
+// it, if it had it, and missing counts those last didn't have.
+func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, missing *TaggedMissing, err error) {
c, cancel := context.WithTimeout(ctx, taggedWait)
defer cancel()
type item struct {
@@ -182,14 +224,14 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, err
"per_page": {strconv.Itoa(taggedPage)}, "page": {strconv.Itoa(page)}}
b, err := fetchJSON(c, githubAPI+"/search/repositories?"+v.Encode(), 4<<20)
if err != nil {
- return nil, false, err
+ return nil, false, nil, err
}
var p struct {
Total int `json:"total_count"`
Items []item `json:"items"`
}
if err := json.Unmarshal(b, &p); err != nil {
- return nil, false, err
+ return nil, false, nil, err
}
for _, it := range p.Items {
// a repository starred between two pages can show on both
@@ -211,8 +253,9 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, err
r.URL, r.Owner, r.OwnerAvatar, r.Description, r.Stars, r.License, r.Pushed = t.URL, t.Owner, t.OwnerAvatar, t.Description, t.Stars, t.License, t.Pushed
return r
}
- var mu sync.Mutex // out, keep, done; late once the page stops waiting
+ var mu sync.Mutex // out, keep, done, why; late once the page stops waiting
late := false
+ var why error // a read GitHub or npm didn't answer, the last one
out := make([]Tagged, len(r.Items))
keep := make([]bool, len(r.Items))
done := make([]bool, len(r.Items))
@@ -255,7 +298,7 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, err
defer func() { <-sem }()
pc, cancel := context.WithTimeout(bg, taggedEach)
defer cancel()
- got, ok, read := readTagged(pc, t, branch)
+ got, ok, read, rerr := readTagged(pc, t, branch)
if read {
readMu.Lock()
if ok {
@@ -270,6 +313,9 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, err
out[i], keep[i] = got, ok
// a package.json GitHub didn't answer for: not known
done[i] = ok || read
+ if !done[i] && rerr != nil {
+ why = rerr
+ }
}
mu.Unlock()
}(i, t, it.DefaultBranch)
@@ -292,6 +338,7 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, err
}
readMu.Unlock()
l, whole = []Tagged{}, true
+ gone := 0
for i := range out {
switch {
case keep[i]:
@@ -300,23 +347,37 @@ func askTagged(ctx context.Context, last []Tagged) (l []Tagged, whole bool, err
whole = false
if was, ok := before[strings.ToLower(asked[i].Repo)]; ok {
l = append(l, fresh(was, asked[i]))
+ } else {
+ gone++
}
}
}
- return l, whole, nil
+ if gone > 0 {
+ mu.Lock()
+ w := why
+ mu.Unlock()
+ if w == nil {
+ w = fmt.Errorf("no answer in %s", taggedWait)
+ }
+ missing = &TaggedMissing{Missing: gone, Why: w}
+ }
+ return l, whole, missing, nil
}
// readTagged reads a tagged repository's package.json (on branch), and
// npm's copy of the package it names: t as it is installed, whether it is
// kept, and whether that is known (GitHub or npm not answering is not,
-// and it is read again next time).
-func readTagged(pc context.Context, t Tagged, branch string) (Tagged, bool, bool) {
+// and it is read again next time), with the read that didn't answer.
+func readTagged(pc context.Context, t Tagged, branch string) (Tagged, bool, bool, error) {
if branch == "" {
branch = "HEAD"
}
pb, err := fetchJSON(pc, githubRaw+"/"+t.Repo+"/"+url.PathEscape(branch)+"/package.json", 2<<20)
if err != nil {
- return t, false, errors.Is(err, errNotFound)
+ if errors.Is(err, errNotFound) {
+ return t, false, true, nil
+ }
+ return t, false, false, err
}
var pj struct {
Name string `json:"name"`
@@ -333,7 +394,7 @@ func readTagged(pc context.Context, t Tagged, branch string) (Tagged, bool, bool
} `json:"magpie"`
}
if json.Unmarshal(pb, &pj) != nil || !pkgName.MatchString(pj.Name) {
- return t, false, true
+ return t, false, true, nil
}
t.Package, t.Version = pj.Name, pj.Version
// a picture, as host.js's iconOf takes one: magpie checks and
@@ -354,6 +415,10 @@ func readTagged(pc context.Context, t Tagged, branch string) (Tagged, bool, bool
// built only to publish), so it is installed from npm
b, err := fetchJSON(pc, npmRegistry+"/"+npmPath(pj.Name)+"/latest", 1<<20)
known := err == nil || errors.Is(err, errNotFound)
+ var unknown error
+ if !known {
+ unknown = err
+ }
// npmNames is whether npm's copy names what it loads (main,
// exports); one that names nothing loads an index file
npmNames := false
@@ -387,12 +452,14 @@ func readTagged(pc context.Context, t Tagged, branch string) (Tagged, bool, bool
if path.Ext(entry) != "" {
_, err := fetchJSON(pc, githubRaw+"/"+t.Repo+"/"+url.PathEscape(branch)+"/"+strings.TrimPrefix(path.Clean("/"+entry), "/"), 1)
if errors.Is(err, errNotFound) {
- return t, false, known
+ return t, false, known, unknown
+ }
+ if err != nil {
+ known, unknown = false, err
}
- known = known && err == nil
}
}
- return t, true, known
+ return t, true, known, unknown
}
// pkgEntry is the file importing a package loads, as Bun resolves it: its
diff --git a/internal/plugin/github_test.go b/internal/plugin/github_test.go
index 49729024e..bc79fb622 100644
--- a/internal/plugin/github_test.go
+++ b/internal/plugin/github_test.go
@@ -3,6 +3,7 @@ package plugin
import (
"context"
"encoding/json"
+ "errors"
"net/http"
"net/http/httptest"
"net/url"
@@ -142,6 +143,11 @@ func fakeGitHub(t *testing.T, up *atomic.Bool, asked *atomic.Int32) {
// Every page of the search is read: with a page short of what is tagged,
// the list is the same as from one page, in GitHub's order (HoGee_xxl on X:
// 59 tagged, and the 9 past GitHub's first page of 50 never showed).
+func taggedList(ctx context.Context) []Tagged {
+ l, _ := TaggedRepos(ctx)
+ return l
+}
+
func TestTaggedReposEveryPage(t *testing.T) {
var up atomic.Bool
var asked atomic.Int32
@@ -152,7 +158,7 @@ func TestTaggedReposEveryPage(t *testing.T) {
t.Cleanup(func() { taggedPage = was })
var repos []string
- for _, x := range TaggedRepos(context.Background()) {
+ for _, x := range taggedList(context.Background()) {
repos = append(repos, x.Repo)
}
want := "rynfar/meridian griffinmartin/opencode-claude-auth slkiser/opencode-quota someone/word-guard someone/agent-aider someone/page-glance someone/flaky-entry"
@@ -172,7 +178,7 @@ func TestTaggedRepos(t *testing.T) {
fakeGitHub(t, &up, &asked)
ctx := context.Background()
- l := TaggedRepos(ctx)
+ l := taggedList(ctx)
var repos []string
for _, x := range l {
repos = append(repos, x.Repo)
@@ -231,7 +237,7 @@ func TestTaggedRepos(t *testing.T) {
t.Fatalf("%s isn't a spec Add installs from git", x.Spec)
}
- TaggedRepos(ctx)
+ taggedList(ctx)
if n := asked.Load(); n != 1 {
t.Fatalf("asked %d times in ten minutes", n)
}
@@ -243,7 +249,7 @@ func TestTaggedRepos(t *testing.T) {
taggedAt = time.Now().Add(-taggedTTL - time.Second)
taggedMu.Unlock()
before := reads.Load()
- if l := TaggedRepos(ctx); len(l) != 7 || asked.Load() != 2 {
+ if l := taggedList(ctx); len(l) != 7 || asked.Load() != 2 {
t.Fatalf("ten minutes on: asked %d times, %d repos", asked.Load(), len(l))
}
if n := reads.Load() - before; n != 3 {
@@ -258,7 +264,7 @@ func TestTaggedRepos(t *testing.T) {
if err := os.Chtimes(taggedCache(), time.Now(), time.Now()); err != nil {
t.Fatal(err)
}
- if l := TaggedRepos(ctx); len(l) != 7 || asked.Load() != 1 {
+ if l := taggedList(ctx); len(l) != 7 || asked.Load() != 1 {
t.Fatalf("restarted: asked %d times, %d repos", asked.Load(), len(l))
}
@@ -269,17 +275,36 @@ func TestTaggedRepos(t *testing.T) {
}
up.Store(false)
forget()
- if l := TaggedRepos(ctx); len(l) != 7 || l[1].Package != "opencode-claude-auth" || asked.Load() != 1 {
- t.Fatalf("down: %+v", l)
+ // what GitHub said before, and that it couldn't be asked now
+ if l, err := TaggedRepos(ctx); len(l) != 7 || l[1].Package != "opencode-claude-auth" || asked.Load() != 1 || err == nil || !strings.Contains(err.Error(), "503") {
+ t.Fatalf("down: %v %+v", err, l)
+ }
+ // the minute it is held, still said
+ if _, err := TaggedRepos(ctx); err == nil {
+ t.Fatal("down, held: no error")
}
- // nothing ever kept, GitHub down: none, not an error
+ // nothing ever kept, GitHub down: none, and an error, never none
+ // passed off as GitHub's answer (till on Discord: the section just
+ // wasn't there)
t.Setenv("XDG_CACHE_HOME", t.TempDir())
taggedMu.Lock()
taggedL = nil
taggedMu.Unlock()
- if l := TaggedRepos(ctx); l == nil || len(l) != 0 {
- t.Fatalf("down, nothing kept: %#v", l)
+ if l, err := TaggedRepos(ctx); l == nil || len(l) != 0 || err == nil {
+ t.Fatalf("down, nothing kept: %v %#v", err, l)
+ }
+ // Try again asks GitHub at once, up again: its answer, no error
+ up.Store(true)
+ TaggedAskAgain()
+ if l, err := TaggedRepos(ctx); len(l) != 7 || err != nil {
+ t.Fatalf("again: %v, %d repos", err, len(l))
+ }
+ // a whole answer stands its ten minutes: Try again doesn't ask
+ before = asked.Load()
+ TaggedAskAgain()
+ if _, err := TaggedRepos(ctx); err != nil || asked.Load() != before {
+ t.Fatalf("again on a whole answer: %v, asked %d", err, asked.Load()-before)
}
// off: GitHub never asked
@@ -289,7 +314,7 @@ func TestTaggedRepos(t *testing.T) {
taggedL = nil
taggedMu.Unlock()
before = asked.Load()
- if l := TaggedRepos(ctx); len(l) != 0 || asked.Load() != before {
+ if l := taggedList(ctx); len(l) != 0 || asked.Load() != before {
t.Fatalf("off: %+v", l)
}
}
@@ -304,7 +329,7 @@ func TestTaggedReposSlowRead(t *testing.T) {
up.Store(true)
fakeGitHub(t, &up, &asked)
ctx := context.Background()
- if l := TaggedRepos(ctx); len(l) != 7 {
+ if l := taggedList(ctx); len(l) != 7 {
t.Fatalf("%d repos", len(l))
}
@@ -336,7 +361,7 @@ func TestTaggedReposSlowRead(t *testing.T) {
return nil
}
cutAsked := time.Now()
- l := TaggedRepos(ctx)
+ l := taggedList(ctx)
cutDone := time.Now()
if x := has(l, "someone/word-guard"); len(l) != 7 || x == nil || x.Kind != "middleware" {
t.Fatalf("slow package.json: %d repos, word-guard %+v", len(l), x)
@@ -388,13 +413,13 @@ func TestTaggedReposSlowRead(t *testing.T) {
taggedMu.Lock()
taggedAt = time.Now().Add(-taggedTTL + taggedRetry)
taggedMu.Unlock()
- TaggedRepos(ctx)
+ taggedList(ctx)
if n := asked.Load(); n != 2 {
t.Fatalf("asked again at once: %d", n)
}
time.Sleep(taggedRetry + 100*time.Millisecond)
before := reads.Load()
- l = TaggedRepos(ctx)
+ l = taggedList(ctx)
if n := asked.Load(); n != 3 {
t.Fatalf("a list cut short kept: asked %d times", n)
}
@@ -454,3 +479,44 @@ func TestPkgEntry(t *testing.T) {
}
}
}
+
+// GitHub's search answers but raw.githubusercontent.com doesn't (blocked
+// on the reader's line, as it is in places where api.github.com isn't),
+// and nothing was kept: the list is empty, and the error says how many
+// couldn't be read and why, not that none are tagged (till on Discord:
+// on their desktop the section wasn't there, on their server it had many).
+// One read before is kept and isn't counted.
+func TestTaggedReposUnreadable(t *testing.T) {
+ var up atomic.Bool
+ var asked atomic.Int32
+ up.Store(true)
+ fakeGitHub(t, &up, &asked)
+ ctx := context.Background()
+ rawWas := githubRaw
+ githubRaw = "http://127.0.0.1:1/raw" // nothing listens: refused at once
+ t.Cleanup(func() { githubRaw = rawWas })
+ l, err := TaggedRepos(ctx)
+ var m *TaggedMissing
+ if len(l) != 0 || !errors.As(err, &m) || m.Missing != 12 || !strings.Contains(m.Why.Error(), "127.0.0.1:1") {
+ t.Fatalf("raw unreachable: %v (%+v), %d repos", err, m, len(l))
+ }
+ // Try again, raw back: the list, no error
+ githubRaw = rawWas
+ TaggedAskAgain()
+ if l, err := TaggedRepos(ctx); len(l) != 7 || err != nil {
+ t.Fatalf("again: %v, %d repos", err, len(l))
+ }
+ // asked again, raw gone again: the seven read before are as they were,
+ // and only what was never read counts as missing
+ taggedMu.Lock()
+ taggedAt = time.Time{}
+ taggedMu.Unlock()
+ readMu.Lock()
+ clear(readRepo)
+ readMu.Unlock()
+ githubRaw = "http://127.0.0.1:1/raw"
+ l, err = TaggedRepos(ctx)
+ if len(l) != 7 || !errors.As(err, &m) || m.Missing != 5 {
+ t.Fatalf("raw gone, seven kept: %v (%+v), %d repos", err, m, len(l))
+ }
+}