Skip to content

AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.

fix: avoid shared GitHub asset rate limits

tianma-if91bd1247c2cb
diff --git a/apps/api/src/plugin-distribution-routes.test.mjs b/apps/api/src/plugin-distribution-routes.test.mjs
index 51253787..a833494e 100644
--- a/apps/api/src/plugin-distribution-routes.test.mjs
+++ b/apps/api/src/plugin-distribution-routes.test.mjs
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
-import { downloadGithubReleaseAsset } from "./plugin-distribution-routes.ts";
+import { downloadGithubReleaseAsset, downloadGithubReleaseAssetByTag } from "./plugin-distribution-routes.ts";
describe("GitHub plugin release asset proxy", () => {
test("downloads a supported release asset on the server", async () => {
@@ -28,4 +28,41 @@ describe("GitHub plugin release asset proxy", () => {
request: async () => new Response("", { headers: { "content-length": String(300 * 1024) } }),
})).rejects.toThrow("exceeds the allowed package size");
});
+
+ test("downloads a public release asset without consuming the GitHub REST API quota", async () => {
+ const calls = [];
+ const buffer = await downloadGithubReleaseAssetByTag({
+ owner: "example",
+ repository: "edgeever-plugin",
+ releaseTag: "v1.2.3-preview.1",
+ assetName: "main.js",
+ request: async (input) => {
+ calls.push(String(input));
+ return new Response("export default {};", { headers: { "content-length": "18" } });
+ },
+ });
+
+ expect(new TextDecoder().decode(buffer)).toBe("export default {};");
+ expect(calls).toEqual([
+ "https://github.com/example/edgeever-plugin/releases/download/v1.2.3-preview.1/main.js",
+ ]);
+ });
+
+ test("rejects release coordinates that could escape the GitHub download path", async () => {
+ await expect(downloadGithubReleaseAssetByTag({
+ owner: "example",
+ repository: "edgeever-plugin",
+ releaseTag: "../../latest",
+ assetName: "main.js",
+ request: async () => new Response("should not be fetched"),
+ })).rejects.toThrow("Invalid GitHub release coordinates");
+
+ await expect(downloadGithubReleaseAssetByTag({
+ owner: "example",
+ repository: "..",
+ releaseTag: "v1.2.3",
+ assetName: "main.js",
+ request: async () => new Response("should not be fetched"),
+ })).rejects.toThrow("Invalid GitHub release coordinates");
+ });
});
diff --git a/apps/api/src/plugin-distribution-routes.ts b/apps/api/src/plugin-distribution-routes.ts
index 9ec8bb7b..a0d38dfd 100644
--- a/apps/api/src/plugin-distribution-routes.ts
+++ b/apps/api/src/plugin-distribution-routes.ts
@@ -6,6 +6,7 @@ const GITHUB_API_VERSION = "2022-11-28";
const OWNER_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,38})$/i;
const REPOSITORY_PATTERN = /^[a-z0-9._-]+$/i;
const ASSET_ID_PATTERN = /^\d+$/;
+const RELEASE_TAG_PATTERN = /^v?\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?$/i;
const ASSET_LIMITS = {
"manifest.json": 256 * 1024,
"main.js": 5 * 1024 * 1024,
@@ -14,6 +15,22 @@ const ASSET_LIMITS = {
type SupportedPluginAssetName = keyof typeof ASSET_LIMITS;
+const hasValidRepositoryCoordinates = (owner: string, repository: string) =>
+ OWNER_PATTERN.test(owner)
+ && REPOSITORY_PATTERN.test(repository)
+ && repository !== "."
+ && repository !== "..";
+
+const readBoundedAsset = async (response: Response, assetName: SupportedPluginAssetName) => {
+ if (!response.ok) throw new Error(`GitHub asset ${assetName} failed with HTTP ${response.status}.`);
+ const maximumBytes = ASSET_LIMITS[assetName];
+ const declaredSize = Number(response.headers.get("content-length") ?? 0);
+ if (declaredSize > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);
+ const buffer = await response.arrayBuffer();
+ if (buffer.byteLength > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);
+ return buffer;
+};
+
export const downloadGithubReleaseAsset = async ({
owner,
repository,
@@ -27,7 +44,6 @@ export const downloadGithubReleaseAsset = async ({
assetName: SupportedPluginAssetName;
request?: typeof fetch;
}) => {
- const maximumBytes = ASSET_LIMITS[assetName];
const response = await request(`https://api.github.com/repos/${owner}/${repository}/releases/assets/${assetId}`, {
redirect: "follow",
headers: {
@@ -36,21 +52,71 @@ export const downloadGithubReleaseAsset = async ({
"X-GitHub-Api-Version": GITHUB_API_VERSION,
},
});
- if (!response.ok) throw new Error(`GitHub asset ${assetName} failed with HTTP ${response.status}.`);
- const declaredSize = Number(response.headers.get("content-length") ?? 0);
- if (declaredSize > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);
- const buffer = await response.arrayBuffer();
- if (buffer.byteLength > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);
- return buffer;
+ return readBoundedAsset(response, assetName);
+};
+
+export const downloadGithubReleaseAssetByTag = async ({
+ owner,
+ repository,
+ releaseTag,
+ assetName,
+ request = fetch,
+}: {
+ owner: string;
+ repository: string;
+ releaseTag: string;
+ assetName: SupportedPluginAssetName;
+ request?: typeof fetch;
+}) => {
+ if (!hasValidRepositoryCoordinates(owner, repository) || !RELEASE_TAG_PATTERN.test(releaseTag)) {
+ throw new Error("Invalid GitHub release coordinates.");
+ }
+ const url = `https://github.com/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/releases/download/${encodeURIComponent(releaseTag)}/${encodeURIComponent(assetName)}`;
+ const response = await request(url, {
+ redirect: "follow",
+ headers: { "User-Agent": "EdgeEver" },
+ });
+ return readBoundedAsset(response, assetName);
};
export const registerPluginDistributionRoutes = (app: Hono<AppEnv>) => {
+ app.get("/api/v1/plugins/github/:owner/:repository/releases/:releaseTag/assets/:assetName", async (context) => {
+ const owner = context.req.param("owner");
+ const repository = context.req.param("repository");
+ const releaseTag = context.req.param("releaseTag");
+ const assetName = context.req.param("assetName");
+ if (!hasValidRepositoryCoordinates(owner, repository) || !RELEASE_TAG_PATTERN.test(releaseTag)) {
+ return badRequest(context, "Invalid GitHub release coordinates.");
+ }
+ if (!(assetName in ASSET_LIMITS)) return badRequest(context, "Unsupported plugin release asset.");
+
+ try {
+ const buffer = await downloadGithubReleaseAssetByTag({
+ owner,
+ repository,
+ releaseTag,
+ assetName: assetName as SupportedPluginAssetName,
+ });
+ return context.body(buffer, 200, {
+ "Cache-Control": "private, max-age=300",
+ "Content-Type": "application/octet-stream",
+ });
+ } catch (error) {
+ return apiError(
+ context,
+ "github_asset_download_failed",
+ error instanceof Error ? error.message : "GitHub release asset download failed.",
+ 502,
+ );
+ }
+ });
+
app.get("/api/v1/plugins/github/:owner/:repository/assets/:assetId/:assetName", async (context) => {
const owner = context.req.param("owner");
const repository = context.req.param("repository");
const assetId = context.req.param("assetId");
const assetName = context.req.param("assetName");
- if (!OWNER_PATTERN.test(owner) || !REPOSITORY_PATTERN.test(repository) || !ASSET_ID_PATTERN.test(assetId)) {
+ if (!hasValidRepositoryCoordinates(owner, repository) || !ASSET_ID_PATTERN.test(assetId)) {
return badRequest(context, "Invalid GitHub release asset coordinates.");
}
if (!(assetName in ASSET_LIMITS)) return badRequest(context, "Unsupported plugin release asset.");
diff --git a/apps/web/src/lib/plugins/github-plugin-distribution.test.mjs b/apps/web/src/lib/plugins/github-plugin-distribution.test.mjs
index f416d0d2..596b1317 100644
--- a/apps/web/src/lib/plugins/github-plugin-distribution.test.mjs
+++ b/apps/web/src/lib/plugins/github-plugin-distribution.test.mjs
@@ -41,9 +41,13 @@ describe("GitHub plugin distribution", () => {
}
return new Response(null, { status: 500 });
};
- const downloadAsset = async (_coordinates, asset) => new TextEncoder().encode(
- asset.id === 1 ? JSON.stringify(manifest) : "export default { activate() {} };",
- ).buffer;
+ const assetDownloads = [];
+ const downloadAsset = async (_coordinates, releaseTag, asset) => {
+ assetDownloads.push([releaseTag, asset.name]);
+ return new TextEncoder().encode(
+ asset.id === 1 ? JSON.stringify(manifest) : "export default { activate() {} };",
+ ).buffer;
+ };
const downloaded = await downloadGithubExtension("https://github.com/example/edgeever-plugin", request, downloadAsset);
@@ -51,6 +55,10 @@ describe("GitHub plugin distribution", () => {
expect(downloaded.pluginPackage?.pluginId).toBe("org.edgeever.github-test");
expect(downloaded.pluginPackage?.mainJs).toContain("activate");
expect(downloaded.checksums.mainJs).toHaveLength(64);
+ expect(assetDownloads).toEqual([
+ ["v1.2.3", "manifest.json"],
+ ["v1.2.3", "main.js"],
+ ]);
expect(calls).toContain("https://api.github.com/repos/example/edgeever-plugin/contents/manifest.json");
expect(calls).toContain("https://api.github.com/repos/example/edgeever-plugin/releases/tags/1.2.3");
expect(calls).not.toContain("https://api.github.com/assets/1");
@@ -85,7 +93,7 @@ describe("GitHub plugin distribution", () => {
return new Response(null, { status: 404 });
};
const releaseManifest = { ...manifest, permissions: [...manifest.permissions, "network"], networkHosts: ["api.example.com"] };
- const downloadAsset = async (_coordinates, asset) => new TextEncoder().encode(
+ const downloadAsset = async (_coordinates, _releaseTag, asset) => new TextEncoder().encode(
asset.id === 1 ? JSON.stringify(releaseManifest) : "export default { activate() {} };",
).buffer;
diff --git a/apps/web/src/lib/plugins/github-plugin-distribution.ts b/apps/web/src/lib/plugins/github-plugin-distribution.ts
index 1cbdce34..e222d311 100644
--- a/apps/web/src/lib/plugins/github-plugin-distribution.ts
+++ b/apps/web/src/lib/plugins/github-plugin-distribution.ts
@@ -92,25 +92,27 @@ const requireAsset = (release: GithubReleaseResponse, name: string) => {
type GithubAssetDownloader = (
coordinates: GithubRepositoryCoordinates,
+ releaseTag: string,
asset: GithubReleaseAsset,
) => Promise<ArrayBuffer>;
-const downloadGithubAssetThroughApi: GithubAssetDownloader = (coordinates, asset) =>
+const downloadGithubAssetThroughApi: GithubAssetDownloader = (coordinates, releaseTag, asset) =>
api.downloadGithubPluginAsset(
coordinates.owner,
coordinates.repository,
- asset.id,
+ releaseTag,
asset.name as "manifest.json" | "main.js" | "styles.css",
);
const downloadAsset = async (
coordinates: GithubRepositoryCoordinates,
+ releaseTag: string,
asset: GithubReleaseAsset,
maximumBytes: number,
download: GithubAssetDownloader,
) => {
if (asset.size > maximumBytes) throw new Error(`${asset.name} exceeds the allowed package size.`);
- const buffer = await download(coordinates, asset);
+ const buffer = await download(coordinates, releaseTag, asset);
if (buffer.byteLength > maximumBytes) throw new Error(`${asset.name} exceeds the allowed package size.`);
const checksum = await sha256Hex(buffer);
if (asset.digest?.startsWith("sha256:") && asset.digest.slice(7).toLocaleLowerCase() !== checksum) {
@@ -194,9 +196,9 @@ export const downloadGithubExtension = async (
const mainAsset = requireAsset(release, "main.js");
const stylesAsset = release.assets.find((asset) => asset.name === "styles.css") ?? null;
const [downloadedManifest, downloadedMain, downloadedStyles] = await Promise.all([
- downloadAsset(coordinates, manifestAsset, 256 * 1024, downloadAssetBytes),
- downloadAsset(coordinates, mainAsset, MAX_MAIN_JS_BYTES, downloadAssetBytes),
- stylesAsset ? downloadAsset(coordinates, stylesAsset, MAX_STYLES_CSS_BYTES, downloadAssetBytes) : Promise.resolve(null),
+ downloadAsset(coordinates, release.tag_name, manifestAsset, 256 * 1024, downloadAssetBytes),
+ downloadAsset(coordinates, release.tag_name, mainAsset, MAX_MAIN_JS_BYTES, downloadAssetBytes),
+ stylesAsset ? downloadAsset(coordinates, release.tag_name, stylesAsset, MAX_STYLES_CSS_BYTES, downloadAssetBytes) : Promise.resolve(null),
]);
const releaseManifest = parseExtensionManifest(JSON.parse(new TextDecoder().decode(downloadedManifest.buffer)) as unknown);
assertReleaseManifest(repositoryManifest, releaseManifest);
diff --git a/packages/client/src/index.test.mjs b/packages/client/src/index.test.mjs
index c6f7025d..872ae481 100644
--- a/packages/client/src/index.test.mjs
+++ b/packages/client/src/index.test.mjs
@@ -149,6 +149,28 @@ describe("EdgeEver client HTTP contract", () => {
expect(Array.from(new Uint8Array(await response.arrayBuffer()))).toEqual([1, 2, 3]);
});
+ test("downloads GitHub plugin assets through the release-tag route", async () => {
+ let requestUrl;
+ const client = createEdgeEverClient({
+ fetch: async (input) => {
+ requestUrl = String(input);
+ return new Response(new Uint8Array([1, 2, 3]));
+ },
+ });
+
+ const buffer = await client.downloadGithubPluginAsset(
+ "example-owner",
+ "example-plugin",
+ "v1.2.3-preview.1",
+ "main.js",
+ );
+
+ expect(requestUrl).toBe(
+ "/api/v1/plugins/github/example-owner/example-plugin/releases/v1.2.3-preview.1/assets/main.js",
+ );
+ expect(Array.from(new Uint8Array(buffer))).toEqual([1, 2, 3]);
+ });
+
test("streams restored resources through bounded multipart requests", async () => {
const calls = [];
const client = createEdgeEverClient({
diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts
index b025dc05..429ca8e5 100644
--- a/packages/client/src/index.ts
+++ b/packages/client/src/index.ts
@@ -1230,10 +1230,10 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => {
downloadGithubPluginAsset: (
owner: string,
repository: string,
- assetId: number,
+ releaseTag: string,
assetName: "manifest.json" | "main.js" | "styles.css",
) => requestArrayBuffer(
- `/api/v1/plugins/github/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/assets/${assetId}/${encodeURIComponent(assetName)}`,
+ `/api/v1/plugins/github/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/releases/${encodeURIComponent(releaseTag)}/assets/${encodeURIComponent(assetName)}`,
),
uploadMemoResource: (memoId: string, file: Blob | FormData) => {