AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.
HTML ai-agentcloudflarecloudflare-d1cloudflare-r2cloudflare-workersdockerelectronevernoteevernote-alternativeknowledge-basemcpnote-takingobsidian-alternativeoffline-firstpersonal-knowledge-basepwaself-hostedsqlitetiptapweb-clipper
fix: avoid shared GitHub asset rate limits
tianma-if91bd1247c2cb
diff --git a/apps/api/src/plugin-distribution-routes.test.mjs b/apps/api/src/plugin-distribution-routes.test.mjsindex 51253787..a833494e 100644--- a/apps/api/src/plugin-distribution-routes.test.mjs+++ b/apps/api/src/plugin-distribution-routes.test.mjs@@ -1,5 +1,5 @@import { describe, expect, test } from "bun:test";-import { downloadGithubReleaseAsset } from "./plugin-distribution-routes.ts";+import { downloadGithubReleaseAsset, downloadGithubReleaseAssetByTag } from "./plugin-distribution-routes.ts";describe("GitHub plugin release asset proxy", () => {test("downloads a supported release asset on the server", async () => {@@ -28,4 +28,41 @@ describe("GitHub plugin release asset proxy", () => {request: async () => new Response("", { headers: { "content-length": String(300 * 1024) } }),})).rejects.toThrow("exceeds the allowed package size");});++ test("downloads a public release asset without consuming the GitHub REST API quota", async () => {+ const calls = [];+ const buffer = await downloadGithubReleaseAssetByTag({+ owner: "example",+ repository: "edgeever-plugin",+ releaseTag: "v1.2.3-preview.1",+ assetName: "main.js",+ request: async (input) => {+ calls.push(String(input));+ return new Response("export default {};", { headers: { "content-length": "18" } });+ },+ });++ expect(new TextDecoder().decode(buffer)).toBe("export default {};");+ expect(calls).toEqual([+ "https://github.com/example/edgeever-plugin/releases/download/v1.2.3-preview.1/main.js",+ ]);+ });++ test("rejects release coordinates that could escape the GitHub download path", async () => {+ await expect(downloadGithubReleaseAssetByTag({+ owner: "example",+ repository: "edgeever-plugin",+ releaseTag: "../../latest",+ assetName: "main.js",+ request: async () => new Response("should not be fetched"),+ })).rejects.toThrow("Invalid GitHub release coordinates");++ await expect(downloadGithubReleaseAssetByTag({+ owner: "example",+ repository: "..",+ releaseTag: "v1.2.3",+ assetName: "main.js",+ request: async () => new Response("should not be fetched"),+ })).rejects.toThrow("Invalid GitHub release coordinates");+ });});diff --git a/apps/api/src/plugin-distribution-routes.ts b/apps/api/src/plugin-distribution-routes.tsindex 9ec8bb7b..a0d38dfd 100644--- a/apps/api/src/plugin-distribution-routes.ts+++ b/apps/api/src/plugin-distribution-routes.ts@@ -6,6 +6,7 @@ const GITHUB_API_VERSION = "2022-11-28";const OWNER_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,38})$/i;const REPOSITORY_PATTERN = /^[a-z0-9._-]+$/i;const ASSET_ID_PATTERN = /^\d+$/;+const RELEASE_TAG_PATTERN = /^v?\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?$/i;const ASSET_LIMITS = {"manifest.json": 256 * 1024,"main.js": 5 * 1024 * 1024,@@ -14,6 +15,22 @@ const ASSET_LIMITS = {type SupportedPluginAssetName = keyof typeof ASSET_LIMITS;+const hasValidRepositoryCoordinates = (owner: string, repository: string) =>+ OWNER_PATTERN.test(owner)+ && REPOSITORY_PATTERN.test(repository)+ && repository !== "."+ && repository !== "..";++const readBoundedAsset = async (response: Response, assetName: SupportedPluginAssetName) => {+ if (!response.ok) throw new Error(`GitHub asset ${assetName} failed with HTTP ${response.status}.`);+ const maximumBytes = ASSET_LIMITS[assetName];+ const declaredSize = Number(response.headers.get("content-length") ?? 0);+ if (declaredSize > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);+ const buffer = await response.arrayBuffer();+ if (buffer.byteLength > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);+ return buffer;+};+export const downloadGithubReleaseAsset = async ({owner,repository,@@ -27,7 +44,6 @@ export const downloadGithubReleaseAsset = async ({assetName: SupportedPluginAssetName;request?: typeof fetch;}) => {- const maximumBytes = ASSET_LIMITS[assetName];const response = await request(`https://api.github.com/repos/${owner}/${repository}/releases/assets/${assetId}`, {redirect: "follow",headers: {@@ -36,21 +52,71 @@ export const downloadGithubReleaseAsset = async ({"X-GitHub-Api-Version": GITHUB_API_VERSION,},});- if (!response.ok) throw new Error(`GitHub asset ${assetName} failed with HTTP ${response.status}.`);- const declaredSize = Number(response.headers.get("content-length") ?? 0);- if (declaredSize > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);- const buffer = await response.arrayBuffer();- if (buffer.byteLength > maximumBytes) throw new Error(`${assetName} exceeds the allowed package size.`);- return buffer;+ return readBoundedAsset(response, assetName);+};++export const downloadGithubReleaseAssetByTag = async ({+ owner,+ repository,+ releaseTag,+ assetName,+ request = fetch,+}: {+ owner: string;+ repository: string;+ releaseTag: string;+ assetName: SupportedPluginAssetName;+ request?: typeof fetch;+}) => {+ if (!hasValidRepositoryCoordinates(owner, repository) || !RELEASE_TAG_PATTERN.test(releaseTag)) {+ throw new Error("Invalid GitHub release coordinates.");+ }+ const url = `https://github.com/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/releases/download/${encodeURIComponent(releaseTag)}/${encodeURIComponent(assetName)}`;+ const response = await request(url, {+ redirect: "follow",+ headers: { "User-Agent": "EdgeEver" },+ });+ return readBoundedAsset(response, assetName);};export const registerPluginDistributionRoutes = (app: Hono<AppEnv>) => {+ app.get("/api/v1/plugins/github/:owner/:repository/releases/:releaseTag/assets/:assetName", async (context) => {+ const owner = context.req.param("owner");+ const repository = context.req.param("repository");+ const releaseTag = context.req.param("releaseTag");+ const assetName = context.req.param("assetName");+ if (!hasValidRepositoryCoordinates(owner, repository) || !RELEASE_TAG_PATTERN.test(releaseTag)) {+ return badRequest(context, "Invalid GitHub release coordinates.");+ }+ if (!(assetName in ASSET_LIMITS)) return badRequest(context, "Unsupported plugin release asset.");++ try {+ const buffer = await downloadGithubReleaseAssetByTag({+ owner,+ repository,+ releaseTag,+ assetName: assetName as SupportedPluginAssetName,+ });+ return context.body(buffer, 200, {+ "Cache-Control": "private, max-age=300",+ "Content-Type": "application/octet-stream",+ });+ } catch (error) {+ return apiError(+ context,+ "github_asset_download_failed",+ error instanceof Error ? error.message : "GitHub release asset download failed.",+ 502,+ );+ }+ });+app.get("/api/v1/plugins/github/:owner/:repository/assets/:assetId/:assetName", async (context) => {const owner = context.req.param("owner");const repository = context.req.param("repository");const assetId = context.req.param("assetId");const assetName = context.req.param("assetName");- if (!OWNER_PATTERN.test(owner) || !REPOSITORY_PATTERN.test(repository) || !ASSET_ID_PATTERN.test(assetId)) {+ if (!hasValidRepositoryCoordinates(owner, repository) || !ASSET_ID_PATTERN.test(assetId)) {return badRequest(context, "Invalid GitHub release asset coordinates.");}if (!(assetName in ASSET_LIMITS)) return badRequest(context, "Unsupported plugin release asset.");diff --git a/apps/web/src/lib/plugins/github-plugin-distribution.test.mjs b/apps/web/src/lib/plugins/github-plugin-distribution.test.mjsindex f416d0d2..596b1317 100644--- a/apps/web/src/lib/plugins/github-plugin-distribution.test.mjs+++ b/apps/web/src/lib/plugins/github-plugin-distribution.test.mjs@@ -41,9 +41,13 @@ describe("GitHub plugin distribution", () => {}return new Response(null, { status: 500 });};- const downloadAsset = async (_coordinates, asset) => new TextEncoder().encode(- asset.id === 1 ? JSON.stringify(manifest) : "export default { activate() {} };",- ).buffer;+ const assetDownloads = [];+ const downloadAsset = async (_coordinates, releaseTag, asset) => {+ assetDownloads.push([releaseTag, asset.name]);+ return new TextEncoder().encode(+ asset.id === 1 ? JSON.stringify(manifest) : "export default { activate() {} };",+ ).buffer;+ };const downloaded = await downloadGithubExtension("https://github.com/example/edgeever-plugin", request, downloadAsset);@@ -51,6 +55,10 @@ describe("GitHub plugin distribution", () => {expect(downloaded.pluginPackage?.pluginId).toBe("org.edgeever.github-test");expect(downloaded.pluginPackage?.mainJs).toContain("activate");expect(downloaded.checksums.mainJs).toHaveLength(64);+ expect(assetDownloads).toEqual([+ ["v1.2.3", "manifest.json"],+ ["v1.2.3", "main.js"],+ ]);expect(calls).toContain("https://api.github.com/repos/example/edgeever-plugin/contents/manifest.json");expect(calls).toContain("https://api.github.com/repos/example/edgeever-plugin/releases/tags/1.2.3");expect(calls).not.toContain("https://api.github.com/assets/1");@@ -85,7 +93,7 @@ describe("GitHub plugin distribution", () => {return new Response(null, { status: 404 });};const releaseManifest = { ...manifest, permissions: [...manifest.permissions, "network"], networkHosts: ["api.example.com"] };- const downloadAsset = async (_coordinates, asset) => new TextEncoder().encode(+ const downloadAsset = async (_coordinates, _releaseTag, asset) => new TextEncoder().encode(asset.id === 1 ? JSON.stringify(releaseManifest) : "export default { activate() {} };",).buffer;diff --git a/apps/web/src/lib/plugins/github-plugin-distribution.ts b/apps/web/src/lib/plugins/github-plugin-distribution.tsindex 1cbdce34..e222d311 100644--- a/apps/web/src/lib/plugins/github-plugin-distribution.ts+++ b/apps/web/src/lib/plugins/github-plugin-distribution.ts@@ -92,25 +92,27 @@ const requireAsset = (release: GithubReleaseResponse, name: string) => {type GithubAssetDownloader = (coordinates: GithubRepositoryCoordinates,+ releaseTag: string,asset: GithubReleaseAsset,) => Promise<ArrayBuffer>;-const downloadGithubAssetThroughApi: GithubAssetDownloader = (coordinates, asset) =>+const downloadGithubAssetThroughApi: GithubAssetDownloader = (coordinates, releaseTag, asset) =>api.downloadGithubPluginAsset(coordinates.owner,coordinates.repository,- asset.id,+ releaseTag,asset.name as "manifest.json" | "main.js" | "styles.css",);const downloadAsset = async (coordinates: GithubRepositoryCoordinates,+ releaseTag: string,asset: GithubReleaseAsset,maximumBytes: number,download: GithubAssetDownloader,) => {if (asset.size > maximumBytes) throw new Error(`${asset.name} exceeds the allowed package size.`);- const buffer = await download(coordinates, asset);+ const buffer = await download(coordinates, releaseTag, asset);if (buffer.byteLength > maximumBytes) throw new Error(`${asset.name} exceeds the allowed package size.`);const checksum = await sha256Hex(buffer);if (asset.digest?.startsWith("sha256:") && asset.digest.slice(7).toLocaleLowerCase() !== checksum) {@@ -194,9 +196,9 @@ export const downloadGithubExtension = async (const mainAsset = requireAsset(release, "main.js");const stylesAsset = release.assets.find((asset) => asset.name === "styles.css") ?? null;const [downloadedManifest, downloadedMain, downloadedStyles] = await Promise.all([- downloadAsset(coordinates, manifestAsset, 256 * 1024, downloadAssetBytes),- downloadAsset(coordinates, mainAsset, MAX_MAIN_JS_BYTES, downloadAssetBytes),- stylesAsset ? downloadAsset(coordinates, stylesAsset, MAX_STYLES_CSS_BYTES, downloadAssetBytes) : Promise.resolve(null),+ downloadAsset(coordinates, release.tag_name, manifestAsset, 256 * 1024, downloadAssetBytes),+ downloadAsset(coordinates, release.tag_name, mainAsset, MAX_MAIN_JS_BYTES, downloadAssetBytes),+ stylesAsset ? downloadAsset(coordinates, release.tag_name, stylesAsset, MAX_STYLES_CSS_BYTES, downloadAssetBytes) : Promise.resolve(null),]);const releaseManifest = parseExtensionManifest(JSON.parse(new TextDecoder().decode(downloadedManifest.buffer)) as unknown);assertReleaseManifest(repositoryManifest, releaseManifest);diff --git a/packages/client/src/index.test.mjs b/packages/client/src/index.test.mjsindex c6f7025d..872ae481 100644--- a/packages/client/src/index.test.mjs+++ b/packages/client/src/index.test.mjs@@ -149,6 +149,28 @@ describe("EdgeEver client HTTP contract", () => {expect(Array.from(new Uint8Array(await response.arrayBuffer()))).toEqual([1, 2, 3]);});+ test("downloads GitHub plugin assets through the release-tag route", async () => {+ let requestUrl;+ const client = createEdgeEverClient({+ fetch: async (input) => {+ requestUrl = String(input);+ return new Response(new Uint8Array([1, 2, 3]));+ },+ });++ const buffer = await client.downloadGithubPluginAsset(+ "example-owner",+ "example-plugin",+ "v1.2.3-preview.1",+ "main.js",+ );++ expect(requestUrl).toBe(+ "/api/v1/plugins/github/example-owner/example-plugin/releases/v1.2.3-preview.1/assets/main.js",+ );+ expect(Array.from(new Uint8Array(buffer))).toEqual([1, 2, 3]);+ });+test("streams restored resources through bounded multipart requests", async () => {const calls = [];const client = createEdgeEverClient({diff --git a/packages/client/src/index.ts b/packages/client/src/index.tsindex b025dc05..429ca8e5 100644--- a/packages/client/src/index.ts+++ b/packages/client/src/index.ts@@ -1230,10 +1230,10 @@ export const createEdgeEverClient = (options: EdgeEverClientOptions = {}) => {downloadGithubPluginAsset: (owner: string,repository: string,- assetId: number,+ releaseTag: string,assetName: "manifest.json" | "main.js" | "styles.css",) => requestArrayBuffer(- `/api/v1/plugins/github/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/assets/${assetId}/${encodeURIComponent(assetName)}`,+ `/api/v1/plugins/github/${encodeURIComponent(owner)}/${encodeURIComponent(repository)}/releases/${encodeURIComponent(releaseTag)}/assets/${encodeURIComponent(assetName)}`,),uploadMemoResource: (memoId: string, file: Blob | FormData) => {