Context7 Platform -- Up-to-date docs for AI agents
fix(cli): use GITHUB_TOKEN for skill downloads to avoid rate limits (#2370)
Fahreddin Özcan89d4862ebdf4
diff --git a/.changeset/fix-gh-ratelimit.md b/.changeset/fix-gh-ratelimit.mdnew file mode 100644index 00000000..5532b0c1--- /dev/null+++ b/.changeset/fix-gh-ratelimit.md@@ -0,0 +1,5 @@+---+"ctx7": patch+---++Use GITHUB_TOKEN/GH_TOKEN or gh CLI auth for skill downloads to avoid GitHub API rate limits and support private reposdiff --git a/packages/cli/src/commands/skill.ts b/packages/cli/src/commands/skill.tsindex bd1c09f4..c04d76dc 100644--- a/packages/cli/src/commands/skill.ts+++ b/packages/cli/src/commands/skill.ts@@ -30,6 +30,7 @@ import {getTrustLabel,} from "../utils/prompts.js";import { installSkillFiles, symlinkSkill } from "../utils/installer.js";+import { listSkillsFromGitHub, getSkillFromGitHub } from "../utils/github.js";import { trackEvent } from "../utils/tracking.js";import { registerGenerateCommand } from "./generate.js";import type {@@ -233,26 +234,50 @@ async function installCommand(if (skillData.error === "prompt_injection_detected") {spinner.fail(pc.red(`Prompt injection detected in skill: ${skillName}`));log.warn("This skill contains potentially malicious content and cannot be installed.");- } else {+ return;+ }++ spinner.text = `Fetching skill from GitHub: ${skillName}...`;+ const ghResult = await getSkillFromGitHub(repo, skillName);+ if (ghResult.status === "repo_not_found") {+ spinner.fail(pc.red(`Repository not found: ${repo}`));+ return;+ }+ if (ghResult.status !== "ok" || !ghResult.skill) {spinner.fail(pc.red(`Skill not found: ${skillName}`));+ return;}- return;- }- spinner.succeed(`Found skill: ${skillName}`);- selectedSkills = [- {- name: skillData.name,- description: skillData.description,- url: skillData.url,- project: repo,- },- ];+ spinner.succeed(`Found skill: ${skillName}`);+ selectedSkills = [ghResult.skill];+ } else {+ spinner.succeed(`Found skill: ${skillName}`);+ selectedSkills = [+ {+ name: skillData.name,+ description: skillData.description,+ url: skillData.url,+ project: repo,+ },+ ];+ }} else {// Fetch all skills when no specific names provided- const data = await listProjectSkills(repo);+ let data = await listProjectSkills(repo);++ if ((data.error || !data.skills || data.skills.length === 0) && !data.blockedSkillsCount) {+ spinner.text = `Fetching skills from GitHub...`;+ const ghResult = await listSkillsFromGitHub(repo);+ if (ghResult.status === "repo_not_found") {+ spinner.fail(pc.red(`Repository not found: ${repo}`));+ return;+ }+ if (ghResult.status === "ok" && ghResult.skills.length > 0) {+ data = { project: repo, skills: ghResult.skills };+ }+ }- if (data.error) {+ if (data.error && (!data.skills || data.skills.length === 0)) {spinner.fail(pc.red(`Error: ${data.message || data.error}`));return;}diff --git a/packages/cli/src/utils/api.ts b/packages/cli/src/utils/api.tsindex 0221cf7e..870a4afa 100644--- a/packages/cli/src/utils/api.ts+++ b/packages/cli/src/utils/api.ts@@ -11,7 +11,7 @@ import type {SkillQuotaResponse,ContextResponse,} from "../types.js";-import { downloadSkillFromGitHub } from "./github.js";+import { downloadSkillFromGitHub, getSkillFromGitHub } from "./github.js";import { VERSION } from "../constants.js";let baseUrl = "https://context7.com";@@ -62,11 +62,21 @@ export async function downloadSkill(project: string, skillName: string): Promiseconst skillData = await getSkill(project, skillName);if (skillData.error) {- return {- skill: { name: skillName, description: "", url: "", project },- files: [],- error: skillData.message || skillData.error,- };+ // handle private repo skills with env var+ const ghResult = await getSkillFromGitHub(project, skillName);+ if (ghResult.status !== "ok" || !ghResult.skill) {+ return {+ skill: { name: skillName, description: "", url: "", project },+ files: [],+ error: skillData.message || skillData.error,+ };+ }++ const { files, error } = await downloadSkillFromGitHub(ghResult.skill);+ if (error) {+ return { skill: ghResult.skill, files: [], error };+ }+ return { skill: ghResult.skill, files };}const skill = {diff --git a/packages/cli/src/utils/github.ts b/packages/cli/src/utils/github.tsindex cf876d3e..6248677f 100644--- a/packages/cli/src/utils/github.ts+++ b/packages/cli/src/utils/github.ts@@ -1,3 +1,4 @@+import { execSync } from "node:child_process";import type { SkillFile, Skill } from "../types.js";const GITHUB_API = "https://api.github.com";@@ -79,6 +80,160 @@ function parseGitHubUrl(url: string): {}}+function getGitHubToken(): string | undefined {+ const envToken = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;+ if (envToken) return envToken;+ try {+ return execSync("gh auth token", { stdio: ["pipe", "pipe", "ignore"] })+ .toString()+ .trim();+ } catch {+ return undefined;+ }+}++function parseSkillFrontmatter(content: string): { name: string; description: string } | null {+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);+ if (!frontmatterMatch) return null;++ const frontmatter = frontmatterMatch[1];++ const nameMatch = frontmatter.match(/^name:\s*(.+)$/m);+ if (!nameMatch) return null;+ const name = nameMatch[1]+ .trim()+ .replace(/^["']|["']$/g, "")+ .toLowerCase();++ let description = "";+ const multiLineMatch = frontmatter.match(/^description:\s*([|>])-?\s*$/m);++ if (multiLineMatch) {+ const descLineIndex = frontmatter.indexOf("description:");+ const lines = frontmatter.slice(descLineIndex).split("\n").slice(1);+ const indentedLines: string[] = [];+ for (const line of lines) {+ if (line.trim() === "") {+ indentedLines.push("");+ continue;+ }+ if (/^\s+/.test(line)) {+ indentedLines.push(line);+ } else {+ break;+ }+ }+ const firstNonEmpty = indentedLines.find((l) => l.trim().length > 0);+ const indent = firstNonEmpty?.match(/^(\s+)/)?.[1].length ?? 0;+ description = indentedLines+ .map((line) => line.slice(indent))+ .join(" ")+ .replace(/\s+/g, " ")+ .trim();+ } else {+ const singleMatch = frontmatter.match(/^description:\s*(.+)$/m);+ if (singleMatch) {+ const value = singleMatch[1].trim();+ if (!["|", ">", "|-", ">-"].includes(value)) {+ description = value.replace(/^["']|["']$/g, "");+ }+ }+ }++ if (!description) return null;+ return { name, description };+}++function getGitHubHeaders(): Record<string, string> {+ const ghToken = getGitHubToken();+ return {+ Accept: "application/vnd.github.v3+json",+ "User-Agent": "context7-cli",+ ...(ghToken && { Authorization: `token ${ghToken}` }),+ };+}++async function fetchRepoTree(+ owner: string,+ repo: string,+ branch: string,+ headers: Record<string, string>+): Promise<GitHubTreeResponse | null> {+ const treeUrl = `${GITHUB_API}/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`;+ const response = await fetch(treeUrl, { headers });+ if (!response.ok) return null;+ return (await response.json()) as GitHubTreeResponse;+}++async function fetchDefaultBranch(+ owner: string,+ repo: string,+ headers: Record<string, string>+): Promise<{ branch: string } | { status: number }> {+ const response = await fetch(`${GITHUB_API}/repos/${owner}/${repo}`, { headers });+ if (!response.ok) return { status: response.status };+ const data = (await response.json()) as { default_branch: string };+ return { branch: data.default_branch };+}++type GitHubSkillsResult =+ | { status: "ok"; skills: (Skill & { project: string })[] }+ | { status: "repo_not_found" }+ | { status: "error"; error: string };++export async function listSkillsFromGitHub(project: string): Promise<GitHubSkillsResult> {+ try {+ const parts = project.split("/").filter(Boolean);+ if (parts.length < 2) return { status: "error", error: "Invalid project format" };+ const [owner, repo] = parts;++ const headers = getGitHubHeaders();+ const branchResult = await fetchDefaultBranch(owner, repo, headers);+ if ("status" in branchResult) return { status: "repo_not_found" };++ const treeData = await fetchRepoTree(owner, repo, branchResult.branch, headers);+ if (!treeData) return { status: "error", error: "Could not fetch repository tree" };++ const skillMdFiles = treeData.tree.filter(+ (item) => item.type === "blob" && item.path.toLowerCase().endsWith("skill.md")+ );++ const skills: (Skill & { project: string })[] = [];+ for (const item of skillMdFiles) {+ const rawUrl = `${GITHUB_RAW}/${owner}/${repo}/${branchResult.branch}/${item.path}`;+ const response = await fetch(rawUrl, { headers });+ if (!response.ok) continue;++ const content = await response.text();+ const meta = parseSkillFrontmatter(content);+ if (!meta) continue;++ const skillDir = item.path.split("/").slice(0, -1).join("/");+ skills.push({+ name: meta.name,+ description: meta.description,+ url: `https://github.com/${owner}/${repo}/tree/${branchResult.branch}/${skillDir}`,+ project,+ });+ }++ return { status: "ok", skills };+ } catch (err) {+ const message = err instanceof Error ? err.message : String(err);+ return { status: "error", error: message };+ }+}++export async function getSkillFromGitHub(+ project: string,+ skillName: string+): Promise<GitHubSkillsResult & { skill?: Skill & { project: string } }> {+ const result = await listSkillsFromGitHub(project);+ if (result.status !== "ok") return result;+ const skill = result.skills.find((s) => s.name === skillName.toLowerCase());+ return { ...result, skill };+}+export async function downloadSkillFromGitHub(skill: Skill & { project: string }): Promise<{ files: SkillFile[]; error?: string }> {@@ -91,20 +246,13 @@ export async function downloadSkillFromGitHub(const { owner, repo, branch, path: skillPath } = parsed;- const treeUrl = `${GITHUB_API}/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`;- const treeResponse = await fetch(treeUrl, {- headers: {- Accept: "application/vnd.github.v3+json",- "User-Agent": "context7-cli",- },- });+ const ghHeaders = getGitHubHeaders();- if (!treeResponse.ok) {- return { files: [], error: `GitHub API error: ${treeResponse.status}` };+ const treeData = await fetchRepoTree(owner, repo, branch, ghHeaders);+ if (!treeData) {+ return { files: [], error: `GitHub API error` };}- const treeData = (await treeResponse.json()) as GitHubTreeResponse;-const skillFiles = treeData.tree.filter((item) => item.type === "blob" && item.path.startsWith(skillPath + "/"));@@ -116,7 +264,7 @@ export async function downloadSkillFromGitHub(const files: SkillFile[] = [];for (const item of skillFiles) {const rawUrl = `${GITHUB_RAW}/${owner}/${repo}/${branch}/${item.path}`;- const fileResponse = await fetch(rawUrl);+ const fileResponse = await fetch(rawUrl, { headers: ghHeaders });if (!fileResponse.ok) {console.warn(`Failed to fetch ${item.path}: ${fileResponse.status}`);