Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): fetch only the requested subpath when importing a git skill (#6892)

Nick Gomezde87325e4c38
diff --git a/.changeset/skill-import-fetches-only-subpath.md b/.changeset/skill-import-fetches-only-subpath.md
new file mode 100644
index 000000000..48a433616
--- /dev/null
+++ b/.changeset/skill-import-fetches-only-subpath.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Importing or updating a skill from one folder of a git repository, such as `your-org/team-skills/plugins/eng/skills/tdd`, now downloads that folder instead of the whole repository, which makes imports from large repositories much faster.
diff --git a/packages/core/src/skills-catalog/acquire/fetch-git.test.ts b/packages/core/src/skills-catalog/acquire/fetch-git.test.ts
new file mode 100644
index 000000000..f89f36715
--- /dev/null
+++ b/packages/core/src/skills-catalog/acquire/fetch-git.test.ts
@@ -0,0 +1,283 @@
+import { type ChildProcess, execFileSync, spawn, spawnSync } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
+import {
+ existsSync,
+ lstatSync,
+ mkdirSync,
+ mkdtempSync,
+ rmSync,
+ statSync,
+ symlinkSync,
+ writeFileSync,
+} from 'node:fs';
+import { type AddressInfo, createServer, type Server } from 'node:net';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import {
+ afterAll,
+ afterEach,
+ beforeAll,
+ beforeEach,
+ describe,
+ expect,
+ type MockInstance,
+ test,
+ vi,
+} from 'vitest';
+import { configureTestGitRepository } from '../../../../../test-support/configure-git-fixture.test-helper.ts';
+import { fetchSource } from './fetch.ts';
+import { parseSkillDir } from './parse.ts';
+
+const OUTSIDE_SUBPATH_BLOB_BYTES = 2_000_000;
+const POSIX_NAMES = process.platform !== 'win32';
+
+const hermeticGitEnv: NodeJS.ProcessEnv = {
+ ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_'))),
+ GIT_CONFIG_GLOBAL: '/dev/null',
+ GIT_CONFIG_SYSTEM: '/dev/null',
+ GIT_CONFIG_NOSYSTEM: '1',
+};
+
+let root: string;
+let server: Server;
+let port: number;
+let bytesServed = 0;
+let sparseCloneUsesCone: boolean;
+let warn: MockInstance<typeof console.warn>;
+const daemons = new Set<ChildProcess>();
+
+function git(cwd: string, args: readonly string[]): void {
+ execFileSync('git', args, { cwd, stdio: 'pipe', env: hermeticGitEnv });
+}
+
+function writeRepoFile(repo: string, path: string, body: string | Buffer): void {
+ mkdirSync(dirname(join(repo, path)), { recursive: true });
+ writeFileSync(join(repo, path), body);
+}
+
+function sparseCloneSetsConeMode(repo: string): boolean {
+ const probe = join(root, 'cone-probe');
+ git(root, ['clone', '-q', '--sparse', '--no-checkout', repo, probe]);
+ const cone = spawnSync('git', ['-C', probe, 'config', '--get', 'core.sparseCheckoutCone'], {
+ env: hermeticGitEnv,
+ encoding: 'utf8',
+ });
+ if (cone.status === 1) return false;
+ if (cone.status !== 0) {
+ throw new Error(
+ `git config --get core.sparseCheckoutCone exited ${cone.status}: ${cone.stderr}`,
+ );
+ }
+ return cone.stdout.trim() === 'true';
+}
+
+function servedUrl(repo: 'filtering' | 'non-filtering'): string {
+ return `git://127.0.0.1:${port}/${repo}.git`;
+}
+
+function untilClosed(child: ChildProcess): Promise<void> {
+ if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
+ return new Promise((resolve) => child.once('close', () => resolve()));
+}
+
+beforeAll(async () => {
+ root = mkdtempSync(join(tmpdir(), 'ok-fetch-git-'));
+ const source = join(root, 'source');
+ mkdirSync(source);
+ git(source, ['init', '-q']);
+ configureTestGitRepository(source);
+ writeRepoFile(source, 'skills/alpha/SKILL.md', '---\nname: alpha\ndescription: a\n---\n');
+ writeRepoFile(source, 'skills/alpha/references/guide.md', 'guide\n');
+ writeRepoFile(source, 'skills/beta/SKILL.md', '---\nname: beta\ndescription: b\n---\n');
+ writeRepoFile(source, 'assets/blob.bin', randomBytes(OUTSIDE_SUBPATH_BLOB_BYTES));
+ if (POSIX_NAMES) {
+ symlinkSync('/etc/passwd', join(source, 'skills/alpha/link'));
+ writeRepoFile(source, '"quoted"/SKILL.md', '---\nname: quoted\ndescription: q\n---\n');
+ }
+ git(source, ['add', '-A']);
+ git(source, ['-c', '[email protected]', '-c', 'user.name=t', 'commit', '-q', '-m', 'init']);
+
+ const served = join(root, 'served');
+ for (const [name, allowFilter] of [
+ ['filtering', 'true'],
+ ['non-filtering', 'false'],
+ ] as const) {
+ const bare = join(served, `${name}.git`);
+ git(root, ['clone', '-q', '--bare', source, bare]);
+ configureTestGitRepository(bare);
+ git(bare, ['config', 'uploadpack.allowFilter', allowFilter]);
+ git(bare, ['config', 'uploadpack.allowAnySHA1InWant', 'false']);
+ git(bare, ['config', 'uploadpack.allowReachableSHA1InWant', 'false']);
+ }
+ sparseCloneUsesCone = sparseCloneSetsConeMode(join(served, 'filtering.git'));
+
+ server = createServer((socket) => {
+ const daemon = spawn(
+ 'git',
+ ['daemon', '--inetd', `--base-path=${served}`, '--export-all', served],
+ { env: hermeticGitEnv, stdio: ['pipe', 'pipe', 'ignore'] },
+ );
+ daemons.add(daemon);
+ daemon.once('close', () => {
+ daemons.delete(daemon);
+ socket.end();
+ });
+ daemon.stdout.on('data', (chunk: Buffer) => {
+ bytesServed += chunk.length;
+ });
+ socket.on('error', () => daemon.stdin.destroy());
+ daemon.stdin.on('error', () => {});
+ socket.pipe(daemon.stdin);
+ daemon.stdout.pipe(socket);
+ });
+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+ port = (server.address() as AddressInfo).port;
+});
+
+afterAll(async () => {
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ for (const daemon of daemons) daemon.kill();
+ await Promise.all([...daemons].map(untilClosed));
+ rmSync(root, { recursive: true, force: true });
+});
+
+beforeEach(() => {
+ bytesServed = 0;
+ warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ for (const name of Object.keys(process.env)) {
+ if (name.startsWith('GIT_')) vi.stubEnv(name, undefined);
+ }
+ vi.stubEnv('GIT_CONFIG_GLOBAL', '/dev/null');
+ vi.stubEnv('GIT_CONFIG_SYSTEM', '/dev/null');
+ vi.stubEnv('GIT_CONFIG_NOSYSTEM', '1');
+});
+
+afterEach(async () => {
+ vi.unstubAllEnvs();
+ warn.mockRestore();
+ await Promise.all([...daemons].map(untilClosed));
+});
+
+describe('fetchSource from a git server', () => {
+ test('a subpath import neither checks out nor downloads the other folders', async () => {
+ const fetched = await fetchSource({
+ kind: 'git',
+ url: servedUrl('filtering'),
+ subpath: 'skills/alpha',
+ });
+ try {
+ expect(parseSkillDir(fetched.dir)?.name).toBe('alpha');
+ expect(existsSync(join(fetched.dir, 'references', 'guide.md'))).toBe(true);
+ expect(existsSync(join(fetched.dir, '..', 'beta'))).toBe(false);
+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(warn).not.toHaveBeenCalled();
+ } finally {
+ fetched.cleanup();
+ }
+ });
+
+ test('an import without a subpath still checks out and downloads the whole repository', async () => {
+ const fetched = await fetchSource({ kind: 'git', url: servedUrl('filtering') });
+ try {
+ expect(parseSkillDir(join(fetched.dir, 'skills', 'alpha'))?.name).toBe('alpha');
+ expect(parseSkillDir(join(fetched.dir, 'skills', 'beta'))?.name).toBe('beta');
+ expect(statSync(join(fetched.dir, 'assets', 'blob.bin')).size).toBe(
+ OUTSIDE_SUBPATH_BLOB_BYTES,
+ );
+ expect(bytesServed).toBeGreaterThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ } finally {
+ fetched.cleanup();
+ }
+ });
+
+ test('a subpath missing from the repository is still reported as not found', async () => {
+ await expect(
+ fetchSource({ kind: 'git', url: servedUrl('filtering'), subpath: 'skills/missing' }),
+ ).rejects.toThrow('subpath not found in repo: skills/missing');
+ expect(bytesServed).toBeGreaterThan(0);
+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ });
+
+ test('a subpath that escapes the repository is still refused', async () => {
+ await expect(
+ fetchSource({ kind: 'git', url: servedUrl('filtering'), subpath: '../../../etc' }),
+ ).rejects.toThrow('subpath escapes the clone: ../../../etc');
+ });
+
+ test.runIf(POSIX_NAMES)('a symlink inside the subpath arrives as a plain file', async () => {
+ const fetched = await fetchSource({
+ kind: 'git',
+ url: servedUrl('filtering'),
+ subpath: 'skills/alpha',
+ });
+ try {
+ expect(existsSync(join(fetched.dir, 'link'))).toBe(true);
+ expect(lstatSync(join(fetched.dir, 'link')).isSymbolicLink()).toBe(false);
+ } finally {
+ fetched.cleanup();
+ }
+ });
+
+ test('a server that ignores the blob filter still yields the subpath', async () => {
+ const fetched = await fetchSource({
+ kind: 'git',
+ url: servedUrl('non-filtering'),
+ subpath: 'skills/alpha',
+ });
+ try {
+ expect(bytesServed).toBeGreaterThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(parseSkillDir(fetched.dir)?.name).toBe('alpha');
+ expect(existsSync(join(fetched.dir, 'references', 'guide.md'))).toBe(true);
+ } finally {
+ fetched.cleanup();
+ }
+ });
+
+ test('a server that cannot send the subpath on demand falls back to the whole repository', async () => {
+ vi.stubEnv('GIT_CONFIG_COUNT', '1');
+ vi.stubEnv('GIT_CONFIG_KEY_0', 'protocol.version');
+ vi.stubEnv('GIT_CONFIG_VALUE_0', '0');
+ const fetched = await fetchSource({
+ kind: 'git',
+ url: servedUrl('filtering'),
+ subpath: 'skills/alpha',
+ });
+ try {
+ expect(bytesServed).toBeGreaterThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(warn).toHaveBeenCalledWith(
+ '[skills-catalog] sparse clone failed, cloning the whole repository',
+ expect.objectContaining({ subpath: 'skills/alpha', err: expect.any(Error) }),
+ );
+ expect(parseSkillDir(fetched.dir)?.name).toBe('alpha');
+ expect(existsSync(join(fetched.dir, 'references', 'guide.md'))).toBe(true);
+ } finally {
+ fetched.cleanup();
+ }
+ });
+
+ test.runIf(POSIX_NAMES)(
+ 'a quoted subpath is imported, falling back to the whole repository where cone mode cannot select it',
+ async () => {
+ const fetched = await fetchSource({
+ kind: 'git',
+ url: servedUrl('filtering'),
+ subpath: '"quoted"',
+ });
+ try {
+ expect(parseSkillDir(fetched.dir)?.name).toBe('quoted');
+ if (sparseCloneUsesCone) {
+ expect(bytesServed).toBeGreaterThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(warn).toHaveBeenCalledWith(
+ '[skills-catalog] sparse checkout left the subpath unselected, cloning the whole repository',
+ expect.objectContaining({ subpath: '"quoted"' }),
+ );
+ } else {
+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(warn).not.toHaveBeenCalled();
+ }
+ } finally {
+ fetched.cleanup();
+ }
+ },
+ );
+});
diff --git a/packages/core/src/skills-catalog/acquire/fetch.ts b/packages/core/src/skills-catalog/acquire/fetch.ts
index c8efb8f8a..37c384819 100644
--- a/packages/core/src/skills-catalog/acquire/fetch.ts
+++ b/packages/core/src/skills-catalog/acquire/fetch.ts
@@ -1,7 +1,7 @@
import { execFile, execFileSync } from 'node:child_process';
import { existsSync, mkdtempSync, rmSync } from 'node:fs';
import { homedir, tmpdir } from 'node:os';
-import { isAbsolute, join, relative, resolve } from 'node:path';
+import { isAbsolute, join, relative, resolve, sep } from 'node:path';
import { promisify } from 'node:util';
import { isLocalSkillSource } from '../source-fields.ts';
import { SkillFetchError } from './errors.ts';
@@ -25,6 +25,8 @@ export interface SkillsShSourceRef {
export { SkillFetchError } from './errors.ts';
+const GIT_FETCH_BUDGET_MS = 60_000;
+
const GITHUB_SHORTHAND = /^(?:github\s+)?([\w.-]+)\/([\w.-]+?)(?:\.git)?(?:\/(.+))?$/;
export const ALLOWED_GIT_TRANSPORTS: readonly RegExp[] = [
@@ -110,22 +112,10 @@ export async function fetchSource(
if (hasTransportScheme && !ALLOWED_GIT_TRANSPORTS.some((p) => p.test(spec.url))) {
throw new SkillFetchError(`Unsupported git transport: ${spec.url}`);
}
- const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));
- try {
- await execFileAsync(
- 'git',
- ['-c', 'core.symlinks=false', 'clone', '-q', '--depth', '1', '--', spec.url, tmp],
- {
- timeout: 60_000,
- maxBuffer: 16 * 1024 * 1024,
- env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
- },
- );
- } catch (e) {
- rmSync(tmp, { recursive: true, force: true });
- const msg = e instanceof Error ? e.message : String(e);
- throw new SkillFetchError(`git clone failed for ${spec.url}: ${msg}`);
- }
+ const deadline = performance.now() + GIT_FETCH_BUDGET_MS;
+ const tmp =
+ (spec.subpath ? await cloneSubpathOnly(spec.url, spec.subpath, deadline) : undefined) ??
+ (await cloneWholeRepository(spec.url, deadline));
let ref: string | undefined;
try {
ref = execFileSync('git', ['-C', tmp, 'rev-parse', 'HEAD'], { stdio: 'pipe' })
@@ -144,3 +134,95 @@ export async function fetchSource(
}
return { dir, ref, cleanup: () => rmSync(tmp, { recursive: true, force: true }) };
}
+
+function runGit(args: readonly string[], deadline: number, stdin?: string): Promise<unknown> {
+ const timeout = Math.floor(deadline - performance.now());
+ if (timeout <= 0) {
+ return Promise.reject(
+ new Error(`git ${args.join(' ')} not started: the ${GIT_FETCH_BUDGET_MS}ms budget is spent`),
+ );
+ }
+ const run = execFileAsync('git', ['-c', 'core.symlinks=false', ...args], {
+ timeout,
+ maxBuffer: 16 * 1024 * 1024,
+ env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
+ });
+ if (stdin !== undefined) {
+ run.child.stdin?.on('error', () => {});
+ run.child.stdin?.end(stdin);
+ }
+ return run;
+}
+
+function gitCloneFailed(url: string, e: unknown): SkillFetchError {
+ const msg = e instanceof Error ? e.message : String(e);
+ return new SkillFetchError(`git clone failed for ${url}: ${msg}`);
+}
+
+async function cloneWholeRepository(url: string, deadline: number): Promise<string> {
+ const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));
+ try {
+ await runGit(['clone', '-q', '--depth', '1', '--', url, tmp], deadline);
+ } catch (e) {
+ rmSync(tmp, { recursive: true, force: true });
+ throw gitCloneFailed(url, e);
+ }
+ return tmp;
+}
+
+async function cloneSubpathOnly(
+ url: string,
+ subpath: string,
+ deadline: number,
+): Promise<string | undefined> {
+ const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));
+ const inside = relative(tmp, resolve(tmp, subpath));
+ if (inside !== '' && !inside.startsWith('..') && !isAbsolute(inside)) {
+ const sparsePath = inside.split(sep).join('/');
+ try {
+ await runGit(
+ ['clone', '-q', '--depth', '1', '--filter=blob:none', '--sparse', '--', url, tmp],
+ deadline,
+ );
+ await runGit(
+ ['-C', tmp, 'sparse-checkout', 'set', '--skip-checks', '--stdin'],
+ deadline,
+ `${sparsePath}\n`,
+ );
+ if (existsSync(join(tmp, inside)) || headCommitLacks(tmp, sparsePath)) return tmp;
+ console.warn(
+ '[skills-catalog] sparse checkout left the subpath unselected, cloning the whole repository',
+ { url, subpath },
+ );
+ } catch (e) {
+ if (budgetSpent(e, deadline)) {
+ rmSync(tmp, { recursive: true, force: true });
+ throw gitCloneFailed(url, e);
+ }
+ console.warn('[skills-catalog] sparse clone failed, cloning the whole repository', {
+ url,
+ subpath,
+ err: e,
+ });
+ }
+ }
+ rmSync(tmp, { recursive: true, force: true });
+ return undefined;
+}
+
+function budgetSpent(e: unknown, deadline: number): boolean {
+ return (
+ performance.now() >= deadline || (e instanceof Error && 'killed' in e && e.killed === true)
+ );
+}
+
+function headCommitLacks(repo: string, path: string): boolean {
+ try {
+ execFileSync('git', ['-C', repo, 'rev-parse', '--verify', '--quiet', `HEAD:${path}`], {
+ stdio: 'pipe',
+ });
+ return false;
+ } catch (e) {
+ return e instanceof Error && 'status' in e && e.status === 1;
+ }
+}