Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): fetch one skill folder when importing a repo by name (#6916)

Nick Gomezf0bb0bceb14b
diff --git a/.changeset/skill-import-by-repository-name-fetches-one-folder.md b/.changeset/skill-import-by-repository-name-fetches-one-folder.md
new file mode 100644
index 000000000..20df5e45d
--- /dev/null
+++ b/.changeset/skill-import-by-repository-name-fetches-one-folder.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Importing and updating skills from a large repository by name, such as `your-org/team-skills`, no longer fails with "Could not fetch source." at the 60-second limit, and the skill picker now lists that repository's skills even when it contains a folder named `SKILL.md`.
diff --git a/packages/core/src/skills-catalog/acquire/acquire.test.ts b/packages/core/src/skills-catalog/acquire/acquire.test.ts
index 17556b9bf..6cc1fec89 100644
--- a/packages/core/src/skills-catalog/acquire/acquire.test.ts
+++ b/packages/core/src/skills-catalog/acquire/acquire.test.ts
@@ -162,6 +162,15 @@ describe('discoverSkillDirs', () => {
);
expect(discoverSkillDirs(repo).map((s) => s.name)).toEqual(['grill-me', 'tdd']);
});
+
+ test('a folder whose SKILL.md is a directory is not a skill', () => {
+ const repo = join(root, 'skill-md-directory');
+ writeSkill(join(repo, 'skills', 'real'), 'name: real\ndescription: d');
+ mkdirSync(join(repo, 'reports', 'copied', 'SKILL.md'), { recursive: true });
+ writeFileSync(join(repo, 'reports', 'copied', 'SKILL.md', 'notes.txt'), 'notes\n');
+ expect(discoverSkillDirs(repo).map((s) => s.name)).toEqual(['real']);
+ expect(discoverSkillDirs(join(repo, 'reports', 'copied'))).toEqual([]);
+ });
});
describe('parseSource', () => {
diff --git a/packages/core/src/skills-catalog/acquire/errors.ts b/packages/core/src/skills-catalog/acquire/errors.ts
index f636d5434..12c346e16 100644
--- a/packages/core/src/skills-catalog/acquire/errors.ts
+++ b/packages/core/src/skills-catalog/acquire/errors.ts
@@ -1 +1,3 @@
export class SkillFetchError extends Error {}
+
+export class SkillSubpathNotFoundError extends SkillFetchError {}
diff --git a/packages/core/src/skills-catalog/acquire/fetch-git.test.ts b/packages/core/src/skills-catalog/acquire/fetch-git.test.ts
index f89f36715..c9b19f491 100644
--- a/packages/core/src/skills-catalog/acquire/fetch-git.test.ts
+++ b/packages/core/src/skills-catalog/acquire/fetch-git.test.ts
@@ -5,6 +5,7 @@ import {
lstatSync,
mkdirSync,
mkdtempSync,
+ readdirSync,
rmSync,
statSync,
symlinkSync,
@@ -12,7 +13,7 @@ import {
} from 'node:fs';
import { type AddressInfo, createServer, type Server } from 'node:net';
import { tmpdir } from 'node:os';
-import { dirname, join } from 'node:path';
+import { dirname, join, relative, sep } from 'node:path';
import {
afterAll,
afterEach,
@@ -21,12 +22,19 @@ import {
describe,
expect,
type MockInstance,
+ onTestFinished,
test,
vi,
} from 'vitest';
import { configureTestGitRepository } from '../../../../../test-support/configure-git-fixture.test-helper.ts';
-import { fetchSource } from './fetch.ts';
-import { parseSkillDir } from './parse.ts';
+import { REFUSED_LOOPBACK_ORIGIN } from '../../../../../test-support/refused-loopback.test-helper.ts';
+import {
+ fetchSource,
+ fetchSourceListing,
+ SkillFetchError,
+ SkillSubpathNotFoundError,
+} from './fetch.ts';
+import { discoverSkillDirs, parseSkillDir, readSkillDirMeta } from './parse.ts';
const OUTSIDE_SUBPATH_BLOB_BYTES = 2_000_000;
const POSIX_NAMES = process.platform !== 'win32';
@@ -50,6 +58,10 @@ function git(cwd: string, args: readonly string[]): void {
execFileSync('git', args, { cwd, stdio: 'pipe', env: hermeticGitEnv });
}
+function gitWithInput(cwd: string, args: readonly string[], input: string): string {
+ return execFileSync('git', args, { cwd, input, env: hermeticGitEnv }).toString().trim();
+}
+
function writeRepoFile(repo: string, path: string, body: string | Buffer): void {
mkdirSync(dirname(join(repo, path)), { recursive: true });
writeFileSync(join(repo, path), body);
@@ -75,6 +87,17 @@ function servedUrl(repo: 'filtering' | 'non-filtering'): string {
return `git://127.0.0.1:${port}/${repo}.git`;
}
+function listedSkills(dir: string): Array<{ folder: string; name: string; description: string }> {
+ return discoverSkillDirs(dir).map((d) => {
+ const meta = readSkillDirMeta(d.dir);
+ return {
+ folder: relative(dir, d.dir).split(sep).join('/'),
+ name: meta?.name ?? d.name,
+ description: meta?.description ?? '',
+ };
+ });
+}
+
function untilClosed(child: ChildProcess): Promise<void> {
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
return new Promise((resolve) => child.once('close', () => resolve()));
@@ -89,10 +112,22 @@ beforeAll(async () => {
writeRepoFile(source, 'skills/alpha/SKILL.md', '---\nname: alpha\ndescription: a\n---\n');
writeRepoFile(source, 'skills/alpha/references/guide.md', 'guide\n');
writeRepoFile(source, 'skills/beta/SKILL.md', '---\nname: beta\ndescription: b\n---\n');
+ writeRepoFile(
+ source,
+ 'skills/beta/upstream/SKILL.md',
+ '---\nname: nested\ndescription: n\n---\n',
+ );
+ writeRepoFile(
+ source,
+ 'tools/scripts/helper/SKILL.md',
+ '---\nname: helper\ndescription: h\n---\n',
+ );
writeRepoFile(source, 'assets/blob.bin', randomBytes(OUTSIDE_SUBPATH_BLOB_BYTES));
if (POSIX_NAMES) {
symlinkSync('/etc/passwd', join(source, 'skills/alpha/link'));
writeRepoFile(source, '"quoted"/SKILL.md', '---\nname: quoted\ndescription: q\n---\n');
+ mkdirSync(join(source, 'linked'));
+ symlinkSync('/etc/passwd', join(source, 'linked/SKILL.md'));
}
git(source, ['add', '-A']);
git(source, ['-c', '[email protected]', '-c', 'user.name=t', 'commit', '-q', '-m', 'init']);
@@ -150,6 +185,7 @@ beforeEach(() => {
vi.stubEnv('GIT_CONFIG_GLOBAL', '/dev/null');
vi.stubEnv('GIT_CONFIG_SYSTEM', '/dev/null');
vi.stubEnv('GIT_CONFIG_NOSYSTEM', '1');
+ vi.stubEnv('LC_ALL', 'C');
});
afterEach(async () => {
@@ -281,3 +317,300 @@ describe('fetchSource from a git server', () => {
},
);
});
+
+async function wholeCloneSkills(): Promise<ReturnType<typeof listedSkills>> {
+ const whole = await fetchSource({ kind: 'git', url: servedUrl('filtering') });
+ try {
+ return listedSkills(whole.dir);
+ } finally {
+ whole.cleanup();
+ bytesServed = 0;
+ }
+}
+
+describe('fetchSourceListing from a git server', () => {
+ test('a repository listing finds the same skills as a whole clone but downloads only their SKILL.md files', async () => {
+ const expected = await wholeCloneSkills();
+ expect(expected.map((s) => s.folder).sort()).toEqual(
+ [...(POSIX_NAMES ? ['"quoted"', 'linked'] : []), 'skills/alpha', 'skills/beta'].sort(),
+ );
+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });
+ try {
+ expect(listedSkills(listing.dir)).toEqual(expected);
+ expect(existsSync(join(listing.dir, 'skills', 'alpha', 'SKILL.md'))).toBe(true);
+ expect(existsSync(join(listing.dir, 'skills', 'alpha', 'references', 'guide.md'))).toBe(
+ false,
+ );
+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(warn).not.toHaveBeenCalled();
+ } finally {
+ listing.cleanup();
+ }
+ });
+
+ test('a listing from a server that ignores the blob filter finds the same skills', async () => {
+ const expected = await wholeCloneSkills();
+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('non-filtering') });
+ try {
+ expect(bytesServed).toBeGreaterThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(listedSkills(listing.dir)).toEqual(expected);
+ expect(warn).not.toHaveBeenCalled();
+ } finally {
+ listing.cleanup();
+ }
+ });
+
+ test('a listing from a server that cannot send SKILL.md files on demand falls back to the whole repository', async () => {
+ const expected = await wholeCloneSkills();
+ vi.stubEnv('GIT_CONFIG_COUNT', '1');
+ vi.stubEnv('GIT_CONFIG_KEY_0', 'protocol.version');
+ vi.stubEnv('GIT_CONFIG_VALUE_0', '0');
+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });
+ try {
+ expect(warn).toHaveBeenCalledWith(
+ '[skills-catalog] skill listing clone failed, cloning the whole repository',
+ expect.objectContaining({ err: expect.any(Error) }),
+ );
+ expect(listing.fetchFolders).toBeUndefined();
+ expect(listedSkills(listing.dir)).toEqual(expected);
+ expect(existsSync(join(listing.dir, 'skills', 'alpha', 'references', 'guide.md'))).toBe(true);
+ } finally {
+ listing.cleanup();
+ }
+ });
+
+ test.runIf(POSIX_NAMES)('a symlinked SKILL.md in a listing arrives as a plain file', async () => {
+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });
+ try {
+ expect(existsSync(join(listing.dir, 'linked', 'SKILL.md'))).toBe(true);
+ expect(lstatSync(join(listing.dir, 'linked', 'SKILL.md')).isSymbolicLink()).toBe(false);
+ } finally {
+ listing.cleanup();
+ }
+ });
+
+ test('a listing refuses a transport outside the allow-list without running git', async () => {
+ const marker = join(root, 'ext-transport-ran');
+ await expect(
+ fetchSourceListing({ kind: 'git', url: `ext::sh -c touch% ${marker}` }),
+ ).rejects.toThrow('Unsupported git transport');
+ expect(existsSync(marker)).toBe(false);
+ });
+
+ test('a source that names a folder is fetched exactly as fetchSource fetches it', async () => {
+ const listing = await fetchSourceListing({
+ kind: 'git',
+ url: servedUrl('filtering'),
+ subpath: 'skills/alpha',
+ });
+ try {
+ expect(listing.fetchFolders).toBeUndefined();
+ expect(parseSkillDir(listing.dir)?.name).toBe('alpha');
+ expect(existsSync(join(listing.dir, 'references', 'guide.md'))).toBe(true);
+ expect(existsSync(join(listing.dir, '..', 'beta'))).toBe(false);
+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ } finally {
+ listing.cleanup();
+ }
+ });
+
+ test('fetchFolders checks out only the chosen skill folders, whole, without downloading the rest', async () => {
+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });
+ const fetchFolders = listing.fetchFolders;
+ if (fetchFolders === undefined) throw new Error('expected a partial listing');
+ try {
+ const content = await fetchFolders(['skills/alpha', 'skills/beta']);
+ try {
+ expect(parseSkillDir(join(content.dir, 'skills', 'alpha'))?.name).toBe('alpha');
+ expect(existsSync(join(content.dir, 'skills', 'alpha', 'references', 'guide.md'))).toBe(
+ true,
+ );
+ expect(parseSkillDir(join(content.dir, 'skills', 'beta'))?.name).toBe('beta');
+ expect(existsSync(join(content.dir, 'assets', 'blob.bin'))).toBe(false);
+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);
+ expect(content.ref).toBe(listing.ref);
+ if (POSIX_NAMES) {
+ expect(lstatSync(join(content.dir, 'skills', 'alpha', 'link')).isSymbolicLink()).toBe(
+ false,
+ );
+ }
+ } finally {
+ content.cleanup();
+ }
+ } finally {
+ listing.cleanup();
+ }
+ });
+
+ test('fetchFolders refuses a folder that escapes the repository before contacting the server', async () => {
+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });
+ const fetchFolders = listing.fetchFolders;
+ if (fetchFolders === undefined) throw new Error('expected a partial listing');
+ try {
+ const served = bytesServed;
+ await expect(fetchFolders(['skills/alpha', '../../../etc'])).rejects.toThrow(
+ 'subpath escapes the clone: ../../../etc',
+ );
+ expect(bytesServed).toBe(served);
+ } finally {
+ listing.cleanup();
+ }
+ });
+
+ test('fetchFolders reports a listed folder removed upstream since the listing as not found and keeps no clone', async () => {
+ const name = `changing-${randomBytes(6).toString('hex')}`;
+ const bare = join(root, 'served', `${name}.git`);
+ git(root, ['clone', '-q', '--bare', join(root, 'source'), bare]);
+ configureTestGitRepository(bare);
+ git(bare, ['config', 'uploadpack.allowFilter', 'true']);
+ const listing = await fetchSourceListing({
+ kind: 'git',
+ url: `git://127.0.0.1:${port}/${name}.git`,
+ });
+ const fetchFolders = listing.fetchFolders;
+ if (fetchFolders === undefined) throw new Error('expected a partial listing');
+ try {
+ expect(listedSkills(listing.dir).map((s) => s.folder)).toContain('skills/beta');
+ const work = join(root, `${name}-work`);
+ git(root, ['clone', '-q', bare, work]);
+ configureTestGitRepository(work);
+ git(work, ['rm', '-q', '-r', 'skills/beta']);
+ git(work, ['-c', '[email protected]', '-c', 'user.name=t', 'commit', '-q', '-m', 'drop beta']);
+ git(work, ['push', '-q', 'origin', 'HEAD']);
+
+ const scratch = mkdtempSync(join(root, 'fetch-folders-tmp-'));
+ vi.stubEnv('TMPDIR', scratch);
+ vi.stubEnv('TMP', scratch);
+ vi.stubEnv('TEMP', scratch);
+ const present = await fetchFolders(['skills/alpha']);
+ try {
+ expect(dirname(present.dir)).toBe(scratch);
+ expect(parseSkillDir(join(present.dir, 'skills', 'alpha'))?.name).toBe('alpha');
+ } finally {
+ present.cleanup();
+ }
+ expect(readdirSync(scratch)).toEqual([]);
+
+ const failure = await fetchFolders(['skills/alpha', 'skills/beta']).then(
+ () => undefined,
+ (e: unknown) => e,
+ );
+ expect(failure).toBeInstanceOf(SkillSubpathNotFoundError);
+ expect(failure).toMatchObject({ message: 'subpath not found in repo: skills/beta' });
+ expect(readdirSync(scratch)).toEqual([]);
+ } finally {
+ listing.cleanup();
+ }
+ });
+});
+
+describe('git fetch errors', () => {
+ test('a missing subpath is reported with its own error type', async () => {
+ await expect(
+ fetchSource({ kind: 'git', url: servedUrl('filtering'), subpath: 'skills/missing' }),
+ ).rejects.toBeInstanceOf(SkillSubpathNotFoundError);
+ });
+
+ test('a whole-repository clone failure gives the reason git printed once, without the git command line or temporary folder', async () => {
+ const missing = `git://127.0.0.1:${port}/missing.git`;
+ for (const attempt of [
+ () => fetchSource({ kind: 'git', url: missing, subpath: 'skills/alpha' }),
+ () => fetchSourceListing({ kind: 'git', url: missing }),
+ ]) {
+ const error = await attempt().then(
+ () => undefined,
+ (e: unknown) => e,
+ );
+ expect(error).toBeInstanceOf(SkillFetchError);
+ const message = (error as Error).message;
+ expect(message).toMatch(
+ /^git clone failed for git:\/\/127\.0\.0\.1:\d+\/missing\.git: fatal: remote error: [^\n]*\/missing\.git$/,
+ );
+ expect(message.match(/fatal: remote error/g)).toHaveLength(1);
+ expect(message).not.toContain('Fetching only part of the repository');
+ expect(message).not.toContain('Command failed');
+ expect(message).not.toContain('ok-skill-import-');
+ }
+ });
+
+ test('when the whole-repository clone fails for another reason, the error also says why the partial fetch failed first', async () => {
+ const name = `unsafe-path-${randomBytes(6).toString('hex')}`;
+ const bare = join(root, 'served', `${name}.git`);
+ git(root, ['init', '-q', '--bare', bare]);
+ configureTestGitRepository(bare);
+ git(bare, ['config', 'uploadpack.allowFilter', 'true']);
+ const skill = gitWithInput(
+ bare,
+ ['hash-object', '-w', '--stdin'],
+ '---\nname: unsafe\ndescription: u\n---\n',
+ );
+ const folder = gitWithInput(bare, ['mktree'], `100644 blob ${skill}\tSKILL.md\n`);
+ const reserved = gitWithInput(bare, ['mktree'], `040000 tree ${folder}\tGIT~1\n`);
+ const top = gitWithInput(bare, ['mktree'], `040000 tree ${reserved}\tunsafe\n`);
+ const commit = gitWithInput(
+ bare,
+ ['-c', '[email protected]', '-c', 'user.name=t', 'commit-tree', '-m', 'init', top],
+ '',
+ );
+ git(bare, ['update-ref', 'refs/heads/main', commit]);
+ git(bare, ['symbolic-ref', 'HEAD', 'refs/heads/main']);
+ const url = `git://127.0.0.1:${port}/${name}.git`;
+
+ const error = await fetchSourceListing({ kind: 'git', url }).then(
+ () => undefined,
+ (e: unknown) => e,
+ );
+
+ expect(error).toBeInstanceOf(SkillFetchError);
+ const message = (error as Error).message;
+ const partialReason = "error: invalid path 'unsafe/GIT~1/SKILL.md'";
+ const opening = `git clone failed for ${url}: ${partialReason}\n`;
+ expect(message.slice(0, opening.length)).toBe(opening);
+ expect(message).toContain('fatal: unable to checkout working tree');
+ const closing = `. Fetching only part of the repository had failed first: ${partialReason}`;
+ expect(message.slice(-closing.length)).toBe(closing);
+ expect(message).not.toContain('Command failed');
+ expect(message).not.toContain('ok-skill-import-');
+ });
+
+ test('a fetch that runs out of time says it hit the 60-second limit instead of listing git arguments', async () => {
+ for (const attempt of [
+ () => fetchSource({ kind: 'git', url: servedUrl('filtering') }),
+ () => fetchSource({ kind: 'git', url: servedUrl('filtering'), subpath: 'skills/alpha' }),
+ () => fetchSourceListing({ kind: 'git', url: servedUrl('filtering') }),
+ ]) {
+ const clock = vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValue(61_000);
+ onTestFinished(() => clock.mockRestore());
+ const error = await attempt().then(
+ () => undefined,
+ (e: unknown) => e,
+ );
+ clock.mockRestore();
+ expect(error).toBeInstanceOf(SkillFetchError);
+ expect((error as Error).message).toBe(
+ `Fetching ${servedUrl('filtering')} hit the 60-second limit.`,
+ );
+ }
+ expect(bytesServed).toBe(0);
+ });
+
+ test('credentials in a source URL stay out of errors and warnings', async () => {
+ const url = `${REFUSED_LOOPBACK_ORIGIN.replace('http://', 'http://someone:hunter2-secret@')}/team/skills.git`;
+ for (const attempt of [
+ () => fetchSource({ kind: 'git', url, subpath: 'skills/alpha' }),
+ () => fetchSourceListing({ kind: 'git', url }),
+ ]) {
+ const error = await attempt().then(
+ () => undefined,
+ (e: unknown) => e,
+ );
+ expect(error).toBeInstanceOf(SkillFetchError);
+ expect((error as Error).message).toContain('http://***@127.0.0.1:4/team/skills.git');
+ expect((error as Error).message).not.toContain('hunter2-secret');
+ }
+ expect(warn).toHaveBeenCalledTimes(2);
+ expect(
+ JSON.stringify(warn.mock.calls, (_k, v) => (v instanceof Error ? v.message : v)),
+ ).not.toContain('hunter2-secret');
+ });
+});
diff --git a/packages/core/src/skills-catalog/acquire/fetch.ts b/packages/core/src/skills-catalog/acquire/fetch.ts
index 37c384819..b4e03d91b 100644
--- a/packages/core/src/skills-catalog/acquire/fetch.ts
+++ b/packages/core/src/skills-catalog/acquire/fetch.ts
@@ -1,10 +1,10 @@
import { execFile, execFileSync } from 'node:child_process';
-import { existsSync, mkdtempSync, rmSync } from 'node:fs';
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { homedir, tmpdir } from 'node:os';
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
import { promisify } from 'node:util';
import { isLocalSkillSource } from '../source-fields.ts';
-import { SkillFetchError } from './errors.ts';
+import { SkillFetchError, SkillSubpathNotFoundError } from './errors.ts';
import {
fetchWellKnownSkill,
type WellKnownFetchOptions,
@@ -23,7 +23,7 @@ export interface SkillsShSourceRef {
skill: string;
}
-export { SkillFetchError } from './errors.ts';
+export { SkillFetchError, SkillSubpathNotFoundError } from './errors.ts';
const GIT_FETCH_BUDGET_MS = 60_000;
@@ -108,20 +108,10 @@ export async function fetchSource(
if (!existsSync(dir)) throw new SkillFetchError(`Local path not found: ${dir}`);
return { dir, cleanup: () => {} };
}
- const hasTransportScheme = /^[a-z][a-z0-9+.-]+:/i.test(spec.url);
- if (hasTransportScheme && !ALLOWED_GIT_TRANSPORTS.some((p) => p.test(spec.url))) {
- throw new SkillFetchError(`Unsupported git transport: ${spec.url}`);
- }
+ assertAllowedGitTransport(spec.url);
const deadline = performance.now() + GIT_FETCH_BUDGET_MS;
- const tmp =
- (spec.subpath ? await cloneSubpathOnly(spec.url, spec.subpath, deadline) : undefined) ??
- (await cloneWholeRepository(spec.url, deadline));
- let ref: string | undefined;
- try {
- ref = execFileSync('git', ['-C', tmp, 'rev-parse', 'HEAD'], { stdio: 'pipe' })
- .toString()
- .trim();
- } catch {}
+ const tmp = await cloneFolders(spec.url, spec.subpath ? [spec.subpath] : [], deadline);
+ const ref = headRef(tmp);
const dir = spec.subpath ? resolve(tmp, spec.subpath) : tmp;
const containment = relative(tmp, dir);
if (containment.startsWith('..') || isAbsolute(containment)) {
@@ -130,11 +120,74 @@ export async function fetchSource(
}
if (!existsSync(dir)) {
rmSync(tmp, { recursive: true, force: true });
- throw new SkillFetchError(`subpath not found in repo: ${spec.subpath}`);
+ throw new SkillSubpathNotFoundError(`subpath not found in repo: ${spec.subpath}`);
}
return { dir, ref, cleanup: () => rmSync(tmp, { recursive: true, force: true }) };
}
+export interface FetchedListing extends Fetched {
+ readonly fetchFolders?: (folders: readonly string[]) => Promise<Fetched>;
+}
+
+export async function fetchSourceListing(
+ spec: SourceSpec,
+ opts: WellKnownFetchOptions = {},
+): Promise<FetchedListing> {
+ if (spec.kind !== 'git' || spec.subpath) return fetchSource(spec, opts);
+ const { url } = spec;
+ assertAllowedGitTransport(url);
+ const deadline = performance.now() + GIT_FETCH_BUDGET_MS;
+ const listed = await cloneSkillFilesOnly(url, deadline);
+ const tmp =
+ typeof listed === 'string' ? listed : await cloneWholeRepository(url, deadline, listed.failure);
+ const fetched = {
+ dir: tmp,
+ ref: headRef(tmp),
+ cleanup: () => rmSync(tmp, { recursive: true, force: true }),
+ };
+ if (typeof listed !== 'string') return fetched;
+ return {
+ ...fetched,
+ fetchFolders: async (folders) => {
+ const escaping = folders.find((folder) => {
+ const inside = relative(tmp, resolve(tmp, folder));
+ return inside.startsWith('..') || isAbsolute(inside);
+ });
+ if (escaping !== undefined) {
+ throw new SkillFetchError(`subpath escapes the clone: ${escaping}`);
+ }
+ const dir = await cloneFolders(url, folders, deadline);
+ const missing = folders.find((folder) => !existsSync(resolve(dir, folder)));
+ if (missing !== undefined) {
+ rmSync(dir, { recursive: true, force: true });
+ throw new SkillSubpathNotFoundError(`subpath not found in repo: ${missing}`);
+ }
+ return {
+ dir,
+ ref: headRef(dir),
+ cleanup: () => rmSync(dir, { recursive: true, force: true }),
+ };
+ },
+ };
+}
+
+function assertAllowedGitTransport(url: string): void {
+ const hasTransportScheme = /^[a-z][a-z0-9+.-]+:/i.test(url);
+ if (hasTransportScheme && !ALLOWED_GIT_TRANSPORTS.some((p) => p.test(url))) {
+ throw new SkillFetchError(`Unsupported git transport: ${url}`);
+ }
+}
+
+function headRef(repo: string): string | undefined {
+ try {
+ return execFileSync('git', ['-C', repo, 'rev-parse', 'HEAD'], { stdio: 'pipe' })
+ .toString()
+ .trim();
+ } catch {
+ return undefined;
+ }
+}
+
function runGit(args: readonly string[], deadline: number, stdin?: string): Promise<unknown> {
const timeout = Math.floor(deadline - performance.now());
if (timeout <= 0) {
@@ -154,60 +207,142 @@ function runGit(args: readonly string[], deadline: number, stdin?: string): Prom
return run;
}
-function gitCloneFailed(url: string, e: unknown): SkillFetchError {
- const msg = e instanceof Error ? e.message : String(e);
- return new SkillFetchError(`git clone failed for ${url}: ${msg}`);
+function withoutUrlCredentials(text: string): string {
+ return text.replace(/(\bhttps?:\/\/)[^\s/@]+@/gi, '$1***@');
+}
+
+function gitFailureReason(e: unknown): string {
+ const stderr =
+ e instanceof Error && 'stderr' in e && typeof e.stderr === 'string' ? e.stderr.trim() : '';
+ return withoutUrlCredentials(stderr !== '' ? stderr : e instanceof Error ? e.message : String(e));
+}
+
+function gitCloneFailed(url: string, e: unknown, earlierFailure?: string): SkillFetchError {
+ const reason = gitFailureReason(e);
+ const earlier =
+ earlierFailure === undefined || earlierFailure === reason
+ ? ''
+ : `. Fetching only part of the repository had failed first: ${earlierFailure}`;
+ return new SkillFetchError(
+ withoutUrlCredentials(`git clone failed for ${url}: ${reason}${earlier}`),
+ );
+}
+
+function gitFetchTimedOut(url: string): SkillFetchError {
+ return new SkillFetchError(
+ withoutUrlCredentials(`Fetching ${url} hit the ${GIT_FETCH_BUDGET_MS / 1000}-second limit.`),
+ );
}
-async function cloneWholeRepository(url: string, deadline: number): Promise<string> {
+async function cloneWholeRepository(
+ url: string,
+ deadline: number,
+ earlierFailure?: string,
+): Promise<string> {
const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));
try {
await runGit(['clone', '-q', '--depth', '1', '--', url, tmp], deadline);
} catch (e) {
rmSync(tmp, { recursive: true, force: true });
- throw gitCloneFailed(url, e);
+ throw budgetSpent(e, deadline) ? gitFetchTimedOut(url) : gitCloneFailed(url, e, earlierFailure);
}
return tmp;
}
-async function cloneSubpathOnly(
+async function cloneFolders(
url: string,
- subpath: string,
+ folders: readonly string[],
deadline: number,
-): Promise<string | undefined> {
+): Promise<string> {
+ const sparse = folders.length > 0 ? await cloneOnlyFolders(url, folders, deadline) : {};
+ return typeof sparse === 'string' ? sparse : cloneWholeRepository(url, deadline, sparse.failure);
+}
+
+async function cloneOnlyFolders(
+ url: string,
+ folders: readonly string[],
+ deadline: number,
+): Promise<string | { failure?: string }> {
const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));
- const inside = relative(tmp, resolve(tmp, subpath));
- if (inside !== '' && !inside.startsWith('..') && !isAbsolute(inside)) {
- const sparsePath = inside.split(sep).join('/');
- try {
- await runGit(
- ['clone', '-q', '--depth', '1', '--filter=blob:none', '--sparse', '--', url, tmp],
- deadline,
- );
- await runGit(
- ['-C', tmp, 'sparse-checkout', 'set', '--skip-checks', '--stdin'],
- deadline,
- `${sparsePath}\n`,
- );
- if (existsSync(join(tmp, inside)) || headCommitLacks(tmp, sparsePath)) return tmp;
- console.warn(
- '[skills-catalog] sparse checkout left the subpath unselected, cloning the whole repository',
- { url, subpath },
- );
- } catch (e) {
- if (budgetSpent(e, deadline)) {
- rmSync(tmp, { recursive: true, force: true });
- throw gitCloneFailed(url, e);
- }
- console.warn('[skills-catalog] sparse clone failed, cloning the whole repository', {
- url,
- subpath,
- err: e,
- });
- }
+ const inside = folders.map((folder) => relative(tmp, resolve(tmp, folder)));
+ if (inside.some((p) => p === '' || p.startsWith('..') || isAbsolute(p))) {
+ rmSync(tmp, { recursive: true, force: true });
+ return {};
+ }
+ const sparsePaths = inside.map((p) => p.split(sep).join('/'));
+ const subpath = folders.join(', ');
+ try {
+ await runGit(
+ ['clone', '-q', '--depth', '1', '--filter=blob:none', '--sparse', '--', url, tmp],
+ deadline,
+ );
+ await runGit(
+ ['-C', tmp, 'sparse-checkout', 'set', '--skip-checks', '--stdin'],
+ deadline,
+ sparsePaths.map((p) => `${p}\n`).join(''),
+ );
+ const unselected = folders.filter(
+ (_, i) =>
+ !existsSync(join(tmp, inside[i] as string)) &&
+ !headCommitLacks(tmp, sparsePaths[i] as string),
+ );
+ if (unselected.length === 0) return tmp;
+ console.warn(
+ '[skills-catalog] sparse checkout left the subpath unselected, cloning the whole repository',
+ { url: withoutUrlCredentials(url), subpath: unselected.join(', ') },
+ );
+ rmSync(tmp, { recursive: true, force: true });
+ return { failure: `sparse checkout left ${unselected.join(', ')} unselected` };
+ } catch (e) {
+ rmSync(tmp, { recursive: true, force: true });
+ if (budgetSpent(e, deadline)) throw gitFetchTimedOut(url);
+ const failure = gitFailureReason(e);
+ console.warn('[skills-catalog] sparse clone failed, cloning the whole repository', {
+ url: withoutUrlCredentials(url),
+ subpath,
+ err: new Error(failure),
+ });
+ return { failure };
+ }
+}
+
+async function cloneSkillFilesOnly(
+ url: string,
+ deadline: number,
+): Promise<string | { failure: string }> {
+ const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));
+ try {
+ await runGit(
+ ['clone', '-q', '--depth', '1', '--filter=blob:none', '--no-checkout', '--', url, tmp],
+ deadline,
+ );
+ mkdirSync(join(tmp, '.git', 'info'), { recursive: true });
+ writeFileSync(join(tmp, '.git', 'info', 'sparse-checkout'), 'SKILL.md\n');
+ await runGit(
+ [
+ '-c',
+ 'core.sparseCheckout=true',
+ '-c',
+ 'core.sparseCheckoutCone=false',
+ '-C',
+ tmp,
+ 'read-tree',
+ '-mu',
+ 'HEAD',
+ ],
+ deadline,
+ );
+ return tmp;
+ } catch (e) {
+ rmSync(tmp, { recursive: true, force: true });
+ if (budgetSpent(e, deadline)) throw gitFetchTimedOut(url);
+ const failure = gitFailureReason(e);
+ console.warn('[skills-catalog] skill listing clone failed, cloning the whole repository', {
+ url: withoutUrlCredentials(url),
+ err: new Error(failure),
+ });
+ return { failure };
}
- rmSync(tmp, { recursive: true, force: true });
- return undefined;
}
function budgetSpent(e: unknown, deadline: number): boolean {
diff --git a/packages/core/src/skills-catalog/acquire/lockfile.ts b/packages/core/src/skills-catalog/acquire/lockfile.ts
index 62b6a1711..1874d20fb 100644
--- a/packages/core/src/skills-catalog/acquire/lockfile.ts
+++ b/packages/core/src/skills-catalog/acquire/lockfile.ts
@@ -16,6 +16,7 @@ export const SkillLockEntrySchema = z.looseObject({
source: z.string(),
pluginProvider: z.string().optional(),
skill: z.string().optional(),
+ subpath: z.string().optional(),
ref: z.string().optional(),
contentHash: z.string(),
files: z.array(z.string()).optional(),
diff --git a/packages/core/src/skills-catalog/acquire/parse.ts b/packages/core/src/skills-catalog/acquire/parse.ts
index 5622c7b95..57760ca6a 100644
--- a/packages/core/src/skills-catalog/acquire/parse.ts
+++ b/packages/core/src/skills-catalog/acquire/parse.ts
@@ -190,15 +190,23 @@ function acquiredSkill(dir: string, skillMd: string, files: AcquiredFile[]): Acq
return { name, description, skillMd, files, contentHash: hash.digest('hex') };
}
+function hasSkillFile(dir: string): boolean {
+ try {
+ return statSync(join(dir, 'SKILL.md')).isFile();
+ } catch {
+ return false;
+ }
+}
+
export function discoverSkillDirs(root: string): Array<{ name: string; dir: string }> {
- if (existsSync(join(root, 'SKILL.md'))) return [{ name: basename(root), dir: root }];
+ if (hasSkillFile(root)) return [{ name: basename(root), dir: root }];
const found: Array<{ name: string; dir: string }> = [];
const MAX_DEPTH = 8;
const MAX_DIRS = 5_000;
let visited = 0;
const walk = (dir: string, depth: number) => {
if (depth > MAX_DEPTH || ++visited > MAX_DIRS) return;
- if (existsSync(join(dir, 'SKILL.md'))) {
+ if (hasSkillFile(dir)) {
found.push({ name: basename(dir), dir });
return;
}
diff --git a/packages/core/src/skills-catalog/index.ts b/packages/core/src/skills-catalog/index.ts
index cf4d8f5f0..826cbb4c8 100644
--- a/packages/core/src/skills-catalog/index.ts
+++ b/packages/core/src/skills-catalog/index.ts
@@ -1,10 +1,13 @@
export {
ALLOWED_GIT_TRANSPORTS,
type Fetched,
+ type FetchedListing,
fetchSource,
+ fetchSourceListing,
parseSkillsShSource,
parseSource,
SkillFetchError,
+ SkillSubpathNotFoundError,
type SkillsShSourceRef,
type SourceSpec,
} from './acquire/fetch.ts';
diff --git a/packages/server/src/api-skill-git-source-composition.test.ts b/packages/server/src/api-skill-git-source-composition.test.ts
new file mode 100644
index 000000000..d42a00c96
--- /dev/null
+++ b/packages/server/src/api-skill-git-source-composition.test.ts
@@ -0,0 +1,327 @@
+import { randomBytes } from 'node:crypto';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, afterEach, beforeAll, beforeEach, expect, test, vi } from 'vitest';
+import {
+ type ServedGitRepos,
+ serveGitRepos,
+} from '../../../test-support/served-git-repos.test-helper.ts';
+import type { BootedServer } from './boot.ts';
+import { bootCompositionRig, rawRequest } from './composition-rig.test-helper.ts';
+
+const OUTSIDE_SKILLS_BYTES = 2_000_000;
+const LARGE_SKILL_MD_BYTES = 1_000_000;
+const skillMd = (name: string, revision = 'first') =>
+ `---\nname: ${name}\ndescription: ${name} skill\n---\n\n${revision} revision of ${name}.\n`;
+
+function publishTeamRepo(repo: string, folders: Readonly<Record<string, string>>): string {
+ const files: Record<string, string | Buffer> = {
+ 'README.md': 'Team skills\n',
+ 'assets/blob.bin': randomBytes(OUTSIDE_SKILLS_BYTES),
+ };
+ for (const [folder, name] of Object.entries(folders)) {
+ files[`${folder}/SKILL.md`] = skillMd(name);
+ files[`${folder}/references/guide.md`] = `${name} guide\n`;
+ }
+ repos.publish(repo, files);
+ return repo;
+}
+
+let root: string;
+let server: BootedServer;
+let repos: ServedGitRepos;
+
+beforeAll(async () => {
+ repos = await serveGitRepos();
+ root = mkdtempSync(join(tmpdir(), 'ok-git-source-contract-'));
+ mkdirSync(join(root, '.claude'));
+ server = await bootCompositionRig(root);
+ await server.ready;
+}, 60_000);
+
+beforeEach(() => {
+ for (const name of Object.keys(process.env)) {
+ if (name.startsWith('GIT_')) vi.stubEnv(name, undefined);
+ }
+ for (const [name, value] of Object.entries(repos.githubRedirectEnv())) vi.stubEnv(name, value);
+ repos.resetBytesServed();
+});
+
+afterEach(() => {
+ vi.unstubAllEnvs();
+});
+
+afterAll(async () => {
+ await server?.destroy();
+ await repos?.close();
+ rmSync(root, { recursive: true, force: true });
+});
+
+async function post(path: string, body: object) {
+ const result = await rawRequest(server.port, path, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ expect(result.status, result.body).toBe(200);
+ return JSON.parse(result.body);
+}
+
+const lockPath = () => join(root, '.ok/skills-lock.json');
+
+function lockEntry(name: string): Record<string, unknown> | undefined {
+ return JSON.parse(readFileSync(lockPath(), 'utf8')).skills[name];
+}
+
+function forgetRecordedFolder(name: string): void {
+ const lock = JSON.parse(readFileSync(lockPath(), 'utf8'));
+ delete lock.skills[name].subpath;
+ writeFileSync(lockPath(), `${JSON.stringify(lock, null, 2)}\n`);
+}
+
+const installed = (name: string, file = 'SKILL.md') =>
+ readFileSync(join(root, '.claude/skills', name, file), 'utf8');
+
+test('the picker lists a repository by name without downloading it whole', async () => {
+ const repo = publishTeamRepo('fixture-org/listing', {
+ 'skills/alpha': 'alpha',
+ 'skills/beta': 'beta',
+ 'skills/beta/upstream': 'nested',
+ 'tools/scripts/helper': 'helper',
+ 'plugins/eng/skills/gamma': 'gamma',
+ 'archive/one/dup': 'dup',
+ 'archive/two/dup': 'dup',
+ });
+ repos.resetBytesServed();
+ const result = await rawRequest(
+ server.port,
+ `/api/skills/discover?source=${encodeURIComponent(repo)}`,
+ );
+ expect(result.status, result.body).toBe(200);
+ expect(JSON.parse(result.body).skills).toEqual([
+ { name: 'alpha', description: 'alpha skill' },
+ { name: 'beta', description: 'beta skill' },
+ { name: 'dup', description: 'dup skill' },
+ { name: 'dup', description: 'dup skill' },
+ { name: 'gamma', description: 'gamma skill' },
+ ]);
+ expect(repos.bytesServed()).toBeGreaterThan(0);
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+});
+
+test('importing a repository by name and picking a skill records its folder and downloads only that folder', async () => {
+ const repo = publishTeamRepo('fixture-org/import', {
+ 'plugins/eng/skills/imp-gamma': 'imp-gamma',
+ 'skills/imp-other': 'imp-other',
+ });
+ repos.resetBytesServed();
+ const result = await post('/api/skill/import', {
+ source: repo,
+ skill: 'imp-gamma',
+ install: false,
+ });
+ expect(result).toMatchObject({ name: 'imp-gamma', created: true, provenance: { source: repo } });
+ expect(installed('imp-gamma')).toBe(skillMd('imp-gamma'));
+ expect(installed('imp-gamma', 'references/guide.md')).toBe('imp-gamma guide\n');
+ expect(existsSync(join(root, '.claude/skills/imp-other'))).toBe(false);
+ expect(lockEntry('imp-gamma')).toMatchObject({
+ source: repo,
+ skill: 'imp-gamma',
+ subpath: 'plugins/eng/skills/imp-gamma',
+ });
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+});
+
+test('a source that already names a folder is recorded as before, without a separate folder', async () => {
+ const repo = publishTeamRepo('fixture-org/named', { 'skills/named-alpha': 'named-alpha' });
+ repos.resetBytesServed();
+ const source = `${repo}/skills/named-alpha`;
+ const result = await post('/api/skill/import', { source, install: false });
+ expect(result).toMatchObject({ name: 'named-alpha', provenance: { source } });
+ expect(installed('named-alpha', 'references/guide.md')).toBe('named-alpha guide\n');
+ const entry = lockEntry('named-alpha');
+ expect(entry).toMatchObject({ source, skill: 'named-alpha' });
+ expect(entry).not.toHaveProperty('subpath');
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+});
+
+test('the update check and update of a skill with a recorded folder fetch only that folder', async () => {
+ const repo = publishTeamRepo('fixture-org/update', { 'skills/upd-beta': 'upd-beta' });
+ repos.publish(repo, {
+ 'skills/upd-large/SKILL.md': `${skillMd('upd-large')}\n${randomBytes(LARGE_SKILL_MD_BYTES).toString('base64')}\n`,
+ });
+ await post('/api/skill/import', { source: repo, skill: 'upd-beta', install: false });
+ repos.publish(repo, { 'skills/upd-beta/SKILL.md': skillMd('upd-beta', 'second') });
+ const lockBefore = readFileSync(lockPath(), 'utf8');
+ repos.resetBytesServed();
+
+ const check = await post('/api/skill/reimport', {
+ name: 'upd-beta',
+ scope: 'project',
+ dryRun: true,
+ });
+ expect(check).toMatchObject({ updated: true });
+ expect(check.upstreamBody).toContain('second revision of upd-beta.');
+ expect(readFileSync(lockPath(), 'utf8')).toBe(lockBefore);
+ expect(repos.bytesServed()).toBeLessThan(LARGE_SKILL_MD_BYTES);
+
+ repos.resetBytesServed();
+ expect(await post('/api/skill/reimport', { name: 'upd-beta', scope: 'project' })).toMatchObject({
+ updated: true,
+ source: repo,
+ });
+ expect(installed('upd-beta')).toBe(skillMd('upd-beta', 'second'));
+ expect(lockEntry('upd-beta')).toMatchObject({ subpath: 'skills/upd-beta' });
+ expect(repos.bytesServed()).toBeLessThan(LARGE_SKILL_MD_BYTES);
+});
+
+test('an import recorded without a folder keeps a fast update check and gains its folder at its next update', async () => {
+ const repo = publishTeamRepo('fixture-org/legacy', {
+ 'skills/leg-alpha': 'leg-alpha',
+ 'skills/leg-other': 'leg-other',
+ });
+ await post('/api/skill/import', { source: repo, skill: 'leg-alpha', install: false });
+ forgetRecordedFolder('leg-alpha');
+ repos.publish(repo, { 'skills/leg-alpha/SKILL.md': skillMd('leg-alpha', 'second') });
+ const lockBefore = readFileSync(lockPath(), 'utf8');
+ repos.resetBytesServed();
+
+ const check = await post('/api/skill/reimport', {
+ name: 'leg-alpha',
+ scope: 'project',
+ dryRun: true,
+ });
+ expect(check).toMatchObject({ updated: true });
+ expect(readFileSync(lockPath(), 'utf8')).toBe(lockBefore);
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+
+ repos.resetBytesServed();
+ expect(await post('/api/skill/reimport', { name: 'leg-alpha', scope: 'project' })).toMatchObject({
+ updated: true,
+ });
+ expect(installed('leg-alpha')).toBe(skillMd('leg-alpha', 'second'));
+ expect(installed('leg-alpha', 'references/guide.md')).toBe('leg-alpha guide\n');
+ expect(lockEntry('leg-alpha')).toMatchObject({ subpath: 'skills/leg-alpha' });
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+});
+
+test('an import recorded without a folder whose name matches two folders is not given one', async () => {
+ const repo = publishTeamRepo('fixture-org/ambiguous', {
+ 'archive/one/amb-dup': 'amb-dup',
+ 'archive/two/amb-dup': 'amb-dup',
+ });
+ await post('/api/skill/import', { source: repo, skill: 'amb-dup', install: false });
+ forgetRecordedFolder('amb-dup');
+ repos.publish(repo, {
+ 'archive/one/amb-dup/SKILL.md': skillMd('amb-dup', 'second'),
+ 'archive/two/amb-dup/SKILL.md': skillMd('amb-dup', 'second'),
+ });
+ expect(await post('/api/skill/reimport', { name: 'amb-dup', scope: 'project' })).toMatchObject({
+ updated: true,
+ });
+ expect(installed('amb-dup')).toBe(skillMd('amb-dup', 'second'));
+ expect(lockEntry('amb-dup')).not.toHaveProperty('subpath');
+});
+
+test('a recorded folder that moved is found again by name and recorded anew', async () => {
+ const repo = publishTeamRepo('fixture-org/moved', {
+ 'plugins/eng/skills/mov-gamma': 'mov-gamma',
+ });
+ await post('/api/skill/import', { source: repo, skill: 'mov-gamma', install: false });
+ expect(lockEntry('mov-gamma')).toMatchObject({ subpath: 'plugins/eng/skills/mov-gamma' });
+ repos.publish(repo, {
+ 'plugins/eng/skills/mov-gamma': null,
+ 'plugins/ops/skills/mov-gamma/SKILL.md': skillMd('mov-gamma', 'moved'),
+ });
+ expect(await post('/api/skill/reimport', { name: 'mov-gamma', scope: 'project' })).toMatchObject({
+ updated: true,
+ });
+ expect(installed('mov-gamma')).toBe(skillMd('mov-gamma', 'moved'));
+ expect(lockEntry('mov-gamma')).toMatchObject({ subpath: 'plugins/ops/skills/mov-gamma' });
+});
+
+test('a recorded folder that is gone, whose name now matches two folders, is not kept in the lock entry', async () => {
+ const repo = publishTeamRepo('fixture-org/gone-ambiguous', {
+ 'archive/one/gone-dup': 'gone-dup',
+ });
+ await post('/api/skill/import', { source: repo, skill: 'gone-dup', install: false });
+ expect(lockEntry('gone-dup')).toMatchObject({ subpath: 'archive/one/gone-dup' });
+ repos.publish(repo, {
+ 'archive/one/gone-dup': null,
+ 'archive/two/gone-dup/SKILL.md': skillMd('gone-dup', 'second'),
+ 'archive/three/gone-dup/SKILL.md': skillMd('gone-dup', 'second'),
+ });
+ expect(await post('/api/skill/reimport', { name: 'gone-dup', scope: 'project' })).toMatchObject({
+ updated: true,
+ });
+ expect(installed('gone-dup')).toBe(skillMd('gone-dup', 'second'));
+ const entry = lockEntry('gone-dup');
+ expect(entry).toMatchObject({ source: repo, skill: 'gone-dup' });
+ expect(entry).not.toHaveProperty('subpath');
+});
+
+test('Update all from this source fetches only the folders of the skills it updates', async () => {
+ const repo = publishTeamRepo('fixture-org/update-all', {
+ 'skills/all-one': 'all-one',
+ 'plugins/eng/skills/all-two': 'all-two',
+ });
+ await post('/api/skill/import', { source: repo, skill: 'all-one', install: false });
+ await post('/api/skill/import', { source: repo, skill: 'all-two', install: false });
+ forgetRecordedFolder('all-two');
+ repos.publish(repo, {
+ 'skills/all-one/SKILL.md': skillMd('all-one', 'second'),
+ 'plugins/eng/skills/all-two/SKILL.md': skillMd('all-two', 'second'),
+ });
+ repos.resetBytesServed();
+ const result = await post('/api/skills/reimport-bulk', {
+ names: ['all-one', 'all-two'],
+ scope: 'project',
+ });
+ expect(result).toMatchObject({ updated: 2, failed: 0 });
+ expect(installed('all-one')).toBe(skillMd('all-one', 'second'));
+ expect(installed('all-two')).toBe(skillMd('all-two', 'second'));
+ expect(lockEntry('all-one')).toMatchObject({ subpath: 'skills/all-one' });
+ expect(lockEntry('all-two')).toMatchObject({ subpath: 'plugins/eng/skills/all-two' });
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+});
+
+test('a bulk import by repository name records each folder and downloads only those folders', async () => {
+ const repo = publishTeamRepo('fixture-org/bulk-import', {
+ 'skills/bulk-one': 'bulk-one',
+ 'plugins/eng/skills/bulk-two': 'bulk-two',
+ });
+ repos.resetBytesServed();
+ const result = await post('/api/skills/import-bulk', {
+ source: repo,
+ skills: ['bulk-one', 'bulk-two', 'missing'],
+ install: false,
+ });
+ expect(result).toMatchObject({ imported: 2, failed: 1 });
+ expect(installed('bulk-two', 'references/guide.md')).toBe('bulk-two guide\n');
+ expect(lockEntry('bulk-one')).toMatchObject({ source: repo, subpath: 'skills/bulk-one' });
+ expect(lockEntry('bulk-two')).toMatchObject({
+ source: repo,
+ subpath: 'plugins/eng/skills/bulk-two',
+ });
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+});
+
+test('a recorded folder that would leave the repository is ignored, and the skill is found by name', async () => {
+ const repo = publishTeamRepo('fixture-org/escape', { 'skills/esc-alpha': 'esc-alpha' });
+ await post('/api/skill/import', { source: repo, skill: 'esc-alpha', install: false });
+ const lock = JSON.parse(readFileSync(lockPath(), 'utf8'));
+ lock.skills['esc-alpha'].subpath = '../../../../../../etc';
+ writeFileSync(lockPath(), `${JSON.stringify(lock, null, 2)}\n`);
+ repos.publish(repo, { 'skills/esc-alpha/SKILL.md': skillMd('esc-alpha', 'second') });
+ repos.resetBytesServed();
+
+ const check = await post('/api/skill/reimport', {
+ name: 'esc-alpha',
+ scope: 'project',
+ dryRun: true,
+ });
+ expect(check).toMatchObject({ updated: true });
+ expect(check.upstreamBody).toContain('second revision of esc-alpha.');
+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);
+});
diff --git a/packages/server/src/http/skills-import-routes.ts b/packages/server/src/http/skills-import-routes.ts
index aad18e1f5..3013039c6 100644
--- a/packages/server/src/http/skills-import-routes.ts
+++ b/packages/server/src/http/skills-import-routes.ts
@@ -14,7 +14,9 @@ import {
} from '@inkeep/open-knowledge-core';
import {
discoverSkillDirs,
+ type FetchedListing,
fetchSource,
+ fetchSourceListing,
parseSkillDir,
parseSource,
readSkillDirMeta,
@@ -33,6 +35,11 @@ import { createMultipartParser, type MultipartParser } from '../multipart.ts';
import type { SkillImportOutcome, SkillImportService } from '../services/skill-import.ts';
import { rejectDisallowedGitSpec } from '../skill-git-spec-guard.ts';
import { resolveSkillInstallReportSettings } from '../skill-install-report-config.ts';
+import {
+ fetchPickedFolders,
+ recordsSkillFolder,
+ skillFolderWithin,
+} from '../skill-source-folders.ts';
import { reportSkillInstall } from '../skills-sh-install-report.ts';
import { type ApiRouteGroup, createApiRouteGroup } from './api-pipeline.ts';
import { errorResponse } from './error-response.ts';
@@ -113,6 +120,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
let ref: string | undefined;
let publisher: string | undefined;
let upstreamSkill: string | undefined;
+ let subpath: string | undefined;
let resolvedSourceForReport = body.source;
{
@@ -138,10 +146,10 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
return;
}
if (rejectDisallowedGitSpec(res, spec, 'skill-import')) return;
- const fetched = await fetchSource(spec);
- cleanup = fetched.cleanup;
- ref = fetched.ref;
- const dirs = discoverSkillDirs(fetched.dir);
+ const listing = await fetchSourceListing(spec);
+ cleanup = listing.cleanup;
+ ref = listing.ref;
+ const dirs = discoverSkillDirs(listing.dir);
if (dirs.length === 0) {
errorResponse(res, 404, 'urn:ok:error:not-found', 'No SKILL.md found in source.', {
handler: 'skill-import',
@@ -172,7 +180,16 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
);
return;
}
- acquiredDir = pick.dir;
+ const folder = skillFolderWithin(listing.dir, pick.dir);
+ const content = await fetchPickedFolders(listing, [pick.dir]);
+ const listingCleanup = cleanup;
+ cleanup = () => {
+ content.cleanup();
+ listingCleanup();
+ };
+ ref = content.ref;
+ acquiredDir = content.dirs[0] ?? pick.dir;
+ if (recordsSkillFolder(spec) && folder !== '') subpath = folder;
upstreamSkill = pick.name;
sourceLabel = rawSource;
publisher = skillsSh?.publisher ?? publisherFromSource(resolvedSource);
@@ -201,6 +218,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
ref,
publisher,
upstreamSkill,
+ ...(subpath !== undefined ? { subpath } : {}),
actor,
skipProjection: body.install === false,
});
@@ -260,6 +278,11 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
let ref: string | undefined;
let publisher: string | undefined;
let resolvedSourceForReport = rawSource;
+ const fetchedDirs = new Map<string, { dir: string; subpath?: string }>();
+ const findRequested = (requested: string) =>
+ dirs.find((d) => d.name === requested) ??
+ dirs.find((d) => parseSkillDir(d.dir)?.name === requested) ??
+ dirs.find((d) => d.name === RENAMED_PACK_SKILLS[requested]);
try {
const skillsSh = await resolveSkillsShImportSource(rawSource, body.skills[0]);
const resolvedSource = skillsSh?.source ?? rawSource;
@@ -279,10 +302,27 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
siteSpec = spec;
siteIndex = await readWellKnownIndex(spec.origin);
} else {
- const fetched = await fetchSource(spec);
- cleanup = fetched.cleanup;
- ref = fetched.ref;
- dirs = discoverSkillDirs(fetched.dir);
+ const listing: FetchedListing = await fetchSourceListing(spec);
+ cleanup = listing.cleanup;
+ ref = listing.ref;
+ dirs = discoverSkillDirs(listing.dir);
+ const picked = [...new Set(body.skills)].flatMap((requested) => {
+ const found = findRequested(requested);
+ return found ? [found.dir] : [];
+ });
+ const content = await fetchPickedFolders(listing, picked);
+ cleanup = () => {
+ content.cleanup();
+ listing.cleanup();
+ };
+ ref = content.ref;
+ picked.forEach((dir, i) => {
+ const folder = skillFolderWithin(listing.dir, dir);
+ fetchedDirs.set(dir, {
+ dir: content.dirs[i] ?? dir,
+ ...(recordsSkillFolder(spec) && folder !== '' ? { subpath: folder } : {}),
+ });
+ });
}
} catch (e) {
if (e instanceof SkillFetchError) {
@@ -304,6 +344,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
for (const requested of new Set(body.skills)) {
let acquiredDir: string;
let upstreamSkill: string;
+ let subpath: string | undefined;
let perSkill: () => void = () => {};
if (siteSpec !== null) {
try {
@@ -325,15 +366,14 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
continue;
}
} else {
- const found =
- dirs.find((d) => d.name === requested) ??
- dirs.find((d) => parseSkillDir(d.dir)?.name === requested) ??
- dirs.find((d) => d.name === RENAMED_PACK_SKILLS[requested]);
+ const found = findRequested(requested);
if (!found) {
results.push({ requested, status: 'not-found', warnings: [], warningCodes: [] });
continue;
}
- acquiredDir = found.dir;
+ const fetched = fetchedDirs.get(found.dir);
+ acquiredDir = fetched?.dir ?? found.dir;
+ subpath = fetched?.subpath;
upstreamSkill = found.name;
}
try {
@@ -344,6 +384,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG
ref,
publisher,
upstreamSkill,
+ ...(subpath !== undefined ? { subpath } : {}),
actor,
skipProjection: body.install === false,
});
diff --git a/packages/server/src/http/skills-recovery-routes.ts b/packages/server/src/http/skills-recovery-routes.ts
index fdda4d34c..8054711a9 100644
--- a/packages/server/src/http/skills-recovery-routes.ts
+++ b/packages/server/src/http/skills-recovery-routes.ts
@@ -16,8 +16,6 @@ import {
} from '@inkeep/open-knowledge-core';
import { resolveProjectIdentity } from '@inkeep/open-knowledge-core/shadow-repo-layout';
import {
- discoverSkillDirs,
- fetchSource,
parseSkillDir,
parseSource,
resolvePluginUpdateSource,
@@ -32,16 +30,20 @@ import type { ContentFilter } from '../content-filter.ts';
import { extractActorIdentity } from '../extract-actor-identity.ts';
import { scanGlobalInPlaceSkills, scanInPlaceSkills } from '../in-place-skills.ts';
import { getLogger } from '../logger.ts';
-import type { SkillReimportDeps, SkillReimportService } from '../services/skill-reimport.ts';
+import type {
+ RecordedFolderUpdate,
+ SkillReimportDeps,
+ SkillReimportService,
+} from '../services/skill-reimport.ts';
import {
groupReimportNamesBySource,
- pickReimportDir,
type SkillReimportOutcome,
} from '../services/skill-reimport.ts';
import type { ShadowRef } from '../shadow-repo.ts';
import { isDisallowedGitSpec, rejectDisallowedGitSpec } from '../skill-git-spec-guard.ts';
import { validateSkillForInstall } from '../skill-projection.ts';
import { restoreSkillVersion } from '../skill-restore.ts';
+import { acquireReimportSources, type ReimportAcquisition } from '../skill-source-folders.ts';
import { mutateSkillsLock, readSkillsLockFile } from '../skills-lock-store.ts';
import { type ApiRouteGroup, createApiRouteGroup } from './api-pipeline.ts';
import { errorResponse } from './error-response.ts';
@@ -377,6 +379,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo
}
let ref: string | undefined;
let acquiredDir: string | null = null;
+ let folder: RecordedFolderUpdate | undefined;
try {
const recordedSkill = entry.skill ?? body.name;
const renamedSkill = RENAMED_PACK_SKILLS[recordedSkill];
@@ -401,26 +404,28 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo
return;
}
if (rejectDisallowedGitSpec(res, spec, 'skill-reimport')) return;
- const fetched = await fetchSource(spec);
- cleanup = fetched.cleanup;
- ref = fetched.ref;
- const dirs = discoverSkillDirs(fetched.dir);
- const pick = pickReimportDir(dirs, {
- ...(resolvedSkill !== undefined ? { recordedSkill: resolvedSkill } : {}),
- localName: body.name,
- frontmatterNameOf: (dir) => parseSkillDir(dir)?.name,
- });
+ const acquired = await acquireReimportSources(spec, [
+ {
+ name: body.name,
+ entry,
+ ...(resolvedSkill !== undefined ? { recordedSkill: resolvedSkill } : {}),
+ },
+ ]);
+ cleanup = acquired.cleanup;
+ ref = acquired.ref;
+ const pick = acquired.picks.get(body.name);
if (!pick) {
errorResponse(
res,
404,
'urn:ok:error:not-found',
'Could not locate this skill in its source anymore.',
- { handler: 'skill-reimport', detail: dirs.map((d) => d.name).join(', ') },
+ { handler: 'skill-reimport', detail: acquired.dirs.map((d) => d.name).join(', ') },
);
return;
}
acquiredDir = pick.dir;
+ folder = pick.folder;
} catch (e) {
if (e instanceof SkillFetchError) {
errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Could not fetch source.', {
@@ -444,6 +449,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo
entry,
lockPath,
ref,
+ ...(folder ? { folder } : {}),
actor,
...(body.dryRun !== undefined ? { dryRun: body.dryRun } : {}),
});
@@ -518,8 +524,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo
for (const group of bySource) {
let cleanup: () => void = () => {};
try {
- let ref: string | undefined;
- let dirs: ReturnType<typeof discoverSkillDirs> = [];
+ let acquired: ReimportAcquisition;
try {
const probe = group.names[0] as string;
const skillsSh = await resolveSkillsShImportSource(group.source, probe).catch(
@@ -549,10 +554,27 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo
}
continue;
}
- const fetched = await fetchSource(spec);
- cleanup = fetched.cleanup;
- ref = fetched.ref;
- dirs = discoverSkillDirs(fetched.dir);
+ acquired = await acquireReimportSources(
+ spec,
+ group.names.flatMap((name) => {
+ const entry = resolveReimportLockEntry(
+ body.scope,
+ name,
+ effectiveSkillRoot(body.scope, name).root,
+ lock,
+ );
+ return entry
+ ? [
+ {
+ name,
+ entry,
+ ...(entry.skill !== undefined ? { recordedSkill: entry.skill } : {}),
+ },
+ ]
+ : [];
+ }),
+ );
+ cleanup = acquired.cleanup;
} catch (e) {
getLogger('skills-reimport-bulk').warn(
{ source: group.source, err: e, requestId: getRequestId(_req) },
@@ -586,11 +608,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo
});
continue;
}
- const pick = pickReimportDir(dirs, {
- ...(entry.skill !== undefined ? { recordedSkill: entry.skill } : {}),
- localName: name,
- frontmatterNameOf: (dir) => parseSkillDir(dir)?.name,
- });
+ const pick = acquired.picks.get(name);
if (!pick) {
results.push({
requested: name,
@@ -608,7 +626,8 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo
scope: body.scope,
entry,
lockPath,
- ref,
+ ref: acquired.ref,
+ ...(pick.folder ? { folder: pick.folder } : {}),
actor,
});
if (!outcome.ok) {
diff --git a/packages/server/src/services/skill-import.ts b/packages/server/src/services/skill-import.ts
index ef20f4474..10b0635ef 100644
--- a/packages/server/src/services/skill-import.ts
+++ b/packages/server/src/services/skill-import.ts
@@ -98,6 +98,7 @@ export interface SkillImportService {
ref?: string;
publisher?: string;
upstreamSkill?: string;
+ subpath?: string;
actor: ActorIdentity;
skipProjection?: boolean;
}): Promise<SkillImportOutcome>;
@@ -110,7 +111,8 @@ export function createSkillImportService(deps: SkillImportDeps): SkillImportServ
return {
async runSkillImport(params) {
- const { acquiredDir, scope, sourceLabel, ref, publisher, upstreamSkill, actor } = params;
+ const { acquiredDir, scope, sourceLabel, ref, publisher, upstreamSkill, subpath, actor } =
+ params;
if (!deps.projectDir) {
return {
ok: false,
@@ -277,6 +279,7 @@ export function createSkillImportService(deps: SkillImportDeps): SkillImportServ
publisher,
importedAt: new Date().toISOString(),
...(upstreamSkill !== undefined ? { skill: upstreamSkill } : {}),
+ ...(subpath !== undefined ? { subpath } : {}),
}),
);
diff --git a/packages/server/src/services/skill-reimport.ts b/packages/server/src/services/skill-reimport.ts
index 45f76ab49..4591c82c8 100644
--- a/packages/server/src/services/skill-reimport.ts
+++ b/packages/server/src/services/skill-reimport.ts
@@ -66,6 +66,19 @@ export interface SkillReimportDeps {
signalFiles: () => void;
}
+export type RecordedFolderUpdate =
+ | { readonly kind: 'set'; readonly subpath: string }
+ | { readonly kind: 'clear' };
+
+function withRecordedFolder(
+ entry: SkillLockEntry,
+ folder: RecordedFolderUpdate | undefined,
+): SkillLockEntry {
+ if (folder === undefined) return entry;
+ const { subpath: _replaced, ...rest } = entry;
+ return folder.kind === 'set' ? { ...rest, subpath: folder.subpath } : rest;
+}
+
export interface SkillReimportService {
runSkillReimport(params: {
acquiredDir: string;
@@ -74,27 +87,42 @@ export interface SkillReimportService {
entry: SkillLockEntry;
lockPath: string;
ref?: string;
+ folder?: RecordedFolderUpdate;
actor: ActorIdentity;
dryRun?: boolean;
}): Promise<SkillReimportOutcome>;
}
+interface ReimportPickOptions {
+ recordedSkill?: string;
+ localName: string;
+ frontmatterNameOf: (dir: string) => string | undefined;
+}
+
export function pickReimportDir<T extends { name: string; dir: string }>(
dirs: readonly T[],
- opts: {
- recordedSkill?: string;
- localName: string;
- frontmatterNameOf: (dir: string) => string | undefined;
- },
+ opts: ReimportPickOptions,
): T | undefined {
+ return reimportCandidates(dirs, opts)[0];
+}
+
+export function reimportCandidates<T extends { name: string; dir: string }>(
+ dirs: readonly T[],
+ opts: ReimportPickOptions,
+): T[] {
const { recordedSkill, localName, frontmatterNameOf } = opts;
- return (
- (recordedSkill ? dirs.find((d) => d.name === recordedSkill) : undefined) ??
- dirs.find((d) => d.name === localName) ??
- dirs.find((d) => frontmatterNameOf(d.dir) === localName) ??
- dirs.find((d) => d.name === RENAMED_PACK_SKILLS[recordedSkill ?? localName]) ??
- (dirs.length === 1 ? dirs[0] : undefined)
- );
+ const renamed = RENAMED_PACK_SKILLS[recordedSkill ?? localName];
+ const rules: Array<(d: T) => boolean> = [
+ (d) => recordedSkill !== undefined && recordedSkill !== '' && d.name === recordedSkill,
+ (d) => d.name === localName,
+ (d) => frontmatterNameOf(d.dir) === localName,
+ (d) => renamed !== undefined && d.name === renamed,
+ ];
+ for (const rule of rules) {
+ const matches = dirs.filter(rule);
+ if (matches.length > 0) return matches;
+ }
+ return dirs.length === 1 ? [...dirs] : [];
}
export function planReimportDiff(input: {
@@ -144,7 +172,7 @@ export function groupReimportNamesBySource(
export function createSkillReimportService(deps: SkillReimportDeps): SkillReimportService {
return {
async runSkillReimport(params) {
- const { acquiredDir, name, scope, entry, lockPath, ref, actor } = params;
+ const { acquiredDir, name, scope, entry, lockPath, ref, folder, actor } = params;
const reimportBase = scope === 'global' ? deps.skillsHome : deps.contentDir;
const {
root: skillsRoot,
@@ -317,7 +345,7 @@ export function createSkillReimportService(deps: SkillReimportDeps): SkillReimpo
: undefined;
await mutateSkillsLock(lockPath, (current: SkillsLock) =>
upsertLockEntry(current, name, {
- ...(current.skills[name] ?? entry),
+ ...withRecordedFolder(current.skills[name] ?? entry, folder),
contentHash: acquired.contentHash,
files: acquired.files.map((file) => file.relPath),
...(reimportLocalHash !== undefined ? { localHash: reimportLocalHash } : {}),
diff --git a/packages/server/src/skill-source-cache.ts b/packages/server/src/skill-source-cache.ts
index e06171d4a..74ff581ae 100644
--- a/packages/server/src/skill-source-cache.ts
+++ b/packages/server/src/skill-source-cache.ts
@@ -1,5 +1,5 @@
import type { Fetched, SourceSpec } from '@inkeep/open-knowledge-core/skills-catalog';
-import { fetchSource } from '@inkeep/open-knowledge-core/skills-catalog';
+import { fetchSource, fetchSourceListing } from '@inkeep/open-knowledge-core/skills-catalog';
const TTL_MS = 30_000;
@@ -26,12 +26,20 @@ function sweepExpired(now: number): void {
export async function fetchCachedSource(spec: SourceSpec): Promise<Fetched> {
if (spec.kind === 'local') return fetchSource(spec);
+ return cached(sourceKey(spec), () => fetchSource(spec));
+}
+
+export async function fetchCachedListing(spec: SourceSpec): Promise<Fetched> {
+ if (spec.kind !== 'git' || spec.subpath) return fetchCachedSource(spec);
+ return cached(`listing\n${sourceKey(spec)}`, () => fetchSourceListing(spec));
+}
+
+async function cached(key: string, fetchOnce: () => Promise<Fetched>): Promise<Fetched> {
const now = Date.now();
sweepExpired(now);
- const key = sourceKey(spec);
let entry = cache.get(key);
if (!entry) {
- const fetched = fetchSource(spec);
+ const fetched = fetchOnce();
entry = { fetched, expiresAt: now + TTL_MS };
cache.set(key, entry);
const self = entry;
diff --git a/packages/server/src/skill-source-folders.ts b/packages/server/src/skill-source-folders.ts
new file mode 100644
index 000000000..83d374995
--- /dev/null
+++ b/packages/server/src/skill-source-folders.ts
@@ -0,0 +1,180 @@
+import { existsSync } from 'node:fs';
+import { join, relative, sep } from 'node:path';
+import {
+ discoverSkillDirs,
+ type FetchedListing,
+ fetchSource,
+ fetchSourceListing,
+ parseSkillDir,
+ type SkillLockEntry,
+ SkillSubpathNotFoundError,
+ type SourceSpec,
+} from '@inkeep/open-knowledge-core/skills-catalog';
+import {
+ pickReimportDir,
+ type RecordedFolderUpdate,
+ reimportCandidates,
+} from './services/skill-reimport.ts';
+
+type SkillDir = { name: string; dir: string };
+
+export function recordsSkillFolder(spec: SourceSpec): boolean {
+ return spec.kind === 'git' && !spec.subpath;
+}
+
+export function skillFolderWithin(root: string, dir: string): string {
+ return relative(root, dir).split(sep).join('/');
+}
+
+function usableRecordedSubpath(subpath: string | undefined): string | undefined {
+ if (subpath === undefined || /^[A-Za-z]:/.test(subpath)) return undefined;
+ const segments = subpath.split('/');
+ return segments.every((s) => s !== '' && s !== '.' && s !== '..' && !/[\\\0\r\n]/.test(s))
+ ? subpath
+ : undefined;
+}
+
+export async function fetchPickedFolders(
+ listing: FetchedListing,
+ pickedDirs: readonly string[],
+): Promise<{ dirs: string[]; ref: string | undefined; cleanup: () => void }> {
+ if (!listing.fetchFolders || pickedDirs.length === 0) {
+ return { dirs: [...pickedDirs], ref: listing.ref, cleanup: () => {} };
+ }
+ const folders = pickedDirs.map((dir) => skillFolderWithin(listing.dir, dir));
+ const content = await listing.fetchFolders([...new Set(folders)]);
+ return {
+ dirs: folders.map((folder) => join(content.dir, folder)),
+ ref: content.ref,
+ cleanup: content.cleanup,
+ };
+}
+
+export interface ReimportWanted {
+ readonly name: string;
+ readonly entry: SkillLockEntry;
+ readonly recordedSkill?: string;
+}
+
+export interface ReimportAcquisition {
+ readonly dirs: readonly SkillDir[];
+ readonly picks: ReadonlyMap<string, { dir: string; folder?: RecordedFolderUpdate }>;
+ readonly ref: string | undefined;
+ readonly cleanup: () => void;
+}
+
+function pickOptions(wanted: ReimportWanted) {
+ return {
+ ...(wanted.recordedSkill !== undefined ? { recordedSkill: wanted.recordedSkill } : {}),
+ localName: wanted.name,
+ frontmatterNameOf: (dir: string) => parseSkillDir(dir)?.name,
+ };
+}
+
+async function fromRecordedFolder(
+ spec: SourceSpec & { kind: 'git' },
+ wanted: ReimportWanted,
+): Promise<ReimportAcquisition | undefined> {
+ const subpath = usableRecordedSubpath(wanted.entry.subpath);
+ if (subpath === undefined) return undefined;
+ let fetched: Awaited<ReturnType<typeof fetchSource>>;
+ try {
+ fetched = await fetchSource({ ...spec, subpath });
+ } catch (e) {
+ if (e instanceof SkillSubpathNotFoundError) return undefined;
+ throw e;
+ }
+ try {
+ const dirs = discoverSkillDirs(fetched.dir);
+ const pick = pickReimportDir(dirs, pickOptions(wanted));
+ if (!pick) {
+ fetched.cleanup();
+ return undefined;
+ }
+ return {
+ dirs,
+ picks: new Map([[wanted.name, { dir: pick.dir }]]),
+ ref: fetched.ref,
+ cleanup: fetched.cleanup,
+ };
+ } catch (e) {
+ fetched.cleanup();
+ throw e;
+ }
+}
+
+export async function acquireReimportSources(
+ spec: SourceSpec,
+ wanted: readonly ReimportWanted[],
+): Promise<ReimportAcquisition> {
+ if (spec.kind !== 'git' || spec.subpath) {
+ const fetched = await fetchSource(spec);
+ try {
+ const dirs = discoverSkillDirs(fetched.dir);
+ const picks = new Map<string, { dir: string }>();
+ for (const want of wanted) {
+ const pick = pickReimportDir(dirs, pickOptions(want));
+ if (pick) picks.set(want.name, { dir: pick.dir });
+ }
+ return { dirs, picks, ref: fetched.ref, cleanup: fetched.cleanup };
+ } catch (e) {
+ fetched.cleanup();
+ throw e;
+ }
+ }
+ const single = wanted.length === 1 ? (wanted[0] as ReimportWanted) : undefined;
+ const recorded = single ? await fromRecordedFolder(spec, single) : undefined;
+ if (recorded) return recorded;
+
+ const listing = await fetchSourceListing(spec);
+ try {
+ const dirs = discoverSkillDirs(listing.dir);
+ const chosen: Array<{ name: string; dir: string; folder?: RecordedFolderUpdate }> = [];
+ for (const want of wanted) {
+ const recordedFolder = usableRecordedSubpath(want.entry.subpath);
+ if (
+ recordedFolder !== undefined &&
+ existsSync(join(listing.dir, recordedFolder, 'SKILL.md'))
+ ) {
+ chosen.push({ name: want.name, dir: join(listing.dir, recordedFolder) });
+ continue;
+ }
+ const candidates = reimportCandidates(dirs, pickOptions(want));
+ const pick = candidates[0];
+ if (!pick) continue;
+ const folder = skillFolderWithin(listing.dir, pick.dir);
+ const found: RecordedFolderUpdate | undefined =
+ candidates.length === 1 && folder !== ''
+ ? { kind: 'set', subpath: folder }
+ : want.entry.subpath !== undefined
+ ? { kind: 'clear' }
+ : undefined;
+ chosen.push({ name: want.name, dir: pick.dir, ...(found ? { folder: found } : {}) });
+ }
+ const content = await fetchPickedFolders(
+ listing,
+ chosen.map((c) => c.dir),
+ );
+ const picks = new Map(
+ chosen.map((c, i) => [
+ c.name,
+ {
+ dir: content.dirs[i] as string,
+ ...(c.folder ? { folder: c.folder } : {}),
+ },
+ ]),
+ );
+ return {
+ dirs,
+ picks,
+ ref: content.ref,
+ cleanup: () => {
+ content.cleanup();
+ listing.cleanup();
+ },
+ };
+ } catch (e) {
+ listing.cleanup();
+ throw e;
+ }
+}
diff --git a/packages/server/src/skills-sh-handlers.ts b/packages/server/src/skills-sh-handlers.ts
index d6ae4913c..e0a3333c3 100644
--- a/packages/server/src/skills-sh-handlers.ts
+++ b/packages/server/src/skills-sh-handlers.ts
@@ -14,7 +14,7 @@ import {
acquiredBundleTooLarge,
discoverSkillDirs,
discoverWellKnownSkills,
- fetchSource,
+ fetchSourceListing,
inspectPluginBundleDir,
inspectPluginSource,
parseGitHubRepoSearch,
@@ -35,7 +35,7 @@ import { successResponse } from './http/success-response.ts';
import { isAllowedGitUrl } from './local-op-security.ts';
import type { PinoLogger } from './logger.ts';
import { rejectDisallowedGitSpec } from './skill-git-spec-guard.ts';
-import { fetchCachedSource } from './skill-source-cache.ts';
+import { fetchCachedListing, fetchCachedSource } from './skill-source-cache.ts';
import { getPopularSkills, getPublisherSkills } from './skills-leaderboard.ts';
import { readSkillsLockFile } from './skills-lock-store.ts';
@@ -400,7 +400,7 @@ export function createSkillsShHandlers(deps: SkillsShHandlerDeps): SkillsShHandl
}
if (rejectDisallowedGitSpec(res, spec, 'skills-discover')) return;
try {
- const fetched = await fetchCachedSource(spec);
+ const fetched = await fetchCachedListing(spec);
const dirs = discoverSkillDirs(fetched.dir);
const skills = dirs.map((d) => {
const meta = readSkillDirMeta(d.dir);
@@ -471,7 +471,7 @@ export function createSkillsShHandlers(deps: SkillsShHandlerDeps): SkillsShHandl
if (spec && allowed) {
let cleanup: (() => void) | undefined;
try {
- const fetched = await fetchSource(spec);
+ const fetched = await fetchSourceListing(spec);
cleanup = fetched.cleanup;
const names = discoverSkillDirs(fetched.dir).map(
(d) => readSkillDirMeta(d.dir)?.name ?? d.name,
diff --git a/test-support/served-git-repos.test-helper.ts b/test-support/served-git-repos.test-helper.ts
new file mode 100644
index 000000000..8aa7f3734
--- /dev/null
+++ b/test-support/served-git-repos.test-helper.ts
@@ -0,0 +1,119 @@
+import { type ChildProcess, execFileSync, spawn } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { type AddressInfo, createServer } from 'node:net';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { configureTestGitRepository } from './configure-git-fixture.test-helper.ts';
+
+const hermeticGitEnv: Readonly<Record<string, string>> = {
+ GIT_CONFIG_GLOBAL: '/dev/null',
+ GIT_CONFIG_SYSTEM: '/dev/null',
+ GIT_CONFIG_NOSYSTEM: '1',
+};
+
+const childEnv: NodeJS.ProcessEnv = {
+ ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_'))),
+ ...hermeticGitEnv,
+};
+
+export interface ServedGitRepos {
+ readonly port: number;
+ readonly bytesServed: () => number;
+ readonly resetBytesServed: () => void;
+ readonly url: (name: string) => string;
+ readonly githubRedirectEnv: () => Record<string, string>;
+ readonly publish: (name: string, files: Readonly<Record<string, string | Buffer | null>>) => void;
+ readonly close: () => Promise<void>;
+}
+
+function git(cwd: string, args: readonly string[]): void {
+ execFileSync('git', args, { cwd, stdio: 'pipe', env: childEnv });
+}
+
+function untilClosed(child: ChildProcess): Promise<void> {
+ if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
+ return new Promise((resolve) => child.once('close', () => resolve()));
+}
+
+export async function serveGitRepos(): Promise<ServedGitRepos> {
+ const root = mkdtempSync(join(tmpdir(), 'ok-served-git-'));
+ const served = join(root, 'served');
+ mkdirSync(served);
+ const daemons = new Set<ChildProcess>();
+ let bytes = 0;
+
+ const server = createServer((socket) => {
+ const daemon = spawn(
+ 'git',
+ ['daemon', '--inetd', `--base-path=${served}`, '--export-all', served],
+ { env: childEnv, stdio: ['pipe', 'pipe', 'ignore'] },
+ );
+ daemons.add(daemon);
+ daemon.once('close', () => {
+ daemons.delete(daemon);
+ socket.end();
+ });
+ daemon.stdout.on('data', (chunk: Buffer) => {
+ bytes += chunk.length;
+ });
+ socket.on('error', () => daemon.stdin.destroy());
+ daemon.stdin.on('error', () => {});
+ socket.pipe(daemon.stdin);
+ daemon.stdout.pipe(socket);
+ });
+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+ const port = (server.address() as AddressInfo).port;
+
+ const publish = (name: string, files: Readonly<Record<string, string | Buffer | null>>) => {
+ const source = join(root, 'sources', name);
+ const bare = join(served, `${name}.git`);
+ if (!existsSync(source)) {
+ mkdirSync(source, { recursive: true });
+ git(source, ['init', '-q']);
+ git(source, ['symbolic-ref', 'HEAD', 'refs/heads/main']);
+ configureTestGitRepository(source);
+ }
+ for (const [path, body] of Object.entries(files)) {
+ const target = join(source, path);
+ if (body === null) {
+ rmSync(target, { recursive: true, force: true });
+ continue;
+ }
+ mkdirSync(dirname(target), { recursive: true });
+ writeFileSync(target, body);
+ }
+ git(source, ['add', '-A']);
+ git(source, ['-c', '[email protected]', '-c', 'user.name=t', 'commit', '-q', '-m', 'publish']);
+ if (!existsSync(bare)) {
+ git(root, ['clone', '-q', '--bare', source, bare]);
+ configureTestGitRepository(bare);
+ git(bare, ['config', 'uploadpack.allowFilter', 'true']);
+ git(bare, ['config', 'uploadpack.allowAnySHA1InWant', 'false']);
+ git(bare, ['config', 'uploadpack.allowReachableSHA1InWant', 'false']);
+ return;
+ }
+ git(source, ['push', '-q', '--force', bare, 'HEAD:refs/heads/main']);
+ };
+
+ return {
+ port,
+ bytesServed: () => bytes,
+ resetBytesServed: () => {
+ bytes = 0;
+ },
+ url: (name) => `git://127.0.0.1:${port}/${name}.git`,
+ githubRedirectEnv: () => ({
+ ...hermeticGitEnv,
+ GIT_CONFIG_COUNT: '1',
+ GIT_CONFIG_KEY_0: `url.git://127.0.0.1:${port}/.insteadOf`,
+ GIT_CONFIG_VALUE_0: 'https://github.com/',
+ }),
+ publish,
+ close: async () => {
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ for (const daemon of daemons) daemon.kill();
+ await Promise.all([...daemons].map(untilClosed));
+ rmSync(root, { recursive: true, force: true });
+ },
+ };
+}