Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): fetch one skill folder when importing a repo by name (#6916)
Nick Gomezf0bb0bceb14b
diff --git a/.changeset/skill-import-by-repository-name-fetches-one-folder.md b/.changeset/skill-import-by-repository-name-fetches-one-folder.mdnew file mode 100644index 000000000..20df5e45d--- /dev/null+++ b/.changeset/skill-import-by-repository-name-fetches-one-folder.md@@ -0,0 +1,5 @@+---+'@inkeep/open-knowledge': patch+---++Importing and updating skills from a large repository by name, such as `your-org/team-skills`, no longer fails with "Could not fetch source." at the 60-second limit, and the skill picker now lists that repository's skills even when it contains a folder named `SKILL.md`.diff --git a/packages/core/src/skills-catalog/acquire/acquire.test.ts b/packages/core/src/skills-catalog/acquire/acquire.test.tsindex 17556b9bf..6cc1fec89 100644--- a/packages/core/src/skills-catalog/acquire/acquire.test.ts+++ b/packages/core/src/skills-catalog/acquire/acquire.test.ts@@ -162,6 +162,15 @@ describe('discoverSkillDirs', () => {);expect(discoverSkillDirs(repo).map((s) => s.name)).toEqual(['grill-me', 'tdd']);});++ test('a folder whose SKILL.md is a directory is not a skill', () => {+ const repo = join(root, 'skill-md-directory');+ writeSkill(join(repo, 'skills', 'real'), 'name: real\ndescription: d');+ mkdirSync(join(repo, 'reports', 'copied', 'SKILL.md'), { recursive: true });+ writeFileSync(join(repo, 'reports', 'copied', 'SKILL.md', 'notes.txt'), 'notes\n');+ expect(discoverSkillDirs(repo).map((s) => s.name)).toEqual(['real']);+ expect(discoverSkillDirs(join(repo, 'reports', 'copied'))).toEqual([]);+ });});describe('parseSource', () => {diff --git a/packages/core/src/skills-catalog/acquire/errors.ts b/packages/core/src/skills-catalog/acquire/errors.tsindex f636d5434..12c346e16 100644--- a/packages/core/src/skills-catalog/acquire/errors.ts+++ b/packages/core/src/skills-catalog/acquire/errors.ts@@ -1 +1,3 @@export class SkillFetchError extends Error {}++export class SkillSubpathNotFoundError extends SkillFetchError {}diff --git a/packages/core/src/skills-catalog/acquire/fetch-git.test.ts b/packages/core/src/skills-catalog/acquire/fetch-git.test.tsindex f89f36715..c9b19f491 100644--- a/packages/core/src/skills-catalog/acquire/fetch-git.test.ts+++ b/packages/core/src/skills-catalog/acquire/fetch-git.test.ts@@ -5,6 +5,7 @@ import {lstatSync,mkdirSync,mkdtempSync,+ readdirSync,rmSync,statSync,symlinkSync,@@ -12,7 +13,7 @@ import {} from 'node:fs';import { type AddressInfo, createServer, type Server } from 'node:net';import { tmpdir } from 'node:os';-import { dirname, join } from 'node:path';+import { dirname, join, relative, sep } from 'node:path';import {afterAll,afterEach,@@ -21,12 +22,19 @@ import {describe,expect,type MockInstance,+ onTestFinished,test,vi,} from 'vitest';import { configureTestGitRepository } from '../../../../../test-support/configure-git-fixture.test-helper.ts';-import { fetchSource } from './fetch.ts';-import { parseSkillDir } from './parse.ts';+import { REFUSED_LOOPBACK_ORIGIN } from '../../../../../test-support/refused-loopback.test-helper.ts';+import {+ fetchSource,+ fetchSourceListing,+ SkillFetchError,+ SkillSubpathNotFoundError,+} from './fetch.ts';+import { discoverSkillDirs, parseSkillDir, readSkillDirMeta } from './parse.ts';const OUTSIDE_SUBPATH_BLOB_BYTES = 2_000_000;const POSIX_NAMES = process.platform !== 'win32';@@ -50,6 +58,10 @@ function git(cwd: string, args: readonly string[]): void {execFileSync('git', args, { cwd, stdio: 'pipe', env: hermeticGitEnv });}+function gitWithInput(cwd: string, args: readonly string[], input: string): string {+ return execFileSync('git', args, { cwd, input, env: hermeticGitEnv }).toString().trim();+}+function writeRepoFile(repo: string, path: string, body: string | Buffer): void {mkdirSync(dirname(join(repo, path)), { recursive: true });writeFileSync(join(repo, path), body);@@ -75,6 +87,17 @@ function servedUrl(repo: 'filtering' | 'non-filtering'): string {return `git://127.0.0.1:${port}/${repo}.git`;}+function listedSkills(dir: string): Array<{ folder: string; name: string; description: string }> {+ return discoverSkillDirs(dir).map((d) => {+ const meta = readSkillDirMeta(d.dir);+ return {+ folder: relative(dir, d.dir).split(sep).join('/'),+ name: meta?.name ?? d.name,+ description: meta?.description ?? '',+ };+ });+}+function untilClosed(child: ChildProcess): Promise<void> {if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();return new Promise((resolve) => child.once('close', () => resolve()));@@ -89,10 +112,22 @@ beforeAll(async () => {writeRepoFile(source, 'skills/alpha/SKILL.md', '---\nname: alpha\ndescription: a\n---\n');writeRepoFile(source, 'skills/alpha/references/guide.md', 'guide\n');writeRepoFile(source, 'skills/beta/SKILL.md', '---\nname: beta\ndescription: b\n---\n');+ writeRepoFile(+ source,+ 'skills/beta/upstream/SKILL.md',+ '---\nname: nested\ndescription: n\n---\n',+ );+ writeRepoFile(+ source,+ 'tools/scripts/helper/SKILL.md',+ '---\nname: helper\ndescription: h\n---\n',+ );writeRepoFile(source, 'assets/blob.bin', randomBytes(OUTSIDE_SUBPATH_BLOB_BYTES));if (POSIX_NAMES) {symlinkSync('/etc/passwd', join(source, 'skills/alpha/link'));writeRepoFile(source, '"quoted"/SKILL.md', '---\nname: quoted\ndescription: q\n---\n');+ mkdirSync(join(source, 'linked'));+ symlinkSync('/etc/passwd', join(source, 'linked/SKILL.md'));}git(source, ['add', '-A']);git(source, ['-c', '[email protected]', '-c', 'user.name=t', 'commit', '-q', '-m', 'init']);@@ -150,6 +185,7 @@ beforeEach(() => {vi.stubEnv('GIT_CONFIG_GLOBAL', '/dev/null');vi.stubEnv('GIT_CONFIG_SYSTEM', '/dev/null');vi.stubEnv('GIT_CONFIG_NOSYSTEM', '1');+ vi.stubEnv('LC_ALL', 'C');});afterEach(async () => {@@ -281,3 +317,300 @@ describe('fetchSource from a git server', () => {},);});++async function wholeCloneSkills(): Promise<ReturnType<typeof listedSkills>> {+ const whole = await fetchSource({ kind: 'git', url: servedUrl('filtering') });+ try {+ return listedSkills(whole.dir);+ } finally {+ whole.cleanup();+ bytesServed = 0;+ }+}++describe('fetchSourceListing from a git server', () => {+ test('a repository listing finds the same skills as a whole clone but downloads only their SKILL.md files', async () => {+ const expected = await wholeCloneSkills();+ expect(expected.map((s) => s.folder).sort()).toEqual(+ [...(POSIX_NAMES ? ['"quoted"', 'linked'] : []), 'skills/alpha', 'skills/beta'].sort(),+ );+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });+ try {+ expect(listedSkills(listing.dir)).toEqual(expected);+ expect(existsSync(join(listing.dir, 'skills', 'alpha', 'SKILL.md'))).toBe(true);+ expect(existsSync(join(listing.dir, 'skills', 'alpha', 'references', 'guide.md'))).toBe(+ false,+ );+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);+ expect(warn).not.toHaveBeenCalled();+ } finally {+ listing.cleanup();+ }+ });++ test('a listing from a server that ignores the blob filter finds the same skills', async () => {+ const expected = await wholeCloneSkills();+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('non-filtering') });+ try {+ expect(bytesServed).toBeGreaterThan(OUTSIDE_SUBPATH_BLOB_BYTES);+ expect(listedSkills(listing.dir)).toEqual(expected);+ expect(warn).not.toHaveBeenCalled();+ } finally {+ listing.cleanup();+ }+ });++ test('a listing from a server that cannot send SKILL.md files on demand falls back to the whole repository', async () => {+ const expected = await wholeCloneSkills();+ vi.stubEnv('GIT_CONFIG_COUNT', '1');+ vi.stubEnv('GIT_CONFIG_KEY_0', 'protocol.version');+ vi.stubEnv('GIT_CONFIG_VALUE_0', '0');+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });+ try {+ expect(warn).toHaveBeenCalledWith(+ '[skills-catalog] skill listing clone failed, cloning the whole repository',+ expect.objectContaining({ err: expect.any(Error) }),+ );+ expect(listing.fetchFolders).toBeUndefined();+ expect(listedSkills(listing.dir)).toEqual(expected);+ expect(existsSync(join(listing.dir, 'skills', 'alpha', 'references', 'guide.md'))).toBe(true);+ } finally {+ listing.cleanup();+ }+ });++ test.runIf(POSIX_NAMES)('a symlinked SKILL.md in a listing arrives as a plain file', async () => {+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });+ try {+ expect(existsSync(join(listing.dir, 'linked', 'SKILL.md'))).toBe(true);+ expect(lstatSync(join(listing.dir, 'linked', 'SKILL.md')).isSymbolicLink()).toBe(false);+ } finally {+ listing.cleanup();+ }+ });++ test('a listing refuses a transport outside the allow-list without running git', async () => {+ const marker = join(root, 'ext-transport-ran');+ await expect(+ fetchSourceListing({ kind: 'git', url: `ext::sh -c touch% ${marker}` }),+ ).rejects.toThrow('Unsupported git transport');+ expect(existsSync(marker)).toBe(false);+ });++ test('a source that names a folder is fetched exactly as fetchSource fetches it', async () => {+ const listing = await fetchSourceListing({+ kind: 'git',+ url: servedUrl('filtering'),+ subpath: 'skills/alpha',+ });+ try {+ expect(listing.fetchFolders).toBeUndefined();+ expect(parseSkillDir(listing.dir)?.name).toBe('alpha');+ expect(existsSync(join(listing.dir, 'references', 'guide.md'))).toBe(true);+ expect(existsSync(join(listing.dir, '..', 'beta'))).toBe(false);+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);+ } finally {+ listing.cleanup();+ }+ });++ test('fetchFolders checks out only the chosen skill folders, whole, without downloading the rest', async () => {+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });+ const fetchFolders = listing.fetchFolders;+ if (fetchFolders === undefined) throw new Error('expected a partial listing');+ try {+ const content = await fetchFolders(['skills/alpha', 'skills/beta']);+ try {+ expect(parseSkillDir(join(content.dir, 'skills', 'alpha'))?.name).toBe('alpha');+ expect(existsSync(join(content.dir, 'skills', 'alpha', 'references', 'guide.md'))).toBe(+ true,+ );+ expect(parseSkillDir(join(content.dir, 'skills', 'beta'))?.name).toBe('beta');+ expect(existsSync(join(content.dir, 'assets', 'blob.bin'))).toBe(false);+ expect(bytesServed).toBeLessThan(OUTSIDE_SUBPATH_BLOB_BYTES);+ expect(content.ref).toBe(listing.ref);+ if (POSIX_NAMES) {+ expect(lstatSync(join(content.dir, 'skills', 'alpha', 'link')).isSymbolicLink()).toBe(+ false,+ );+ }+ } finally {+ content.cleanup();+ }+ } finally {+ listing.cleanup();+ }+ });++ test('fetchFolders refuses a folder that escapes the repository before contacting the server', async () => {+ const listing = await fetchSourceListing({ kind: 'git', url: servedUrl('filtering') });+ const fetchFolders = listing.fetchFolders;+ if (fetchFolders === undefined) throw new Error('expected a partial listing');+ try {+ const served = bytesServed;+ await expect(fetchFolders(['skills/alpha', '../../../etc'])).rejects.toThrow(+ 'subpath escapes the clone: ../../../etc',+ );+ expect(bytesServed).toBe(served);+ } finally {+ listing.cleanup();+ }+ });++ test('fetchFolders reports a listed folder removed upstream since the listing as not found and keeps no clone', async () => {+ const name = `changing-${randomBytes(6).toString('hex')}`;+ const bare = join(root, 'served', `${name}.git`);+ git(root, ['clone', '-q', '--bare', join(root, 'source'), bare]);+ configureTestGitRepository(bare);+ git(bare, ['config', 'uploadpack.allowFilter', 'true']);+ const listing = await fetchSourceListing({+ kind: 'git',+ url: `git://127.0.0.1:${port}/${name}.git`,+ });+ const fetchFolders = listing.fetchFolders;+ if (fetchFolders === undefined) throw new Error('expected a partial listing');+ try {+ expect(listedSkills(listing.dir).map((s) => s.folder)).toContain('skills/beta');+ const work = join(root, `${name}-work`);+ git(root, ['clone', '-q', bare, work]);+ configureTestGitRepository(work);+ git(work, ['rm', '-q', '-r', 'skills/beta']);+ git(work, ['-c', '[email protected]', '-c', 'user.name=t', 'commit', '-q', '-m', 'drop beta']);+ git(work, ['push', '-q', 'origin', 'HEAD']);++ const scratch = mkdtempSync(join(root, 'fetch-folders-tmp-'));+ vi.stubEnv('TMPDIR', scratch);+ vi.stubEnv('TMP', scratch);+ vi.stubEnv('TEMP', scratch);+ const present = await fetchFolders(['skills/alpha']);+ try {+ expect(dirname(present.dir)).toBe(scratch);+ expect(parseSkillDir(join(present.dir, 'skills', 'alpha'))?.name).toBe('alpha');+ } finally {+ present.cleanup();+ }+ expect(readdirSync(scratch)).toEqual([]);++ const failure = await fetchFolders(['skills/alpha', 'skills/beta']).then(+ () => undefined,+ (e: unknown) => e,+ );+ expect(failure).toBeInstanceOf(SkillSubpathNotFoundError);+ expect(failure).toMatchObject({ message: 'subpath not found in repo: skills/beta' });+ expect(readdirSync(scratch)).toEqual([]);+ } finally {+ listing.cleanup();+ }+ });+});++describe('git fetch errors', () => {+ test('a missing subpath is reported with its own error type', async () => {+ await expect(+ fetchSource({ kind: 'git', url: servedUrl('filtering'), subpath: 'skills/missing' }),+ ).rejects.toBeInstanceOf(SkillSubpathNotFoundError);+ });++ test('a whole-repository clone failure gives the reason git printed once, without the git command line or temporary folder', async () => {+ const missing = `git://127.0.0.1:${port}/missing.git`;+ for (const attempt of [+ () => fetchSource({ kind: 'git', url: missing, subpath: 'skills/alpha' }),+ () => fetchSourceListing({ kind: 'git', url: missing }),+ ]) {+ const error = await attempt().then(+ () => undefined,+ (e: unknown) => e,+ );+ expect(error).toBeInstanceOf(SkillFetchError);+ const message = (error as Error).message;+ expect(message).toMatch(+ /^git clone failed for git:\/\/127\.0\.0\.1:\d+\/missing\.git: fatal: remote error: [^\n]*\/missing\.git$/,+ );+ expect(message.match(/fatal: remote error/g)).toHaveLength(1);+ expect(message).not.toContain('Fetching only part of the repository');+ expect(message).not.toContain('Command failed');+ expect(message).not.toContain('ok-skill-import-');+ }+ });++ test('when the whole-repository clone fails for another reason, the error also says why the partial fetch failed first', async () => {+ const name = `unsafe-path-${randomBytes(6).toString('hex')}`;+ const bare = join(root, 'served', `${name}.git`);+ git(root, ['init', '-q', '--bare', bare]);+ configureTestGitRepository(bare);+ git(bare, ['config', 'uploadpack.allowFilter', 'true']);+ const skill = gitWithInput(+ bare,+ ['hash-object', '-w', '--stdin'],+ '---\nname: unsafe\ndescription: u\n---\n',+ );+ const folder = gitWithInput(bare, ['mktree'], `100644 blob ${skill}\tSKILL.md\n`);+ const reserved = gitWithInput(bare, ['mktree'], `040000 tree ${folder}\tGIT~1\n`);+ const top = gitWithInput(bare, ['mktree'], `040000 tree ${reserved}\tunsafe\n`);+ const commit = gitWithInput(+ bare,+ ['-c', '[email protected]', '-c', 'user.name=t', 'commit-tree', '-m', 'init', top],+ '',+ );+ git(bare, ['update-ref', 'refs/heads/main', commit]);+ git(bare, ['symbolic-ref', 'HEAD', 'refs/heads/main']);+ const url = `git://127.0.0.1:${port}/${name}.git`;++ const error = await fetchSourceListing({ kind: 'git', url }).then(+ () => undefined,+ (e: unknown) => e,+ );++ expect(error).toBeInstanceOf(SkillFetchError);+ const message = (error as Error).message;+ const partialReason = "error: invalid path 'unsafe/GIT~1/SKILL.md'";+ const opening = `git clone failed for ${url}: ${partialReason}\n`;+ expect(message.slice(0, opening.length)).toBe(opening);+ expect(message).toContain('fatal: unable to checkout working tree');+ const closing = `. Fetching only part of the repository had failed first: ${partialReason}`;+ expect(message.slice(-closing.length)).toBe(closing);+ expect(message).not.toContain('Command failed');+ expect(message).not.toContain('ok-skill-import-');+ });++ test('a fetch that runs out of time says it hit the 60-second limit instead of listing git arguments', async () => {+ for (const attempt of [+ () => fetchSource({ kind: 'git', url: servedUrl('filtering') }),+ () => fetchSource({ kind: 'git', url: servedUrl('filtering'), subpath: 'skills/alpha' }),+ () => fetchSourceListing({ kind: 'git', url: servedUrl('filtering') }),+ ]) {+ const clock = vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValue(61_000);+ onTestFinished(() => clock.mockRestore());+ const error = await attempt().then(+ () => undefined,+ (e: unknown) => e,+ );+ clock.mockRestore();+ expect(error).toBeInstanceOf(SkillFetchError);+ expect((error as Error).message).toBe(+ `Fetching ${servedUrl('filtering')} hit the 60-second limit.`,+ );+ }+ expect(bytesServed).toBe(0);+ });++ test('credentials in a source URL stay out of errors and warnings', async () => {+ const url = `${REFUSED_LOOPBACK_ORIGIN.replace('http://', 'http://someone:hunter2-secret@')}/team/skills.git`;+ for (const attempt of [+ () => fetchSource({ kind: 'git', url, subpath: 'skills/alpha' }),+ () => fetchSourceListing({ kind: 'git', url }),+ ]) {+ const error = await attempt().then(+ () => undefined,+ (e: unknown) => e,+ );+ expect(error).toBeInstanceOf(SkillFetchError);+ expect((error as Error).message).toContain('http://***@127.0.0.1:4/team/skills.git');+ expect((error as Error).message).not.toContain('hunter2-secret');+ }+ expect(warn).toHaveBeenCalledTimes(2);+ expect(+ JSON.stringify(warn.mock.calls, (_k, v) => (v instanceof Error ? v.message : v)),+ ).not.toContain('hunter2-secret');+ });+});diff --git a/packages/core/src/skills-catalog/acquire/fetch.ts b/packages/core/src/skills-catalog/acquire/fetch.tsindex 37c384819..b4e03d91b 100644--- a/packages/core/src/skills-catalog/acquire/fetch.ts+++ b/packages/core/src/skills-catalog/acquire/fetch.ts@@ -1,10 +1,10 @@import { execFile, execFileSync } from 'node:child_process';-import { existsSync, mkdtempSync, rmSync } from 'node:fs';+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';import { homedir, tmpdir } from 'node:os';import { isAbsolute, join, relative, resolve, sep } from 'node:path';import { promisify } from 'node:util';import { isLocalSkillSource } from '../source-fields.ts';-import { SkillFetchError } from './errors.ts';+import { SkillFetchError, SkillSubpathNotFoundError } from './errors.ts';import {fetchWellKnownSkill,type WellKnownFetchOptions,@@ -23,7 +23,7 @@ export interface SkillsShSourceRef {skill: string;}-export { SkillFetchError } from './errors.ts';+export { SkillFetchError, SkillSubpathNotFoundError } from './errors.ts';const GIT_FETCH_BUDGET_MS = 60_000;@@ -108,20 +108,10 @@ export async function fetchSource(if (!existsSync(dir)) throw new SkillFetchError(`Local path not found: ${dir}`);return { dir, cleanup: () => {} };}- const hasTransportScheme = /^[a-z][a-z0-9+.-]+:/i.test(spec.url);- if (hasTransportScheme && !ALLOWED_GIT_TRANSPORTS.some((p) => p.test(spec.url))) {- throw new SkillFetchError(`Unsupported git transport: ${spec.url}`);- }+ assertAllowedGitTransport(spec.url);const deadline = performance.now() + GIT_FETCH_BUDGET_MS;- const tmp =- (spec.subpath ? await cloneSubpathOnly(spec.url, spec.subpath, deadline) : undefined) ??- (await cloneWholeRepository(spec.url, deadline));- let ref: string | undefined;- try {- ref = execFileSync('git', ['-C', tmp, 'rev-parse', 'HEAD'], { stdio: 'pipe' })- .toString()- .trim();- } catch {}+ const tmp = await cloneFolders(spec.url, spec.subpath ? [spec.subpath] : [], deadline);+ const ref = headRef(tmp);const dir = spec.subpath ? resolve(tmp, spec.subpath) : tmp;const containment = relative(tmp, dir);if (containment.startsWith('..') || isAbsolute(containment)) {@@ -130,11 +120,74 @@ export async function fetchSource(}if (!existsSync(dir)) {rmSync(tmp, { recursive: true, force: true });- throw new SkillFetchError(`subpath not found in repo: ${spec.subpath}`);+ throw new SkillSubpathNotFoundError(`subpath not found in repo: ${spec.subpath}`);}return { dir, ref, cleanup: () => rmSync(tmp, { recursive: true, force: true }) };}+export interface FetchedListing extends Fetched {+ readonly fetchFolders?: (folders: readonly string[]) => Promise<Fetched>;+}++export async function fetchSourceListing(+ spec: SourceSpec,+ opts: WellKnownFetchOptions = {},+): Promise<FetchedListing> {+ if (spec.kind !== 'git' || spec.subpath) return fetchSource(spec, opts);+ const { url } = spec;+ assertAllowedGitTransport(url);+ const deadline = performance.now() + GIT_FETCH_BUDGET_MS;+ const listed = await cloneSkillFilesOnly(url, deadline);+ const tmp =+ typeof listed === 'string' ? listed : await cloneWholeRepository(url, deadline, listed.failure);+ const fetched = {+ dir: tmp,+ ref: headRef(tmp),+ cleanup: () => rmSync(tmp, { recursive: true, force: true }),+ };+ if (typeof listed !== 'string') return fetched;+ return {+ ...fetched,+ fetchFolders: async (folders) => {+ const escaping = folders.find((folder) => {+ const inside = relative(tmp, resolve(tmp, folder));+ return inside.startsWith('..') || isAbsolute(inside);+ });+ if (escaping !== undefined) {+ throw new SkillFetchError(`subpath escapes the clone: ${escaping}`);+ }+ const dir = await cloneFolders(url, folders, deadline);+ const missing = folders.find((folder) => !existsSync(resolve(dir, folder)));+ if (missing !== undefined) {+ rmSync(dir, { recursive: true, force: true });+ throw new SkillSubpathNotFoundError(`subpath not found in repo: ${missing}`);+ }+ return {+ dir,+ ref: headRef(dir),+ cleanup: () => rmSync(dir, { recursive: true, force: true }),+ };+ },+ };+}++function assertAllowedGitTransport(url: string): void {+ const hasTransportScheme = /^[a-z][a-z0-9+.-]+:/i.test(url);+ if (hasTransportScheme && !ALLOWED_GIT_TRANSPORTS.some((p) => p.test(url))) {+ throw new SkillFetchError(`Unsupported git transport: ${url}`);+ }+}++function headRef(repo: string): string | undefined {+ try {+ return execFileSync('git', ['-C', repo, 'rev-parse', 'HEAD'], { stdio: 'pipe' })+ .toString()+ .trim();+ } catch {+ return undefined;+ }+}+function runGit(args: readonly string[], deadline: number, stdin?: string): Promise<unknown> {const timeout = Math.floor(deadline - performance.now());if (timeout <= 0) {@@ -154,60 +207,142 @@ function runGit(args: readonly string[], deadline: number, stdin?: string): Promreturn run;}-function gitCloneFailed(url: string, e: unknown): SkillFetchError {- const msg = e instanceof Error ? e.message : String(e);- return new SkillFetchError(`git clone failed for ${url}: ${msg}`);+function withoutUrlCredentials(text: string): string {+ return text.replace(/(\bhttps?:\/\/)[^\s/@]+@/gi, '$1***@');+}++function gitFailureReason(e: unknown): string {+ const stderr =+ e instanceof Error && 'stderr' in e && typeof e.stderr === 'string' ? e.stderr.trim() : '';+ return withoutUrlCredentials(stderr !== '' ? stderr : e instanceof Error ? e.message : String(e));+}++function gitCloneFailed(url: string, e: unknown, earlierFailure?: string): SkillFetchError {+ const reason = gitFailureReason(e);+ const earlier =+ earlierFailure === undefined || earlierFailure === reason+ ? ''+ : `. Fetching only part of the repository had failed first: ${earlierFailure}`;+ return new SkillFetchError(+ withoutUrlCredentials(`git clone failed for ${url}: ${reason}${earlier}`),+ );+}++function gitFetchTimedOut(url: string): SkillFetchError {+ return new SkillFetchError(+ withoutUrlCredentials(`Fetching ${url} hit the ${GIT_FETCH_BUDGET_MS / 1000}-second limit.`),+ );}-async function cloneWholeRepository(url: string, deadline: number): Promise<string> {+async function cloneWholeRepository(+ url: string,+ deadline: number,+ earlierFailure?: string,+): Promise<string> {const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));try {await runGit(['clone', '-q', '--depth', '1', '--', url, tmp], deadline);} catch (e) {rmSync(tmp, { recursive: true, force: true });- throw gitCloneFailed(url, e);+ throw budgetSpent(e, deadline) ? gitFetchTimedOut(url) : gitCloneFailed(url, e, earlierFailure);}return tmp;}-async function cloneSubpathOnly(+async function cloneFolders(url: string,- subpath: string,+ folders: readonly string[],deadline: number,-): Promise<string | undefined> {+): Promise<string> {+ const sparse = folders.length > 0 ? await cloneOnlyFolders(url, folders, deadline) : {};+ return typeof sparse === 'string' ? sparse : cloneWholeRepository(url, deadline, sparse.failure);+}++async function cloneOnlyFolders(+ url: string,+ folders: readonly string[],+ deadline: number,+): Promise<string | { failure?: string }> {const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));- const inside = relative(tmp, resolve(tmp, subpath));- if (inside !== '' && !inside.startsWith('..') && !isAbsolute(inside)) {- const sparsePath = inside.split(sep).join('/');- try {- await runGit(- ['clone', '-q', '--depth', '1', '--filter=blob:none', '--sparse', '--', url, tmp],- deadline,- );- await runGit(- ['-C', tmp, 'sparse-checkout', 'set', '--skip-checks', '--stdin'],- deadline,- `${sparsePath}\n`,- );- if (existsSync(join(tmp, inside)) || headCommitLacks(tmp, sparsePath)) return tmp;- console.warn(- '[skills-catalog] sparse checkout left the subpath unselected, cloning the whole repository',- { url, subpath },- );- } catch (e) {- if (budgetSpent(e, deadline)) {- rmSync(tmp, { recursive: true, force: true });- throw gitCloneFailed(url, e);- }- console.warn('[skills-catalog] sparse clone failed, cloning the whole repository', {- url,- subpath,- err: e,- });- }+ const inside = folders.map((folder) => relative(tmp, resolve(tmp, folder)));+ if (inside.some((p) => p === '' || p.startsWith('..') || isAbsolute(p))) {+ rmSync(tmp, { recursive: true, force: true });+ return {};+ }+ const sparsePaths = inside.map((p) => p.split(sep).join('/'));+ const subpath = folders.join(', ');+ try {+ await runGit(+ ['clone', '-q', '--depth', '1', '--filter=blob:none', '--sparse', '--', url, tmp],+ deadline,+ );+ await runGit(+ ['-C', tmp, 'sparse-checkout', 'set', '--skip-checks', '--stdin'],+ deadline,+ sparsePaths.map((p) => `${p}\n`).join(''),+ );+ const unselected = folders.filter(+ (_, i) =>+ !existsSync(join(tmp, inside[i] as string)) &&+ !headCommitLacks(tmp, sparsePaths[i] as string),+ );+ if (unselected.length === 0) return tmp;+ console.warn(+ '[skills-catalog] sparse checkout left the subpath unselected, cloning the whole repository',+ { url: withoutUrlCredentials(url), subpath: unselected.join(', ') },+ );+ rmSync(tmp, { recursive: true, force: true });+ return { failure: `sparse checkout left ${unselected.join(', ')} unselected` };+ } catch (e) {+ rmSync(tmp, { recursive: true, force: true });+ if (budgetSpent(e, deadline)) throw gitFetchTimedOut(url);+ const failure = gitFailureReason(e);+ console.warn('[skills-catalog] sparse clone failed, cloning the whole repository', {+ url: withoutUrlCredentials(url),+ subpath,+ err: new Error(failure),+ });+ return { failure };+ }+}++async function cloneSkillFilesOnly(+ url: string,+ deadline: number,+): Promise<string | { failure: string }> {+ const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-import-'));+ try {+ await runGit(+ ['clone', '-q', '--depth', '1', '--filter=blob:none', '--no-checkout', '--', url, tmp],+ deadline,+ );+ mkdirSync(join(tmp, '.git', 'info'), { recursive: true });+ writeFileSync(join(tmp, '.git', 'info', 'sparse-checkout'), 'SKILL.md\n');+ await runGit(+ [+ '-c',+ 'core.sparseCheckout=true',+ '-c',+ 'core.sparseCheckoutCone=false',+ '-C',+ tmp,+ 'read-tree',+ '-mu',+ 'HEAD',+ ],+ deadline,+ );+ return tmp;+ } catch (e) {+ rmSync(tmp, { recursive: true, force: true });+ if (budgetSpent(e, deadline)) throw gitFetchTimedOut(url);+ const failure = gitFailureReason(e);+ console.warn('[skills-catalog] skill listing clone failed, cloning the whole repository', {+ url: withoutUrlCredentials(url),+ err: new Error(failure),+ });+ return { failure };}- rmSync(tmp, { recursive: true, force: true });- return undefined;}function budgetSpent(e: unknown, deadline: number): boolean {diff --git a/packages/core/src/skills-catalog/acquire/lockfile.ts b/packages/core/src/skills-catalog/acquire/lockfile.tsindex 62b6a1711..1874d20fb 100644--- a/packages/core/src/skills-catalog/acquire/lockfile.ts+++ b/packages/core/src/skills-catalog/acquire/lockfile.ts@@ -16,6 +16,7 @@ export const SkillLockEntrySchema = z.looseObject({source: z.string(),pluginProvider: z.string().optional(),skill: z.string().optional(),+ subpath: z.string().optional(),ref: z.string().optional(),contentHash: z.string(),files: z.array(z.string()).optional(),diff --git a/packages/core/src/skills-catalog/acquire/parse.ts b/packages/core/src/skills-catalog/acquire/parse.tsindex 5622c7b95..57760ca6a 100644--- a/packages/core/src/skills-catalog/acquire/parse.ts+++ b/packages/core/src/skills-catalog/acquire/parse.ts@@ -190,15 +190,23 @@ function acquiredSkill(dir: string, skillMd: string, files: AcquiredFile[]): Acqreturn { name, description, skillMd, files, contentHash: hash.digest('hex') };}+function hasSkillFile(dir: string): boolean {+ try {+ return statSync(join(dir, 'SKILL.md')).isFile();+ } catch {+ return false;+ }+}+export function discoverSkillDirs(root: string): Array<{ name: string; dir: string }> {- if (existsSync(join(root, 'SKILL.md'))) return [{ name: basename(root), dir: root }];+ if (hasSkillFile(root)) return [{ name: basename(root), dir: root }];const found: Array<{ name: string; dir: string }> = [];const MAX_DEPTH = 8;const MAX_DIRS = 5_000;let visited = 0;const walk = (dir: string, depth: number) => {if (depth > MAX_DEPTH || ++visited > MAX_DIRS) return;- if (existsSync(join(dir, 'SKILL.md'))) {+ if (hasSkillFile(dir)) {found.push({ name: basename(dir), dir });return;}diff --git a/packages/core/src/skills-catalog/index.ts b/packages/core/src/skills-catalog/index.tsindex cf4d8f5f0..826cbb4c8 100644--- a/packages/core/src/skills-catalog/index.ts+++ b/packages/core/src/skills-catalog/index.ts@@ -1,10 +1,13 @@export {ALLOWED_GIT_TRANSPORTS,type Fetched,+ type FetchedListing,fetchSource,+ fetchSourceListing,parseSkillsShSource,parseSource,SkillFetchError,+ SkillSubpathNotFoundError,type SkillsShSourceRef,type SourceSpec,} from './acquire/fetch.ts';diff --git a/packages/server/src/api-skill-git-source-composition.test.ts b/packages/server/src/api-skill-git-source-composition.test.tsnew file mode 100644index 000000000..d42a00c96--- /dev/null+++ b/packages/server/src/api-skill-git-source-composition.test.ts@@ -0,0 +1,327 @@+import { randomBytes } from 'node:crypto';+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterAll, afterEach, beforeAll, beforeEach, expect, test, vi } from 'vitest';+import {+ type ServedGitRepos,+ serveGitRepos,+} from '../../../test-support/served-git-repos.test-helper.ts';+import type { BootedServer } from './boot.ts';+import { bootCompositionRig, rawRequest } from './composition-rig.test-helper.ts';++const OUTSIDE_SKILLS_BYTES = 2_000_000;+const LARGE_SKILL_MD_BYTES = 1_000_000;+const skillMd = (name: string, revision = 'first') =>+ `---\nname: ${name}\ndescription: ${name} skill\n---\n\n${revision} revision of ${name}.\n`;++function publishTeamRepo(repo: string, folders: Readonly<Record<string, string>>): string {+ const files: Record<string, string | Buffer> = {+ 'README.md': 'Team skills\n',+ 'assets/blob.bin': randomBytes(OUTSIDE_SKILLS_BYTES),+ };+ for (const [folder, name] of Object.entries(folders)) {+ files[`${folder}/SKILL.md`] = skillMd(name);+ files[`${folder}/references/guide.md`] = `${name} guide\n`;+ }+ repos.publish(repo, files);+ return repo;+}++let root: string;+let server: BootedServer;+let repos: ServedGitRepos;++beforeAll(async () => {+ repos = await serveGitRepos();+ root = mkdtempSync(join(tmpdir(), 'ok-git-source-contract-'));+ mkdirSync(join(root, '.claude'));+ server = await bootCompositionRig(root);+ await server.ready;+}, 60_000);++beforeEach(() => {+ for (const name of Object.keys(process.env)) {+ if (name.startsWith('GIT_')) vi.stubEnv(name, undefined);+ }+ for (const [name, value] of Object.entries(repos.githubRedirectEnv())) vi.stubEnv(name, value);+ repos.resetBytesServed();+});++afterEach(() => {+ vi.unstubAllEnvs();+});++afterAll(async () => {+ await server?.destroy();+ await repos?.close();+ rmSync(root, { recursive: true, force: true });+});++async function post(path: string, body: object) {+ const result = await rawRequest(server.port, path, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ expect(result.status, result.body).toBe(200);+ return JSON.parse(result.body);+}++const lockPath = () => join(root, '.ok/skills-lock.json');++function lockEntry(name: string): Record<string, unknown> | undefined {+ return JSON.parse(readFileSync(lockPath(), 'utf8')).skills[name];+}++function forgetRecordedFolder(name: string): void {+ const lock = JSON.parse(readFileSync(lockPath(), 'utf8'));+ delete lock.skills[name].subpath;+ writeFileSync(lockPath(), `${JSON.stringify(lock, null, 2)}\n`);+}++const installed = (name: string, file = 'SKILL.md') =>+ readFileSync(join(root, '.claude/skills', name, file), 'utf8');++test('the picker lists a repository by name without downloading it whole', async () => {+ const repo = publishTeamRepo('fixture-org/listing', {+ 'skills/alpha': 'alpha',+ 'skills/beta': 'beta',+ 'skills/beta/upstream': 'nested',+ 'tools/scripts/helper': 'helper',+ 'plugins/eng/skills/gamma': 'gamma',+ 'archive/one/dup': 'dup',+ 'archive/two/dup': 'dup',+ });+ repos.resetBytesServed();+ const result = await rawRequest(+ server.port,+ `/api/skills/discover?source=${encodeURIComponent(repo)}`,+ );+ expect(result.status, result.body).toBe(200);+ expect(JSON.parse(result.body).skills).toEqual([+ { name: 'alpha', description: 'alpha skill' },+ { name: 'beta', description: 'beta skill' },+ { name: 'dup', description: 'dup skill' },+ { name: 'dup', description: 'dup skill' },+ { name: 'gamma', description: 'gamma skill' },+ ]);+ expect(repos.bytesServed()).toBeGreaterThan(0);+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);+});++test('importing a repository by name and picking a skill records its folder and downloads only that folder', async () => {+ const repo = publishTeamRepo('fixture-org/import', {+ 'plugins/eng/skills/imp-gamma': 'imp-gamma',+ 'skills/imp-other': 'imp-other',+ });+ repos.resetBytesServed();+ const result = await post('/api/skill/import', {+ source: repo,+ skill: 'imp-gamma',+ install: false,+ });+ expect(result).toMatchObject({ name: 'imp-gamma', created: true, provenance: { source: repo } });+ expect(installed('imp-gamma')).toBe(skillMd('imp-gamma'));+ expect(installed('imp-gamma', 'references/guide.md')).toBe('imp-gamma guide\n');+ expect(existsSync(join(root, '.claude/skills/imp-other'))).toBe(false);+ expect(lockEntry('imp-gamma')).toMatchObject({+ source: repo,+ skill: 'imp-gamma',+ subpath: 'plugins/eng/skills/imp-gamma',+ });+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);+});++test('a source that already names a folder is recorded as before, without a separate folder', async () => {+ const repo = publishTeamRepo('fixture-org/named', { 'skills/named-alpha': 'named-alpha' });+ repos.resetBytesServed();+ const source = `${repo}/skills/named-alpha`;+ const result = await post('/api/skill/import', { source, install: false });+ expect(result).toMatchObject({ name: 'named-alpha', provenance: { source } });+ expect(installed('named-alpha', 'references/guide.md')).toBe('named-alpha guide\n');+ const entry = lockEntry('named-alpha');+ expect(entry).toMatchObject({ source, skill: 'named-alpha' });+ expect(entry).not.toHaveProperty('subpath');+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);+});++test('the update check and update of a skill with a recorded folder fetch only that folder', async () => {+ const repo = publishTeamRepo('fixture-org/update', { 'skills/upd-beta': 'upd-beta' });+ repos.publish(repo, {+ 'skills/upd-large/SKILL.md': `${skillMd('upd-large')}\n${randomBytes(LARGE_SKILL_MD_BYTES).toString('base64')}\n`,+ });+ await post('/api/skill/import', { source: repo, skill: 'upd-beta', install: false });+ repos.publish(repo, { 'skills/upd-beta/SKILL.md': skillMd('upd-beta', 'second') });+ const lockBefore = readFileSync(lockPath(), 'utf8');+ repos.resetBytesServed();++ const check = await post('/api/skill/reimport', {+ name: 'upd-beta',+ scope: 'project',+ dryRun: true,+ });+ expect(check).toMatchObject({ updated: true });+ expect(check.upstreamBody).toContain('second revision of upd-beta.');+ expect(readFileSync(lockPath(), 'utf8')).toBe(lockBefore);+ expect(repos.bytesServed()).toBeLessThan(LARGE_SKILL_MD_BYTES);++ repos.resetBytesServed();+ expect(await post('/api/skill/reimport', { name: 'upd-beta', scope: 'project' })).toMatchObject({+ updated: true,+ source: repo,+ });+ expect(installed('upd-beta')).toBe(skillMd('upd-beta', 'second'));+ expect(lockEntry('upd-beta')).toMatchObject({ subpath: 'skills/upd-beta' });+ expect(repos.bytesServed()).toBeLessThan(LARGE_SKILL_MD_BYTES);+});++test('an import recorded without a folder keeps a fast update check and gains its folder at its next update', async () => {+ const repo = publishTeamRepo('fixture-org/legacy', {+ 'skills/leg-alpha': 'leg-alpha',+ 'skills/leg-other': 'leg-other',+ });+ await post('/api/skill/import', { source: repo, skill: 'leg-alpha', install: false });+ forgetRecordedFolder('leg-alpha');+ repos.publish(repo, { 'skills/leg-alpha/SKILL.md': skillMd('leg-alpha', 'second') });+ const lockBefore = readFileSync(lockPath(), 'utf8');+ repos.resetBytesServed();++ const check = await post('/api/skill/reimport', {+ name: 'leg-alpha',+ scope: 'project',+ dryRun: true,+ });+ expect(check).toMatchObject({ updated: true });+ expect(readFileSync(lockPath(), 'utf8')).toBe(lockBefore);+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);++ repos.resetBytesServed();+ expect(await post('/api/skill/reimport', { name: 'leg-alpha', scope: 'project' })).toMatchObject({+ updated: true,+ });+ expect(installed('leg-alpha')).toBe(skillMd('leg-alpha', 'second'));+ expect(installed('leg-alpha', 'references/guide.md')).toBe('leg-alpha guide\n');+ expect(lockEntry('leg-alpha')).toMatchObject({ subpath: 'skills/leg-alpha' });+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);+});++test('an import recorded without a folder whose name matches two folders is not given one', async () => {+ const repo = publishTeamRepo('fixture-org/ambiguous', {+ 'archive/one/amb-dup': 'amb-dup',+ 'archive/two/amb-dup': 'amb-dup',+ });+ await post('/api/skill/import', { source: repo, skill: 'amb-dup', install: false });+ forgetRecordedFolder('amb-dup');+ repos.publish(repo, {+ 'archive/one/amb-dup/SKILL.md': skillMd('amb-dup', 'second'),+ 'archive/two/amb-dup/SKILL.md': skillMd('amb-dup', 'second'),+ });+ expect(await post('/api/skill/reimport', { name: 'amb-dup', scope: 'project' })).toMatchObject({+ updated: true,+ });+ expect(installed('amb-dup')).toBe(skillMd('amb-dup', 'second'));+ expect(lockEntry('amb-dup')).not.toHaveProperty('subpath');+});++test('a recorded folder that moved is found again by name and recorded anew', async () => {+ const repo = publishTeamRepo('fixture-org/moved', {+ 'plugins/eng/skills/mov-gamma': 'mov-gamma',+ });+ await post('/api/skill/import', { source: repo, skill: 'mov-gamma', install: false });+ expect(lockEntry('mov-gamma')).toMatchObject({ subpath: 'plugins/eng/skills/mov-gamma' });+ repos.publish(repo, {+ 'plugins/eng/skills/mov-gamma': null,+ 'plugins/ops/skills/mov-gamma/SKILL.md': skillMd('mov-gamma', 'moved'),+ });+ expect(await post('/api/skill/reimport', { name: 'mov-gamma', scope: 'project' })).toMatchObject({+ updated: true,+ });+ expect(installed('mov-gamma')).toBe(skillMd('mov-gamma', 'moved'));+ expect(lockEntry('mov-gamma')).toMatchObject({ subpath: 'plugins/ops/skills/mov-gamma' });+});++test('a recorded folder that is gone, whose name now matches two folders, is not kept in the lock entry', async () => {+ const repo = publishTeamRepo('fixture-org/gone-ambiguous', {+ 'archive/one/gone-dup': 'gone-dup',+ });+ await post('/api/skill/import', { source: repo, skill: 'gone-dup', install: false });+ expect(lockEntry('gone-dup')).toMatchObject({ subpath: 'archive/one/gone-dup' });+ repos.publish(repo, {+ 'archive/one/gone-dup': null,+ 'archive/two/gone-dup/SKILL.md': skillMd('gone-dup', 'second'),+ 'archive/three/gone-dup/SKILL.md': skillMd('gone-dup', 'second'),+ });+ expect(await post('/api/skill/reimport', { name: 'gone-dup', scope: 'project' })).toMatchObject({+ updated: true,+ });+ expect(installed('gone-dup')).toBe(skillMd('gone-dup', 'second'));+ const entry = lockEntry('gone-dup');+ expect(entry).toMatchObject({ source: repo, skill: 'gone-dup' });+ expect(entry).not.toHaveProperty('subpath');+});++test('Update all from this source fetches only the folders of the skills it updates', async () => {+ const repo = publishTeamRepo('fixture-org/update-all', {+ 'skills/all-one': 'all-one',+ 'plugins/eng/skills/all-two': 'all-two',+ });+ await post('/api/skill/import', { source: repo, skill: 'all-one', install: false });+ await post('/api/skill/import', { source: repo, skill: 'all-two', install: false });+ forgetRecordedFolder('all-two');+ repos.publish(repo, {+ 'skills/all-one/SKILL.md': skillMd('all-one', 'second'),+ 'plugins/eng/skills/all-two/SKILL.md': skillMd('all-two', 'second'),+ });+ repos.resetBytesServed();+ const result = await post('/api/skills/reimport-bulk', {+ names: ['all-one', 'all-two'],+ scope: 'project',+ });+ expect(result).toMatchObject({ updated: 2, failed: 0 });+ expect(installed('all-one')).toBe(skillMd('all-one', 'second'));+ expect(installed('all-two')).toBe(skillMd('all-two', 'second'));+ expect(lockEntry('all-one')).toMatchObject({ subpath: 'skills/all-one' });+ expect(lockEntry('all-two')).toMatchObject({ subpath: 'plugins/eng/skills/all-two' });+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);+});++test('a bulk import by repository name records each folder and downloads only those folders', async () => {+ const repo = publishTeamRepo('fixture-org/bulk-import', {+ 'skills/bulk-one': 'bulk-one',+ 'plugins/eng/skills/bulk-two': 'bulk-two',+ });+ repos.resetBytesServed();+ const result = await post('/api/skills/import-bulk', {+ source: repo,+ skills: ['bulk-one', 'bulk-two', 'missing'],+ install: false,+ });+ expect(result).toMatchObject({ imported: 2, failed: 1 });+ expect(installed('bulk-two', 'references/guide.md')).toBe('bulk-two guide\n');+ expect(lockEntry('bulk-one')).toMatchObject({ source: repo, subpath: 'skills/bulk-one' });+ expect(lockEntry('bulk-two')).toMatchObject({+ source: repo,+ subpath: 'plugins/eng/skills/bulk-two',+ });+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);+});++test('a recorded folder that would leave the repository is ignored, and the skill is found by name', async () => {+ const repo = publishTeamRepo('fixture-org/escape', { 'skills/esc-alpha': 'esc-alpha' });+ await post('/api/skill/import', { source: repo, skill: 'esc-alpha', install: false });+ const lock = JSON.parse(readFileSync(lockPath(), 'utf8'));+ lock.skills['esc-alpha'].subpath = '../../../../../../etc';+ writeFileSync(lockPath(), `${JSON.stringify(lock, null, 2)}\n`);+ repos.publish(repo, { 'skills/esc-alpha/SKILL.md': skillMd('esc-alpha', 'second') });+ repos.resetBytesServed();++ const check = await post('/api/skill/reimport', {+ name: 'esc-alpha',+ scope: 'project',+ dryRun: true,+ });+ expect(check).toMatchObject({ updated: true });+ expect(check.upstreamBody).toContain('second revision of esc-alpha.');+ expect(repos.bytesServed()).toBeLessThan(OUTSIDE_SKILLS_BYTES);+});diff --git a/packages/server/src/http/skills-import-routes.ts b/packages/server/src/http/skills-import-routes.tsindex aad18e1f5..3013039c6 100644--- a/packages/server/src/http/skills-import-routes.ts+++ b/packages/server/src/http/skills-import-routes.ts@@ -14,7 +14,9 @@ import {} from '@inkeep/open-knowledge-core';import {discoverSkillDirs,+ type FetchedListing,fetchSource,+ fetchSourceListing,parseSkillDir,parseSource,readSkillDirMeta,@@ -33,6 +35,11 @@ import { createMultipartParser, type MultipartParser } from '../multipart.ts';import type { SkillImportOutcome, SkillImportService } from '../services/skill-import.ts';import { rejectDisallowedGitSpec } from '../skill-git-spec-guard.ts';import { resolveSkillInstallReportSettings } from '../skill-install-report-config.ts';+import {+ fetchPickedFolders,+ recordsSkillFolder,+ skillFolderWithin,+} from '../skill-source-folders.ts';import { reportSkillInstall } from '../skills-sh-install-report.ts';import { type ApiRouteGroup, createApiRouteGroup } from './api-pipeline.ts';import { errorResponse } from './error-response.ts';@@ -113,6 +120,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGlet ref: string | undefined;let publisher: string | undefined;let upstreamSkill: string | undefined;+ let subpath: string | undefined;let resolvedSourceForReport = body.source;{@@ -138,10 +146,10 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGreturn;}if (rejectDisallowedGitSpec(res, spec, 'skill-import')) return;- const fetched = await fetchSource(spec);- cleanup = fetched.cleanup;- ref = fetched.ref;- const dirs = discoverSkillDirs(fetched.dir);+ const listing = await fetchSourceListing(spec);+ cleanup = listing.cleanup;+ ref = listing.ref;+ const dirs = discoverSkillDirs(listing.dir);if (dirs.length === 0) {errorResponse(res, 404, 'urn:ok:error:not-found', 'No SKILL.md found in source.', {handler: 'skill-import',@@ -172,7 +180,16 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteG);return;}- acquiredDir = pick.dir;+ const folder = skillFolderWithin(listing.dir, pick.dir);+ const content = await fetchPickedFolders(listing, [pick.dir]);+ const listingCleanup = cleanup;+ cleanup = () => {+ content.cleanup();+ listingCleanup();+ };+ ref = content.ref;+ acquiredDir = content.dirs[0] ?? pick.dir;+ if (recordsSkillFolder(spec) && folder !== '') subpath = folder;upstreamSkill = pick.name;sourceLabel = rawSource;publisher = skillsSh?.publisher ?? publisherFromSource(resolvedSource);@@ -201,6 +218,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGref,publisher,upstreamSkill,+ ...(subpath !== undefined ? { subpath } : {}),actor,skipProjection: body.install === false,});@@ -260,6 +278,11 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGlet ref: string | undefined;let publisher: string | undefined;let resolvedSourceForReport = rawSource;+ const fetchedDirs = new Map<string, { dir: string; subpath?: string }>();+ const findRequested = (requested: string) =>+ dirs.find((d) => d.name === requested) ??+ dirs.find((d) => parseSkillDir(d.dir)?.name === requested) ??+ dirs.find((d) => d.name === RENAMED_PACK_SKILLS[requested]);try {const skillsSh = await resolveSkillsShImportSource(rawSource, body.skills[0]);const resolvedSource = skillsSh?.source ?? rawSource;@@ -279,10 +302,27 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGsiteSpec = spec;siteIndex = await readWellKnownIndex(spec.origin);} else {- const fetched = await fetchSource(spec);- cleanup = fetched.cleanup;- ref = fetched.ref;- dirs = discoverSkillDirs(fetched.dir);+ const listing: FetchedListing = await fetchSourceListing(spec);+ cleanup = listing.cleanup;+ ref = listing.ref;+ dirs = discoverSkillDirs(listing.dir);+ const picked = [...new Set(body.skills)].flatMap((requested) => {+ const found = findRequested(requested);+ return found ? [found.dir] : [];+ });+ const content = await fetchPickedFolders(listing, picked);+ cleanup = () => {+ content.cleanup();+ listing.cleanup();+ };+ ref = content.ref;+ picked.forEach((dir, i) => {+ const folder = skillFolderWithin(listing.dir, dir);+ fetchedDirs.set(dir, {+ dir: content.dirs[i] ?? dir,+ ...(recordsSkillFolder(spec) && folder !== '' ? { subpath: folder } : {}),+ });+ });}} catch (e) {if (e instanceof SkillFetchError) {@@ -304,6 +344,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGfor (const requested of new Set(body.skills)) {let acquiredDir: string;let upstreamSkill: string;+ let subpath: string | undefined;let perSkill: () => void = () => {};if (siteSpec !== null) {try {@@ -325,15 +366,14 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGcontinue;}} else {- const found =- dirs.find((d) => d.name === requested) ??- dirs.find((d) => parseSkillDir(d.dir)?.name === requested) ??- dirs.find((d) => d.name === RENAMED_PACK_SKILLS[requested]);+ const found = findRequested(requested);if (!found) {results.push({ requested, status: 'not-found', warnings: [], warningCodes: [] });continue;}- acquiredDir = found.dir;+ const fetched = fetchedDirs.get(found.dir);+ acquiredDir = fetched?.dir ?? found.dir;+ subpath = fetched?.subpath;upstreamSkill = found.name;}try {@@ -344,6 +384,7 @@ export function createSkillsImportRoutes(deps: SkillsImportRouteDeps): ApiRouteGref,publisher,upstreamSkill,+ ...(subpath !== undefined ? { subpath } : {}),actor,skipProjection: body.install === false,});diff --git a/packages/server/src/http/skills-recovery-routes.ts b/packages/server/src/http/skills-recovery-routes.tsindex fdda4d34c..8054711a9 100644--- a/packages/server/src/http/skills-recovery-routes.ts+++ b/packages/server/src/http/skills-recovery-routes.ts@@ -16,8 +16,6 @@ import {} from '@inkeep/open-knowledge-core';import { resolveProjectIdentity } from '@inkeep/open-knowledge-core/shadow-repo-layout';import {- discoverSkillDirs,- fetchSource,parseSkillDir,parseSource,resolvePluginUpdateSource,@@ -32,16 +30,20 @@ import type { ContentFilter } from '../content-filter.ts';import { extractActorIdentity } from '../extract-actor-identity.ts';import { scanGlobalInPlaceSkills, scanInPlaceSkills } from '../in-place-skills.ts';import { getLogger } from '../logger.ts';-import type { SkillReimportDeps, SkillReimportService } from '../services/skill-reimport.ts';+import type {+ RecordedFolderUpdate,+ SkillReimportDeps,+ SkillReimportService,+} from '../services/skill-reimport.ts';import {groupReimportNamesBySource,- pickReimportDir,type SkillReimportOutcome,} from '../services/skill-reimport.ts';import type { ShadowRef } from '../shadow-repo.ts';import { isDisallowedGitSpec, rejectDisallowedGitSpec } from '../skill-git-spec-guard.ts';import { validateSkillForInstall } from '../skill-projection.ts';import { restoreSkillVersion } from '../skill-restore.ts';+import { acquireReimportSources, type ReimportAcquisition } from '../skill-source-folders.ts';import { mutateSkillsLock, readSkillsLockFile } from '../skills-lock-store.ts';import { type ApiRouteGroup, createApiRouteGroup } from './api-pipeline.ts';import { errorResponse } from './error-response.ts';@@ -377,6 +379,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo}let ref: string | undefined;let acquiredDir: string | null = null;+ let folder: RecordedFolderUpdate | undefined;try {const recordedSkill = entry.skill ?? body.name;const renamedSkill = RENAMED_PACK_SKILLS[recordedSkill];@@ -401,26 +404,28 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRoreturn;}if (rejectDisallowedGitSpec(res, spec, 'skill-reimport')) return;- const fetched = await fetchSource(spec);- cleanup = fetched.cleanup;- ref = fetched.ref;- const dirs = discoverSkillDirs(fetched.dir);- const pick = pickReimportDir(dirs, {- ...(resolvedSkill !== undefined ? { recordedSkill: resolvedSkill } : {}),- localName: body.name,- frontmatterNameOf: (dir) => parseSkillDir(dir)?.name,- });+ const acquired = await acquireReimportSources(spec, [+ {+ name: body.name,+ entry,+ ...(resolvedSkill !== undefined ? { recordedSkill: resolvedSkill } : {}),+ },+ ]);+ cleanup = acquired.cleanup;+ ref = acquired.ref;+ const pick = acquired.picks.get(body.name);if (!pick) {errorResponse(res,404,'urn:ok:error:not-found','Could not locate this skill in its source anymore.',- { handler: 'skill-reimport', detail: dirs.map((d) => d.name).join(', ') },+ { handler: 'skill-reimport', detail: acquired.dirs.map((d) => d.name).join(', ') },);return;}acquiredDir = pick.dir;+ folder = pick.folder;} catch (e) {if (e instanceof SkillFetchError) {errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Could not fetch source.', {@@ -444,6 +449,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRoentry,lockPath,ref,+ ...(folder ? { folder } : {}),actor,...(body.dryRun !== undefined ? { dryRun: body.dryRun } : {}),});@@ -518,8 +524,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRofor (const group of bySource) {let cleanup: () => void = () => {};try {- let ref: string | undefined;- let dirs: ReturnType<typeof discoverSkillDirs> = [];+ let acquired: ReimportAcquisition;try {const probe = group.names[0] as string;const skillsSh = await resolveSkillsShImportSource(group.source, probe).catch(@@ -549,10 +554,27 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo}continue;}- const fetched = await fetchSource(spec);- cleanup = fetched.cleanup;- ref = fetched.ref;- dirs = discoverSkillDirs(fetched.dir);+ acquired = await acquireReimportSources(+ spec,+ group.names.flatMap((name) => {+ const entry = resolveReimportLockEntry(+ body.scope,+ name,+ effectiveSkillRoot(body.scope, name).root,+ lock,+ );+ return entry+ ? [+ {+ name,+ entry,+ ...(entry.skill !== undefined ? { recordedSkill: entry.skill } : {}),+ },+ ]+ : [];+ }),+ );+ cleanup = acquired.cleanup;} catch (e) {getLogger('skills-reimport-bulk').warn({ source: group.source, err: e, requestId: getRequestId(_req) },@@ -586,11 +608,7 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRo});continue;}- const pick = pickReimportDir(dirs, {- ...(entry.skill !== undefined ? { recordedSkill: entry.skill } : {}),- localName: name,- frontmatterNameOf: (dir) => parseSkillDir(dir)?.name,- });+ const pick = acquired.picks.get(name);if (!pick) {results.push({requested: name,@@ -608,7 +626,8 @@ export function createSkillsRecoveryRoutes(deps: SkillsRecoveryRouteDeps): ApiRoscope: body.scope,entry,lockPath,- ref,+ ref: acquired.ref,+ ...(pick.folder ? { folder: pick.folder } : {}),actor,});if (!outcome.ok) {diff --git a/packages/server/src/services/skill-import.ts b/packages/server/src/services/skill-import.tsindex ef20f4474..10b0635ef 100644--- a/packages/server/src/services/skill-import.ts+++ b/packages/server/src/services/skill-import.ts@@ -98,6 +98,7 @@ export interface SkillImportService {ref?: string;publisher?: string;upstreamSkill?: string;+ subpath?: string;actor: ActorIdentity;skipProjection?: boolean;}): Promise<SkillImportOutcome>;@@ -110,7 +111,8 @@ export function createSkillImportService(deps: SkillImportDeps): SkillImportServreturn {async runSkillImport(params) {- const { acquiredDir, scope, sourceLabel, ref, publisher, upstreamSkill, actor } = params;+ const { acquiredDir, scope, sourceLabel, ref, publisher, upstreamSkill, subpath, actor } =+ params;if (!deps.projectDir) {return {ok: false,@@ -277,6 +279,7 @@ export function createSkillImportService(deps: SkillImportDeps): SkillImportServpublisher,importedAt: new Date().toISOString(),...(upstreamSkill !== undefined ? { skill: upstreamSkill } : {}),+ ...(subpath !== undefined ? { subpath } : {}),}),);diff --git a/packages/server/src/services/skill-reimport.ts b/packages/server/src/services/skill-reimport.tsindex 45f76ab49..4591c82c8 100644--- a/packages/server/src/services/skill-reimport.ts+++ b/packages/server/src/services/skill-reimport.ts@@ -66,6 +66,19 @@ export interface SkillReimportDeps {signalFiles: () => void;}+export type RecordedFolderUpdate =+ | { readonly kind: 'set'; readonly subpath: string }+ | { readonly kind: 'clear' };++function withRecordedFolder(+ entry: SkillLockEntry,+ folder: RecordedFolderUpdate | undefined,+): SkillLockEntry {+ if (folder === undefined) return entry;+ const { subpath: _replaced, ...rest } = entry;+ return folder.kind === 'set' ? { ...rest, subpath: folder.subpath } : rest;+}+export interface SkillReimportService {runSkillReimport(params: {acquiredDir: string;@@ -74,27 +87,42 @@ export interface SkillReimportService {entry: SkillLockEntry;lockPath: string;ref?: string;+ folder?: RecordedFolderUpdate;actor: ActorIdentity;dryRun?: boolean;}): Promise<SkillReimportOutcome>;}+interface ReimportPickOptions {+ recordedSkill?: string;+ localName: string;+ frontmatterNameOf: (dir: string) => string | undefined;+}+export function pickReimportDir<T extends { name: string; dir: string }>(dirs: readonly T[],- opts: {- recordedSkill?: string;- localName: string;- frontmatterNameOf: (dir: string) => string | undefined;- },+ opts: ReimportPickOptions,): T | undefined {+ return reimportCandidates(dirs, opts)[0];+}++export function reimportCandidates<T extends { name: string; dir: string }>(+ dirs: readonly T[],+ opts: ReimportPickOptions,+): T[] {const { recordedSkill, localName, frontmatterNameOf } = opts;- return (- (recordedSkill ? dirs.find((d) => d.name === recordedSkill) : undefined) ??- dirs.find((d) => d.name === localName) ??- dirs.find((d) => frontmatterNameOf(d.dir) === localName) ??- dirs.find((d) => d.name === RENAMED_PACK_SKILLS[recordedSkill ?? localName]) ??- (dirs.length === 1 ? dirs[0] : undefined)- );+ const renamed = RENAMED_PACK_SKILLS[recordedSkill ?? localName];+ const rules: Array<(d: T) => boolean> = [+ (d) => recordedSkill !== undefined && recordedSkill !== '' && d.name === recordedSkill,+ (d) => d.name === localName,+ (d) => frontmatterNameOf(d.dir) === localName,+ (d) => renamed !== undefined && d.name === renamed,+ ];+ for (const rule of rules) {+ const matches = dirs.filter(rule);+ if (matches.length > 0) return matches;+ }+ return dirs.length === 1 ? [...dirs] : [];}export function planReimportDiff(input: {@@ -144,7 +172,7 @@ export function groupReimportNamesBySource(export function createSkillReimportService(deps: SkillReimportDeps): SkillReimportService {return {async runSkillReimport(params) {- const { acquiredDir, name, scope, entry, lockPath, ref, actor } = params;+ const { acquiredDir, name, scope, entry, lockPath, ref, folder, actor } = params;const reimportBase = scope === 'global' ? deps.skillsHome : deps.contentDir;const {root: skillsRoot,@@ -317,7 +345,7 @@ export function createSkillReimportService(deps: SkillReimportDeps): SkillReimpo: undefined;await mutateSkillsLock(lockPath, (current: SkillsLock) =>upsertLockEntry(current, name, {- ...(current.skills[name] ?? entry),+ ...withRecordedFolder(current.skills[name] ?? entry, folder),contentHash: acquired.contentHash,files: acquired.files.map((file) => file.relPath),...(reimportLocalHash !== undefined ? { localHash: reimportLocalHash } : {}),diff --git a/packages/server/src/skill-source-cache.ts b/packages/server/src/skill-source-cache.tsindex e06171d4a..74ff581ae 100644--- a/packages/server/src/skill-source-cache.ts+++ b/packages/server/src/skill-source-cache.ts@@ -1,5 +1,5 @@import type { Fetched, SourceSpec } from '@inkeep/open-knowledge-core/skills-catalog';-import { fetchSource } from '@inkeep/open-knowledge-core/skills-catalog';+import { fetchSource, fetchSourceListing } from '@inkeep/open-knowledge-core/skills-catalog';const TTL_MS = 30_000;@@ -26,12 +26,20 @@ function sweepExpired(now: number): void {export async function fetchCachedSource(spec: SourceSpec): Promise<Fetched> {if (spec.kind === 'local') return fetchSource(spec);+ return cached(sourceKey(spec), () => fetchSource(spec));+}++export async function fetchCachedListing(spec: SourceSpec): Promise<Fetched> {+ if (spec.kind !== 'git' || spec.subpath) return fetchCachedSource(spec);+ return cached(`listing\n${sourceKey(spec)}`, () => fetchSourceListing(spec));+}++async function cached(key: string, fetchOnce: () => Promise<Fetched>): Promise<Fetched> {const now = Date.now();sweepExpired(now);- const key = sourceKey(spec);let entry = cache.get(key);if (!entry) {- const fetched = fetchSource(spec);+ const fetched = fetchOnce();entry = { fetched, expiresAt: now + TTL_MS };cache.set(key, entry);const self = entry;diff --git a/packages/server/src/skill-source-folders.ts b/packages/server/src/skill-source-folders.tsnew file mode 100644index 000000000..83d374995--- /dev/null+++ b/packages/server/src/skill-source-folders.ts@@ -0,0 +1,180 @@+import { existsSync } from 'node:fs';+import { join, relative, sep } from 'node:path';+import {+ discoverSkillDirs,+ type FetchedListing,+ fetchSource,+ fetchSourceListing,+ parseSkillDir,+ type SkillLockEntry,+ SkillSubpathNotFoundError,+ type SourceSpec,+} from '@inkeep/open-knowledge-core/skills-catalog';+import {+ pickReimportDir,+ type RecordedFolderUpdate,+ reimportCandidates,+} from './services/skill-reimport.ts';++type SkillDir = { name: string; dir: string };++export function recordsSkillFolder(spec: SourceSpec): boolean {+ return spec.kind === 'git' && !spec.subpath;+}++export function skillFolderWithin(root: string, dir: string): string {+ return relative(root, dir).split(sep).join('/');+}++function usableRecordedSubpath(subpath: string | undefined): string | undefined {+ if (subpath === undefined || /^[A-Za-z]:/.test(subpath)) return undefined;+ const segments = subpath.split('/');+ return segments.every((s) => s !== '' && s !== '.' && s !== '..' && !/[\\\0\r\n]/.test(s))+ ? subpath+ : undefined;+}++export async function fetchPickedFolders(+ listing: FetchedListing,+ pickedDirs: readonly string[],+): Promise<{ dirs: string[]; ref: string | undefined; cleanup: () => void }> {+ if (!listing.fetchFolders || pickedDirs.length === 0) {+ return { dirs: [...pickedDirs], ref: listing.ref, cleanup: () => {} };+ }+ const folders = pickedDirs.map((dir) => skillFolderWithin(listing.dir, dir));+ const content = await listing.fetchFolders([...new Set(folders)]);+ return {+ dirs: folders.map((folder) => join(content.dir, folder)),+ ref: content.ref,+ cleanup: content.cleanup,+ };+}++export interface ReimportWanted {+ readonly name: string;+ readonly entry: SkillLockEntry;+ readonly recordedSkill?: string;+}++export interface ReimportAcquisition {+ readonly dirs: readonly SkillDir[];+ readonly picks: ReadonlyMap<string, { dir: string; folder?: RecordedFolderUpdate }>;+ readonly ref: string | undefined;+ readonly cleanup: () => void;+}++function pickOptions(wanted: ReimportWanted) {+ return {+ ...(wanted.recordedSkill !== undefined ? { recordedSkill: wanted.recordedSkill } : {}),+ localName: wanted.name,+ frontmatterNameOf: (dir: string) => parseSkillDir(dir)?.name,+ };+}++async function fromRecordedFolder(+ spec: SourceSpec & { kind: 'git' },+ wanted: ReimportWanted,+): Promise<ReimportAcquisition | undefined> {+ const subpath = usableRecordedSubpath(wanted.entry.subpath);+ if (subpath === undefined) return undefined;+ let fetched: Awaited<ReturnType<typeof fetchSource>>;+ try {+ fetched = await fetchSource({ ...spec, subpath });+ } catch (e) {+ if (e instanceof SkillSubpathNotFoundError) return undefined;+ throw e;+ }+ try {+ const dirs = discoverSkillDirs(fetched.dir);+ const pick = pickReimportDir(dirs, pickOptions(wanted));+ if (!pick) {+ fetched.cleanup();+ return undefined;+ }+ return {+ dirs,+ picks: new Map([[wanted.name, { dir: pick.dir }]]),+ ref: fetched.ref,+ cleanup: fetched.cleanup,+ };+ } catch (e) {+ fetched.cleanup();+ throw e;+ }+}++export async function acquireReimportSources(+ spec: SourceSpec,+ wanted: readonly ReimportWanted[],+): Promise<ReimportAcquisition> {+ if (spec.kind !== 'git' || spec.subpath) {+ const fetched = await fetchSource(spec);+ try {+ const dirs = discoverSkillDirs(fetched.dir);+ const picks = new Map<string, { dir: string }>();+ for (const want of wanted) {+ const pick = pickReimportDir(dirs, pickOptions(want));+ if (pick) picks.set(want.name, { dir: pick.dir });+ }+ return { dirs, picks, ref: fetched.ref, cleanup: fetched.cleanup };+ } catch (e) {+ fetched.cleanup();+ throw e;+ }+ }+ const single = wanted.length === 1 ? (wanted[0] as ReimportWanted) : undefined;+ const recorded = single ? await fromRecordedFolder(spec, single) : undefined;+ if (recorded) return recorded;++ const listing = await fetchSourceListing(spec);+ try {+ const dirs = discoverSkillDirs(listing.dir);+ const chosen: Array<{ name: string; dir: string; folder?: RecordedFolderUpdate }> = [];+ for (const want of wanted) {+ const recordedFolder = usableRecordedSubpath(want.entry.subpath);+ if (+ recordedFolder !== undefined &&+ existsSync(join(listing.dir, recordedFolder, 'SKILL.md'))+ ) {+ chosen.push({ name: want.name, dir: join(listing.dir, recordedFolder) });+ continue;+ }+ const candidates = reimportCandidates(dirs, pickOptions(want));+ const pick = candidates[0];+ if (!pick) continue;+ const folder = skillFolderWithin(listing.dir, pick.dir);+ const found: RecordedFolderUpdate | undefined =+ candidates.length === 1 && folder !== ''+ ? { kind: 'set', subpath: folder }+ : want.entry.subpath !== undefined+ ? { kind: 'clear' }+ : undefined;+ chosen.push({ name: want.name, dir: pick.dir, ...(found ? { folder: found } : {}) });+ }+ const content = await fetchPickedFolders(+ listing,+ chosen.map((c) => c.dir),+ );+ const picks = new Map(+ chosen.map((c, i) => [+ c.name,+ {+ dir: content.dirs[i] as string,+ ...(c.folder ? { folder: c.folder } : {}),+ },+ ]),+ );+ return {+ dirs,+ picks,+ ref: content.ref,+ cleanup: () => {+ content.cleanup();+ listing.cleanup();+ },+ };+ } catch (e) {+ listing.cleanup();+ throw e;+ }+}diff --git a/packages/server/src/skills-sh-handlers.ts b/packages/server/src/skills-sh-handlers.tsindex d6ae4913c..e0a3333c3 100644--- a/packages/server/src/skills-sh-handlers.ts+++ b/packages/server/src/skills-sh-handlers.ts@@ -14,7 +14,7 @@ import {acquiredBundleTooLarge,discoverSkillDirs,discoverWellKnownSkills,- fetchSource,+ fetchSourceListing,inspectPluginBundleDir,inspectPluginSource,parseGitHubRepoSearch,@@ -35,7 +35,7 @@ import { successResponse } from './http/success-response.ts';import { isAllowedGitUrl } from './local-op-security.ts';import type { PinoLogger } from './logger.ts';import { rejectDisallowedGitSpec } from './skill-git-spec-guard.ts';-import { fetchCachedSource } from './skill-source-cache.ts';+import { fetchCachedListing, fetchCachedSource } from './skill-source-cache.ts';import { getPopularSkills, getPublisherSkills } from './skills-leaderboard.ts';import { readSkillsLockFile } from './skills-lock-store.ts';@@ -400,7 +400,7 @@ export function createSkillsShHandlers(deps: SkillsShHandlerDeps): SkillsShHandl}if (rejectDisallowedGitSpec(res, spec, 'skills-discover')) return;try {- const fetched = await fetchCachedSource(spec);+ const fetched = await fetchCachedListing(spec);const dirs = discoverSkillDirs(fetched.dir);const skills = dirs.map((d) => {const meta = readSkillDirMeta(d.dir);@@ -471,7 +471,7 @@ export function createSkillsShHandlers(deps: SkillsShHandlerDeps): SkillsShHandlif (spec && allowed) {let cleanup: (() => void) | undefined;try {- const fetched = await fetchSource(spec);+ const fetched = await fetchSourceListing(spec);cleanup = fetched.cleanup;const names = discoverSkillDirs(fetched.dir).map((d) => readSkillDirMeta(d.dir)?.name ?? d.name,diff --git a/test-support/served-git-repos.test-helper.ts b/test-support/served-git-repos.test-helper.tsnew file mode 100644index 000000000..8aa7f3734--- /dev/null+++ b/test-support/served-git-repos.test-helper.ts@@ -0,0 +1,119 @@+import { type ChildProcess, execFileSync, spawn } from 'node:child_process';+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { type AddressInfo, createServer } from 'node:net';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { configureTestGitRepository } from './configure-git-fixture.test-helper.ts';++const hermeticGitEnv: Readonly<Record<string, string>> = {+ GIT_CONFIG_GLOBAL: '/dev/null',+ GIT_CONFIG_SYSTEM: '/dev/null',+ GIT_CONFIG_NOSYSTEM: '1',+};++const childEnv: NodeJS.ProcessEnv = {+ ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_'))),+ ...hermeticGitEnv,+};++export interface ServedGitRepos {+ readonly port: number;+ readonly bytesServed: () => number;+ readonly resetBytesServed: () => void;+ readonly url: (name: string) => string;+ readonly githubRedirectEnv: () => Record<string, string>;+ readonly publish: (name: string, files: Readonly<Record<string, string | Buffer | null>>) => void;+ readonly close: () => Promise<void>;+}++function git(cwd: string, args: readonly string[]): void {+ execFileSync('git', args, { cwd, stdio: 'pipe', env: childEnv });+}++function untilClosed(child: ChildProcess): Promise<void> {+ if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();+ return new Promise((resolve) => child.once('close', () => resolve()));+}++export async function serveGitRepos(): Promise<ServedGitRepos> {+ const root = mkdtempSync(join(tmpdir(), 'ok-served-git-'));+ const served = join(root, 'served');+ mkdirSync(served);+ const daemons = new Set<ChildProcess>();+ let bytes = 0;++ const server = createServer((socket) => {+ const daemon = spawn(+ 'git',+ ['daemon', '--inetd', `--base-path=${served}`, '--export-all', served],+ { env: childEnv, stdio: ['pipe', 'pipe', 'ignore'] },+ );+ daemons.add(daemon);+ daemon.once('close', () => {+ daemons.delete(daemon);+ socket.end();+ });+ daemon.stdout.on('data', (chunk: Buffer) => {+ bytes += chunk.length;+ });+ socket.on('error', () => daemon.stdin.destroy());+ daemon.stdin.on('error', () => {});+ socket.pipe(daemon.stdin);+ daemon.stdout.pipe(socket);+ });+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));+ const port = (server.address() as AddressInfo).port;++ const publish = (name: string, files: Readonly<Record<string, string | Buffer | null>>) => {+ const source = join(root, 'sources', name);+ const bare = join(served, `${name}.git`);+ if (!existsSync(source)) {+ mkdirSync(source, { recursive: true });+ git(source, ['init', '-q']);+ git(source, ['symbolic-ref', 'HEAD', 'refs/heads/main']);+ configureTestGitRepository(source);+ }+ for (const [path, body] of Object.entries(files)) {+ const target = join(source, path);+ if (body === null) {+ rmSync(target, { recursive: true, force: true });+ continue;+ }+ mkdirSync(dirname(target), { recursive: true });+ writeFileSync(target, body);+ }+ git(source, ['add', '-A']);+ git(source, ['-c', '[email protected]', '-c', 'user.name=t', 'commit', '-q', '-m', 'publish']);+ if (!existsSync(bare)) {+ git(root, ['clone', '-q', '--bare', source, bare]);+ configureTestGitRepository(bare);+ git(bare, ['config', 'uploadpack.allowFilter', 'true']);+ git(bare, ['config', 'uploadpack.allowAnySHA1InWant', 'false']);+ git(bare, ['config', 'uploadpack.allowReachableSHA1InWant', 'false']);+ return;+ }+ git(source, ['push', '-q', '--force', bare, 'HEAD:refs/heads/main']);+ };++ return {+ port,+ bytesServed: () => bytes,+ resetBytesServed: () => {+ bytes = 0;+ },+ url: (name) => `git://127.0.0.1:${port}/${name}.git`,+ githubRedirectEnv: () => ({+ ...hermeticGitEnv,+ GIT_CONFIG_COUNT: '1',+ GIT_CONFIG_KEY_0: `url.git://127.0.0.1:${port}/.insteadOf`,+ GIT_CONFIG_VALUE_0: 'https://github.com/',+ }),+ publish,+ close: async () => {+ await new Promise<void>((resolve) => server.close(() => resolve()));+ for (const daemon of daemons) daemon.kill();+ await Promise.all([...daemons].map(untilClosed));+ rmSync(root, { recursive: true, force: true });+ },+ };+}