Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(PRD-10109): Bound website skill downloads while streaming (#7861)

Andrew Mikofalvy2df018b3ade3
diff --git a/.changeset/website-skill-download-budgets.md b/.changeset/website-skill-download-budgets.md
new file mode 100644
index 000000000..1f83dac1a
--- /dev/null
+++ b/.changeset/website-skill-download-budgets.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Stop oversized website skill downloads early and cancel the remaining downloads when a bundle exceeds its size limits.
diff --git a/packages/core/src/skills-catalog/acquire/well-known.test.ts b/packages/core/src/skills-catalog/acquire/well-known.test.ts
new file mode 100644
index 000000000..0be721cf5
--- /dev/null
+++ b/packages/core/src/skills-catalog/acquire/well-known.test.ts
@@ -0,0 +1,259 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { createServer } from 'node:http';
+import { join } from 'node:path';
+import { setImmediate } from 'node:timers/promises';
+import { gzipSync } from 'node:zlib';
+import { describe, expect, test } from 'vitest';
+import { SKILL_IMPORT_MAX_FILE_BYTES, SKILL_IMPORT_MAX_TOTAL_BYTES } from '../import-limits.ts';
+import { fetchWellKnownSkill, readWellKnownIndex } from './well-known.ts';
+
+const origin = 'https://skills.example.com';
+const mib = 1024 * 1024;
+
+function streamedResponse(size: number, headers?: HeadersInit) {
+ let consumed = 0;
+ let cancelled = false;
+ const chunk = new Uint8Array(mib / 4);
+ const response = new Response(
+ new ReadableStream<Uint8Array>(
+ {
+ pull(controller) {
+ if (consumed === size) {
+ controller.close();
+ return;
+ }
+ const bytes = chunk.subarray(0, Math.min(chunk.length, size - consumed));
+ consumed += bytes.length;
+ controller.enqueue(bytes);
+ },
+ cancel() {
+ cancelled = true;
+ },
+ },
+ { highWaterMark: 0 },
+ ),
+ { headers },
+ );
+ return { response, consumed: () => consumed, cancelled: () => cancelled };
+}
+
+function indexFor(files: string[]) {
+ return {
+ basePath: '/.well-known/skills',
+ skills: [{ name: 'alpha', description: '', files }],
+ };
+}
+
+describe('website skill streaming budgets', () => {
+ test('cancels oversized indexes before consuming the whole body, including fallback', async () => {
+ const streams: ReturnType<typeof streamedResponse>[] = [];
+ const fetchImpl: typeof fetch = async () => {
+ const stream = streamedResponse(2 * mib);
+ streams.push(stream);
+ return stream.response;
+ };
+ await expect(readWellKnownIndex(origin, { fetchImpl })).rejects.toThrow('1048576 bytes');
+ expect(streams).toHaveLength(2);
+ for (const stream of streams) {
+ expect(stream.cancelled()).toBe(true);
+ expect(stream.consumed()).toBeLessThanOrEqual(mib + mib / 4);
+ }
+ });
+
+ test('accepts an index at the exact byte limit and preserves UTF-8 text', async () => {
+ const json = JSON.stringify({
+ skills: [{ name: 'alpha', description: 'café', files: ['SKILL.md'] }],
+ });
+ const raw = json + ' '.repeat(mib - Buffer.byteLength(json));
+ const fetchImpl: typeof fetch = async () => new Response(raw);
+ await expect(readWellKnownIndex(origin, { fetchImpl })).resolves.toEqual({
+ basePath: '/.well-known/agent-skills',
+ skills: [{ name: 'alpha', description: 'café', files: ['SKILL.md'] }],
+ });
+ });
+
+ test.each([undefined, { 'content-length': '1' }])(
+ 'cancels a file exceeding the actual byte cap with headers %j',
+ async (headers) => {
+ const stream = streamedResponse(2 * SKILL_IMPORT_MAX_FILE_BYTES, headers);
+ const fetchImpl: typeof fetch = async () => stream.response;
+ await expect(
+ fetchWellKnownSkill(
+ { kind: 'well-known', origin, skill: 'alpha' },
+ { fetchImpl, index: indexFor(['SKILL.md']) },
+ ),
+ ).rejects.toThrow(`file exceeds ${SKILL_IMPORT_MAX_FILE_BYTES} bytes`);
+ expect(stream.cancelled()).toBe(true);
+ expect(stream.consumed()).toBeLessThanOrEqual(SKILL_IMPORT_MAX_FILE_BYTES + mib / 4);
+ },
+ );
+
+ test('rejects an oversized declared file without reading its body', async () => {
+ const stream = streamedResponse(mib, {
+ 'content-length': String(SKILL_IMPORT_MAX_FILE_BYTES + 1),
+ });
+ const fetchImpl: typeof fetch = async () => stream.response;
+ await expect(
+ fetchWellKnownSkill(
+ { kind: 'well-known', origin, skill: 'alpha' },
+ { fetchImpl, index: indexFor(['SKILL.md']) },
+ ),
+ ).rejects.toThrow('file exceeds');
+ expect(stream.consumed()).toBe(0);
+ expect(stream.cancelled()).toBe(true);
+ });
+
+ test('enforces the bundle budget across partial concurrent files and aborts peers', async () => {
+ const files = ['SKILL.md', ...Array.from({ length: 9 }, (_, n) => `file-${n}.bin`)];
+ const streams: ReturnType<typeof streamedResponse>[] = [];
+ const signals: Array<AbortSignal | null | undefined> = [];
+ const fetchImpl: typeof fetch = async (_input, init) => {
+ signals.push(init?.signal);
+ const stream = streamedResponse(SKILL_IMPORT_MAX_FILE_BYTES);
+ streams.push(stream);
+ return stream.response;
+ };
+ await expect(
+ fetchWellKnownSkill(
+ { kind: 'well-known', origin, skill: 'alpha' },
+ { fetchImpl, index: indexFor(files) },
+ ),
+ ).rejects.toThrow(`bundle exceeds ${SKILL_IMPORT_MAX_TOTAL_BYTES} bytes`);
+ expect(streams).toHaveLength(8);
+ expect(signals.every((signal) => signal?.aborted === true)).toBe(true);
+ expect(streams.every((stream) => stream.cancelled())).toBe(true);
+ expect(streams.reduce((sum, stream) => sum + stream.consumed(), 0)).toBeLessThanOrEqual(
+ SKILL_IMPORT_MAX_TOTAL_BYTES + (8 * mib) / 4,
+ );
+ });
+
+ test('accepts exact per-file and bundle limits and cleans up binary files', async () => {
+ const files = ['SKILL.md', 'a.bin', 'b.bin', 'c.bin'];
+ const fetchImpl: typeof fetch = async () =>
+ streamedResponse(SKILL_IMPORT_MAX_FILE_BYTES).response;
+ const fetched = await fetchWellKnownSkill(
+ { kind: 'well-known', origin, skill: 'alpha' },
+ { fetchImpl, index: indexFor(files) },
+ );
+ try {
+ for (const file of files) {
+ expect(
+ readFileSync(join(fetched.dir, file)).equals(Buffer.alloc(SKILL_IMPORT_MAX_FILE_BYTES)),
+ ).toBe(true);
+ }
+ } finally {
+ fetched.cleanup();
+ }
+ expect(existsSync(fetched.dir)).toBe(false);
+ });
+
+ test('does not request queued files when peers finish after another worker aborts', async () => {
+ const files = ['SKILL.md', ...Array.from({ length: 9 }, (_, n) => `file-${n}.bin`)];
+ const requested: string[] = [];
+ const fetchImpl: typeof fetch = async (input, init) => {
+ requested.push(String(input));
+ if (String(input).endsWith('SKILL.md')) {
+ return new Response('oversized', {
+ headers: { 'content-length': String(SKILL_IMPORT_MAX_FILE_BYTES + 1) },
+ });
+ }
+ const signal = init?.signal;
+ if (!signal) throw new Error('Missing request signal');
+ if (!signal.aborted) {
+ await new Promise<void>((resolve) =>
+ signal.addEventListener('abort', () => resolve(), { once: true }),
+ );
+ }
+ return new Response('peer finished');
+ };
+ await expect(
+ fetchWellKnownSkill(
+ { kind: 'well-known', origin, skill: 'alpha' },
+ { fetchImpl, index: indexFor(files) },
+ ),
+ ).rejects.toThrow('file exceeds');
+ expect(requested).toHaveLength(8);
+ });
+
+ test('waits for aborted peers to settle before rejecting the bundle', async () => {
+ const { promise: aborted, resolve: observeAbort } = Promise.withResolvers<void>();
+ const { promise: cleanup, resolve: finishCleanup } = Promise.withResolvers<void>();
+ const fetchImpl: typeof fetch = async (input, init) => {
+ if (String(input).endsWith('SKILL.md')) {
+ return new Response('oversized', {
+ headers: { 'content-length': String(SKILL_IMPORT_MAX_FILE_BYTES + 1) },
+ });
+ }
+ const signal = init?.signal;
+ if (!signal) throw new Error('Missing request signal');
+ signal.addEventListener('abort', () => observeAbort(), { once: true });
+ await cleanup;
+ signal.throwIfAborted();
+ throw new Error('Peer was not aborted');
+ };
+ let settled = false;
+ const outcome = fetchWellKnownSkill(
+ { kind: 'well-known', origin, skill: 'alpha' },
+ { fetchImpl, index: indexFor(['SKILL.md', 'peer.bin']) },
+ ).then(
+ () => {
+ settled = true;
+ return null;
+ },
+ (error: unknown) => {
+ settled = true;
+ return error;
+ },
+ );
+ try {
+ await Promise.race([aborted, outcome]);
+ await setImmediate();
+ expect(settled).toBe(false);
+ } finally {
+ finishCleanup();
+ await outcome;
+ }
+ expect(await outcome).toHaveProperty('message', expect.stringContaining('file exceeds'));
+ });
+
+ test('rejects decoded overflow and aborts an in-flight native-fetch peer', async () => {
+ const signals: Array<AbortSignal | null | undefined> = [];
+ const { promise: peerClosed, resolve: closePeer } = Promise.withResolvers<void>();
+ const { promise: peerStarted, resolve: startOversized } = Promise.withResolvers<void>();
+ const compressed = gzipSync(Buffer.alloc(SKILL_IMPORT_MAX_FILE_BYTES + 1));
+ const server = createServer((request, response) => {
+ if (request.url?.endsWith('peer.bin')) {
+ response.on('close', closePeer);
+ response.write('waiting');
+ startOversized();
+ } else {
+ void peerStarted.then(() => {
+ response.writeHead(200, { 'content-encoding': 'gzip' });
+ response.end(compressed);
+ });
+ }
+ });
+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+ const address = server.address();
+ if (address === null || typeof address === 'string') throw new Error('Missing TCP address');
+ const fetchImpl: typeof fetch = (input, init) => {
+ signals.push(init?.signal);
+ return fetch(`http://127.0.0.1:${address.port}${new URL(String(input)).pathname}`, init);
+ };
+ try {
+ await expect(
+ fetchWellKnownSkill(
+ { kind: 'well-known', origin, skill: 'alpha' },
+ { fetchImpl, index: indexFor(['SKILL.md', 'peer.bin']) },
+ ),
+ ).rejects.toThrow(`file exceeds ${SKILL_IMPORT_MAX_FILE_BYTES} bytes`);
+ expect(signals.every((signal) => signal?.aborted === true)).toBe(true);
+ await peerClosed;
+ } finally {
+ server.closeAllConnections();
+ await new Promise<void>((resolve, reject) =>
+ server.close((error) => (error ? reject(error) : resolve())),
+ );
+ }
+ });
+});
diff --git a/packages/core/src/skills-catalog/acquire/well-known.ts b/packages/core/src/skills-catalog/acquire/well-known.ts
index 51035145f..e5b9b6640 100644
--- a/packages/core/src/skills-catalog/acquire/well-known.ts
+++ b/packages/core/src/skills-catalog/acquire/well-known.ts
@@ -135,6 +135,39 @@ function childUrl(origin: string, basePath: string, skill: string, file: string)
return new URL(`${basePath}/${path}`, origin).href;
}
+async function readBoundedBody(
+ response: Response,
+ maxBytes: number,
+ tooLarge: string,
+ consumeBytes?: (count: number) => void,
+): Promise<Uint8Array> {
+ const reader = response.body?.getReader();
+ if (!reader) return new Uint8Array();
+ const chunks: Uint8Array[] = [];
+ let byteLength = 0;
+ try {
+ const declaredLength = Number(response.headers.get('content-length'));
+ if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+ throw new SkillFetchError(tooLarge);
+ }
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ byteLength += value.byteLength;
+ if (byteLength > maxBytes) throw new SkillFetchError(tooLarge);
+ consumeBytes?.(value.byteLength);
+ chunks.push(value);
+ }
+ return Buffer.concat(chunks, byteLength);
+ } finally {
+ try {
+ await reader.cancel();
+ } finally {
+ reader.releaseLock();
+ }
+ }
+}
+
async function readIndex(
origin: string,
fetchImpl: typeof fetch,
@@ -151,11 +184,12 @@ async function readIndex(
failures.push(`${indexPath}: HTTP ${response.status}`);
continue;
}
- const raw = await response.text();
- if (Buffer.byteLength(raw) > MAX_INDEX_BYTES) {
- failures.push(`${indexPath}: index exceeds ${MAX_INDEX_BYTES} bytes`);
- continue;
- }
+ const bytes = await readBoundedBody(
+ response,
+ MAX_INDEX_BYTES,
+ `index exceeds ${MAX_INDEX_BYTES} bytes`,
+ );
+ const raw = new TextDecoder().decode(bytes);
let value: unknown;
try {
value = JSON.parse(raw);
@@ -214,46 +248,49 @@ export async function fetchWellKnownSkill(
const tmp = mkdtempSync(join(tmpdir(), 'ok-skill-site-'));
const skillDir = join(tmp, entry.name);
let totalBytes = 0;
+ const controller = new AbortController();
try {
const pending = [...entry.files];
const fetchOne = async (file: string): Promise<void> => {
const url = childUrl(origin, basePath, entry.name, file);
const response = await fetchWithinOrigin(fetchImpl, url, origin, {
- signal: AbortSignal.timeout(timeoutMs),
+ signal: AbortSignal.any([AbortSignal.timeout(timeoutMs), controller.signal]),
});
if (!response.ok) {
throw new SkillFetchError(`Website skill file returned HTTP ${response.status}: ${file}`);
}
- const declaredLength = Number(response.headers.get('content-length'));
- if (Number.isFinite(declaredLength) && declaredLength > SKILL_IMPORT_MAX_FILE_BYTES) {
- throw new SkillFetchError(
- `Website skill file exceeds ${SKILL_IMPORT_MAX_FILE_BYTES} bytes: ${file}`,
- );
- }
- const bytes = new Uint8Array(await response.arrayBuffer());
- if (bytes.byteLength > SKILL_IMPORT_MAX_FILE_BYTES) {
- throw new SkillFetchError(
- `Website skill file exceeds ${SKILL_IMPORT_MAX_FILE_BYTES} bytes: ${file}`,
- );
- }
- totalBytes += bytes.byteLength;
- if (totalBytes > SKILL_IMPORT_MAX_TOTAL_BYTES) {
- throw new SkillFetchError(
- `Website skill bundle exceeds ${SKILL_IMPORT_MAX_TOTAL_BYTES} bytes.`,
- );
- }
+ const bytes = await readBoundedBody(
+ response,
+ SKILL_IMPORT_MAX_FILE_BYTES,
+ `Website skill file exceeds ${SKILL_IMPORT_MAX_FILE_BYTES} bytes: ${file}`,
+ (count) => {
+ totalBytes += count;
+ if (totalBytes > SKILL_IMPORT_MAX_TOTAL_BYTES) {
+ throw new SkillFetchError(
+ `Website skill bundle exceeds ${SKILL_IMPORT_MAX_TOTAL_BYTES} bytes.`,
+ );
+ }
+ },
+ );
const destination = join(skillDir, ...file.split('/'));
mkdirSync(dirname(destination), { recursive: true });
writeFileSync(destination, bytes);
};
const worker = async (): Promise<void> => {
- for (let file = pending.shift(); file !== undefined; file = pending.shift()) {
- await fetchOne(file);
+ try {
+ for (let file = pending.shift(); file !== undefined; file = pending.shift()) {
+ controller.signal.throwIfAborted();
+ await fetchOne(file);
+ }
+ } catch (error) {
+ controller.abort(error);
+ throw error;
}
};
- await Promise.all(
+ await Promise.allSettled(
Array.from({ length: Math.min(SITE_FILE_CONCURRENCY, pending.length) }, worker),
);
+ controller.signal.throwIfAborted();
return { dir: skillDir, cleanup: () => rmSync(tmp, { recursive: true, force: true }) };
} catch (error) {
rmSync(tmp, { recursive: true, force: true });