Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): catalog loads registry items a few at a time so a slow link no longer times them out (#5829)
Miguel Ángel95a686c23d79
diff --git a/packages/cli/src/commands/lambda/render-batch.test.ts b/packages/cli/src/commands/lambda/render-batch.test.tsindex 81fa926db7..c8acd1fc35 100644--- a/packages/cli/src/commands/lambda/render-batch.test.ts+++ b/packages/cli/src/commands/lambda/render-batch.test.ts@@ -6,9 +6,9 @@ import { CliRuntimeError } from "../../utils/commandResult.js";import {buildLambdaBatchRenderConfig,parseBatchFile,- runWithConcurrencyLimit,type RenderBatchArgs,} from "./render-batch.js";+import { runWithConcurrencyLimit } from "../../utils/concurrency.js";let tmpDir: string;diff --git a/packages/cli/src/commands/lambda/render-batch.ts b/packages/cli/src/commands/lambda/render-batch.tsindex 8aff5360e2..6296fb6801 100644--- a/packages/cli/src/commands/lambda/render-batch.ts+++ b/packages/cli/src/commands/lambda/render-batch.ts@@ -39,6 +39,7 @@ import {} from "../../utils/variables.js";import { normalizeErrorMessage } from "../../utils/errorMessage.js";import { warnOnDimensionMismatch } from "./_dimensions.js";+import { runWithConcurrencyLimit } from "../../utils/concurrency.js";import { requireStack } from "./state.js";// Dynamic-import the SDK so tsup keeps it out of the static-import head of@@ -451,38 +452,3 @@ export function parseBatchFile(path: string): Array<{ entry: BatchEntry; lineNum}return out;}--/**- * Run `worker` against every input with at most `limit` concurrent- * invocations. Preserves input order in the returned array; each output- * is positional with its input.- *- * Implementation: index-cursor + N concurrent producers each picking the- * next index until the input is drained. Uses `Promise.all` and- * propagates the first worker rejection — partial-failure isolation is- * the caller's responsibility (in this file, `startEntry` wraps the- * per-entry render in try/catch so a single failure surfaces in the- * manifest rather than aborting the batch).- *- * Exported so unit tests can pin the concurrency cap independently of- * the AWS-dependent fan-out path.- */-export async function runWithConcurrencyLimit<T, R>(- inputs: readonly T[],- limit: number,- worker: (input: T, index: number) => Promise<R>,-): Promise<R[]> {- if (limit < 1) throw new Error(`runWithConcurrencyLimit: limit must be ≥ 1, got ${limit}`);- const results = new Array<R>(inputs.length);- let cursor = 0;- const workerCount = Math.min(limit, inputs.length);- await Promise.all(- Array.from({ length: workerCount }, async () => {- while (cursor < inputs.length) {- const idx = cursor++;- results[idx] = await worker(inputs[idx]!, idx);- }- }),- );- return results;-}diff --git a/packages/cli/src/registry/remote.ts b/packages/cli/src/registry/remote.tsindex 4b6745b8f3..715cdf6f65 100644--- a/packages/cli/src/registry/remote.ts+++ b/packages/cli/src/registry/remote.ts@@ -151,6 +151,7 @@ export async function fetchItemManifest(name: string,type: ItemType,baseUrl: string = DEFAULT_REGISTRY_URL,+ options?: { cacheOnly?: boolean },): Promise<RegistryItem> {if (!validRegistryName(name) || !Object.hasOwn(ITEM_TYPE_DIRS, type))throw new Error("Invalid registry item name or type");@@ -159,7 +160,8 @@ export async function fetchItemManifest(const url = registryPathUrl(baseUrl, dir, name, "registry-item.json");const cacheFile = cachePath(baseUrl, `${dir}__${name}`);const cached = readCacheEntry(cacheFile, validate);- if (cached && isFresh(cached)) return cached.data;+ if (cached && (isFresh(cached) || options?.cacheOnly)) return cached.data;+ if (options?.cacheOnly) throw new Error("not fetched after an earlier registry timeout");try {const item = await fetchJson(url, validate);diff --git a/packages/cli/src/registry/resolver.test.ts b/packages/cli/src/registry/resolver.test.tsindex d8f6b2611e..f1340488c4 100644--- a/packages/cli/src/registry/resolver.test.ts+++ b/packages/cli/src/registry/resolver.test.ts@@ -132,6 +132,49 @@ describe("registry resolver", () => {});});+ describe("loadAllItems on a slow registry", () => {+ const entries = Array.from({ length: 100 }, (_, i) => ({+ name: `item-${i}`,+ type: "hyperframes:block" as const,+ }));+ const itemName = (url: string) => /\/blocks\/([^/]+)\/registry-item\.json$/.exec(url)?.[1];++ it("keeps at most 32 item fetches in flight", async () => {+ let inFlight = 0;+ let peak = 0;+ vi.stubGlobal(+ "fetch",+ vi.fn(async (url: string) => {+ inFlight += 1;+ peak = Math.max(peak, inFlight);+ await new Promise((resolve) => setImmediate(resolve));+ inFlight -= 1;+ return new Response(JSON.stringify(buildItem(itemName(url)!, "hyperframes:block")));+ }),+ );+ const items = await loadAllItems(entries, { baseUrl: uniqueBaseUrl() });+ expect(items).toHaveLength(100);+ expect(peak).toBe(32);+ });++ it("stops fetching after a timeout and warns once for every skipped item", async () => {+ const fetchMock = vi.fn(async () => {+ throw new DOMException("The operation was aborted due to timeout", "TimeoutError");+ });+ vi.stubGlobal("fetch", fetchMock);+ const warnings: string[] = [];+ const items = await loadAllItems(entries, {+ baseUrl: uniqueBaseUrl(),+ onWarn: (m) => warnings.push(m),+ });+ expect(items).toEqual([]);+ expect(fetchMock).toHaveBeenCalledTimes(32);+ expect(warnings).toEqual([+ 'skipped 100 items ("item-0", "item-1", "item-2", ...): TimeoutError: The operation was aborted due to timeout',+ ]);+ });+ });+describe("resolveItem", () => {it("returns the full manifest for a known item", async () => {const baseUrl = uniqueBaseUrl();diff --git a/packages/cli/src/registry/resolver.ts b/packages/cli/src/registry/resolver.tsindex a41502198c..27e2c7a5f8 100644--- a/packages/cli/src/registry/resolver.ts+++ b/packages/cli/src/registry/resolver.ts@@ -6,15 +6,16 @@import type { ItemType, RegistryItem, RegistryManifestEntry } from "@hyperframes/core";import { fetchItemManifest, fetchRegistryManifest, DEFAULT_REGISTRY_URL } from "./remote.js";+import { runWithConcurrencyLimit } from "../utils/concurrency.js";export interface ResolveOptions {baseUrl?: string;/** Bypass the 24h manifest cache and fetch fresh data from the registry. */skipCache?: boolean;/**- * Called once per item that fails to load inside `loadAllItems`. Defaults- * to writing a diagnostic line to stderr. Pass a quieter implementation- * when rendering structured output (clack prompts, JSON, etc.).+ * Called once with a summary when items fail to load inside `loadAllItems`.+ * Defaults to writing a diagnostic line to stderr. Pass a quieter+ * implementation when rendering structured output (clack prompts, JSON, etc.).*/onWarn?: (message: string) => void;}@@ -38,11 +39,13 @@ export async function listRegistryItems(return manifest.items.filter((item) => item.type === filter.type);}+// Hundreds of item manifests sharing one slow link all started their timeouts at once.+const ITEM_FETCH_CONCURRENCY = 32;+/**- * Load every item's full manifest in parallel. Used by the interactive init- * picker to populate titles/descriptions for all examples at once. Items that- * fail to load are skipped with a warning so one missing manifest doesn't- * break the picker.+ * Load every item's full manifest, a bounded number at a time. Items that fail+ * to load are skipped with one summary warning so one missing manifest doesn't+ * break the picker; after a timeout the rest come from cache only.*/export async function loadAllItems(entries: RegistryManifestEntry[],@@ -50,19 +53,25 @@ export async function loadAllItems(): Promise<RegistryItem[]> {const baseUrl = options.baseUrl ?? DEFAULT_REGISTRY_URL;const warn = options.onWarn ?? defaultWarn;- const results = await Promise.allSettled(- entries.map((e) => fetchItemManifest(e.name, e.type, baseUrl)),- );- const items: RegistryItem[] = [];- results.forEach((r, i) => {- if (r.status === "fulfilled") {- items.push(r.value);- } else {- const name = entries[i]?.name ?? "<unknown>";- warn(`skipped item "${name}": ${String(r.reason)}`);+ const skipped: string[] = [];+ let firstReason = "";+ let timedOut = false;+ const loaded = await runWithConcurrencyLimit(entries, ITEM_FETCH_CONCURRENCY, async (e) => {+ try {+ return await fetchItemManifest(e.name, e.type, baseUrl, { cacheOnly: timedOut });+ } catch (err) {+ if (err instanceof Error && err.name === "TimeoutError") timedOut = true;+ skipped.push(`"${e.name}"`);+ firstReason ||= String(err);+ return undefined;}});- return items;+ if (skipped.length === 1) warn(`skipped item ${skipped[0]}: ${firstReason}`);+ else if (skipped.length > 1)+ warn(+ `skipped ${skipped.length} items (${skipped.slice(0, 3).join(", ")}${skipped.length > 3 ? ", ..." : ""}): ${firstReason}`,+ );+ return loaded.filter((item): item is RegistryItem => item !== undefined);}/**diff --git a/packages/cli/src/utils/concurrency.ts b/packages/cli/src/utils/concurrency.tsnew file mode 100644index 0000000000..f6853b0817--- /dev/null+++ b/packages/cli/src/utils/concurrency.ts@@ -0,0 +1,21 @@+/** Runs `worker` over `inputs` with at most `limit` in flight. Results keep input order, and the+ * first rejection rejects the run. */+export async function runWithConcurrencyLimit<T, R>(+ inputs: readonly T[],+ limit: number,+ worker: (input: T, index: number) => Promise<R>,+): Promise<R[]> {+ if (limit < 1) throw new Error(`runWithConcurrencyLimit: limit must be ≥ 1, got ${limit}`);+ const results = new Array<R>(inputs.length);+ let cursor = 0;+ const workerCount = Math.min(limit, inputs.length);+ await Promise.all(+ Array.from({ length: workerCount }, async () => {+ while (cursor < inputs.length) {+ const idx = cursor++;+ results[idx] = await worker(inputs[idx]!, idx);+ }+ }),+ );+ return results;+}