Skip to content

Write HTML. Render video. Built for agents.

perf(packages): ship npm packages without source maps and guard their size in CI (#5403)

Miguel Ángelbb9e507e8408
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e586ed3f8d..420dffcd16 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -69,6 +69,7 @@ jobs:
code: ${{ (github.event_name == 'workflow_dispatch' && inputs.catalog_publish) && 'true' || steps.filter.outputs.code }}
docs_catalog: ${{ steps.filter.outputs.docs_catalog }}
cli: ${{ steps.filter.outputs.cli }}
+ cli_bundle: ${{ steps.filter.outputs.cli_bundle }}
skills: ${{ steps.filter.outputs.skills }}
catalog_index: ${{ steps.filter.outputs.catalog_index }}
codex_plugin: ${{ steps.filter.outputs.codex_plugin }}
@@ -94,6 +95,11 @@ jobs:
git rev-parse --verify -q HEAD^2 > /dev/null || { echo "::error::HEAD is not the pull request merge commit"; exit 1; }
sha="$(git rev-parse --verify HEAD^1)"
echo "sha=$sha" >> "$GITHUB_OUTPUT"
+ - name: Reject large binaries outside LFS
+ if: github.event_name == 'pull_request'
+ env:
+ BASE_SHA: ${{ steps.base.outputs.sha }}
+ run: sh scripts/check-large-files.sh --range "$BASE_SHA" HEAD
- uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4
id: filter
with:
@@ -131,6 +137,12 @@ jobs:
- "package.json"
- "bun.lock"
- ".github/workflows/ci.yml"
+ # Skills packages/cli/scripts/build-copy.mjs bundles into the CLI tarball (size budget).
+ cli_bundle:
+ - "skills/hyperframes/**"
+ - "skills/hyperframes-cli/**"
+ - "skills/gsap/**"
+ - "skills/media-use/**"
skills:
- "scripts/generate-captured-font-copies.mjs"
- "packages/cli/src/media-use/**"
@@ -243,14 +255,14 @@ jobs:
name: Build
needs: changes
# A skipped required check passes, so a broken change detection must fail it instead.
- if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
+ if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true' || needs.changes.outputs.cli_bundle == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Require change detection
if: needs.changes.result != 'success'
run: |
- echo "::error::Change detection did not succeed, so this required check cannot pass."
+ echo "::error::Detect changes did not succeed (it also rejects large binaries outside LFS); see its log. This required check cannot pass."
exit 1
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
@@ -437,7 +449,7 @@ jobs:
- name: Require change detection
if: needs.changes.result != 'success'
run: |
- echo "::error::Change detection did not succeed, so this required check cannot pass."
+ echo "::error::Detect changes did not succeed (it also rejects large binaries outside LFS); see its log. This required check cannot pass."
exit 1
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
@@ -543,7 +555,7 @@ jobs:
- name: Require change detection
if: needs.changes.result != 'success'
run: |
- echo "::error::Change detection did not succeed, so this required check cannot pass."
+ echo "::error::Detect changes did not succeed (it also rejects large binaries outside LFS); see its log. This required check cannot pass."
exit 1
- name: Require producer source tests
if: needs.changes.outputs.code == 'true' && needs.producer-source-tests.result != 'success'
@@ -772,7 +784,7 @@ jobs:
- name: Require change detection
if: needs.changes.result != 'success'
run: |
- echo "::error::Change detection did not succeed, so this required check cannot pass."
+ echo "::error::Detect changes did not succeed (it also rejects large binaries outside LFS); see its log. This required check cannot pass."
exit 1
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
diff --git a/lefthook.yml b/lefthook.yml
index 23d99d917c..e5b885e4a9 100644
--- a/lefthook.yml
+++ b/lefthook.yml
@@ -33,11 +33,8 @@ pre-commit:
# The prune keeps one audit base cache per checkout; the commit gets the audit's own status.
run: env -u GIT_DIR -u GIT_INDEX_FILE -u GIT_WORK_TREE sh -c 'bunx fallow audit --base origin/main --fail-on-issues; status=$?; bun scripts/prune-fallow-audit-caches.mjs origin/main; exit $status'
largefiles:
- # Reject large binaries committed straight into the git pack instead of
- # LFS. The script reads the staged set itself (via `git diff --cached`)
- # rather than taking lefthook's `{staged_files}` expansion, which would
- # split paths containing spaces. It skips files under the limit, anything
- # routed through LFS, and registry/ assets. Tune via HF_MAX_NONLFS_KB.
+ # Rejects staged binaries over the limit that are not LFS pointers; see
+ # the script header. CI runs the same script with --range.
run: ./scripts/check-large-files.sh
tracked-artifacts:
# Keep ignored dependency trees and platform metadata out of commits.
diff --git a/packages/aws-lambda/build.mjs b/packages/aws-lambda/build.mjs
index f6918a2480..88393f1ce4 100644
--- a/packages/aws-lambda/build.mjs
+++ b/packages/aws-lambda/build.mjs
@@ -29,7 +29,6 @@ const sharedOpts = {
target: "node22",
format: "esm",
minify: false,
- sourcemap: true,
external: [
"@aws-sdk/client-s3",
"@aws-sdk/client-sfn",
diff --git a/packages/aws-lambda/tsconfig.build.json b/packages/aws-lambda/tsconfig.build.json
index c93bd474e6..6da0256de2 100644
--- a/packages/aws-lambda/tsconfig.build.json
+++ b/packages/aws-lambda/tsconfig.build.json
@@ -5,8 +5,6 @@
"noEmit": false,
"outDir": "./dist",
"rootDir": "./src",
- "declaration": true,
- "declarationMap": true,
- "sourceMap": true
+ "declaration": true
}
}
diff --git a/packages/cli/scripts/build-copy.mjs b/packages/cli/scripts/build-copy.mjs
index 84026383fa..d7b0517b10 100644
--- a/packages/cli/scripts/build-copy.mjs
+++ b/packages/cli/scripts/build-copy.mjs
@@ -66,7 +66,11 @@ async function main() {
const studioDist = resolve(CLI_ROOT, "..", "studio", "dist");
await waitForStudioDist(studioDist);
- copyDirContents(studioDist, join(DIST, "studio"));
+ // Only the SPA studioServer.ts serves (index.html, /assets, /icons, /favicon.svg);
+ // the rest of studio/dist is the npm library build.
+ for (const entry of ["index.html", "assets", "icons", "favicon.svg"]) {
+ copyDir(join(studioDist, entry), join(DIST, "studio", entry));
+ }
for (const tmpl of ["blank", "from-file", "_shared"]) {
copyDir(join(CLI_ROOT, "src", "templates", tmpl), join(DIST, "templates", tmpl));
diff --git a/packages/core/package.json b/packages/core/package.json
index 4624f65b34..a24bb21e4b 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -10,8 +10,6 @@
"files": [
"dist/**/*.js",
"dist/**/*.d.ts",
- "dist/**/*.d.ts.map",
- "dist/**/*.js.map",
"!dist/hyperframe.runtime.mjs",
"docs",
"schemas",
diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json
index 20b8ada6c2..487985d2ff 100644
--- a/packages/core/tsconfig.json
+++ b/packages/core/tsconfig.json
@@ -8,8 +8,6 @@
"esModuleInterop": true,
"skipLibCheck": true,
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"outDir": "./dist",
"rootDir": "./src"
},
diff --git a/packages/engine/tsconfig.json b/packages/engine/tsconfig.json
index c3e695fc6e..ff972b1b62 100644
--- a/packages/engine/tsconfig.json
+++ b/packages/engine/tsconfig.json
@@ -10,8 +10,6 @@
"outDir": "./dist",
"rootDir": "./src",
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"types": ["@webgpu/types"]
},
"include": ["src/**/*"],
diff --git a/packages/gcp-cloud-run/build.mjs b/packages/gcp-cloud-run/build.mjs
index 16ab6c1fd9..7d4c024c0a 100644
--- a/packages/gcp-cloud-run/build.mjs
+++ b/packages/gcp-cloud-run/build.mjs
@@ -30,7 +30,6 @@ const sharedOpts = {
target: "node22",
format: "esm",
minify: false,
- sourcemap: true,
external: [
"@google-cloud/storage",
"@google-cloud/workflows",
diff --git a/packages/gcp-cloud-run/tsconfig.build.json b/packages/gcp-cloud-run/tsconfig.build.json
index c93bd474e6..6da0256de2 100644
--- a/packages/gcp-cloud-run/tsconfig.build.json
+++ b/packages/gcp-cloud-run/tsconfig.build.json
@@ -5,8 +5,6 @@
"noEmit": false,
"outDir": "./dist",
"rootDir": "./src",
- "declaration": true,
- "declarationMap": true,
- "sourceMap": true
+ "declaration": true
}
}
diff --git a/packages/lint/tsconfig.json b/packages/lint/tsconfig.json
index 96f38bce00..5b48ae9a5d 100644
--- a/packages/lint/tsconfig.json
+++ b/packages/lint/tsconfig.json
@@ -8,8 +8,6 @@
"esModuleInterop": true,
"skipLibCheck": true,
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"outDir": "./dist",
"rootDir": "./src"
},
diff --git a/packages/lint/tsup.config.ts b/packages/lint/tsup.config.ts
index e4948a4d3c..93e18ed988 100644
--- a/packages/lint/tsup.config.ts
+++ b/packages/lint/tsup.config.ts
@@ -9,7 +9,6 @@ export default defineConfig([
platform: "node",
bundle: true,
splitting: false,
- sourcemap: true,
clean: true,
dts: true,
},
@@ -24,7 +23,6 @@ export default defineConfig([
platform: "browser",
bundle: true,
splitting: false,
- sourcemap: true,
clean: false,
dts: true,
},
diff --git a/packages/parsers/tsconfig.json b/packages/parsers/tsconfig.json
index 572403d6d7..ad4170aaf5 100644
--- a/packages/parsers/tsconfig.json
+++ b/packages/parsers/tsconfig.json
@@ -8,8 +8,6 @@
"esModuleInterop": true,
"skipLibCheck": true,
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"outDir": "./dist",
"rootDir": "./src"
},
diff --git a/packages/parsers/tsup.config.ts b/packages/parsers/tsup.config.ts
index c286cb09a8..ba477ed713 100644
--- a/packages/parsers/tsup.config.ts
+++ b/packages/parsers/tsup.config.ts
@@ -30,7 +30,6 @@ export default defineConfig({
platform: "node",
bundle: true,
splitting: false,
- sourcemap: true,
clean: true,
dts: true,
});
diff --git a/packages/player/tsconfig.json b/packages/player/tsconfig.json
index d1972c9fd1..3c47fb18d7 100644
--- a/packages/player/tsconfig.json
+++ b/packages/player/tsconfig.json
@@ -5,8 +5,6 @@
"moduleResolution": "bundler",
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
diff --git a/packages/player/tsup.config.ts b/packages/player/tsup.config.ts
index 8b6f443556..033147592f 100644
--- a/packages/player/tsup.config.ts
+++ b/packages/player/tsup.config.ts
@@ -12,7 +12,6 @@ export default defineConfig({
dts: true,
clean: true,
minify: true,
- sourcemap: true,
define: {
__HYPERFRAMES_RUNTIME_CDN_URL__: JSON.stringify(
`https://cdn.jsdelivr.net/npm/@hyperframes/core@${packageVersion}/dist/hyperframe.runtime.iife.js`,
diff --git a/packages/sdk/tsconfig.json b/packages/sdk/tsconfig.json
index 96f38bce00..5b48ae9a5d 100644
--- a/packages/sdk/tsconfig.json
+++ b/packages/sdk/tsconfig.json
@@ -8,8 +8,6 @@
"esModuleInterop": true,
"skipLibCheck": true,
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"outDir": "./dist",
"rootDir": "./src"
},
diff --git a/packages/shader-transitions/tsconfig.json b/packages/shader-transitions/tsconfig.json
index d1972c9fd1..3c47fb18d7 100644
--- a/packages/shader-transitions/tsconfig.json
+++ b/packages/shader-transitions/tsconfig.json
@@ -5,8 +5,6 @@
"moduleResolution": "bundler",
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
diff --git a/packages/shader-transitions/tsup.config.ts b/packages/shader-transitions/tsup.config.ts
index c4c7c3b6e5..593667575d 100644
--- a/packages/shader-transitions/tsup.config.ts
+++ b/packages/shader-transitions/tsup.config.ts
@@ -8,5 +8,4 @@ export default defineConfig({
dts: true,
clean: true,
minify: true,
- sourcemap: true,
});
diff --git a/packages/studio-server/tsconfig.json b/packages/studio-server/tsconfig.json
index 96f38bce00..5b48ae9a5d 100644
--- a/packages/studio-server/tsconfig.json
+++ b/packages/studio-server/tsconfig.json
@@ -8,8 +8,6 @@
"esModuleInterop": true,
"skipLibCheck": true,
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"outDir": "./dist",
"rootDir": "./src"
},
diff --git a/packages/studio-server/tsup.config.ts b/packages/studio-server/tsup.config.ts
index 1f828c16cc..bdad8ff3f1 100644
--- a/packages/studio-server/tsup.config.ts
+++ b/packages/studio-server/tsup.config.ts
@@ -28,7 +28,6 @@ export default defineConfig({
// With splitting off, each entry inlined its own copy and a pre-warm from
// media-proxy-preview couldn't dedupe against a route's proxy-transcoder.
splitting: true,
- sourcemap: true,
clean: true,
dts: true,
});
diff --git a/packages/studio/tsconfig.json b/packages/studio/tsconfig.json
index 8af20cf3f5..ee7066da64 100644
--- a/packages/studio/tsconfig.json
+++ b/packages/studio/tsconfig.json
@@ -12,8 +12,6 @@
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"declaration": true,
- "declarationMap": true,
- "sourceMap": true,
"outDir": "dist",
"rootDir": "..",
"types": ["vite/client", "node"],
diff --git a/packages/studio/tsup.config.ts b/packages/studio/tsup.config.ts
index 05fd4a1926..ea45a7745d 100644
--- a/packages/studio/tsup.config.ts
+++ b/packages/studio/tsup.config.ts
@@ -10,7 +10,6 @@ export default defineConfig({
},
format: ["esm"],
dts: true,
- sourcemap: true,
clean: false,
splitting: true,
external: [
diff --git a/scripts/check-large-files.sh b/scripts/check-large-files.sh
index 5f562ce10c..cf7361189f 100755
--- a/scripts/check-large-files.sh
+++ b/scripts/check-large-files.sh
@@ -14,43 +14,105 @@
# Usage:
# check-large-files.sh # default: check the staged file set
# check-large-files.sh <file> [<file>] # explicit files (handy for testing)
+# check-large-files.sh --range <base> <head> # files a commit range adds or changes (CI)
+#
+# The staged and range modes size the blob git stores, not the file on disk, so
+# a file matching an LFS pattern but added without git-lfs installed is caught.
#
# We read the staged set ourselves rather than taking lefthook's {staged_files}
-# expansion: that expands to a bare space-separated string, which splits paths
-# containing spaces into separate args. `git diff --cached` + a line-based read
-# keeps whole paths intact (only a literal newline in a filename would break it,
-# which git quotes/escapes anyway).
+# expansion, which splits paths containing spaces into separate args.
set -u
MAX_KB="${HF_MAX_NONLFS_KB:-500}"
-# Emit the list of paths to check, one per line.
+# staged (default): the index; range: what <head> adds over <base>; files: paths on disk.
+MODE=staged
+RANGE_BASE=""
+RANGE_HEAD=""
+if [ "${1:-}" = "--range" ]; then
+ [ "$#" -eq 3 ] || { echo "usage: $0 --range <base> <head>" >&2; exit 2; }
+ MODE=range
+ RANGE_BASE="$2"
+ RANGE_HEAD="$3"
+ shift 3
+elif [ "$#" -gt 0 ]; then
+ MODE=files
+fi
+# Git modes name paths from the repository root; files mode keeps the caller's paths.
+if [ "$MODE" != files ]; then
+ cd "$(git rev-parse --show-toplevel)" || { echo "ERROR: not inside a git repository." >&2; exit 2; }
+fi
+
+# Emit a "<raw diff header>" record then a path record per entry, NUL-separated and
+# unquoted. The header carries the new mode and blob id, so blobs are read by id,
+# never by a path git could parse as revision syntax. Renames count as additions.
list_files() {
- if [ "$#" -gt 0 ]; then
- printf '%s\n' "$@"
+ case "$MODE" in
+ range) git diff -z --raw --no-abbrev --no-renames --diff-filter=ACMT "$RANGE_BASE" "$RANGE_HEAD" ;;
+ staged) git diff -z --raw --no-abbrev --no-renames --diff-filter=ACMT --cached ;;
+ files) for f in "$@"; do printf 'file\0%s\0' "$f"; done ;;
+ esac
+}
+
+# Blob ids under registry/ in the tree being checked, listed on first need.
+list_registry_ids() {
+ [ -s "$regids" ] && return
+ if [ "$MODE" = range ]; then
+ git ls-tree -r "$RANGE_HEAD" -- registry | awk '{print $3}'
else
- # Added/Copied/Modified/Renamed staged paths (skip Deleted — nothing to size).
- git diff --cached --name-only --diff-filter=ACMR
- fi
+ git ls-files -s -- registry | awk '{print $2}'
+ fi > "$regids"
+}
+
+read_bytes() {
+ if [ "$MODE" = files ]; then cat -- "$1"; else git cat-file blob "$2"; fi
}
violations="$(mktemp)"
-trap 'rm -f "$violations"' EXIT INT TERM
+errors="$(mktemp)"
+paths="$(mktemp)"
+blob="$(mktemp)"
+regids="$(mktemp)"
+trap 'rm -f "$violations" "$errors" "$paths" "$blob" "$regids"' EXIT INT TERM
-list_files "$@" | while IFS= read -r f; do
+# A git error (bad ref, not a repo) must fail the check, not pass an empty list.
+list_files "$@" > "$paths" || { echo "ERROR: could not list the files to check." >&2; exit 2; }
+# Records are NUL-separated, so a newline byte can only come from a filename; sh cannot
+# keep such a name whole, so refuse it rather than check the wrong files.
+if [ "$(tr -cd '\n' < "$paths" | wc -c)" -gt 0 ]; then
+ echo "ERROR: a file name contains a newline; rename it." >&2
+ exit 2
+fi
+
+tr '\0' '\n' < "$paths" | while IFS= read -r header && IFS= read -r f; do
[ -n "$f" ] || continue
- # Skip symlinks: `wc -c` would measure the link *target's* bytes, so a symlink
- # to a large LFS-tracked asset could be flagged even though the real blob is a
- # tiny pointer. Symlinks themselves are never the bloat we're hunting.
- [ -L "$f" ] && continue
- [ -f "$f" ] || continue
+ id=""
+ if [ "$MODE" = files ]; then
+ # Skip symlinks: `wc -c` would measure the link *target's* bytes, so a symlink
+ # to a large LFS-tracked asset could be flagged even though the real blob is a
+ # tiny pointer.
+ [ -L "$f" ] && continue
+ [ -f "$f" ] || continue
+ else
+ # ":<old mode> <new mode> <old id> <new id> <status>"
+ set -- $header
+ # A submodule (160000) records another repository's commit: nothing stored here.
+ [ "$2" = 160000 ] && continue
+ id="$4"
+ fi
# registry/ intentionally ships raw binary assets (block backgrounds, avatar
# PNGs, .glb models, audio) so installed blocks stay portable without an LFS
# round-trip. Those are the product, not accidental bloat — skip them here.
case "$f" in registry/*) continue ;; esac
+ # docs/public/catalog/ mirrors those assets for the docs site: exempt only an exact copy.
+ case "$f" in docs/public/catalog/*)
+ [ -n "$id" ] || id="$(git hash-object -- "$f")"
+ list_registry_ids
+ grep -qxF "$id" "$regids" && continue ;;
+ esac
# Text is exempt, whatever its size, because the cost this hook exists to stop
# is a binary one. Git delta-compresses text, so a file that grows by a few KB
@@ -63,30 +125,48 @@ list_files "$@" | while IFS= read -r f; do
# `grep -I` treats a file containing NUL bytes as binary, the same heuristic
# git uses to print "Binary files differ". A generated blob of text is still
# caught by review, not here.
- grep -qI . "$f" 2>/dev/null && continue
+ # Read once into a file so a failed read is an error, never an empty pass.
+ read_bytes "$f" "$id" > "$blob" || { printf '%s\n' "$f" >> "$errors"; continue; }
+ grep -qI . "$blob" && continue
- bytes="$(wc -c < "$f" | tr -d ' ')"
+ bytes="$(wc -c < "$blob" | tr -d ' ')"
# Ceiling division: a sub-1024-byte file must report >=1 KB, never 0, so it
# can't slip past a strict threshold (e.g. HF_MAX_NONLFS_KB=0). Plain
# `bytes / 1024` would round a 512-byte binary down to 0 and pass it.
kb=$(( (bytes + 1023) / 1024 ))
[ "$kb" -le "$MAX_KB" ] && continue
- # Is this path routed through LFS? `git check-attr` reads .gitattributes.
+ # A stored blob over the limit is raw bytes even when .gitattributes routes
+ # the path through LFS (git-lfs was not installed). A file on disk under an
+ # LFS pattern is the smudged copy of a pointer, so it passes.
filter="$(git check-attr filter -- "$f" | sed 's/.*: //')"
- [ "$filter" = "lfs" ] && continue
+ note=""
+ if [ "$filter" = "lfs" ]; then
+ [ "$MODE" = files ] && continue
+ note=" — matches an LFS pattern but was stored raw; run \`git lfs install\` and re-add it"
+ fi
- printf '%s\t%s\n' "$kb" "$f" >> "$violations"
+ printf '%s\t%s\n' "$kb" "$f$note" >> "$violations"
done
+if [ -s "$errors" ]; then
+ echo "ERROR: could not read these files from git:" >&2
+ sed 's/^/ • /' "$errors" >&2
+ exit 2
+fi
+
# `while` ran in a pipeline subshell, so it couldn't set a parent-shell flag —
# the violations file is the durable signal.
if [ -s "$violations" ]; then
echo "ERROR: large binaries are being committed to git instead of LFS." >&2
- echo " (limit: ${MAX_KB} KB — override per-commit with HF_MAX_NONLFS_KB)" >&2
+ if [ "$MODE" = range ]; then
+ echo " (limit: ${MAX_KB} KB)" >&2
+ else
+ echo " (limit: ${MAX_KB} KB — override per-commit with HF_MAX_NONLFS_KB)" >&2
+ fi
echo >&2
while IFS=' ' read -r kb f; do
- echo " • ${f} (${kb} KB)" >&2
+ printf ' • %s (%s KB)\n' "$f" "$kb" >&2
done < "$violations"
echo >&2
echo "Fix: add an LFS pattern for it in .gitattributes, e.g." >&2
diff --git a/scripts/check-large-files.test.mjs b/scripts/check-large-files.test.mjs
index 237dbae7b1..dce317b947 100644
--- a/scripts/check-large-files.test.mjs
+++ b/scripts/check-large-files.test.mjs
@@ -1,13 +1,64 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import {
+ chmodSync,
+ mkdirSync,
+ mkdtempSync,
+ rmSync,
+ symlinkSync,
+ truncateSync,
+ writeFileSync,
+} from "node:fs";
import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
import { describe, it } from "node:test";
const SCRIPT = join(import.meta.dirname, "check-large-files.sh");
const OVER_LIMIT_BYTES = 2 * 1024 * 1024;
+/** Commit files into a throwaway repo with no global or system git config (so no LFS filter). */
+function withRepo(run) {
+ const dir = mkdtempSync(join(tmpdir(), "hf-largefiles-repo-"));
+ const env = { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" };
+ const git = (...args) =>
+ spawnSync("git", ["-c", "user.name=t", "-c", "user.email=t@t", ...args], {
+ cwd: dir,
+ env,
+ encoding: "utf-8",
+ });
+ const write = (files) => {
+ for (const [name, contents] of Object.entries(files)) {
+ mkdirSync(dirname(join(dir, name)), { recursive: true });
+ writeFileSync(join(dir, name), contents);
+ }
+ };
+ const commit = (files) => {
+ write(files);
+ git("add", "-A");
+ git("commit", "-q", "-m", "c");
+ return git("rev-parse", "HEAD").stdout.trim();
+ };
+ // `files` null checks the index as it stands (git add -A would drop a gitlink with no checkout).
+ const staged = (files) => {
+ if (files) {
+ write(files);
+ git("add", "-A");
+ }
+ const result = spawnSync(SCRIPT, [], { cwd: dir, env, encoding: "utf-8" });
+ return { ok: result.status === 0, stderr: result.stderr ?? "" };
+ };
+ const range = (base, head) => {
+ const result = spawnSync(SCRIPT, ["--range", base, head], { cwd: dir, env, encoding: "utf-8" });
+ return { ok: result.status === 0, stderr: result.stderr ?? "" };
+ };
+ try {
+ git("init", "-q");
+ run({ commit, staged, range, dir, git });
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
+
/** Run the checker over explicit paths. Exit 0 means "nothing to complain about". */
function check(...paths) {
const result = spawnSync(SCRIPT, paths, { encoding: "utf-8" });
@@ -29,6 +80,8 @@ function withFiles(files, run) {
}
const bigBinary = Buffer.alloc(OVER_LIMIT_BYTES);
+/** A NUL-filled (so binary) buffer that differs from bigBinary by its first byte. */
+const binaryVariant = (first) => Buffer.concat([Buffer.from([first]), bigBinary.subarray(1)]);
const bigText = "the quick brown fox jumps over the lazy dog\n".repeat(50_000);
describe("check-large-files", () => {
@@ -65,4 +118,164 @@ describe("check-large-files", () => {
assert.match(stderr, /b\.bin/);
});
});
+
+ it("--range rejects a binary the range adds, and ignores text and untouched files", () => {
+ withRepo(({ commit, range }) => {
+ const base = commit({ "old.bin": bigBinary });
+ const head = commit({ "big.bin": bigBinary, "big.txt": bigText });
+ const { ok, stderr } = range(base, head);
+ assert.equal(ok, false);
+ assert.match(stderr, /big\.bin/);
+ assert.doesNotMatch(stderr, /big\.txt|old\.bin/);
+ });
+ });
+
+ it("--range rejects a raw blob even when .gitattributes routes it through LFS", () => {
+ withRepo(({ commit, range }) => {
+ const base = commit({ ".gitattributes": "*.mp4 filter=lfs diff=lfs merge=lfs -text\n" });
+ const head = commit({ "clip.mp4": bigBinary });
+ const { ok, stderr } = range(base, head);
+ assert.equal(ok, false);
+ assert.match(stderr, /clip\.mp4 .*stored raw/);
+ });
+ });
+
+ it("--range rejects binaries whose names git would quote", () => {
+ withRepo(({ commit, range }) => {
+ const base = commit({ "seed.txt": "x" });
+ const head = commit({
+ "café.bin": bigBinary,
+ 'quo"te.bin': bigBinary,
+ "tab\tname.bin": bigBinary,
+ });
+ const { ok, stderr } = range(base, head);
+ assert.equal(ok, false);
+ assert.match(stderr, /café\.bin/);
+ assert.match(stderr, /quo"te\.bin/);
+ assert.match(stderr, /tab\tname\.bin/);
+ });
+ });
+
+ it("--range fails on a ref git cannot resolve instead of passing", () => {
+ withRepo(({ commit, range }) => {
+ const head = commit({ "seed.txt": "x" });
+ assert.equal(range("no-such-ref", head).ok, false);
+ });
+ });
+
+ it("--range refuses a file name with a newline instead of checking the wrong files", () => {
+ withRepo(({ commit, range }) => {
+ const base = commit({ left: "l", right: "r" });
+ const head = commit({ "left\nright": bigBinary });
+ const { ok, stderr } = range(base, head);
+ assert.equal(ok, false);
+ assert.match(stderr, /newline/);
+ });
+ });
+
+ it("--range rejects a symlink replaced by a large binary", () => {
+ withRepo(({ commit, range, dir }) => {
+ writeFileSync(join(dir, "target"), "t");
+ symlinkSync("target", join(dir, "clip.bin"));
+ const base = commit({});
+ rmSync(join(dir, "clip.bin"));
+ const head = commit({ "clip.bin": bigBinary });
+ const { ok, stderr } = range(base, head);
+ assert.equal(ok, false);
+ assert.match(stderr, /clip\.bin/);
+ });
+ });
+
+ it("--range fails when a blob cannot be read", () => {
+ withRepo(({ commit, range, dir, git }) => {
+ const base = commit({ "seed.txt": "x" });
+ const head = commit({ "big.bin": bigBinary });
+ const id = git("rev-parse", `${head}:big.bin`).stdout.trim();
+ const object = join(dir, ".git", "objects", id.slice(0, 2), id.slice(2));
+ chmodSync(object, 0o644);
+ truncateSync(object, 64);
+ assert.equal(range(base, head).ok, false);
+ });
+ });
+
+ it("staged mode rejects a large binary and passes text", () => {
+ withRepo(({ commit, staged }) => {
+ commit({ "seed.txt": "x" });
+ const { ok, stderr } = staged({ "big.bin": bigBinary, "big.txt": bigText });
+ assert.equal(ok, false);
+ assert.match(stderr, /big\.bin/);
+ assert.doesNotMatch(stderr, /big\.txt/);
+ });
+ });
+
+ it("staged mode sizes a file named like revision syntax, not the file it aliases", () => {
+ withRepo(({ commit, staged }) => {
+ commit({ "clip.bin": "small" });
+ const { ok, stderr } = staged({ "0:clip.bin": bigBinary });
+ assert.equal(ok, false);
+ assert.match(stderr, /0:clip\.bin/);
+ });
+ });
+
+ it("staged mode checks modified and type-changed files", () => {
+ withRepo(({ commit, staged, dir }) => {
+ writeFileSync(join(dir, "target"), "t");
+ symlinkSync("target", join(dir, "link.bin"));
+ commit({ "clip.bin": "small" });
+ rmSync(join(dir, "link.bin"));
+ const { ok, stderr } = staged({ "clip.bin": bigBinary, "link.bin": bigBinary });
+ assert.equal(ok, false);
+ assert.match(stderr, /clip\.bin/);
+ assert.match(stderr, /link\.bin/);
+ });
+ });
+
+ it("skips a submodule whose commit is not in this repository", () => {
+ withRepo(({ commit, staged, range, git }) => {
+ const base = commit({ "seed.txt": "x" });
+ const child = "1234567890abcdef1234567890abcdef12345678";
+ git("update-index", "--add", "--cacheinfo", `160000,${child},sub`);
+ assert.equal(staged(null).ok, true);
+ git("commit", "-q", "-m", "sub");
+ const head = git("rev-parse", "HEAD").stdout.trim();
+ assert.equal(range(base, head).ok, true);
+ });
+ });
+
+ it("staged mode exempts a new registry asset staged with its catalog copy", () => {
+ withRepo(({ commit, staged }) => {
+ commit({ "seed.txt": "x" });
+ const { ok, stderr } = staged({
+ "registry/a/bg.png": bigBinary,
+ "docs/public/catalog/a/bg.png": bigBinary,
+ "docs/public/catalog/c/random.mp4": binaryVariant(2),
+ });
+ assert.equal(ok, false);
+ assert.match(stderr, /catalog\/c\/random\.mp4/);
+ assert.doesNotMatch(stderr, /registry\/|catalog\/a\/bg\.png/);
+ });
+ });
+
+ it("exempts registry/ binaries and catalog copies of them, but not other catalog files", () => {
+ withRepo(({ commit, range }) => {
+ const base = commit({ "registry/a/bg.png": bigBinary });
+ const head = commit({
+ "registry/b/new.png": binaryVariant(1),
+ "docs/public/catalog/a/bg.png": bigBinary,
+ "docs/public/catalog/c/random.mp4": binaryVariant(2),
+ });
+ const { ok, stderr } = range(base, head);
+ assert.equal(ok, false);
+ assert.match(stderr, /docs\/public\/catalog\/c\/random\.mp4/);
+ assert.doesNotMatch(stderr, /registry\/|catalog\/a\/bg\.png|HF_MAX_NONLFS_KB/);
+ });
+ });
+
+ it("prints file names with backslashes intact", () => {
+ withRepo(({ commit, range }) => {
+ const base = commit({ "seed.txt": "x" });
+ const head = commit({ "a\\cb.bin": bigBinary });
+ assert.match(range(base, head).stderr, /a\\cb\.bin/);
+ });
+ });
});
diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json
new file mode 100644
index 0000000000..ba28f903b7
--- /dev/null
+++ b/scripts/package-size-budgets.json
@@ -0,0 +1,15 @@
+{
+ "@hyperframes/aws-lambda": { "packed": 63198, "unpacked": 230499 },
+ "@hyperframes/cli": { "packed": 7517641, "unpacked": 22401400 },
+ "@hyperframes/core": { "packed": 1636442, "unpacked": 3903211 },
+ "@hyperframes/engine": { "packed": 428347, "unpacked": 1436293 },
+ "@hyperframes/gcp-cloud-run": { "packed": 49412, "unpacked": 192672 },
+ "@hyperframes/lint": { "packed": 168745, "unpacked": 654607 },
+ "@hyperframes/parsers": { "packed": 190591, "unpacked": 831243 },
+ "@hyperframes/player": { "packed": 157145, "unpacked": 541601 },
+ "@hyperframes/producer": { "packed": 729380, "unpacked": 2672493 },
+ "@hyperframes/sdk": { "packed": 90662, "unpacked": 325069 },
+ "@hyperframes/shader-transitions": { "packed": 212864, "unpacked": 825937 },
+ "@hyperframes/studio": { "packed": 5973698, "unpacked": 23915921 },
+ "@hyperframes/studio-server": { "packed": 134964, "unpacked": 528620 }
+}
diff --git a/scripts/verify-packed-manifests.mjs b/scripts/verify-packed-manifests.mjs
index 83e72a6748..1a20f43f46 100644
--- a/scripts/verify-packed-manifests.mjs
+++ b/scripts/verify-packed-manifests.mjs
@@ -11,6 +11,7 @@ import {
readdirSync,
readFileSync,
rmSync,
+ statSync,
writeFileSync,
} from "node:fs";
import { extname, join, posix } from "node:path";
@@ -22,6 +23,11 @@ const PACKAGES_DIR = join(ROOT, "packages");
const DEP_FIELDS = ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"];
const RUNTIME_IMPORT_EXTENSIONS = new Set([".js", ".mjs", ".cjs", ".json", ".wasm", ".node"]);
const PACKED_JAVASCRIPT_FILE_PATTERN = /\.(?:js|mjs|cjs)$/;
+const SIZE_BUDGETS_FILE = "scripts/package-size-budgets.json";
+// A budget sits 5% above the measured size; a package that shrinks 15% below
+// its budget fails until the budget follows it down, so budgets only ratchet down.
+const SIZE_BUDGET_HEADROOM = 1.05;
+const SIZE_BUDGET_RATCHET = 0.85;
function listWorkspacePackageDirs() {
return readdirSync(PACKAGES_DIR)
@@ -467,6 +473,51 @@ function verifyPackedConsumer(packDir, packedWorkspaces) {
);
}
+function measurePackedSize(filename) {
+ return {
+ packed: statSync(filename).size,
+ unpacked: execFileSync("tar", ["-xOf", filename], { cwd: ROOT, maxBuffer: 1 << 30 }).length,
+ };
+}
+
+export function listPackageSizeIssues(name, sizes, budget) {
+ const suggest = (bytes) => Math.ceil(bytes * SIZE_BUDGET_HEADROOM);
+ if (!budget) {
+ return [
+ `${name} has no size budget: add "${name}": { "packed": ${suggest(sizes.packed)}, "unpacked": ${suggest(sizes.unpacked)} } to ${SIZE_BUDGETS_FILE}`,
+ ];
+ }
+ return ["packed", "unpacked"].flatMap((kind) => {
+ if (!Number.isFinite(budget[kind]) || budget[kind] <= 0) {
+ return [`${name} has an invalid ${kind} budget in ${SIZE_BUDGETS_FILE}: use a byte count.`];
+ }
+ if (sizes[kind] > budget[kind]) {
+ return [
+ `${name} ${kind} size ${sizes[kind]} bytes is over its budget of ${budget[kind]}. Keep source maps, tests and unused assets out of the tarball; raise the budget in ${SIZE_BUDGETS_FILE} only with the reason in the PR.`,
+ ];
+ }
+ if (sizes[kind] < budget[kind] * SIZE_BUDGET_RATCHET) {
+ return [
+ `${name} ${kind} size ${sizes[kind]} bytes is well under its budget of ${budget[kind]}: lower it to ${suggest(sizes[kind])} in ${SIZE_BUDGETS_FILE} so the saving stays.`,
+ ];
+ }
+ return [];
+ });
+}
+
+function verifyPackageSizes(packedWorkspaces) {
+ const budgets = JSON.parse(readFileSync(join(ROOT, SIZE_BUDGETS_FILE), "utf8"));
+ const issues = packedWorkspaces.flatMap(({ filename, packedPackage }) =>
+ listPackageSizeIssues(
+ packedPackage.name,
+ measurePackedSize(filename),
+ budgets[packedPackage.name],
+ ),
+ );
+ if (issues.length > 0) throw new Error(`Package size budgets:\n${issues.join("\n")}`);
+ console.log(`Verified ${packedWorkspaces.length} packages are within their size budgets.`);
+}
+
function packAndVerifyWorkspace(workspace, packDir) {
const sourcePackageJson = readWorkspacePackage(workspace);
if (sourcePackageJson.private) return null;
@@ -486,6 +537,7 @@ function main() {
.map((workspace) => packAndVerifyWorkspace(workspace, packDir))
.filter(Boolean);
verifyPackedConsumer(packDir, packedWorkspaces);
+ verifyPackageSizes(packedWorkspaces);
} finally {
rmSync(packDir, { force: true, recursive: true });
}
diff --git a/scripts/verify-packed-manifests.test.mjs b/scripts/verify-packed-manifests.test.mjs
index e9e2c4812a..bebae1ea47 100644
--- a/scripts/verify-packed-manifests.test.mjs
+++ b/scripts/verify-packed-manifests.test.mjs
@@ -6,6 +6,7 @@ import { dirname, join } from "node:path";
import { describe, it } from "node:test";
import {
listPackedExportContracts,
+ listPackageSizeIssues,
listPackedJavaScriptImportIssues,
packageExportSpecifier,
renderBrowserConsumer,
@@ -13,6 +14,45 @@ import {
} from "./verify-packed-manifests.mjs";
describe("packed manifest verifier", () => {
+ it("passes a package inside its size budget", () => {
+ const budget = { packed: 1000, unpacked: 4000 };
+ assert.deepEqual(listPackageSizeIssues("pkg", { packed: 950, unpacked: 3900 }, budget), []);
+ });
+
+ it("fails a package over its size budget", () => {
+ const issues = listPackageSizeIssues(
+ "pkg",
+ { packed: 1001, unpacked: 3900 },
+ { packed: 1000, unpacked: 4000 },
+ );
+ assert.equal(issues.length, 1);
+ assert.match(issues[0], /pkg packed size 1001 bytes is over its budget of 1000/);
+ });
+
+ it("makes a budget follow a package down once it shrinks", () => {
+ const issues = listPackageSizeIssues(
+ "pkg",
+ { packed: 500, unpacked: 3900 },
+ { packed: 1000, unpacked: 4000 },
+ );
+ assert.equal(issues.length, 1);
+ assert.match(issues[0], /lower it to 525/);
+ });
+
+ it("fails a malformed budget instead of skipping the metric", () => {
+ const sizes = { packed: 100, unpacked: 400 };
+ for (const budget of [{}, { paked: 100, unpacked: 400 }, { packed: "NaN", unpacked: 400 }]) {
+ assert.match(listPackageSizeIssues("pkg", sizes, budget).join("\n"), /invalid packed budget/);
+ }
+ });
+
+ it("asks for a budget for a new published package", () => {
+ assert.match(
+ listPackageSizeIssues("pkg", { packed: 100, unpacked: 400 }, undefined)[0],
+ /"pkg": \{ "packed": 105, "unpacked": 420 \}/,
+ );
+ });
+
it("requires the CLI package and tarball to declare Apache-2.0", () => {
assert.throws(
() => verifyCliLicense("packages/cli", {}, {}),