Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
ci(studio): gates read the base the run measured, and drag frames on a quiet runner (#4872)
Miguel Ángeld218cec091a7
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex a72ef42c90..28fb97023a 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -851,14 +851,7 @@ jobs:- name: Read the base branch's work-count ceilings# A ceiling may not disappear, or rise without a new reason, against the branch this PR merges into.if: github.event_name == 'pull_request'- env:- BASE_REF: ${{ github.base_ref }}- run: |- git fetch --no-tags --depth=1 origin "$BASE_REF"- if ! git show FETCH_HEAD:packages/studio/tests/e2e/perf-ceilings.json > /tmp/base-perf-ceilings.json; then- rm -f /tmp/base-perf-ceilings.json- echo "The base branch has no perf-ceilings.json yet"- fi+ run: bash scripts/ci/base-file.sh packages/studio/tests/e2e/perf-ceilings.json /tmp/base-perf-ceilings.json- name: Run both arms of the timeline viewport gaterun: |set -euo pipefail@@ -1020,14 +1013,9 @@ jobs:# Studio and the producer both run in the engine's pinned headless shell.- run: node packages/cli/dist/cli.js browser ensure- name: Read the base branch's edit accuracy baseline- env:- BASE_REF: ${{ github.base_ref }}run: |- git fetch --no-tags --depth=1 origin "$BASE_REF"- if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then- echo '{"cases":{}}' > /tmp/base-edit-accuracy.json- echo "The base branch has no edit accuracy baseline yet"- fi+ bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json+ [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json- name: Run this shard, then re-run twice each case whose verdict differs from the base branchrun: |set -euo pipefail@@ -1062,14 +1050,9 @@ jobs:with:node-version: 22- name: Read the base branch's edit accuracy baseline- env:- BASE_REF: ${{ github.base_ref }}run: |- git fetch --no-tags --depth=1 origin "$BASE_REF"- if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then- echo '{"cases":{}}' > /tmp/base-edit-accuracy.json- echo "The base branch has no edit accuracy baseline yet"- fi+ bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json+ [ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4with:pattern: edit-accuracy-shard-*diff --git a/.github/workflows/studio-drag-frames.yml b/.github/workflows/studio-drag-frames.ymlnew file mode 100644index 0000000000..9c95f2cd75--- /dev/null+++ b/.github/workflows/studio-drag-frames.yml@@ -0,0 +1,76 @@+name: Studio drag frames++permissions:+ contents: read++env:+ HYPERFRAMES_NO_TELEMETRY: "1"++# Measures drag smoothness on a runner with nothing else on it. The bench measures a blank page per case+# in the same run, and the box only counts as quiet if that control dropped no frame anywhere.+on:+ workflow_dispatch:+ inputs:+ ref:+ description: Commit, branch or tag to measure+ default: main+ pull_request:+ paths:+ - ".github/workflows/studio-drag-frames.yml"++jobs:+ drag-frames:+ name: "Studio: drag frames"+ runs-on: ubuntu-latest+ timeout-minutes: 60+ env:+ BENCH_REF: ${{ inputs.ref || 'main' }}+ steps:+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ with:+ ref: ${{ env.BENCH_REF }}+ lfs: true+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version: 22+ - run: corepack enable+ - run: corepack prepare [email protected] --activate+ - uses: ./.github/actions/prepare-ffmpeg-bin+ - run: bash scripts/ci/install-workspace-dependencies.sh+ - run: bun run build+ - run: node packages/cli/dist/cli.js browser ensure+ - name: Run the drag cases five times+ working-directory: packages/studio+ run: |+ set -euo pipefail+ echo "Measuring $(git rev-parse HEAD)"+ for n in 1 2 3 4 5; do+ bun tests/e2e/edit-accuracy/run.mjs --grid pr \+ --filter '^(move|resize|rotate|crop)-(none|tween)-px-r0-root-z100$' \+ --jobs 1 --port 5961 --cli "$GITHUB_WORKSPACE/packages/cli/dist/cli.js" \+ --out "$RUNNER_TEMP/drag-frames/run-$n"+ done+ - name: Require a quiet box (the blank page dropped no frame)+ run: |+ # shellcheck disable=SC2016 # the backticks are JavaScript template strings+ node -e '+ const fs = require("fs"), dir = process.env.RUNNER_TEMP + "/drag-frames";+ let noisy = 0;+ for (const run of fs.readdirSync(dir)) {+ const { cases } = JSON.parse(fs.readFileSync(`${dir}/${run}/results.json`, "utf8"));+ for (const c of cases) {+ const dropped = c.smooth?.control?.dropped;+ if (dropped !== 0) { noisy++; console.log(`${run} ${c.id} control dropped=${dropped}`); }+ }+ }+ if (noisy) { console.log(`::error::the blank page dropped frames, or went unmeasured, in ${noisy} case(s); this box is not quiet`); process.exit(1); }+ console.log("the blank page dropped 0 frames in every case of every run");+ '+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ if: always()+ with:+ name: studio-drag-frames+ path: ${{ runner.temp }}/drag-frames+ if-no-files-found: error+ retention-days: 30diff --git a/package.json b/package.jsonindex dafda9bd95..642414e008 100644--- a/package.json+++ b/package.json@@ -52,7 +52,7 @@"player:perf": "bun run --filter @hyperframes/player perf","format:check": "oxfmt --check .","knip": "knip",- "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",+ "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts","typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json","test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'","generate:previews": "tsx scripts/generate-template-previews.ts",diff --git a/scripts/ci/base-file.sh b/scripts/ci/base-file.shnew file mode 100755index 0000000000..0d3c280cf1--- /dev/null+++ b/scripts/ci/base-file.sh@@ -0,0 +1,14 @@+#!/usr/bin/env bash+# base-file.sh <path> <out>: writes <path> as it was on the base this pull_request run measured, the first parent of+# the checked-out merge commit. The base branch tip moves while a run is in flight, so it is never read. A base without+# the file removes <out>; a failed fetch fails the step.+set -euo pipefail+base="$(git cat-file -p HEAD | awk '/^parent /{print $2; exit}')"+echo "Reading $1 from the base this run measured: $base"+git fetch --no-tags --depth=1 origin "$base"+if git cat-file -e "$base:$1" 2>/dev/null; then+ git show "$base:$1" > "$2"+else+ rm -f "$2"+ echo "The base has no $1 yet"+fidiff --git a/scripts/ci/base-file.test.mjs b/scripts/ci/base-file.test.mjsnew file mode 100644index 0000000000..a0e0df7936--- /dev/null+++ b/scripts/ci/base-file.test.mjs@@ -0,0 +1,68 @@+import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { test } from "node:test";++const script = join(import.meta.dirname, "base-file.sh");+const git = (cwd, ...args) =>+ execFileSync(+ "git",+ ["-c", "user.email=t@t", "-c", "user.name=t", "-c", "commit.gpgsign=false", ...args],+ {+ cwd,+ encoding: "utf8",+ },+ ).trim();++// A PR run checks out the merge commit shallowly; the base branch then moves before the gate reads its baseline.+function raceFixture(baseline) {+ const root = mkdtempSync(join(tmpdir(), "base-file-"));+ const origin = join(root, "origin.git");+ const work = join(root, "work");+ git(root, "init", "-q", "--bare", origin);+ git(root, "init", "-q", "-b", "main", work);+ git(work, "remote", "add", "origin", origin);+ if (baseline) writeFileSync(join(work, "baseline.json"), baseline);+ writeFileSync(join(work, "readme"), "base");+ git(work, "add", ".");+ git(work, "commit", "-qm", "base");+ const base = git(work, "rev-parse", "HEAD");+ git(work, "switch", "-qc", "pr");+ writeFileSync(join(work, "pr"), "change");+ git(work, "add", ".");+ git(work, "commit", "-qm", "pr");+ git(work, "switch", "-q", "main");+ git(work, "merge", "-q", "--no-ff", "-m", "merge", "pr");+ const merge = git(work, "rev-parse", "HEAD");+ git(work, "push", "-q", "origin", `${merge}:refs/pull/1/merge`);+ git(work, "reset", "-q", "--hard", base);+ writeFileSync(join(work, "baseline.json"), '{"total":852}');+ git(work, "add", ".");+ git(work, "commit", "-qm", "base moves on");+ git(work, "push", "-q", "origin", "HEAD:main");+ const checkout = join(root, "checkout");+ git(root, "init", "-q", checkout);+ git(checkout, "remote", "add", "origin", `file://${origin}`);+ git(checkout, "fetch", "-q", "--depth=1", "origin", "refs/pull/1/merge");+ git(checkout, "checkout", "-q", "FETCH_HEAD");+ return { checkout, base, out: join(root, "out.json") };+}++test("reads the base the run measured, not the base branch tip that moved since", () => {+ const { checkout, base, out } = raceFixture('{"total":846}');+ const log = execFileSync("bash", [script, "baseline.json", out], {+ cwd: checkout,+ encoding: "utf8",+ });+ assert.equal(readFileSync(out, "utf8"), '{"total":846}');+ assert.match(log, new RegExp(`measured: ${base}`));+});++test("a base without the file leaves no output", () => {+ const { checkout, out } = raceFixture(null);+ writeFileSync(out, "stale");+ execFileSync("bash", [script, "baseline.json", out], { cwd: checkout, encoding: "utf8" });+ assert.equal(existsSync(out), false);+});