Skip to content

Open source CapCut alternative.

TypeScript

ci: every eval a runner can carry runs nightly

David Ledac54c99f694
diff --git a/.github/workflows/nightly-cut-perf.yml b/.github/workflows/nightly-cut-perf.yml
deleted file mode 100644
index 76f14737..00000000
--- a/.github/workflows/nightly-cut-perf.yml
+++ /dev/null
@@ -1,128 +0,0 @@
-name: Nightly Cut Perf
-
-# Drives the preview performance eval (scripts/eval-cut-perf.ts) every night
-# in a real Chrome against a dev server, on a machine whose display and load
-# never change. The scrub cases are held to the `ci` machine profile's
-# budgets: the runner has no hardware video decoder, and that profile names
-# what a scrub costs on one. A case that breaches runs again, to three runs,
-# and the majority decides, so one hiccup on the shared host is one run. The
-# playback cases run for the record on the same profile — their lag budgets
-# describe a machine with a decoder — and the full report, with every scrub
-# case's trace, is kept as an artifact.
-
-on:
- workflow_dispatch:
- schedule:
- - cron: "0 10 * * *"
-
-permissions:
- contents: read
-
-concurrency:
- group: nightly-cut-perf
- cancel-in-progress: true
-
-jobs:
- perf:
- runs-on: ubuntu-latest
- timeout-minutes: 60
- defaults:
- run:
- # Names bash so the shell runs with pipefail: a step's status is the
- # eval's exit code, whatever tee behind it returns.
- shell: bash
- working-directory: site
- steps:
- - uses: actions/checkout@v7
-
- - uses: actions/setup-node@v7
- with:
- node-version: 24
- cache: npm
- cache-dependency-path: site/package-lock.json
-
- - name: Install dependencies
- run: npm ci
-
- - name: Generate Prisma client
- run: npx prisma generate
-
- - name: Install ffmpeg
- run: |
- sudo apt-get update -qq
- sudo apt-get install -y -qq ffmpeg
-
- # Placeholder values: the eval's project lives on the dev server's own
- # disk and the page answers its session check itself, so nothing reads
- # these — they only have to exist for the dev server to boot.
- - name: Write placeholder env
- run: |
- for key in BETTER_AUTH_SECRET GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET \
- R2_ACCOUNT_ID R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY \
- CUT_MEDIA_SIGNING_SECRET; do
- echo "$key=placeholder" >> .env
- done
- echo "DATABASE_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env
- echo "DIRECT_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env
-
- - name: Start dev server
- env:
- DONKEY_DEV_AUTH_BYPASS: "1"
- run: |
- npm run dev > /tmp/next-dev.log 2>&1 &
- for i in $(seq 1 60); do
- if curl -sf -o /dev/null "http://localhost:3000/api/cut/projects?u=donkey-dev-auth-bypass"; then exit 0; fi
- sleep 2
- done
- echo "dev server never answered" >&2
- tail -50 /tmp/next-dev.log >&2
- exit 1
-
- - name: Scrub cases, held to budget
- run: npm run eval:cut-perf -- --bucket scrub --machine ci --enforce-budgets --dump-trace --out /tmp/cut-perf-scrub.json | tee /tmp/cut-perf-scrub.log
-
- - name: Full eval, for the record
- if: always()
- run: npm run eval:cut-perf -- --machine ci --out /tmp/cut-perf.json | tee /tmp/cut-perf.log
-
- - name: Keys after a drop
- if: always()
- run: npm run eval:cut-keys -- --out /tmp/cut-keys.json | tee /tmp/cut-keys.log
-
- - name: Summary
- if: always()
- run: |
- {
- echo '## Scrub gate, ci profile'
- echo '```'
- grep -E '^\[(ok|FAIL)|breached' /tmp/cut-perf-scrub.log || true
- echo '```'
- echo '## Full eval, for the record'
- echo '```'
- grep -E '^\[(ok|FAIL)' /tmp/cut-perf.log || true
- echo '```'
- echo '## Cut keys'
- echo '```'
- grep -E '^\[(ok|FAIL)' /tmp/cut-keys.log || true
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
-
- - uses: actions/upload-artifact@v7
- if: always()
- with:
- name: cut-perf-report
- path: |
- /tmp/cut-perf.json
- /tmp/cut-perf-scrub.json
- /tmp/cut-perf.log
- /tmp/cut-perf-scrub.log
- /tmp/cut-keys.json
- /tmp/cut-keys.log
- dist/cut-perf/trace-*.json
- dist/cut-perf/log-*.json
- dist/cut-perf/net-*.json
- if-no-files-found: ignore
-
- - name: Dev server log on failure
- if: failure()
- run: tail -100 /tmp/next-dev.log
diff --git a/.github/workflows/nightly-evals.yml b/.github/workflows/nightly-evals.yml
new file mode 100644
index 00000000..1d6a1ad1
--- /dev/null
+++ b/.github/workflows/nightly-evals.yml
@@ -0,0 +1,247 @@
+name: Nightly Evals
+
+# Every eval that runs on a GitHub runner, on one schedule. The jobs are
+# grouped by what an eval needs: a process, a browser, or a dev server. A group
+# that needs more than the last one costs more to run, so a failure's first
+# clue is which group went red.
+#
+# The evals that ask a live model are run by hand, and the export eval has its
+# own workflow because Chrome encodes AAC through the platform and only a Mac
+# carries that encoder. `site/scripts/lib/cut-eval/README.md` says how.
+#
+# The scrub cases are held to the `ci` machine profile's budgets: the runner
+# has no hardware video decoder, and that profile names what a scrub costs on
+# one. A case that breaches runs again, to three runs, and the majority
+# decides, so one hiccup on the shared host is one run. The playback cases run
+# for the record on the same profile — their lag budgets describe a machine
+# with a decoder — and the full report, with every scrub case's trace, is kept
+# as an artifact.
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "0 10 * * *"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: nightly-evals
+ cancel-in-progress: true
+
+jobs:
+ # Unit tests and the evals that build their own truth in process: a script of
+ # known onsets, a take of known words, a brief run through fake models. No
+ # browser, no server, no network, no credits — a couple of minutes for the
+ # broadest coverage the project has.
+ units:
+ name: Units and pure evals
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ defaults:
+ run:
+ shell: bash
+ working-directory: site
+ env:
+ DATABASE_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder
+ DIRECT_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder
+ steps:
+ - uses: actions/checkout@v7
+
+ - uses: actions/setup-node@v7
+ with:
+ node-version: 24
+ cache: npm
+ cache-dependency-path: site/package-lock.json
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Generate Prisma client
+ run: npx prisma generate
+
+ - name: Unit tests
+ run: npm test
+
+ - name: Caption timing
+ if: always()
+ run: npm run eval:cut-sync
+
+ - name: Speech cut edges
+ if: always()
+ run: npm run eval:cut-edges
+
+ - name: Brief-to-video self-test
+ if: always()
+ run: npm run test:genvideo
+
+ - name: Model comparison over fake suites
+ if: always()
+ run: npm run eval:genvideo
+
+ # Real editor components under real pointer and key gestures. Each fixture
+ # bundles the components and serves them to Chrome itself, so these need a
+ # browser and nothing behind it. ffmpeg is here for the widget's sample clip.
+ components:
+ name: Component evals
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ defaults:
+ run:
+ shell: bash
+ working-directory: site
+ env:
+ DATABASE_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder
+ DIRECT_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder
+ steps:
+ - uses: actions/checkout@v7
+
+ - uses: actions/setup-node@v7
+ with:
+ node-version: 24
+ cache: npm
+ cache-dependency-path: site/package-lock.json
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Generate Prisma client
+ run: npx prisma generate
+
+ - name: Install ffmpeg
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -y -qq ffmpeg
+
+ # The fixture evals drive the Chrome the runner already carries. The two
+ # ChatGPT evals drive Playwright's own Chromium, which is a download.
+ - name: Install Chromium
+ run: npx playwright install --with-deps chromium
+
+ - name: Timeline operations
+ run: npm run eval:cut-timeline-operations
+
+ - name: Timeline rows
+ if: always()
+ run: npm run eval:cut-timeline-rows
+
+ - name: Preview selection
+ if: always()
+ run: npm run eval:cut-preview-selection
+
+ - name: ChatGPT widget
+ if: always()
+ run: npm run eval:chatgpt-widget
+
+ - name: ChatGPT OAuth consent
+ if: always()
+ run: npm run eval:chatgpt-oauth
+
+ # The preview under load and the keys after a drop, driven against a dev
+ # server whose project lives on its own disk. The export eval belongs to this
+ # tier and runs on its own workflow, because Chrome encodes AAC through the
+ # platform and only a Mac has that encoder.
+ browser:
+ name: Preview and keys
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ defaults:
+ run:
+ # Names bash so the shell runs with pipefail: a step's status is the
+ # eval's exit code, whatever tee behind it returns.
+ shell: bash
+ working-directory: site
+ steps:
+ - uses: actions/checkout@v7
+
+ - uses: actions/setup-node@v7
+ with:
+ node-version: 24
+ cache: npm
+ cache-dependency-path: site/package-lock.json
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Generate Prisma client
+ run: npx prisma generate
+
+ - name: Install ffmpeg
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -y -qq ffmpeg
+
+ # Placeholder values: the eval's project lives on the dev server's own
+ # disk and the page answers its session check itself, so nothing reads
+ # these — they only have to exist for the dev server to boot.
+ - name: Write placeholder env
+ run: |
+ for key in BETTER_AUTH_SECRET GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET \
+ R2_ACCOUNT_ID R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY \
+ CUT_MEDIA_SIGNING_SECRET; do
+ echo "$key=placeholder" >> .env
+ done
+ echo "DATABASE_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env
+ echo "DIRECT_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env
+
+ - name: Start dev server
+ env:
+ DONKEY_DEV_AUTH_BYPASS: "1"
+ run: |
+ npm run dev > /tmp/next-dev.log 2>&1 &
+ for i in $(seq 1 60); do
+ if curl -sf -o /dev/null "http://localhost:3000/api/cut/projects?u=donkey-dev-auth-bypass"; then exit 0; fi
+ sleep 2
+ done
+ echo "dev server never answered" >&2
+ tail -50 /tmp/next-dev.log >&2
+ exit 1
+
+ - name: Scrub cases, held to budget
+ run: npm run eval:cut-perf -- --bucket scrub --machine ci --enforce-budgets --dump-trace --out /tmp/cut-perf-scrub.json | tee /tmp/cut-perf-scrub.log
+
+ - name: Full eval, for the record
+ if: always()
+ run: npm run eval:cut-perf -- --machine ci --out /tmp/cut-perf.json | tee /tmp/cut-perf.log
+
+ - name: Keys after a drop
+ if: always()
+ run: npm run eval:cut-keys -- --out /tmp/cut-keys.json | tee /tmp/cut-keys.log
+
+ - name: Summary
+ if: always()
+ run: |
+ {
+ echo '## Scrub gate, ci profile'
+ echo '```'
+ grep -E '^\[(ok|FAIL)|breached' /tmp/cut-perf-scrub.log || true
+ echo '```'
+ echo '## Full eval, for the record'
+ echo '```'
+ grep -E '^\[(ok|FAIL)' /tmp/cut-perf.log || true
+ echo '```'
+ echo '## Cut keys'
+ echo '```'
+ grep -E '^\[(ok|FAIL)' /tmp/cut-keys.log || true
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ - uses: actions/upload-artifact@v7
+ if: always()
+ with:
+ name: cut-perf-report
+ path: |
+ /tmp/cut-perf.json
+ /tmp/cut-perf-scrub.json
+ /tmp/cut-perf.log
+ /tmp/cut-perf-scrub.log
+ /tmp/cut-keys.json
+ /tmp/cut-keys.log
+ dist/cut-perf/trace-*.json
+ dist/cut-perf/log-*.json
+ dist/cut-perf/net-*.json
+ if-no-files-found: ignore
+
+ - name: Dev server log on failure
+ if: failure()
+ run: tail -100 /tmp/next-dev.log
diff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.md
index 3099c61d..d7eeff7a 100644
--- a/site/scripts/lib/cut-eval/README.md
+++ b/site/scripts/lib/cut-eval/README.md
@@ -69,6 +69,13 @@ class decides whether the render ladder may drop to its text-only rung and
whether a music prompt is worth rewriting, so a case is added whenever a new
error reached us and was read wrong.
+## Nightly
+
+`.github/workflows/nightly-evals.yml` runs the evals a GitHub runner can carry,
+grouped by what each needs: a process, a browser, a dev server. The evals that
+ask a live model are run by hand, because a runner has no key, no database of
+its own and no budget to spend.
+
## The report
A full run writes `evals/cut-chat.latest-report.json` (schema