Open source CapCut alternative.
TypeScript
ci: every eval a runner can carry runs nightly
David Ledac54c99f694
diff --git a/.github/workflows/nightly-cut-perf.yml b/.github/workflows/nightly-cut-perf.ymldeleted file mode 100644index 76f14737..00000000--- a/.github/workflows/nightly-cut-perf.yml+++ /dev/null@@ -1,128 +0,0 @@-name: Nightly Cut Perf--# Drives the preview performance eval (scripts/eval-cut-perf.ts) every night-# in a real Chrome against a dev server, on a machine whose display and load-# never change. The scrub cases are held to the `ci` machine profile's-# budgets: the runner has no hardware video decoder, and that profile names-# what a scrub costs on one. A case that breaches runs again, to three runs,-# and the majority decides, so one hiccup on the shared host is one run. The-# playback cases run for the record on the same profile — their lag budgets-# describe a machine with a decoder — and the full report, with every scrub-# case's trace, is kept as an artifact.--on:- workflow_dispatch:- schedule:- - cron: "0 10 * * *"--permissions:- contents: read--concurrency:- group: nightly-cut-perf- cancel-in-progress: true--jobs:- perf:- runs-on: ubuntu-latest- timeout-minutes: 60- defaults:- run:- # Names bash so the shell runs with pipefail: a step's status is the- # eval's exit code, whatever tee behind it returns.- shell: bash- working-directory: site- steps:- - uses: actions/checkout@v7-- - uses: actions/setup-node@v7- with:- node-version: 24- cache: npm- cache-dependency-path: site/package-lock.json-- - name: Install dependencies- run: npm ci-- - name: Generate Prisma client- run: npx prisma generate-- - name: Install ffmpeg- run: |- sudo apt-get update -qq- sudo apt-get install -y -qq ffmpeg-- # Placeholder values: the eval's project lives on the dev server's own- # disk and the page answers its session check itself, so nothing reads- # these — they only have to exist for the dev server to boot.- - name: Write placeholder env- run: |- for key in BETTER_AUTH_SECRET GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET \- R2_ACCOUNT_ID R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY \- CUT_MEDIA_SIGNING_SECRET; do- echo "$key=placeholder" >> .env- done- echo "DATABASE_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env- echo "DIRECT_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env-- - name: Start dev server- env:- DONKEY_DEV_AUTH_BYPASS: "1"- run: |- npm run dev > /tmp/next-dev.log 2>&1 &- for i in $(seq 1 60); do- if curl -sf -o /dev/null "http://localhost:3000/api/cut/projects?u=donkey-dev-auth-bypass"; then exit 0; fi- sleep 2- done- echo "dev server never answered" >&2- tail -50 /tmp/next-dev.log >&2- exit 1-- - name: Scrub cases, held to budget- run: npm run eval:cut-perf -- --bucket scrub --machine ci --enforce-budgets --dump-trace --out /tmp/cut-perf-scrub.json | tee /tmp/cut-perf-scrub.log-- - name: Full eval, for the record- if: always()- run: npm run eval:cut-perf -- --machine ci --out /tmp/cut-perf.json | tee /tmp/cut-perf.log-- - name: Keys after a drop- if: always()- run: npm run eval:cut-keys -- --out /tmp/cut-keys.json | tee /tmp/cut-keys.log-- - name: Summary- if: always()- run: |- {- echo '## Scrub gate, ci profile'- echo '```'- grep -E '^\[(ok|FAIL)|breached' /tmp/cut-perf-scrub.log || true- echo '```'- echo '## Full eval, for the record'- echo '```'- grep -E '^\[(ok|FAIL)' /tmp/cut-perf.log || true- echo '```'- echo '## Cut keys'- echo '```'- grep -E '^\[(ok|FAIL)' /tmp/cut-keys.log || true- echo '```'- } >> "$GITHUB_STEP_SUMMARY"-- - uses: actions/upload-artifact@v7- if: always()- with:- name: cut-perf-report- path: |- /tmp/cut-perf.json- /tmp/cut-perf-scrub.json- /tmp/cut-perf.log- /tmp/cut-perf-scrub.log- /tmp/cut-keys.json- /tmp/cut-keys.log- dist/cut-perf/trace-*.json- dist/cut-perf/log-*.json- dist/cut-perf/net-*.json- if-no-files-found: ignore-- - name: Dev server log on failure- if: failure()- run: tail -100 /tmp/next-dev.logdiff --git a/.github/workflows/nightly-evals.yml b/.github/workflows/nightly-evals.ymlnew file mode 100644index 00000000..1d6a1ad1--- /dev/null+++ b/.github/workflows/nightly-evals.yml@@ -0,0 +1,247 @@+name: Nightly Evals++# Every eval that runs on a GitHub runner, on one schedule. The jobs are+# grouped by what an eval needs: a process, a browser, or a dev server. A group+# that needs more than the last one costs more to run, so a failure's first+# clue is which group went red.+#+# The evals that ask a live model are run by hand, and the export eval has its+# own workflow because Chrome encodes AAC through the platform and only a Mac+# carries that encoder. `site/scripts/lib/cut-eval/README.md` says how.+#+# The scrub cases are held to the `ci` machine profile's budgets: the runner+# has no hardware video decoder, and that profile names what a scrub costs on+# one. A case that breaches runs again, to three runs, and the majority+# decides, so one hiccup on the shared host is one run. The playback cases run+# for the record on the same profile — their lag budgets describe a machine+# with a decoder — and the full report, with every scrub case's trace, is kept+# as an artifact.++on:+ workflow_dispatch:+ schedule:+ - cron: "0 10 * * *"++permissions:+ contents: read++concurrency:+ group: nightly-evals+ cancel-in-progress: true++jobs:+ # Unit tests and the evals that build their own truth in process: a script of+ # known onsets, a take of known words, a brief run through fake models. No+ # browser, no server, no network, no credits — a couple of minutes for the+ # broadest coverage the project has.+ units:+ name: Units and pure evals+ runs-on: ubuntu-latest+ timeout-minutes: 20+ defaults:+ run:+ shell: bash+ working-directory: site+ env:+ DATABASE_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder+ DIRECT_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder+ steps:+ - uses: actions/checkout@v7++ - uses: actions/setup-node@v7+ with:+ node-version: 24+ cache: npm+ cache-dependency-path: site/package-lock.json++ - name: Install dependencies+ run: npm ci++ - name: Generate Prisma client+ run: npx prisma generate++ - name: Unit tests+ run: npm test++ - name: Caption timing+ if: always()+ run: npm run eval:cut-sync++ - name: Speech cut edges+ if: always()+ run: npm run eval:cut-edges++ - name: Brief-to-video self-test+ if: always()+ run: npm run test:genvideo++ - name: Model comparison over fake suites+ if: always()+ run: npm run eval:genvideo++ # Real editor components under real pointer and key gestures. Each fixture+ # bundles the components and serves them to Chrome itself, so these need a+ # browser and nothing behind it. ffmpeg is here for the widget's sample clip.+ components:+ name: Component evals+ runs-on: ubuntu-latest+ timeout-minutes: 30+ defaults:+ run:+ shell: bash+ working-directory: site+ env:+ DATABASE_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder+ DIRECT_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder+ steps:+ - uses: actions/checkout@v7++ - uses: actions/setup-node@v7+ with:+ node-version: 24+ cache: npm+ cache-dependency-path: site/package-lock.json++ - name: Install dependencies+ run: npm ci++ - name: Generate Prisma client+ run: npx prisma generate++ - name: Install ffmpeg+ run: |+ sudo apt-get update -qq+ sudo apt-get install -y -qq ffmpeg++ # The fixture evals drive the Chrome the runner already carries. The two+ # ChatGPT evals drive Playwright's own Chromium, which is a download.+ - name: Install Chromium+ run: npx playwright install --with-deps chromium++ - name: Timeline operations+ run: npm run eval:cut-timeline-operations++ - name: Timeline rows+ if: always()+ run: npm run eval:cut-timeline-rows++ - name: Preview selection+ if: always()+ run: npm run eval:cut-preview-selection++ - name: ChatGPT widget+ if: always()+ run: npm run eval:chatgpt-widget++ - name: ChatGPT OAuth consent+ if: always()+ run: npm run eval:chatgpt-oauth++ # The preview under load and the keys after a drop, driven against a dev+ # server whose project lives on its own disk. The export eval belongs to this+ # tier and runs on its own workflow, because Chrome encodes AAC through the+ # platform and only a Mac has that encoder.+ browser:+ name: Preview and keys+ runs-on: ubuntu-latest+ timeout-minutes: 60+ defaults:+ run:+ # Names bash so the shell runs with pipefail: a step's status is the+ # eval's exit code, whatever tee behind it returns.+ shell: bash+ working-directory: site+ steps:+ - uses: actions/checkout@v7++ - uses: actions/setup-node@v7+ with:+ node-version: 24+ cache: npm+ cache-dependency-path: site/package-lock.json++ - name: Install dependencies+ run: npm ci++ - name: Generate Prisma client+ run: npx prisma generate++ - name: Install ffmpeg+ run: |+ sudo apt-get update -qq+ sudo apt-get install -y -qq ffmpeg++ # Placeholder values: the eval's project lives on the dev server's own+ # disk and the page answers its session check itself, so nothing reads+ # these — they only have to exist for the dev server to boot.+ - name: Write placeholder env+ run: |+ for key in BETTER_AUTH_SECRET GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET \+ R2_ACCOUNT_ID R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY \+ CUT_MEDIA_SIGNING_SECRET; do+ echo "$key=placeholder" >> .env+ done+ echo "DATABASE_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env+ echo "DIRECT_URL=postgresql://placeholder:placeholder@localhost:5432/placeholder" >> .env++ - name: Start dev server+ env:+ DONKEY_DEV_AUTH_BYPASS: "1"+ run: |+ npm run dev > /tmp/next-dev.log 2>&1 &+ for i in $(seq 1 60); do+ if curl -sf -o /dev/null "http://localhost:3000/api/cut/projects?u=donkey-dev-auth-bypass"; then exit 0; fi+ sleep 2+ done+ echo "dev server never answered" >&2+ tail -50 /tmp/next-dev.log >&2+ exit 1++ - name: Scrub cases, held to budget+ run: npm run eval:cut-perf -- --bucket scrub --machine ci --enforce-budgets --dump-trace --out /tmp/cut-perf-scrub.json | tee /tmp/cut-perf-scrub.log++ - name: Full eval, for the record+ if: always()+ run: npm run eval:cut-perf -- --machine ci --out /tmp/cut-perf.json | tee /tmp/cut-perf.log++ - name: Keys after a drop+ if: always()+ run: npm run eval:cut-keys -- --out /tmp/cut-keys.json | tee /tmp/cut-keys.log++ - name: Summary+ if: always()+ run: |+ {+ echo '## Scrub gate, ci profile'+ echo '```'+ grep -E '^\[(ok|FAIL)|breached' /tmp/cut-perf-scrub.log || true+ echo '```'+ echo '## Full eval, for the record'+ echo '```'+ grep -E '^\[(ok|FAIL)' /tmp/cut-perf.log || true+ echo '```'+ echo '## Cut keys'+ echo '```'+ grep -E '^\[(ok|FAIL)' /tmp/cut-keys.log || true+ echo '```'+ } >> "$GITHUB_STEP_SUMMARY"++ - uses: actions/upload-artifact@v7+ if: always()+ with:+ name: cut-perf-report+ path: |+ /tmp/cut-perf.json+ /tmp/cut-perf-scrub.json+ /tmp/cut-perf.log+ /tmp/cut-perf-scrub.log+ /tmp/cut-keys.json+ /tmp/cut-keys.log+ dist/cut-perf/trace-*.json+ dist/cut-perf/log-*.json+ dist/cut-perf/net-*.json+ if-no-files-found: ignore++ - name: Dev server log on failure+ if: failure()+ run: tail -100 /tmp/next-dev.logdiff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.mdindex 3099c61d..d7eeff7a 100644--- a/site/scripts/lib/cut-eval/README.md+++ b/site/scripts/lib/cut-eval/README.md@@ -69,6 +69,13 @@ class decides whether the render ladder may drop to its text-only rung andwhether a music prompt is worth rewriting, so a case is added whenever a newerror reached us and was read wrong.+## Nightly++`.github/workflows/nightly-evals.yml` runs the evals a GitHub runner can carry,+grouped by what each needs: a process, a browser, a dev server. The evals that+ask a live model are run by hand, because a runner has no key, no database of+its own and no budget to spend.+## The reportA full run writes `evals/cut-chat.latest-report.json` (schema