Lightpanda: the headless browser designed for AI and automation
ci: generate orderfile before release
Pierre Tachoire8fc4598ff9fb
diff --git a/.github/actions/orderfile/action.yml b/.github/actions/orderfile/action.ymlnew file mode 100644index 000000000..299418d70--- /dev/null+++ b/.github/actions/orderfile/action.yml@@ -0,0 +1,79 @@+name: "Orderfile"+description: "Regenerate orderfile/lightpanda.ld and orderfile/v8.txt from a profile of the CDP bench"++# Runs orderfile/tools/regen.sh, which writes the two files back into the+# checkout. What the caller does with them (commit, or just build) is its own+# business. See orderfile/README.md.+#+# Needs a Linux runner with sudo (the script flips+# /sys/kernel/debug/fault_around_bytes), plus node, go, python3 and binutils.+# Call it after ./.github/actions/install and after the v8 snapshot.++inputs:+ build-args:+ description: 'zig build args for the profiling build: the release build args minus -Dorderfile'+ required: true+ runs:+ description: 'CDP bench iterations'+ required: false+ default: '100'+ demo-repository:+ description: 'Checkout holding the CDP bench (demo/puppeteer/cdp.js)'+ required: false+ default: 'lightpanda-io/demo'++outputs:+ out-dir:+ description: 'Scratch directory with the profile: resident.json, hot.text, hot.rodata, gen_order.stats, result.txt and the generated lightpanda.ld / v8.txt'+ value: ${{ steps.out.outputs.dir }}++runs:+ using: "composite"++ steps:+ # The bench the profile is taken from.+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3+ with:+ repository: ${{ inputs.demo-repository }}+ path: demo++ - run: npm install+ shell: bash+ working-directory: demo++ # Its own step, so the path is still an output when the profile fails.+ - name: profile scratch directory+ id: out+ shell: bash+ run: echo "dir=$RUNNER_TEMP/orderfile-regen" >> "$GITHUB_OUTPUT"++ - name: regenerate the profile+ shell: bash+ env:+ DEMO_DIR: demo+ RUNS: ${{ inputs.runs }}+ OUT: ${{ steps.out.outputs.dir }}+ BUILD_ARGS: ${{ inputs.build-args }}+ run: |+ # BUILD_ARGS is a list of flags, it has to word-split.+ # shellcheck disable=SC2086+ orderfile/tools/regen.sh $BUILD_ARGS++ # $OUT dies with the runner, and a failed profile is the one case where+ # the page dump and the bench output are worth reading afterwards.+ - name: upload profile+ if: failure()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: orderfile-profile+ path: |+ ${{ steps.out.outputs.dir }}/result.txt+ ${{ steps.out.outputs.dir }}/gen_order.stats+ ${{ steps.out.outputs.dir }}/hot.text+ ${{ steps.out.outputs.dir }}/hot.rodata+ ${{ steps.out.outputs.dir }}/resident.json+ ${{ steps.out.outputs.dir }}/link.line+ ${{ steps.out.outputs.dir }}/bench.out+ ${{ steps.out.outputs.dir }}/lightpanda.ld+ ${{ steps.out.outputs.dir }}/v8.txt+ retention-days: 10diff --git a/.github/workflows/orderfile.yml b/.github/workflows/orderfile.ymlindex eb1e20dec..d02f7423d 100644--- a/.github/workflows/orderfile.yml+++ b/.github/workflows/orderfile.yml@@ -3,6 +3,9 @@ name: orderfile# Regenerates orderfile/lightpanda.ld + orderfile/v8.txt from a profile of the# CDP bench and opens a pull request with the result, every night or on demand# from the Actions tab. See orderfile/README.md.+#+# release.yml runs the same ./.github/actions/orderfile, but builds with the+# profile instead of committing it.env:LIGHTPANDA_DISABLE_TELEMETRY: true@@ -34,46 +37,16 @@ jobs:steps:- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3with:- ref: mainfetch-depth: 0-- - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3- with:- repository: 'lightpanda-io/demo'- path: demo+ token: ${{ steps.app-token.outputs.token }}- uses: ./.github/actions/install- uses: ./.github/actions/v8-snapshot- - run: npm install- working-directory: demo-- - name: regenerate the profile- env:- DEMO_DIR: demo- OUT: ${{ runner.temp }}/orderfile-regen- run: |- orderfile/tools/regen.sh -Dsnapshot_path=../../snapshot.bin -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64-- # Only on failure: a run that got as far as the PR has its result in- # the PR itself, and $OUT dies with the runner, so this is the one case- # where the page dump and the bench output are worth keeping.- - name: upload profile- if: failure()- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ - uses: ./.github/actions/orderfile+ id: orderfilewith:- name: orderfile-profile- path: |- ${{ runner.temp }}/orderfile-regen/result.txt- ${{ runner.temp }}/orderfile-regen/gen_order.stats- ${{ runner.temp }}/orderfile-regen/hot.text- ${{ runner.temp }}/orderfile-regen/hot.rodata- ${{ runner.temp }}/orderfile-regen/resident.json- ${{ runner.temp }}/orderfile-regen/link.line- ${{ runner.temp }}/orderfile-regen/bench.out- ${{ runner.temp }}/orderfile-regen/lightpanda.ld- ${{ runner.temp }}/orderfile-regen/v8.txt- retention-days: 10+ build-args: -Dsnapshot_path=../../snapshot.bin -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64- name: open a pull requestenv:diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex e268c7f0a..ba77b6686 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -26,6 +26,47 @@ permissions:packages: readjobs:+ # The hot-code profile the Linux binaries are laid out with. Regenerated on+ # every release rather than taken from the repository, so the layout matches+ # the code being shipped. Nothing is committed here; orderfile.yml is what+ # keeps the checked-in profile fresh.+ orderfile:+ name: orderfile profile++ runs-on: ubuntu-22.04+ timeout-minutes: 45++ # A profile is an optimization, not a requirement. When this job fails the+ # build jobs find no artifact and fall back to orderfile/lightpanda.ld as+ # committed, so a missing kernel knob or a flaky bench never blocks a+ # release.+ continue-on-error: true++ steps:+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3+ with:+ fetch-depth: 0++ - uses: ./.github/actions/install++ - name: v8 snapshot+ run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast snapshot_creator -- src/snapshot.bin++ - uses: ./.github/actions/orderfile+ with:+ build-args: -Dsnapshot_path=../../snapshot.bin -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64++ # Both arches use this one profile: it is a list of symbol names, and+ # orderfile/README.md applies the x86_64 profile to aarch64 as-is.+ - name: share the profile with the build jobs+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ with:+ name: orderfile+ path: |+ orderfile/lightpanda.ld+ orderfile/v8.txt+ retention-days: 1+build-linux:strategy:fail-fast: false@@ -47,6 +88,11 @@ jobs:runs-on: ${{ matrix.runner }}timeout-minutes: 30+ needs: orderfile+ # The profile job is allowed to fail, and a skipped build is not an+ # acceptable answer to that.+ if: ${{ !cancelled() }}+steps:- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3with:@@ -57,6 +103,16 @@ jobs:os: ${{env.OS}}arch: ${{env.ARCH}}+ # The fresh profile, when the orderfile job produced one. It overwrites+ # the committed files; with no artifact the build keeps what is in the+ # repository.+ - name: fresh orderfile+ continue-on-error: true+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0+ with:+ name: orderfile+ path: orderfile+- name: v8 snapshotrun: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast snapshot_creator -- src/snapshot.bindiff --git a/orderfile/README.md b/orderfile/README.mdindex 7a61991e7..1cc7d92ad 100644--- a/orderfile/README.md+++ b/orderfile/README.md@@ -19,8 +19,10 @@ below), with no change in run duration.## How it is applied- Opt-in via `-Dorderfile=orderfile/lightpanda.ld`; CI passes it for the Linux- release artifact and the e2e bench build. Local release builds don't, so- they keep the sub-second link. Linux/LLD only (it is an ELF linker script).+ release artifact and the e2e bench build. The release regenerates the profile+ first and builds with that one, falling back to the committed file when that+ job fails. Local release builds don't pass the flag, so they keep the+ sub-second link. Linux/LLD only (it is an ELF linker script).- The exe and every C library it links are built with`-ffunction-sections -fdata-sections` so each function/datum has its owninput section for the script to address.