Skip to content

Lightpanda: the headless browser designed for AI and automation

ci: generate orderfile before release

Pierre Tachoire8fc4598ff9fb
diff --git a/.github/actions/orderfile/action.yml b/.github/actions/orderfile/action.yml
new file mode 100644
index 000000000..299418d70
--- /dev/null
+++ b/.github/actions/orderfile/action.yml
@@ -0,0 +1,79 @@
+name: "Orderfile"
+description: "Regenerate orderfile/lightpanda.ld and orderfile/v8.txt from a profile of the CDP bench"
+
+# Runs orderfile/tools/regen.sh, which writes the two files back into the
+# checkout. What the caller does with them (commit, or just build) is its own
+# business. See orderfile/README.md.
+#
+# Needs a Linux runner with sudo (the script flips
+# /sys/kernel/debug/fault_around_bytes), plus node, go, python3 and binutils.
+# Call it after ./.github/actions/install and after the v8 snapshot.
+
+inputs:
+ build-args:
+ description: 'zig build args for the profiling build: the release build args minus -Dorderfile'
+ required: true
+ runs:
+ description: 'CDP bench iterations'
+ required: false
+ default: '100'
+ demo-repository:
+ description: 'Checkout holding the CDP bench (demo/puppeteer/cdp.js)'
+ required: false
+ default: 'lightpanda-io/demo'
+
+outputs:
+ out-dir:
+ description: 'Scratch directory with the profile: resident.json, hot.text, hot.rodata, gen_order.stats, result.txt and the generated lightpanda.ld / v8.txt'
+ value: ${{ steps.out.outputs.dir }}
+
+runs:
+ using: "composite"
+
+ steps:
+ # The bench the profile is taken from.
+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ repository: ${{ inputs.demo-repository }}
+ path: demo
+
+ - run: npm install
+ shell: bash
+ working-directory: demo
+
+ # Its own step, so the path is still an output when the profile fails.
+ - name: profile scratch directory
+ id: out
+ shell: bash
+ run: echo "dir=$RUNNER_TEMP/orderfile-regen" >> "$GITHUB_OUTPUT"
+
+ - name: regenerate the profile
+ shell: bash
+ env:
+ DEMO_DIR: demo
+ RUNS: ${{ inputs.runs }}
+ OUT: ${{ steps.out.outputs.dir }}
+ BUILD_ARGS: ${{ inputs.build-args }}
+ run: |
+ # BUILD_ARGS is a list of flags, it has to word-split.
+ # shellcheck disable=SC2086
+ orderfile/tools/regen.sh $BUILD_ARGS
+
+ # $OUT dies with the runner, and a failed profile is the one case where
+ # the page dump and the bench output are worth reading afterwards.
+ - name: upload profile
+ if: failure()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: orderfile-profile
+ path: |
+ ${{ steps.out.outputs.dir }}/result.txt
+ ${{ steps.out.outputs.dir }}/gen_order.stats
+ ${{ steps.out.outputs.dir }}/hot.text
+ ${{ steps.out.outputs.dir }}/hot.rodata
+ ${{ steps.out.outputs.dir }}/resident.json
+ ${{ steps.out.outputs.dir }}/link.line
+ ${{ steps.out.outputs.dir }}/bench.out
+ ${{ steps.out.outputs.dir }}/lightpanda.ld
+ ${{ steps.out.outputs.dir }}/v8.txt
+ retention-days: 10
diff --git a/.github/workflows/orderfile.yml b/.github/workflows/orderfile.yml
index eb1e20dec..d02f7423d 100644
--- a/.github/workflows/orderfile.yml
+++ b/.github/workflows/orderfile.yml
@@ -3,6 +3,9 @@ name: orderfile
# Regenerates orderfile/lightpanda.ld + orderfile/v8.txt from a profile of the
# CDP bench and opens a pull request with the result, every night or on demand
# from the Actions tab. See orderfile/README.md.
+#
+# release.yml runs the same ./.github/actions/orderfile, but builds with the
+# profile instead of committing it.
env:
LIGHTPANDA_DISABLE_TELEMETRY: true
@@ -34,46 +37,16 @@ jobs:
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
- ref: main
fetch-depth: 0
-
- - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- repository: 'lightpanda-io/demo'
- path: demo
+ token: ${{ steps.app-token.outputs.token }}
- uses: ./.github/actions/install
- uses: ./.github/actions/v8-snapshot
- - run: npm install
- working-directory: demo
-
- - name: regenerate the profile
- env:
- DEMO_DIR: demo
- OUT: ${{ runner.temp }}/orderfile-regen
- run: |
- orderfile/tools/regen.sh -Dsnapshot_path=../../snapshot.bin -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64
-
- # Only on failure: a run that got as far as the PR has its result in
- # the PR itself, and $OUT dies with the runner, so this is the one case
- # where the page dump and the bench output are worth keeping.
- - name: upload profile
- if: failure()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ - uses: ./.github/actions/orderfile
+ id: orderfile
with:
- name: orderfile-profile
- path: |
- ${{ runner.temp }}/orderfile-regen/result.txt
- ${{ runner.temp }}/orderfile-regen/gen_order.stats
- ${{ runner.temp }}/orderfile-regen/hot.text
- ${{ runner.temp }}/orderfile-regen/hot.rodata
- ${{ runner.temp }}/orderfile-regen/resident.json
- ${{ runner.temp }}/orderfile-regen/link.line
- ${{ runner.temp }}/orderfile-regen/bench.out
- ${{ runner.temp }}/orderfile-regen/lightpanda.ld
- ${{ runner.temp }}/orderfile-regen/v8.txt
- retention-days: 10
+ build-args: -Dsnapshot_path=../../snapshot.bin -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64
- name: open a pull request
env:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e268c7f0a..ba77b6686 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -26,6 +26,47 @@ permissions:
packages: read
jobs:
+ # The hot-code profile the Linux binaries are laid out with. Regenerated on
+ # every release rather than taken from the repository, so the layout matches
+ # the code being shipped. Nothing is committed here; orderfile.yml is what
+ # keeps the checked-in profile fresh.
+ orderfile:
+ name: orderfile profile
+
+ runs-on: ubuntu-22.04
+ timeout-minutes: 45
+
+ # A profile is an optimization, not a requirement. When this job fails the
+ # build jobs find no artifact and fall back to orderfile/lightpanda.ld as
+ # committed, so a missing kernel knob or a flaky bench never blocks a
+ # release.
+ continue-on-error: true
+
+ steps:
+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ fetch-depth: 0
+
+ - uses: ./.github/actions/install
+
+ - name: v8 snapshot
+ run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast snapshot_creator -- src/snapshot.bin
+
+ - uses: ./.github/actions/orderfile
+ with:
+ build-args: -Dsnapshot_path=../../snapshot.bin -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64
+
+ # Both arches use this one profile: it is a list of symbol names, and
+ # orderfile/README.md applies the x86_64 profile to aarch64 as-is.
+ - name: share the profile with the build jobs
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: orderfile
+ path: |
+ orderfile/lightpanda.ld
+ orderfile/v8.txt
+ retention-days: 1
+
build-linux:
strategy:
fail-fast: false
@@ -47,6 +88,11 @@ jobs:
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
+ needs: orderfile
+ # The profile job is allowed to fail, and a skipped build is not an
+ # acceptable answer to that.
+ if: ${{ !cancelled() }}
+
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
@@ -57,6 +103,16 @@ jobs:
os: ${{env.OS}}
arch: ${{env.ARCH}}
+ # The fresh profile, when the orderfile job produced one. It overwrites
+ # the committed files; with no artifact the build keeps what is in the
+ # repository.
+ - name: fresh orderfile
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+ with:
+ name: orderfile
+ path: orderfile
+
- name: v8 snapshot
run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast snapshot_creator -- src/snapshot.bin
diff --git a/orderfile/README.md b/orderfile/README.md
index 7a61991e7..1cc7d92ad 100644
--- a/orderfile/README.md
+++ b/orderfile/README.md
@@ -19,8 +19,10 @@ below), with no change in run duration.
## How it is applied
- Opt-in via `-Dorderfile=orderfile/lightpanda.ld`; CI passes it for the Linux
- release artifact and the e2e bench build. Local release builds don't, so
- they keep the sub-second link. Linux/LLD only (it is an ELF linker script).
+ release artifact and the e2e bench build. The release regenerates the profile
+ first and builds with that one, falling back to the committed file when that
+ job fails. Local release builds don't pass the flag, so they keep the
+ sub-second link. Linux/LLD only (it is an ELF linker script).
- The exe and every C library it links are built with
`-ffunction-sections -fdata-sections` so each function/datum has its own
input section for the script to address.