Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
build(aws-lambda): pinned native ffmpeg build for an arm64 Lambda image (#6062)
Miguel Ángelbb19f342d3e4
diff --git a/.github/workflows/lambda-ffmpeg.yml b/.github/workflows/lambda-ffmpeg.ymlnew file mode 100644index 0000000000..00c0ce4edc--- /dev/null+++ b/.github/workflows/lambda-ffmpeg.yml@@ -0,0 +1,47 @@+name: Lambda arm64 ffmpeg++permissions:+ contents: read++# Builds the native ffmpeg for the arm64 Lambda video function on two separate ARM runners.+# Each build runs the codec check on the bare Lambda base; the tarballs must be byte-identical.+on:+ pull_request:+ paths:+ - "packages/aws-lambda/image/**"+ - ".github/workflows/lambda-ffmpeg.yml"+ workflow_dispatch:++concurrency:+ group: lambda-ffmpeg-${{ github.ref }}+ cancel-in-progress: true++jobs:+ build:+ name: Build ${{ matrix.copy }}+ runs-on: ubuntu-24.04-arm+ timeout-minutes: 90+ strategy:+ matrix:+ copy: [a, b]+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ - name: Build and check+ run: >-+ docker buildx build --progress=plain --platform linux/arm64 --target artifact+ --output type=local,dest=out packages/aws-lambda/image+ - run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: hf-media-${{ matrix.copy }}+ path: out/+ retention-days: 7++ reproducible:+ name: Builds match+ needs: build+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ - run: cmp hf-media-a/hf-media-linux-aarch64.tar.gz hf-media-b/hf-media-linux-aarch64.tar.gzdiff --git a/packages/aws-lambda/image/Dockerfile b/packages/aws-lambda/image/Dockerfilenew file mode 100644index 0000000000..1cc7b124ba--- /dev/null+++ b/packages/aws-lambda/image/Dockerfile@@ -0,0 +1,34 @@+# The arm64 Lambda video function's native ffmpeg, built on the Lambda Node 22 base it runs on:+# docker buildx build --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/image+# The artifact stage depends on the check stage, so no tarball comes out of a build whose codecs fail.+ARG BASE=public.ecr.aws/lambda/nodejs:22@sha256:d138eff445f6ea04c51091d8e6cbb064d5663e850cddfa62396061586f98b126++FROM ${BASE} AS build+ENV SOURCE_DATE_EPOCH=1722643200+RUN rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \+ microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \+ gcc gcc-c++ make cmake meson ninja-build autoconf automake libtool pkgconf-pkg-config diffutils \+ git xz bzip2 tar gzip which patchelf gnutls-devel zlib-devel $([ "$(uname -m)" = x86_64 ] && echo nasm) && \+ microdnf clean all && mkdir /build && mv /build-base-rpms.txt /build/base-rpms.txt+COPY sources.txt build-ffmpeg.sh /build/+RUN bash /build/build-ffmpeg.sh++FROM ${BASE} AS certs+# certtool, not openssl: the base's openssl-libs cannot run the repo's openssl CLI.+RUN microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \+ printf 'cn = check-ca\nca\ncert_signing_key\nexpiration_days = 2\n' >ca.tmpl && \+ printf 'cn = localhost\ndns_name = localhost\ntls_www_server\nexpiration_days = 2\n' >server.tmpl && \+ certtool -p --outfile ca.key && certtool -s --load-privkey ca.key --template ca.tmpl --outfile ca.crt && \+ certtool -p --outfile server.key && \+ certtool -c --load-privkey server.key --load-ca-certificate ca.crt --load-ca-privkey ca.key \+ --template server.tmpl --outfile server.crt++FROM ${BASE} AS check+COPY --from=build /opt/hf-media /opt/hf-media+COPY --from=certs /certs /certs+COPY check-ffmpeg.sh /check-ffmpeg.sh+RUN cp /certs/ca.crt /etc/pki/ca-trust/source/anchors/ && update-ca-trust && bash /check-ffmpeg.sh >/check.txt++FROM scratch AS artifact+COPY --from=build /out/ /+COPY --from=check /check.txt /hf-media-check.txtdiff --git a/packages/aws-lambda/image/build-ffmpeg.sh b/packages/aws-lambda/image/build-ffmpeg.shnew file mode 100755index 0000000000..65492cfe22--- /dev/null+++ b/packages/aws-lambda/image/build-ffmpeg.sh@@ -0,0 +1,92 @@+#!/bin/bash+# Builds ffmpeg + ffprobe into /opt/hf-media and packs /out/hf-media-linux-<arch>.tar.gz.+# Codec libraries link statically; glibc and the TLS stack stay dynamic (a static glibc build crashed on Lambda).+set -euo pipefail++PREFIX=/opt/hf-media+DEPS=/build/deps+SRC=/build/src+JOBS=$(nproc)+export PKG_CONFIG_PATH="$DEPS/lib/pkgconfig:$DEPS/lib64/pkgconfig"+export CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC"+mkdir -p "$SRC" "$DEPS" "$PREFIX/share/licenses" /out++fetch() {+ local name=$1 sum url+ read -r sum url < <(awk -v n="$name" '$1 == n { print $2, $3 }' /build/sources.txt)+ curl -fsSL --retry 3 -o "$SRC/$name" "$url"+ echo "$sum $SRC/$name" | sha256sum -c --quiet+ tar -xf "$SRC/$name" -C "$SRC"+}++license() { mkdir -p "$PREFIX/share/licenses/$1" && cp "${@:2}" "$PREFIX/share/licenses/$1/"; }++read -r x264_commit x264_url < <(awk '$1 == "x264" { print $2, $3 }' /build/sources.txt)+git clone -q "$x264_url" "$SRC/x264"+git -C "$SRC/x264" checkout -q "$x264_commit"+[ "$(git -C "$SRC/x264" rev-parse HEAD)" = "$x264_commit" ]+(cd "$SRC/x264" && ./configure --prefix="$DEPS" --enable-static --enable-pic --disable-cli &&+ make -j"$JOBS" && make install)+license x264 "$SRC/x264/COPYING"++fetch x265_4.1.tar.gz+# 8-bit x265 with the 10-bit library linked in (x265's multilib layout): HDR renders encode yuv420p10le.+# SVE2 kernels need -march=armv9-a, which the base's gcc 11 does not know (x86 ignores the flag).+x265=(-S "$SRC/x265_4.1/source" -DCMAKE_BUILD_TYPE=Release -DENABLE_SHARED=OFF -DENABLE_CLI=OFF -DENABLE_SVE2=OFF)+cmake "${x265[@]}" -B "$SRC/x265-10" -DHIGH_BIT_DEPTH=ON -DEXPORT_C_API=OFF+cmake --build "$SRC/x265-10" -j"$JOBS"+cmake "${x265[@]}" -B "$SRC/x265-8" -DCMAKE_INSTALL_PREFIX="$DEPS" -DLINKED_10BIT=ON \+ -DEXTRA_LIB="$SRC/x265-10/libx265.a"+cmake --build "$SRC/x265-8" -j"$JOBS"+cmake --install "$SRC/x265-8"+printf 'CREATE %s\nADDLIB %s\nADDLIB %s\nSAVE\nEND\n' "$DEPS/lib/libx265.a" \+ "$SRC/x265-8/libx265.a" "$SRC/x265-10/libx265.a" | ar -M+license x265 "$SRC/x265_4.1/COPYING"++fetch libvpx-1.15.0.tar.gz+(cd "$SRC/libvpx-1.15.0" && ./configure --prefix="$DEPS" --enable-static --disable-shared --enable-pic \+ --enable-vp9-highbitdepth --disable-examples --disable-tools --disable-docs --disable-unit-tests &&+ make -j"$JOBS" && make install)+license libvpx "$SRC/libvpx-1.15.0/LICENSE" "$SRC/libvpx-1.15.0/PATENTS"++fetch opus-1.5.2.tar.gz+(cd "$SRC/opus-1.5.2" && ./configure --prefix="$DEPS" --enable-static --disable-shared --with-pic \+ --disable-doc --disable-extra-programs && make -j"$JOBS" && make install)+license opus "$SRC/opus-1.5.2/COPYING"++fetch zimg-3.0.5.tar.gz+(cd "$SRC/zimg-release-3.0.5" && ./autogen.sh && ./configure --prefix="$DEPS" --enable-static --disable-shared \+ --with-pic && make -j"$JOBS" && make install)+license zimg "$SRC/zimg-release-3.0.5/COPYING"++fetch dav1d-1.5.1.tar.xz+meson setup "$SRC/dav1d-build" "$SRC/dav1d-1.5.1" --prefix="$DEPS" --libdir=lib --buildtype=release \+ --default-library=static -Denable_tools=false -Denable_tests=false+ninja -C "$SRC/dav1d-build" install+license dav1d "$SRC/dav1d-1.5.1/COPYING"++fetch ffmpeg-7.0.2.tar.xz+# No --static pkg-config: gnutls.pc then names a libgmp.a from its own RPM build tree. The deps above+# exist only as .a, so they link statically anyway; --extra-libs carries their private libraries.+(cd "$SRC/ffmpeg-7.0.2" && ./configure --prefix="$PREFIX" \+ --extra-cflags="-I$DEPS/include" --extra-ldflags="-L$DEPS/lib" --extra-libs="-lstdc++ -lpthread -lm -ldl" \+ --enable-gpl --enable-version3 --disable-debug --disable-doc --disable-ffplay --disable-autodetect \+ --enable-gnutls --enable-zlib --enable-libx264 --enable-libx265 --enable-libvpx --enable-libopus \+ --enable-libzimg --enable-libdav1d && make -j"$JOBS" && make install)+license ffmpeg "$SRC/ffmpeg-7.0.2/COPYING.GPLv3" "$SRC/ffmpeg-7.0.2/LICENSE.md"+rm -rf "$PREFIX/include" "$PREFIX/lib" "$PREFIX/share/ffmpeg"+strip "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"++# Bundle every shared library the bare base image lacks (gnutls, nettle); the rpath finds them.+mkdir -p "$PREFIX/lib"+ldd "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe" | awk '$2 == "=>" { print $3 }' | sort -u | while read -r lib; do+ if ! grep -qx "$(rpm -qf --qf '%{NAME}' "$lib")" /build/base-rpms.txt; then+ cp -L "$lib" "$PREFIX/lib/" && patchelf --set-rpath '$ORIGIN' "$PREFIX/lib/$(basename "$lib")"+ echo "$(rpm -qf "$lib") $(basename "$lib")" >>"$PREFIX/share/licenses/bundled-libraries.txt"+ fi+done+patchelf --set-rpath '$ORIGIN/../lib' "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"+cp /build/sources.txt "$PREFIX/share/licenses/sources.txt"++tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --numeric-owner -C /opt -cf - hf-media |+ gzip -n -9 >"/out/hf-media-linux-$(uname -m).tar.gz"diff --git a/packages/aws-lambda/image/check-ffmpeg.sh b/packages/aws-lambda/image/check-ffmpeg.shnew file mode 100755index 0000000000..3aa601ebbd--- /dev/null+++ b/packages/aws-lambda/image/check-ffmpeg.sh@@ -0,0 +1,61 @@+#!/bin/bash+# Runs on the bare Lambda base with only /opt/hf-media added: every codec the engine calls must work.+# Prints a deterministic summary to stdout; the artifact stage keeps it.+set -euo pipefail+ff() { /opt/hf-media/bin/ffmpeg -hide_banner -loglevel error -nostdin "$@"; }+probe() { local input=$1; shift; /opt/hf-media/bin/ffprobe -hide_banner -loglevel error "$@" -- "$input"; }+listed() { # listed <ffmpeg list flag> <awk field> <name...>+ local names+ names=$(/opt/hf-media/bin/ffmpeg -hide_banner "$1" | awk -v f="$2" '{ print $f }')+ for name in "${@:3}"; do grep -qx -- "$name" <<<"$names" || { echo "missing $1 $name" >&2; exit 1; }; done+ echo "${1#-}: ${*:3}"+}+frames() { probe "$1" -count_frames -select_streams v:0 -show_entries stream=nb_read_frames -of csv=p=0 "${@:2}"; }++cd "$(mktemp -d)"+listed -encoders 2 libx264 libx265 libvpx-vp9 libopus prores_ks aac png mjpeg+listed -decoders 2 libvpx-vp9 libvpx libdav1d h264 hevc prores aac opus png mjpeg+listed -filters 2 zscale tonemap setparams ebur128 alphamerge overlay+listed -protocols 1 https tls file++src=(-f lavfi -i "testsrc2=s=320x240:r=25:d=1")+ff "${src[@]}" -c:v libx264 -pix_fmt yuv420p -bf 0 h264.mp4+ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 -x265-params log-level=error h265.mp4+ff "${src[@]}" -c:v prores_ks -profile:v 4444 -pix_fmt yuva444p10le prores.mov+ff -f lavfi -i "sine=d=1" -c:a libopus -b:a 128k opus.webm+ff -f lavfi -i "sine=d=1" -c:a aac -b:a 192k aac.m4a+ff "${src[@]}" -frames:v 1 frame.png+for f in h264.mp4 h265.mp4 prores.mov; do [ "$(frames "$f")" = 25 ] || { echo "decode $f" >&2; exit 1; }; done+for f in opus.webm aac.m4a; do ff -i "$f" -f null -; done+[ "$(probe frame.png -show_entries stream=width -of csv=p=0)" = 320 ]+echo "encode+decode: h264 h265 prores opus aac png"++# VP9 with alpha: left half transparent, decoded through libvpx-vp9 as the engine does.+ff "${src[@]}" -vf "format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='if(lt(X,W/2),0,255)'" \+ -c:v libvpx-vp9 -pix_fmt yuva420p -auto-alt-ref 0 -b:v 0 -crf 30 alpha.webm+ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:20:100 -f rawvideo -pix_fmt gray left.raw+ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:260:100 -f rawvideo -pix_fmt gray right.raw+node -e '+ const fs = require("fs"), mean = (f) => fs.readFileSync(f).reduce((a, b) => a + b, 0) / 1600;+ if (!(mean("left.raw") < 16 && mean("right.raw") > 240)) throw new Error("vp9 alpha lost");'+echo "vp9 alpha: encode+decode"++# The engine's HDR to SDR chain (core/src/hdrToneMap.ts) on tagged 10-bit HEVC.+ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p10le -color_primaries bt2020 -color_trc smpte2084 \+ -colorspace bt2020nc -x265-params log-level=error hdr.mp4+[ "$(probe hdr.mp4 -select_streams v:0 -show_entries stream=pix_fmt -of csv=p=0)" = yuv420p10le ]+ff -i hdr.mp4 -vf "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv" \+ -f null -+echo "hdr: 10-bit hevc encode, zscale tonemap"++# TLS against the system trust store: a trusted host passes, a wrong host name is refused.+exec 4< <(node -e '+ const fs = require("fs");+ require("https").createServer({ key: fs.readFileSync("/certs/server.key"), cert: fs.readFileSync("/certs/server.crt") },+ (_, res) => res.end(fs.readFileSync("h264.mp4"))).listen(8443, () => console.log("ready"));')+server=$!+trap 'kill $server' EXIT+read -r -t 30 -u 4 ready && [ "$ready" = ready ]+[ "$(frames https://localhost:8443/h264.mp4 -tls_verify 1)" = 25 ]+if frames https://127.0.0.1:8443/h264.mp4 -tls_verify 1 2>/dev/null; then echo "tls accepted a wrong host" >&2; exit 1; fi+echo "tls: verified, wrong host refused"diff --git a/packages/aws-lambda/image/sources.txt b/packages/aws-lambda/image/sources.txtnew file mode 100644index 0000000000..66e2683544--- /dev/null+++ b/packages/aws-lambda/image/sources.txt@@ -0,0 +1,8 @@+# name sha256 url (x264 has no stable tarballs: its line pins a git commit, r3191 as in the x86 ffmpeg-static)+ffmpeg-7.0.2.tar.xz 8646515b638a3ad303e23af6a3587734447cb8fc0a0c064ecdb8e95c4fd8b389 https://ffmpeg.org/releases/ffmpeg-7.0.2.tar.xz+x264 4613ac3c15fd75cebc4b9f65b7fb95e70a3acce1 https://code.videolan.org/videolan/x264.git+x265_4.1.tar.gz a31699c6a89806b74b0151e5e6a7df65de4b49050482fe5ebf8a4379d7af8f29 https://bitbucket.org/multicoreware/x265_git/downloads/x265_4.1.tar.gz+libvpx-1.15.0.tar.gz e935eded7d81631a538bfae703fd1e293aad1c7fd3407ba00440c95105d2011e https://github.com/webmproject/libvpx/archive/refs/tags/v1.15.0.tar.gz+opus-1.5.2.tar.gz 65c1d2f78b9f2fb20082c38cbe47c951ad5839345876e46941612ee87f9a7ce1 https://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz+zimg-3.0.5.tar.gz a9a0226bf85e0d83c41a8ebe4e3e690e1348682f6a2a7838f1b8cbff1b799bcf https://github.com/sekrit-twc/zimg/archive/refs/tags/release-3.0.5.tar.gz+dav1d-1.5.1.tar.xz 401813f1f89fa8fd4295805aa5284d9aed9bc7fc1fdbe554af4292f64cbabe21 https://downloads.videolan.org/pub/videolan/dav1d/1.5.1/dav1d-1.5.1.tar.xz