A feature-rich command-line audio/video downloader
[build] Refactor Linux build jobs (#14275)
bashonlye2d37bcc8e84
diff --git a/.github/workflows/build.yml b/.github/workflows/build.ymlindex 8e382f9007..5371bf140e 100644--- a/.github/workflows/build.yml+++ b/.github/workflows/build.yml@@ -84,6 +84,8 @@ jobs:origin: ${{ steps.process_inputs.outputs.origin }}timestamp: ${{ steps.process_inputs.outputs.timestamp }}version: ${{ steps.process_inputs.outputs.version }}+ linux_matrix: ${{ steps.linux_matrix.outputs.matrix }}+steps:- name: Process inputsid: process_inputs@@ -118,6 +120,69 @@ jobs:with open(os.environ['GITHUB_OUTPUT'], 'a') as f:f.write('\n'.join(f'{key}={value}' for key, value in outputs.items()))+ - name: Build Linux matrix+ id: linux_matrix+ env:+ INPUTS: ${{ toJSON(inputs) }}+ PYTHON_VERSION: '3.13'+ UPDATE_TO: yt-dlp/[email protected]+ shell: python+ run: |+ import json+ import os+ EXE_MAP = {+ 'linux': [{+ 'os': 'linux',+ 'arch': 'x86_64',+ 'runner': 'ubuntu-24.04',+ }, {+ 'os': 'linux',+ 'arch': 'aarch64',+ 'runner': 'ubuntu-24.04-arm',+ }],+ 'linux_armv7l': [{+ 'os': 'linux',+ 'arch': 'armv7l',+ 'runner': 'ubuntu-24.04-arm',+ 'qemu_platform': 'linux/arm/v7',+ 'onefile': False,+ 'cache_requirements': True,+ 'update_to': 'yt-dlp/[email protected]',+ }],+ 'musllinux': [{+ 'os': 'musllinux',+ 'arch': 'x86_64',+ 'runner': 'ubuntu-24.04',+ }, {+ 'os': 'musllinux',+ 'arch': 'aarch64',+ 'runner': 'ubuntu-24.04-arm',+ }],+ }+ INPUTS = json.loads(os.environ['INPUTS'])+ matrix = [exe for key, group in EXE_MAP.items() for exe in group if INPUTS.get(key)]+ if not matrix:+ # If we send an empty matrix when no linux inputs are given, the entire workflow fails+ matrix = [EXE_MAP['linux'][0]]+ for exe in matrix:+ exe['exe'] = '_'.join(filter(None, (+ 'yt-dlp',+ exe['os'],+ exe['arch'] != 'x86_64' and exe['arch'],+ )))+ exe.setdefault('qemu_platform', None)+ exe.setdefault('onefile', True)+ exe.setdefault('onedir', True)+ exe.setdefault('cache_requirements', False)+ exe.setdefault('python_version', os.environ['PYTHON_VERSION'])+ exe.setdefault('update_to', os.environ['UPDATE_TO'])+ if not any(INPUTS.get(key) for key in EXE_MAP):+ print('skipping linux job')+ else:+ print(json.dumps(matrix, indent=2))+ with open(os.environ['GITHUB_OUTPUT'], 'a') as f:+ f.write(f'matrix={json.dumps(matrix)}')+unix:needs: processif: inputs.unix@@ -127,24 +192,30 @@ jobs:ORIGIN: ${{ needs.process.outputs.origin }}VERSION: ${{ needs.process.outputs.version }}UPDATE_TO: yt-dlp/[email protected]+steps:- uses: actions/checkout@v4with:fetch-depth: 0 # Needed for changelog+- uses: actions/setup-python@v6with:python-version: "3.10"+- name: Install Requirementsrun: |sudo apt -y install zip pandoc man sed+- name: Preparerun: |python devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}"python devscripts/update_changelog.py -vvpython devscripts/make_lazy_extractors.py+- name: Build Unix platform-independent binaryrun: |make all tar+- name: Verify --update-toif: vars.UPDATE_TO_VERIFICATIONrun: |@@ -154,6 +225,7 @@ jobs:./yt-dlp_downgraded -v --update-to "${UPDATE_TO}"downgraded_version="$(./yt-dlp_downgraded --version)"[[ "${version}" != "${downgraded_version}" ]]+- name: Upload artifactsuses: actions/upload-artifact@v4with:@@ -164,154 +236,71 @@ jobs:compression-level: 0linux:+ name: ${{ matrix.os }} (${{ matrix.arch }})+ if: inputs.linux || inputs.linux_armv7l || inputs.musllinuxneeds: process- if: inputs.linuxruns-on: ${{ matrix.runner }}strategy:fail-fast: falsematrix:- include:- - exe: yt-dlp_linux- platform: x86_64- runner: ubuntu-24.04- - exe: yt-dlp_linux_aarch64- platform: aarch64- runner: ubuntu-24.04-arm+ include: ${{ fromJSON(needs.process.outputs.linux_matrix) }}env:CHANNEL: ${{ inputs.channel }}ORIGIN: ${{ needs.process.outputs.origin }}VERSION: ${{ needs.process.outputs.version }}EXE_NAME: ${{ matrix.exe }}- UPDATE_TO: yt-dlp/[email protected]- steps:- - uses: actions/checkout@v4- - name: Build executable- env:- SERVICE: linux_${{ matrix.platform }}- run: |- mkdir -p ./dist- pushd bundle/docker- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"- popd- sudo chown "${USER}:docker" "./dist/${EXE_NAME}"- - name: Verify executable in container- if: vars.UPDATE_TO_VERIFICATION- env:- SERVICE: linux_${{ matrix.platform }}_verify- run: |- cd bundle/docker- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"- - name: Verify --update-to- if: vars.UPDATE_TO_VERIFICATION- run: |- chmod +x "./dist/${EXE_NAME}"- mkdir -p ~/testing- cp "./dist/${EXE_NAME}" ~/testing/"${EXE_NAME}_downgraded"- version="$("./dist/${EXE_NAME}" --version)"- ~/testing/"${EXE_NAME}_downgraded" -v --update-to "${UPDATE_TO}"- downgraded_version="$(~/testing/"${EXE_NAME}_downgraded" --version)"- [[ "${version}" != "${downgraded_version}" ]]- - name: Upload artifacts- uses: actions/upload-artifact@v4- with:- name: build-bin-${{ github.job }}_${{ matrix.platform }}- path: |- dist/${{ matrix.exe }}*- compression-level: 0+ PYTHON_VERSION: ${{ matrix.python_version }}+ UPDATE_TO: ${{ (vars.UPDATE_TO_VERIFICATION && matrix.update_to) || '' }}+ SKIP_ONEDIR_BUILD: ${{ (!matrix.onedir && '1') || '' }}+ SKIP_ONEFILE_BUILD: ${{ (!matrix.onefile && '1') || '' }}- linux_armv7l:- needs: process- if: inputs.linux_armv7l- permissions:- contents: read- runs-on: ubuntu-24.04-arm- env:- CHANNEL: ${{ inputs.channel }}- ORIGIN: ${{ needs.process.outputs.origin }}- VERSION: ${{ needs.process.outputs.version }}- EXE_NAME: yt-dlp_linux_armv7lsteps:- uses: actions/checkout@v4+- name: Cache requirements+ if: matrix.cache_requirementsid: cache-venvuses: actions/cache@v4env:SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1with:path: |- ~/yt-dlp-build-venv- key: cache-reqs-${{ github.job }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }}+ venv+ key: cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }}restore-keys: |- cache-reqs-${{ github.job }}-${{ github.ref }}-- cache-reqs-${{ github.job }}-+ cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}-+ cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-+- name: Set up QEMU+ if: matrix.qemu_platformuses: docker/setup-qemu-action@v3with:- platforms: linux/arm/v7- - name: Build executable- env:- SERVICE: linux_armv7l- run: |- mkdir -p ./dist- mkdir -p ~/yt-dlp-build-venv- cd bundle/docker- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"- - name: Verify executable in container- if: vars.UPDATE_TO_VERIFICATION- env:- SERVICE: linux_armv7l_verify- run: |- cd bundle/docker- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"- - name: Upload artifacts- uses: actions/upload-artifact@v4- with:- name: build-bin-${{ github.job }}- path: |- dist/yt-dlp_linux_armv7l.zip- compression-level: 0+ platforms: ${{ matrix.qemu_platform }}- musllinux:- needs: process- if: inputs.musllinux- runs-on: ${{ matrix.runner }}- strategy:- fail-fast: false- matrix:- include:- - exe: yt-dlp_musllinux- platform: x86_64- runner: ubuntu-24.04- - exe: yt-dlp_musllinux_aarch64- platform: aarch64- runner: ubuntu-24.04-arm- env:- CHANNEL: ${{ inputs.channel }}- ORIGIN: ${{ needs.process.outputs.origin }}- VERSION: ${{ needs.process.outputs.version }}- EXE_NAME: ${{ matrix.exe }}- steps:- - uses: actions/checkout@v4- name: Build executableenv:- SERVICE: musllinux_${{ matrix.platform }}+ SERVICE: ${{ matrix.os }}_${{ matrix.arch }}run: |+ mkdir -p ./venvmkdir -p ./distpushd bundle/dockerdocker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"popd- sudo chown "${USER}:docker" "./dist/${EXE_NAME}"+ if [[ -z "${SKIP_ONEFILE_BUILD}" ]]; then+ sudo chown "${USER}:docker" "./dist/${EXE_NAME}"+ fi+- name: Verify executable in container- if: vars.UPDATE_TO_VERIFICATIONenv:- SERVICE: musllinux_${{ matrix.platform }}_verify+ SERVICE: ${{ matrix.os }}_${{ matrix.arch }}_verifyrun: |cd bundle/dockerdocker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"+- name: Upload artifactsuses: actions/upload-artifact@v4with:- name: build-bin-${{ github.job }}_${{ matrix.platform }}+ name: build-bin-${{ matrix.os }}_${{ matrix.arch }}path: |dist/${{ matrix.exe }}*compression-level: 0@@ -542,8 +531,6 @@ jobs:- process- unix- linux- - linux_armv7l- - musllinux- macos- windowsruns-on: ubuntu-latestdiff --git a/bundle/docker/compose.yml b/bundle/docker/compose.ymlindex 77062f594a..19a011d7a2 100644--- a/bundle/docker/compose.yml+++ b/bundle/docker/compose.yml@@ -13,6 +13,9 @@ services:CHANNEL: ${CHANNEL:?}ORIGIN: ${ORIGIN:?}VERSION:+ PYTHON_VERSION:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../..:/yt-dlp@@ -27,6 +30,8 @@ services:environment:EXE_NAME: ${EXE_NAME:?}UPDATE_TO:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../../dist:/build@@ -43,6 +48,9 @@ services:CHANNEL: ${CHANNEL:?}ORIGIN: ${ORIGIN:?}VERSION:+ PYTHON_VERSION:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../..:/yt-dlp@@ -57,6 +65,8 @@ services:environment:EXE_NAME: ${EXE_NAME:?}UPDATE_TO:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../../dist:/build@@ -73,10 +83,12 @@ services:CHANNEL: ${CHANNEL:?}ORIGIN: ${ORIGIN:?}VERSION:- SKIP_ONEFILE_BUILD: "1"+ PYTHON_VERSION:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../..:/yt-dlp- - ~/yt-dlp-build-venv:/yt-dlp-build-venv+ - ../../venv:/yt-dlp-build-venvlinux_armv7l_verify:build:@@ -89,7 +101,8 @@ services:environment:EXE_NAME: ${EXE_NAME:?}UPDATE_TO:- TEST_ONEDIR_BUILD: "1"+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../../dist:/build@@ -106,6 +119,9 @@ services:CHANNEL: ${CHANNEL:?}ORIGIN: ${ORIGIN:?}VERSION:+ PYTHON_VERSION:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../..:/yt-dlp@@ -120,6 +136,8 @@ services:environment:EXE_NAME: ${EXE_NAME:?}UPDATE_TO:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../../dist:/build@@ -136,6 +154,9 @@ services:CHANNEL: ${CHANNEL:?}ORIGIN: ${ORIGIN:?}VERSION:+ PYTHON_VERSION:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:EXCLUDE_CURL_CFFI: "1"volumes:- ../..:/yt-dlp@@ -151,5 +172,7 @@ services:environment:EXE_NAME: ${EXE_NAME:?}UPDATE_TO:+ SKIP_ONEDIR_BUILD:+ SKIP_ONEFILE_BUILD:volumes:- ../../dist:/builddiff --git a/bundle/docker/linux/build.sh b/bundle/docker/linux/build.shindex 1ce330a5b0..b278ebf60f 100755--- a/bundle/docker/linux/build.sh+++ b/bundle/docker/linux/build.sh@@ -1,16 +1,17 @@#!/bin/bashset -exuo pipefail-if [[ -z "${USE_PYTHON_VERSION:-}" ]]; then- USE_PYTHON_VERSION="3.13"+if [[ -z "${PYTHON_VERSION:-}" ]]; then+ PYTHON_VERSION="3.13"+ echo "Defaulting to using Python ${PYTHON_VERSION}"fifunction runpy {- "/opt/shared-cpython-${USE_PYTHON_VERSION}/bin/python${USE_PYTHON_VERSION}" "$@"+ "/opt/shared-cpython-${PYTHON_VERSION}/bin/python${PYTHON_VERSION}" "$@"}function venvpy {- "python${USE_PYTHON_VERSION}" "$@"+ "python${PYTHON_VERSION}" "$@"}INCLUDES=(diff --git a/bundle/docker/linux/verify.sh b/bundle/docker/linux/verify.shindex 062a576f9d..ecb071a8c5 100755--- a/bundle/docker/linux/verify.sh+++ b/bundle/docker/linux/verify.sh@@ -1,7 +1,11 @@#!/bin/shset -eu-if [ -n "${TEST_ONEDIR_BUILD:-}" ]; then+if [ -n "${SKIP_ONEFILE_BUILD:-}" ]; then+ if [ -n "${SKIP_ONEDIR_BUILD:-}" ]; then+ echo "All executable builds were skipped"+ exit 1+ fiecho "Extracting zip to verify onedir build"if command -v python3 >/dev/null 2>&1; thenpython3 -m zipfile -e "/build/${EXE_NAME}.zip" ./@@ -22,23 +26,22 @@ if [ -n "${TEST_ONEDIR_BUILD:-}" ]; thenfiunzip "/build/${EXE_NAME}.zip" -d ./fi-else- echo "Verifying onefile build"- cp "/build/${EXE_NAME}" ./+ chmod +x "./${EXE_NAME}"+ "./${EXE_NAME}" -v || true+ "./${EXE_NAME}" --version+ exit 0fi+echo "Verifying onefile build"+cp "/build/${EXE_NAME}" ./chmod +x "./${EXE_NAME}"-if [ -n "${SKIP_UPDATE_TO:-}" ] || [ -n "${TEST_ONEDIR_BUILD:-}" ]; then+if [ -z "${UPDATE_TO:-}" ]; then"./${EXE_NAME}" -v || true"./${EXE_NAME}" --versionexit 0fi-if [ -z "${UPDATE_TO:-}" ]; then- UPDATE_TO="yt-dlp/[email protected]"-fi-cp "./${EXE_NAME}" "./${EXE_NAME}_downgraded"version="$("./${EXE_NAME}" --version)""./${EXE_NAME}_downgraded" -v --update-to "${UPDATE_TO}"