Skip to content

A feature-rich command-line audio/video downloader

[build] Refactor Linux build jobs (#14275)

bashonlye2d37bcc8e84
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8e382f9007..5371bf140e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -84,6 +84,8 @@ jobs:
origin: ${{ steps.process_inputs.outputs.origin }}
timestamp: ${{ steps.process_inputs.outputs.timestamp }}
version: ${{ steps.process_inputs.outputs.version }}
+ linux_matrix: ${{ steps.linux_matrix.outputs.matrix }}
+
steps:
- name: Process inputs
id: process_inputs
@@ -118,6 +120,69 @@ jobs:
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
f.write('\n'.join(f'{key}={value}' for key, value in outputs.items()))
+ - name: Build Linux matrix
+ id: linux_matrix
+ env:
+ INPUTS: ${{ toJSON(inputs) }}
+ PYTHON_VERSION: '3.13'
+ UPDATE_TO: yt-dlp/[email protected]
+ shell: python
+ run: |
+ import json
+ import os
+ EXE_MAP = {
+ 'linux': [{
+ 'os': 'linux',
+ 'arch': 'x86_64',
+ 'runner': 'ubuntu-24.04',
+ }, {
+ 'os': 'linux',
+ 'arch': 'aarch64',
+ 'runner': 'ubuntu-24.04-arm',
+ }],
+ 'linux_armv7l': [{
+ 'os': 'linux',
+ 'arch': 'armv7l',
+ 'runner': 'ubuntu-24.04-arm',
+ 'qemu_platform': 'linux/arm/v7',
+ 'onefile': False,
+ 'cache_requirements': True,
+ 'update_to': 'yt-dlp/[email protected]',
+ }],
+ 'musllinux': [{
+ 'os': 'musllinux',
+ 'arch': 'x86_64',
+ 'runner': 'ubuntu-24.04',
+ }, {
+ 'os': 'musllinux',
+ 'arch': 'aarch64',
+ 'runner': 'ubuntu-24.04-arm',
+ }],
+ }
+ INPUTS = json.loads(os.environ['INPUTS'])
+ matrix = [exe for key, group in EXE_MAP.items() for exe in group if INPUTS.get(key)]
+ if not matrix:
+ # If we send an empty matrix when no linux inputs are given, the entire workflow fails
+ matrix = [EXE_MAP['linux'][0]]
+ for exe in matrix:
+ exe['exe'] = '_'.join(filter(None, (
+ 'yt-dlp',
+ exe['os'],
+ exe['arch'] != 'x86_64' and exe['arch'],
+ )))
+ exe.setdefault('qemu_platform', None)
+ exe.setdefault('onefile', True)
+ exe.setdefault('onedir', True)
+ exe.setdefault('cache_requirements', False)
+ exe.setdefault('python_version', os.environ['PYTHON_VERSION'])
+ exe.setdefault('update_to', os.environ['UPDATE_TO'])
+ if not any(INPUTS.get(key) for key in EXE_MAP):
+ print('skipping linux job')
+ else:
+ print(json.dumps(matrix, indent=2))
+ with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+ f.write(f'matrix={json.dumps(matrix)}')
+
unix:
needs: process
if: inputs.unix
@@ -127,24 +192,30 @@ jobs:
ORIGIN: ${{ needs.process.outputs.origin }}
VERSION: ${{ needs.process.outputs.version }}
UPDATE_TO: yt-dlp/[email protected]
+
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Needed for changelog
+
- uses: actions/setup-python@v6
with:
python-version: "3.10"
+
- name: Install Requirements
run: |
sudo apt -y install zip pandoc man sed
+
- name: Prepare
run: |
python devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}"
python devscripts/update_changelog.py -vv
python devscripts/make_lazy_extractors.py
+
- name: Build Unix platform-independent binary
run: |
make all tar
+
- name: Verify --update-to
if: vars.UPDATE_TO_VERIFICATION
run: |
@@ -154,6 +225,7 @@ jobs:
./yt-dlp_downgraded -v --update-to "${UPDATE_TO}"
downgraded_version="$(./yt-dlp_downgraded --version)"
[[ "${version}" != "${downgraded_version}" ]]
+
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
@@ -164,154 +236,71 @@ jobs:
compression-level: 0
linux:
+ name: ${{ matrix.os }} (${{ matrix.arch }})
+ if: inputs.linux || inputs.linux_armv7l || inputs.musllinux
needs: process
- if: inputs.linux
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
- include:
- - exe: yt-dlp_linux
- platform: x86_64
- runner: ubuntu-24.04
- - exe: yt-dlp_linux_aarch64
- platform: aarch64
- runner: ubuntu-24.04-arm
+ include: ${{ fromJSON(needs.process.outputs.linux_matrix) }}
env:
CHANNEL: ${{ inputs.channel }}
ORIGIN: ${{ needs.process.outputs.origin }}
VERSION: ${{ needs.process.outputs.version }}
EXE_NAME: ${{ matrix.exe }}
- UPDATE_TO: yt-dlp/[email protected]
- steps:
- - uses: actions/checkout@v4
- - name: Build executable
- env:
- SERVICE: linux_${{ matrix.platform }}
- run: |
- mkdir -p ./dist
- pushd bundle/docker
- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"
- popd
- sudo chown "${USER}:docker" "./dist/${EXE_NAME}"
- - name: Verify executable in container
- if: vars.UPDATE_TO_VERIFICATION
- env:
- SERVICE: linux_${{ matrix.platform }}_verify
- run: |
- cd bundle/docker
- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"
- - name: Verify --update-to
- if: vars.UPDATE_TO_VERIFICATION
- run: |
- chmod +x "./dist/${EXE_NAME}"
- mkdir -p ~/testing
- cp "./dist/${EXE_NAME}" ~/testing/"${EXE_NAME}_downgraded"
- version="$("./dist/${EXE_NAME}" --version)"
- ~/testing/"${EXE_NAME}_downgraded" -v --update-to "${UPDATE_TO}"
- downgraded_version="$(~/testing/"${EXE_NAME}_downgraded" --version)"
- [[ "${version}" != "${downgraded_version}" ]]
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: build-bin-${{ github.job }}_${{ matrix.platform }}
- path: |
- dist/${{ matrix.exe }}*
- compression-level: 0
+ PYTHON_VERSION: ${{ matrix.python_version }}
+ UPDATE_TO: ${{ (vars.UPDATE_TO_VERIFICATION && matrix.update_to) || '' }}
+ SKIP_ONEDIR_BUILD: ${{ (!matrix.onedir && '1') || '' }}
+ SKIP_ONEFILE_BUILD: ${{ (!matrix.onefile && '1') || '' }}
- linux_armv7l:
- needs: process
- if: inputs.linux_armv7l
- permissions:
- contents: read
- runs-on: ubuntu-24.04-arm
- env:
- CHANNEL: ${{ inputs.channel }}
- ORIGIN: ${{ needs.process.outputs.origin }}
- VERSION: ${{ needs.process.outputs.version }}
- EXE_NAME: yt-dlp_linux_armv7l
steps:
- uses: actions/checkout@v4
+
- name: Cache requirements
+ if: matrix.cache_requirements
id: cache-venv
uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1
with:
path: |
- ~/yt-dlp-build-venv
- key: cache-reqs-${{ github.job }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }}
+ venv
+ key: cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }}
restore-keys: |
- cache-reqs-${{ github.job }}-${{ github.ref }}-
- cache-reqs-${{ github.job }}-
+ cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}-
+ cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-
+
- name: Set up QEMU
+ if: matrix.qemu_platform
uses: docker/setup-qemu-action@v3
with:
- platforms: linux/arm/v7
- - name: Build executable
- env:
- SERVICE: linux_armv7l
- run: |
- mkdir -p ./dist
- mkdir -p ~/yt-dlp-build-venv
- cd bundle/docker
- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"
- - name: Verify executable in container
- if: vars.UPDATE_TO_VERIFICATION
- env:
- SERVICE: linux_armv7l_verify
- run: |
- cd bundle/docker
- docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: build-bin-${{ github.job }}
- path: |
- dist/yt-dlp_linux_armv7l.zip
- compression-level: 0
+ platforms: ${{ matrix.qemu_platform }}
- musllinux:
- needs: process
- if: inputs.musllinux
- runs-on: ${{ matrix.runner }}
- strategy:
- fail-fast: false
- matrix:
- include:
- - exe: yt-dlp_musllinux
- platform: x86_64
- runner: ubuntu-24.04
- - exe: yt-dlp_musllinux_aarch64
- platform: aarch64
- runner: ubuntu-24.04-arm
- env:
- CHANNEL: ${{ inputs.channel }}
- ORIGIN: ${{ needs.process.outputs.origin }}
- VERSION: ${{ needs.process.outputs.version }}
- EXE_NAME: ${{ matrix.exe }}
- steps:
- - uses: actions/checkout@v4
- name: Build executable
env:
- SERVICE: musllinux_${{ matrix.platform }}
+ SERVICE: ${{ matrix.os }}_${{ matrix.arch }}
run: |
+ mkdir -p ./venv
mkdir -p ./dist
pushd bundle/docker
docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"
popd
- sudo chown "${USER}:docker" "./dist/${EXE_NAME}"
+ if [[ -z "${SKIP_ONEFILE_BUILD}" ]]; then
+ sudo chown "${USER}:docker" "./dist/${EXE_NAME}"
+ fi
+
- name: Verify executable in container
- if: vars.UPDATE_TO_VERIFICATION
env:
- SERVICE: musllinux_${{ matrix.platform }}_verify
+ SERVICE: ${{ matrix.os }}_${{ matrix.arch }}_verify
run: |
cd bundle/docker
docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"
+
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
- name: build-bin-${{ github.job }}_${{ matrix.platform }}
+ name: build-bin-${{ matrix.os }}_${{ matrix.arch }}
path: |
dist/${{ matrix.exe }}*
compression-level: 0
@@ -542,8 +531,6 @@ jobs:
- process
- unix
- linux
- - linux_armv7l
- - musllinux
- macos
- windows
runs-on: ubuntu-latest
diff --git a/bundle/docker/compose.yml b/bundle/docker/compose.yml
index 77062f594a..19a011d7a2 100644
--- a/bundle/docker/compose.yml
+++ b/bundle/docker/compose.yml
@@ -13,6 +13,9 @@ services:
CHANNEL: ${CHANNEL:?}
ORIGIN: ${ORIGIN:?}
VERSION:
+ PYTHON_VERSION:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../..:/yt-dlp
@@ -27,6 +30,8 @@ services:
environment:
EXE_NAME: ${EXE_NAME:?}
UPDATE_TO:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../../dist:/build
@@ -43,6 +48,9 @@ services:
CHANNEL: ${CHANNEL:?}
ORIGIN: ${ORIGIN:?}
VERSION:
+ PYTHON_VERSION:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../..:/yt-dlp
@@ -57,6 +65,8 @@ services:
environment:
EXE_NAME: ${EXE_NAME:?}
UPDATE_TO:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../../dist:/build
@@ -73,10 +83,12 @@ services:
CHANNEL: ${CHANNEL:?}
ORIGIN: ${ORIGIN:?}
VERSION:
- SKIP_ONEFILE_BUILD: "1"
+ PYTHON_VERSION:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../..:/yt-dlp
- - ~/yt-dlp-build-venv:/yt-dlp-build-venv
+ - ../../venv:/yt-dlp-build-venv
linux_armv7l_verify:
build:
@@ -89,7 +101,8 @@ services:
environment:
EXE_NAME: ${EXE_NAME:?}
UPDATE_TO:
- TEST_ONEDIR_BUILD: "1"
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../../dist:/build
@@ -106,6 +119,9 @@ services:
CHANNEL: ${CHANNEL:?}
ORIGIN: ${ORIGIN:?}
VERSION:
+ PYTHON_VERSION:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../..:/yt-dlp
@@ -120,6 +136,8 @@ services:
environment:
EXE_NAME: ${EXE_NAME:?}
UPDATE_TO:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../../dist:/build
@@ -136,6 +154,9 @@ services:
CHANNEL: ${CHANNEL:?}
ORIGIN: ${ORIGIN:?}
VERSION:
+ PYTHON_VERSION:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
EXCLUDE_CURL_CFFI: "1"
volumes:
- ../..:/yt-dlp
@@ -151,5 +172,7 @@ services:
environment:
EXE_NAME: ${EXE_NAME:?}
UPDATE_TO:
+ SKIP_ONEDIR_BUILD:
+ SKIP_ONEFILE_BUILD:
volumes:
- ../../dist:/build
diff --git a/bundle/docker/linux/build.sh b/bundle/docker/linux/build.sh
index 1ce330a5b0..b278ebf60f 100755
--- a/bundle/docker/linux/build.sh
+++ b/bundle/docker/linux/build.sh
@@ -1,16 +1,17 @@
#!/bin/bash
set -exuo pipefail
-if [[ -z "${USE_PYTHON_VERSION:-}" ]]; then
- USE_PYTHON_VERSION="3.13"
+if [[ -z "${PYTHON_VERSION:-}" ]]; then
+ PYTHON_VERSION="3.13"
+ echo "Defaulting to using Python ${PYTHON_VERSION}"
fi
function runpy {
- "/opt/shared-cpython-${USE_PYTHON_VERSION}/bin/python${USE_PYTHON_VERSION}" "$@"
+ "/opt/shared-cpython-${PYTHON_VERSION}/bin/python${PYTHON_VERSION}" "$@"
}
function venvpy {
- "python${USE_PYTHON_VERSION}" "$@"
+ "python${PYTHON_VERSION}" "$@"
}
INCLUDES=(
diff --git a/bundle/docker/linux/verify.sh b/bundle/docker/linux/verify.sh
index 062a576f9d..ecb071a8c5 100755
--- a/bundle/docker/linux/verify.sh
+++ b/bundle/docker/linux/verify.sh
@@ -1,7 +1,11 @@
#!/bin/sh
set -eu
-if [ -n "${TEST_ONEDIR_BUILD:-}" ]; then
+if [ -n "${SKIP_ONEFILE_BUILD:-}" ]; then
+ if [ -n "${SKIP_ONEDIR_BUILD:-}" ]; then
+ echo "All executable builds were skipped"
+ exit 1
+ fi
echo "Extracting zip to verify onedir build"
if command -v python3 >/dev/null 2>&1; then
python3 -m zipfile -e "/build/${EXE_NAME}.zip" ./
@@ -22,23 +26,22 @@ if [ -n "${TEST_ONEDIR_BUILD:-}" ]; then
fi
unzip "/build/${EXE_NAME}.zip" -d ./
fi
-else
- echo "Verifying onefile build"
- cp "/build/${EXE_NAME}" ./
+ chmod +x "./${EXE_NAME}"
+ "./${EXE_NAME}" -v || true
+ "./${EXE_NAME}" --version
+ exit 0
fi
+echo "Verifying onefile build"
+cp "/build/${EXE_NAME}" ./
chmod +x "./${EXE_NAME}"
-if [ -n "${SKIP_UPDATE_TO:-}" ] || [ -n "${TEST_ONEDIR_BUILD:-}" ]; then
+if [ -z "${UPDATE_TO:-}" ]; then
"./${EXE_NAME}" -v || true
"./${EXE_NAME}" --version
exit 0
fi
-if [ -z "${UPDATE_TO:-}" ]; then
- UPDATE_TO="yt-dlp/[email protected]"
-fi
-
cp "./${EXE_NAME}" "./${EXE_NAME}_downgraded"
version="$("./${EXE_NAME}" --version)"
"./${EXE_NAME}_downgraded" -v --update-to "${UPDATE_TO}"