diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec5d4020ab..913e6f1de5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,10 +12,13 @@ on: unix: default: true type: boolean - linux_static: + linux: default: true type: boolean - linux_arm: + linux_armv7l: + default: true + type: boolean + musllinux: default: true type: boolean macos: @@ -52,12 +55,16 @@ on: description: yt-dlp, yt-dlp.tar.gz default: true type: boolean - linux_static: - description: yt-dlp_linux + linux: + description: yt-dlp_linux, yt-dlp_linux.zip, yt-dlp_linux_aarch64, yt-dlp_linux_aarch64.zip default: true type: boolean - linux_arm: - description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l + linux_armv7l: + description: yt-dlp_linux_armv7l.zip + default: true + type: boolean + musllinux: + description: yt-dlp_musllinux, yt-dlp_musllinux.zip, yt-dlp_musllinux_aarch64, yt-dlp_musllinux_aarch64.zip default: true type: boolean macos: @@ -134,99 +141,187 @@ jobs: yt-dlp.tar.gz compression-level: 0 - linux_static: + linux: needs: process - if: inputs.linux_static - runs-on: ubuntu-latest + if: inputs.linux + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - exe: yt-dlp_linux + platform: x86_64 + runner: ubuntu-24.04 + - exe: yt-dlp_linux_aarch64 + platform: aarch64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 - - name: Build static executable + - name: Build executable env: - channel: ${{ inputs.channel }} - origin: ${{ needs.process.outputs.origin }} - version: ${{ inputs.version }} + EXE_NAME: ${{ matrix.exe }} + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ inputs.version }} + SERVICE: linux_${{ matrix.platform }} run: | - mkdir ~/build + mkdir -p ~/build cd bundle/docker - docker compose up --build static - sudo chown "${USER}:docker" ~/build/yt-dlp_linux + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + docker compose down + sudo chown "${USER}:docker" ~/build/${{ matrix.exe }} + - name: Verify executable in container + if: vars.UPDATE_TO_VERIFICATION + env: + EXE_NAME: ${{ matrix.exe }} + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ inputs.version }} + SERVICE: linux_${{ matrix.platform }}_verify + run: | + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + docker compose down - name: Verify --update-to if: vars.UPDATE_TO_VERIFICATION run: | - chmod +x ~/build/yt-dlp_linux - cp ~/build/yt-dlp_linux ~/build/yt-dlp_linux_downgraded - version="$(~/build/yt-dlp_linux --version)" - ~/build/yt-dlp_linux_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 - downgraded_version="$(~/build/yt-dlp_linux_downgraded --version)" + chmod +x ~/build/${{ matrix.exe }} + mkdir -p ~/testing + cp ~/build/${{ matrix.exe }} ~/testing/${{ matrix.exe }}_downgraded + version="$(~/build/${{ matrix.exe }} --version)" + ~/testing/${{ matrix.exe }}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(~/testing/${{ matrix.exe }}_downgraded --version)" [[ "$version" != "$downgraded_version" ]] - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-bin-${{ github.job }} + name: build-bin-${{ github.job }}_${{ matrix.platform }} path: | - ~/build/yt-dlp_linux + ~/build/${{ matrix.exe }}* compression-level: 0 - linux_arm: + linux_armv7l: needs: process - if: inputs.linux_arm + if: inputs.linux_armv7l permissions: contents: read - packages: write # for creating cache - runs-on: ubuntu-latest - strategy: - matrix: - architecture: - - armv7 - - aarch64 - + actions: write # For cleaning up cache + runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 + + - name: Restore cached requirements + id: restore-cache + uses: actions/cache/restore@v4 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 + with: + path: | + ~/yt-dlp-build-venv + key: cache-reqs-${{ github.job }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + + - name: Build executable + env: + EXE_NAME: yt-dlp_linux_armv7l + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ inputs.version }} + SERVICE: linux_armv7l + run: | + mkdir -p ~/build + mkdir -p ~/yt-dlp-build-venv + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + docker compose down + + - name: Verify executable in container + if: vars.UPDATE_TO_VERIFICATION + env: + EXE_NAME: yt-dlp_linux_armv7l + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ inputs.version }} + SERVICE: linux_armv7l_verify + run: | + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + docker compose down + + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: - path: ./repo - - name: Virtualized Install, Prepare & Build - uses: yt-dlp/run-on-arch-action@v3 + name: build-bin-${{ github.job }} + path: | + ~/build/yt-dlp_linux_armv7l.zip + compression-level: 0 + + - name: Cleanup cache + if: steps.restore-cache.outputs.cache-hit == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + cache_key: cache-reqs-${{ github.job }} + run: | + gh cache delete "${cache_key}" + + - name: Cache requirements + uses: actions/cache/save@v4 with: - # Ref: https://github.com/uraimo/run-on-arch-action/issues/55 - env: | - GITHUB_WORKFLOW: build - githubToken: ${{ github.token }} # To cache image - arch: ${{ matrix.architecture }} - distro: ubuntu20.04 # Standalone executable should be built on minimum supported OS - dockerRunArgs: --volume "${PWD}/repo:/repo" - install: | # Installing Python 3.10 from the Deadsnakes repo raises errors - apt update - apt -y install zlib1g-dev libffi-dev python3.9 python3.9-dev python3.9-distutils python3-pip \ - python3-secretstorage # Cannot build cryptography wheel in virtual armv7 environment - python3.9 -m pip install -U pip wheel 'setuptools>=71.0.2' - # XXX: Keep this in sync with pyproject.toml (it can't be accessed at this stage) and exclude secretstorage - python3.9 -m pip install -U Pyinstaller mutagen pycryptodomex brotli certifi cffi \ - 'requests>=2.32.2,<3' 'urllib3>=2.0.2,<3' 'websockets>=13.0' - - run: | - cd repo - python3.9 devscripts/install_deps.py -o --include build - python3.9 devscripts/install_deps.py --include pyinstaller # Cached versions may be out of date - python3.9 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" - python3.9 devscripts/make_lazy_extractors.py - python3.9 -m bundle.pyinstaller - - if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then - arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}" - chmod +x ./dist/yt-dlp_linux_${arch} - cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded - version="$(./dist/yt-dlp_linux_${arch} --version)" - ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 - downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)" - [[ "$version" != "$downgraded_version" ]] - fi + path: | + ~/yt-dlp-build-venv + key: cache-reqs-${{ github.job }} + musllinux: + needs: process + if: inputs.musllinux + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - exe: yt-dlp_musllinux + platform: x86_64 + runner: ubuntu-24.04 + - exe: yt-dlp_musllinux_aarch64 + platform: aarch64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + - name: Build executable + env: + EXE_NAME: ${{ matrix.exe }} + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ inputs.version }} + SERVICE: musllinux_${{ matrix.platform }} + run: | + mkdir -p ~/build + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + docker compose down + sudo chown "${USER}:docker" ~/build/${{ matrix.exe }} + - name: Verify executable in container + if: vars.UPDATE_TO_VERIFICATION + env: + EXE_NAME: ${{ matrix.exe }} + CHANNEL: ${{ inputs.channel }} + ORIGIN: ${{ needs.process.outputs.origin }} + VERSION: ${{ inputs.version }} + SERVICE: musllinux_${{ matrix.platform }}_verify + run: | + cd bundle/docker + docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" + docker compose down - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: build-bin-linux_${{ matrix.architecture }} - path: | # run-on-arch-action designates armv7l as armv7 - repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} + name: build-bin-${{ github.job }}_${{ matrix.platform }} + path: | + ~/build/${{ matrix.exe }}* compression-level: 0 macos: @@ -436,8 +531,9 @@ jobs: needs: - process - unix - - linux_static - - linux_arm + - linux + - linux_armv7l + - musllinux - macos - windows - windows32 @@ -473,6 +569,7 @@ jobs: lock 2024.10.22 zip Python 3\.8 lock 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lock 2025.08.11 darwin_legacy_exe .+ + lock 2025.09.01 linux_armv7l_exe .+ lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6 lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7 lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server) @@ -481,6 +578,7 @@ jobs: lockV2 yt-dlp/yt-dlp 2024.10.22 zip Python 3\.8 lockV2 yt-dlp/yt-dlp 2024.10.22 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lockV2 yt-dlp/yt-dlp 2025.08.11 darwin_legacy_exe .+ + lockV2 yt-dlp/yt-dlp 2025.09.01 linux_armv7l_exe .+ lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7 lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server) lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 py2exe .+ @@ -488,6 +586,7 @@ jobs: lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 zip Python 3\.8 lockV2 yt-dlp/yt-dlp-nightly-builds 2024.10.22.051025 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lockV2 yt-dlp/yt-dlp-nightly-builds 2025.08.12.233030 darwin_legacy_exe .+ + lockV2 yt-dlp/yt-dlp-nightly-builds 2025.09.01.000000 linux_armv7l_exe .+ lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7 lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server) lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.045052 py2exe .+ @@ -495,6 +594,7 @@ jobs: lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 zip Python 3\.8 lockV2 yt-dlp/yt-dlp-master-builds 2024.10.22.060347 win(?:_x86)?_exe Python 3\.[78].+ Windows-(?:7-|2008ServerR2) lockV2 yt-dlp/yt-dlp-master-builds 2025.08.12.232447 darwin_legacy_exe .+ + lockV2 yt-dlp/yt-dlp-master-builds 2025.09.01.000000 linux_armv7l_exe .+ EOF - name: Sign checksum files diff --git a/.github/workflows/cache-warmer.yml b/.github/workflows/cache-warmer.yml new file mode 100644 index 0000000000..e51c880a1b --- /dev/null +++ b/.github/workflows/cache-warmer.yml @@ -0,0 +1,24 @@ +name: Keep cache warm +on: + workflow_dispatch: + schedule: + - cron: '0 22 1,6,11,16,21,27 * *' + +jobs: + build: + if: | + vars.KEEP_CACHE_WARM != '' || github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/build.yml + with: + version: '123456' + channel: stable + unix: false + linux: false + linux_armv7l: true + musllinux: false + macos: true + windows: false + windows32: false + permissions: + contents: read + actions: write # For cleaning up cache diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 78445e417e..132fc4a928 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -24,7 +24,6 @@ jobs: source: master permissions: contents: write - packages: write # For package cache actions: write # For cleaning up cache id-token: write # mandatory for trusted publishing secrets: inherit diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index 8f72844058..d35e4ef077 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -37,7 +37,6 @@ jobs: source: nightly permissions: contents: write - packages: write # For package cache actions: write # For cleaning up cache id-token: write # mandatory for trusted publishing secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26b93e429c..662f641dce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,10 @@ on: required: false default: '' type: string + linux_armv7l: + required: false + default: false + type: boolean prerelease: required: false default: true @@ -43,6 +47,10 @@ on: required: false default: '' type: string + linux_armv7l: + description: Include linux_armv7l + default: true + type: boolean prerelease: description: Pre-release default: false @@ -226,9 +234,9 @@ jobs: version: ${{ needs.prepare.outputs.version }} channel: ${{ needs.prepare.outputs.channel }} origin: ${{ needs.prepare.outputs.target_repo }} + linux_armv7l: ${{ inputs.linux_armv7l }} permissions: contents: read - packages: write # For package cache actions: write # For cleaning up cache secrets: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} diff --git a/README.md b/README.md index aa8b1d4f24..6b3831e87e 100644 --- a/README.md +++ b/README.md @@ -106,11 +106,17 @@ File|Description File|Description :---|:--- [yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Win8+) standalone x86 (32-bit) binary -[yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux standalone x64 binary -[yt-dlp_linux_armv7l](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_armv7l)|Linux standalone armv7l (32-bit) binary -[yt-dlp_linux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64)|Linux standalone aarch64 (64-bit) binary -[yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update) +[yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux (glibc 2.17+) standalone x86_64 binary +[yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Linux (glibc 2.17+) x86_64 executable (no auto-update) +[yt-dlp_linux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64)|Linux (glibc 2.17+) standalone aarch64 binary +[yt-dlp_linux_aarch64.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64.zip)|Unpackaged Linux (glibc 2.17+) aarch64 executable (no auto-update) +[yt-dlp_musllinux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux)|Linux (musl 1.2+) standalone x86_64 binary +[yt-dlp_musllinux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux.zip)|Unpackaged Linux (musl 1.2+) x86_64 executable (no auto-update) +[yt-dlp_musllinux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux_aarch64)|Linux (musl 1.2+) standalone aarch64 binary +[yt-dlp_musllinux_aarch64.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_musllinux_aarch64.zip)|Unpackaged Linux (musl 1.2+) aarch64 executable (no auto-update) +[yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows (Win8+) executable (no auto-update) [yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update) +[yt-dlp_linux_armv7l.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_armv7l.zip)|Unpackaged Linux (glibc 2.31+) armv7l executable (no auto-update) #### Misc @@ -203,7 +209,7 @@ The following provide support for impersonating browser requests. This may be re * [**curl_cffi**](https://github.com/lexiforest/curl_cffi) (recommended) - Python binding for [curl-impersonate](https://github.com/lexiforest/curl-impersonate). Provides impersonation targets for Chrome, Edge and Safari. Licensed under [MIT](https://github.com/lexiforest/curl_cffi/blob/main/LICENSE) * Can be installed with the `curl-cffi` group, e.g. `pip install "yt-dlp[default,curl-cffi]"` - * Currently included in `yt-dlp.exe`, `yt-dlp_linux` and `yt-dlp_macos` builds + * Currently included in most builds *except* `yt-dlp` (Unix zipimport binary), `yt-dlp_x86` (Windows 32-bit) and `yt-dlp_musllinux_aarch64` ### Metadata diff --git a/bundle/docker/compose.yml b/bundle/docker/compose.yml index 5f89ca6d09..b2db7e1895 100644 --- a/bundle/docker/compose.yml +++ b/bundle/docker/compose.yml @@ -1,10 +1,154 @@ services: - static: - build: static + + linux_x86_64: + build: + context: linux + platforms: + - "linux/amd64" + args: + BASEIMAGE: ghcr.io/bashonly/manylinux2014_x86_64_builds:latest environment: - channel: ${channel} - origin: ${origin} - version: ${version} + EXE_NAME: ${EXE_NAME} + CHANNEL: ${CHANNEL} + ORIGIN: ${ORIGIN} + VERSION: ${VERSION} volumes: - ~/build:/build - ../..:/yt-dlp + + linux_x86_64_verify: + build: + context: linux + dockerfile: verify.Dockerfile + platforms: + - "linux/amd64" + args: + IMAGE: quay.io/pypa/manylinux2014_x86_64:latest + environment: + EXE_NAME: ${EXE_NAME} + volumes: + - ~/build:/build + + linux_aarch64: + build: + context: linux + platforms: + - "linux/arm64" + args: + BASEIMAGE: ghcr.io/bashonly/manylinux2014_aarch64_builds:latest + environment: + EXE_NAME: ${EXE_NAME} + CHANNEL: ${CHANNEL} + ORIGIN: ${ORIGIN} + VERSION: ${VERSION} + volumes: + - ~/build:/build + - ../..:/yt-dlp + + linux_aarch64_verify: + build: + context: linux + dockerfile: verify.Dockerfile + platforms: + - "linux/arm64" + args: + IMAGE: quay.io/pypa/manylinux2014_aarch64:latest + environment: + EXE_NAME: ${EXE_NAME} + SKIP_UPDATE_TO: "1" # TODO: remove when there is a glibc2.17 aarch64 release to --update-to + volumes: + - ~/build:/build + + linux_armv7l: + build: + context: linux + platforms: + - "linux/arm/v7" + args: + BASEIMAGE: ghcr.io/bashonly/manylinux_2_31_armv7l_builds:latest + environment: + EXE_NAME: ${EXE_NAME} + CHANNEL: ${CHANNEL} + ORIGIN: ${ORIGIN} + VERSION: ${VERSION} + SKIP_ONEFILE_BUILD: "1" + volumes: + - ~/build:/build + - ../..:/yt-dlp + - ~/yt-dlp-build-venv:/yt-dlp-build-venv + + linux_armv7l_verify: + build: + context: linux + dockerfile: verify.Dockerfile + platforms: + - "linux/arm/v7" + args: + IMAGE: arm32v7/debian:bullseye + environment: + EXE_NAME: ${EXE_NAME} + TEST_ONEDIR_BUILD: "1" + volumes: + - ~/build:/build + + musllinux_x86_64: + build: + context: linux + platforms: + - "linux/amd64" + args: + BASEIMAGE: ghcr.io/bashonly/musllinux_1_2_x86_64_builds:latest + environment: + EXE_NAME: ${EXE_NAME} + CHANNEL: ${CHANNEL} + ORIGIN: ${ORIGIN} + VERSION: ${VERSION} + volumes: + - ~/build:/build + - ../..:/yt-dlp + + musllinux_x86_64_verify: + build: + context: linux + dockerfile: verify.Dockerfile + platforms: + - "linux/amd64" + args: + IMAGE: alpine:3.22 + environment: + EXE_NAME: ${EXE_NAME} + SKIP_UPDATE_TO: "1" # TODO: remove when there is a musllinux_aarch64 release to --update-to + volumes: + - ~/build:/build + + musllinux_aarch64: + build: + context: linux + platforms: + - "linux/arm64" + args: + BASEIMAGE: ghcr.io/bashonly/musllinux_1_2_aarch64_builds:latest + environment: + EXE_NAME: ${EXE_NAME} + CHANNEL: ${CHANNEL} + ORIGIN: ${ORIGIN} + VERSION: ${VERSION} + EXCLUDE_CURL_CFFI: "1" + volumes: + - ~/build:/build + - ../..:/yt-dlp + + musllinux_aarch64_verify: + build: + context: linux + dockerfile: verify.Dockerfile + platforms: + - "linux/arm64" + args: + IMAGE: alpine:3.22 + environment: + EXE_NAME: ${EXE_NAME} + SKIP_UPDATE_TO: "1" # TODO: remove when there is a musllinux_aarch64 release to --update-to + EXCLUDE_CURL_CFFI: "1" + volumes: + - ~/build:/build diff --git a/bundle/docker/linux/Dockerfile b/bundle/docker/linux/Dockerfile new file mode 100644 index 0000000000..d989390b05 --- /dev/null +++ b/bundle/docker/linux/Dockerfile @@ -0,0 +1,7 @@ +ARG BASEIMAGE=ghcr.io/bashonly/manylinux2014_x86_64_builds:latest + +FROM $BASEIMAGE AS base + +WORKDIR /yt-dlp +COPY entrypoint.sh /entrypoint.sh +ENTRYPOINT /entrypoint.sh diff --git a/bundle/docker/linux/entrypoint.sh b/bundle/docker/linux/entrypoint.sh new file mode 100755 index 0000000000..e6e8710f85 --- /dev/null +++ b/bundle/docker/linux/entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -exuo pipefail + +function runpy { + case ${USE_PYTHON_VERSION:-} in + "3.11") python3.11 "$@";; + "3.13") python3.13 "$@";; + *) python3 "$@";; + esac +} + +INCLUDES=( + --include pyinstaller + --include secretstorage +) + +if [[ -z "${EXCLUDE_CURL_CFFI:-}" ]]; then + INCLUDES+=(--include curl-cffi) +fi + +runpy -m venv /yt-dlp-build-venv +source /yt-dlp-build-venv/bin/activate +runpy -m devscripts.install_deps -o --include build +runpy -m devscripts.install_deps "${INCLUDES[@]}" +runpy -m devscripts.make_lazy_extractors +runpy devscripts/update-version.py -c "${CHANNEL}" -r "${ORIGIN}" "${VERSION}" + +if [[ -z "${SKIP_ONEDIR_BUILD:-}" ]]; then + runpy -m bundle.pyinstaller --onedir + pushd "./dist/${EXE_NAME}" + runpy -m zipfile -c "/build/${EXE_NAME}.zip" ./ + popd +fi + +if [[ -z "${SKIP_ONEFILE_BUILD:-}" ]]; then + runpy -m bundle.pyinstaller + mv "./dist/${EXE_NAME}" /build/ +fi diff --git a/bundle/docker/linux/verify.Dockerfile b/bundle/docker/linux/verify.Dockerfile new file mode 100644 index 0000000000..82c1e34db9 --- /dev/null +++ b/bundle/docker/linux/verify.Dockerfile @@ -0,0 +1,7 @@ +ARG IMAGE=alpine:3.22 + +FROM $IMAGE AS image + +WORKDIR /testing +COPY verify.sh /verify.sh +ENTRYPOINT /verify.sh diff --git a/bundle/docker/linux/verify.sh b/bundle/docker/linux/verify.sh new file mode 100755 index 0000000000..9263bf7b1c --- /dev/null +++ b/bundle/docker/linux/verify.sh @@ -0,0 +1,49 @@ +#!/bin/sh +set -eu + +if [ -n "${TEST_ONEDIR_BUILD:-}" ]; then + echo "Extracting zip to verify onedir build" + if command -v python3 >/dev/null 2>&1; then + python3 -m zipfile -e "/build/${EXE_NAME}.zip" ./ + else + echo "Attempting to install unzip" + if command -v dnf >/dev/null 2>&1; then + dnf -y install --allowerasing unzip + elif command -v yum >/dev/null 2>&1; then + yum -y install unzip + elif command -v apt-get >/dev/null 2>&1; then + DEBIAN_FRONTEND=noninteractive apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends unzip + elif command -v apk >/dev/null 2>&1; then + apk add --no-cache unzip + else + echo "Unsupported image" + exit 1 + fi + unzip "/build/${EXE_NAME}.zip" -d ./ + fi +else + echo "Verifying onefile build" + cp "/build/${EXENAME}" ./ +fi + +chmod +x "./${EXE_NAME}" + +if [ -z "${EXCLUDE_CURL_CFFI:-}" ]; then + "./${EXE_NAME}" -v --print-traffic --impersonate chrome "https://tls.browserleaks.com/json" -o ./resp.json + cat ./resp.json +fi + +if [ -n "${SKIP_UPDATE_TO:-}" ] || [ -n "${TEST_ONEDIR_BUILD:-}" ]; then + "./${EXE_NAME}" -v || true + "./${EXE_NAME}" --version + exit 0 +fi + +cp "./${EXE_NAME}" "./${EXE_NAME}_downgraded" +version="$("./${EXE_NAME}" --version)" +"./${EXE_NAME}_downgraded" -v --update-to yt-dlp/yt-dlp@2023.03.04 +downgraded_version="$("./${EXE_NAME}_downgraded" --version)" +if [ "${version}" = "${downgraded_version}" ]; then + exit 1 +fi diff --git a/bundle/docker/static/Dockerfile b/bundle/docker/static/Dockerfile deleted file mode 100644 index dae2dff3d8..0000000000 --- a/bundle/docker/static/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM alpine:3.19 as base - -RUN apk --update add --no-cache \ - build-base \ - python3 \ - pipx \ - ; - -RUN pipx install pyinstaller -# Requires above step to prepare the shared venv -RUN ~/.local/share/pipx/shared/bin/python -m pip install -U wheel -RUN apk --update add --no-cache \ - scons \ - patchelf \ - binutils \ - ; -RUN pipx install staticx - -WORKDIR /yt-dlp -COPY entrypoint.sh /entrypoint.sh -ENTRYPOINT /entrypoint.sh diff --git a/bundle/docker/static/entrypoint.sh b/bundle/docker/static/entrypoint.sh deleted file mode 100755 index 8049e68205..0000000000 --- a/bundle/docker/static/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/ash -set -e - -source ~/.local/share/pipx/venvs/pyinstaller/bin/activate -python -m devscripts.install_deps -o --include build -python -m devscripts.install_deps --include secretstorage --include curl-cffi -python -m devscripts.make_lazy_extractors -python devscripts/update-version.py -c "${channel}" -r "${origin}" "${version}" -python -m bundle.pyinstaller -deactivate - -source ~/.local/share/pipx/venvs/staticx/bin/activate -staticx /yt-dlp/dist/yt-dlp_linux /build/yt-dlp_linux -deactivate diff --git a/bundle/pyinstaller.py b/bundle/pyinstaller.py index 0597f602d0..8286f077ab 100755 --- a/bundle/pyinstaller.py +++ b/bundle/pyinstaller.py @@ -13,6 +13,8 @@ from PyInstaller.__main__ import run as run_pyinstaller from devscripts.utils import read_version OS_NAME, MACHINE, ARCH = sys.platform, platform.machine().lower(), platform.architecture()[0][:2] +if OS_NAME == 'linux' and platform.libc_ver()[0] != 'glibc': + OS_NAME = 'musllinux' if MACHINE in ('x86', 'x86_64', 'amd64', 'i386', 'i686'): MACHINE = 'x86' if ARCH == '32' else '' diff --git a/yt_dlp/update.py b/yt_dlp/update.py index dd948cd521..9483887a9f 100644 --- a/yt_dlp/update.py +++ b/yt_dlp/update.py @@ -58,15 +58,28 @@ def _get_variant_and_executable_path(): """@returns (variant, executable_path)""" if getattr(sys, 'frozen', False): path = sys.executable - # py2exe is unsupported but we should still correctly identify it for debugging purposes + + # py2exe: No longer officially supported, but still identify it to block updates if not hasattr(sys, '_MEIPASS'): return 'py2exe', path - if sys._MEIPASS == os.path.dirname(path): - return f'{sys.platform}_dir', path - if sys.platform == 'darwin': + + # staticx builds: sys.executable returns a /tmp/ path + # No longer officially supported, but still identify them to block updates + # Ref: https://staticx.readthedocs.io/en/latest/usage.html#run-time-information + if static_exe_path := os.getenv('STATICX_PROG_PATH'): + return 'linux_static_exe', static_exe_path + + # We know it's a PyInstaller bundle, but is it "onedir" or "onefile"? + suffix = 'dir' if sys._MEIPASS == os.path.dirname(path) else 'exe' + system_platform = remove_end(sys.platform, '32') + + if system_platform == 'darwin': # darwin_legacy_exe is no longer supported, but still identify it to block updates machine = '_legacy' if version_tuple(platform.mac_ver()[0]) < (10, 15) else '' - return f'darwin{machine}_exe', path + return f'darwin{machine}_{suffix}', path + + if system_platform == 'linux' and platform.libc_ver()[0] != 'glibc': + system_platform = 'musllinux' machine = f'_{platform.machine().lower()}' is_64bits = sys.maxsize > 2**32 @@ -77,12 +90,8 @@ def _get_variant_and_executable_path(): # See: https://github.com/yt-dlp/yt-dlp/issues/11813 elif machine[1:] == 'aarch64' and not is_64bits: machine = '_armv7l' - # sys.executable returns a /tmp/ path for staticx builds (linux_static) - # Ref: https://staticx.readthedocs.io/en/latest/usage.html#run-time-information - if static_exe_path := os.getenv('STATICX_PROG_PATH'): - path = static_exe_path - return f'{remove_end(sys.platform, "32")}{machine}_exe', path + return f'{system_platform}{machine}_{suffix}', path path = os.path.dirname(__file__) if isinstance(__loader__, zipimporter): @@ -117,7 +126,8 @@ _FILE_SUFFIXES = { 'darwin_exe': '_macos', 'linux_exe': '_linux', 'linux_aarch64_exe': '_linux_aarch64', - 'linux_armv7l_exe': '_linux_armv7l', + 'musllinux_exe': '_musllinux', + 'musllinux_aarch64_exe': '_musllinux_aarch64', } _NON_UPDATEABLE_REASONS = { @@ -145,21 +155,6 @@ def _get_binary_name(): def _get_system_deprecation(): MIN_SUPPORTED, MIN_RECOMMENDED = (3, 9), (3, 10) - EXE_MSG_TMPL = ('Support for {} has been deprecated. ' - 'See https://github.com/yt-dlp/yt-dlp/{} for details.\n{}') - STOP_MSG = 'You may stop receiving updates on this version at any time!' - variant = detect_variant() - - # Temporary until linux_armv7l executable builds are discontinued - if variant == 'linux_armv7l_exe': - return EXE_MSG_TMPL.format( - f'{variant} (the PyInstaller-bundled executable for the Linux armv7l platform)', - 'issues/13976', STOP_MSG) - - # Temporary until linux_aarch64_exe is built with Python >=3.10 instead of Python 3.9 - if variant == 'linux_aarch64_exe': - return None - if sys.version_info > MIN_RECOMMENDED: return None