Merge branch 'ytdl-org:master' into chelseafc-extractor
01001110
1 year ago
committed by
GitHub
commit
1250269c74
@ -1,81 +1,445 @@
|
||||
name: CI
|
||||
on: [push, pull_request]
|
||||
|
||||
env:
|
||||
all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12
|
||||
main-cpython-versions: 2.7, 3.2, 3.5, 3.9, 3.11
|
||||
pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
|
||||
cpython-versions: main
|
||||
test-set: core
|
||||
|
||||
on:
|
||||
push:
|
||||
inputs:
|
||||
cpython-versions:
|
||||
type: string
|
||||
default: all
|
||||
test-set:
|
||||
type: string
|
||||
default: core
|
||||
pull_request:
|
||||
inputs:
|
||||
cpython-versions:
|
||||
type: string
|
||||
default: main
|
||||
test-set:
|
||||
type: string
|
||||
default: both
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
cpython-versions:
|
||||
type: choice
|
||||
description: CPython versions (main = 2.7, 3.2, 3.5, 3.9, 3.11)
|
||||
options:
|
||||
- all
|
||||
- main
|
||||
required: true
|
||||
default: main
|
||||
test-set:
|
||||
type: choice
|
||||
description: core, download
|
||||
options:
|
||||
- both
|
||||
- core
|
||||
- download
|
||||
required: true
|
||||
default: both
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
select:
|
||||
name: Select tests from inputs
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
cpython-versions: ${{ steps.run.outputs.cpython-versions }}
|
||||
test-set: ${{ steps.run.outputs.test-set }}
|
||||
own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
|
||||
steps:
|
||||
- name: Make version array
|
||||
id: run
|
||||
run: |
|
||||
# Make a JSON Array from comma/space-separated string (no extra escaping)
|
||||
json_list() { \
|
||||
ret=""; IFS="${IFS},"; set -- $*; \
|
||||
for a in "$@"; do \
|
||||
ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \
|
||||
done; \
|
||||
printf '[%s]' "$ret"; }
|
||||
tests="${{ inputs.test-set || env.test-set }}"
|
||||
[ $tests = both ] && tests="core download"
|
||||
printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT"
|
||||
versions="${{ inputs.cpython-versions || env.cpython-versions }}"
|
||||
if [ "$versions" = all ]; then \
|
||||
versions="${{ env.all-cpython-versions }}"; else \
|
||||
versions="${{ env.main-cpython-versions }}"; \
|
||||
fi
|
||||
printf 'cpython-versions=%s\n' \
|
||||
"$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT"
|
||||
# versions with a special get-pip.py in a per-version subdirectory
|
||||
printf 'own-pip-versions=%s\n' \
|
||||
"$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"
|
||||
tests:
|
||||
name: Tests
|
||||
name: Run tests
|
||||
needs: select
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
PIP: python -m pip
|
||||
PIP_DISABLE_PIP_VERSION_CHECK: true
|
||||
PIP_NO_PYTHON_VERSION_WARNING: true
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
os: [ubuntu-18.04]
|
||||
# TODO: python 2.6
|
||||
python-version: [2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
|
||||
os: [ubuntu-20.04]
|
||||
python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }}
|
||||
python-impl: [cpython]
|
||||
ytdl-test-set: [core, download]
|
||||
ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }}
|
||||
run-tests-ext: [sh]
|
||||
include:
|
||||
# python 3.2 is only available on windows via setup-python
|
||||
- os: windows-2019
|
||||
python-version: 3.2
|
||||
python-version: 3.4
|
||||
python-impl: cpython
|
||||
ytdl-test-set: core
|
||||
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
|
||||
run-tests-ext: bat
|
||||
- os: windows-2019
|
||||
python-version: 3.2
|
||||
python-version: 3.4
|
||||
python-impl: cpython
|
||||
ytdl-test-set: download
|
||||
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }}
|
||||
run-tests-ext: bat
|
||||
# jython
|
||||
- os: ubuntu-18.04
|
||||
- os: ubuntu-20.04
|
||||
python-version: 2.7
|
||||
python-impl: jython
|
||||
ytdl-test-set: core
|
||||
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
|
||||
run-tests-ext: sh
|
||||
- os: ubuntu-18.04
|
||||
- os: ubuntu-20.04
|
||||
python-version: 2.7
|
||||
python-impl: jython
|
||||
ytdl-test-set: download
|
||||
ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }}
|
||||
run-tests-ext: sh
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
if: ${{ matrix.python-impl == 'cpython' }}
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
#-------- Python 3 -----
|
||||
- name: Set up supported Python ${{ matrix.python-version }}
|
||||
id: setup-python
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != '3.12'}}
|
||||
# wrap broken actions/setup-python@v4
|
||||
uses: ytdl-org/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache-build: true
|
||||
allow-build: info
|
||||
- name: Locate supported Python ${{ matrix.python-version }}
|
||||
if: ${{ env.pythonLocation }}
|
||||
shell: bash
|
||||
run: |
|
||||
echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV"
|
||||
export expected="${{ steps.setup-python.outputs.python-path }}"
|
||||
dirname() { printf '%s\n' \
|
||||
'import os, sys' \
|
||||
'print(os.path.dirname(sys.argv[1]))' \
|
||||
| ${expected} - "$1"; }
|
||||
expd="$(dirname "$expected")"
|
||||
export python="$(command -v python)"
|
||||
[ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV"
|
||||
[ -x "$python" ] || printf '%s\n' \
|
||||
'import os' \
|
||||
'exp = os.environ["expected"]' \
|
||||
'python = os.environ["python"]' \
|
||||
'exps = os.path.split(exp)' \
|
||||
'if python and (os.path.dirname(python) == exp[0]):' \
|
||||
' exit(0)' \
|
||||
'exps[1] = "python" + os.path.splitext(exps[1])[1]' \
|
||||
'python = os.path.join(*exps)' \
|
||||
'try:' \
|
||||
' os.symlink(exp, python)' \
|
||||
'except AttributeError:' \
|
||||
' os.rename(exp, python)' \
|
||||
| ${expected} -
|
||||
printf '%s\n' \
|
||||
'import sys' \
|
||||
'print(sys.path)' \
|
||||
| ${expected} -
|
||||
#-------- Python 3.12 -
|
||||
- name: Set up CPython 3.12 environment
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
|
||||
shell: bash
|
||||
run: |
|
||||
PYENV_ROOT=$HOME/.local/share/pyenv
|
||||
echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
|
||||
- name: Cache Python 3.12
|
||||
id: cache312
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
key: python-3.12
|
||||
path: |
|
||||
${{ env.PYENV_ROOT }}
|
||||
- name: Build and set up Python 3.12
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
|
||||
# dl and build locally
|
||||
shell: bash
|
||||
run: |
|
||||
# Install build environment
|
||||
sudo apt-get install -y build-essential llvm libssl-dev tk-dev \
|
||||
libncursesw5-dev libreadline-dev libsqlite3-dev \
|
||||
libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
|
||||
# Download PyEnv from its GitHub repository.
|
||||
export PYENV_ROOT=${{ env.PYENV_ROOT }}
|
||||
export PATH=$PYENV_ROOT/bin:$PATH
|
||||
git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
|
||||
pyenv install 3.12.0b4
|
||||
- name: Locate Python 3.12
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
|
||||
shell: bash
|
||||
run: |
|
||||
PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4"
|
||||
echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
|
||||
echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
|
||||
#-------- Python 2.7 --
|
||||
- name: Set up Python 2.7
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }}
|
||||
# install 2.7
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install -y python2 python-is-python2
|
||||
echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
|
||||
#-------- Python 2.6 --
|
||||
- name: Set up Python 2.6 environment
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
|
||||
shell: bash
|
||||
run: |
|
||||
openssl_name=openssl-1.0.2u
|
||||
echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV"
|
||||
openssl_dir=$HOME/.local/opt/$openssl_name
|
||||
echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV"
|
||||
PYENV_ROOT=$HOME/.local/share/pyenv
|
||||
echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
|
||||
sudo apt-get install -y openssl ca-certificates
|
||||
- name: Cache Python 2.6
|
||||
id: cache26
|
||||
if: ${{ matrix.python-version == '2.6' }}
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
key: python-2.6.9
|
||||
path: |
|
||||
${{ env.openssl_dir }}
|
||||
${{ env.PYENV_ROOT }}
|
||||
- name: Build and set up Python 2.6
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
|
||||
# dl and build locally
|
||||
shell: bash
|
||||
run: |
|
||||
# Install build environment
|
||||
sudo apt-get install -y build-essential llvm libssl-dev tk-dev \
|
||||
libncursesw5-dev libreadline-dev libsqlite3-dev \
|
||||
libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
|
||||
# Download and install OpenSSL 1.0.2, back in time
|
||||
openssl_name=${{ env.openssl_name }}
|
||||
openssl_targz=${openssl_name}.tar.gz
|
||||
openssl_dir=${{ env.openssl_dir }}
|
||||
openssl_inc=$openssl_dir/include
|
||||
openssl_lib=$openssl_dir/lib
|
||||
openssl_ssl=$openssl_dir/ssl
|
||||
curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz
|
||||
tar -xf $openssl_targz
|
||||
( cd $openssl_name; \
|
||||
./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \
|
||||
--libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \
|
||||
make && \
|
||||
make install )
|
||||
rm -rf $openssl_name
|
||||
rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs
|
||||
# Download PyEnv from its GitHub repository.
|
||||
export PYENV_ROOT=${{ env.PYENV_ROOT }}
|
||||
export PATH=$PYENV_ROOT/bin:$PATH
|
||||
git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
|
||||
# Prevent pyenv build trying (and failing) to update pip
|
||||
export GET_PIP=get-pip-2.6.py
|
||||
echo 'import sys; sys.exit(0)' > ${GET_PIP}
|
||||
GET_PIP=$(realpath $GET_PIP)
|
||||
# Build and install Python
|
||||
export CFLAGS="-I$openssl_inc"
|
||||
export LDFLAGS="-L$openssl_lib"
|
||||
export LD_LIBRARY_PATH="$openssl_lib"
|
||||
pyenv install 2.6.9
|
||||
- name: Locate Python 2.6
|
||||
if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
|
||||
shell: bash
|
||||
run: |
|
||||
PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9"
|
||||
echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
|
||||
echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
|
||||
echo "LD_LIBRARY_PATH=${{ env.openssl_dir }}/lib${LD_LIBRARY_PATH:+:}${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
|
||||
#-------- Jython ------
|
||||
- name: Set up Java 8
|
||||
if: ${{ matrix.python-impl == 'jython' }}
|
||||
uses: actions/setup-java@v1
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: 8
|
||||
- name: Install Jython
|
||||
distribution: 'zulu'
|
||||
- name: Setup Jython environment
|
||||
if: ${{ matrix.python-impl == 'jython' }}
|
||||
shell: bash
|
||||
run: |
|
||||
wget https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar -O jython-installer.jar
|
||||
java -jar jython-installer.jar -s -d "$HOME/jython"
|
||||
echo "$HOME/jython/bin" >> $GITHUB_PATH
|
||||
- name: Install nose
|
||||
if: ${{ matrix.python-impl != 'jython' }}
|
||||
run: pip install nose
|
||||
- name: Install nose (Jython)
|
||||
if: ${{ matrix.python-impl == 'jython' }}
|
||||
# Working around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
|
||||
echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV"
|
||||
echo "PIP=pip" >> "$GITHUB_ENV"
|
||||
- name: Cache Jython
|
||||
id: cachejy
|
||||
if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }}
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
# 2.7.3 now available, may solve SNI issue
|
||||
key: jython-2.7.1
|
||||
path: |
|
||||
${{ env.JYTHON_ROOT }}
|
||||
- name: Install Jython
|
||||
if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }}
|
||||
shell: bash
|
||||
run: |
|
||||
JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
|
||||
curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar
|
||||
java -jar jython-installer.jar -s -d "${JYTHON_ROOT}"
|
||||
echo "${JYTHON_ROOT}/bin" >> "$GITHUB_PATH"
|
||||
- name: Set up cached Jython
|
||||
if: ${{ steps.cachejy.outputs.cache-hit }}
|
||||
shell: bash
|
||||
run: |
|
||||
JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
|
||||
echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
|
||||
- name: Install supporting Python 2.7 if possible
|
||||
if: ${{ steps.cachejy.outputs.cache-hit }}
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install -y python2.7 || true
|
||||
#-------- pip ---------
|
||||
- name: Set up supported Python ${{ matrix.python-version }} pip
|
||||
if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
|
||||
# This step may run in either Linux or Windows
|
||||
shell: bash
|
||||
run: |
|
||||
echo "$PATH"
|
||||
echo "$PYTHONHOME"
|
||||
# curl is available on both Windows and Linux, -L follows redirects, -O gets name
|
||||
python -m ensurepip || python -m pip --version || { \
|
||||
get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \
|
||||
curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \
|
||||
python get-pip.py; }
|
||||
- name: Set up Python 2.6 pip
|
||||
if: ${{ matrix.python-version == '2.6' }}
|
||||
shell: bash
|
||||
run: |
|
||||
wget https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl
|
||||
pip install nose-1.3.7-py2-none-any.whl
|
||||
python -m pip --version || { \
|
||||
curl -L -O "https://bootstrap.pypa.io/pip/2.6/get-pip.py"; \
|
||||
curl -L -O "https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl"; \
|
||||
python get-pip.py --no-setuptools --no-wheel pip-9.0.3-py2.py3-none-any.whl; }
|
||||
# work-around to invoke pip module on 2.6: https://bugs.python.org/issue2751
|
||||
echo "PIP=python -m pip.__main__" >> "$GITHUB_ENV"
|
||||
- name: Set up other Python ${{ matrix.python-version }} pip
|
||||
if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
|
||||
shell: bash
|
||||
run: |
|
||||
python -m pip --version || { \
|
||||
curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \
|
||||
curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \
|
||||
python get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; }
|
||||
#-------- unittest ----
|
||||
- name: Upgrade Unittest for Python 2.6
|
||||
if: ${{ matrix.python-version == '2.6' }}
|
||||
shell: bash
|
||||
run: |
|
||||
# Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
|
||||
$PIP -qq show unittest2 || { \
|
||||
for u in "65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl" \
|
||||
"f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl" \
|
||||
"c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl" \
|
||||
"17/0a/6ac05a3723017a967193456a2efa0aa9ac4b51456891af1e2353bb9de21e/traceback2-1.4.0-py2.py3-none-any.whl" \
|
||||
"72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"; do \
|
||||
curl -L -O "https://files.pythonhosted.org/packages/${u}"; \
|
||||
$PIP install ${u##*/}; \
|
||||
done; }
|
||||
# make tests use unittest2
|
||||
for test in ./test/test_*.py ./test/helper.py; do
|
||||
sed -r -i -e '/^import unittest$/s/test/test2 as unittest/' "$test"
|
||||
done
|
||||
#-------- nose --------
|
||||
- name: Install nose for Python ${{ matrix.python-version }}
|
||||
if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == '3.12')) }}
|
||||
shell: bash
|
||||
run: |
|
||||
echo "$PATH"
|
||||
echo "$PYTHONHOME"
|
||||
# Use PyNose for recent Pythons instead of Nose
|
||||
py3ver="${{ matrix.python-version }}"
|
||||
py3ver=${py3ver#3.}
|
||||
[ "$py3ver" != "${{ matrix.python-version }}" ] && py3ver=${py3ver%.*} || py3ver=0
|
||||
[ "$py3ver" -ge 9 ] && nose=pynose || nose=nose
|
||||
$PIP -qq show $nose || $PIP install $nose
|
||||
- name: Install nose for other Python 2
|
||||
if: ${{ matrix.python-impl == 'jython' || (matrix.python-impl == 'cpython' && matrix.python-version == '2.6') }}
|
||||
shell: bash
|
||||
run: |
|
||||
# Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
|
||||
$PIP -qq show nose || { \
|
||||
curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \
|
||||
$PIP install nose-1.3.7-py2-none-any.whl; }
|
||||
- name: Install nose for other Python 3
|
||||
if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
|
||||
shell: bash
|
||||
run: |
|
||||
$PIP -qq show nose || { \
|
||||
curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \
|
||||
$PIP install nose-1.3.7-py3-none-any.whl; }
|
||||
- name: Set up nosetest test
|
||||
if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
|
||||
shell: bash
|
||||
run: |
|
||||
# set PYTHON_VER
|
||||
PYTHON_VER=${{ matrix.python-version }}
|
||||
[ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}"
|
||||
echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV"
|
||||
echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV"
|
||||
# define a test to validate the Python version used by nosetests
|
||||
printf '%s\n' \
|
||||
'from __future__ import unicode_literals' \
|
||||
'import sys, os, platform' \
|
||||
'try:' \
|
||||
' import unittest2 as unittest' \
|
||||
'except ImportError:' \
|
||||
' import unittest' \
|
||||
'class TestPython(unittest.TestCase):' \
|
||||
' def setUp(self):' \
|
||||
' self.ver = os.environ["PYTHON_VER"].split("-")' \
|
||||
' def test_python_ver(self):' \
|
||||
' self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \
|
||||
' self.assertTrue(sys.version.startswith(self.ver[-1]))' \
|
||||
' self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \
|
||||
' def test_python_impl(self):' \
|
||||
' self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
|
||||
> test/test_python.py
|
||||
#-------- TESTS -------
|
||||
- name: Run tests
|
||||
if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
|
||||
continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
|
||||
env:
|
||||
YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
|
||||
run: ./devscripts/run_tests.${{ matrix.run-tests-ext }}
|
||||
run: |
|
||||
./devscripts/run_tests.${{ matrix.run-tests-ext }}
|
||||
flake8:
|
||||
name: Linter
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install flake8
|
||||
run: pip install flake8
|
||||
- name: Run flake8
|
||||
run: flake8 .
|
||||
|
||||
|
||||
@ -0,0 +1 @@
|
||||
# Empty file needed to make devscripts.utils properly importable from outside
|
||||
@ -0,0 +1,62 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import os.path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
dirn = os.path.dirname
|
||||
|
||||
sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
|
||||
|
||||
from youtube_dl.compat import (
|
||||
compat_kwargs,
|
||||
compat_open as open,
|
||||
)
|
||||
|
||||
|
||||
def read_file(fname):
|
||||
with open(fname, encoding='utf-8') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def write_file(fname, content, mode='w'):
|
||||
with open(fname, mode, encoding='utf-8') as f:
|
||||
return f.write(content)
|
||||
|
||||
|
||||
def read_version(fname='youtube_dl/version.py'):
|
||||
"""Get the version without importing the package"""
|
||||
exec(compile(read_file(fname), fname, 'exec'))
|
||||
return locals()['__version__']
|
||||
|
||||
|
||||
def get_filename_args(has_infile=False, default_outfile=None):
|
||||
parser = argparse.ArgumentParser()
|
||||
if has_infile:
|
||||
parser.add_argument('infile', help='Input file')
|
||||
kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {}
|
||||
kwargs['help'] = 'Output file'
|
||||
parser.add_argument('outfile', **compat_kwargs(kwargs))
|
||||
|
||||
opts = parser.parse_args()
|
||||
if has_infile:
|
||||
return opts.infile, opts.outfile
|
||||
return opts.outfile
|
||||
|
||||
|
||||
def compose_functions(*functions):
|
||||
return lambda x: functools.reduce(lambda y, f: f(y), functions, x)
|
||||
|
||||
|
||||
def run_process(*args, **kwargs):
|
||||
kwargs.setdefault('text', True)
|
||||
kwargs.setdefault('check', True)
|
||||
kwargs.setdefault('capture_output', True)
|
||||
if kwargs['text']:
|
||||
kwargs.setdefault('encoding', 'utf-8')
|
||||
kwargs.setdefault('errors', 'replace')
|
||||
kwargs = compat_kwargs(kwargs)
|
||||
return subprocess.run(args, **kwargs)
|
||||
@ -0,0 +1,69 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_str
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
merge_dicts,
|
||||
T,
|
||||
traverse_obj,
|
||||
unified_timestamp,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class ClipchampIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?clipchamp\.com/watch/(?P<id>[\w-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://clipchamp.com/watch/gRXZ4ZhdDaU',
|
||||
'info_dict': {
|
||||
'id': 'gRXZ4ZhdDaU',
|
||||
'ext': 'mp4',
|
||||
'title': 'Untitled video',
|
||||
'uploader': 'Alexander Schwartz',
|
||||
'timestamp': 1680805580,
|
||||
'upload_date': '20230406',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'm3u8',
|
||||
'format': 'bestvideo',
|
||||
},
|
||||
}]
|
||||
|
||||
_STREAM_URL_TMPL = 'https://%s.cloudflarestream.com/%s/manifest/video.%s'
|
||||
_STREAM_URL_QUERY = {'parentOrigin': 'https://clipchamp.com'}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['video']
|
||||
|
||||
storage_location = data.get('storage_location')
|
||||
if storage_location != 'cf_stream':
|
||||
raise ExtractorError('Unsupported clip storage location "%s"' % (storage_location,))
|
||||
|
||||
path = data['download_url']
|
||||
iframe = self._download_webpage(
|
||||
'https://iframe.cloudflarestream.com/' + path, video_id, 'Downloading player iframe')
|
||||
subdomain = self._search_regex(
|
||||
r'''\bcustomer-domain-prefix\s*=\s*("|')(?P<sd>[\w-]+)\1''', iframe,
|
||||
'subdomain', group='sd', fatal=False) or 'customer-2ut9yn3y6fta1yxe'
|
||||
|
||||
formats = self._extract_mpd_formats(
|
||||
self._STREAM_URL_TMPL % (subdomain, path, 'mpd'), video_id,
|
||||
query=self._STREAM_URL_QUERY, fatal=False, mpd_id='dash')
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
self._STREAM_URL_TMPL % (subdomain, path, 'm3u8'), video_id, 'mp4',
|
||||
query=self._STREAM_URL_QUERY, fatal=False, m3u8_id='hls'))
|
||||
|
||||
return merge_dicts({
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'uploader': ' '.join(traverse_obj(data, ('creator', ('first_name', 'last_name'), T(compat_str)))) or None,
|
||||
}, traverse_obj(data, {
|
||||
'title': ('project', 'project_name', T(compat_str)),
|
||||
'timestamp': ('created_at', T(unified_timestamp)),
|
||||
'thumbnail': ('thumbnail_url', T(url_or_none)),
|
||||
}), rev=True)
|
||||
@ -0,0 +1,204 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import (
|
||||
compat_str,
|
||||
)
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
extract_attributes,
|
||||
int_or_none,
|
||||
merge_dicts,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
variadic,
|
||||
)
|
||||
|
||||
|
||||
class DLFBaseIE(InfoExtractor):
|
||||
_VALID_URL_BASE = r'https?://(?:www\.)?deutschlandfunk\.de/'
|
||||
_BUTTON_REGEX = r'(<button[^>]+alt="Anhören"[^>]+data-audio-diraid[^>]*>)'
|
||||
|
||||
def _parse_button_attrs(self, button, audio_id=None):
|
||||
attrs = extract_attributes(button)
|
||||
audio_id = audio_id or attrs['data-audio-diraid']
|
||||
|
||||
url = traverse_obj(
|
||||
attrs, 'data-audio-download-src', 'data-audio', 'data-audioreference',
|
||||
'data-audio-src', expected_type=url_or_none)
|
||||
ext = determine_ext(url)
|
||||
formats = (self._extract_m3u8_formats(url, audio_id, fatal=False)
|
||||
if ext == 'm3u8' else [{'url': url, 'ext': ext, 'vcodec': 'none'}])
|
||||
self._sort_formats(formats)
|
||||
|
||||
def traverse_attrs(path):
|
||||
path = list(variadic(path))
|
||||
t = path.pop() if callable(path[-1]) else None
|
||||
return traverse_obj(attrs, path, expected_type=t, get_all=False)
|
||||
|
||||
def txt_or_none(v, default=None):
|
||||
return default if v is None else (compat_str(v).strip() or default)
|
||||
|
||||
return merge_dicts(*reversed([{
|
||||
'id': audio_id,
|
||||
# 'extractor_key': DLFIE.ie_key(),
|
||||
# 'extractor': DLFIE.IE_NAME,
|
||||
'formats': formats,
|
||||
}, dict((k, traverse_attrs(v)) for k, v in {
|
||||
'title': (('data-audiotitle', 'data-audio-title', 'data-audio-download-tracking-title'), txt_or_none),
|
||||
'duration': (('data-audioduration', 'data-audio-duration'), int_or_none),
|
||||
'thumbnail': ('data-audioimage', url_or_none),
|
||||
'uploader': 'data-audio-producer',
|
||||
'series': 'data-audio-series',
|
||||
'channel': 'data-audio-origin-site-name',
|
||||
'webpage_url': ('data-audio-download-tracking-path', url_or_none),
|
||||
}.items())]))
|
||||
|
||||
|
||||
class DLFIE(DLFBaseIE):
|
||||
IE_NAME = 'dlf'
|
||||
_VALID_URL = DLFBaseIE._VALID_URL_BASE + r'[\w-]+-dlf-(?P<id>[\da-f]{8})-100\.html'
|
||||
_TESTS = [
|
||||
# Audio as an HLS stream
|
||||
{
|
||||
'url': 'https://www.deutschlandfunk.de/tanz-der-saiteninstrumente-das-wild-strings-trio-aus-slowenien-dlf-03a3eb19-100.html',
|
||||
'info_dict': {
|
||||
'id': '03a3eb19',
|
||||
'title': r're:Tanz der Saiteninstrumente [-/] Das Wild Strings Trio aus Slowenien',
|
||||
'ext': 'm4a',
|
||||
'duration': 3298,
|
||||
'thumbnail': 'https://assets.deutschlandfunk.de/FALLBACK-IMAGE-AUDIO/512x512.png?t=1603714364673',
|
||||
'uploader': 'Deutschlandfunk',
|
||||
'series': 'On Stage',
|
||||
'channel': 'deutschlandfunk'
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'm3u8'
|
||||
},
|
||||
'skip': 'This webpage no longer exists'
|
||||
}, {
|
||||
'url': 'https://www.deutschlandfunk.de/russische-athleten-kehren-zurueck-auf-die-sportbuehne-ein-gefaehrlicher-tueroeffner-dlf-d9cc1856-100.html',
|
||||
'info_dict': {
|
||||
'id': 'd9cc1856',
|
||||
'title': 'Russische Athleten kehren zurück auf die Sportbühne: Ein gefährlicher Türöffner',
|
||||
'ext': 'mp3',
|
||||
'duration': 291,
|
||||
'thumbnail': 'https://assets.deutschlandfunk.de/FALLBACK-IMAGE-AUDIO/512x512.png?t=1603714364673',
|
||||
'uploader': 'Deutschlandfunk',
|
||||
'series': 'Kommentare und Themen der Woche',
|
||||
'channel': 'deutschlandfunk'
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
audio_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, audio_id)
|
||||
|
||||
return self._parse_button_attrs(
|
||||
self._search_regex(self._BUTTON_REGEX, webpage, 'button'), audio_id)
|
||||
|
||||
|
||||
class DLFCorpusIE(DLFBaseIE):
|
||||
IE_NAME = 'dlf:corpus'
|
||||
IE_DESC = 'DLF Multi-feed Archives'
|
||||
_VALID_URL = DLFBaseIE._VALID_URL_BASE + r'(?P<id>(?![\w-]+-dlf-[\da-f]{8})[\w-]+-\d+)\.html'
|
||||
_TESTS = [
|
||||
# Recorded news broadcast with referrals to related broadcasts
|
||||
{
|
||||
'url': 'https://www.deutschlandfunk.de/fechten-russland-belarus-ukraine-protest-100.html',
|
||||
'info_dict': {
|
||||
'id': 'fechten-russland-belarus-ukraine-protest-100',
|
||||
'title': r're:Wiederzulassung als neutrale Athleten [-/] Was die Rückkehr russischer und belarussischer Sportler beim Fechten bedeutet',
|
||||
'description': 'md5:91340aab29c71aa7518ad5be13d1e8ad'
|
||||
},
|
||||
'playlist_mincount': 5,
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '1fc5d64a',
|
||||
'title': r're:Wiederzulassung als neutrale Athleten [-/] Was die Rückkehr russischer und belarussischer Sportler beim Fechten bedeutet',
|
||||
'ext': 'mp3',
|
||||
'duration': 252,
|
||||
'thumbnail': 'https://assets.deutschlandfunk.de/aad16241-6b76-4a09-958b-96d0ee1d6f57/512x512.jpg?t=1679480020313',
|
||||
'uploader': 'Deutschlandfunk',
|
||||
'series': 'Sport',
|
||||
'channel': 'deutschlandfunk'
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '2ada145f',
|
||||
'title': r're:(?:Sportpolitik / )?Fechtverband votiert für Rückkehr russischer Athleten',
|
||||
'ext': 'mp3',
|
||||
'duration': 336,
|
||||
'thumbnail': 'https://assets.deutschlandfunk.de/FILE_93982766f7317df30409b8a184ac044a/512x512.jpg?t=1678547581005',
|
||||
'uploader': 'Deutschlandfunk',
|
||||
'series': 'Deutschlandfunk Nova',
|
||||
'channel': 'deutschlandfunk-nova'
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '5e55e8c9',
|
||||
'title': r're:Wiederzulassung von Russland und Belarus [-/] "Herumlavieren" des Fechter-Bundes sorgt für Unverständnis',
|
||||
'ext': 'mp3',
|
||||
'duration': 187,
|
||||
'thumbnail': 'https://assets.deutschlandfunk.de/a595989d-1ed1-4a2e-8370-b64d7f11d757/512x512.jpg?t=1679173825412',
|
||||
'uploader': 'Deutschlandfunk',
|
||||
'series': 'Sport am Samstag',
|
||||
'channel': 'deutschlandfunk'
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '47e1a096',
|
||||
'title': r're:Rückkehr Russlands im Fechten [-/] "Fassungslos, dass es einfach so passiert ist"',
|
||||
'ext': 'mp3',
|
||||
'duration': 602,
|
||||
'thumbnail': 'https://assets.deutschlandfunk.de/da4c494a-21cc-48b4-9cc7-40e09fd442c2/512x512.jpg?t=1678562155770',
|
||||
'uploader': 'Deutschlandfunk',
|
||||
'series': 'Sport am Samstag',
|
||||
'channel': 'deutschlandfunk'
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '5e55e8c9',
|
||||
'title': r're:Wiederzulassung von Russland und Belarus [-/] "Herumlavieren" des Fechter-Bundes sorgt für Unverständnis',
|
||||
'ext': 'mp3',
|
||||
'duration': 187,
|
||||
'thumbnail': 'https://assets.deutschlandfunk.de/a595989d-1ed1-4a2e-8370-b64d7f11d757/512x512.jpg?t=1679173825412',
|
||||
'uploader': 'Deutschlandfunk',
|
||||
'series': 'Sport am Samstag',
|
||||
'channel': 'deutschlandfunk'
|
||||
}
|
||||
}]
|
||||
},
|
||||
# Podcast feed with tag buttons, playlist count fluctuates
|
||||
{
|
||||
'url': 'https://www.deutschlandfunk.de/kommentare-und-themen-der-woche-100.html',
|
||||
'info_dict': {
|
||||
'id': 'kommentare-und-themen-der-woche-100',
|
||||
'title': 'Meinung - Kommentare und Themen der Woche',
|
||||
'description': 'md5:2901bbd65cd2d45e116d399a099ce5d5',
|
||||
},
|
||||
'playlist_mincount': 10,
|
||||
},
|
||||
# Podcast feed with no description
|
||||
{
|
||||
'url': 'https://www.deutschlandfunk.de/podcast-tolle-idee-100.html',
|
||||
'info_dict': {
|
||||
'id': 'podcast-tolle-idee-100',
|
||||
'title': 'Wissenschaftspodcast - Tolle Idee! - Was wurde daraus?',
|
||||
},
|
||||
'playlist_mincount': 11,
|
||||
},
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
playlist_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, playlist_id)
|
||||
|
||||
return self.playlist_result(
|
||||
map(self._parse_button_attrs, re.findall(self._BUTTON_REGEX, webpage)),
|
||||
playlist_id, self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None),
|
||||
self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage, default=None))
|
||||
@ -0,0 +1,273 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
join_nonempty,
|
||||
merge_dicts,
|
||||
parse_duration,
|
||||
str_or_none,
|
||||
T,
|
||||
traverse_obj,
|
||||
unified_strdate,
|
||||
unified_timestamp,
|
||||
urlhandle_detect_ext,
|
||||
)
|
||||
|
||||
|
||||
class GlobalPlayerBaseIE(InfoExtractor):
|
||||
|
||||
def _get_page_props(self, url, video_id):
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
return self._search_nextjs_data(webpage, video_id)['props']['pageProps']
|
||||
|
||||
def _request_ext(self, url, video_id):
|
||||
return urlhandle_detect_ext(self._request_webpage( # Server rejects HEAD requests
|
||||
url, video_id, note='Determining source extension'))
|
||||
|
||||
@staticmethod
|
||||
def _clean_desc(x):
|
||||
x = clean_html(x)
|
||||
if x:
|
||||
x = x.replace('\xa0', ' ')
|
||||
return x
|
||||
|
||||
def _extract_audio(self, episode, series):
|
||||
|
||||
return merge_dicts({
|
||||
'vcodec': 'none',
|
||||
}, traverse_obj(series, {
|
||||
'series': 'title',
|
||||
'series_id': 'id',
|
||||
'thumbnail': 'imageUrl',
|
||||
'uploader': 'itunesAuthor', # podcasts only
|
||||
}), traverse_obj(episode, {
|
||||
'id': 'id',
|
||||
'description': ('description', T(self._clean_desc)),
|
||||
'duration': ('duration', T(parse_duration)),
|
||||
'thumbnail': 'imageUrl',
|
||||
'url': 'streamUrl',
|
||||
'timestamp': (('pubDate', 'startDate'), T(unified_timestamp)),
|
||||
'title': 'title',
|
||||
}, get_all=False), rev=True)
|
||||
|
||||
|
||||
class GlobalPlayerLiveIE(GlobalPlayerBaseIE):
|
||||
_VALID_URL = r'https?://www\.globalplayer\.com/live/(?P<id>\w+)/\w+'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.globalplayer.com/live/smoothchill/uk/',
|
||||
'info_dict': {
|
||||
'id': '2mx1E',
|
||||
'ext': 'aac',
|
||||
'display_id': 'smoothchill-uk',
|
||||
'title': 're:^Smooth Chill.+$',
|
||||
'thumbnail': 'https://herald.musicradio.com/media/f296ade8-50c9-4f60-911f-924e96873620.png',
|
||||
'description': 'Music To Chill To',
|
||||
# 'live_status': 'is_live',
|
||||
'is_live': True,
|
||||
},
|
||||
}, {
|
||||
# national station
|
||||
'url': 'https://www.globalplayer.com/live/heart/uk/',
|
||||
'info_dict': {
|
||||
'id': '2mwx4',
|
||||
'ext': 'aac',
|
||||
'description': 'turn up the feel good!',
|
||||
'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
|
||||
# 'live_status': 'is_live',
|
||||
'is_live': True,
|
||||
'title': 're:^Heart UK.+$',
|
||||
'display_id': 'heart-uk',
|
||||
},
|
||||
}, {
|
||||
# regional variation
|
||||
'url': 'https://www.globalplayer.com/live/heart/london/',
|
||||
'info_dict': {
|
||||
'id': 'AMqg',
|
||||
'ext': 'aac',
|
||||
'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
|
||||
'title': 're:^Heart London.+$',
|
||||
# 'live_status': 'is_live',
|
||||
'is_live': True,
|
||||
'display_id': 'heart-london',
|
||||
'description': 'turn up the feel good!',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
station = self._get_page_props(url, video_id)['station']
|
||||
stream_url = station['streamUrl']
|
||||
|
||||
return merge_dicts({
|
||||
'id': station['id'],
|
||||
'display_id': (
|
||||
join_nonempty('brandSlug', 'slug', from_dict=station)
|
||||
or station.get('legacyStationPrefix')),
|
||||
'url': stream_url,
|
||||
'ext': self._request_ext(stream_url, video_id),
|
||||
'vcodec': 'none',
|
||||
'is_live': True,
|
||||
}, {
|
||||
'title': self._live_title(traverse_obj(
|
||||
station, (('name', 'brandName'), T(str_or_none)),
|
||||
get_all=False)),
|
||||
}, traverse_obj(station, {
|
||||
'description': 'tagline',
|
||||
'thumbnail': 'brandLogo',
|
||||
}), rev=True)
|
||||
|
||||
|
||||
class GlobalPlayerLivePlaylistIE(GlobalPlayerBaseIE):
|
||||
_VALID_URL = r'https?://www\.globalplayer\.com/playlists/(?P<id>\w+)'
|
||||
_TESTS = [{
|
||||
# "live playlist"
|
||||
'url': 'https://www.globalplayer.com/playlists/8bLk/',
|
||||
'info_dict': {
|
||||
'id': '8bLk',
|
||||
'ext': 'aac',
|
||||
# 'live_status': 'is_live',
|
||||
'is_live': True,
|
||||
'description': r're:(?s).+\bclassical\b.+\bClassic FM Hall [oO]f Fame\b',
|
||||
'thumbnail': 'https://images.globalplayer.com/images/551379?width=450&signature=oMLPZIoi5_dBSHnTMREW0Xg76mA=',
|
||||
'title': 're:Classic FM Hall of Fame.+$'
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
station = self._get_page_props(url, video_id)['playlistData']
|
||||
stream_url = station['streamUrl']
|
||||
|
||||
return merge_dicts({
|
||||
'id': video_id,
|
||||
'url': stream_url,
|
||||
'ext': self._request_ext(stream_url, video_id),
|
||||
'vcodec': 'none',
|
||||
'is_live': True,
|
||||
}, traverse_obj(station, {
|
||||
'title': 'title',
|
||||
'description': ('description', T(self._clean_desc)),
|
||||
'thumbnail': 'image',
|
||||
}), rev=True)
|
||||
|
||||
|
||||
class GlobalPlayerAudioIE(GlobalPlayerBaseIE):
|
||||
_VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)/|catchup/\w+/\w+/)(?P<id>\w+)/?(?:$|[?#])'
|
||||
_TESTS = [{
|
||||
# podcast
|
||||
'url': 'https://www.globalplayer.com/podcasts/42KuaM/',
|
||||
'playlist_mincount': 5,
|
||||
'info_dict': {
|
||||
'id': '42KuaM',
|
||||
'title': 'Filthy Ritual',
|
||||
'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
|
||||
'categories': ['Society & Culture', 'True Crime'],
|
||||
'uploader': 'Global',
|
||||
'description': r're:(?s).+\bscam\b.+?\bseries available now\b',
|
||||
},
|
||||
}, {
|
||||
# radio catchup
|
||||
'url': 'https://www.globalplayer.com/catchup/lbc/uk/46vyD7z/',
|
||||
'playlist_mincount': 2,
|
||||
'info_dict': {
|
||||
'id': '46vyD7z',
|
||||
'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
|
||||
'title': 'Nick Ferrari',
|
||||
'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
|
||||
props = self._get_page_props(url, video_id)
|
||||
series = props['podcastInfo'] if podcast else props['catchupInfo']
|
||||
|
||||
return merge_dicts({
|
||||
'_type': 'playlist',
|
||||
'id': video_id,
|
||||
'entries': [self._extract_audio(ep, series) for ep in traverse_obj(
|
||||
series, ('episodes', lambda _, v: v['id'] and v['streamUrl']))],
|
||||
'categories': traverse_obj(series, ('categories', Ellipsis, 'name')) or None,
|
||||
}, traverse_obj(series, {
|
||||
'description': ('description', T(self._clean_desc)),
|
||||
'thumbnail': 'imageUrl',
|
||||
'title': 'title',
|
||||
'uploader': 'itunesAuthor', # podcasts only
|
||||
}), rev=True)
|
||||
|
||||
|
||||
class GlobalPlayerAudioEpisodeIE(GlobalPlayerBaseIE):
|
||||
_VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)|catchup/\w+/\w+)/episodes/(?P<id>\w+)/?(?:$|[?#])'
|
||||
_TESTS = [{
|
||||
# podcast
|
||||
'url': 'https://www.globalplayer.com/podcasts/episodes/7DrfNnE/',
|
||||
'info_dict': {
|
||||
'id': '7DrfNnE',
|
||||
'ext': 'mp3',
|
||||
'title': 'Filthy Ritual - Trailer',
|
||||
'description': 'md5:1f1562fd0f01b4773b590984f94223e0',
|
||||
'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
|
||||
'duration': 225.0,
|
||||
'timestamp': 1681254900,
|
||||
'series': 'Filthy Ritual',
|
||||
'series_id': '42KuaM',
|
||||
'upload_date': '20230411',
|
||||
'uploader': 'Global',
|
||||
},
|
||||
}, {
|
||||
# radio catchup
|
||||
'url': 'https://www.globalplayer.com/catchup/lbc/uk/episodes/2zGq26Vcv1fCWhddC4JAwETXWe/',
|
||||
'only_matching': True,
|
||||
# expired: refresh the details with a current show for a full test
|
||||
'info_dict': {
|
||||
'id': '2zGq26Vcv1fCWhddC4JAwETXWe',
|
||||
'ext': 'm4a',
|
||||
'timestamp': 1682056800,
|
||||
'series': 'Nick Ferrari',
|
||||
'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
|
||||
'upload_date': '20230421',
|
||||
'series_id': '46vyD7z',
|
||||
'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
|
||||
'title': 'Nick Ferrari',
|
||||
'duration': 10800.0,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
|
||||
props = self._get_page_props(url, video_id)
|
||||
episode = props['podcastEpisode'] if podcast else props['catchupEpisode']
|
||||
|
||||
return self._extract_audio(
|
||||
episode, traverse_obj(episode, 'podcast', 'show', expected_type=dict) or {})
|
||||
|
||||
|
||||
class GlobalPlayerVideoIE(GlobalPlayerBaseIE):
|
||||
_VALID_URL = r'https?://www\.globalplayer\.com/videos/(?P<id>\w+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.globalplayer.com/videos/2JsSZ7Gm2uP/',
|
||||
'info_dict': {
|
||||
'id': '2JsSZ7Gm2uP',
|
||||
'ext': 'mp4',
|
||||
'description': 'md5:6a9f063c67c42f218e42eee7d0298bfd',
|
||||
'thumbnail': 'md5:d4498af48e15aae4839ce77b97d39550',
|
||||
'upload_date': '20230420',
|
||||
'title': 'Treble Malakai Bayoh sings a sublime Handel aria at Classic FM Live',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
meta = self._get_page_props(url, video_id)['videoData']
|
||||
|
||||
return merge_dicts({
|
||||
'id': video_id,
|
||||
}, traverse_obj(meta, {
|
||||
'url': 'url',
|
||||
'thumbnail': ('image', 'url'),
|
||||
'title': 'title',
|
||||
'upload_date': ('publish_date', T(unified_strdate)),
|
||||
'description': 'description',
|
||||
}), rev=True)
|
||||
@ -0,0 +1,124 @@
|
||||
# coding: utf-8
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from functools import partial as partial_f
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
float_or_none,
|
||||
merge_dicts,
|
||||
T,
|
||||
traverse_obj,
|
||||
txt_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class S4CIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?s4c\.cymru/clic/programme/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.s4c.cymru/clic/programme/861362209',
|
||||
'info_dict': {
|
||||
'id': '861362209',
|
||||
'ext': 'mp4',
|
||||
'title': 'Y Swn',
|
||||
'description': 'md5:f7681a30e4955b250b3224aa9fe70cf0',
|
||||
'duration': 5340,
|
||||
'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Y_Swn_2023S4C_099_ii.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.s4c.cymru/clic/programme/856636948',
|
||||
'info_dict': {
|
||||
'id': '856636948',
|
||||
'ext': 'mp4',
|
||||
'title': 'Am Dro',
|
||||
'duration': 2880,
|
||||
'description': 'md5:100d8686fc9a632a0cb2db52a3433ffe',
|
||||
'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Am_Dro_2022-23S4C_P6_4005.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
details = self._download_json(
|
||||
'https://www.s4c.cymru/df/full_prog_details',
|
||||
video_id, query={
|
||||
'lang': 'e',
|
||||
'programme_id': video_id,
|
||||
}, fatal=False)
|
||||
|
||||
player_config = self._download_json(
|
||||
'https://player-api.s4c-cdn.co.uk/player-configuration/prod', video_id, query={
|
||||
'programme_id': video_id,
|
||||
'signed': '0',
|
||||
'lang': 'en',
|
||||
'mode': 'od',
|
||||
'appId': 'clic',
|
||||
'streamName': '',
|
||||
}, note='Downloading player config JSON')
|
||||
|
||||
m3u8_url = self._download_json(
|
||||
'https://player-api.s4c-cdn.co.uk/streaming-urls/prod', video_id, query={
|
||||
'mode': 'od',
|
||||
'application': 'clic',
|
||||
'region': 'WW',
|
||||
'extra': 'false',
|
||||
'thirdParty': 'false',
|
||||
'filename': player_config['filename'],
|
||||
}, note='Downloading streaming urls JSON')['hls']
|
||||
formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native')
|
||||
self._sort_formats(formats)
|
||||
|
||||
subtitles = {}
|
||||
for sub in traverse_obj(player_config, ('subtitles', lambda _, v: url_or_none(v['0']))):
|
||||
subtitles.setdefault(sub.get('3', 'en'), []).append({
|
||||
'url': sub['0'],
|
||||
'name': sub.get('1'),
|
||||
})
|
||||
|
||||
return merge_dicts({
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnail': url_or_none(player_config.get('poster')),
|
||||
}, traverse_obj(details, ('full_prog_details', 0, {
|
||||
'title': (('programme_title', 'series_title'), T(txt_or_none)),
|
||||
'description': ('full_billing', T(txt_or_none)),
|
||||
'duration': ('duration', T(partial_f(float_or_none, invscale=60))),
|
||||
}), get_all=False),
|
||||
rev=True)
|
||||
|
||||
|
||||
class S4CSeriesIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?s4c\.cymru/clic/series/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.s4c.cymru/clic/series/864982911',
|
||||
'playlist_mincount': 6,
|
||||
'info_dict': {
|
||||
'id': '864982911',
|
||||
'title': 'Iaith ar Daith',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.s4c.cymru/clic/series/866852587',
|
||||
'playlist_mincount': 8,
|
||||
'info_dict': {
|
||||
'id': '866852587',
|
||||
'title': 'FFIT Cymru',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
series_id = self._match_id(url)
|
||||
series_details = self._download_json(
|
||||
'https://www.s4c.cymru/df/series_details', series_id, query={
|
||||
'lang': 'e',
|
||||
'series_id': series_id,
|
||||
'show_prog_in_series': 'Y'
|
||||
}, note='Downloading series details JSON')
|
||||
|
||||
return self.playlist_result(
|
||||
(self.url_result('https://www.s4c.cymru/clic/programme/' + episode_id, S4CIE, episode_id)
|
||||
for episode_id in traverse_obj(series_details, ('other_progs_in_series', Ellipsis, 'id'))),
|
||||
playlist_id=series_id, playlist_title=traverse_obj(
|
||||
series_details, ('full_prog_details', 0, 'series_title', T(txt_or_none))))
|
||||
@ -0,0 +1,55 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
float_or_none,
|
||||
merge_dicts,
|
||||
str_or_none,
|
||||
T,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class WhypIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?whyp\.it/tracks/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.whyp.it/tracks/18337/home-page-example-track-b4kq7',
|
||||
'md5': 'c1187b42ebf8605284e3dc92aeb33d16',
|
||||
'info_dict': {
|
||||
'url': 'https://cdn.whyp.it/50eb17cc-e9ff-4e18-b89b-dc9206a95cb1.mp3',
|
||||
'id': '18337',
|
||||
'title': 'Home Page Example Track',
|
||||
'description': r're:(?s).+\bexample track\b',
|
||||
'ext': 'mp3',
|
||||
'duration': 52.82,
|
||||
'uploader': 'Brad',
|
||||
'uploader_id': '1',
|
||||
'thumbnail': 'https://cdn.whyp.it/a537bb36-3373-4c61-96c8-27fc1b2f427a.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.whyp.it/tracks/18337',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
unique_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, unique_id)
|
||||
data = self._search_nuxt_data(webpage, unique_id)['rawTrack']
|
||||
|
||||
return merge_dicts({
|
||||
'url': data['audio_url'],
|
||||
'id': unique_id,
|
||||
}, traverse_obj(data, {
|
||||
'title': 'title',
|
||||
'description': 'description',
|
||||
'duration': ('duration', T(float_or_none)),
|
||||
'uploader': ('user', 'username'),
|
||||
'uploader_id': ('user', 'id', T(str_or_none)),
|
||||
'thumbnail': ('artwork_url', T(url_or_none)),
|
||||
}), {
|
||||
'ext': 'mp3',
|
||||
'vcodec': 'none',
|
||||
'http_headers': {'Referer': 'https://whyp.it/'},
|
||||
}, rev=True)
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue