From c1714454313e01c94a7e55e1cb99d439ff933a43 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 27 Apr 2022 13:45:45 +0530 Subject: [PATCH] [cleanup,build] Cleanup some build-related code Fixes an issue in 7ab56be2c7309a2d11d4ee28c71f8fb29da21ef7 --- Makefile | 5 +++- devscripts/bash-completion.py | 2 +- devscripts/fish-completion.py | 2 +- devscripts/make_issue_template.py | 24 +++++++++--------- devscripts/make_lazy_extractors.py | 2 +- devscripts/zsh-completion.py | 2 +- pyinst.py | 39 ++++++++++++++++++------------ setup.py | 22 ++++++++++++----- 8 files changed, 60 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index 0e911feba..179aaff57 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,9 @@ clean-dist: rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \ yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap clean-cache: - find . \( -name "*.pyc" -o -name "*.class" \) -delete + find . \( \ + -type d -name .pytest_cache -o -type d -name __pycache__ -o -name "*.pyc" -o -name "*.class" \ + \) -prune -exec rm -rf {} \; completion-bash: completions/bash/yt-dlp completion-fish: completions/fish/yt-dlp.fish @@ -131,6 +133,7 @@ yt-dlp.tar.gz: all --exclude '*.pyo' \ --exclude '*~' \ --exclude '__pycache__' \ + --exclude '.pytest_cache' \ --exclude '.git' \ -- \ README.md supportedsites.md Changelog.md LICENSE \ diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 27ec7ca7a..268e8a2ae 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -24,5 +24,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = yt_dlp.parseOpts()[0] +parser = yt_dlp.parseOpts(ignore_config_files=True)[0] build_completion(parser) diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index dcb1d6582..d9c0048e2 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -44,5 +44,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = yt_dlp.parseOpts()[0] +parser = yt_dlp.parseOpts(ignore_config_files=True)[0] build_completion(parser) diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 878b94166..811a3e9b5 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -3,6 +3,17 @@ import io import optparse +def read(fname): + with open(fname, encoding='utf-8') as f: + return f.read() + + +# Get the version from yt_dlp/version.py without importing the package +def read_version(fname): + exec(compile(read(fname), fname, 'exec')) + return locals()['__version__'] + + def main(): parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') options, args = parser.parse_args() @@ -10,18 +21,9 @@ def main(): parser.error('Expected an input and an output filename') infile, outfile = args - - with open(infile, encoding='utf-8') as inf: - issue_template_tmpl = inf.read() - - # Get the version from yt_dlp/version.py without importing the package - exec(compile(open('yt_dlp/version.py').read(), - 'yt_dlp/version.py', 'exec')) - - out = issue_template_tmpl % {'version': locals()['__version__']} - with open(outfile, 'w', encoding='utf-8') as outf: - outf.write(out) + outf.write( + read(infile) % {'version': read_version('yt_dlp/version.py')}) if __name__ == '__main__': diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 5e2070602..6dc8fed90 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -21,7 +21,7 @@ from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor if os.path.exists(plugins_blocked_dirname): os.rename(plugins_blocked_dirname, plugins_dirname) -with open('devscripts/lazy_load_template.py') as f: +with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: module_template = f.read() CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 06660d8fd..59faea06a 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -43,5 +43,5 @@ def build_completion(opt_parser): f.write(template) -parser = yt_dlp.parseOpts()[0] +parser = yt_dlp.parseOpts(ignore_config_files=True)[0] build_completion(parser) diff --git a/pyinst.py b/pyinst.py index c63d879a0..bc3c58ff8 100644 --- a/pyinst.py +++ b/pyinst.py @@ -3,7 +3,7 @@ import os import platform import sys -from PyInstaller.utils.hooks import collect_submodules +from PyInstaller.__main__ import run as run_pyinstaller OS_NAME = platform.system() if OS_NAME == 'Windows': @@ -20,18 +20,22 @@ if OS_NAME == 'Windows': elif OS_NAME == 'Darwin': pass else: - raise Exception('{OS_NAME} is not supported') + raise Exception(f'{OS_NAME} is not supported') ARCH = platform.architecture()[0][:2] def main(): opts = parse_options() - version = read_version() + version = read_version('yt_dlp/version.py') + + onedir = '--onedir' in opts or '-D' in opts + if not onedir and '-F' not in opts and '--onefile' not in opts: + opts.append('--onefile') suffix = '_macos' if OS_NAME == 'Darwin' else '_x86' if ARCH == '32' else '' final_file = 'dist/%syt-dlp%s%s' % ( - 'yt-dlp/' if '--onedir' in opts else '', suffix, '.exe' if OS_NAME == 'Windows' else '') + 'yt-dlp/' if onedir else '', suffix, '.exe' if OS_NAME == 'Windows' else '') print(f'Building yt-dlp v{version} {ARCH}bit for {OS_NAME} with options {opts}') print('Remember to update the version using "devscripts/update-version.py"') @@ -45,17 +49,16 @@ def main(): '--icon=devscripts/logo.ico', '--upx-exclude=vcruntime140.dll', '--noconfirm', + # NB: Modules that are only imported dynamically must be added here. + # --collect-submodules may not work correctly if user has a yt-dlp installed via PIP + '--hidden-import=yt_dlp.compat._legacy', *dependency_options(), *opts, - '--collect-submodules=yt_dlp', 'yt_dlp/__main__.py', ] - print(f'Running PyInstaller with {opts}') - - import PyInstaller.__main__ - - PyInstaller.__main__.run(opts) + print(f'Running PyInstaller with {opts}') + run_pyinstaller(opts) set_version_info(final_file, version) @@ -66,12 +69,14 @@ def parse_options(): if ARCH != opts[0]: raise Exception(f'{opts[0]}bit executable cannot be built on a {ARCH}bit system') opts = opts[1:] - return opts or ['--onefile'] + return opts -def read_version(): - exec(compile(open('yt_dlp/version.py').read(), 'yt_dlp/version.py', 'exec')) - return locals()['__version__'] +# Get the version from yt_dlp/version.py without importing the package +def read_version(fname): + with open(fname, encoding='utf-8') as f: + exec(compile(f.read(), fname, 'exec')) + return locals()['__version__'] def version_to_list(version): @@ -80,10 +85,12 @@ def version_to_list(version): def dependency_options(): - dependencies = [pycryptodome_module(), 'mutagen', 'brotli', 'certifi'] + collect_submodules('websockets') - excluded_modules = ['test', 'ytdlp_plugins', 'youtube-dl', 'youtube-dlc'] + # Due to the current implementation, these are auto-detected, but explicitly add them just in case + dependencies = [pycryptodome_module(), 'mutagen', 'brotli', 'certifi', 'websockets'] + excluded_modules = ['test', 'ytdlp_plugins', 'youtube_dl', 'youtube_dlc'] yield from (f'--hidden-import={module}' for module in dependencies) + yield '--collect-submodules=websockets' yield from (f'--exclude-module={module}' for module in excluded_modules) diff --git a/setup.py b/setup.py index 141cb238f..89b819b1a 100644 --- a/setup.py +++ b/setup.py @@ -11,18 +11,28 @@ except ImportError: setuptools_available = False from distutils.spawn import spawn + +def read(fname): + with open(fname, encoding='utf-8') as f: + return f.read() + + # Get the version from yt_dlp/version.py without importing the package -exec(compile(open('yt_dlp/version.py').read(), 'yt_dlp/version.py', 'exec')) +def read_version(fname): + exec(compile(read(fname), fname, 'exec')) + return locals()['__version__'] + +VERSION = read_version('yt_dlp/version.py') DESCRIPTION = 'A youtube-dl fork with additional features and patches' LONG_DESCRIPTION = '\n\n'.join(( 'Official repository: ', '**PS**: Some links in this document will not work since this is a copy of the README.md from Github', - open('README.md', encoding='utf-8').read())) + read('README.md'))) -REQUIREMENTS = open('requirements.txt', encoding='utf-8').read().splitlines() +REQUIREMENTS = read('requirements.txt').splitlines() if sys.argv[1:2] == ['py2exe']: @@ -34,11 +44,11 @@ if sys.argv[1:2] == ['py2exe']: 'console': [{ 'script': './yt_dlp/__main__.py', 'dest_base': 'yt-dlp', - 'version': __version__, + 'version': VERSION, 'description': DESCRIPTION, 'comments': LONG_DESCRIPTION.split('\n')[0], 'product_name': 'yt-dlp', - 'product_version': __version__, + 'product_version': VERSION, }], 'options': { 'py2exe': { @@ -107,7 +117,7 @@ else: setup( name='yt-dlp', - version=__version__, + version=VERSION, maintainer='pukkandan', maintainer_email='pukkandan.ytdlp@gmail.com', description=DESCRIPTION,