From 8c297d184ccf81fedbadb99140e5c15236e841f2 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Mon, 16 Jan 2023 04:53:41 +0100 Subject: [PATCH] Its lazy now at least? --- yt_dlp/YoutubeDL.py | 3 +- yt_dlp/__init__.py | 3 +- yt_dlp/extractor/__init__.py | 20 +++++++---- yt_dlp/extractor/extractors.py | 15 +++++++-- yt_dlp/globals.py | 4 +-- yt_dlp/plugins.py | 61 +++++++++++++++++----------------- 6 files changed, 62 insertions(+), 44 deletions(-) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 27eaefdc4..badae37b1 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -28,7 +28,7 @@ from .compat import compat_os_name, compat_shlex_quote from .cookies import load_cookies from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name from .downloader.rtmp import rtmpdump_version -from .extractor import gen_extractor_classes, get_info_extractor +from .extractor import gen_extractor_classes, get_info_extractor, import_extractors from .extractor.common import UnsupportedURLIE from .extractor.openload import PhantomJSwrapper from .globals import ( @@ -3777,6 +3777,7 @@ class YoutubeDL: if not IN_CLI.get(): write_debug(f'params: {self.params}') + import_extractors() lazy_extractors = LAZY_EXTRACTORS.get() if lazy_extractors is None: write_debug('Lazy loading extractors is disabled') diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index d52ed1bf3..ad44cbfbe 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -19,7 +19,7 @@ from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS from .downloader.external import get_external_downloader from .extractor import list_extractor_classes from .extractor.adobepass import MSO_INFO -from .globals import IN_CLI +from .globals import IN_CLI, plugin_dirs from .options import parseOpts from .plugins import load_all_plugin_types from .postprocessor.ffmpeg import ( @@ -931,6 +931,7 @@ def _real_main(argv=None): FFmpegPostProcessor._ffmpeg_location.set(opts.ffmpeg_location) # load all plugins into the global lookup + plugin_dirs.set(opts.plugin_dirs) load_all_plugin_types() with YoutubeDL(ydl_opts) as ydl: diff --git a/yt_dlp/extractor/__init__.py b/yt_dlp/extractor/__init__.py index 515b7d60c..233fbf0c4 100644 --- a/yt_dlp/extractor/__init__.py +++ b/yt_dlp/extractor/__init__.py @@ -1,16 +1,17 @@ -from .extractors import * -from ..globals import extractors as _extractor_classes +from ..compat.compat_utils import passthrough_module +from ..globals import extractors as _extractors_context -# from ..compat.compat_utils import passthrough_module -# passthrough_module(__name__, '.extractors') -# del passthrough_module + +passthrough_module(__name__, '.extractors') +del passthrough_module def gen_extractor_classes(): """ Return a list of supported extractors. The order does matter; the first extractor matched is the one handling the URL. """ - return list(_extractor_classes.get().values()) + import_extractors() + return list(_extractors_context.get().values()) def gen_extractors(): @@ -37,4 +38,9 @@ def list_extractors(age_limit=None): def get_info_extractor(ie_name): """Returns the info extractor class with the given ie_name""" - return _extractor_classes.get()[f'{ie_name}IE'] + import_extractors() + return _extractors_context.get()[f'{ie_name}IE'] + + +def import_extractors(): + from . import extractors # noqa: F401 diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py index edf2deb02..bf806fef4 100644 --- a/yt_dlp/extractor/extractors.py +++ b/yt_dlp/extractor/extractors.py @@ -1,7 +1,7 @@ import inspect import os -from ..globals import LAZY_EXTRACTORS, extractors +from ..globals import LAZY_EXTRACTORS, extractors as _extractors_context _CLASS_LOOKUP = None if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'): @@ -21,5 +21,14 @@ if not _CLASS_LOOKUP: } _CLASS_LOOKUP['GenericIE'] = _extractors.GenericIE -extractors.set(_CLASS_LOOKUP) -globals().update(_CLASS_LOOKUP) +# We want to append to the main lookup +_current = _extractors_context.get() +for name, ie in _CLASS_LOOKUP.items(): + _current.setdefault(name, ie) + + +def __getattr__(name): + value = _CLASS_LOOKUP.get(name) + if not value: + raise AttributeError(f'module {__name__} has no attribute {name}') + return value diff --git a/yt_dlp/globals.py b/yt_dlp/globals.py index fb01b5f63..52e98fb71 100644 --- a/yt_dlp/globals.py +++ b/yt_dlp/globals.py @@ -7,9 +7,9 @@ postprocessors = ContextVar('postprocessors', default={}) extractors = ContextVar('extractors', default={}) IN_CLI = ContextVar('IN_CLI', default=False) # `False`=force, `None`=disabled, `True`=enabled -LAZY_EXTRACTORS = ContextVar('LAZY_EXTRACTORS') +LAZY_EXTRACTORS = ContextVar('LAZY_EXTRACTORS', default=False) -plugin_dirs = ContextVar('plugin_dirs') +plugin_dirs = ContextVar('plugin_dirs', default=(..., )) plugin_ies = ContextVar('plugin_ies', default={}) plugin_overrides = ContextVar('plugin_overrides', default=defaultdict(list)) plugin_pps = ContextVar('plugin_pps', default={}) diff --git a/yt_dlp/plugins.py b/yt_dlp/plugins.py index 43de2b5d0..43e4ecf2f 100644 --- a/yt_dlp/plugins.py +++ b/yt_dlp/plugins.py @@ -28,11 +28,13 @@ from .utils import ( get_system_config_dirs, get_user_config_dirs, merge_dicts, + orderedSet, write_string, ) PACKAGE_NAME = 'yt_dlp_plugins' COMPAT_PACKAGE_NAME = 'ytdlp_plugins' +_BASE_PACKAGE_PATH = Path(__file__).parent class PluginType(enum.Enum): @@ -55,41 +57,43 @@ class PluginLoader(importlib.abc.Loader): @functools.cache def dirs_in_zip(archive): - with contextlib.suppress(FileNotFoundError): - with ZipFile(archive) as zip: + try: + with ZipFile(archive) as zip_: return set(itertools.chain.from_iterable( - Path(file).parents for file in zip.namelist())) + Path(file).parents for file in zip_.namelist())) + except FileNotFoundError: + pass + except Exception as e: + write_string(f'WARNING: Could not read zip file {archive}: {e}\n') return () def default_plugin_paths(): - seen = set() - - def _get_unique_package_paths(*root_paths, containing_folder): - for config_dir in map(Path, root_paths): - plugin_dir = config_dir / containing_folder - # if plugin_dir in seen: - # continue - seen.add(plugin_dir) - if not plugin_dir.is_dir(): + def _get_package_paths(*root_paths, containing_folder): + for config_dir in orderedSet(map(Path, root_paths), lazy=True): + # We need to filter the base path added when running __main__.py directly + if config_dir == _BASE_PACKAGE_PATH: continue - yield from plugin_dir.iterdir() + with contextlib.suppress(OSError): + for item in (config_dir / containing_folder).iterdir(): + yield item # Load from yt-dlp config folders - yield from _get_unique_package_paths( + yield from _get_package_paths( *get_user_config_dirs('yt-dlp'), *get_system_config_dirs('yt-dlp'), containing_folder='plugins') # Load from yt-dlp-plugins folders - yield from _get_unique_package_paths( + yield from _get_package_paths( get_executable_path(), *get_user_config_dirs(''), *get_system_config_dirs(''), containing_folder='yt-dlp-plugins') # Load from PYTHONPATH folders - yield from map(Path, sys.path) + yield from (path for path in map(Path, sys.path) if path != _BASE_PACKAGE_PATH) + # yield from _get_package_paths(*sys.path, containing_folder='') class PluginFinder(importlib.abc.MetaPathFinder): @@ -109,26 +113,21 @@ class PluginFinder(importlib.abc.MetaPathFinder): candidate_locations = itertools.chain.from_iterable( default_plugin_paths() if candidate is ... else Path(candidate).iterdir() - for candidate in plugin_dirs.get((..., ))) + for candidate in plugin_dirs.get()) parts = Path(*fullname.split('.')) - locations = dict() - for path in dict.fromkeys(candidate_locations): + for path in orderedSet(candidate_locations, lazy=True): candidate = path / parts - # print(candidate) if candidate.is_dir(): - locations[candidate] = None - elif path.name and any(path.with_suffix(suffix).is_file() for suffix in {'.zip', '.egg', '.whl'}): - if parts in dirs_in_zip(path): - locations[candidate] = None - - return list(map(str, locations)) + yield candidate + elif path.suffix in ('.zip', '.egg', '.whl') and parts in dirs_in_zip(path): + yield candidate def find_spec(self, fullname, path=None, target=None): if fullname not in self.packages: return None - search_locations = self.search_locations(fullname) + search_locations = list(map(str, self.search_locations(fullname))) if not search_locations: return None @@ -200,10 +199,12 @@ def load_plugins(plugin_type: PluginType): spec.loader.exec_module(plugins) classes.update(load_module(plugins, spec.name, suffix)) + regular_plugins = {} # __init_subclass__ was removed so we manually add overrides for name, klass in classes.items(): plugin_name = getattr(klass, '_plugin_name', None) if not plugin_name: + regular_plugins[name] = klass continue # FIXME: Most likely something wrong here @@ -217,13 +218,13 @@ def load_plugins(plugin_type: PluginType): plugin_overrides.get()[super_class].append(klass) # Add the classes into the global plugin lookup - plugin_destination.set(classes) + plugin_destination.set(regular_plugins) # We want to prepend to the main lookup current = destination.get() - result = merge_dicts(classes, current) + result = merge_dicts(regular_plugins, current) destination.set(result) - return classes + return result def load_all_plugin_types():