mirror of https://github.com/yt-dlp/yt-dlp
Improve plugin architecture (#5553)
to make plugins easier to develop and use: * Plugins are now loaded as namespace packages. * Plugins can be loaded in any distribution of yt-dlp (binary, pip, source, etc.). * Plugin packages can be installed and managed via pip, or dropped into any of the documented locations. * Users do not need to edit any code files to install plugins. * Backwards-compatible with previous plugin architecture. As a side-effect, yt-dlp will now search in a few more locations for config files. Closes https://github.com/yt-dlp/yt-dlp/issues/1389 Authored by: flashdagger, coletdjnz, pukkandan, Grub4K Co-authored-by: Marcel <flashdagger@googlemail.com> Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com> Co-authored-by: Simon Sawicki <accounts@grub4k.xyz>pull/5921/head
Matthew
2 years ago
committed by
GitHub
parent
2fb0f85868
commit
8e40b9d1ec
@ -0,0 +1,73 @@
|
|||||||
|
import importlib
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
TEST_DATA_DIR = Path(os.path.dirname(os.path.abspath(__file__)), 'testdata')
|
||||||
|
sys.path.append(str(TEST_DATA_DIR))
|
||||||
|
importlib.invalidate_caches()
|
||||||
|
|
||||||
|
from yt_dlp.plugins import PACKAGE_NAME, directories, load_plugins
|
||||||
|
|
||||||
|
|
||||||
|
class TestPlugins(unittest.TestCase):
|
||||||
|
|
||||||
|
TEST_PLUGIN_DIR = TEST_DATA_DIR / PACKAGE_NAME
|
||||||
|
|
||||||
|
def test_directories_containing_plugins(self):
|
||||||
|
self.assertIn(self.TEST_PLUGIN_DIR, map(Path, directories()))
|
||||||
|
|
||||||
|
def test_extractor_classes(self):
|
||||||
|
for module_name in tuple(sys.modules):
|
||||||
|
if module_name.startswith(f'{PACKAGE_NAME}.extractor'):
|
||||||
|
del sys.modules[module_name]
|
||||||
|
plugins_ie = load_plugins('extractor', 'IE')
|
||||||
|
|
||||||
|
self.assertIn(f'{PACKAGE_NAME}.extractor.normal', sys.modules.keys())
|
||||||
|
self.assertIn('NormalPluginIE', plugins_ie.keys())
|
||||||
|
|
||||||
|
# don't load modules with underscore prefix
|
||||||
|
self.assertFalse(
|
||||||
|
f'{PACKAGE_NAME}.extractor._ignore' in sys.modules.keys(),
|
||||||
|
'loaded module beginning with underscore')
|
||||||
|
self.assertNotIn('IgnorePluginIE', plugins_ie.keys())
|
||||||
|
|
||||||
|
# Don't load extractors with underscore prefix
|
||||||
|
self.assertNotIn('_IgnoreUnderscorePluginIE', plugins_ie.keys())
|
||||||
|
|
||||||
|
# Don't load extractors not specified in __all__ (if supplied)
|
||||||
|
self.assertNotIn('IgnoreNotInAllPluginIE', plugins_ie.keys())
|
||||||
|
self.assertIn('InAllPluginIE', plugins_ie.keys())
|
||||||
|
|
||||||
|
def test_postprocessor_classes(self):
|
||||||
|
plugins_pp = load_plugins('postprocessor', 'PP')
|
||||||
|
self.assertIn('NormalPluginPP', plugins_pp.keys())
|
||||||
|
|
||||||
|
def test_importing_zipped_module(self):
|
||||||
|
zip_path = TEST_DATA_DIR / 'zipped_plugins.zip'
|
||||||
|
shutil.make_archive(str(zip_path)[:-4], 'zip', str(zip_path)[:-4])
|
||||||
|
sys.path.append(str(zip_path)) # add zip to search paths
|
||||||
|
importlib.invalidate_caches() # reset the import caches
|
||||||
|
|
||||||
|
try:
|
||||||
|
for plugin_type in ('extractor', 'postprocessor'):
|
||||||
|
package = importlib.import_module(f'{PACKAGE_NAME}.{plugin_type}')
|
||||||
|
self.assertIn(zip_path / PACKAGE_NAME / plugin_type, map(Path, package.__path__))
|
||||||
|
|
||||||
|
plugins_ie = load_plugins('extractor', 'IE')
|
||||||
|
self.assertIn('ZippedPluginIE', plugins_ie.keys())
|
||||||
|
|
||||||
|
plugins_pp = load_plugins('postprocessor', 'PP')
|
||||||
|
self.assertIn('ZippedPluginPP', plugins_pp.keys())
|
||||||
|
|
||||||
|
finally:
|
||||||
|
sys.path.remove(str(zip_path))
|
||||||
|
os.remove(zip_path)
|
||||||
|
importlib.invalidate_caches() # reset the import caches
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from yt_dlp.extractor.common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class IgnorePluginIE(InfoExtractor):
|
||||||
|
pass
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
from yt_dlp.extractor.common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class IgnoreNotInAllPluginIE(InfoExtractor):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class InAllPluginIE(InfoExtractor):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['InAllPluginIE']
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
from yt_dlp.extractor.common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class NormalPluginIE(InfoExtractor):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _IgnoreUnderscorePluginIE(InfoExtractor):
|
||||||
|
pass
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from yt_dlp.postprocessor.common import PostProcessor
|
||||||
|
|
||||||
|
|
||||||
|
class NormalPluginPP(PostProcessor):
|
||||||
|
pass
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from yt_dlp.extractor.common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class ZippedPluginIE(InfoExtractor):
|
||||||
|
pass
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from yt_dlp.postprocessor.common import PostProcessor
|
||||||
|
|
||||||
|
|
||||||
|
class ZippedPluginPP(PostProcessor):
|
||||||
|
pass
|
||||||
@ -0,0 +1,171 @@
|
|||||||
|
import contextlib
|
||||||
|
import importlib
|
||||||
|
import importlib.abc
|
||||||
|
import importlib.machinery
|
||||||
|
import importlib.util
|
||||||
|
import inspect
|
||||||
|
import itertools
|
||||||
|
import os
|
||||||
|
import pkgutil
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
import zipimport
|
||||||
|
from pathlib import Path
|
||||||
|
from zipfile import ZipFile
|
||||||
|
|
||||||
|
from .compat import functools # isort: split
|
||||||
|
from .compat import compat_expanduser
|
||||||
|
from .utils import (
|
||||||
|
get_executable_path,
|
||||||
|
get_system_config_dirs,
|
||||||
|
get_user_config_dirs,
|
||||||
|
write_string,
|
||||||
|
)
|
||||||
|
|
||||||
|
PACKAGE_NAME = 'yt_dlp_plugins'
|
||||||
|
COMPAT_PACKAGE_NAME = 'ytdlp_plugins'
|
||||||
|
|
||||||
|
|
||||||
|
class PluginLoader(importlib.abc.Loader):
|
||||||
|
"""Dummy loader for virtual namespace packages"""
|
||||||
|
|
||||||
|
def exec_module(self, module):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@functools.cache
|
||||||
|
def dirs_in_zip(archive):
|
||||||
|
with ZipFile(archive) as zip:
|
||||||
|
return set(itertools.chain.from_iterable(
|
||||||
|
Path(file).parents for file in zip.namelist()))
|
||||||
|
|
||||||
|
|
||||||
|
class PluginFinder(importlib.abc.MetaPathFinder):
|
||||||
|
"""
|
||||||
|
This class provides one or multiple namespace packages.
|
||||||
|
It searches in sys.path and yt-dlp config folders for
|
||||||
|
the existing subdirectories from which the modules can be imported
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, *packages):
|
||||||
|
self._zip_content_cache = {}
|
||||||
|
self.packages = set(itertools.chain.from_iterable(
|
||||||
|
itertools.accumulate(name.split('.'), lambda a, b: '.'.join((a, b)))
|
||||||
|
for name in packages))
|
||||||
|
|
||||||
|
def search_locations(self, fullname):
|
||||||
|
candidate_locations = []
|
||||||
|
|
||||||
|
def _get_package_paths(*root_paths, containing_folder='plugins'):
|
||||||
|
for config_dir in map(Path, root_paths):
|
||||||
|
plugin_dir = config_dir / containing_folder
|
||||||
|
if not plugin_dir.is_dir():
|
||||||
|
continue
|
||||||
|
yield from plugin_dir.iterdir()
|
||||||
|
|
||||||
|
# Load from yt-dlp config folders
|
||||||
|
candidate_locations.extend(_get_package_paths(
|
||||||
|
*get_user_config_dirs('yt-dlp'), *get_system_config_dirs('yt-dlp'),
|
||||||
|
containing_folder='plugins'))
|
||||||
|
|
||||||
|
# Load from yt-dlp-plugins folders
|
||||||
|
candidate_locations.extend(_get_package_paths(
|
||||||
|
get_executable_path(),
|
||||||
|
compat_expanduser('~'),
|
||||||
|
'/etc',
|
||||||
|
os.getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config'),
|
||||||
|
containing_folder='yt-dlp-plugins'))
|
||||||
|
|
||||||
|
candidate_locations.extend(map(Path, sys.path)) # PYTHONPATH
|
||||||
|
|
||||||
|
parts = Path(*fullname.split('.'))
|
||||||
|
locations = set()
|
||||||
|
for path in dict.fromkeys(candidate_locations):
|
||||||
|
candidate = path / parts
|
||||||
|
if candidate.is_dir():
|
||||||
|
locations.add(str(candidate))
|
||||||
|
elif path.name and any(path.with_suffix(suffix).is_file() for suffix in {'.zip', '.egg', '.whl'}):
|
||||||
|
with contextlib.suppress(FileNotFoundError):
|
||||||
|
if parts in dirs_in_zip(path):
|
||||||
|
locations.add(str(candidate))
|
||||||
|
return locations
|
||||||
|
|
||||||
|
def find_spec(self, fullname, path=None, target=None):
|
||||||
|
if fullname not in self.packages:
|
||||||
|
return None
|
||||||
|
|
||||||
|
search_locations = self.search_locations(fullname)
|
||||||
|
if not search_locations:
|
||||||
|
return None
|
||||||
|
|
||||||
|
spec = importlib.machinery.ModuleSpec(fullname, PluginLoader(), is_package=True)
|
||||||
|
spec.submodule_search_locations = search_locations
|
||||||
|
return spec
|
||||||
|
|
||||||
|
def invalidate_caches(self):
|
||||||
|
dirs_in_zip.cache_clear()
|
||||||
|
for package in self.packages:
|
||||||
|
if package in sys.modules:
|
||||||
|
del sys.modules[package]
|
||||||
|
|
||||||
|
|
||||||
|
def directories():
|
||||||
|
spec = importlib.util.find_spec(PACKAGE_NAME)
|
||||||
|
return spec.submodule_search_locations if spec else []
|
||||||
|
|
||||||
|
|
||||||
|
def iter_modules(subpackage):
|
||||||
|
fullname = f'{PACKAGE_NAME}.{subpackage}'
|
||||||
|
with contextlib.suppress(ModuleNotFoundError):
|
||||||
|
pkg = importlib.import_module(fullname)
|
||||||
|
yield from pkgutil.iter_modules(path=pkg.__path__, prefix=f'{fullname}.')
|
||||||
|
|
||||||
|
|
||||||
|
def load_module(module, module_name, suffix):
|
||||||
|
return inspect.getmembers(module, lambda obj: (
|
||||||
|
inspect.isclass(obj)
|
||||||
|
and obj.__name__.endswith(suffix)
|
||||||
|
and obj.__module__.startswith(module_name)
|
||||||
|
and not obj.__name__.startswith('_')
|
||||||
|
and obj.__name__ in getattr(module, '__all__', [obj.__name__])))
|
||||||
|
|
||||||
|
|
||||||
|
def load_plugins(name, suffix):
|
||||||
|
classes = {}
|
||||||
|
|
||||||
|
for finder, module_name, _ in iter_modules(name):
|
||||||
|
if any(x.startswith('_') for x in module_name.split('.')):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if sys.version_info < (3, 10) and isinstance(finder, zipimport.zipimporter):
|
||||||
|
# zipimporter.load_module() is deprecated in 3.10 and removed in 3.12
|
||||||
|
# The exec_module branch below is the replacement for >= 3.10
|
||||||
|
# See: https://docs.python.org/3/library/zipimport.html#zipimport.zipimporter.exec_module
|
||||||
|
module = finder.load_module(module_name)
|
||||||
|
else:
|
||||||
|
spec = finder.find_spec(module_name)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[module_name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
except Exception:
|
||||||
|
write_string(f'Error while importing module {module_name!r}\n{traceback.format_exc(limit=-1)}')
|
||||||
|
continue
|
||||||
|
classes.update(load_module(module, module_name, suffix))
|
||||||
|
|
||||||
|
# Compat: old plugin system using __init__.py
|
||||||
|
# Note: plugins imported this way do not show up in directories()
|
||||||
|
# nor are considered part of the yt_dlp_plugins namespace package
|
||||||
|
with contextlib.suppress(FileNotFoundError):
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
name, Path(get_executable_path(), COMPAT_PACKAGE_NAME, name, '__init__.py'))
|
||||||
|
plugins = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = plugins
|
||||||
|
spec.loader.exec_module(plugins)
|
||||||
|
classes.update(load_module(plugins, spec.name, suffix))
|
||||||
|
|
||||||
|
return classes
|
||||||
|
|
||||||
|
|
||||||
|
sys.meta_path.insert(0, PluginFinder(f'{PACKAGE_NAME}.extractor', f'{PACKAGE_NAME}.postprocessor'))
|
||||||
|
|
||||||
|
__all__ = ['directories', 'load_plugins', 'PACKAGE_NAME', 'COMPAT_PACKAGE_NAME']
|
||||||
Loading…
Reference in New Issue