Mirroring
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp
1
0
Fork 0
Code Issues Releases Wiki Activity

Allow multiple and nested configuration files

Browse Source
pull/2222/head
pukkandan 3 years ago
parent b62fa6d75f
commit 06e57990f7
No known key found for this signature in database
GPG Key ID: 0F00D95A001F4698
4 changed files with 172 additions and 141 deletions
Show Stats Download Patch File Download Diff File

26
test/test_options.py
Unescape Escape View File

@ -1,26 +0,0 @@
# coding: utf-8
from __future__ import unicode_literals
# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from yt_dlp.options import _hide_login_info
class TestOptions(unittest.TestCase):
def test_hide_login_info(self):
self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
['-u', 'PRIVATE', '-p', 'PRIVATE'])
self.assertEqual(_hide_login_info(['-u']), ['-u'])
self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']),
['-u', 'PRIVATE', '-u', 'PRIVATE'])
self.assertEqual(_hide_login_info(['--username=foo']),
['--username=PRIVATE'])
if __name__ == '__main__':
unittest.main()

10
test/test_utils.py
Unescape Escape View File

@ -23,6 +23,7 @@ from yt_dlp.utils import (
caesar,
clean_html,
clean_podcast_url,
Config,
date_from_str,
datetime_from_str,
DateRange,
@ -1701,6 +1702,15 @@ Line 1
self.assertEqual(format_bytes(1024**7), '1.00ZiB')
self.assertEqual(format_bytes(1024**8), '1.00YiB')
def test_hide_login_info(self):
self.assertEqual(Config.hide_login_info(['-u', 'foo', '-p', 'bar']),
['-u', 'PRIVATE', '-p', 'PRIVATE'])
self.assertEqual(Config.hide_login_info(['-u']), ['-u'])
self.assertEqual(Config.hide_login_info(['-u', 'foo', '-u', 'bar']),
['-u', 'PRIVATE', '-u', 'PRIVATE'])
self.assertEqual(Config.hide_login_info(['--username=foo']),
['--username=PRIVATE'])
if __name__ == '__main__':
unittest.main()

187
yt_dlp/options.py
Unescape Escape View File

@ -13,11 +13,11 @@ from .compat import (
compat_shlex_split,
)
from .utils import (
Config,
expand_path,
get_executable_path,
OUTTMPL_TYPES,
POSTPROCESS_WHEN,
preferredencoding,
remove_end,
write_string,
)
@ -35,39 +35,16 @@ from .postprocessor import (
from .postprocessor.modify_chapters import DEFAULT_SPONSORBLOCK_CHAPTER_TITLE
def _hide_login_info(opts):
PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
parser = create_parser()
root = Config(parser)
def _scrub_eq(o):
m = eqre.match(o)
if m:
return m.group('key') + '=PRIVATE'
if ignore_config_files == 'if_override':
ignore_config_files = overrideArguments is not None
if overrideArguments:
root.append_config(overrideArguments, label='Override')
else:
return o
opts = list(map(_scrub_eq, opts))
for idx, opt in enumerate(opts):
if opt in PRIVATE_OPTS and idx + 1 < len(opts):
opts[idx + 1] = 'PRIVATE'
return opts
def parseOpts(overrideArguments=None):
def _readOptions(filename_bytes, default=[]):
try:
optionf = open(filename_bytes)
except IOError:
return default # silently skip if file is not present
try:
# FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
contents = optionf.read()
if sys.version_info < (3,):
contents = contents.decode(preferredencoding())
res = compat_shlex_split(contents, comments=True)
finally:
optionf.close()
return res
root.append_config(sys.argv[1:], label='Command-line')
def _readUserConf(package_name, default=[]):
# .config
@ -75,7 +52,7 @@ def parseOpts(overrideArguments=None):
userConfFile = os.path.join(xdg_config_home, package_name, 'config')
if not os.path.isfile(userConfFile):
userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name)
userConf = _readOptions(userConfFile, default=None)
userConf = Config.read_file(userConfFile, default=None)
if userConf is not None:
return userConf, userConfFile
@ -83,24 +60,64 @@ def parseOpts(overrideArguments=None):
appdata_dir = compat_getenv('appdata')
if appdata_dir:
userConfFile = os.path.join(appdata_dir, package_name, 'config')
userConf = _readOptions(userConfFile, default=None)
userConf = Config.read_file(userConfFile, default=None)
if userConf is None:
userConfFile += '.txt'
userConf = _readOptions(userConfFile, default=None)
userConf = Config.read_file(userConfFile, default=None)
if userConf is not None:
return userConf, userConfFile
# home
userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
userConf = _readOptions(userConfFile, default=None)
userConf = Config.read_file(userConfFile, default=None)
if userConf is None:
userConfFile += '.txt'
userConf = _readOptions(userConfFile, default=None)
userConf = Config.read_file(userConfFile, default=None)
if userConf is not None:
return userConf, userConfFile
return default, None
def add_config(label, path, user=False):
""" Adds config and returns whether to continue """
if root.parse_args()[0].ignoreconfig:
return False
# Multiple package names can be given here
# Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
# the configuration file of any of these three packages
for package in ('yt-dlp',):
if user:
args, current_path = _readUserConf(package, default=None)
else:
current_path = os.path.join(path, '%s.conf' % package)
args = Config.read_file(current_path, default=None)
if args is not None:
root.append_config(args, current_path, label=label)
return True
return True
def load_configs():
yield not ignore_config_files
yield add_config('Portable', get_executable_path())
yield add_config('Home', expand_path(root.parse_args()[0].paths.get('home', '')).strip())
yield add_config('User', None, user=True)
yield add_config('System', '/etc')
if all(load_configs()):
# If ignoreconfig is found inside the system configuration file,
# the user configuration is removed
if root.parse_args()[0].ignoreconfig:
user_conf = next((i for i, conf in enumerate(root.configs) if conf.label == 'User'), None)
if user_conf is not None:
root.configs.pop(user_conf)
opts, args = root.parse_args()
if opts.verbose:
write_string(f'\n{root}'.replace('\n| ', '\n[debug] ')[1:] + '\n')
return parser, opts, args
def create_parser():
def _format_option_string(option):
''' ('-o', '--option') -> -o, --format METAVAR'''
@ -244,14 +261,20 @@ def parseOpts(overrideArguments=None):
'--ignore-config', '--no-config',
action='store_true', dest='ignoreconfig',
help=(
'Disable loading any configuration files except the one provided by --config-location. '
'When given inside a configuration file, no further configuration files are loaded. '
'Additionally, (for backward compatibility) if this option is found inside the '
'system configuration file, the user configuration is not loaded'))
'Disable loading any further configuration files except the one provided by --config-locations. '
'For backward compatibility, if this option is found inside the system configuration file, the user configuration is not loaded'))
general.add_option(
'--config-location',
dest='config_location', metavar='PATH',
help='Location of the main configuration file; either the path to the config or its containing directory')
'--no-config-locations',
action='store_const', dest='config_locations', const=[],
help=(
'Do not load any custom configuration files (default). When given inside a '
'configuration file, ignore all previous --config-locations defined in the current file'))
general.add_option(
'--config-locations',
dest='config_locations', metavar='PATH', action='append',
help=(
'Location of the main configuration file; either the path to the config or its containing directory. '
'Can be used multiple times and inside other configuration files'))
general.add_option(
'--flat-playlist',
action='store_const', dest='extract_flat', const='in_playlist', default=False,
@ -1634,75 +1657,11 @@ def parseOpts(overrideArguments=None):
parser.add_option_group(sponsorblock)
parser.add_option_group(extractor)
if overrideArguments is not None:
opts, args = parser.parse_args(overrideArguments)
if opts.verbose:
write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
else:
def compat_conf(conf):
if sys.version_info < (3,):
return [a.decode(preferredencoding(), 'replace') for a in conf]
return conf
return parser
configs = {
'command-line': compat_conf(sys.argv[1:]),
'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []}
paths = {'command-line': False}
def read_options(name, path, user=False):
''' loads config files and returns ignoreconfig '''
# Multiple package names can be given here
# Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
# the configuration file of any of these three packages
for package in ('yt-dlp',):
if user:
config, current_path = _readUserConf(package, default=None)
else:
current_path = os.path.join(path, '%s.conf' % package)
config = _readOptions(current_path, default=None)
if config is not None:
current_path = os.path.realpath(current_path)
if current_path in paths.values():
return False
configs[name], paths[name] = config, current_path
return parser.parse_args(config)[0].ignoreconfig
return False
def get_configs():
opts, _ = parser.parse_args(configs['command-line'])
if opts.config_location is not None:
location = compat_expanduser(opts.config_location)
if os.path.isdir(location):
location = os.path.join(location, 'yt-dlp.conf')
if not os.path.exists(location):
parser.error('config-location %s does not exist.' % location)
config = _readOptions(location, default=None)
if config:
configs['custom'], paths['custom'] = config, location
if opts.ignoreconfig:
return
if parser.parse_args(configs['custom'])[0].ignoreconfig:
return
if read_options('portable', get_executable_path()):
return
opts, _ = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line'])
if read_options('home', expand_path(opts.paths.get('home', '')).strip()):
return
if read_options('system', '/etc'):
return
if read_options('user', None, user=True):
configs['system'], paths['system'] = [], None
get_configs()
argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line']
opts, args = parser.parse_args(argv)
if opts.verbose:
for label in ('Command-line', 'Custom', 'Portable', 'Home', 'User', 'System'):
key = label.lower()
if paths.get(key):
write_string(f'[debug] {label} config file: {paths[key]}\n')
if paths.get(key) is not None:
write_string(f'[debug] {label} config: {_hide_login_info(configs[key])!r}\n')
return parser, opts, args
def _hide_login_info(opts):
write_string(
'DeprecationWarning: "yt_dlp.options._hide_login_info" is deprecated and may be removed in a future version. '
'Use "yt_dlp.utils.Config.hide_login_info" instead\n')
return Config.hide_login_info(opts)

88
yt_dlp/utils.py
Unescape Escape View File

@ -58,6 +58,7 @@ from .compat import (
compat_kwargs,
compat_os_name,
compat_parse_qs,
compat_shlex_split,
compat_shlex_quote,
compat_str,
compat_struct_pack,
@ -5100,3 +5101,90 @@ def join_nonempty(*values, delim='-', from_dict=None):
if from_dict is not None:
values = map(from_dict.get, values)
return delim.join(map(str, filter(None, values)))
class Config:
own_args = None
filename = None
__initialized = False
def __init__(self, parser, label=None):
self._parser, self.label = parser, label
self._loaded_paths, self.configs = set(), []
def init(self, args=None, filename=None):
assert not self.__initialized
if filename:
location = os.path.realpath(filename)
if location in self._loaded_paths:
return False
self._loaded_paths.add(location)
self.__initialized = True
self.own_args, self.filename = args, filename
for location in self._parser.parse_args(args)[0].config_locations or []:
location = compat_expanduser(location)
if os.path.isdir(location):
location = os.path.join(location, 'yt-dlp.conf')
if not os.path.exists(location):
self._parser.error(f'config location {location} does not exist')
self.append_config(self.read_file(location), location)
return True
def __str__(self):
label = join_nonempty(
self.label, 'config', f'"{self.filename}"' if self.filename else '',
delim=' ')
return join_nonempty(
self.own_args is not None and f'{label[0].upper()}{label[1:]}: {self.hide_login_info(self.own_args)}',
*(f'\n{c}'.replace('\n', '\n| ')[1:] for c in self.configs),
delim='\n')
@staticmethod
def read_file(filename, default=[]):
try:
optionf = open(filename)
except IOError:
return default # silently skip if file is not present
try:
# FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
contents = optionf.read()
if sys.version_info < (3,):
contents = contents.decode(preferredencoding())
res = compat_shlex_split(contents, comments=True)
finally:
optionf.close()
return res
@staticmethod
def hide_login_info(opts):
PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
def _scrub_eq(o):
m = eqre.match(o)
if m:
return m.group('key') + '=PRIVATE'
else:
return o
opts = list(map(_scrub_eq, opts))
for idx, opt in enumerate(opts):
if opt in PRIVATE_OPTS and idx + 1 < len(opts):
opts[idx + 1] = 'PRIVATE'
return opts
def append_config(self, *args, label=None):
config = type(self)(self._parser, label)
config._loaded_paths = self._loaded_paths
if config.init(*args):
self.configs.append(config)
@property
def all_args(self):
for config in reversed(self.configs):
yield from config.all_args
yield from self.own_args or []
def parse_args(self):
return self._parser.parse_args(list(self.all_args))

Write Preview
Loading…
Cancel
Save