Mirroring
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp
1
0
Fork 0
Code Issues Releases Wiki Activity

[movefiles] Fix bugs and make more robust

Browse Source
pull/173/head
pukkandan 4 years ago
parent e3c076970e
commit dcf64d43e0
No known key found for this signature in database
GPG Key ID: 0F00D95A001F4698
4 changed files with 40 additions and 30 deletions
Show Stats Download Patch File Download Diff File

37
yt_dlp/YoutubeDL.py
Unescape Escape View File

@ -2075,6 +2075,7 @@ class YoutubeDL(object):
info_dict = self.pre_process(info_dict) info_dict = self.pre_process(info_dict)
# info_dict['_filename'] needs to be set for backward compatibility
info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True) info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True)
temp_filename = self.prepare_filename(info_dict, 'temp') temp_filename = self.prepare_filename(info_dict, 'temp')
files_to_move = {} files_to_move = {}
@ -2159,6 +2160,7 @@ class YoutubeDL(object):
sub_filename_final = subtitles_filename(sub_fn, sub_lang, sub_format, info_dict.get('ext')) sub_filename_final = subtitles_filename(sub_fn, sub_lang, sub_format, info_dict.get('ext'))
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)): if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format)) self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
sub_info['filepath'] = sub_filename
files_to_move[sub_filename] = sub_filename_final files_to_move[sub_filename] = sub_filename_final
else: else:
self.to_screen('[info] Writing video subtitles to: ' + sub_filename) self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
@ -2168,13 +2170,15 @@ class YoutubeDL(object):
# See https://github.com/ytdl-org/youtube-dl/issues/10268 # See https://github.com/ytdl-org/youtube-dl/issues/10268
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile: with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
subfile.write(sub_info['data']) subfile.write(sub_info['data'])
sub_info['filepath'] = sub_filename
files_to_move[sub_filename] = sub_filename_final files_to_move[sub_filename] = sub_filename_final
except (OSError, IOError): except (OSError, IOError):
self.report_error('Cannot write subtitles file ' + sub_filename) self.report_error('Cannot write subtitles file ' + sub_filename)
return return
else: else:
try: try:
dl(sub_filename, sub_info, subtitle=True) dl(sub_filename, sub_info.copy(), subtitle=True)
sub_info['filepath'] = sub_filename
files_to_move[sub_filename] = sub_filename_final files_to_move[sub_filename] = sub_filename_final
except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_warning('Unable to download subtitle for "%s": %s' % self.report_warning('Unable to download subtitle for "%s": %s' %
@ -2223,7 +2227,7 @@ class YoutubeDL(object):
for thumb_ext in self._write_thumbnails(info_dict, thumb_fn_temp): for thumb_ext in self._write_thumbnails(info_dict, thumb_fn_temp):
thumb_filename_temp = replace_extension(thumb_fn_temp, thumb_ext, info_dict.get('ext')) thumb_filename_temp = replace_extension(thumb_fn_temp, thumb_ext, info_dict.get('ext'))
thumb_filename = replace_extension(thumbfn, thumb_ext, info_dict.get('ext')) thumb_filename = replace_extension(thumbfn, thumb_ext, info_dict.get('ext'))
files_to_move[thumb_filename_temp] = info_dict['__thumbnail_filename'] = thumb_filename files_to_move[thumb_filename_temp] = thumb_filename
# Write internet shortcut files # Write internet shortcut files
url_link = webloc_link = desktop_link = False url_link = webloc_link = desktop_link = False
@ -2529,15 +2533,17 @@ class YoutubeDL(object):
(k, v) for k, v in info_dict.items() (k, v) for k, v in info_dict.items()
if (k[0] != '_' or k == '_type') and k not in fields_to_remove) if (k[0] != '_' or k == '_type') and k not in fields_to_remove)
def run_pp(self, pp, infodict, files_to_move={}): def run_pp(self, pp, infodict):
files_to_delete = [] files_to_delete = []
if '__files_to_move' not in infodict:
infodict['__files_to_move'] = {}
files_to_delete, infodict = pp.run(infodict) files_to_delete, infodict = pp.run(infodict)
if not files_to_delete: if not files_to_delete:
return files_to_move, infodict return infodict
if self.params.get('keepvideo', False): if self.params.get('keepvideo', False):
for f in files_to_delete: for f in files_to_delete:
files_to_move.setdefault(f, '') infodict['__files_to_move'].setdefault(f, '')
else: else:
for old_filename in set(files_to_delete): for old_filename in set(files_to_delete):
self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename) self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
@ -2545,9 +2551,9 @@ class YoutubeDL(object):
os.remove(encodeFilename(old_filename)) os.remove(encodeFilename(old_filename))
except (IOError, OSError): except (IOError, OSError):
self.report_warning('Unable to remove downloaded original file') self.report_warning('Unable to remove downloaded original file')
if old_filename in files_to_move: if old_filename in infodict['__files_to_move']:
del files_to_move[old_filename] del infodict['__files_to_move'][old_filename]
return files_to_move, infodict return infodict
@staticmethod @staticmethod
def post_extract(info_dict): def post_extract(info_dict):
@ -2570,20 +2576,21 @@ class YoutubeDL(object):
def pre_process(self, ie_info): def pre_process(self, ie_info):
info = dict(ie_info) info = dict(ie_info)
for pp in self._pps['beforedl']: for pp in self._pps['beforedl']:
info = self.run_pp(pp, info)[1] info = self.run_pp(pp, info)
return info return info
def post_process(self, filename, ie_info, files_to_move={}): def post_process(self, filename, ie_info, files_to_move=None):
"""Run all the postprocessors on the given file.""" """Run all the postprocessors on the given file."""
info = dict(ie_info) info = dict(ie_info)
info['filepath'] = filename info['filepath'] = filename
info['__files_to_move'] = {} info['__files_to_move'] = files_to_move or {}
for pp in ie_info.get('__postprocessors', []) + self._pps['normal']: for pp in ie_info.get('__postprocessors', []) + self._pps['normal']:
files_to_move, info = self.run_pp(pp, info, files_to_move) info = self.run_pp(pp, info)
info = self.run_pp(MoveFilesAfterDownloadPP(self, files_to_move), info)[1] info = self.run_pp(MoveFilesAfterDownloadPP(self), info)
del info['__files_to_move']
for pp in self._pps['aftermove']: for pp in self._pps['aftermove']:
info = self.run_pp(pp, info, {})[1] info = self.run_pp(pp, info)
def _make_archive_id(self, info_dict): def _make_archive_id(self, info_dict):
video_id = info_dict.get('id') video_id = info_dict.get('id')
@ -2951,7 +2958,7 @@ class YoutubeDL(object):
thumb_ext = determine_ext(t['url'], 'jpg') thumb_ext = determine_ext(t['url'], 'jpg')
suffix = '%s.' % t['id'] if multiple else '' suffix = '%s.' % t['id'] if multiple else ''
thumb_display_id = '%s ' % t['id'] if multiple else '' thumb_display_id = '%s ' % t['id'] if multiple else ''
t['filename'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext')) t['filepath'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext'))
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)): if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
ret.append(suffix + thumb_ext) ret.append(suffix + thumb_ext)

7
yt_dlp/postprocessor/embedthumbnail.py
Unescape Escape View File

@ -47,7 +47,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
self.to_screen('There aren\'t any thumbnails to embed') self.to_screen('There aren\'t any thumbnails to embed')
return [], info return [], info
original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filename'] initial_thumbnail = original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filepath']
if not os.path.exists(encodeFilename(thumbnail_filename)): if not os.path.exists(encodeFilename(thumbnail_filename)):
self.report_warning('Skipping embedding the thumbnail because the file is missing.') self.report_warning('Skipping embedding the thumbnail because the file is missing.')
@ -65,6 +65,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
if thumbnail_ext != 'webp' and is_webp(thumbnail_filename): if thumbnail_ext != 'webp' and is_webp(thumbnail_filename):
self.to_screen('Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename) self.to_screen('Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename)
thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp') thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp')
if os.path.exists(thumbnail_webp_filename):
os.remove(thumbnail_webp_filename)
os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename)) os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename))
original_thumbnail = thumbnail_filename = thumbnail_webp_filename original_thumbnail = thumbnail_filename = thumbnail_webp_filename
thumbnail_ext = 'webp' thumbnail_ext = 'webp'
@ -194,7 +196,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
files_to_delete = [thumbnail_filename] files_to_delete = [thumbnail_filename]
if self._already_have_thumbnail: if self._already_have_thumbnail:
info['__files_to_move'][original_thumbnail] = replace_extension( info['__files_to_move'][original_thumbnail] = replace_extension(
info['__thumbnail_filename'], os.path.splitext(original_thumbnail)[1][1:]) info['__files_to_move'][initial_thumbnail],
os.path.splitext(original_thumbnail)[1][1:])
if original_thumbnail == thumbnail_filename: if original_thumbnail == thumbnail_filename:
files_to_delete = [] files_to_delete = []
elif original_thumbnail != thumbnail_filename: elif original_thumbnail != thumbnail_filename:

17
yt_dlp/postprocessor/ffmpeg.py
Unescape Escape View File

@ -486,7 +486,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
self.report_warning('JSON subtitles cannot be embedded') self.report_warning('JSON subtitles cannot be embedded')
elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt':
sub_langs.append(lang) sub_langs.append(lang)
sub_filenames.append(subtitles_filename(filename, lang, sub_ext, ext)) sub_filenames.append(sub_info['filepath'])
else: else:
if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt': if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt':
webm_vtt_warn = True webm_vtt_warn = True
@ -732,9 +732,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
'You have requested to convert json subtitles into another format, ' 'You have requested to convert json subtitles into another format, '
'which is currently not possible') 'which is currently not possible')
continue continue
old_file = subtitles_filename(filename, lang, ext, info.get('ext')) old_file = sub['filepath']
sub_filenames.append(old_file) sub_filenames.append(old_file)
new_file = subtitles_filename(filename, lang, new_ext, info.get('ext')) new_file = replace_extension(old_file, new_ext)
if ext in ('dfxp', 'ttml', 'tt'): if ext in ('dfxp', 'ttml', 'tt'):
self.report_warning( self.report_warning(
@ -742,7 +742,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
'which results in style information loss') 'which results in style information loss')
dfxp_file = old_file dfxp_file = old_file
srt_file = subtitles_filename(filename, lang, 'srt', info.get('ext')) srt_file = replace_extension(old_file, 'srt')
with open(dfxp_file, 'rb') as f: with open(dfxp_file, 'rb') as f:
srt_data = dfxp2srt(f.read()) srt_data = dfxp2srt(f.read())
@ -753,7 +753,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
subs[lang] = { subs[lang] = {
'ext': 'srt', 'ext': 'srt',
'data': srt_data 'data': srt_data,
'filepath': srt_file,
} }
if new_ext == 'srt': if new_ext == 'srt':
@ -767,8 +768,12 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
subs[lang] = { subs[lang] = {
'ext': new_ext, 'ext': new_ext,
'data': f.read(), 'data': f.read(),
'filepath': new_file,
} }
info['__files_to_move'][new_file] = replace_extension(
info['__files_to_move'][old_file], new_ext)
return sub_filenames, info return sub_filenames, info
@ -789,7 +794,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor):
if not self._downloader._ensure_dir_exists(encodeFilename(destination)): if not self._downloader._ensure_dir_exists(encodeFilename(destination)):
return return
chapter['_filename'] = destination chapter['filepath'] = destination
self.to_screen('Chapter %03d; Destination: %s' % (number, destination)) self.to_screen('Chapter %03d; Destination: %s' % (number, destination))
return ( return (
destination, destination,

9
yt_dlp/postprocessor/movefilesafterdownload.py
Unescape Escape View File

@ -13,10 +13,6 @@ from ..utils import (
class MoveFilesAfterDownloadPP(PostProcessor): class MoveFilesAfterDownloadPP(PostProcessor):
def __init__(self, downloader, files_to_move):
PostProcessor.__init__(self, downloader)
self.files_to_move = files_to_move
@classmethod @classmethod
def pp_key(cls): def pp_key(cls):
return 'MoveFiles' return 'MoveFiles'
@ -25,11 +21,10 @@ class MoveFilesAfterDownloadPP(PostProcessor):
dl_path, dl_name = os.path.split(encodeFilename(info['filepath'])) dl_path, dl_name = os.path.split(encodeFilename(info['filepath']))
finaldir = info.get('__finaldir', dl_path) finaldir = info.get('__finaldir', dl_path)
finalpath = os.path.join(finaldir, dl_name) finalpath = os.path.join(finaldir, dl_name)
self.files_to_move.update(info['__files_to_move']) info['__files_to_move'][info['filepath']] = decodeFilename(finalpath)
self.files_to_move[info['filepath']] = decodeFilename(finalpath)
make_newfilename = lambda old: decodeFilename(os.path.join(finaldir, os.path.basename(encodeFilename(old)))) make_newfilename = lambda old: decodeFilename(os.path.join(finaldir, os.path.basename(encodeFilename(old))))
for oldfile, newfile in self.files_to_move.items(): for oldfile, newfile in info['__files_to_move'].items():
if not newfile: if not newfile:
newfile = make_newfilename(oldfile) newfile = make_newfilename(oldfile)
if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)): if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)):

Write Preview
Loading…
Cancel
Save