diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 91312e4e5f..4f253883c0 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -962,32 +962,33 @@ class TestYoutubeDL(unittest.TestCase): }), r'^30fps$') def test_postprocessors(self): - filename = 'post-processor-testfile.mp4' - audiofile = filename + '.mp3' + filename = 'post-processor-testfile' + video_file = filename + '.mp4' + audio_file = filename + '.mp3' class SimplePP(PostProcessor): def run(self, info): - with open(audiofile, 'w') as f: + with open(audio_file, 'w') as f: f.write('EXAMPLE') return [info['filepath']], info def run_pp(params, pp): - with open(filename, 'w') as f: + with open(video_file, 'w') as f: f.write('EXAMPLE') ydl = YoutubeDL(params) ydl.add_post_processor(pp()) - ydl.post_process(filename, {'filepath': filename}) + ydl.post_process(video_file, {'filepath': video_file}) - run_pp({'keepvideo': True}, SimplePP) - self.assertTrue(os.path.exists(filename), f'{filename} doesn\'t exist') - self.assertTrue(os.path.exists(audiofile), f'{audiofile} doesn\'t exist') - os.unlink(filename) - os.unlink(audiofile) + run_pp({'keepvideo': True, 'outtmpl': filename}, SimplePP) + self.assertTrue(os.path.exists(video_file), f'{video_file} doesn\'t exist') + self.assertTrue(os.path.exists(audio_file), f'{audio_file} doesn\'t exist') + os.unlink(video_file) + os.unlink(audio_file) - run_pp({'keepvideo': False}, SimplePP) - self.assertFalse(os.path.exists(filename), f'{filename} exists') - self.assertTrue(os.path.exists(audiofile), f'{audiofile} doesn\'t exist') - os.unlink(audiofile) + run_pp({'keepvideo': False, 'outtmpl': filename}, SimplePP) + self.assertFalse(os.path.exists(video_file), f'{video_file} exists') + self.assertTrue(os.path.exists(audio_file), f'{audio_file} doesn\'t exist') + os.unlink(audio_file) class ModifierPP(PostProcessor): def run(self, info): @@ -995,9 +996,9 @@ class TestYoutubeDL(unittest.TestCase): f.write('MODIFIED') return [], info - run_pp({'keepvideo': False}, ModifierPP) - self.assertTrue(os.path.exists(filename), f'{filename} doesn\'t exist') - os.unlink(filename) + run_pp({'keepvideo': False, 'outtmpl': filename}, ModifierPP) + self.assertTrue(os.path.exists(video_file), f'{video_file} doesn\'t exist') + os.unlink(video_file) def test_match_filter(self): first = { diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 5985d2ec76..ecd6b6d9fc 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -3296,7 +3296,6 @@ class YoutubeDL: # info_dict['_filename'] needs to be set for backward compatibility info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True) temp_filename = self.prepare_filename(info_dict, 'temp') - files_to_move = {} # Forced printings self.__forced_printings(info_dict, full_filename, incomplete=('format' not in info_dict)) @@ -3324,13 +3323,11 @@ class YoutubeDL: sub_files = self._write_subtitles(info_dict, temp_filename) if sub_files is None: return - files_to_move.update(dict(sub_files)) thumb_files = self._write_thumbnails( 'video', info_dict, temp_filename, self.prepare_filename(info_dict, 'thumbnail')) if thumb_files is None: return - files_to_move.update(dict(thumb_files)) infofn = self.prepare_filename(info_dict, 'infojson') _infojson_written = self._write_info_json('video', info_dict, infofn) @@ -3404,13 +3401,12 @@ class YoutubeDL: for link_type, should_write in write_links.items()): return - new_info, files_to_move = self.pre_process(info_dict, 'before_dl', files_to_move) + new_info, _ = self.pre_process(info_dict, 'before_dl') replace_info_dict(new_info) if self.params.get('skip_download'): info_dict['filepath'] = temp_filename info_dict['__finaldir'] = os.path.dirname(os.path.abspath(full_filename)) - info_dict['__files_to_move'] = files_to_move replace_info_dict(self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict)) info_dict['__write_download_archive'] = self.params.get('force_write_download_archive') else: @@ -3524,9 +3520,6 @@ class YoutubeDL: info_dict['__files_to_merge'] = downloaded # Even if there were no downloads, it is being merged only now info_dict['__real_download'] = True - else: - for file in downloaded: - files_to_move[file] = None else: # Just a single file dl_filename = existing_video_file(full_filename, temp_filename) @@ -3611,7 +3604,7 @@ class YoutubeDL: fixup() try: - replace_info_dict(self.post_process(dl_filename, info_dict, files_to_move)) + replace_info_dict(self.post_process(dl_filename, info_dict)) except PostProcessingError as err: self.report_error(f'Postprocessing: {err}') return @@ -3736,8 +3729,6 @@ class YoutubeDL: os.remove(filename) except OSError: self.report_warning(f'Unable to delete file {filename}') - if filename in info.get('__files_to_move', []): # NB: Delete even if None - del info['__files_to_move'][filename] @staticmethod def post_extract(info_dict): @@ -3754,8 +3745,7 @@ class YoutubeDL: def run_pp(self, pp, infodict): files_to_delete = [] - if '__files_to_move' not in infodict: - infodict['__files_to_move'] = {} + try: files_to_delete, infodict = pp.run(infodict) except PostProcessingError as e: @@ -3767,10 +3757,7 @@ class YoutubeDL: if not files_to_delete: return infodict - if self.params.get('keepvideo', False): - for f in files_to_delete: - infodict['__files_to_move'].setdefault(f, '') - else: + if not self.params.get('keepvideo', False): self._delete_downloaded_files( *files_to_delete, info=infodict, msg='Deleting original file %s (pass -k to keep)') return infodict @@ -3783,23 +3770,27 @@ class YoutubeDL: return info def pre_process(self, ie_info, key='pre_process', files_to_move=None): + if files_to_move is not None: + self.report_warning('[pre_process] "files_to_move" is deprecated and may be removed in a future version') + info = dict(ie_info) - info['__files_to_move'] = files_to_move or {} try: info = self.run_all_pps(key, info) except PostProcessingError as err: msg = f'Preprocessing: {err}' info.setdefault('__pending_error', msg) self.report_error(msg, is_error=False) - return info, info.pop('__files_to_move', None) + return info, files_to_move def post_process(self, filename, info, files_to_move=None): """Run all the postprocessors on the given file.""" + if files_to_move is not None: + self.report_warning('[post_process] "files_to_move" is deprecated and may be removed in a future version') + info['filepath'] = filename - info['__files_to_move'] = files_to_move or {} info = self.run_all_pps('post_process', info, additional_pps=info.get('__postprocessors')) info = self.run_pp(MoveFilesAfterDownloadPP(self), info) - del info['__files_to_move'] + info.pop('__multiple_thumbnails', None) return self.run_all_pps('after_move', info) def _make_archive_id(self, info_dict): @@ -4404,10 +4395,11 @@ class YoutubeDL: sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext')) sub_filename_final = subtitles_filename(sub_filename_base, sub_lang, sub_format, info_dict.get('ext')) existing_sub = self.existing_file((sub_filename_final, sub_filename)) + if existing_sub: self.to_screen(f'[info] Video subtitle {sub_lang}.{sub_format} is already present') sub_info['filepath'] = existing_sub - ret.append((existing_sub, sub_filename_final)) + ret.append(existing_sub) continue self.to_screen(f'[info] Writing video subtitles to: {sub_filename}') @@ -4418,7 +4410,7 @@ class YoutubeDL: with open(sub_filename, 'w', encoding='utf-8', newline='') as subfile: subfile.write(sub_info['data']) sub_info['filepath'] = sub_filename - ret.append((sub_filename, sub_filename_final)) + ret.append(sub_filename) continue except OSError: self.report_error(f'Cannot write video subtitles file {sub_filename}') @@ -4429,7 +4421,7 @@ class YoutubeDL: sub_copy.setdefault('http_headers', info_dict.get('http_headers')) self.dl(sub_filename, sub_copy, subtitle=True) sub_info['filepath'] = sub_filename - ret.append((sub_filename, sub_filename_final)) + ret.append(sub_filename) except (DownloadError, ExtractorError, OSError, ValueError, *network_exceptions) as err: msg = f'Unable to download video subtitles for {sub_lang!r}: {err}' if self.params.get('ignoreerrors') is not True: # False or 'only_download' @@ -4449,6 +4441,7 @@ class YoutubeDL: self.to_screen(f'[info] There are no {label} thumbnails to download') return ret multiple = write_all and len(thumbnails) > 1 + info_dict['__multiple_thumbnails'] = multiple if thumb_filename_base is None: thumb_filename_base = filename @@ -4472,7 +4465,7 @@ class YoutubeDL: self.to_screen('[info] {} is already present'.format(( thumb_display_id if multiple else f'{label} thumbnail').capitalize())) t['filepath'] = existing_thumb - ret.append((existing_thumb, thumb_filename_final)) + ret.append(existing_thumb) else: self.to_screen(f'[info] Downloading {thumb_display_id} ...') try: @@ -4480,7 +4473,7 @@ class YoutubeDL: self.to_screen(f'[info] Writing {thumb_display_id} to: {thumb_filename}') with open(thumb_filename, 'wb') as thumbf: shutil.copyfileobj(uf, thumbf) - ret.append((thumb_filename, thumb_filename_final)) + ret.append(thumb_filename) t['filepath'] = thumb_filename except network_exceptions as err: if isinstance(err, HTTPError) and err.status == 404: @@ -4490,4 +4483,5 @@ class YoutubeDL: thumbnails.pop(idx) if ret and not write_all: break + return ret diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py index 39e8826c6f..5eea451515 100644 --- a/yt_dlp/postprocessor/embedthumbnail.py +++ b/yt_dlp/postprocessor/embedthumbnail.py @@ -230,4 +230,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor): thumbnail_filename if converted or not self._already_have_thumbnail else None, original_thumbnail if converted and not self._already_have_thumbnail else None, info=info) + + if not self._already_have_thumbnail: + info['thumbnails'][idx].pop('filepath', None) + return [], info diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 59a49aa578..8901557dd0 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -662,6 +662,10 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): self.run_ffmpeg_multiple_files(input_files, temp_filename, opts) os.replace(temp_filename, filename) + if not self._already_have_subtitle: + for _, subtitle in subtitles.items(): + subtitle.pop('filepath', None) + files_to_delete = [] if self._already_have_subtitle else sub_filenames return files_to_delete, info @@ -698,6 +702,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): infojson_filename = info.get('infojson_filename') options.extend(self._get_infojson_opts(info, infojson_filename)) if not infojson_filename: + info.pop('infojson_filename', None) files_to_delete.append(info.get('infojson_filename')) elif self._add_infojson is True: self.to_screen('The info-json can only be attached to mkv/mka files') @@ -1015,9 +1020,6 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): 'filepath': new_file, } - info['__files_to_move'][new_file] = replace_extension( - info['__files_to_move'][sub['filepath']], new_ext) - return sub_filenames, info @@ -1082,16 +1084,15 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): return imghdr.what(path) == 'webp' def fixup_webp(self, info, idx=-1): - thumbnail_filename = info['thumbnails'][idx]['filepath'] + thumbnail = info['thumbnails'][idx] + thumbnail_filename = thumbnail['filepath'] _, thumbnail_ext = os.path.splitext(thumbnail_filename) if thumbnail_ext: if thumbnail_ext.lower() != '.webp' and imghdr.what(thumbnail_filename) == 'webp': self.to_screen(f'Correcting thumbnail "{thumbnail_filename}" extension to webp') webp_filename = replace_extension(thumbnail_filename, 'webp') os.replace(thumbnail_filename, webp_filename) - info['thumbnails'][idx]['filepath'] = webp_filename - info['__files_to_move'][webp_filename] = replace_extension( - info['__files_to_move'].pop(thumbnail_filename), 'webp') + thumbnail['filepath'] = webp_filename @staticmethod def _options(target_ext): @@ -1129,8 +1130,6 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): continue thumbnail_dict['filepath'] = self.convert_thumbnail(original_thumbnail, target_ext) files_to_delete.append(original_thumbnail) - info['__files_to_move'][thumbnail_dict['filepath']] = replace_extension( - info['__files_to_move'][original_thumbnail], target_ext) if not has_thumbnail: self.to_screen('There aren\'t any thumbnails to convert') diff --git a/yt_dlp/postprocessor/movefilesafterdownload.py b/yt_dlp/postprocessor/movefilesafterdownload.py index 964ca1f921..149d684479 100644 --- a/yt_dlp/postprocessor/movefilesafterdownload.py +++ b/yt_dlp/postprocessor/movefilesafterdownload.py @@ -1,14 +1,22 @@ import os +from pathlib import Path from .common import PostProcessor from ..compat import shutil from ..utils import ( PostProcessingError, make_dir, + replace_extension, ) class MoveFilesAfterDownloadPP(PostProcessor): + # Map of the keys that contain moveable files and the 'type' of the file + # for generating the output filename + CHILD_KEYS = { + 'thumbnails': 'thumbnail', + 'requested_subtitles': 'subtitle', + } def __init__(self, downloader=None, downloaded=True): PostProcessor.__init__(self, downloader) @@ -18,33 +26,77 @@ class MoveFilesAfterDownloadPP(PostProcessor): def pp_key(cls): return 'MoveFiles' + def move_file_and_write_to_info(self, info_dict, relevant_dict=None, output_file_type=None): + relevant_dict = relevant_dict or info_dict + if 'filepath' not in relevant_dict: + return + + output_file_type = output_file_type or '' + current_filepath, final_filepath = self.determine_filepath(info_dict, relevant_dict, output_file_type) + move_result = self.move_file(info_dict, current_filepath, final_filepath) + + if move_result: + relevant_dict['filepath'] = move_result + else: + del relevant_dict['filepath'] + + def determine_filepath(self, info_dict, relevant_dict, output_file_type): + current_filepath = relevant_dict['filepath'] + prepared_filepath = self._downloader.prepare_filename(info_dict, output_file_type) + + if (output_file_type == 'thumbnail' and info_dict['__multiple_thumbnails']) or output_file_type == 'subtitle': + desired_extension = ''.join(Path(current_filepath).suffixes[-2:]) + else: + desired_extension = Path(current_filepath).suffix + + return current_filepath, replace_extension(prepared_filepath, desired_extension[1:]) + + def move_file(self, info_dict, current_filepath, final_filepath): + if not current_filepath or not final_filepath: + return + + dl_parent_folder = os.path.split(info_dict['filepath'])[0] + finaldir = info_dict.get('__finaldir', os.path.abspath(dl_parent_folder)) + + if not os.path.isabs(current_filepath): + current_filepath = os.path.join(finaldir, current_filepath) + + if not os.path.isabs(final_filepath): + final_filepath = os.path.join(finaldir, final_filepath) + + if current_filepath == final_filepath: + return final_filepath + + if not os.path.exists(current_filepath): + self.report_warning(f'File "{current_filepath}" cannot be found') + return + + if os.path.exists(final_filepath): + if self.get_param('overwrites', True): + self.report_warning(f'Replacing existing file "{final_filepath}"') + os.remove(final_filepath) + else: + self.report_warning(f'Cannot move file "{current_filepath}" out of temporary directory since "{final_filepath}" already exists. ') + return + + make_dir(final_filepath, PostProcessingError) + self.to_screen(f'Moving file "{current_filepath}" to "{final_filepath}"') + shutil.move(current_filepath, final_filepath) # os.rename cannot move between volumes + + return final_filepath + def run(self, info): - dl_path, dl_name = os.path.split(info['filepath']) - finaldir = info.get('__finaldir', dl_path) - finalpath = os.path.join(finaldir, dl_name) - if self._downloaded: - info['__files_to_move'][info['filepath']] = finalpath - - make_newfilename = lambda old: os.path.join(finaldir, os.path.basename(old)) - for oldfile, newfile in info['__files_to_move'].items(): - if not newfile: - newfile = make_newfilename(oldfile) - if os.path.abspath(oldfile) == os.path.abspath(newfile): - continue - if not os.path.exists(oldfile): - self.report_warning(f'File "{oldfile}" cannot be found') + # This represents the main media file (using the 'filepath' key) + self.move_file_and_write_to_info(info) + + for key, output_file_type in self.CHILD_KEYS.items(): + if key not in info: continue - if os.path.exists(newfile): - if self.get_param('overwrites', True): - self.report_warning(f'Replacing existing file "{newfile}"') - os.remove(newfile) - else: - self.report_warning( - f'Cannot move file "{oldfile}" out of temporary directory since "{newfile}" already exists. ') - continue - make_dir(newfile, PostProcessingError) - self.to_screen(f'Moving file "{oldfile}" to "{newfile}"') - shutil.move(oldfile, newfile) # os.rename cannot move between volumes - - info['filepath'] = finalpath + + if isinstance(info[key], (dict, list)): + iterable = info[key].values() if isinstance(info[key], dict) else info[key] + + for file_dict in iterable: + self.move_file_and_write_to_info(info, file_dict, output_file_type) + return [], info