From e38f28ab002fab75e3f8cf9ff121547f0e512a76 Mon Sep 17 00:00:00 2001 From: Matthias Sweertvaegher Date: Mon, 15 Oct 2018 21:04:43 +0200 Subject: [PATCH 1/3] pass on extractor's info to postprocessors pass on extractor's info to postprocessors so track metadata can be used in postprocessing_args --- youtube_dl/postprocessor/embedthumbnail.py | 2 +- youtube_dl/postprocessor/ffmpeg.py | 30 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py index 56be914b8..3f4f9518a 100644 --- a/youtube_dl/postprocessor/embedthumbnail.py +++ b/youtube_dl/postprocessor/embedthumbnail.py @@ -48,7 +48,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor): self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename) - self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options) + self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options, info) if not self._already_have_thumbnail: os.remove(encodeFilename(thumbnail_filename)) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 757b496a1..5d4e9f06b 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -175,13 +175,13 @@ class FFmpegPostProcessor(PostProcessor): return audio_codec return None - def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, info): self.check_version() oldest_mtime = min( os.stat(encodeFilename(path)).st_mtime for path in input_paths) - opts += self._configuration_args() + opts += map(lambda s: s % info, self._configuration_args()) files_cmd = [] for path in input_paths: @@ -204,8 +204,8 @@ class FFmpegPostProcessor(PostProcessor): raise FFmpegPostProcessorError(msg) self.try_utime(out_path, oldest_mtime, oldest_mtime) - def run_ffmpeg(self, path, out_path, opts): - self.run_ffmpeg_multiple_files([path], out_path, opts) + def run_ffmpeg(self, path, out_path, opts, info): + self.run_ffmpeg_multiple_files([path], out_path, opts, info) def _ffmpeg_filename_argument(self, fn): # Always use 'file:' because the filename may contain ':' (ffmpeg @@ -224,14 +224,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): self._preferredquality = preferredquality self._nopostoverwrites = nopostoverwrites - def run_ffmpeg(self, path, out_path, codec, more_opts): + def run_ffmpeg(self, path, out_path, codec, more_opts, info): if codec is None: acodec_opts = [] else: acodec_opts = ['-acodec', codec] opts = ['-vn'] + acodec_opts + more_opts try: - FFmpegPostProcessor.run_ffmpeg(self, path, out_path, opts) + FFmpegPostProcessor.run_ffmpeg(self, path, out_path, opts, info) except FFmpegPostProcessorError as err: raise AudioConversionError(err.msg) @@ -302,7 +302,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): try: self._downloader.to_screen('[ffmpeg] Destination: ' + new_path) - self.run_ffmpeg(path, new_path, acodec, more_opts) + self.run_ffmpeg(path, new_path, acodec, more_opts, information) except AudioConversionError as e: raise PostProcessingError( 'audio conversion failed: ' + e.msg) @@ -334,7 +334,7 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor): prefix, sep, ext = path.rpartition('.') outpath = prefix + sep + self._preferedformat self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath) - self.run_ffmpeg(path, outpath, options) + self.run_ffmpeg(path, outpath, options, information) information['filepath'] = outpath information['format'] = self._preferedformat information['ext'] = self._preferedformat @@ -390,7 +390,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): temp_filename = prepend_extension(filename, 'temp') self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename) - self.run_ffmpeg_multiple_files(input_files, temp_filename, opts) + self.run_ffmpeg_multiple_files(input_files, temp_filename, opts, info) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) @@ -462,7 +462,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): options.extend(['-map_metadata', '1']) self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename) - self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options) + self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options, info) if chapters: os.remove(metadata_filename) os.remove(encodeFilename(filename)) @@ -476,7 +476,7 @@ class FFmpegMergerPP(FFmpegPostProcessor): temp_filename = prepend_extension(filename, 'temp') args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0'] self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename) - self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args) + self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args, info) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return info['__files_to_merge'], info @@ -509,7 +509,7 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor): options = ['-c', 'copy', '-aspect', '%f' % stretched_ratio] self._downloader.to_screen('[ffmpeg] Fixing aspect ratio in "%s"' % filename) - self.run_ffmpeg(filename, temp_filename, options) + self.run_ffmpeg(filename, temp_filename, options, info) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) @@ -527,7 +527,7 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor): options = ['-c', 'copy', '-f', 'mp4'] self._downloader.to_screen('[ffmpeg] Correcting container in "%s"' % filename) - self.run_ffmpeg(filename, temp_filename, options) + self.run_ffmpeg(filename, temp_filename, options, info) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) @@ -543,7 +543,7 @@ class FFmpegFixupM3u8PP(FFmpegPostProcessor): options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc'] self._downloader.to_screen('[ffmpeg] Fixing malformed AAC bitstream in "%s"' % filename) - self.run_ffmpeg(filename, temp_filename, options) + self.run_ffmpeg(filename, temp_filename, options, info) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) @@ -602,7 +602,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): else: sub_filenames.append(srt_file) - self.run_ffmpeg(old_file, new_file, ['-f', new_format]) + self.run_ffmpeg(old_file, new_file, ['-f', new_format], info) with io.open(new_file, 'rt', encoding='utf-8') as f: subs[lang] = { From 7a080138ce9c4aa849e046ad8d43da3dae368f29 Mon Sep 17 00:00:00 2001 From: Matthias Sweertvaegher Date: Tue, 16 Oct 2018 19:45:11 +0200 Subject: [PATCH 2/3] info var does not exist here, is called information instead --- youtube_dl/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 5d4e9f06b..13a7d57de 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -390,7 +390,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): temp_filename = prepend_extension(filename, 'temp') self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename) - self.run_ffmpeg_multiple_files(input_files, temp_filename, opts, info) + self.run_ffmpeg_multiple_files(input_files, temp_filename, opts, information) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) From 2e45a51464a13ef8726b8c34e36c8a2c78a28727 Mon Sep 17 00:00:00 2001 From: Matthias Sweertvaegher Date: Tue, 4 Dec 2018 21:20:54 +0100 Subject: [PATCH 3/3] do similar stuff as for output template --- youtube_dl/postprocessor/ffmpeg.py | 97 +++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 13a7d57de..8b56a0aae 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -5,7 +5,23 @@ import os import subprocess import time import re - +import collections +from string import ascii_letters +import random +from ..utils import expand_path +# from youtube_dl import YoutubeDL +# todo fail to import youtubedl class for _NUMERIC_FIELDS +# http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html +_NUMERIC_FIELDS = set(( + 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx', + 'timestamp', 'upload_year', 'upload_month', 'upload_day', + 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count', + 'average_rating', 'comment_count', 'age_limit', + 'start_time', 'end_time', + 'chapter_number', 'season_number', 'episode_number', + 'track_number', 'disc_number', 'release_year', + 'playlist_index', +)) from .common import AudioConversionError, PostProcessor @@ -181,7 +197,7 @@ class FFmpegPostProcessor(PostProcessor): oldest_mtime = min( os.stat(encodeFilename(path)).st_mtime for path in input_paths) - opts += map(lambda s: s % info, self._configuration_args()) + opts += map(lambda s: self._resolve_postprocessor_arg_var(s, info), self._configuration_args()) files_cmd = [] for path in input_paths: @@ -214,6 +230,83 @@ class FFmpegPostProcessor(PostProcessor): # Also leave '-' intact in order not to break streaming to stdout. return 'file:' + fn if fn != '-' else fn + def _resolve_postprocessor_arg_var(self, arg, info): + # map(lambda s: s % info, args) + if "%(" not in arg: + return arg + + template_dict = dict(info) + template_dict['epoch'] = int(time.time()) + # autonumber_size = self.params.get('autonumber_size') + # if autonumber_size is None: + autonumber_size = 5 + # + # template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads + # if template_dict.get('resolution') is None: + # if template_dict.get('width') and template_dict.get('height'): + # template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height']) + # elif template_dict.get('height'): + # template_dict['resolution'] = '%sp' % template_dict['height'] + # elif template_dict.get('width'): + # template_dict['resolution'] = '%dx?' % template_dict['width'] + # sanitize = lambda k, v: sanitize_filename( + # compat_str(v), + # restricted=self.params.get('restrictfilenames'), + # is_id=(k == 'id' or k.endswith('_id'))) + # template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v)) + # for k, v in template_dict.items() + # if v is not None and not isinstance(v, (list, tuple, dict))) + template_dict = collections.defaultdict(lambda: 'NA', template_dict) + outtmpl = arg + + # For fields playlist_index and autonumber convert all occurrences + # of %(field)s to %(field)0Nd for backward compatibility + field_size_compat_map = { + 'playlist_index': len(str(template_dict['n_entries'])), + 'autonumber': autonumber_size, + } + FIELD_SIZE_COMPAT_RE = r'(?autonumber|playlist_index)\)s' + mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl) + if mobj: + outtmpl = re.sub( + FIELD_SIZE_COMPAT_RE, + r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')], + outtmpl) + # Missing numeric fields used together with integer presentation types + # in format specification will break the argument substitution since + # string 'NA' is returned for missing fields. We will patch output + # template for missing fields to meet string presentation type. + for numeric_field in _NUMERIC_FIELDS: + if numeric_field not in template_dict: + # As of [1] format syntax is: + # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type + # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting + FORMAT_RE = r'''(?x) + (?