From 0a5a191a2a33e3b305aaf684576b7129ba5173a0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 27 Apr 2022 21:52:57 +0530 Subject: [PATCH] Improve `--clean-infojson` It should not removes fields that may be needed for `--load-infojson`. Eg: `_ffmpeg_args`, `_has_drm` --- test/test_YoutubeDL.py | 2 +- yt_dlp/YoutubeDL.py | 15 ++++++++------- yt_dlp/downloader/external.py | 7 +++++-- yt_dlp/extractor/common.py | 6 ++++-- yt_dlp/extractor/nbc.py | 2 +- yt_dlp/extractor/radiko.py | 2 +- yt_dlp/extractor/turner.py | 2 +- 7 files changed, 21 insertions(+), 15 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 051a203acd..1133f61654 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -661,7 +661,7 @@ class TestYoutubeDL(unittest.TestCase): 'duration': 100000, 'playlist_index': 1, 'playlist_autonumber': 2, - '_last_playlist_index': 100, + '__last_playlist_index': 100, 'n_entries': 10, 'formats': [{'id': 'id 1'}, {'id': 'id 2'}, {'id': 'id 3'}] } diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 9acd881711..eadc5d7ecb 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -954,7 +954,7 @@ class YoutubeDL: self.to_screen('Deleting existing file') def raise_no_formats(self, info, forced=False, *, msg=None): - has_drm = info.get('__has_drm') + has_drm = info.get('_has_drm') ignored, expected = self.params.get('ignore_no_formats_error'), bool(msg) msg = msg or has_drm and 'This video is DRM protected' or 'No video formats found!' if forced or not ignored: @@ -1052,7 +1052,7 @@ class YoutubeDL: # For fields playlist_index, playlist_autonumber and autonumber convert all occurrences # of %(field)s to %(field)0Nd for backward compatibility field_size_compat_map = { - 'playlist_index': number_of_digits(info_dict.get('_last_playlist_index') or 0), + 'playlist_index': number_of_digits(info_dict.get('__last_playlist_index') or 0), 'playlist_autonumber': number_of_digits(info_dict.get('n_entries') or 0), 'autonumber': self.params.get('autonumber_size') or 5, } @@ -1764,7 +1764,7 @@ class YoutubeDL: entry['__x_forwarded_for_ip'] = x_forwarded_for extra = { 'n_entries': n_entries, - '_last_playlist_index': max(playlistitems) if playlistitems else (playlistend or n_entries), + '__last_playlist_index': max(playlistitems) if playlistitems else (playlistend or n_entries), 'playlist_count': ie_result.get('playlist_count'), 'playlist_index': playlist_index, 'playlist_autonumber': i, @@ -2436,10 +2436,11 @@ class YoutubeDL: else: formats = info_dict['formats'] - info_dict['__has_drm'] = any(f.get('has_drm') for f in formats) + # or None ensures --clean-infojson removes it + info_dict['_has_drm'] = any(f.get('has_drm') for f in formats) or None if not self.params.get('allow_unplayable_formats'): formats = [f for f in formats if not f.get('has_drm')] - if info_dict['__has_drm'] and all( + if info_dict['_has_drm'] and all( f.get('acodec') == f.get('vcodec') == 'none' for f in formats): self.report_warning( 'This video is DRM protected and only images are available for download. ' @@ -3266,9 +3267,9 @@ class YoutubeDL: info_dict.setdefault('_type', 'video') if remove_private_keys: - reject = lambda k, v: v is None or (k.startswith('_') and k != '_type') or k in { + reject = lambda k, v: v is None or k.startswith('__') or k in { 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries', - 'entries', 'filepath', 'infojson_filename', 'original_url', 'playlist_autonumber', + 'entries', 'filepath', '_filename', 'infojson_filename', 'original_url', 'playlist_autonumber', } else: reject = lambda k, v: False diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index da38e502db..4fe56bb957 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -20,6 +20,7 @@ from ..utils import ( encodeFilename, handle_youtubedl_headers, remove_end, + traverse_obj, ) @@ -363,9 +364,11 @@ class FFmpegFD(ExternalFD): if not self.params.get('verbose'): args += ['-hide_banner'] - args += info_dict.get('_ffmpeg_args', []) + args += traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args'), default=[]) - # This option exists only for compatibility. Extractors should use `_ffmpeg_args` instead + # These exists only for compatibility. Extractors should use + # info_dict['downloader_options']['ffmpeg_args'] instead + args += info_dict.get('_ffmpeg_args') seekable = info_dict.get('_seekable') if seekable is not None: # setting -seekable prevents ffmpeg from guessing if the server diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 8c2fd7fea2..63f7b5d4a6 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -208,8 +208,10 @@ class InfoExtractor: * no_resume The server does not support resuming the (HTTP or RTMP) download. Boolean. * has_drm The format has DRM and cannot be downloaded. Boolean - * downloader_options A dictionary of downloader options as - described in FileDownloader (For internal use only) + * downloader_options A dictionary of downloader options + (For internal use only) + * http_chunk_size Chunk size for HTTP downloads + * ffmpeg_args Extra arguments for ffmpeg downloader RTMP formats can also have the additional fields: page_url, app, play_path, tc_url, flash_version, rtmp_live, rtmp_conn, rtmp_protocol, rtmp_real_time diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py index 8aab80a0f7..365c2e60d5 100644 --- a/yt_dlp/extractor/nbc.py +++ b/yt_dlp/extractor/nbc.py @@ -579,7 +579,7 @@ class NBCOlympicsStreamIE(AdobePassIE): for f in formats: # -http_seekable requires ffmpeg 4.3+ but it doesnt seem possible to # download with ffmpeg without this option - f['_ffmpeg_args'] = ['-seekable', '0', '-http_seekable', '0', '-icy', '0'] + f['downloader_options'] = {'ffmpeg_args': ['-seekable', '0', '-http_seekable', '0', '-icy', '0']} self._sort_formats(formats) return { diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py index a0f5ebdd08..651cfe63b1 100644 --- a/yt_dlp/extractor/radiko.py +++ b/yt_dlp/extractor/radiko.py @@ -125,7 +125,7 @@ class RadikoBaseIE(InfoExtractor): # Prioritize live radio vs playback based on extractor sf['preference'] = 100 if is_onair else -100 if not is_onair and url_attrib['timefree'] == '1' and time_to_skip: - sf['_ffmpeg_args'] = ['-ss', time_to_skip] + sf['downloader_options'] = {'ffmpeg_args': ['-ss', time_to_skip]} formats.extend(subformats) self._sort_formats(formats) diff --git a/yt_dlp/extractor/turner.py b/yt_dlp/extractor/turner.py index 568b6de495..fae8b51e76 100644 --- a/yt_dlp/extractor/turner.py +++ b/yt_dlp/extractor/turner.py @@ -141,7 +141,7 @@ class TurnerBaseIE(AdobePassIE): m3u8_id=format_id or 'hls', fatal=False) if '/secure/' in video_url and '?hdnea=' in video_url: for f in m3u8_formats: - f['_ffmpeg_args'] = ['-seekable', '0'] + f['downloader_options'] = {'ffmpeg_args': ['-seekable', '0']} formats.extend(m3u8_formats) elif ext == 'f4m': formats.extend(self._extract_f4m_formats(