[postprocessor] Add plugin support

Adds option `--use-postprocessor` to enable them

README.md

@@ -837,6 +837,20 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
                                      around the cuts
     --no-force-keyframes-at-cuts     Do not force keyframes around the chapters
                                      when cutting/splitting (default)
+    --use-postprocessor NAME[:ARGS]  The (case sensitive) name of plugin
+                                     postprocessors to be enabled, and
+                                     (optionally) arguments to be passed to it,
+                                     seperated by a colon ":". ARGS are a
+                                     semicolon ";" delimited list of NAME=VALUE.
+                                     The "when" argument determines when the
+                                     postprocessor is invoked. It can be one of
+                                     "pre_process" (after extraction),
+                                     "before_dl" (before video download),
+                                     "post_process" (after video download;
+                                     default) or "after_move" (after moving file
+                                     to their final locations). This option can
+                                     be used multiple times to add different
+                                     postprocessors
 
 ## SponsorBlock Options:
 Make chapter entries for, or remove various segments (sponsor,
@@ -1465,9 +1479,16 @@ NOTE: These options may be changed/removed in the future without concern for bac
 
 # PLUGINS
 
-Plugins are loaded from `<root-dir>/ytdlp_plugins/<type>/__init__.py`. Currently only `extractor` plugins are supported. Support for `downloader` and `postprocessor` plugins may be added in the future. See [ytdlp_plugins](ytdlp_plugins) for example.
+Plugins are loaded from `<root-dir>/ytdlp_plugins/<type>/__init__.py`; where `<root-dir>` is the directory of the binary (`<root-dir>/yt-dlp`), or the root directory of the module if you are running directly from source-code (`<root dir>/yt_dlp/__main__.py`). Plugins are currently not supported for the `pip` version
+
+Plugins can be of `<type>`s `extractor` or `postprocessor`. Extractor plugins do not need to be enabled from the CLI and are automatically invoked when the input URL is suitable for it. Postprocessor plugins can be invoked using `--use-postprocessor NAME`.
+
+See [ytdlp_plugins](ytdlp_plugins) for example plugins.
+
+Note that **all** plugins are imported even if not invoked, and that **there are no checks** performed on plugin code. Use plugins at your own risk and only if you trust the code
+
+If you are a plugin author, add [ytdlp-plugins](https://github.com/topics/ytdlp-plugins) as a topic to your repository for discoverability
 
-**Note**: `<root-dir>` is the directory of the binary (`<root-dir>/yt-dlp`), or the root directory of the module if you are running directly from source-code (`<root dir>/yt_dlp/__main__.py`)
 
 # DEPRECATED OPTIONS
 

yt_dlp/YoutubeDL.py

@@ -123,7 +123,7 @@ from .extractor import (
     gen_extractor_classes,
     get_info_extractor,
     _LAZY_LOADER,
-    _PLUGIN_CLASSES
+    _PLUGIN_CLASSES as plugin_extractors
 )
 from .extractor.openload import PhantomJSwrapper
 from .downloader import (
@@ -142,6 +142,7 @@ from .postprocessor import (
     FFmpegMergerPP,
     FFmpegPostProcessor,
     MoveFilesAfterDownloadPP,
+    _PLUGIN_CLASSES as plugin_postprocessors
 )
 from .update import detect_variant
 from .version import __version__
@@ -3201,9 +3202,10 @@ class YoutubeDL(object):
         self._write_string('[debug] yt-dlp version %s%s\n' % (__version__, '' if source == 'unknown' else f' ({source})'))
         if _LAZY_LOADER:
             self._write_string('[debug] Lazy loading extractors enabled\n')
-        if _PLUGIN_CLASSES:
+        if plugin_extractors or plugin_postprocessors:
-            self._write_string(
+            self._write_string('[debug] Plugins: %s\n' % [
-                '[debug] Plugin Extractors: %s\n' % [ie.ie_key() for ie in _PLUGIN_CLASSES])
+                '%s%s' % (klass.__name__, '' if klass.__name__ == name else f' as {name}')
+                for name, klass in itertools.chain(plugin_extractors.items(), plugin_postprocessors.items())])
         if self.params.get('compat_opts'):
             self._write_string(
                 '[debug] Compatibility options: %s\n' % ', '.join(self.params.get('compat_opts')))

yt_dlp/__init__.py

@@ -418,7 +418,7 @@ def _real_main(argv=None):
         opts.sponskrub = False
 
     # PostProcessors
-    postprocessors = []
+    postprocessors = list(opts.add_postprocessors)
     if sponsorblock_query:
         postprocessors.append({
             'key': 'SponsorBlock',

yt_dlp/extractor/__init__.py

@@ -6,7 +6,7 @@ try:
     from .lazy_extractors import *
     from .lazy_extractors import _ALL_CLASSES
     _LAZY_LOADER = True
-    _PLUGIN_CLASSES = []
+    _PLUGIN_CLASSES = {}
 except ImportError:
     _LAZY_LOADER = False
 
@@ -20,7 +20,7 @@ if not _LAZY_LOADER:
     _ALL_CLASSES.append(GenericIE)
 
     _PLUGIN_CLASSES = load_plugins('extractor', 'IE', globals())
-    _ALL_CLASSES = _PLUGIN_CLASSES + _ALL_CLASSES
+    _ALL_CLASSES = list(_PLUGIN_CLASSES.values()) + _ALL_CLASSES
 
 
 def gen_extractor_classes():

yt_dlp/options.py

@@ -17,6 +17,7 @@ from .utils import (
     get_executable_path,
     OUTTMPL_TYPES,
     preferredencoding,
+    remove_end,
     write_string,
 )
 from .cookies import SUPPORTED_BROWSERS
@@ -1389,6 +1390,25 @@ def parseOpts(overrideArguments=None):
         '--no-force-keyframes-at-cuts',
         action='store_false', dest='force_keyframes_at_cuts',
         help='Do not force keyframes around the chapters when cutting/splitting (default)')
+    _postprocessor_opts_parser = lambda key, val='': (
+        *(item.split('=', 1) for item in (val.split(';') if val else [])),
+        ('key', remove_end(key, 'PP')))
+    postproc.add_option(
+        '--use-postprocessor',
+        metavar='NAME[:ARGS]', dest='add_postprocessors', default=[], type='str',
+        action='callback', callback=_list_from_options_callback,
+        callback_kwargs={
+            'delim': None,
+            'process': lambda val: dict(_postprocessor_opts_parser(*val.split(':', 1)))
+        }, help=(
+            'The (case sensitive) name of plugin postprocessors to be enabled, '
+            'and (optionally) arguments to be passed to it, seperated by a colon ":". '
+            'ARGS are a semicolon ";" delimited list of NAME=VALUE. '
+            'The "when" argument determines when the postprocessor is invoked. '
+            'It can be one of "pre_process" (after extraction), '
+            '"before_dl" (before video download), "post_process" (after video download; default) '
+            'or "after_move" (after moving file to their final locations). '
+            'This option can be used multiple times to add different postprocessors'))
 
     sponsorblock = optparse.OptionGroup(parser, 'SponsorBlock Options', description=(
         'Make chapter entries for, or remove various segments (sponsor, introductions, etc.) '

yt_dlp/postprocessor/__init__.py

@@ -1,6 +1,9 @@
-from __future__ import unicode_literals
+# flake8: noqa: F401
+
+from ..utils import load_plugins
 
 from .embedthumbnail import EmbedThumbnailPP
+from .exec import ExecPP, ExecAfterDownloadPP
 from .ffmpeg import (
     FFmpegPostProcessor,
     FFmpegEmbedSubtitlePP,
@@ -18,48 +21,23 @@ from .ffmpeg import (
     FFmpegVideoConvertorPP,
     FFmpegVideoRemuxerPP,
 )
-from .xattrpp import XAttrMetadataPP
-from .exec import ExecPP, ExecAfterDownloadPP
 from .metadataparser import (
     MetadataFromFieldPP,
     MetadataFromTitlePP,
     MetadataParserPP,
 )
-from .movefilesafterdownload import MoveFilesAfterDownloadPP
-from .sponsorblock import SponsorBlockPP
-from .sponskrub import SponSkrubPP
 from .modify_chapters import ModifyChaptersPP
+from .movefilesafterdownload import MoveFilesAfterDownloadPP
+from .sponskrub import SponSkrubPP
+from .sponsorblock import SponsorBlockPP
+from .xattrpp import XAttrMetadataPP
+
+_PLUGIN_CLASSES = load_plugins('postprocessor', 'PP', globals())
 
 
 def get_postprocessor(key):
     return globals()[key + 'PP']
 
 
-__all__ = [
+__all__ = [name for name in globals().keys() if name.endswith('IE')]
-    'FFmpegPostProcessor',
+__all__.append('FFmpegPostProcessor')
-    'EmbedThumbnailPP',
-    'ExecPP',
-    'ExecAfterDownloadPP',
-    'FFmpegEmbedSubtitlePP',
-    'FFmpegExtractAudioPP',
-    'FFmpegSplitChaptersPP',
-    'FFmpegFixupDurationPP',
-    'FFmpegFixupM3u8PP',
-    'FFmpegFixupM4aPP',
-    'FFmpegFixupStretchedPP',
-    'FFmpegFixupTimestampPP',
-    'FFmpegMergerPP',
-    'FFmpegMetadataPP',
-    'FFmpegSubtitlesConvertorPP',
-    'FFmpegThumbnailsConvertorPP',
-    'FFmpegVideoConvertorPP',
-    'FFmpegVideoRemuxerPP',
-    'MetadataParserPP',
-    'MetadataFromFieldPP',
-    'MetadataFromTitlePP',
-    'MoveFilesAfterDownloadPP',
-    'SponsorBlockPP',
-    'SponSkrubPP',
-    'ModifyChaptersPP',
-    'XAttrMetadataPP',
-]

yt_dlp/utils.py

@@ -6278,7 +6278,7 @@ def get_executable_path():
 
 def load_plugins(name, suffix, namespace):
     plugin_info = [None]
-    classes = []
+    classes = {}
     try:
         plugin_info = imp.find_module(
             name, [os.path.join(get_executable_path(), 'ytdlp_plugins')])
@@ -6289,8 +6289,7 @@ def load_plugins(name, suffix, namespace):
             if not name.endswith(suffix):
                 continue
             klass = getattr(plugins, name)
-            classes.append(klass)
+            classes[name] = namespace[name] = klass
-            namespace[name] = klass
     except ImportError:
         pass
     finally:

ytdlp_plugins/extractor/__init__.py

@@ -1,3 +1,4 @@
-# flake8: noqa
+# flake8: noqa: F401
 
+# ℹ️ The imported name must end in "IE"
 from .sample import SamplePluginIE

ytdlp_plugins/extractor/sample.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 
-from __future__ import unicode_literals
-
 # ⚠ Don't use relative imports
 from yt_dlp.extractor.common import InfoExtractor
 

ytdlp_plugins/postprocessor/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+
+# ℹ️ The imported name must end in "PP" and is the name to be used in --use-postprocessor
+from .sample import SamplePluginPP

ytdlp_plugins/postprocessor/sample.py

@@ -0,0 +1,23 @@
+# coding: utf-8
+
+# ⚠ Don't use relative imports
+from yt_dlp.postprocessor.common import PostProcessor
+
+
+# ℹ️ See the docstring of yt_dlp.postprocessor.common.PostProcessor
+class SamplePluginPP(PostProcessor):
+    def __init__(self, downloader=None, **kwargs):
+        # ⚠ Only kwargs can be passed from the CLI, and all argument values will be string
+        # Also, "downloader", "when" and "key" are reserved names
+        super().__init__(downloader)
+        self._kwargs = kwargs
+
+    # ℹ️ See docstring of yt_dlp.postprocessor.common.PostProcessor.run
+    def run(self, info):
+        filepath = info.get('filepath')
+        if filepath:  # PP was called after download (default)
+            self.to_screen(f'Post-processed {filepath!r} with {self._kwargs}')
+        else:  # PP was called before actual download
+            filepath = info.get('_filename')
+            self.to_screen(f'Pre-processed {filepath!r} with {self._kwargs}')
+        return [], info  # return list_of_files_to_delete, info_dict