[extractor/generic:quoted-html] Add extractor (#5213)

Extracts embeds from escaped HTML within `data-html` attribute.
Related: https://github.com/ytdl-org/youtube-dl/issues/21294, https://github.com/yt-dlp/yt-dlp/pull/5121

Authored by: coletdjnz
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>

yt_dlp/extractor/_extractors.py

@@ -698,7 +698,10 @@ from .hse import (
     HSEShowIE,
     HSEProductIE,
 )
-from .genericembeds import HTML5MediaEmbedIE
+from .genericembeds import (
+    HTML5MediaEmbedIE,
+    QuotedHTMLIE,
+)
 from .huajiao import HuajiaoIE
 from .huya import HuyaLiveIE
 from .huffpost import HuffPostIE
@@ -1884,7 +1887,6 @@ from .tv2 import (
 )
 from .tv24ua import (
     TV24UAVideoIE,
-    TV24UAGenericPassthroughIE
 )
 from .tv2dk import (
     TV2DKIE,

yt_dlp/extractor/generic.py

@@ -1980,22 +1980,6 @@ class GenericIE(InfoExtractor):
             },
             'playlist_count': 6,
         },
-        {
-            # Squarespace video embed, 2019-08-28
-            'url': 'http://ootboxford.com',
-            'info_dict': {
-                'id': 'Tc7b_JGdZfw',
-                'title': 'Out of the Blue, at Childish Things 10',
-                'ext': 'mp4',
-                'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
-                'uploader_id': 'helendouglashouse',
-                'uploader': 'Helen & Douglas House',
-                'upload_date': '20140328',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
         # {
         #     # Zype embed
         #     'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites',
@@ -2784,12 +2768,6 @@ class GenericIE(InfoExtractor):
         # There probably should be a second run of generic extractor on unescaped webpage.
         # webpage = urllib.parse.unquote(webpage)
 
-        # Unescape squarespace embeds to be detected by generic extractor,
-        # see https://github.com/ytdl-org/youtube-dl/issues/21294
-        webpage = re.sub(
-            r'<div[^>]+class=[^>]*?\bsqs-video-wrapper\b[^>]*>',
-            lambda x: unescapeHTML(x.group(0)), webpage)
-
         # TODO: Move to respective extractors
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:

yt_dlp/extractor/genericembeds.py

@@ -1,5 +1,8 @@
+import re
+import urllib.parse
+
 from .common import InfoExtractor
-from ..utils import make_archive_id
+from ..utils import make_archive_id, unescapeHTML
 
 
 class HTML5MediaEmbedIE(InfoExtractor):
@@ -29,3 +32,84 @@ class HTML5MediaEmbedIE(InfoExtractor):
             })
             self._sort_formats(entry['formats'])
             yield entry
+
+
+class QuotedHTMLIE(InfoExtractor):
+    """For common cases of quoted/escaped html parts in the webpage"""
+    _VALID_URL = False
+    IE_NAME = 'generic:quoted-html'
+    IE_DESC = False  # Do not list
+    _WEBPAGE_TESTS = [{
+        # 2 YouTube embeds in data-html
+        'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
+        'info_dict': {
+            'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
+            'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті',
+            'thumbnail': r're:^https?://.*\.jpe?g',
+            'timestamp': float,
+            'upload_date': str,
+            'description': 'md5:6816e1e5a65304bd7898e4c7eb1b26f7',
+            'age_limit': 0,
+        },
+        'playlist_count': 2
+    }, {
+        # Generic iframe embed of TV24UAPlayerIE within data-html
+        'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
+        'info_dict': {
+            'id': '1887584',
+            'ext': 'mp4',
+            'title': 'Харків\'яни згадують місто до війни: щемливе відео',
+            'thumbnail': r're:^https?://.*\.jpe?g',
+        },
+        'params': {'skip_download': True}
+    }, {
+        # YouTube embeds on Squarespace (data-html): https://github.com/ytdl-org/youtube-dl/issues/21294
+        'url': 'https://www.harvardballetcompany.org/past-productions',
+        'info_dict': {
+            'id': 'past-productions',
+            'title': 'Productions — Harvard Ballet Company',
+            'age_limit': 0,
+            'description': 'Past Productions',
+        },
+        'playlist_mincount': 26
+    }, {
+        # Squarespace video embed, 2019-08-28, data-html
+        'url': 'http://ootboxford.com',
+        'info_dict': {
+            'id': 'Tc7b_JGdZfw',
+            'title': 'Out of the Blue, at Childish Things 10',
+            'ext': 'mp4',
+            'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
+            'uploader_id': 'helendouglashouse',
+            'uploader': 'Helen & Douglas House',
+            'upload_date': '20140328',
+            'availability': 'public',
+            'view_count': int,
+            'channel': 'Helen & Douglas House',
+            'comment_count': int,
+            'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
+            'duration': 253,
+            'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
+            'playable_in_embed': True,
+            'age_limit': 0,
+            'channel_follower_count': int,
+            'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
+            'tags': 'count:6',
+            'categories': ['Nonprofits & Activism'],
+            'like_count': int,
+            'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _extract_from_webpage(self, url, webpage):
+        combined = ''
+        for _, html in re.findall(r'(?s)\bdata-html=(["\'])((?:(?!\1).)+)\1', webpage):
+            # unescapeHTML can handle " etc., unquote can handle percent encoding
+            unquoted_html = unescapeHTML(urllib.parse.unquote(html))
+            if unquoted_html != html:
+                combined += unquoted_html
+        if combined:
+            yield from self._extract_generic_embeds(url, combined)

yt_dlp/extractor/tv24ua.py

@@ -1,15 +1,10 @@
-import base64
 import re
-import urllib.parse
 
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
-    extract_attributes,
-    get_elements_html_by_class,
     js_to_json,
     mimetype2ext,
-    smuggle_url,
     traverse_obj,
 )
 
@@ -87,60 +82,3 @@ class TV24UAVideoIE(InfoExtractor):
             'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
             'description': self._og_search_description(webpage, default=None),
         }
-
-
-class TV24UAGenericPassthroughIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[a-zA-Z0-9]+?\.)?24tv\.ua/(?P<id>[^/]+?_n\d+)'
-
-    _TESTS = [{
-        # Generic iframe, not within media_embed
-        'url': 'https://24tv.ua/vipalyuyut-nashi-mista-sela-dsns-pokazali-motoroshni-naslidki_n1883966',
-        'info_dict': {
-            'id': '1883966',
-            'ext': 'mp4',
-            'title': 'Випалюють наші міста та села, – моторошні наслідки обстрілів на Чернігівщині',
-            'thumbnail': r're:^https?://.*\.jpe?g',
-        }
-    }, {
-        # Generic iframe embed of TV24UAPlayerIE, within media_embed
-        'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
-        'info_dict': {
-            'id': 'harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
-            'title': 'Харків\'яни згадують місто до війни: щемливе відео'
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '1887584',
-                'ext': 'mp4',
-                'title': 'Харків\'яни згадують місто до війни: щемливе відео',
-                'thumbnail': r're:^https?://.*\.jpe?g',
-            },
-        }]
-    }, {
-        # 2 media_embeds with YouTube iframes
-        'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
-        'info_dict': {
-            'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
-            'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті',
-        },
-        'playlist_count': 2
-    }, {
-        'url': 'https://men.24tv.ua/fitnes-bloger-sprobuvav-vikonati-trenuvannya-naysilnishoyi-lyudini_n2164538',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        data_urls = []
-        # The site contains escaped iframe embeds within an attribute.
-        # Once escaped, generic can handle them, so we use a data url to pass the escaped html back.
-        for html in get_elements_html_by_class('media_embed', webpage):
-            data = urllib.parse.unquote(extract_attributes(html).get('data-html'))
-            data_urls.append(f'data:text/html;base64,{base64.b64encode(data.encode("utf-8")).decode("utf-8")}')
-
-        if not data_urls:
-            return self.url_result(url, 'Generic')
-        return self.playlist_from_matches(
-            [smuggle_url(url, {'to_generic': True}) for url in data_urls], display_id, ie='Generic',
-            playlist_title=self._og_search_title(webpage) or self._html_extract_title(webpage))