From 6551b10bd6b2b31176ac693c66c9a3289cd26ad2 Mon Sep 17 00:00:00 2001 From: Mozi <29089388+pzhlkj6612@users.noreply.github.com> Date: Sun, 24 Nov 2024 03:38:44 +0000 Subject: [PATCH] simplify traversal on extraction of formats and subtitles --- yt_dlp/extractor/underline.py | 71 ++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/yt_dlp/extractor/underline.py b/yt_dlp/extractor/underline.py index 72aa8423aa..c3e1bd149d 100644 --- a/yt_dlp/extractor/underline.py +++ b/yt_dlp/extractor/underline.py @@ -4,6 +4,7 @@ from .common import InfoExtractor from ..utils import ( OnDemandPagedList, filter_dict, + merge_dicts, update_url, url_or_none, ) @@ -12,7 +13,6 @@ from ..utils.traversal import traverse_obj class UnderlinePosterIE(InfoExtractor): _VALID_URL = r'https://(?:www\.)?underline\.io/events/\d+/posters/\d+/poster/(?P\d+)-[\w-]+/?(?:[#?]|$)' - _TESTS = [{ # Video and PPT 'url': 'https://underline.io/events/342/posters/12863/poster/66466-towards-a-general-purpose-machine-translation-system-for-sranantongo', @@ -21,16 +21,14 @@ class UnderlinePosterIE(InfoExtractor): 'ext': 'mp4', 'title': 'Towards a general purpose machine translation system for Sranantongo', 'thumbnail': 'https://assets.underline.io/lecture/66466/poster_document_thumbnail_extract/2a8d3abd5a5edbf0117d8d9bdf018e6d.jpg', - 'live_status': 'not_live', }, }, { - # PPT only + # no video, PPT only 'url': 'https://underline.io/events/342/posters/12863/poster/66459-low-resourced-multilingual-neural-machine-translation-for-ometo-english', 'info_dict': { 'id': '66459', 'title': 'Low Resourced Multilingual Neural Machine Translation for Ometo-English', 'thumbnail': 'https://assets.underline.io/lecture/66459/poster_document_thumbnail_extract/69a3fd48e3bb137ddcf3099db637d184.jpg', - 'live_status': 'not_live', }, 'params': { 'skip_download': True, @@ -48,33 +46,34 @@ class UnderlinePosterIE(InfoExtractor): data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['fallback'] - m3u8_url = traverse_obj(data, (..., 'data', 'attributes', 'playlist', {url_or_none}), get_all=False) - subtitle_urls = filter_dict(dict(traverse_obj(data, ( - ..., 'data', lambda _, v: v['type'] == 'transcripts', {lambda x: ( - x['relationships']['language']['data']['id'], - url_or_none(x['attributes']['subtitleUrl']), - )})))) + formats = traverse_obj(data, ( + ..., 'data', 'attributes', 'playlist', {url_or_none}, filter, + {lambda x: self._extract_m3u8_formats(x, video_id)}, any)) + + subtitles = {} + subtitle_urls = traverse_obj(data, (..., 'data', lambda _, v: v['type'] == 'transcripts', { + 'id': ('relationships', 'language', 'data', 'id'), + 'url': ('attributes', 'subtitleUrl', {url_or_none}), + }, {lambda x: {x['id']: x['url']}}, all, {lambda x: merge_dicts(*x)}, {filter_dict}), default={}) + if subtitle_urls: + traverse_obj(data, (..., 'included', lambda _, v: v['type'] == 'transcript_languages', { + 'tag': ('attributes', 'locale'), + 'url': ('id', {subtitle_urls.get}), + }, {filter_dict}, {lambda x: self._merge_subtitles({x['tag']: [{'url': x['url']}]}, target=subtitles)})) return { 'id': video_id, - 'title': traverse_obj(data, (..., 'data', 'attributes', 'title', {str}), get_all=False), - 'thumbnail': traverse_obj(data, ( - ..., 'data', 'attributes', 'originalPosterDocumentThumbnailExtractUrl', {url_or_none}), get_all=False), - 'formats': self._extract_m3u8_formats(m3u8_url, video_id) if m3u8_url else [], - 'subtitles': filter_dict(dict(traverse_obj(data, ( - ..., 'included', lambda _, v: v['type'] == 'transcript_languages', {lambda x: ( - x['attributes']['locale'], - [{'url': subtitle_urls[x['id']]}], - )})))), - 'live_status': 'not_live', + 'title': traverse_obj(data, (..., 'data', 'attributes', 'title', {str}, any)), + 'thumbnail': traverse_obj(data, (..., 'data', 'attributes', 'originalPosterDocumentThumbnailExtractUrl', {url_or_none}, any)), + 'formats': formats, + 'subtitles': subtitles, } class UnderlinePosterListIE(InfoExtractor): - _VALID_URL = r'https://(?:www\.)?underline\.io/events/(?P\d+)/posters/?\?(?:[^&]+&)*?eventSessionId=(?P\d+)(?:&|#)?' + _VALID_URL = r'https://(?:www\.)?underline\.io/events/(?P\d+)/posters/?\?(?:[^&]+&)*?eventSessionId=(?P\d+)(?:&|#)?' _HEADERS = {'Accept': 'application/vnd.api+json'} _PAGE_SIZE = 16 - _TESTS = [{ 'url': 'https://underline.io/events/342/posters?eventSessionId=12863', 'info_dict': { @@ -88,23 +87,29 @@ class UnderlinePosterListIE(InfoExtractor): 'only_matching': True, }] - def _fetch_paged_channel_video_list(self, list_url, event_id, event_session_id, page): + def _fetch_paged_channel_video_list(self, list_url, event_id, session_id, page): page_num = page + 1 - video_url_tmpl = f'https://underline.io/events/{event_id}/posters/{event_session_id}/poster/{{id}}-{{slug}}' + video_url_tmpl = f'https://underline.io/events/{event_id}/posters/{session_id}/poster/{{id}}-{{slug}}' page_info = self._download_json(update_url( - list_url, query_update={'page[number]': page_num}), event_session_id, headers=self._HEADERS, + list_url, query_update={'page[number]': page_num}), session_id, headers=self._HEADERS, note=f'Downloading list info (page {page_num})', errnote=f'Failed to download list info (page {page_num})') yield from traverse_obj(page_info, ( 'data', ..., {lambda x: video_url_tmpl.format(id=x['id'], slug=x['attributes']['slug'])}, {self.url_result})) def _real_extract(self, url): - event_id, event_session_id = self._match_valid_url(url).groups() + event_id, session_id = self._match_valid_url(url).groups() - event_info = self._download_json( - f'https://app.underline.io/api/v1/thin_event_sessions/{event_session_id}', event_session_id, headers=self._HEADERS, - note='Downloading event session info', errnote='Failed to download event session info') + session_info = { + **traverse_obj(self._download_json( + f'https://app.underline.io/api/v1/thin_event_sessions/{session_id}', session_id, + headers=self._HEADERS, note='Downloading session info', errnote='Failed to download session info'), ({ + 'title': ('data', 'attributes', 'name'), + 'description': ('data', 'attributes', 'description'), + }, any)), + 'id': session_id, + } return self.playlist_result(OnDemandPagedList(functools.partial( self._fetch_paged_channel_video_list, @@ -115,9 +120,7 @@ class UnderlinePosterListIE(InfoExtractor): 'include': 'sorted_profiles,event_session,tag,event', 'filter[scope]': 'all', 'filter[event_id]': event_id, - 'filter[event_session_id]': event_session_id, + 'filter[event_session_id]': session_id, 'page[size]': self._PAGE_SIZE, - }), event_id, event_session_id), self._PAGE_SIZE), - event_session_id, - traverse_obj(event_info, ('data', 'attributes', 'name'), get_all=False), - traverse_obj(event_info, ('data', 'attributes', 'description'), get_all=False)) + }), event_id, session_id), self._PAGE_SIZE), + **session_info)