Miroirs/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp synced 2024-12-27 21:59:17 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

simplify traversal on extraction of formats and subtitles

Browse source
This commit is contained in:
Mozi 2024-11-24 03:38:44 +00:00
parent a064d31dbe
commit 6551b10bd6
1 changed files with 37 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
yt_dlp/extractor/underline.py
View file

@ -4,6 +4,7 @@ from .common import InfoExtractor
from ..utils import ( from ..utils import (
OnDemandPagedList, OnDemandPagedList,
filter_dict, filter_dict,
merge_dicts,
update_url, update_url,
url_or_none, url_or_none,
) )
@ -12,7 +13,6 @@ from ..utils.traversal import traverse_obj
class UnderlinePosterIE(InfoExtractor): class UnderlinePosterIE(InfoExtractor):
_VALID_URL = r'https://(?:www\.)?underline\.io/events/\d+/posters/\d+/poster/(?P<id>\d+)-[\w-]+/?(?:[#?]|$)' _VALID_URL = r'https://(?:www\.)?underline\.io/events/\d+/posters/\d+/poster/(?P<id>\d+)-[\w-]+/?(?:[#?]|$)'
_TESTS = [{ _TESTS = [{
# Video and PPT # Video and PPT
'url': 'https://underline.io/events/342/posters/12863/poster/66466-towards-a-general-purpose-machine-translation-system-for-sranantongo', 'url': 'https://underline.io/events/342/posters/12863/poster/66466-towards-a-general-purpose-machine-translation-system-for-sranantongo',
@ -21,16 +21,14 @@ class UnderlinePosterIE(InfoExtractor):
'ext': 'mp4', 'ext': 'mp4',
'title': 'Towards a general purpose machine translation system for Sranantongo', 'title': 'Towards a general purpose machine translation system for Sranantongo',
'thumbnail': 'https://assets.underline.io/lecture/66466/poster_document_thumbnail_extract/2a8d3abd5a5edbf0117d8d9bdf018e6d.jpg', 'thumbnail': 'https://assets.underline.io/lecture/66466/poster_document_thumbnail_extract/2a8d3abd5a5edbf0117d8d9bdf018e6d.jpg',
'live_status': 'not_live',
}, },
}, { }, {
# PPT only # no video, PPT only
'url': 'https://underline.io/events/342/posters/12863/poster/66459-low-resourced-multilingual-neural-machine-translation-for-ometo-english', 'url': 'https://underline.io/events/342/posters/12863/poster/66459-low-resourced-multilingual-neural-machine-translation-for-ometo-english',
'info_dict': { 'info_dict': {
'id': '66459', 'id': '66459',
'title': 'Low Resourced Multilingual Neural Machine Translation for Ometo-English', 'title': 'Low Resourced Multilingual Neural Machine Translation for Ometo-English',
'thumbnail': 'https://assets.underline.io/lecture/66459/poster_document_thumbnail_extract/69a3fd48e3bb137ddcf3099db637d184.jpg', 'thumbnail': 'https://assets.underline.io/lecture/66459/poster_document_thumbnail_extract/69a3fd48e3bb137ddcf3099db637d184.jpg',
'live_status': 'not_live',
}, },
'params': { 'params': {
'skip_download': True, 'skip_download': True,
@ -48,33 +46,34 @@ class UnderlinePosterIE(InfoExtractor):
data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['fallback'] data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['fallback']
m3u8_url = traverse_obj(data, (..., 'data', 'attributes', 'playlist', {url_or_none}), get_all=False) formats = traverse_obj(data, (
subtitle_urls = filter_dict(dict(traverse_obj(data, ( ..., 'data', 'attributes', 'playlist', {url_or_none}, filter,
..., 'data', lambda _, v: v['type'] == 'transcripts', {lambda x: ( {lambda x: self._extract_m3u8_formats(x, video_id)}, any))
x['relationships']['language']['data']['id'],
url_or_none(x['attributes']['subtitleUrl']), subtitles = {}
)})))) subtitle_urls = traverse_obj(data, (..., 'data', lambda _, v: v['type'] == 'transcripts', {
'id': ('relationships', 'language', 'data', 'id'),
'url': ('attributes', 'subtitleUrl', {url_or_none}),
}, {lambda x: {x['id']: x['url']}}, all, {lambda x: merge_dicts(*x)}, {filter_dict}), default={})
if subtitle_urls:
traverse_obj(data, (..., 'included', lambda _, v: v['type'] == 'transcript_languages', {
'tag': ('attributes', 'locale'),
'url': ('id', {subtitle_urls.get}),
}, {filter_dict}, {lambda x: self._merge_subtitles({x['tag']: [{'url': x['url']}]}, target=subtitles)}))
return { return {
'id': video_id, 'id': video_id,
'title': traverse_obj(data, (..., 'data', 'attributes', 'title', {str}), get_all=False), 'title': traverse_obj(data, (..., 'data', 'attributes', 'title', {str}, any)),
'thumbnail': traverse_obj(data, ( 'thumbnail': traverse_obj(data, (..., 'data', 'attributes', 'originalPosterDocumentThumbnailExtractUrl', {url_or_none}, any)),
..., 'data', 'attributes', 'originalPosterDocumentThumbnailExtractUrl', {url_or_none}), get_all=False), 'formats': formats,
'formats': self._extract_m3u8_formats(m3u8_url, video_id) if m3u8_url else [], 'subtitles': subtitles,
'subtitles': filter_dict(dict(traverse_obj(data, (
..., 'included', lambda _, v: v['type'] == 'transcript_languages', {lambda x: (
x['attributes']['locale'],
[{'url': subtitle_urls[x['id']]}],
)})))),
'live_status': 'not_live',
} }
class UnderlinePosterListIE(InfoExtractor): class UnderlinePosterListIE(InfoExtractor):
_VALID_URL = r'https://(?:www\.)?underline\.io/events/(?P<event_id>\d+)/posters/?\?(?:[^&]+&)*?eventSessionId=(?P<event_session_id>\d+)(?:&|#)?' _VALID_URL = r'https://(?:www\.)?underline\.io/events/(?P<event_id>\d+)/posters/?\?(?:[^&]+&)*?eventSessionId=(?P<session_id>\d+)(?:&|#)?'
_HEADERS = {'Accept': 'application/vnd.api+json'} _HEADERS = {'Accept': 'application/vnd.api+json'}
_PAGE_SIZE = 16 _PAGE_SIZE = 16
_TESTS = [{ _TESTS = [{
'url': 'https://underline.io/events/342/posters?eventSessionId=12863', 'url': 'https://underline.io/events/342/posters?eventSessionId=12863',
'info_dict': { 'info_dict': {
@ -88,23 +87,29 @@ class UnderlinePosterListIE(InfoExtractor):
'only_matching': True, 'only_matching': True,
}] }]
def _fetch_paged_channel_video_list(self, list_url, event_id, event_session_id, page): def _fetch_paged_channel_video_list(self, list_url, event_id, session_id, page):
page_num = page + 1 page_num = page + 1
video_url_tmpl = f'https://underline.io/events/{event_id}/posters/{event_session_id}/poster/{{id}}-{{slug}}' video_url_tmpl = f'https://underline.io/events/{event_id}/posters/{session_id}/poster/{{id}}-{{slug}}'
page_info = self._download_json(update_url( page_info = self._download_json(update_url(
list_url, query_update={'page[number]': page_num}), event_session_id, headers=self._HEADERS, list_url, query_update={'page[number]': page_num}), session_id, headers=self._HEADERS,
note=f'Downloading list info (page {page_num})', errnote=f'Failed to download list info (page {page_num})') note=f'Downloading list info (page {page_num})', errnote=f'Failed to download list info (page {page_num})')
yield from traverse_obj(page_info, ( yield from traverse_obj(page_info, (
'data', ..., {lambda x: video_url_tmpl.format(id=x['id'], slug=x['attributes']['slug'])}, {self.url_result})) 'data', ..., {lambda x: video_url_tmpl.format(id=x['id'], slug=x['attributes']['slug'])}, {self.url_result}))
def _real_extract(self, url): def _real_extract(self, url):
event_id, event_session_id = self._match_valid_url(url).groups() event_id, session_id = self._match_valid_url(url).groups()
event_info = self._download_json( session_info = {
f'https://app.underline.io/api/v1/thin_event_sessions/{event_session_id}', event_session_id, headers=self._HEADERS, **traverse_obj(self._download_json(
note='Downloading event session info', errnote='Failed to download event session info') f'https://app.underline.io/api/v1/thin_event_sessions/{session_id}', session_id,
headers=self._HEADERS, note='Downloading session info', errnote='Failed to download session info'), ({
'title': ('data', 'attributes', 'name'),
'description': ('data', 'attributes', 'description'),
}, any)),
'id': session_id,
}
return self.playlist_result(OnDemandPagedList(functools.partial( return self.playlist_result(OnDemandPagedList(functools.partial(
self._fetch_paged_channel_video_list, self._fetch_paged_channel_video_list,
@ -115,9 +120,7 @@ class UnderlinePosterListIE(InfoExtractor):
'include': 'sorted_profiles,event_session,tag,event', 'include': 'sorted_profiles,event_session,tag,event',
'filter[scope]': 'all', 'filter[scope]': 'all',
'filter[event_id]': event_id, 'filter[event_id]': event_id,
'filter[event_session_id]': event_session_id, 'filter[event_session_id]': session_id,
'page[size]': self._PAGE_SIZE, 'page[size]': self._PAGE_SIZE,
}), event_id, event_session_id), self._PAGE_SIZE), }), event_id, session_id), self._PAGE_SIZE),
event_session_id, **session_info)
traverse_obj(event_info, ('data', 'attributes', 'name'), get_all=False),
traverse_obj(event_info, ('data', 'attributes', 'description'), get_all=False))