From f3b3fe16af86e3b32a08df9ef4f32ba0f4f84016 Mon Sep 17 00:00:00 2001 From: Henrik Heimbuerger Date: Sun, 15 May 2022 13:55:44 +0200 Subject: [PATCH] [nebula] Add support for subscriptions (#3719) Closes #3609 Authored by: hheimbuerger --- yt_dlp/extractor/extractors.py | 3 +- yt_dlp/extractor/nebula.py | 95 ++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py index ee5ced11ae..3970f1477a 100644 --- a/yt_dlp/extractor/extractors.py +++ b/yt_dlp/extractor/extractors.py @@ -1018,7 +1018,8 @@ from .ndr import ( from .ndtv import NDTVIE from .nebula import ( NebulaIE, - NebulaCollectionIE, + NebulaSubscriptionsIE, + NebulaChannelIE, ) from .nerdcubed import NerdCubedFeedIE from .netzkino import NetzkinoIE diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py index eccf740aad..ff9a2adf07 100644 --- a/yt_dlp/extractor/nebula.py +++ b/yt_dlp/extractor/nebula.py @@ -18,9 +18,8 @@ class NebulaBaseIE(InfoExtractor): _nebula_bearer_token = None _zype_access_token = None - def _perform_nebula_auth(self): - username, password = self._get_login_info() - if not (username and password): + def _perform_nebula_auth(self, username, password): + if not username or not password: self.raise_login_required() data = json.dumps({'email': username, 'password': password}).encode('utf8') @@ -51,7 +50,7 @@ class NebulaBaseIE(InfoExtractor): return response['key'] - def _retrieve_nebula_api_token(self): + def _retrieve_nebula_api_token(self, username=None, password=None): """ Check cookie jar for valid token. Try to authenticate using credentials if no valid token can be found in the cookie jar. @@ -65,7 +64,7 @@ class NebulaBaseIE(InfoExtractor): if nebula_api_token: return nebula_api_token - return self._perform_nebula_auth() + return self._perform_nebula_auth(username, password) def _call_nebula_api(self, url, video_id=None, method='GET', auth_type='api', note=''): assert method in ('GET', 'POST',) @@ -146,8 +145,7 @@ class NebulaBaseIE(InfoExtractor): } def _perform_login(self, username=None, password=None): - # FIXME: username should be passed from here to inner functions - self._nebula_api_token = self._retrieve_nebula_api_token() + self._nebula_api_token = self._retrieve_nebula_api_token(username, password) self._nebula_bearer_token = self._fetch_nebula_bearer_token() self._zype_access_token = self._fetch_zype_access_token() @@ -157,7 +155,7 @@ class NebulaIE(NebulaBaseIE): _TESTS = [ { 'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast', - 'md5': 'fe79c4df8b3aa2fea98a93d027465c7e', + 'md5': '14944cfee8c7beeea106320c47560efc', 'info_dict': { 'id': '5c271b40b13fd613090034fd', 'ext': 'mp4', @@ -169,14 +167,21 @@ class NebulaIE(NebulaBaseIE): 'channel_id': 'lindsayellis', 'uploader': 'Lindsay Ellis', 'uploader_id': 'lindsayellis', - }, - 'params': { - 'usenetrc': True, + 'timestamp': 1533009600, + 'uploader_url': 'https://nebula.app/lindsayellis', + 'series': 'Lindsay Ellis', + 'average_rating': int, + 'display_id': 'that-time-disney-remade-beauty-and-the-beast', + 'channel_url': 'https://nebula.app/lindsayellis', + 'creator': 'Lindsay Ellis', + 'duration': 2212, + 'view_count': int, + 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*', }, }, { 'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore', - 'md5': '6d4edd14ce65720fa63aba5c583fb328', + 'md5': 'd05739cf6c38c09322422f696b569c23', 'info_dict': { 'id': '5e7e78171aaf320001fbd6be', 'ext': 'mp4', @@ -188,14 +193,20 @@ class NebulaIE(NebulaBaseIE): 'channel_id': 'realengineering', 'uploader': 'Real Engineering', 'uploader_id': 'realengineering', - }, - 'params': { - 'usenetrc': True, + 'view_count': int, + 'series': 'Real Engineering', + 'average_rating': int, + 'display_id': 'the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore', + 'creator': 'Real Engineering', + 'duration': 841, + 'channel_url': 'https://nebula.app/realengineering', + 'uploader_url': 'https://nebula.app/realengineering', + 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*', }, }, { 'url': 'https://nebula.app/videos/money-episode-1-the-draw', - 'md5': '8c7d272910eea320f6f8e6d3084eecf5', + 'md5': 'ebe28a7ad822b9ee172387d860487868', 'info_dict': { 'id': '5e779ebdd157bc0001d1c75a', 'ext': 'mp4', @@ -207,9 +218,15 @@ class NebulaIE(NebulaBaseIE): 'channel_id': 'tom-scott-presents-money', 'uploader': 'Tom Scott Presents: Money', 'uploader_id': 'tom-scott-presents-money', - }, - 'params': { - 'usenetrc': True, + 'uploader_url': 'https://nebula.app/tom-scott-presents-money', + 'duration': 825, + 'channel_url': 'https://nebula.app/tom-scott-presents-money', + 'view_count': int, + 'series': 'Tom Scott Presents: Money', + 'display_id': 'money-episode-1-the-draw', + 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*', + 'average_rating': int, + 'creator': 'Tom Scott Presents: Money', }, }, { @@ -230,9 +247,37 @@ class NebulaIE(NebulaBaseIE): return self._build_video_info(video) -class NebulaCollectionIE(NebulaBaseIE): - IE_NAME = 'nebula:collection' - _VALID_URL = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/(?!videos/)(?P[-\w]+)' +class NebulaSubscriptionsIE(NebulaBaseIE): + IE_NAME = 'nebula:subscriptions' + _VALID_URL = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/myshows' + _TESTS = [ + { + 'url': 'https://nebula.app/myshows', + 'playlist_mincount': 1, + 'info_dict': { + 'id': 'myshows', + }, + }, + ] + + def _generate_playlist_entries(self): + next_url = 'https://content.watchnebula.com/library/video/?page_size=100' + page_num = 1 + while next_url: + channel = self._call_nebula_api(next_url, 'myshows', auth_type='bearer', + note=f'Retrieving subscriptions page {page_num}') + for episode in channel['results']: + yield self._build_video_info(episode) + next_url = channel['next'] + page_num += 1 + + def _real_extract(self, url): + return self.playlist_result(self._generate_playlist_entries(), 'myshows') + + +class NebulaChannelIE(NebulaBaseIE): + IE_NAME = 'nebula:channel' + _VALID_URL = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app)/(?!myshows|videos/)(?P[-\w]+)' _TESTS = [ { 'url': 'https://nebula.app/tom-scott-presents-money', @@ -242,9 +287,6 @@ class NebulaCollectionIE(NebulaBaseIE): 'description': 'Tom Scott hosts a series all about trust, negotiation and money.', }, 'playlist_count': 5, - 'params': { - 'usenetrc': True, - }, }, { 'url': 'https://nebula.app/lindsayellis', 'info_dict': { @@ -253,9 +295,6 @@ class NebulaCollectionIE(NebulaBaseIE): 'description': 'Enjoy these hottest of takes on Disney, Transformers, and Musicals.', }, 'playlist_mincount': 100, - 'params': { - 'usenetrc': True, - }, }, ]