From 0dac7cbb092c804f1548c4a60f15ac29a7db06b9 Mon Sep 17 00:00:00 2001 From: Remita Amine Date: Sun, 12 Feb 2017 17:24:45 +0100 Subject: [hotstar] improve extraction(closes #12096) - extract all qualities - detect drm protected videos - extract more metadata --- youtube_dl/extractor/hotstar.py | 46 ++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 14 deletions(-) (limited to 'youtube_dl/extractor/hotstar.py') diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py index f05d765d6..3a7a66a34 100644 --- a/youtube_dl/extractor/hotstar.py +++ b/youtube_dl/extractor/hotstar.py @@ -34,11 +34,9 @@ class HotStarIE(InfoExtractor): 'only_matching': True, }] - _GET_CONTENT_TEMPLATE = 'http://account.hotstar.com/AVS/besc?action=GetAggregatedContentDetails&channel=PCTV&contentId=%s' - _GET_CDN_TEMPLATE = 'http://getcdn.hotstar.com/AVS/besc?action=GetCDN&asJson=Y&channel=%s&id=%s&type=%s' - - def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True): - json_data = super(HotStarIE, self)._download_json(url_or_request, video_id, note, fatal=fatal) + def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True, query=None): + json_data = super(HotStarIE, self)._download_json( + url_or_request, video_id, note, fatal=fatal, query=query) if json_data['resultCode'] != 'OK': if fatal: raise ExtractorError(json_data['errorDescription']) @@ -48,20 +46,37 @@ class HotStarIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) video_data = self._download_json( - self._GET_CONTENT_TEMPLATE % video_id, - video_id)['contentInfo'][0] + 'http://account.hotstar.com/AVS/besc', video_id, query={ + 'action': 'GetAggregatedContentDetails', + 'channel': 'PCTV', + 'contentId': video_id, + })['contentInfo'][0] + title = video_data['episodeTitle'] + + if video_data.get('encrypted') == 'Y': + raise ExtractorError('This video is DRM protected.', expected=True) formats = [] - # PCTV for extracting f4m manifest - for f in ('TABLET',): + for f in ('JIO',): format_data = self._download_json( - self._GET_CDN_TEMPLATE % (f, video_id, 'VOD'), - video_id, 'Downloading %s JSON metadata' % f, fatal=False) + 'http://getcdn.hotstar.com/AVS/besc', + video_id, 'Downloading %s JSON metadata' % f, + fatal=False, query={ + 'action': 'GetCDN', + 'asJson': 'Y', + 'channel': f, + 'id': video_id, + 'type': 'VOD', + }) if format_data: - format_url = format_data['src'] + format_url = format_data.get('src') + if not format_url: + continue ext = determine_ext(format_url) if ext == 'm3u8': - formats.extend(self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)) + formats.extend(self._extract_m3u8_formats( + format_url, video_id, 'mp4', + m3u8_id='hls', fatal=False)) elif ext == 'f4m': # produce broken files continue @@ -75,9 +90,12 @@ class HotStarIE(InfoExtractor): return { 'id': video_id, - 'title': video_data['episodeTitle'], + 'title': title, 'description': video_data.get('description'), 'duration': int_or_none(video_data.get('duration')), 'timestamp': int_or_none(video_data.get('broadcastDate')), 'formats': formats, + 'episode': title, + 'episode_number': int_or_none(video_data.get('episodeNumber')), + 'series': video_data.get('contentTitle'), } -- cgit 1.4.1 From 6e71bbf4abc729cae3b0e428c3bb321690c9e485 Mon Sep 17 00:00:00 2001 From: Sergey M․ Date: Sun, 5 Nov 2017 16:12:56 +0700 Subject: [hotstar] Bypass geo restriction (closes #14672) --- youtube_dl/extractor/hotstar.py | 1 + 1 file changed, 1 insertion(+) (limited to 'youtube_dl/extractor/hotstar.py') diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py index 3a7a66a34..9be958be6 100644 --- a/youtube_dl/extractor/hotstar.py +++ b/youtube_dl/extractor/hotstar.py @@ -11,6 +11,7 @@ from ..utils import ( class HotStarIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P\d{10})' + _GEO_COUNTRIES = ['IN'] _TESTS = [{ 'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273', 'info_dict': { -- cgit 1.4.1 From 477c97f86b5451f384a84a7a8d8237cfd1bec1d2 Mon Sep 17 00:00:00 2001 From: Alpesh Valia Date: Thu, 16 Mar 2017 22:00:11 +0530 Subject: [hotstar:playlist] Add extractor --- youtube_dl/extractor/extractors.py | 5 +++- youtube_dl/extractor/hotstar.py | 58 +++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) (limited to 'youtube_dl/extractor/hotstar.py') diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 92f7e9027..d084707ee 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -432,7 +432,10 @@ from .hitbox import HitboxIE, HitboxLiveIE from .hitrecord import HitRecordIE from .hornbunny import HornBunnyIE from .hotnewhiphop import HotNewHipHopIE -from .hotstar import HotStarIE +from .hotstar import ( + HotStarIE, + HotStarPlaylistIE, +) from .howcast import HowcastIE from .howstuffworks import HowStuffWorksIE from .hrti import ( diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py index 9be958be6..8d8a80a82 100644 --- a/youtube_dl/extractor/hotstar.py +++ b/youtube_dl/extractor/hotstar.py @@ -7,6 +7,7 @@ from ..utils import ( determine_ext, int_or_none, ) +import re class HotStarIE(InfoExtractor): @@ -17,7 +18,7 @@ class HotStarIE(InfoExtractor): 'info_dict': { 'id': '1000076273', 'ext': 'mp4', - 'title': 'On Air With AIB - English', + 'title': 'On Air With AIB', 'description': 'md5:c957d8868e9bc793ccb813691cc4c434', 'timestamp': 1447227000, 'upload_date': '20151111', @@ -100,3 +101,58 @@ class HotStarIE(InfoExtractor): 'episode_number': int_or_none(video_data.get('episodeNumber')), 'series': video_data.get('contentTitle'), } + + +class HotStarPlaylistIE(InfoExtractor): + IE_NAME = 'hotstar:playlist' + _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/(?P.+)/(?P\d+)/episodes/(?P\d{1,})' + + _TESTS = [{ + 'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993', + 'info_dict': { + 'id': '10856', + 'title': 'pow-bandi-yuddh-ke', + }, + 'playlist_mincount': 0, + }, { + 'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993', + 'only_matching': True, + }] + + def _extract_episode_info(self, series_id, playlist_title, video): + + picture_url = video.get('urlPictures') + thumbnail = '' + if picture_url: + thumbnail = 'http://media0-starag.startv.in/r1/thumbs/PCTV/%s/%s/PCTV-%s-hs.jpg' % (picture_url[-2:], picture_url, picture_url) + + episode_title = video.get('episodeTitle', '') + episode_title = episode_title.lower().replace(' ', '-') + url = "http://www.hotstar.com/tv/%s/%s/%s/%s" % (playlist_title, series_id, episode_title, video.get('contentId')) + + info_dict = { + 'id': video.get('contentId'), + 'title': video.get('episodeTitle'), + 'description': video.get('longDescription'), + 'thumbnail': thumbnail, + 'url': url, + '_type': 'url', + } + return info_dict + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + series_id = mobj.group('series_id') + playlist_id = mobj.group('playlist_id') + playlist_title = mobj.group('playlist_title') + + collection = self._download_json( + "http://search.hotstar.com/AVS/besc?action=SearchContents&appVersion=5.0.39&channel=PCTV&moreFilters=series:%s;&query=*&searchOrder=last_broadcast_date+desc,year+asc,title+asc&type=EPISODE" % playlist_id, + playlist_id + ) + + videos = collection.get('resultObj', {}).get('response', {}).get('docs', []) + entries = [ + self._extract_episode_info(series_id, playlist_title, video) + for video in videos if video.get('contentId')] + return self.playlist_result(entries, playlist_id, playlist_title) -- cgit 1.4.1 From 909191de9154bf289b333cfe01b8e88e3ac1fefc Mon Sep 17 00:00:00 2001 From: Sergey M․ Date: Sun, 5 Nov 2017 19:14:48 +0700 Subject: [hotstar:playlist] Fix issues and improve (closes #12465) --- youtube_dl/extractor/hotstar.py | 120 +++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 58 deletions(-) (limited to 'youtube_dl/extractor/hotstar.py') diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py index 8d8a80a82..d28af36ec 100644 --- a/youtube_dl/extractor/hotstar.py +++ b/youtube_dl/extractor/hotstar.py @@ -1,18 +1,41 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor +from ..compat import compat_str from ..utils import ( - ExtractorError, determine_ext, + ExtractorError, int_or_none, ) -import re -class HotStarIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P\d{10})' +class HotStarBaseIE(InfoExtractor): _GEO_COUNTRIES = ['IN'] + + def _download_json(self, *args, **kwargs): + response = super(HotStarBaseIE, self)._download_json(*args, **kwargs) + if response['resultCode'] != 'OK': + if kwargs.get('fatal'): + raise ExtractorError( + response['errorDescription'], expected=True) + return None + return response['resultObj'] + + def _download_content_info(self, content_id): + return self._download_json( + 'https://account.hotstar.com/AVS/besc', content_id, query={ + 'action': 'GetAggregatedContentDetails', + 'appVersion': '5.0.40', + 'channel': 'PCTV', + 'contentId': content_id, + })['contentInfo'][0] + + +class HotStarIE(HotStarBaseIE): + _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P\d{10})' _TESTS = [{ 'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273', 'info_dict': { @@ -36,23 +59,11 @@ class HotStarIE(InfoExtractor): 'only_matching': True, }] - def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True, query=None): - json_data = super(HotStarIE, self)._download_json( - url_or_request, video_id, note, fatal=fatal, query=query) - if json_data['resultCode'] != 'OK': - if fatal: - raise ExtractorError(json_data['errorDescription']) - return None - return json_data['resultObj'] - def _real_extract(self, url): video_id = self._match_id(url) - video_data = self._download_json( - 'http://account.hotstar.com/AVS/besc', video_id, query={ - 'action': 'GetAggregatedContentDetails', - 'channel': 'PCTV', - 'contentId': video_id, - })['contentInfo'][0] + + video_data = self._download_content_info(video_id) + title = video_data['episodeTitle'] if video_data.get('encrypted') == 'Y': @@ -103,56 +114,49 @@ class HotStarIE(InfoExtractor): } -class HotStarPlaylistIE(InfoExtractor): +class HotStarPlaylistIE(HotStarBaseIE): IE_NAME = 'hotstar:playlist' - _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/(?P.+)/(?P\d+)/episodes/(?P\d{1,})' - + _VALID_URL = r'(?Phttps?://(?:www\.)?hotstar\.com/tv/[^/]+/(?P\d+))/(?P[^/]+)/(?P\d+)' _TESTS = [{ - 'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993', + 'url': 'http://www.hotstar.com/tv/pratidaan/14982/episodes/14812/9993', 'info_dict': { - 'id': '10856', - 'title': 'pow-bandi-yuddh-ke', + 'id': '14812', }, - 'playlist_mincount': 0, + 'playlist_mincount': 75, }, { - 'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993', + 'url': 'http://www.hotstar.com/tv/pratidaan/14982/popular-clips/9998/9998', 'only_matching': True, }] - - def _extract_episode_info(self, series_id, playlist_title, video): - - picture_url = video.get('urlPictures') - thumbnail = '' - if picture_url: - thumbnail = 'http://media0-starag.startv.in/r1/thumbs/PCTV/%s/%s/PCTV-%s-hs.jpg' % (picture_url[-2:], picture_url, picture_url) - - episode_title = video.get('episodeTitle', '') - episode_title = episode_title.lower().replace(' ', '-') - url = "http://www.hotstar.com/tv/%s/%s/%s/%s" % (playlist_title, series_id, episode_title, video.get('contentId')) - - info_dict = { - 'id': video.get('contentId'), - 'title': video.get('episodeTitle'), - 'description': video.get('longDescription'), - 'thumbnail': thumbnail, - 'url': url, - '_type': 'url', - } - return info_dict + _ITEM_TYPES = { + 'episodes': 'EPISODE', + 'popular-clips': 'CLIPS', + } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - series_id = mobj.group('series_id') - playlist_id = mobj.group('playlist_id') - playlist_title = mobj.group('playlist_title') + base_url = mobj.group('url') + content_id = mobj.group('content_id') + playlist_type = mobj.group('type') + + content_info = self._download_content_info(content_id) + playlist_id = compat_str(content_info['categoryId']) collection = self._download_json( - "http://search.hotstar.com/AVS/besc?action=SearchContents&appVersion=5.0.39&channel=PCTV&moreFilters=series:%s;&query=*&searchOrder=last_broadcast_date+desc,year+asc,title+asc&type=EPISODE" % playlist_id, - playlist_id - ) + 'https://search.hotstar.com/AVS/besc', playlist_id, query={ + 'action': 'SearchContents', + 'appVersion': '5.0.40', + 'channel': 'PCTV', + 'moreFilters': 'series:%s;' % playlist_id, + 'query': '*', + 'searchOrder': 'last_broadcast_date desc,year desc,title asc', + 'type': self._ITEM_TYPES.get(playlist_type, 'EPISODE'), + }) - videos = collection.get('resultObj', {}).get('response', {}).get('docs', []) entries = [ - self._extract_episode_info(series_id, playlist_title, video) - for video in videos if video.get('contentId')] - return self.playlist_result(entries, playlist_id, playlist_title) + self.url_result( + '%s/_/%s' % (base_url, video['contentId']), + ie=HotStarIE.ie_key(), video_id=video['contentId']) + for video in collection['response']['docs'] + if video.get('contentId')] + + return self.playlist_result(entries, playlist_id) -- cgit 1.4.1