diff options
Diffstat (limited to 'youtube_dl/extractor/youku.py')
| -rw-r--r-- | youtube_dl/extractor/youku.py | 385 |
1 files changed, 180 insertions, 205 deletions
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py index e37f237c7..5b0b248cd 100644 --- a/youtube_dl/extractor/youku.py +++ b/youtube_dl/extractor/youku.py @@ -1,21 +1,18 @@ # coding: utf-8 from __future__ import unicode_literals -import base64 -import itertools import random import re import string import time from .common import InfoExtractor -from ..compat import ( - compat_urllib_parse_urlencode, - compat_ord, -) from ..utils import ( ExtractorError, - get_element_by_attribute, + get_element_by_class, + js_to_json, + str_or_none, + strip_jsonp, ) @@ -24,7 +21,9 @@ class YoukuIE(InfoExtractor): IE_DESC = '优酷' _VALID_URL = r'''(?x) (?: - http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)| + https?://( + (?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)| + video\.tudou\.com/v/)| youku:) (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|) ''' @@ -33,9 +32,15 @@ class YoukuIE(InfoExtractor): # MD5 is unstable 'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html', 'info_dict': { - 'id': 'XMTc1ODE5Njcy_part1', + 'id': 'XMTc1ODE5Njcy', 'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.', - 'ext': 'flv' + 'ext': 'mp4', + 'duration': 74.73, + 'thumbnail': r're:^https?://.*', + 'uploader': '。躲猫猫、', + 'uploader_id': '36017967', + 'uploader_url': 'http://i.youku.com/u/UMTQ0MDcxODY4', + 'tags': list, } }, { 'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf', @@ -44,25 +49,42 @@ class YoukuIE(InfoExtractor): 'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html', 'info_dict': { 'id': 'XODgxNjg1Mzk2', + 'ext': 'mp4', 'title': '武媚娘传奇 85', + 'duration': 1999.61, + 'thumbnail': r're:^https?://.*', + 'uploader': '疯狂豆花', + 'uploader_id': '62583473', + 'uploader_url': 'http://i.youku.com/u/UMjUwMzMzODky', + 'tags': list, }, - 'playlist_count': 11, - 'skip': 'Available in China only', }, { 'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html', 'info_dict': { 'id': 'XMTI1OTczNDM5Mg', + 'ext': 'mp4', 'title': '花千骨 04', + 'duration': 2363, + 'thumbnail': r're:^https?://.*', + 'uploader': '放剧场-花千骨', + 'uploader_id': '772849359', + 'uploader_url': 'http://i.youku.com/u/UMzA5MTM5NzQzNg==', + 'tags': list, }, - 'playlist_count': 13, }, { 'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html', 'note': 'Video protected with password', 'info_dict': { 'id': 'XNjA1NzA2Njgw', + 'ext': 'mp4', 'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起', + 'duration': 7264.5, + 'thumbnail': r're:^https?://.*', + 'uploader': 'FoxJin1006', + 'uploader_id': '322014285', + 'uploader_url': 'http://i.youku.com/u/UMTI4ODA1NzE0MA==', + 'tags': list, }, - 'playlist_count': 19, 'params': { 'videopassword': '100600', }, @@ -71,128 +93,38 @@ class YoukuIE(InfoExtractor): 'url': 'http://v.youku.com/v_show/id_XOTUxMzg4NDMy.html', 'info_dict': { 'id': 'XOTUxMzg4NDMy', + 'ext': 'mp4', 'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft', + 'duration': 702.08, + 'thumbnail': r're:^https?://.*', + 'uploader': '明月庄主moon', + 'uploader_id': '38465621', + 'uploader_url': 'http://i.youku.com/u/UMTUzODYyNDg0', + 'tags': list, + }, + }, { + 'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805', + 'info_dict': { + 'id': 'XMjIyNzAzMTQ4NA', + 'ext': 'mp4', + 'title': '卡马乔国足开大脚长传冲吊集锦', + 'duration': 289, + 'thumbnail': r're:^https?://.*', + 'uploader': '阿卜杜拉之星', + 'uploader_id': '2382249', + 'uploader_url': 'http://i.youku.com/u/UOTUyODk5Ng==', + 'tags': list, }, - 'playlist_count': 6, + }, { + 'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html', + 'only_matching': True, }] - def construct_video_urls(self, data): - # get sid, token - def yk_t(s1, s2): - ls = list(range(256)) - t = 0 - for i in range(256): - t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256 - ls[i], ls[t] = ls[t], ls[i] - s = bytearray() - x, y = 0, 0 - for i in range(len(s2)): - y = (y + 1) % 256 - x = (x + ls[y]) % 256 - ls[x], ls[y] = ls[y], ls[x] - s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256]) - return bytes(s) - - sid, token = yk_t( - b'becaf9be', base64.b64decode(data['security']['encrypt_string'].encode('ascii')) - ).decode('ascii').split('_') - - # get oip - oip = data['security']['ip'] - - fileid_dict = {} - for stream in data['stream']: - if stream.get('channel_type') == 'tail': - continue - format = stream.get('stream_type') - fileid = stream['stream_fileid'] - fileid_dict[format] = fileid - - def get_fileid(format, n): - number = hex(int(str(n), 10))[2:].upper() - if len(number) == 1: - number = '0' + number - streamfileids = fileid_dict[format] - fileid = streamfileids[0:8] + number + streamfileids[10:] - return fileid - - # get ep - def generate_ep(format, n): - fileid = get_fileid(format, n) - ep_t = yk_t( - b'bf7e5f01', - ('%s_%s_%s' % (sid, fileid, token)).encode('ascii') - ) - ep = base64.b64encode(ep_t).decode('ascii') - return ep - - # generate video_urls - video_urls_dict = {} - for stream in data['stream']: - if stream.get('channel_type') == 'tail': - continue - format = stream.get('stream_type') - video_urls = [] - for dt in stream['segs']: - n = str(stream['segs'].index(dt)) - param = { - 'K': dt['key'], - 'hd': self.get_hd(format), - 'myp': 0, - 'ypp': 0, - 'ctype': 12, - 'ev': 1, - 'token': token, - 'oip': oip, - 'ep': generate_ep(format, n) - } - video_url = \ - 'http://k.youku.com/player/getFlvPath/' + \ - 'sid/' + sid + \ - '_00' + \ - '/st/' + self.parse_ext_l(format) + \ - '/fileid/' + get_fileid(format, n) + '?' + \ - compat_urllib_parse_urlencode(param) - video_urls.append(video_url) - video_urls_dict[format] = video_urls - - return video_urls_dict - @staticmethod def get_ysuid(): return '%d%s' % (int(time.time()), ''.join([ random.choice(string.ascii_letters) for i in range(3)])) - def get_hd(self, fm): - hd_id_dict = { - '3gp': '0', - '3gphd': '1', - 'flv': '0', - 'flvhd': '0', - 'mp4': '1', - 'mp4hd': '1', - 'mp4hd2': '1', - 'mp4hd3': '1', - 'hd2': '2', - 'hd3': '3', - } - return hd_id_dict[fm] - - def parse_ext_l(self, fm): - ext_dict = { - '3gp': 'flv', - '3gphd': 'mp4', - 'flv': 'flv', - 'flvhd': 'flv', - 'mp4': 'mp4', - 'mp4hd': 'mp4', - 'mp4hd2': 'flv', - 'mp4hd3': 'flv', - 'hd2': 'flv', - 'hd3': 'flv', - } - return ext_dict[fm] - def get_format_name(self, fm): _dict = { '3gp': 'h6', @@ -206,32 +138,40 @@ class YoukuIE(InfoExtractor): 'hd2': 'h2', 'hd3': 'h1', } - return _dict[fm] + return _dict.get(fm) def _real_extract(self, url): video_id = self._match_id(url) self._set_cookie('youku.com', '__ysuid', self.get_ysuid()) + self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com') - def retrieve_data(req_url, note): - headers = { - 'Referer': req_url, - } - headers.update(self.geo_verification_headers()) - self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com') - - raw_data = self._download_json(req_url, video_id, note=note, headers=headers) + _, urlh = self._download_webpage_handle( + 'https://log.mmstat.com/eg.js', video_id, 'Retrieving cna info') + # The etag header is '"foobar"'; let's remove the double quotes + cna = urlh.headers['etag'][1:-1] - return raw_data['data'] + # request basic data + basic_data_params = { + 'vid': video_id, + 'ccode': '0507', + 'client_ip': '192.168.1.1', + 'utid': cna, + 'client_ts': time.time() / 1000, + } video_password = self._downloader.params.get('videopassword') - - # request basic data - basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id if video_password: - basic_data_url += '&pwd=%s' % video_password + basic_data_params['password'] = video_password - data = retrieve_data(basic_data_url, 'Downloading JSON metadata') + headers = { + 'Referer': url, + } + headers.update(self.geo_verification_headers()) + data = self._download_json( + 'https://ups.youku.com/ups/get.json', video_id, + 'Downloading JSON metadata', + query=basic_data_params, headers=headers)['data'] error = data.get('error') if error: @@ -249,86 +189,121 @@ class YoukuIE(InfoExtractor): raise ExtractorError(msg) # get video title - title = data['video']['title'] - - # generate video_urls_dict - video_urls_dict = self.construct_video_urls(data) - - # construct info - entries = [{ - 'id': '%s_part%d' % (video_id, i + 1), - 'title': title, - 'formats': [], - # some formats are not available for all parts, we have to detect - # which one has all - } for i in range(max(len(v.get('segs')) for v in data['stream']))] - for stream in data['stream']: - if stream.get('channel_type') == 'tail': - continue - fm = stream.get('stream_type') - video_urls = video_urls_dict[fm] - for video_url, seg, entry in zip(video_urls, stream['segs'], entries): - entry['formats'].append({ - 'url': video_url, - 'format_id': self.get_format_name(fm), - 'ext': self.parse_ext_l(fm), - 'filesize': int(seg['size']), - 'width': stream.get('width'), - 'height': stream.get('height'), - }) + video_data = data['video'] + title = video_data['title'] + + formats = [{ + 'url': stream['m3u8_url'], + 'format_id': self.get_format_name(stream.get('stream_type')), + 'ext': 'mp4', + 'protocol': 'm3u8_native', + 'filesize': int(stream.get('size')), + 'width': stream.get('width'), + 'height': stream.get('height'), + } for stream in data['stream'] if stream.get('channel_type') != 'tail'] + self._sort_formats(formats) return { - '_type': 'multi_video', 'id': video_id, 'title': title, - 'entries': entries, + 'formats': formats, + 'duration': video_data.get('seconds'), + 'thumbnail': video_data.get('logo'), + 'uploader': video_data.get('username'), + 'uploader_id': str_or_none(video_data.get('userid')), + 'uploader_url': data.get('uploader', {}).get('homepage'), + 'tags': video_data.get('tags'), } class YoukuShowIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?youku\.com/show_page/id_(?P<id>[0-9a-z]+)\.html' + _VALID_URL = r'https?://list\.youku\.com/show/id_(?P<id>[0-9a-z]+)\.html' IE_NAME = 'youku:show' - _TEST = { - 'url': 'http://www.youku.com/show_page/id_zc7c670be07ff11e48b3f.html', + _TESTS = [{ + 'url': 'http://list.youku.com/show/id_zc7c670be07ff11e48b3f.html', 'info_dict': { 'id': 'zc7c670be07ff11e48b3f', - 'title': '花千骨 未删减版', - 'description': 'md5:578d4f2145ae3f9128d9d4d863312910', + 'title': '花千骨 DVD版', + 'description': 'md5:a1ae6f5618571bbeb5c9821f9c81b558', }, 'playlist_count': 50, - } - - _PAGE_SIZE = 40 + }, { + # Episode number not starting from 1 + 'url': 'http://list.youku.com/show/id_zefbfbd70efbfbd780bef.html', + 'info_dict': { + 'id': 'zefbfbd70efbfbd780bef', + 'title': '超级飞侠3', + 'description': 'md5:275715156abebe5ccc2a1992e9d56b98', + }, + 'playlist_count': 24, + }, { + # Ongoing playlist. The initial page is the last one + 'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html', + 'only_matching': True, + }, { + # No data-id value. + 'url': 'http://list.youku.com/show/id_zefbfbd61237fefbfbdef.html', + 'only_matching': True, + }, { + # Wrong number of reload_id. + 'url': 'http://list.youku.com/show/id_z20eb4acaf5c211e3b2ad.html', + 'only_matching': True, + }] - def _find_videos_in_page(self, webpage): - videos = re.findall( - r'<li><a[^>]+href="(?P<url>https?://v\.youku\.com/[^"]+)"[^>]+title="(?P<title>[^"]+)"', webpage) - return [ - self.url_result(video_url, YoukuIE.ie_key(), title) - for video_url, title in videos] + def _extract_entries(self, playlist_data_url, show_id, note, query): + query['callback'] = 'cb' + playlist_data = self._download_json( + playlist_data_url, show_id, query=query, note=note, + transform_source=lambda s: js_to_json(strip_jsonp(s))).get('html') + if playlist_data is None: + return [None, None] + drama_list = (get_element_by_class('p-drama-grid', playlist_data) or + get_element_by_class('p-drama-half-row', playlist_data)) + if drama_list is None: + raise ExtractorError('No episodes found') + video_urls = re.findall(r'<a[^>]+href="([^"]+)"', drama_list) + return playlist_data, [ + self.url_result(self._proto_relative_url(video_url, 'http:'), YoukuIE.ie_key()) + for video_url in video_urls] def _real_extract(self, url): show_id = self._match_id(url) webpage = self._download_webpage(url, show_id) - entries = self._find_videos_in_page(webpage) - - playlist_title = self._html_search_regex( - r'<span[^>]+class="name">([^<]+)</span>', webpage, 'playlist title', fatal=False) - detail_div = get_element_by_attribute('class', 'detail', webpage) or '' - playlist_description = self._html_search_regex( - r'<span[^>]+style="display:none"[^>]*>([^<]+)</span>', - detail_div, 'playlist description', fatal=False) - - for idx in itertools.count(1): - episodes_page = self._download_webpage( - 'http://www.youku.com/show_episode/id_%s.html' % show_id, - show_id, query={'divid': 'reload_%d' % (idx * self._PAGE_SIZE + 1)}, - note='Downloading episodes page %d' % idx) - new_entries = self._find_videos_in_page(episodes_page) - entries.extend(new_entries) - if len(new_entries) < self._PAGE_SIZE: - break - - return self.playlist_result(entries, show_id, playlist_title, playlist_description) + entries = [] + page_config = self._parse_json(self._search_regex( + r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'), + show_id, transform_source=js_to_json) + first_page, initial_entries = self._extract_entries( + 'http://list.youku.com/show/module', show_id, + note='Downloading initial playlist data page', + query={ + 'id': page_config['showid'], + 'tab': 'showInfo', + }) + first_page_reload_id = self._html_search_regex( + r'<div[^>]+id="(reload_\d+)', first_page, 'first page reload id') + # The first reload_id has the same items as first_page + reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page) + entries.extend(initial_entries) + for idx, reload_id in enumerate(reload_ids): + if reload_id == first_page_reload_id: + continue + _, new_entries = self._extract_entries( + 'http://list.youku.com/show/episode', show_id, + note='Downloading playlist data page %d' % (idx + 1), + query={ + 'id': page_config['showid'], + 'stage': reload_id, + }) + if new_entries is not None: + entries.extend(new_entries) + desc = self._html_search_meta('description', webpage, fatal=False) + playlist_title = desc.split(',')[0] if desc else None + detail_li = get_element_by_class('p-intro', webpage) + playlist_description = get_element_by_class( + 'intro-more', detail_li) if detail_li else None + + return self.playlist_result( + entries, show_id, playlist_title, playlist_description) |