From 3266d08af29bbd6078aca172741458ddee180ab9 Mon Sep 17 00:00:00 2001 From: John Hawkinson Date: Sat, 25 Mar 2017 19:47:48 -0400 Subject: [wsj:article] Add extractor --- youtube_dl/extractor/wsj.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'youtube_dl/extractor/wsj.py') diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py index deb7483ae..ec38a2ad8 100644 --- a/youtube_dl/extractor/wsj.py +++ b/youtube_dl/extractor/wsj.py @@ -10,10 +10,11 @@ from ..utils import ( class WSJIE(InfoExtractor): - _VALID_URL = r'''(?x)https?:// + _VALID_URL = r'''(?x) (?: - video-api\.wsj\.com/api-video/player/iframe\.html\?guid=| - (?:www\.)?wsj\.com/video/[^/]+/ + https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=| + https?://(?:www\.)?wsj\.com/video/[^/]+/| + wsj: ) (?P[a-zA-Z0-9-]+)''' IE_DESC = 'Wall Street Journal' @@ -87,3 +88,24 @@ class WSJIE(InfoExtractor): 'title': title, 'categories': info.get('keywords'), } + + +class WSJArticleIE(InfoExtractor): + _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P\w[^/]+)' + _TESTS = [{ + 'url': 'https://www.wsj.com/articles/dont-like-china-no-pandas-for-you-1490366939?', + 'info_dict': { + 'id': '4B13FA62-1D8C-45DB-8EA1-4105CB20B362', + 'ext': 'mp4', + 'upload_date': '20170221', + 'uploader_id': 'ralcaraz', + 'title': 'Bao Bao the Panda Leaves for China', + } + }] + + def _real_extract(self, url): + article_id = self._match_id(url) + webpage = self._download_webpage(url, article_id) + video_id = self._search_regex(r'data-src=["\']([A-Z0-9\-]+)', + webpage, 'video id') + return self.url_result('wsj:%s' % video_id, WSJIE.ie_key(), video_id) -- cgit 1.4.1 From b2a19e38293206a4ff687315baf0369c205bcd6b Mon Sep 17 00:00:00 2001 From: Sergey M․ Date: Sat, 15 Apr 2017 20:51:47 +0700 Subject: [wsj] Improve and modernize (closes #12558) --- youtube_dl/extractor/wsj.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) (limited to 'youtube_dl/extractor/wsj.py') diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py index ec38a2ad8..45cfca7c5 100644 --- a/youtube_dl/extractor/wsj.py +++ b/youtube_dl/extractor/wsj.py @@ -11,12 +11,13 @@ from ..utils import ( class WSJIE(InfoExtractor): _VALID_URL = r'''(?x) - (?: - https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=| - https?://(?:www\.)?wsj\.com/video/[^/]+/| - wsj: - ) - (?P[a-zA-Z0-9-]+)''' + (?: + https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=| + https?://(?:www\.)?wsj\.com/video/[^/]+/| + wsj: + ) + (?P[a-fA-F0-9-]{36}) + ''' IE_DESC = 'Wall Street Journal' _TESTS = [{ 'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A', @@ -39,12 +40,17 @@ class WSJIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - api_url = ( - 'http://video-api.wsj.com/api-video/find_all_videos.asp?' - 'type=guid&count=1&query=%s&fields=type,hls,videoMP4List,' - 'thumbnailList,author,description,name,duration,videoURL,' - 'titletag,formattedCreationDate,keywords,editor' % video_id) - info = self._download_json(api_url, video_id)['items'][0] + info = self._download_json( + 'http://video-api.wsj.com/api-video/find_all_videos.asp', video_id, + query={ + 'type': 'guid', + 'count': 1, + 'query': video_id, + 'fields': ','.join(( + 'type', 'hls', 'videoMP4List', 'thumbnailList', 'author', + 'description', 'name', 'duration', 'videoURL', 'titletag', + 'formattedCreationDate', 'keywords', 'editor')), + })['items'][0] title = info.get('name', info.get('titletag')) formats = [] @@ -91,8 +97,8 @@ class WSJIE(InfoExtractor): class WSJArticleIE(InfoExtractor): - _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P\w[^/]+)' - _TESTS = [{ + _VALID_URL = r'(?i)https?://(?:www\.)?wsj\.com/articles/(?P[^/?#&]+)' + _TEST = { 'url': 'https://www.wsj.com/articles/dont-like-china-no-pandas-for-you-1490366939?', 'info_dict': { 'id': '4B13FA62-1D8C-45DB-8EA1-4105CB20B362', @@ -101,11 +107,11 @@ class WSJArticleIE(InfoExtractor): 'uploader_id': 'ralcaraz', 'title': 'Bao Bao the Panda Leaves for China', } - }] + } def _real_extract(self, url): article_id = self._match_id(url) webpage = self._download_webpage(url, article_id) - video_id = self._search_regex(r'data-src=["\']([A-Z0-9\-]+)', - webpage, 'video id') + video_id = self._search_regex( + r'data-src=["\']([a-fA-F0-9-]{36})', webpage, 'video id') return self.url_result('wsj:%s' % video_id, WSJIE.ie_key(), video_id) -- cgit 1.4.1 From fd9ee4de8c3aa640d51c5ff42f5d8241970e894d Mon Sep 17 00:00:00 2001 From: Sergey M․ Date: Sun, 25 Jun 2017 02:14:10 +0700 Subject: [wsj] Add support for barrons.com (closes #13470) --- youtube_dl/extractor/wsj.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'youtube_dl/extractor/wsj.py') diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py index 45cfca7c5..9b5487710 100644 --- a/youtube_dl/extractor/wsj.py +++ b/youtube_dl/extractor/wsj.py @@ -13,7 +13,7 @@ class WSJIE(InfoExtractor): _VALID_URL = r'''(?x) (?: https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=| - https?://(?:www\.)?wsj\.com/video/[^/]+/| + https?://(?:www\.)?(?:wsj|barrons)\.com/video/[^/]+/| wsj: ) (?P[a-fA-F0-9-]{36}) @@ -35,6 +35,9 @@ class WSJIE(InfoExtractor): }, { 'url': 'http://www.wsj.com/video/can-alphabet-build-a-smarter-city/359DDAA8-9AC1-489C-82E6-0429C1E430E0.html', 'only_matching': True, + }, { + 'url': 'http://www.barrons.com/video/capitalism-deserves-more-respect-from-millennials/F301217E-6F46-43AE-B8D2-B7180D642EE9.html', + 'only_matching': True, }] def _real_extract(self, url): -- cgit 1.4.1 From a9543e37c8e460e69a8556c8e5004ebd8e9b4da4 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 11 Nov 2017 00:29:08 +0800 Subject: [wsj] Recognize another URL pattern (closes #14704) --- ChangeLog | 6 ++++++ youtube_dl/extractor/wsj.py | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'youtube_dl/extractor/wsj.py') diff --git a/ChangeLog b/ChangeLog index 8af368274..cedab4723 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +version + +Extractors ++ [wsj] Recognize another URL pattern (#14704) + + version 2017.11.06 Core diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py index 9b5487710..67236f377 100644 --- a/youtube_dl/extractor/wsj.py +++ b/youtube_dl/extractor/wsj.py @@ -13,7 +13,7 @@ class WSJIE(InfoExtractor): _VALID_URL = r'''(?x) (?: https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=| - https?://(?:www\.)?(?:wsj|barrons)\.com/video/[^/]+/| + https?://(?:www\.)?(?:wsj|barrons)\.com/video/(?:[^/]+/)+| wsj: ) (?P[a-fA-F0-9-]{36}) @@ -38,6 +38,9 @@ class WSJIE(InfoExtractor): }, { 'url': 'http://www.barrons.com/video/capitalism-deserves-more-respect-from-millennials/F301217E-6F46-43AE-B8D2-B7180D642EE9.html', 'only_matching': True, + }, { + 'url': 'https://www.wsj.com/video/series/a-brief-history-of/the-modern-cell-carrier-how-we-got-here/980E2187-401D-48A1-B82B-1486CEE06CB9', + 'only_matching': True, }] def _real_extract(self, url): -- cgit 1.4.1