From 87446dc6186c7e4247fd7f9bc1046ef41f5d1a0f Mon Sep 17 00:00:00 2001 From: Hannu Lintala Date: Sun, 7 Jun 2015 17:25:30 +0300 Subject: [PATCH 1/8] [tvc] Add extractor (Closes #5795) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/tvc.py | 79 ++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 youtube_dl/extractor/tvc.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 67eb96057..8c4e12904 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -582,6 +582,7 @@ from .tv2 import ( TV2ArticleIE, ) from .tv4 import TV4IE +from .tvc import TVCIE from .tvigle import TvigleIE from .tvp import TvpIE, TvpSeriesIE from .tvplay import TVPlayIE diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py new file mode 100644 index 000000000..b62ab857c --- /dev/null +++ b/youtube_dl/extractor/tvc.py @@ -0,0 +1,79 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + int_or_none, + str_or_none, +) + + +class TVCIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P\d+)' + _TESTS = [ + { + 'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/', + 'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba', + 'info_dict': { + 'id': '74622', + 'display_id': '39702', + 'ext': 'mp4', + 'title': 'События. "События". Эфир от 22.05.2015 14:30', + 'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 1122, + }, + }, + { + 'url': 'http://www.tvc.ru/news/show/id/69944', + 'md5': 'b173128ee7b88b5b06c84e5f7880909f', + 'info_dict': { + 'id': '75399', + 'display_id': '69944', + 'ext': 'mp4', + 'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках', + 'description': 'md5:f675c8eaf23aab9df542d31773ed6518', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 278, + }, + }, + ] + + def _real_extract(self, url): + display_id = self._match_id(url) + + webpage = self._download_webpage(url, display_id) + + video_url = self._og_search_video_url(webpage) + + video_id = self._search_regex( + r'video/iframe/id/(\d+)/', video_url, 'video id') + + video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id) + + video_json = self._download_json(video_json_url, video_id) + + formats = [] + for info in video_json.get('path', {}).get('quality', []): + format_id = self._search_regex( + r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id', + fatal=False) + formats.append({ + 'format_id': str_or_none(format_id), + 'url': info.get('url'), + 'width': int_or_none(info.get('width')), + 'height': int_or_none(info.get('height')), + 'tbr': int_or_none(info.get('bitrate')), + }) + + self._sort_formats(formats) + + return { + 'id': video_id, + 'display_id': display_id, + 'title': self._og_search_title(webpage), + 'description': self._og_search_description(webpage), + 'thumbnail': self._og_search_thumbnail(webpage), + 'duration': int_or_none(video_json.get('duration')), + 'formats': formats, + } From 9f15bdabc85add582d78a6dd57cfbb56cb33baff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Jun 2015 16:13:36 +0600 Subject: [PATCH 2/8] [tvc] Separate embed extractor --- youtube_dl/extractor/__init__.py | 5 +- youtube_dl/extractor/tvc.py | 125 ++++++++++++++++++------------- 2 files changed, 77 insertions(+), 53 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 6dc3cbff4..a8d3a8928 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -584,7 +584,10 @@ from .tv2 import ( TV2ArticleIE, ) from .tv4 import TV4IE -from .tvc import TVCIE +from .tvc import ( + TVCIE, + TVCEmbedIE, +) from .tvigle import TvigleIE from .tvp import TvpIE, TvpSeriesIE from .tvplay import TVPlayIE diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py index b62ab857c..0055f9598 100644 --- a/youtube_dl/extractor/tvc.py +++ b/youtube_dl/extractor/tvc.py @@ -3,77 +3,98 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( + clean_html, int_or_none, - str_or_none, ) -class TVCIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P\d+)' - _TESTS = [ - { - 'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/', - 'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba', - 'info_dict': { - 'id': '74622', - 'display_id': '39702', - 'ext': 'mp4', - 'title': 'События. "События". Эфир от 22.05.2015 14:30', - 'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd', - 'thumbnail': 're:^https?://.*\.jpg$', - 'duration': 1122, - }, - }, - { - 'url': 'http://www.tvc.ru/news/show/id/69944', - 'md5': 'b173128ee7b88b5b06c84e5f7880909f', - 'info_dict': { - 'id': '75399', - 'display_id': '69944', - 'ext': 'mp4', - 'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках', - 'description': 'md5:f675c8eaf23aab9df542d31773ed6518', - 'thumbnail': 're:^https?://.*\.jpg$', - 'duration': 278, - }, +class TVCEmbedIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P\d+)' + _TEST = { + 'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702', + 'md5': 'bbc5ff531d1e90e856f60fc4b3afd708', + 'info_dict': { + 'id': '74622', + 'ext': 'mp4', + 'title': 'События. "События". Эфир от 22.05.2015 14:30', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 1122, }, - ] + } def _real_extract(self, url): - display_id = self._match_id(url) - - webpage = self._download_webpage(url, display_id) - - video_url = self._og_search_video_url(webpage) - - video_id = self._search_regex( - r'video/iframe/id/(\d+)/', video_url, 'video id') + video_id = self._match_id(url) - video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id) - - video_json = self._download_json(video_json_url, video_id) + video = self._download_json( + 'http://www.tvc.ru/video/json/id/%s' % video_id, video_id) formats = [] - for info in video_json.get('path', {}).get('quality', []): + for info in video.get('path', {}).get('quality', []): + video_url = info.get('url') + if not video_url: + continue format_id = self._search_regex( - r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id', - fatal=False) + r'cdnvideo/([^/]+?)(?:-[^/]+?)?/', video_url, + 'format id', default=None) formats.append({ - 'format_id': str_or_none(format_id), - 'url': info.get('url'), + 'url': video_url, + 'format_id': format_id, 'width': int_or_none(info.get('width')), 'height': int_or_none(info.get('height')), 'tbr': int_or_none(info.get('bitrate')), }) - self._sort_formats(formats) return { 'id': video_id, - 'display_id': display_id, - 'title': self._og_search_title(webpage), - 'description': self._og_search_description(webpage), - 'thumbnail': self._og_search_thumbnail(webpage), - 'duration': int_or_none(video_json.get('duration')), + 'title': video['title'], + 'thumbnail': video.get('picture'), + 'duration': int_or_none(video.get('duration')), 'formats': formats, } + + +class TVCIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P[^?#]+)' + _TESTS = [{ + 'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/', + 'info_dict': { + 'id': '74622', + 'ext': 'mp4', + 'title': 'События. "События". Эфир от 22.05.2015 14:30', + 'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 1122, + }, + }, { + 'url': 'http://www.tvc.ru/news/show/id/69944', + 'info_dict': { + 'id': '75399', + 'ext': 'mp4', + 'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках', + 'description': 'md5:f2098f71e21f309e89f69b525fd9846e', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 278, + }, + }, { + 'url': 'http://www.tvc.ru/channel/brand/id/47/show/episodes#', + 'info_dict': { + 'id': '2185', + 'ext': 'mp4', + 'title': 'Ещё не поздно. Эфир от 03.08.2013', + 'description': 'md5:51fae9f3f8cfe67abce014e428e5b027', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 3316, + }, + }] + + def _real_extract(self, url): + webpage = self._download_webpage(url, self._match_id(url)) + return { + '_type': 'url_transparent', + 'ie_key': 'TVCEmbed', + 'url': self._og_search_video_url(webpage), + 'title': clean_html(self._og_search_title(webpage)), + 'description': clean_html(self._og_search_description(webpage)), + 'thumbnail': self._og_search_thumbnail(webpage), + } From 29902c8ec016a7128557d47a7413e82d4e022f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Jun 2015 16:22:23 +0600 Subject: [PATCH 3/8] [tvc:embed] Add embed extraction routine --- youtube_dl/extractor/tvc.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py index 0055f9598..756fec732 100644 --- a/youtube_dl/extractor/tvc.py +++ b/youtube_dl/extractor/tvc.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..utils import ( clean_html, @@ -22,6 +24,13 @@ class TVCEmbedIE(InfoExtractor): }, } + @classmethod + def _extract_url(cls, webpage): + mobj = re.search( + r']+?src=(["\'])(?P(?:http://)?(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage) + if mobj: + return mobj.group('url') + def _real_extract(self, url): video_id = self._match_id(url) From 494f20cbdca8e76e3cb452bb0feabcb855d9b4a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Jun 2015 16:22:46 +0600 Subject: [PATCH 4/8] [extractor/generic] Add support for tvc embeds --- youtube_dl/extractor/generic.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 75526384f..c797c4b52 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -34,6 +34,7 @@ from .brightcove import BrightcoveIE from .nbc import NBCSportsVPlayerIE from .ooyala import OoyalaIE from .rutv import RUTVIE +from .tvc import TVCEmbedIE from .sportbox import SportBoxEmbedIE from .smotri import SmotriIE from .condenast import CondeNastIE @@ -1301,6 +1302,11 @@ class GenericIE(InfoExtractor): if rutv_url: return self.url_result(rutv_url, 'RUTV') + # Look for embedded TVC player + rutv_url = TVCEmbedIE._extract_url(webpage) + if rutv_url: + return self.url_result(rutv_url, 'TVCEmbed') + # Look for embedded SportBox player sportbox_urls = SportBoxEmbedIE._extract_urls(webpage) if sportbox_urls: From 954c1d05299ae7c6a51db46c1ac33ddf150266c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Jun 2015 16:24:13 +0600 Subject: [PATCH 5/8] [tvc] Refactor extractor names --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/generic.py | 4 ++-- youtube_dl/extractor/tvc.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index a8d3a8928..18b1c5e54 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -586,7 +586,7 @@ from .tv2 import ( from .tv4 import TV4IE from .tvc import ( TVCIE, - TVCEmbedIE, + TVCArticleIE, ) from .tvigle import TvigleIE from .tvp import TvpIE, TvpSeriesIE diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index c797c4b52..507e4a571 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -34,7 +34,7 @@ from .brightcove import BrightcoveIE from .nbc import NBCSportsVPlayerIE from .ooyala import OoyalaIE from .rutv import RUTVIE -from .tvc import TVCEmbedIE +from .tvc import TVCIE from .sportbox import SportBoxEmbedIE from .smotri import SmotriIE from .condenast import CondeNastIE @@ -1303,7 +1303,7 @@ class GenericIE(InfoExtractor): return self.url_result(rutv_url, 'RUTV') # Look for embedded TVC player - rutv_url = TVCEmbedIE._extract_url(webpage) + rutv_url = TVCIE._extract_url(webpage) if rutv_url: return self.url_result(rutv_url, 'TVCEmbed') diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py index 756fec732..36c2a3196 100644 --- a/youtube_dl/extractor/tvc.py +++ b/youtube_dl/extractor/tvc.py @@ -10,7 +10,7 @@ from ..utils import ( ) -class TVCEmbedIE(InfoExtractor): +class TVCIE(InfoExtractor): _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P\d+)' _TEST = { 'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702', @@ -63,7 +63,7 @@ class TVCEmbedIE(InfoExtractor): } -class TVCIE(InfoExtractor): +class TVCArticleIE(InfoExtractor): _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P[^?#]+)' _TESTS = [{ 'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/', From 5ccddb7ecfb1015038f2616dd7e0da78a4365c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Jun 2015 16:25:26 +0600 Subject: [PATCH 6/8] [tvc] Fix ie_key --- youtube_dl/extractor/tvc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py index 36c2a3196..6b5d80aee 100644 --- a/youtube_dl/extractor/tvc.py +++ b/youtube_dl/extractor/tvc.py @@ -101,7 +101,7 @@ class TVCArticleIE(InfoExtractor): webpage = self._download_webpage(url, self._match_id(url)) return { '_type': 'url_transparent', - 'ie_key': 'TVCEmbed', + 'ie_key': 'TVC', 'url': self._og_search_video_url(webpage), 'title': clean_html(self._og_search_title(webpage)), 'description': clean_html(self._og_search_description(webpage)), From 2da09ff8b0de3c27a16d9096f5d28d03f44fcf70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Jun 2015 16:26:31 +0600 Subject: [PATCH 7/8] [extractor/generic] Fix tvc ie_key --- youtube_dl/extractor/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 507e4a571..66aceefb8 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -1305,7 +1305,7 @@ class GenericIE(InfoExtractor): # Look for embedded TVC player rutv_url = TVCIE._extract_url(webpage) if rutv_url: - return self.url_result(rutv_url, 'TVCEmbed') + return self.url_result(rutv_url, 'TVC') # Look for embedded SportBox player sportbox_urls = SportBoxEmbedIE._extract_urls(webpage) From f37bdbe537134f1ece0819e2aa677b1fec0c1cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Jun 2015 16:28:45 +0600 Subject: [PATCH 8/8] [extractor/generic] Add test for tvc embed --- youtube_dl/extractor/generic.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 66aceefb8..6be9e6329 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -292,6 +292,15 @@ class GenericIE(InfoExtractor): 'skip_download': True, }, }, + # TVC embed + { + 'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/', + 'info_dict': { + 'id': '55304', + 'ext': 'mp4', + 'title': 'Дошкольное воспитание', + }, + }, # SportBox embed { 'url': 'http://www.vestifinance.ru/articles/25753',