Browse Source
[frontendmasters] Fix issues and improve extraction (closes #3661, closes #16328)
master
[frontendmasters] Fix issues and improve extraction (closes #3661, closes #16328)
master
Sergey M․
6 years ago
No known key found for this signature in database
GPG Key ID: 2C393E0F18A9236D
3 changed files with 266 additions and 274 deletions
Split View
Diff Options
-
7youtube_dl/extractor/extractors.py
-
271youtube_dl/extractor/frontendmaster.py
-
262youtube_dl/extractor/frontendmasters.py
@ -1,271 +0,0 @@ |
|||
# coding: utf-8 |
|||
from __future__ import unicode_literals |
|||
|
|||
import collections |
|||
import re |
|||
|
|||
from .common import InfoExtractor |
|||
from ..compat import ( |
|||
compat_urlparse) |
|||
from ..utils import ( |
|||
ExtractorError, |
|||
urlencode_postdata, |
|||
qualities, unescapeHTML) |
|||
|
|||
|
|||
class FrontEndMasterBaseIE(InfoExtractor): |
|||
_API_BASE = 'https://api.frontendmasters.com/v1/kabuki/courses' |
|||
_VIDEO_BASE = 'http://www.frontendmasters.com/courses' |
|||
_CAPTIONS_BASE = 'https://api.frontendmasters.com/v1/kabuki/transcripts' |
|||
_COOKIES_BASE = 'https://api.frontendmasters.com' |
|||
_LOGIN_URL = 'https://frontendmasters.com/login/' |
|||
|
|||
_QUALITIES_PREFERENCE = ('low', 'medium', 'high') |
|||
_QUALITIES = { |
|||
'low': {'width': 480, 'height': 360}, |
|||
'medium': {'width': 1280, 'height': 720}, |
|||
'high': {'width': 1920, 'height': 1080} |
|||
} |
|||
|
|||
AllowedQuality = collections.namedtuple('AllowedQuality', |
|||
['ext', 'qualities']) |
|||
_ALLOWED_QUALITIES = [ |
|||
AllowedQuality('webm', ['low', 'medium', 'high']), |
|||
AllowedQuality('mp4', ['low', 'medium', 'high']) |
|||
] |
|||
|
|||
def _real_initialize(self): |
|||
self._login() |
|||
|
|||
def _login(self): |
|||
(username, password) = self._get_login_info() |
|||
if username is None: |
|||
return |
|||
|
|||
login_page = self._download_webpage( |
|||
self._LOGIN_URL, None, 'Downloading login page') |
|||
|
|||
login_form = self._hidden_inputs(login_page) |
|||
|
|||
login_form.update({ |
|||
'username': username, |
|||
'password': password |
|||
}) |
|||
|
|||
post_url = self._search_regex( |
|||
r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, |
|||
'post_url', default=self._LOGIN_URL, group='url') |
|||
|
|||
if not post_url.startswith('http'): |
|||
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url) |
|||
|
|||
response = self._download_webpage( |
|||
post_url, None, 'Logging in', |
|||
data=urlencode_postdata(login_form), |
|||
headers={'Content-Type': 'application/x-www-form-urlencoded'} |
|||
) |
|||
|
|||
error = self._search_regex( |
|||
r'<div[^>]+class=["\']Message MessageAlert["\'][^>]*>' |
|||
r'([^<]+)' |
|||
r'</div>', |
|||
response, 'error message', default=None) |
|||
|
|||
if error: |
|||
raise ExtractorError('Unable to login: %s' % unescapeHTML(error), |
|||
expected=True) |
|||
|
|||
def _download_course(self, course_id, url): |
|||
response = self._download_json( |
|||
'%s/%s' % (self._API_BASE, course_id), course_id, |
|||
'Downloading course JSON', |
|||
headers={ |
|||
'Content-Type': 'application/json;charset=utf-8', |
|||
'Referer': url, |
|||
}) |
|||
return response |
|||
|
|||
@staticmethod |
|||
def _pair_section_video_element(lesson_elements): |
|||
sections = {} |
|||
current_section = None |
|||
current_section_number = 0 |
|||
for elem in lesson_elements: |
|||
if not isinstance(elem, int): |
|||
elem_name = elem |
|||
if not isinstance(elem_name, str): |
|||
# convert unicode to str |
|||
elem_name = elem.encode('utf-8') |
|||
(current_section, current_section_number) = \ |
|||
(elem_name, current_section_number + 1) |
|||
else: |
|||
if current_section: |
|||
sections[elem] = (current_section, current_section_number) |
|||
|
|||
return sections |
|||
|
|||
|
|||
class FrontEndMasterIE(FrontEndMasterBaseIE): |
|||
IE_NAME = 'frontend-masters' |
|||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/' \ |
|||
r'(?P<courseid>[a-z\-]+)/' \ |
|||
r'(?P<id>[a-z\-]+)' |
|||
|
|||
_NETRC_MACHINE = 'frontendmasters' |
|||
|
|||
_TEST = { |
|||
'url': 'https://frontendmasters.com/courses/web-development/tools', |
|||
'md5': '7f161159710d6b7016a4f4af6fcb05e2', |
|||
'info_dict': { |
|||
'id': 'tools', |
|||
'title': 'Tools', |
|||
'display_id': 'tools', |
|||
'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7', |
|||
'ext': 'mp4' |
|||
}, |
|||
'skip': 'Requires FrontendMasters account credentials', |
|||
} |
|||
|
|||
def _get_subtitles(self, video_hash, video_id): |
|||
captions = self._download_webpage( |
|||
'%s/%s.vtt' % (self._CAPTIONS_BASE, video_hash), video_id, |
|||
fatal=False) |
|||
if captions: |
|||
return { |
|||
'en': [{ |
|||
'ext': 'vtt', |
|||
'data': captions |
|||
}] |
|||
} |
|||
|
|||
def _real_extract(self, url): |
|||
mobj = re.match(self._VALID_URL, url) |
|||
video_id = mobj.group('id') |
|||
course_id = mobj.group('courseid') |
|||
|
|||
course_json_content = self._download_course(course_id=course_id, |
|||
url=url) |
|||
|
|||
# Necessary to get mandatory informations like title and video_url |
|||
lesson_index = course_json_content.get('lessonSlugs').index(video_id) |
|||
lesson_hash = course_json_content.get('lessonHashes')[lesson_index] |
|||
lesson_data = course_json_content.get('lessonData')[lesson_hash] |
|||
# This is necessary to get the link for the video |
|||
lesson_source_base = lesson_data['sourceBase'] |
|||
|
|||
lesson_title = lesson_data['title'] |
|||
|
|||
# Some optional fields |
|||
lesson_description = lesson_data.get('description') |
|||
lesson_index = lesson_data.get('index') |
|||
lesson_slug = lesson_data.get('slug') |
|||
lesson_thumbnail_url = lesson_data.get('thumbnail') |
|||
lesson_section_elements = course_json_content.get('lessonElements') |
|||
|
|||
try: |
|||
course_sections_pairing = self._pair_section_video_element( |
|||
lesson_section_elements) |
|||
|
|||
lesson_section = \ |
|||
course_sections_pairing.get(lesson_index)[0] |
|||
|
|||
lesson_section_number = \ |
|||
course_sections_pairing.get(lesson_index)[1] |
|||
except Exception: |
|||
lesson_section = None |
|||
lesson_section_number = None |
|||
|
|||
video_request_url = '%s/source' |
|||
video_request_headers = { |
|||
'origin': 'https://frontendmasters.com', |
|||
'referer': lesson_source_base, |
|||
} |
|||
|
|||
quality_key = qualities(self._QUALITIES_PREFERENCE) |
|||
|
|||
formats = [] |
|||
for ext, qualities_ in self._ALLOWED_QUALITIES: |
|||
for quality in qualities_: |
|||
f = self._QUALITIES[quality].copy() |
|||
video_request_params = { |
|||
'r': f['height'], |
|||
'f': ext |
|||
} |
|||
video_response = self._download_json( |
|||
video_request_url % lesson_source_base, video_id, |
|||
query=video_request_params, headers=video_request_headers) |
|||
|
|||
video_url = video_response.get('url') |
|||
clip_f = f.copy() |
|||
clip_f.update({ |
|||
'url': video_url, |
|||
'ext': ext, |
|||
'format_id': '%s-%s' % (ext, quality), |
|||
'quality': quality_key(quality), |
|||
'height': f['height'] |
|||
}) |
|||
formats.append(clip_f) |
|||
|
|||
self._sort_formats(formats) |
|||
|
|||
subtitles = self.extract_subtitles(lesson_hash, video_id) |
|||
|
|||
return { |
|||
'id': video_id, |
|||
'display_id': lesson_slug, |
|||
'title': lesson_title, |
|||
'description': lesson_description, |
|||
'chapter': lesson_section, |
|||
'chapter_number': lesson_section_number, |
|||
'thumbnail': lesson_thumbnail_url, |
|||
'formats': formats, |
|||
'subtitles': subtitles |
|||
} |
|||
|
|||
|
|||
class FrontEndMasterCourseIE(FrontEndMasterBaseIE): |
|||
IE_NAME = 'frontend-masters:course' |
|||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<courseid>[a-z\-]+)/?$' |
|||
|
|||
_NETRC_MACHINE = 'frontendmasters' |
|||
|
|||
_TEST = { |
|||
'url': 'https://frontendmasters.com/courses/javascript-basics/', |
|||
'info_dict': { |
|||
'id': 'javascript-basics', |
|||
'title': 'Introduction to JavaScript Programming', |
|||
'description': 'md5:269412fbb76d86954761599ad8e4cbc9' |
|||
}, |
|||
'playlist_count': 19, |
|||
'skip': 'Requires FrontendMasters account credentials' |
|||
} |
|||
|
|||
@classmethod |
|||
def suitable(cls, url): |
|||
return False if FrontEndMasterIE.suitable(url) else super(FrontEndMasterBaseIE, cls).suitable(url) |
|||
|
|||
def _real_extract(self, url): |
|||
mobj = re.match(self._VALID_URL, url) |
|||
course_id = mobj.group('courseid') |
|||
course_json_content = self._download_course(course_id=course_id, |
|||
url=url) |
|||
|
|||
title = course_json_content.get('title') |
|||
description = course_json_content.get('description') |
|||
course_display_id = course_json_content.get('slug') |
|||
|
|||
videos_data = course_json_content.get('lessonData').values() |
|||
videos_data = sorted(videos_data, key=lambda video: video.get('index')) |
|||
|
|||
entries = [] |
|||
for video in videos_data: |
|||
video_slug = video.get('slug') |
|||
clip_url = '%s/%s/%s' % ( |
|||
self._VIDEO_BASE, course_display_id, video_slug) |
|||
entries.append({ |
|||
'_type': 'url_transparent', |
|||
'url': clip_url, |
|||
'ie_key': FrontEndMasterIE.ie_key() |
|||
}) |
|||
|
|||
return self.playlist_result(entries, course_id, title, description) |
|||
@ -0,0 +1,262 @@ |
|||
# coding: utf-8 |
|||
from __future__ import unicode_literals |
|||
|
|||
import re |
|||
|
|||
from .common import InfoExtractor |
|||
from ..compat import ( |
|||
compat_str, |
|||
compat_urlparse, |
|||
) |
|||
from ..utils import ( |
|||
ExtractorError, |
|||
parse_duration, |
|||
urlencode_postdata, |
|||
) |
|||
|
|||
|
|||
class FrontendMastersBaseIE(InfoExtractor): |
|||
_API_BASE = 'https://api.frontendmasters.com/v1/kabuki' |
|||
_LOGIN_URL = 'https://frontendmasters.com/login/' |
|||
|
|||
_NETRC_MACHINE = 'frontendmasters' |
|||
|
|||
_QUALITIES = { |
|||
'low': {'width': 480, 'height': 360}, |
|||
'mid': {'width': 1280, 'height': 720}, |
|||
'high': {'width': 1920, 'height': 1080} |
|||
} |
|||
|
|||
def _real_initialize(self): |
|||
self._login() |
|||
|
|||
def _login(self): |
|||
(username, password) = self._get_login_info() |
|||
if username is None: |
|||
return |
|||
|
|||
login_page = self._download_webpage( |
|||
self._LOGIN_URL, None, 'Downloading login page') |
|||
|
|||
login_form = self._hidden_inputs(login_page) |
|||
|
|||
login_form.update({ |
|||
'username': username, |
|||
'password': password |
|||
}) |
|||
|
|||
post_url = self._search_regex( |
|||
r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page, |
|||
'post_url', default=self._LOGIN_URL, group='url') |
|||
|
|||
if not post_url.startswith('http'): |
|||
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url) |
|||
|
|||
response = self._download_webpage( |
|||
post_url, None, 'Logging in', data=urlencode_postdata(login_form), |
|||
headers={'Content-Type': 'application/x-www-form-urlencoded'}) |
|||
|
|||
# Successful login |
|||
if any(p in response for p in ( |
|||
'wp-login.php?action=logout', '>Logout')): |
|||
return |
|||
|
|||
error = self._html_search_regex( |
|||
r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<', |
|||
response, 'error message', default=None, group='error') |
|||
if error: |
|||
raise ExtractorError('Unable to login: %s' % error, expected=True) |
|||
raise ExtractorError('Unable to log in') |
|||
|
|||
|
|||
class FrontendMastersPageBaseIE(FrontendMastersBaseIE): |
|||
def _download_course(self, course_name, url): |
|||
return self._download_json( |
|||
'%s/courses/%s' % (self._API_BASE, course_name), course_name, |
|||
'Downloading course JSON', headers={'Referer': url}) |
|||
|
|||
@staticmethod |
|||
def _extract_chapters(course): |
|||
chapters = [] |
|||
lesson_elements = course.get('lessonElements') |
|||
if isinstance(lesson_elements, list): |
|||
chapters = [e for e in lesson_elements if isinstance(e, compat_str)] |
|||
return chapters |
|||
|
|||
@staticmethod |
|||
def _extract_lesson(chapters, lesson_id, lesson): |
|||
title = lesson.get('title') or lesson_id |
|||
display_id = lesson.get('slug') |
|||
description = lesson.get('description') |
|||
thumbnail = lesson.get('thumbnail') |
|||
|
|||
chapter_number = None |
|||
index = lesson.get('index') |
|||
element_index = lesson.get('elementIndex') |
|||
if (isinstance(index, int) and isinstance(element_index, int) and |
|||
index < element_index): |
|||
chapter_number = element_index - index |
|||
chapter = (chapters[chapter_number - 1] |
|||
if chapter_number - 1 < len(chapters) else None) |
|||
|
|||
duration = None |
|||
timestamp = lesson.get('timestamp') |
|||
if isinstance(timestamp, compat_str): |
|||
mobj = re.search( |
|||
r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})', |
|||
timestamp) |
|||
if mobj: |
|||
duration = parse_duration(mobj.group('end')) - parse_duration( |
|||
mobj.group('start')) |
|||
|
|||
return { |
|||
'_type': 'url_transparent', |
|||
'url': 'frontendmasters:%s' % lesson_id, |
|||
'ie_key': FrontendMastersIE.ie_key(), |
|||
'id': lesson_id, |
|||
'display_id': display_id, |
|||
'title': title, |
|||
'description': description, |
|||
'thumbnail': thumbnail, |
|||
'duration': duration, |
|||
'chapter': chapter, |
|||
'chapter_number': chapter_number, |
|||
} |
|||
|
|||
|
|||
class FrontendMastersIE(FrontendMastersBaseIE): |
|||
_VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)' |
|||
_TESTS = [{ |
|||
'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba', |
|||
'md5': '7f161159710d6b7016a4f4af6fcb05e2', |
|||
'info_dict': { |
|||
'id': 'a2qogef6ba', |
|||
'ext': 'mp4', |
|||
'title': 'a2qogef6ba', |
|||
}, |
|||
'skip': 'Requires FrontendMasters account credentials', |
|||
}, { |
|||
'url': 'frontendmasters:a2qogef6ba', |
|||
'only_matching': True, |
|||
}] |
|||
|
|||
def _real_extract(self, url): |
|||
lesson_id = self._match_id(url) |
|||
|
|||
source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id) |
|||
|
|||
formats = [] |
|||
for ext in ('webm', 'mp4'): |
|||
for quality in ('low', 'mid', 'high'): |
|||
resolution = self._QUALITIES[quality].copy() |
|||
format_id = '%s-%s' % (ext, quality) |
|||
format_url = self._download_json( |
|||
source_url, lesson_id, |
|||
'Downloading %s source JSON' % format_id, query={ |
|||
'f': ext, |
|||
'r': resolution['height'], |
|||
}, headers={ |
|||
'Referer': url, |
|||
}, fatal=False)['url'] |
|||
|
|||
if not format_url: |
|||
continue |
|||
|
|||
f = resolution.copy() |
|||
f.update({ |
|||
'url': format_url, |
|||
'ext': ext, |
|||
'format_id': format_id, |
|||
}) |
|||
formats.append(f) |
|||
self._sort_formats(formats) |
|||
|
|||
subtitles = { |
|||
'en': [{ |
|||
'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id), |
|||
}] |
|||
} |
|||
|
|||
return { |
|||
'id': lesson_id, |
|||
'title': lesson_id, |
|||
'formats': formats, |
|||
'subtitles': subtitles |
|||
} |
|||
|
|||
|
|||
class FrontendMastersLessonIE(FrontendMastersPageBaseIE): |
|||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)' |
|||
_TEST = { |
|||
'url': 'https://frontendmasters.com/courses/web-development/tools', |
|||
'info_dict': { |
|||
'id': 'a2qogef6ba', |
|||
'display_id': 'tools', |
|||
'ext': 'mp4', |
|||
'title': 'Tools', |
|||
'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7', |
|||
'thumbnail': r're:^https?://.*\.jpg$', |
|||
'chapter': 'Introduction', |
|||
'chapter_number': 1, |
|||
}, |
|||
'params': { |
|||
'skip_download': True, |
|||
}, |
|||
'skip': 'Requires FrontendMasters account credentials', |
|||
} |
|||
|
|||
def _real_extract(self, url): |
|||
mobj = re.match(self._VALID_URL, url) |
|||
course_name, lesson_name = mobj.group('course_name', 'lesson_name') |
|||
|
|||
course = self._download_course(course_name, url) |
|||
|
|||
lesson_id, lesson = next( |
|||
(video_id, data) |
|||
for video_id, data in course['lessonData'].items() |
|||
if data.get('slug') == lesson_name) |
|||
|
|||
chapters = self._extract_chapters(course) |
|||
return self._extract_lesson(chapters, lesson_id, lesson) |
|||
|
|||
|
|||
class FrontendMastersCourseIE(FrontendMastersPageBaseIE): |
|||
_VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)' |
|||
_TEST = { |
|||
'url': 'https://frontendmasters.com/courses/web-development/', |
|||
'info_dict': { |
|||
'id': 'web-development', |
|||
'title': 'Introduction to Web Development', |
|||
'description': 'md5:9317e6e842098bf725d62360e52d49a6', |
|||
}, |
|||
'playlist_count': 81, |
|||
'skip': 'Requires FrontendMasters account credentials', |
|||
} |
|||
|
|||
@classmethod |
|||
def suitable(cls, url): |
|||
return False if FrontendMastersLessonIE.suitable(url) else super( |
|||
FrontendMastersBaseIE, cls).suitable(url) |
|||
|
|||
def _real_extract(self, url): |
|||
course_name = self._match_id(url) |
|||
|
|||
course = self._download_course(course_name, url) |
|||
|
|||
chapters = self._extract_chapters(course) |
|||
|
|||
lessons = sorted( |
|||
course['lessonData'].values(), key=lambda data: data['index']) |
|||
|
|||
entries = [] |
|||
for lesson in lessons: |
|||
lesson_name = lesson.get('slug') |
|||
if not lesson_name: |
|||
continue |
|||
lesson_id = lesson.get('hash') or lesson.get('statsId') |
|||
entries.append(self._extract_lesson(chapters, lesson_id, lesson)) |
|||
|
|||
title = course.get('title') |
|||
description = course.get('description') |
|||
|
|||
return self.playlist_result(entries, course_name, title, description) |
|||
Write
Preview
Loading…
Cancel
Save