Philipp Hagemeister
11 years ago
3 changed files with 267 additions and 247 deletions
Unified View
Diff Options
-
248youtube_dl/InfoExtractors.py
-
0youtube_dl/extractor/__init__.py
-
266youtube_dl/extractor/common.py
@ -0,0 +1,266 @@ |
|||||
|
from __future__ import absolute_import |
||||
|
|
||||
|
import base64 |
||||
|
import os |
||||
|
import re |
||||
|
import socket |
||||
|
import sys |
||||
|
|
||||
|
from ..utils import ( |
||||
|
compat_http_client, |
||||
|
compat_urllib_error, |
||||
|
compat_urllib_request, |
||||
|
compat_str, |
||||
|
|
||||
|
clean_html, |
||||
|
compiled_regex_type, |
||||
|
ExtractorError, |
||||
|
) |
||||
|
|
||||
|
class InfoExtractor(object): |
||||
|
"""Information Extractor class. |
||||
|
|
||||
|
Information extractors are the classes that, given a URL, extract |
||||
|
information about the video (or videos) the URL refers to. This |
||||
|
information includes the real video URL, the video title, author and |
||||
|
others. The information is stored in a dictionary which is then |
||||
|
passed to the FileDownloader. The FileDownloader processes this |
||||
|
information possibly downloading the video to the file system, among |
||||
|
other possible outcomes. |
||||
|
|
||||
|
The dictionaries must include the following fields: |
||||
|
|
||||
|
id: Video identifier. |
||||
|
url: Final video URL. |
||||
|
title: Video title, unescaped. |
||||
|
ext: Video filename extension. |
||||
|
|
||||
|
The following fields are optional: |
||||
|
|
||||
|
format: The video format, defaults to ext (used for --get-format) |
||||
|
thumbnail: Full URL to a video thumbnail image. |
||||
|
description: One-line video description. |
||||
|
uploader: Full name of the video uploader. |
||||
|
upload_date: Video upload date (YYYYMMDD). |
||||
|
uploader_id: Nickname or id of the video uploader. |
||||
|
location: Physical location of the video. |
||||
|
player_url: SWF Player URL (used for rtmpdump). |
||||
|
subtitles: The subtitle file contents. |
||||
|
urlhandle: [internal] The urlHandle to be used to download the file, |
||||
|
like returned by urllib.request.urlopen |
||||
|
|
||||
|
The fields should all be Unicode strings. |
||||
|
|
||||
|
Subclasses of this one should re-define the _real_initialize() and |
||||
|
_real_extract() methods and define a _VALID_URL regexp. |
||||
|
Probably, they should also be added to the list of extractors. |
||||
|
|
||||
|
_real_extract() must return a *list* of information dictionaries as |
||||
|
described above. |
||||
|
|
||||
|
Finally, the _WORKING attribute should be set to False for broken IEs |
||||
|
in order to warn the users and skip the tests. |
||||
|
""" |
||||
|
|
||||
|
_ready = False |
||||
|
_downloader = None |
||||
|
_WORKING = True |
||||
|
|
||||
|
def __init__(self, downloader=None): |
||||
|
"""Constructor. Receives an optional downloader.""" |
||||
|
self._ready = False |
||||
|
self.set_downloader(downloader) |
||||
|
|
||||
|
@classmethod |
||||
|
def suitable(cls, url): |
||||
|
"""Receives a URL and returns True if suitable for this IE.""" |
||||
|
return re.match(cls._VALID_URL, url) is not None |
||||
|
|
||||
|
@classmethod |
||||
|
def working(cls): |
||||
|
"""Getter method for _WORKING.""" |
||||
|
return cls._WORKING |
||||
|
|
||||
|
def initialize(self): |
||||
|
"""Initializes an instance (authentication, etc).""" |
||||
|
if not self._ready: |
||||
|
self._real_initialize() |
||||
|
self._ready = True |
||||
|
|
||||
|
def extract(self, url): |
||||
|
"""Extracts URL information and returns it in list of dicts.""" |
||||
|
self.initialize() |
||||
|
return self._real_extract(url) |
||||
|
|
||||
|
def set_downloader(self, downloader): |
||||
|
"""Sets the downloader for this IE.""" |
||||
|
self._downloader = downloader |
||||
|
|
||||
|
def _real_initialize(self): |
||||
|
"""Real initialization process. Redefine in subclasses.""" |
||||
|
pass |
||||
|
|
||||
|
def _real_extract(self, url): |
||||
|
"""Real extraction process. Redefine in subclasses.""" |
||||
|
pass |
||||
|
|
||||
|
@property |
||||
|
def IE_NAME(self): |
||||
|
return type(self).__name__[:-2] |
||||
|
|
||||
|
def _request_webpage(self, url_or_request, video_id, note=None, errnote=None): |
||||
|
""" Returns the response handle """ |
||||
|
if note is None: |
||||
|
self.report_download_webpage(video_id) |
||||
|
elif note is not False: |
||||
|
self.to_screen(u'%s: %s' % (video_id, note)) |
||||
|
try: |
||||
|
return compat_urllib_request.urlopen(url_or_request) |
||||
|
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: |
||||
|
if errnote is None: |
||||
|
errnote = u'Unable to download webpage' |
||||
|
raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2]) |
||||
|
|
||||
|
def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None): |
||||
|
""" Returns a tuple (page content as string, URL handle) """ |
||||
|
urlh = self._request_webpage(url_or_request, video_id, note, errnote) |
||||
|
content_type = urlh.headers.get('Content-Type', '') |
||||
|
m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type) |
||||
|
if m: |
||||
|
encoding = m.group(1) |
||||
|
else: |
||||
|
encoding = 'utf-8' |
||||
|
webpage_bytes = urlh.read() |
||||
|
if self._downloader.params.get('dump_intermediate_pages', False): |
||||
|
try: |
||||
|
url = url_or_request.get_full_url() |
||||
|
except AttributeError: |
||||
|
url = url_or_request |
||||
|
self.to_screen(u'Dumping request to ' + url) |
||||
|
dump = base64.b64encode(webpage_bytes).decode('ascii') |
||||
|
self._downloader.to_screen(dump) |
||||
|
content = webpage_bytes.decode(encoding, 'replace') |
||||
|
return (content, urlh) |
||||
|
|
||||
|
def _download_webpage(self, url_or_request, video_id, note=None, errnote=None): |
||||
|
""" Returns the data of the page as a string """ |
||||
|
return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0] |
||||
|
|
||||
|
def to_screen(self, msg): |
||||
|
"""Print msg to screen, prefixing it with '[ie_name]'""" |
||||
|
self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg)) |
||||
|
|
||||
|
def report_extraction(self, id_or_name): |
||||
|
"""Report information extraction.""" |
||||
|
self.to_screen(u'%s: Extracting information' % id_or_name) |
||||
|
|
||||
|
def report_download_webpage(self, video_id): |
||||
|
"""Report webpage download.""" |
||||
|
self.to_screen(u'%s: Downloading webpage' % video_id) |
||||
|
|
||||
|
def report_age_confirmation(self): |
||||
|
"""Report attempt to confirm age.""" |
||||
|
self.to_screen(u'Confirming age') |
||||
|
|
||||
|
#Methods for following #608 |
||||
|
#They set the correct value of the '_type' key |
||||
|
def video_result(self, video_info): |
||||
|
"""Returns a video""" |
||||
|
video_info['_type'] = 'video' |
||||
|
return video_info |
||||
|
def url_result(self, url, ie=None): |
||||
|
"""Returns a url that points to a page that should be processed""" |
||||
|
#TODO: ie should be the class used for getting the info |
||||
|
video_info = {'_type': 'url', |
||||
|
'url': url, |
||||
|
'ie_key': ie} |
||||
|
return video_info |
||||
|
def playlist_result(self, entries, playlist_id=None, playlist_title=None): |
||||
|
"""Returns a playlist""" |
||||
|
video_info = {'_type': 'playlist', |
||||
|
'entries': entries} |
||||
|
if playlist_id: |
||||
|
video_info['id'] = playlist_id |
||||
|
if playlist_title: |
||||
|
video_info['title'] = playlist_title |
||||
|
return video_info |
||||
|
|
||||
|
def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): |
||||
|
""" |
||||
|
Perform a regex search on the given string, using a single or a list of |
||||
|
patterns returning the first matching group. |
||||
|
In case of failure return a default value or raise a WARNING or a |
||||
|
ExtractorError, depending on fatal, specifying the field name. |
||||
|
""" |
||||
|
if isinstance(pattern, (str, compat_str, compiled_regex_type)): |
||||
|
mobj = re.search(pattern, string, flags) |
||||
|
else: |
||||
|
for p in pattern: |
||||
|
mobj = re.search(p, string, flags) |
||||
|
if mobj: break |
||||
|
|
||||
|
if sys.stderr.isatty() and os.name != 'nt': |
||||
|
_name = u'\033[0;34m%s\033[0m' % name |
||||
|
else: |
||||
|
_name = name |
||||
|
|
||||
|
if mobj: |
||||
|
# return the first matching group |
||||
|
return next(g for g in mobj.groups() if g is not None) |
||||
|
elif default is not None: |
||||
|
return default |
||||
|
elif fatal: |
||||
|
raise ExtractorError(u'Unable to extract %s' % _name) |
||||
|
else: |
||||
|
self._downloader.report_warning(u'unable to extract %s; ' |
||||
|
u'please report this issue on GitHub.' % _name) |
||||
|
return None |
||||
|
|
||||
|
def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): |
||||
|
""" |
||||
|
Like _search_regex, but strips HTML tags and unescapes entities. |
||||
|
""" |
||||
|
res = self._search_regex(pattern, string, name, default, fatal, flags) |
||||
|
if res: |
||||
|
return clean_html(res).strip() |
||||
|
else: |
||||
|
return res |
||||
|
|
||||
|
class SearchInfoExtractor(InfoExtractor): |
||||
|
""" |
||||
|
Base class for paged search queries extractors. |
||||
|
They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query} |
||||
|
Instances should define _SEARCH_KEY and _MAX_RESULTS. |
||||
|
""" |
||||
|
|
||||
|
@classmethod |
||||
|
def _make_valid_url(cls): |
||||
|
return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY |
||||
|
|
||||
|
@classmethod |
||||
|
def suitable(cls, url): |
||||
|
return re.match(cls._make_valid_url(), url) is not None |
||||
|
|
||||
|
def _real_extract(self, query): |
||||
|
mobj = re.match(self._make_valid_url(), query) |
||||
|
if mobj is None: |
||||
|
raise ExtractorError(u'Invalid search query "%s"' % query) |
||||
|
|
||||
|
prefix = mobj.group('prefix') |
||||
|
query = mobj.group('query') |
||||
|
if prefix == '': |
||||
|
return self._get_n_results(query, 1) |
||||
|
elif prefix == 'all': |
||||
|
return self._get_n_results(query, self._MAX_RESULTS) |
||||
|
else: |
||||
|
n = int(prefix) |
||||
|
if n <= 0: |
||||
|
raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query)) |
||||
|
elif n > self._MAX_RESULTS: |
||||
|
self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n)) |
||||
|
n = self._MAX_RESULTS |
||||
|
return self._get_n_results(query, n) |
||||
|
|
||||
|
def _get_n_results(self, query, n): |
||||
|
"""Get a specified number of results for a query""" |
||||
|
raise NotImplementedError("This method must be implemented by sublclasses") |
||||
Write
Preview
Loading…
Cancel
Save