|
|
|
@ -1,14 +1,19 @@ |
|
|
|
#!/usr/bin/env python |
|
|
|
# -*- coding: utf-8 -*- |
|
|
|
# Author: Ricardo Garcia Gonzalez |
|
|
|
# Author: Danny Colligan |
|
|
|
# Author: Benjamin Johnson |
|
|
|
# Author: Vasyl' Vavrychuk |
|
|
|
# Author: Witold Baryluk |
|
|
|
# Author: Paweł Paprota |
|
|
|
# Author: Gergely Imreh |
|
|
|
# Author: Philipp Hagemeister <phihag@phihag.de> |
|
|
|
# License: Public domain code |
|
|
|
|
|
|
|
__author__ = ( |
|
|
|
'Ricardo Garcia Gonzalez', |
|
|
|
'Danny Colligan', |
|
|
|
'Benjamin Johnson', |
|
|
|
'Vasyl\' Vavrychuk', |
|
|
|
'Witold Baryluk', |
|
|
|
'Paweł Paprota', |
|
|
|
'Gergely Imreh', |
|
|
|
) |
|
|
|
|
|
|
|
__license__ = 'Public Domain' |
|
|
|
__version__ = '2011.08.24-phihag' |
|
|
|
|
|
|
|
import cookielib |
|
|
|
import datetime |
|
|
|
import gzip |
|
|
|
@ -876,7 +881,7 @@ class FileDownloader(object): |
|
|
|
else: |
|
|
|
# Examine the reported length |
|
|
|
if (content_length is not None and |
|
|
|
(resume_len - 100 < long(content_length) < resume_len + 100)): |
|
|
|
(resume_len - 100 < long(content_length) < resume_len + 100)): |
|
|
|
# The file had already been fully downloaded. |
|
|
|
# Explanation to the above condition: in issue #175 it was revealed that |
|
|
|
# YouTube sometimes adds or removes a few bytes from the end of the file, |
|
|
|
@ -1862,8 +1867,8 @@ class YahooIE(InfoExtractor): |
|
|
|
yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents |
|
|
|
yv_bitrate = '700' # according to Wikipedia this is hard-coded |
|
|
|
request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id + |
|
|
|
'&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + |
|
|
|
'&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') |
|
|
|
'&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + |
|
|
|
'&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') |
|
|
|
try: |
|
|
|
self.report_download_webpage(video_id) |
|
|
|
webpage = urllib2.urlopen(request).read() |
|
|
|
@ -2432,9 +2437,9 @@ class YoutubeUserIE(InfoExtractor): |
|
|
|
video_ids = video_ids[playliststart:] |
|
|
|
else: |
|
|
|
video_ids = video_ids[playliststart:playlistend] |
|
|
|
|
|
|
|
|
|
|
|
self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" % |
|
|
|
(username, all_ids_count, len(video_ids))) |
|
|
|
(username, all_ids_count, len(video_ids))) |
|
|
|
|
|
|
|
for video_id in video_ids: |
|
|
|
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id) |
|
|
|
@ -2942,321 +2947,381 @@ class FFmpegExtractAudioPP(PostProcessor): |
|
|
|
information['filepath'] = new_path |
|
|
|
return information |
|
|
|
|
|
|
|
### MAIN PROGRAM ### |
|
|
|
if __name__ == '__main__': |
|
|
|
|
|
|
|
def updateSelf(downloader, filename): |
|
|
|
''' Update the program file with the latest version from the repository ''' |
|
|
|
# Note: downloader only used for options |
|
|
|
if not os.access(filename, os.W_OK): |
|
|
|
sys.exit('ERROR: no write permissions on %s' % filename) |
|
|
|
|
|
|
|
downloader.to_screen('Updating to latest stable version...') |
|
|
|
|
|
|
|
try: |
|
|
|
# Modules needed only when running the main program |
|
|
|
import getpass |
|
|
|
import optparse |
|
|
|
latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION' |
|
|
|
latest_version = urllib.urlopen(latest_url).read().strip() |
|
|
|
prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version |
|
|
|
newcontent = urllib.urlopen(prog_url).read() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit('ERROR: unable to download latest version') |
|
|
|
|
|
|
|
# Function to update the program file with the latest version from the repository. |
|
|
|
def update_self(downloader, filename): |
|
|
|
# Note: downloader only used for options |
|
|
|
if not os.access(filename, os.W_OK): |
|
|
|
sys.exit('ERROR: no write permissions on %s' % filename) |
|
|
|
try: |
|
|
|
stream = open(filename, 'w') |
|
|
|
stream.write(newcontent) |
|
|
|
stream.close() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit('ERROR: unable to overwrite current version') |
|
|
|
|
|
|
|
downloader.to_screen('Updating to latest stable version...') |
|
|
|
try: |
|
|
|
latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION' |
|
|
|
latest_version = urllib.urlopen(latest_url).read().strip() |
|
|
|
prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version |
|
|
|
newcontent = urllib.urlopen(prog_url).read() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit('ERROR: unable to download latest version') |
|
|
|
try: |
|
|
|
stream = open(filename, 'w') |
|
|
|
stream.write(newcontent) |
|
|
|
stream.close() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit('ERROR: unable to overwrite current version') |
|
|
|
downloader.to_screen('Updated to version %s' % latest_version) |
|
|
|
|
|
|
|
# Parse command line |
|
|
|
parser = optparse.OptionParser( |
|
|
|
usage='Usage: %prog [options] url...', |
|
|
|
version='2011.08.04-phihag', |
|
|
|
conflict_handler='resolve', |
|
|
|
) |
|
|
|
|
|
|
|
parser.add_option('-h', '--help', |
|
|
|
action='help', help='print this help text and exit') |
|
|
|
parser.add_option('-v', '--version', |
|
|
|
action='version', help='print program version and exit') |
|
|
|
parser.add_option('-U', '--update', |
|
|
|
action='store_true', dest='update_self', help='update this program to latest stable version') |
|
|
|
parser.add_option('-i', '--ignore-errors', |
|
|
|
action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) |
|
|
|
parser.add_option('-r', '--rate-limit', |
|
|
|
dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)') |
|
|
|
parser.add_option('-R', '--retries', |
|
|
|
dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10) |
|
|
|
parser.add_option('--playlist-start', |
|
|
|
dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1) |
|
|
|
parser.add_option('--playlist-end', |
|
|
|
dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) |
|
|
|
parser.add_option('--dump-user-agent', |
|
|
|
action='store_true', dest='dump_user_agent', |
|
|
|
help='display the current browser identification', default=False) |
|
|
|
|
|
|
|
authentication = optparse.OptionGroup(parser, 'Authentication Options') |
|
|
|
authentication.add_option('-u', '--username', |
|
|
|
dest='username', metavar='USERNAME', help='account username') |
|
|
|
authentication.add_option('-p', '--password', |
|
|
|
dest='password', metavar='PASSWORD', help='account password') |
|
|
|
authentication.add_option('-n', '--netrc', |
|
|
|
action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) |
|
|
|
parser.add_option_group(authentication) |
|
|
|
|
|
|
|
video_format = optparse.OptionGroup(parser, 'Video Format Options') |
|
|
|
video_format.add_option('-f', '--format', |
|
|
|
action='store', dest='format', metavar='FORMAT', help='video format code') |
|
|
|
video_format.add_option('--all-formats', |
|
|
|
action='store_const', dest='format', help='download all available video formats', const='-1') |
|
|
|
video_format.add_option('--max-quality', |
|
|
|
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') |
|
|
|
parser.add_option_group(video_format) |
|
|
|
|
|
|
|
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') |
|
|
|
verbosity.add_option('-q', '--quiet', |
|
|
|
action='store_true', dest='quiet', help='activates quiet mode', default=False) |
|
|
|
verbosity.add_option('-s', '--simulate', |
|
|
|
action='store_true', dest='simulate', help='do not download video', default=False) |
|
|
|
verbosity.add_option('-g', '--get-url', |
|
|
|
action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) |
|
|
|
verbosity.add_option('-e', '--get-title', |
|
|
|
action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) |
|
|
|
verbosity.add_option('--get-thumbnail', |
|
|
|
action='store_true', dest='getthumbnail', |
|
|
|
help='simulate, quiet but print thumbnail URL', default=False) |
|
|
|
verbosity.add_option('--get-description', |
|
|
|
action='store_true', dest='getdescription', |
|
|
|
help='simulate, quiet but print video description', default=False) |
|
|
|
verbosity.add_option('--get-filename', |
|
|
|
action='store_true', dest='getfilename', |
|
|
|
help='simulate, quiet but print output filename', default=False) |
|
|
|
verbosity.add_option('--no-progress', |
|
|
|
action='store_true', dest='noprogress', help='do not print progress bar', default=False) |
|
|
|
verbosity.add_option('--console-title', |
|
|
|
action='store_true', dest='consoletitle', |
|
|
|
help='display progress in console titlebar', default=False) |
|
|
|
parser.add_option_group(verbosity) |
|
|
|
|
|
|
|
filesystem = optparse.OptionGroup(parser, 'Filesystem Options') |
|
|
|
filesystem.add_option('-t', '--title', |
|
|
|
action='store_true', dest='usetitle', help='use title in file name', default=False) |
|
|
|
filesystem.add_option('-l', '--literal', |
|
|
|
action='store_true', dest='useliteral', help='use literal title in file name', default=False) |
|
|
|
filesystem.add_option('-A', '--auto-number', |
|
|
|
action='store_true', dest='autonumber', |
|
|
|
help='number downloaded files starting from 00000', default=False) |
|
|
|
filesystem.add_option('-o', '--output', |
|
|
|
dest='outtmpl', metavar='TEMPLATE', help='output filename template') |
|
|
|
filesystem.add_option('-a', '--batch-file', |
|
|
|
dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') |
|
|
|
filesystem.add_option('-w', '--no-overwrites', |
|
|
|
action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) |
|
|
|
filesystem.add_option('-c', '--continue', |
|
|
|
action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False) |
|
|
|
filesystem.add_option('--cookies', |
|
|
|
dest='cookiefile', metavar='FILE', help='file to dump cookie jar to') |
|
|
|
filesystem.add_option('--no-part', |
|
|
|
action='store_true', dest='nopart', help='do not use .part files', default=False) |
|
|
|
filesystem.add_option('--no-mtime', |
|
|
|
action='store_false', dest='updatetime', |
|
|
|
help='do not use the Last-modified header to set the file modification time', default=True) |
|
|
|
filesystem.add_option('--write-description', |
|
|
|
action='store_true', dest='writedescription', |
|
|
|
help='write video description to a .description file', default=False) |
|
|
|
filesystem.add_option('--write-info-json', |
|
|
|
action='store_true', dest='writeinfojson', |
|
|
|
help='write video metadata to a .info.json file', default=False) |
|
|
|
parser.add_option_group(filesystem) |
|
|
|
|
|
|
|
postproc = optparse.OptionGroup(parser, 'Post-processing Options') |
|
|
|
postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False, |
|
|
|
help='convert video files to audio-only files (requires ffmpeg and ffprobe)') |
|
|
|
postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', |
|
|
|
help='"best", "aac" or "mp3"; best by default') |
|
|
|
parser.add_option_group(postproc) |
|
|
|
|
|
|
|
(opts, args) = parser.parse_args() |
|
|
|
|
|
|
|
# Open appropriate CookieJar |
|
|
|
if opts.cookiefile is None: |
|
|
|
jar = cookielib.CookieJar() |
|
|
|
else: |
|
|
|
try: |
|
|
|
jar = cookielib.MozillaCookieJar(opts.cookiefile) |
|
|
|
if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK): |
|
|
|
jar.load() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit(u'ERROR: unable to open cookie file') |
|
|
|
downloader.to_screen('Updated to version %s' % latest_version) |
|
|
|
|
|
|
|
# Dump user agent |
|
|
|
if opts.dump_user_agent: |
|
|
|
print std_headers['User-Agent'] |
|
|
|
sys.exit(0) |
|
|
|
def parseOpts(): |
|
|
|
# Deferred imports |
|
|
|
import getpass |
|
|
|
import optparse |
|
|
|
|
|
|
|
# General configuration |
|
|
|
cookie_processor = urllib2.HTTPCookieProcessor(jar) |
|
|
|
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())) |
|
|
|
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) |
|
|
|
def _format_option_string(option): |
|
|
|
''' ('-o', '--option') -> -o, --format METAVAR''' |
|
|
|
|
|
|
|
# Batch file verification |
|
|
|
batchurls = [] |
|
|
|
if opts.batchfile is not None: |
|
|
|
try: |
|
|
|
if opts.batchfile == '-': |
|
|
|
batchfd = sys.stdin |
|
|
|
else: |
|
|
|
batchfd = open(opts.batchfile, 'r') |
|
|
|
batchurls = batchfd.readlines() |
|
|
|
batchurls = [x.strip() for x in batchurls] |
|
|
|
batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] |
|
|
|
except IOError: |
|
|
|
sys.exit(u'ERROR: batch file could not be read') |
|
|
|
all_urls = batchurls + args |
|
|
|
|
|
|
|
# Conflicting, missing and erroneous options |
|
|
|
if opts.usenetrc and (opts.username is not None or opts.password is not None): |
|
|
|
parser.error(u'using .netrc conflicts with giving username/password') |
|
|
|
if opts.password is not None and opts.username is None: |
|
|
|
parser.error(u'account username missing') |
|
|
|
if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber): |
|
|
|
parser.error(u'using output template conflicts with using title, literal title or auto number') |
|
|
|
if opts.usetitle and opts.useliteral: |
|
|
|
parser.error(u'using title conflicts with using literal title') |
|
|
|
if opts.username is not None and opts.password is None: |
|
|
|
opts.password = getpass.getpass(u'Type account password and press return:') |
|
|
|
if opts.ratelimit is not None: |
|
|
|
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) |
|
|
|
if numeric_limit is None: |
|
|
|
parser.error(u'invalid rate limit specified') |
|
|
|
opts.ratelimit = numeric_limit |
|
|
|
if opts.retries is not None: |
|
|
|
try: |
|
|
|
opts.retries = long(opts.retries) |
|
|
|
except (TypeError, ValueError), err: |
|
|
|
parser.error(u'invalid retry count specified') |
|
|
|
opts = [] |
|
|
|
|
|
|
|
if option._short_opts: opts.append(option._short_opts[0]) |
|
|
|
if option._long_opts: opts.append(option._long_opts[0]) |
|
|
|
if len(opts) > 1: opts.insert(1, ', ') |
|
|
|
|
|
|
|
if option.takes_value(): opts.append(' %s' % option.metavar) |
|
|
|
|
|
|
|
return "".join(opts) |
|
|
|
|
|
|
|
def _find_term_columns(): |
|
|
|
columns = os.environ.get('COLUMNS', None) |
|
|
|
if columns: |
|
|
|
return int(columns) |
|
|
|
|
|
|
|
# TODO: Breaks on phihag's system |
|
|
|
#if sys.platform.startswith('linux'): |
|
|
|
# try: |
|
|
|
# return os.popen('stty size', 'r').read().split()[1] |
|
|
|
# except: pass |
|
|
|
return None |
|
|
|
|
|
|
|
max_width = 80 |
|
|
|
max_help_position = 80 |
|
|
|
|
|
|
|
# No need to wrap help messages if we're on a wide console |
|
|
|
columns = _find_term_columns() |
|
|
|
if columns: max_width = columns |
|
|
|
|
|
|
|
fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) |
|
|
|
fmt.format_option_strings = _format_option_string |
|
|
|
|
|
|
|
kw = { |
|
|
|
'version' : __version__, |
|
|
|
'formatter' : fmt, |
|
|
|
'usage' : '%prog [options] url...', |
|
|
|
'conflict_handler' : 'resolve', |
|
|
|
} |
|
|
|
|
|
|
|
parser = optparse.OptionParser(**kw) |
|
|
|
|
|
|
|
# option groups |
|
|
|
general = optparse.OptionGroup(parser, 'General Options') |
|
|
|
authentication = optparse.OptionGroup(parser, 'Authentication Options') |
|
|
|
video_format = optparse.OptionGroup(parser, 'Video Format Options') |
|
|
|
postproc = optparse.OptionGroup(parser, 'Post-processing Options') |
|
|
|
filesystem = optparse.OptionGroup(parser, 'Filesystem Options') |
|
|
|
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') |
|
|
|
|
|
|
|
general.add_option('-h', '--help', |
|
|
|
action='help', help='print this help text and exit') |
|
|
|
general.add_option('-v', '--version', |
|
|
|
action='version', help='print program version and exit') |
|
|
|
general.add_option('-U', '--update', |
|
|
|
action='store_true', dest='update_self', help='update this program to latest stable version') |
|
|
|
general.add_option('-i', '--ignore-errors', |
|
|
|
action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) |
|
|
|
general.add_option('-r', '--rate-limit', |
|
|
|
dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)') |
|
|
|
general.add_option('-R', '--retries', |
|
|
|
dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10) |
|
|
|
general.add_option('--playlist-start', |
|
|
|
dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1) |
|
|
|
general.add_option('--playlist-end', |
|
|
|
dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) |
|
|
|
general.add_option('--dump-user-agent', |
|
|
|
action='store_true', dest='dump_user_agent', |
|
|
|
help='display the current browser identification', default=False) |
|
|
|
|
|
|
|
authentication.add_option('-u', '--username', |
|
|
|
dest='username', metavar='USERNAME', help='account username') |
|
|
|
authentication.add_option('-p', '--password', |
|
|
|
dest='password', metavar='PASSWORD', help='account password') |
|
|
|
authentication.add_option('-n', '--netrc', |
|
|
|
action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) |
|
|
|
|
|
|
|
|
|
|
|
video_format.add_option('-f', '--format', |
|
|
|
action='store', dest='format', metavar='FORMAT', help='video format code') |
|
|
|
video_format.add_option('--all-formats', |
|
|
|
action='store_const', dest='format', help='download all available video formats', const='-1') |
|
|
|
video_format.add_option('--max-quality', |
|
|
|
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') |
|
|
|
|
|
|
|
|
|
|
|
verbosity.add_option('-q', '--quiet', |
|
|
|
action='store_true', dest='quiet', help='activates quiet mode', default=False) |
|
|
|
verbosity.add_option('-s', '--simulate', |
|
|
|
action='store_true', dest='simulate', help='do not download video', default=False) |
|
|
|
verbosity.add_option('-g', '--get-url', |
|
|
|
action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) |
|
|
|
verbosity.add_option('-e', '--get-title', |
|
|
|
action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) |
|
|
|
verbosity.add_option('--get-thumbnail', |
|
|
|
action='store_true', dest='getthumbnail', |
|
|
|
help='simulate, quiet but print thumbnail URL', default=False) |
|
|
|
verbosity.add_option('--get-description', |
|
|
|
action='store_true', dest='getdescription', |
|
|
|
help='simulate, quiet but print video description', default=False) |
|
|
|
verbosity.add_option('--get-filename', |
|
|
|
action='store_true', dest='getfilename', |
|
|
|
help='simulate, quiet but print output filename', default=False) |
|
|
|
verbosity.add_option('--no-progress', |
|
|
|
action='store_true', dest='noprogress', help='do not print progress bar', default=False) |
|
|
|
verbosity.add_option('--console-title', |
|
|
|
action='store_true', dest='consoletitle', |
|
|
|
help='display progress in console titlebar', default=False) |
|
|
|
|
|
|
|
|
|
|
|
filesystem.add_option('-t', '--title', |
|
|
|
action='store_true', dest='usetitle', help='use title in file name', default=False) |
|
|
|
filesystem.add_option('-l', '--literal', |
|
|
|
action='store_true', dest='useliteral', help='use literal title in file name', default=False) |
|
|
|
filesystem.add_option('-A', '--auto-number', |
|
|
|
action='store_true', dest='autonumber', |
|
|
|
help='number downloaded files starting from 00000', default=False) |
|
|
|
filesystem.add_option('-o', '--output', |
|
|
|
dest='outtmpl', metavar='TEMPLATE', help='output filename template') |
|
|
|
filesystem.add_option('-a', '--batch-file', |
|
|
|
dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') |
|
|
|
filesystem.add_option('-w', '--no-overwrites', |
|
|
|
action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) |
|
|
|
filesystem.add_option('-c', '--continue', |
|
|
|
action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False) |
|
|
|
filesystem.add_option('--cookies', |
|
|
|
dest='cookiefile', metavar='FILE', help='file to dump cookie jar to') |
|
|
|
filesystem.add_option('--no-part', |
|
|
|
action='store_true', dest='nopart', help='do not use .part files', default=False) |
|
|
|
filesystem.add_option('--no-mtime', |
|
|
|
action='store_false', dest='updatetime', |
|
|
|
help='do not use the Last-modified header to set the file modification time', default=True) |
|
|
|
filesystem.add_option('--write-description', |
|
|
|
action='store_true', dest='writedescription', |
|
|
|
help='write video description to a .description file', default=False) |
|
|
|
filesystem.add_option('--write-info-json', |
|
|
|
action='store_true', dest='writeinfojson', |
|
|
|
help='write video metadata to a .info.json file', default=False) |
|
|
|
|
|
|
|
|
|
|
|
postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False, |
|
|
|
help='convert video files to audio-only files (requires ffmpeg and ffprobe)') |
|
|
|
postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', |
|
|
|
help='"best", "aac" or "mp3"; best by default') |
|
|
|
|
|
|
|
|
|
|
|
parser.add_option_group(general) |
|
|
|
parser.add_option_group(filesystem) |
|
|
|
parser.add_option_group(verbosity) |
|
|
|
parser.add_option_group(video_format) |
|
|
|
parser.add_option_group(authentication) |
|
|
|
parser.add_option_group(postproc) |
|
|
|
|
|
|
|
opts, args = parser.parse_args() |
|
|
|
|
|
|
|
return parser, opts, args |
|
|
|
|
|
|
|
def main(): |
|
|
|
parser, opts, args = parseOpts() |
|
|
|
|
|
|
|
# Open appropriate CookieJar |
|
|
|
if opts.cookiefile is None: |
|
|
|
jar = cookielib.CookieJar() |
|
|
|
else: |
|
|
|
try: |
|
|
|
opts.playliststart = long(opts.playliststart) |
|
|
|
if opts.playliststart <= 0: |
|
|
|
raise ValueError |
|
|
|
except (TypeError, ValueError), err: |
|
|
|
parser.error(u'invalid playlist start number specified') |
|
|
|
jar = cookielib.MozillaCookieJar(opts.cookiefile) |
|
|
|
if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK): |
|
|
|
jar.load() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit(u'ERROR: unable to open cookie file') |
|
|
|
|
|
|
|
# Dump user agent |
|
|
|
if opts.dump_user_agent: |
|
|
|
print std_headers['User-Agent'] |
|
|
|
sys.exit(0) |
|
|
|
|
|
|
|
# General configuration |
|
|
|
cookie_processor = urllib2.HTTPCookieProcessor(jar) |
|
|
|
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())) |
|
|
|
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) |
|
|
|
|
|
|
|
# Batch file verification |
|
|
|
batchurls = [] |
|
|
|
if opts.batchfile is not None: |
|
|
|
try: |
|
|
|
opts.playlistend = long(opts.playlistend) |
|
|
|
if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): |
|
|
|
raise ValueError |
|
|
|
except (TypeError, ValueError), err: |
|
|
|
parser.error(u'invalid playlist end number specified') |
|
|
|
if opts.extractaudio: |
|
|
|
if opts.audioformat not in ['best', 'aac', 'mp3']: |
|
|
|
parser.error(u'invalid audio format specified') |
|
|
|
|
|
|
|
# Information extractors |
|
|
|
youtube_ie = YoutubeIE() |
|
|
|
metacafe_ie = MetacafeIE(youtube_ie) |
|
|
|
dailymotion_ie = DailymotionIE() |
|
|
|
youtube_pl_ie = YoutubePlaylistIE(youtube_ie) |
|
|
|
youtube_user_ie = YoutubeUserIE(youtube_ie) |
|
|
|
youtube_search_ie = YoutubeSearchIE(youtube_ie) |
|
|
|
google_ie = GoogleIE() |
|
|
|
google_search_ie = GoogleSearchIE(google_ie) |
|
|
|
photobucket_ie = PhotobucketIE() |
|
|
|
yahoo_ie = YahooIE() |
|
|
|
yahoo_search_ie = YahooSearchIE(yahoo_ie) |
|
|
|
deposit_files_ie = DepositFilesIE() |
|
|
|
facebook_ie = FacebookIE() |
|
|
|
bliptv_ie = BlipTVIE() |
|
|
|
generic_ie = GenericIE() |
|
|
|
|
|
|
|
# File downloader |
|
|
|
fd = FileDownloader({ |
|
|
|
'usenetrc': opts.usenetrc, |
|
|
|
'username': opts.username, |
|
|
|
'password': opts.password, |
|
|
|
'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename), |
|
|
|
'forceurl': opts.geturl, |
|
|
|
'forcetitle': opts.gettitle, |
|
|
|
'forcethumbnail': opts.getthumbnail, |
|
|
|
'forcedescription': opts.getdescription, |
|
|
|
'forcefilename': opts.getfilename, |
|
|
|
'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename), |
|
|
|
'format': opts.format, |
|
|
|
'format_limit': opts.format_limit, |
|
|
|
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding())) |
|
|
|
or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s') |
|
|
|
or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s') |
|
|
|
or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') |
|
|
|
or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s') |
|
|
|
or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') |
|
|
|
or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s') |
|
|
|
or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s') |
|
|
|
or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') |
|
|
|
or u'%(id)s.%(ext)s'), |
|
|
|
'ignoreerrors': opts.ignoreerrors, |
|
|
|
'ratelimit': opts.ratelimit, |
|
|
|
'nooverwrites': opts.nooverwrites, |
|
|
|
'retries': opts.retries, |
|
|
|
'continuedl': opts.continue_dl, |
|
|
|
'noprogress': opts.noprogress, |
|
|
|
'playliststart': opts.playliststart, |
|
|
|
'playlistend': opts.playlistend, |
|
|
|
'logtostderr': opts.outtmpl == '-', |
|
|
|
'consoletitle': opts.consoletitle, |
|
|
|
'nopart': opts.nopart, |
|
|
|
'updatetime': opts.updatetime, |
|
|
|
'writedescription': opts.writedescription, |
|
|
|
'writeinfojson': opts.writeinfojson, |
|
|
|
}) |
|
|
|
fd.add_info_extractor(youtube_search_ie) |
|
|
|
fd.add_info_extractor(youtube_pl_ie) |
|
|
|
fd.add_info_extractor(youtube_user_ie) |
|
|
|
fd.add_info_extractor(metacafe_ie) |
|
|
|
fd.add_info_extractor(dailymotion_ie) |
|
|
|
fd.add_info_extractor(youtube_ie) |
|
|
|
fd.add_info_extractor(google_ie) |
|
|
|
fd.add_info_extractor(google_search_ie) |
|
|
|
fd.add_info_extractor(photobucket_ie) |
|
|
|
fd.add_info_extractor(yahoo_ie) |
|
|
|
fd.add_info_extractor(yahoo_search_ie) |
|
|
|
fd.add_info_extractor(deposit_files_ie) |
|
|
|
fd.add_info_extractor(facebook_ie) |
|
|
|
fd.add_info_extractor(bliptv_ie) |
|
|
|
|
|
|
|
# This must come last since it's the |
|
|
|
# fallback if none of the others work |
|
|
|
fd.add_info_extractor(generic_ie) |
|
|
|
|
|
|
|
# PostProcessors |
|
|
|
if opts.extractaudio: |
|
|
|
fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat)) |
|
|
|
|
|
|
|
# Update version |
|
|
|
if opts.update_self: |
|
|
|
update_self(fd, sys.argv[0]) |
|
|
|
|
|
|
|
# Maybe do nothing |
|
|
|
if len(all_urls) < 1: |
|
|
|
if not opts.update_self: |
|
|
|
parser.error(u'you must provide at least one URL') |
|
|
|
if opts.batchfile == '-': |
|
|
|
batchfd = sys.stdin |
|
|
|
else: |
|
|
|
sys.exit() |
|
|
|
retcode = fd.download(all_urls) |
|
|
|
batchfd = open(opts.batchfile, 'r') |
|
|
|
batchurls = batchfd.readlines() |
|
|
|
batchurls = [x.strip() for x in batchurls] |
|
|
|
batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] |
|
|
|
except IOError: |
|
|
|
sys.exit(u'ERROR: batch file could not be read') |
|
|
|
all_urls = batchurls + args |
|
|
|
|
|
|
|
# Conflicting, missing and erroneous options |
|
|
|
if opts.usenetrc and (opts.username is not None or opts.password is not None): |
|
|
|
parser.error(u'using .netrc conflicts with giving username/password') |
|
|
|
if opts.password is not None and opts.username is None: |
|
|
|
parser.error(u'account username missing') |
|
|
|
if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber): |
|
|
|
parser.error(u'using output template conflicts with using title, literal title or auto number') |
|
|
|
if opts.usetitle and opts.useliteral: |
|
|
|
parser.error(u'using title conflicts with using literal title') |
|
|
|
if opts.username is not None and opts.password is None: |
|
|
|
opts.password = getpass.getpass(u'Type account password and press return:') |
|
|
|
if opts.ratelimit is not None: |
|
|
|
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) |
|
|
|
if numeric_limit is None: |
|
|
|
parser.error(u'invalid rate limit specified') |
|
|
|
opts.ratelimit = numeric_limit |
|
|
|
if opts.retries is not None: |
|
|
|
try: |
|
|
|
opts.retries = long(opts.retries) |
|
|
|
except (TypeError, ValueError), err: |
|
|
|
parser.error(u'invalid retry count specified') |
|
|
|
try: |
|
|
|
opts.playliststart = int(opts.playliststart) |
|
|
|
if opts.playliststart <= 0: |
|
|
|
raise ValueError(u'Playlist start must be positive') |
|
|
|
except (TypeError, ValueError), err: |
|
|
|
parser.error(u'invalid playlist start number specified') |
|
|
|
try: |
|
|
|
opts.playlistend = int(opts.playlistend) |
|
|
|
if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): |
|
|
|
raise ValueError(u'Playlist end must be greater than playlist start') |
|
|
|
except (TypeError, ValueError), err: |
|
|
|
parser.error(u'invalid playlist end number specified') |
|
|
|
if opts.extractaudio: |
|
|
|
if opts.audioformat not in ['best', 'aac', 'mp3']: |
|
|
|
parser.error(u'invalid audio format specified') |
|
|
|
|
|
|
|
# Information extractors |
|
|
|
youtube_ie = YoutubeIE() |
|
|
|
metacafe_ie = MetacafeIE(youtube_ie) |
|
|
|
dailymotion_ie = DailymotionIE() |
|
|
|
youtube_pl_ie = YoutubePlaylistIE(youtube_ie) |
|
|
|
youtube_user_ie = YoutubeUserIE(youtube_ie) |
|
|
|
youtube_search_ie = YoutubeSearchIE(youtube_ie) |
|
|
|
google_ie = GoogleIE() |
|
|
|
google_search_ie = GoogleSearchIE(google_ie) |
|
|
|
photobucket_ie = PhotobucketIE() |
|
|
|
yahoo_ie = YahooIE() |
|
|
|
yahoo_search_ie = YahooSearchIE(yahoo_ie) |
|
|
|
deposit_files_ie = DepositFilesIE() |
|
|
|
facebook_ie = FacebookIE() |
|
|
|
bliptv_ie = BlipTVIE() |
|
|
|
generic_ie = GenericIE() |
|
|
|
|
|
|
|
# File downloader |
|
|
|
fd = FileDownloader({ |
|
|
|
'usenetrc': opts.usenetrc, |
|
|
|
'username': opts.username, |
|
|
|
'password': opts.password, |
|
|
|
'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename), |
|
|
|
'forceurl': opts.geturl, |
|
|
|
'forcetitle': opts.gettitle, |
|
|
|
'forcethumbnail': opts.getthumbnail, |
|
|
|
'forcedescription': opts.getdescription, |
|
|
|
'forcefilename': opts.getfilename, |
|
|
|
'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename), |
|
|
|
'format': opts.format, |
|
|
|
'format_limit': opts.format_limit, |
|
|
|
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding())) |
|
|
|
or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s') |
|
|
|
or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s') |
|
|
|
or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') |
|
|
|
or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s') |
|
|
|
or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') |
|
|
|
or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s') |
|
|
|
or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s') |
|
|
|
or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') |
|
|
|
or u'%(id)s.%(ext)s'), |
|
|
|
'ignoreerrors': opts.ignoreerrors, |
|
|
|
'ratelimit': opts.ratelimit, |
|
|
|
'nooverwrites': opts.nooverwrites, |
|
|
|
'retries': opts.retries, |
|
|
|
'continuedl': opts.continue_dl, |
|
|
|
'noprogress': opts.noprogress, |
|
|
|
'playliststart': opts.playliststart, |
|
|
|
'playlistend': opts.playlistend, |
|
|
|
'logtostderr': opts.outtmpl == '-', |
|
|
|
'consoletitle': opts.consoletitle, |
|
|
|
'nopart': opts.nopart, |
|
|
|
'updatetime': opts.updatetime, |
|
|
|
'writedescription': opts.writedescription, |
|
|
|
'writeinfojson': opts.writeinfojson, |
|
|
|
}) |
|
|
|
fd.add_info_extractor(youtube_search_ie) |
|
|
|
fd.add_info_extractor(youtube_pl_ie) |
|
|
|
fd.add_info_extractor(youtube_user_ie) |
|
|
|
fd.add_info_extractor(metacafe_ie) |
|
|
|
fd.add_info_extractor(dailymotion_ie) |
|
|
|
fd.add_info_extractor(youtube_ie) |
|
|
|
fd.add_info_extractor(google_ie) |
|
|
|
fd.add_info_extractor(google_search_ie) |
|
|
|
fd.add_info_extractor(photobucket_ie) |
|
|
|
fd.add_info_extractor(yahoo_ie) |
|
|
|
fd.add_info_extractor(yahoo_search_ie) |
|
|
|
fd.add_info_extractor(deposit_files_ie) |
|
|
|
fd.add_info_extractor(facebook_ie) |
|
|
|
fd.add_info_extractor(bliptv_ie) |
|
|
|
|
|
|
|
# This must come last since it's the |
|
|
|
# fallback if none of the others work |
|
|
|
fd.add_info_extractor(generic_ie) |
|
|
|
|
|
|
|
# PostProcessors |
|
|
|
if opts.extractaudio: |
|
|
|
fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat)) |
|
|
|
|
|
|
|
# Update version |
|
|
|
if opts.update_self: |
|
|
|
updateSelf(fd, sys.argv[0]) |
|
|
|
|
|
|
|
# Maybe do nothing |
|
|
|
if len(all_urls) < 1: |
|
|
|
if not opts.update_self: |
|
|
|
parser.error(u'you must provide at least one URL') |
|
|
|
else: |
|
|
|
sys.exit() |
|
|
|
retcode = fd.download(all_urls) |
|
|
|
|
|
|
|
# Dump cookie jar if requested |
|
|
|
if opts.cookiefile is not None: |
|
|
|
try: |
|
|
|
jar.save() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit(u'ERROR: unable to save cookie jar') |
|
|
|
# Dump cookie jar if requested |
|
|
|
if opts.cookiefile is not None: |
|
|
|
try: |
|
|
|
jar.save() |
|
|
|
except (IOError, OSError), err: |
|
|
|
sys.exit(u'ERROR: unable to save cookie jar') |
|
|
|
|
|
|
|
sys.exit(retcode) |
|
|
|
|
|
|
|
sys.exit(retcode) |
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
try: |
|
|
|
main() |
|
|
|
except DownloadError: |
|
|
|
sys.exit(1) |
|
|
|
except SameFileError: |
|
|
|
sys.exit(u'ERROR: fixed output name but more than one file to download') |
|
|
|
except KeyboardInterrupt: |
|
|
|
sys.exit(u'\nERROR: Interrupted by user') |
|
|
|
|
|
|
|
# vim: set ts=4 sw=4 sts=4 noet ai si filetype=python: |
Philipp Hagemeister
13 years ago