1
0
mirror of https://code.hackerspace.pl/q3k/youtube-dl synced 2025-03-16 11:43:02 +00:00

Merge branch 'master' into prefer-webm

This commit is contained in:
Rogério Brito 2011-01-29 03:45:21 -02:00
commit 9e1ee3364a
2 changed files with 395 additions and 147 deletions

View File

@ -1 +1 @@
2010.11.19 2010.12.09

View File

@ -3,8 +3,14 @@
# Author: Ricardo Garcia Gonzalez # Author: Ricardo Garcia Gonzalez
# Author: Danny Colligan # Author: Danny Colligan
# Author: Benjamin Johnson # Author: Benjamin Johnson
# Author: Vasyl' Vavrychuk
# Author: Witold Baryluk
# License: Public domain code # License: Public domain code
import cookielib import cookielib
import ctypes
import datetime
import email.utils
import gzip
import htmlentitydefs import htmlentitydefs
import httplib import httplib
import locale import locale
@ -15,11 +21,13 @@ import os.path
import re import re
import socket import socket
import string import string
import StringIO
import subprocess import subprocess
import sys import sys
import time import time
import urllib import urllib
import urllib2 import urllib2
import zlib
# parse_qs was moved from the cgi module to the urlparse module recently. # parse_qs was moved from the cgi module to the urlparse module recently.
try: try:
@ -31,26 +39,12 @@ std_headers = {
'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12', 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-us,en;q=0.5', 'Accept-Language': 'en-us,en;q=0.5',
} }
simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii') simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
month_name_to_number = {
'January': '01',
'February': '02',
'March': '03',
'April': '04',
'May': '05',
'June': '06',
'July': '07',
'August': '08',
'September': '09',
'October': '10',
'November': '11',
'December': '12',
}
def preferredencoding(): def preferredencoding():
"""Get preferred encoding. """Get preferred encoding.
@ -69,7 +63,7 @@ def preferredencoding():
def htmlentity_transform(matchobj): def htmlentity_transform(matchobj):
"""Transforms an HTML entity to a Unicode character. """Transforms an HTML entity to a Unicode character.
This function receives a match object and is intended to be used with This function receives a match object and is intended to be used with
the re.sub() function. the re.sub() function.
""" """
@ -124,10 +118,17 @@ def sanitize_open(filename, open_mode):
stream = open(filename, open_mode) stream = open(filename, open_mode)
return (stream, filename) return (stream, filename)
def timeconvert(timestr):
"""Convert RFC 2822 defined time string into system timestamp"""
timestamp = None
timetuple = email.utils.parsedate_tz(timestr)
if timetuple is not None:
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
class DownloadError(Exception): class DownloadError(Exception):
"""Download Error exception. """Download Error exception.
This exception may be thrown by FileDownloader objects if they are not This exception may be thrown by FileDownloader objects if they are not
configured to continue on errors. They will contain the appropriate configured to continue on errors. They will contain the appropriate
error message. error message.
@ -173,6 +174,64 @@ class ContentTooShortError(Exception):
self.downloaded = downloaded self.downloaded = downloaded
self.expected = expected self.expected = expected
class YoutubeDLHandler(urllib2.HTTPHandler):
"""Handler for HTTP requests and responses.
This class, when installed with an OpenerDirector, automatically adds
the standard headers to every HTTP request and handles gzipped and
deflated responses from web servers. If compression is to be avoided in
a particular request, the original request in the program code only has
to include the HTTP header "Youtubedl-No-Compression", which will be
removed before making the real request.
Part of this code was copied from:
http://techknack.net/python-urllib2-handlers/
Andrew Rowls, the author of that code, agreed to release it to the
public domain.
"""
@staticmethod
def deflate(data):
try:
return zlib.decompress(data, -zlib.MAX_WBITS)
except zlib.error:
return zlib.decompress(data)
@staticmethod
def addinfourl_wrapper(stream, headers, url, code):
if hasattr(urllib2.addinfourl, 'getcode'):
return urllib2.addinfourl(stream, headers, url, code)
ret = urllib2.addinfourl(stream, headers, url)
ret.code = code
return ret
def http_request(self, req):
for h in std_headers:
if h in req.headers:
del req.headers[h]
req.add_header(h, std_headers[h])
if 'Youtubedl-no-compression' in req.headers:
if 'Accept-encoding' in req.headers:
del req.headers['Accept-encoding']
del req.headers['Youtubedl-no-compression']
return req
def http_response(self, req, resp):
old_resp = resp
# gzip
if resp.headers.get('Content-encoding', '') == 'gzip':
gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
# deflate
if resp.headers.get('Content-encoding', '') == 'deflate':
gz = StringIO.StringIO(self.deflate(resp.read()))
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
return resp
class FileDownloader(object): class FileDownloader(object):
"""File Downloader class. """File Downloader class.
@ -208,6 +267,7 @@ class FileDownloader(object):
forcetitle: Force printing title. forcetitle: Force printing title.
forcethumbnail: Force printing thumbnail URL. forcethumbnail: Force printing thumbnail URL.
forcedescription: Force printing description. forcedescription: Force printing description.
forcefilename: Force printing final filename.
simulate: Do not download the video files. simulate: Do not download the video files.
format: Video format code. format: Video format code.
format_limit: Highest quality format to try. format_limit: Highest quality format to try.
@ -221,6 +281,9 @@ class FileDownloader(object):
playliststart: Playlist item to start at. playliststart: Playlist item to start at.
playlistend: Playlist item to end at. playlistend: Playlist item to end at.
logtostderr: Log messages to stderr instead of stdout. logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
nopart: Do not use temporary .part files.
updatetime: Use the Last-modified header to set output file timestamps.
""" """
params = None params = None
@ -238,7 +301,7 @@ class FileDownloader(object):
self._num_downloads = 0 self._num_downloads = 0
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self.params = params self.params = params
@staticmethod @staticmethod
def pmkdir(filename): def pmkdir(filename):
"""Create directory components in filename. Similar to Unix "mkdir -p".""" """Create directory components in filename. Similar to Unix "mkdir -p"."""
@ -248,7 +311,7 @@ class FileDownloader(object):
for dir in aggregate: for dir in aggregate:
if not os.path.exists(dir): if not os.path.exists(dir):
os.mkdir(dir) os.mkdir(dir)
@staticmethod @staticmethod
def format_bytes(bytes): def format_bytes(bytes):
if bytes is None: if bytes is None:
@ -317,12 +380,12 @@ class FileDownloader(object):
"""Add an InfoExtractor object to the end of the list.""" """Add an InfoExtractor object to the end of the list."""
self._ies.append(ie) self._ies.append(ie)
ie.set_downloader(self) ie.set_downloader(self)
def add_post_processor(self, pp): def add_post_processor(self, pp):
"""Add a PostProcessor object to the end of the chain.""" """Add a PostProcessor object to the end of the chain."""
self._pps.append(pp) self._pps.append(pp)
pp.set_downloader(self) pp.set_downloader(self)
def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False): def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
"""Print message to stdout if not in quiet mode.""" """Print message to stdout if not in quiet mode."""
try: try:
@ -333,11 +396,22 @@ class FileDownloader(object):
except (UnicodeEncodeError), err: except (UnicodeEncodeError), err:
if not ignore_encoding_errors: if not ignore_encoding_errors:
raise raise
def to_stderr(self, message): def to_stderr(self, message):
"""Print message to stderr.""" """Print message to stderr."""
print >>sys.stderr, message.encode(preferredencoding()) print >>sys.stderr, message.encode(preferredencoding())
def to_cons_title(self, message):
"""Set console/terminal window title to message."""
if not self.params.get('consoletitle', False):
return
if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
# c_wchar_p() might not be necessary if `message` is
# already of type unicode()
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
elif 'TERM' in os.environ:
sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
def fixed_template(self): def fixed_template(self):
"""Checks if the output template is fixed.""" """Checks if the output template is fixed."""
return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None) return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
@ -368,49 +442,101 @@ class FileDownloader(object):
if speed > rate_limit: if speed > rate_limit:
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
def temp_name(self, filename):
"""Returns a temporary filename for the given filename."""
if self.params.get('nopart', False) or filename == u'-' or \
(os.path.exists(filename) and not os.path.isfile(filename)):
return filename
return filename + u'.part'
def undo_temp_name(self, filename):
if filename.endswith(u'.part'):
return filename[:-len(u'.part')]
return filename
def try_rename(self, old_filename, new_filename):
try:
if old_filename == new_filename:
return
os.rename(old_filename, new_filename)
except (IOError, OSError), err:
self.trouble(u'ERROR: unable to rename file')
def try_utime(self, filename, last_modified_hdr):
"""Try to set the last-modified time of the given file."""
if last_modified_hdr is None:
return
if not os.path.isfile(filename):
return
timestr = last_modified_hdr
if timestr is None:
return
filetime = timeconvert(timestr)
if filetime is None:
return
try:
os.utime(filename,(time.time(), filetime))
except:
pass
def report_destination(self, filename): def report_destination(self, filename):
"""Report destination filename.""" """Report destination filename."""
self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True) self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
def report_progress(self, percent_str, data_len_str, speed_str, eta_str): def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
"""Report download progress.""" """Report download progress."""
if self.params.get('noprogress', False): if self.params.get('noprogress', False):
return return
self.to_screen(u'\r[download] %s of %s at %s ETA %s' % self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
(percent_str, data_len_str, speed_str, eta_str), skip_eol=True) (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
def report_resuming_byte(self, resume_len): def report_resuming_byte(self, resume_len):
"""Report attempt to resume at given byte.""" """Report attempt to resume at given byte."""
self.to_screen(u'[download] Resuming download at byte %s' % resume_len) self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
def report_retry(self, count, retries): def report_retry(self, count, retries):
"""Report retry in case of HTTP error 5xx""" """Report retry in case of HTTP error 5xx"""
self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
def report_file_already_downloaded(self, file_name): def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded.""" """Report file has already been fully downloaded."""
try: try:
self.to_screen(u'[download] %s has already been downloaded' % file_name) self.to_screen(u'[download] %s has already been downloaded' % file_name)
except (UnicodeEncodeError), err: except (UnicodeEncodeError), err:
self.to_screen(u'[download] The file has already been downloaded') self.to_screen(u'[download] The file has already been downloaded')
def report_unable_to_resume(self): def report_unable_to_resume(self):
"""Report it was impossible to resume download.""" """Report it was impossible to resume download."""
self.to_screen(u'[download] Unable to resume') self.to_screen(u'[download] Unable to resume')
def report_finish(self): def report_finish(self):
"""Report download finished.""" """Report download finished."""
if self.params.get('noprogress', False): if self.params.get('noprogress', False):
self.to_screen(u'[download] Download completed') self.to_screen(u'[download] Download completed')
else: else:
self.to_screen(u'') self.to_screen(u'')
def increment_downloads(self): def increment_downloads(self):
"""Increment the ordinal that assigns a number to each file.""" """Increment the ordinal that assigns a number to each file."""
self._num_downloads += 1 self._num_downloads += 1
def prepare_filename(self, info_dict):
"""Generate the output filename."""
try:
template_dict = dict(info_dict)
template_dict['epoch'] = unicode(long(time.time()))
template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
filename = self.params['outtmpl'] % template_dict
return filename
except (ValueError, KeyError), err:
self.trouble(u'ERROR: invalid system charset or erroneous output template')
return None
def process_info(self, info_dict): def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor.""" """Process a single dictionary returned by an InfoExtractor."""
filename = self.prepare_filename(info_dict)
# Do nothing else if in simulate mode # Do nothing else if in simulate mode
if self.params.get('simulate', False): if self.params.get('simulate', False):
# Forced printings # Forced printings
@ -422,16 +548,12 @@ class FileDownloader(object):
print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace') print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
if self.params.get('forcedescription', False) and 'description' in info_dict: if self.params.get('forcedescription', False) and 'description' in info_dict:
print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace') print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
if self.params.get('forcefilename', False) and filename is not None:
print filename.encode(preferredencoding(), 'xmlcharrefreplace')
return return
try: if filename is None:
template_dict = dict(info_dict)
template_dict['epoch'] = unicode(long(time.time()))
template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
filename = self.params['outtmpl'] % template_dict
except (ValueError, KeyError), err:
self.trouble(u'ERROR: invalid system charset or erroneous output template')
return return
if self.params.get('nooverwrites', False) and os.path.exists(filename): if self.params.get('nooverwrites', False) and os.path.exists(filename):
self.to_stderr(u'WARNING: file exists and will be skipped') self.to_stderr(u'WARNING: file exists and will be skipped')
@ -495,9 +617,10 @@ class FileDownloader(object):
info = pp.run(info) info = pp.run(info)
if info is None: if info is None:
break break
def _download_with_rtmpdump(self, filename, url, player_url): def _download_with_rtmpdump(self, filename, url, player_url):
self.report_destination(filename) self.report_destination(filename)
tmpfilename = self.temp_name(filename)
# Check for rtmpdump first # Check for rtmpdump first
try: try:
@ -509,36 +632,46 @@ class FileDownloader(object):
# Download using rtmpdump. rtmpdump returns exit code 2 when # Download using rtmpdump. rtmpdump returns exit code 2 when
# the connection was interrumpted and resuming appears to be # the connection was interrumpted and resuming appears to be
# possible. This is part of rtmpdump's normal usage, AFAIK. # possible. This is part of rtmpdump's normal usage, AFAIK.
basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename] basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]) retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
while retval == 2 or retval == 1: while retval == 2 or retval == 1:
prevsize = os.path.getsize(filename) prevsize = os.path.getsize(tmpfilename)
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
time.sleep(5.0) # This seems to be needed time.sleep(5.0) # This seems to be needed
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
cursize = os.path.getsize(filename) cursize = os.path.getsize(tmpfilename)
if prevsize == cursize and retval == 1: if prevsize == cursize and retval == 1:
break break
if retval == 0: if retval == 0:
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename)) self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
self.try_rename(tmpfilename, filename)
return True return True
else: else:
self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
return False return False
def _do_download(self, filename, url, player_url): def _do_download(self, filename, url, player_url):
# Check file already present
if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
self.report_file_already_downloaded(filename)
return True
# Attempt to download using rtmpdump # Attempt to download using rtmpdump
if url.startswith('rtmp'): if url.startswith('rtmp'):
return self._download_with_rtmpdump(filename, url, player_url) return self._download_with_rtmpdump(filename, url, player_url)
tmpfilename = self.temp_name(filename)
stream = None stream = None
open_mode = 'wb' open_mode = 'wb'
basic_request = urllib2.Request(url, None, std_headers)
request = urllib2.Request(url, None, std_headers) # Do not include the Accept-Encoding header
headers = {'Youtubedl-no-compression': 'True'}
basic_request = urllib2.Request(url, None, headers)
request = urllib2.Request(url, None, headers)
# Establish possible resume length # Establish possible resume length
if os.path.isfile(filename): if os.path.isfile(tmpfilename):
resume_len = os.path.getsize(filename) resume_len = os.path.getsize(tmpfilename)
else: else:
resume_len = 0 resume_len = 0
@ -580,6 +713,7 @@ class FileDownloader(object):
# completely downloaded if the file size differs less than 100 bytes from # completely downloaded if the file size differs less than 100 bytes from
# the one in the hard drive. # the one in the hard drive.
self.report_file_already_downloaded(filename) self.report_file_already_downloaded(filename)
self.try_rename(tmpfilename, filename)
return True return True
else: else:
# The length does not match, we start the download over # The length does not match, we start the download over
@ -596,8 +730,10 @@ class FileDownloader(object):
return False return False
data_len = data.info().get('Content-length', None) data_len = data.info().get('Content-length', None)
if data_len is not None:
data_len = long(data_len) + resume_len
data_len_str = self.format_bytes(data_len) data_len_str = self.format_bytes(data_len)
byte_counter = 0 byte_counter = 0 + resume_len
block_size = 1024 block_size = 1024
start = time.time() start = time.time()
while True: while True:
@ -605,15 +741,15 @@ class FileDownloader(object):
before = time.time() before = time.time()
data_block = data.read(block_size) data_block = data.read(block_size)
after = time.time() after = time.time()
data_block_len = len(data_block) if len(data_block) == 0:
if data_block_len == 0:
break break
byte_counter += data_block_len byte_counter += len(data_block)
# Open file just in time # Open file just in time
if stream is None: if stream is None:
try: try:
(stream, filename) = sanitize_open(filename, open_mode) (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
filename = self.undo_temp_name(tmpfilename)
self.report_destination(filename) self.report_destination(filename)
except (OSError, IOError), err: except (OSError, IOError), err:
self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
@ -623,20 +759,27 @@ class FileDownloader(object):
except (IOError, OSError), err: except (IOError, OSError), err:
self.trouble(u'\nERROR: unable to write data: %s' % str(err)) self.trouble(u'\nERROR: unable to write data: %s' % str(err))
return False return False
block_size = self.best_block_size(after - before, data_block_len) block_size = self.best_block_size(after - before, len(data_block))
# Progress message # Progress message
percent_str = self.calc_percent(byte_counter, data_len) percent_str = self.calc_percent(byte_counter, data_len)
eta_str = self.calc_eta(start, time.time(), data_len, byte_counter) eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
speed_str = self.calc_speed(start, time.time(), byte_counter) speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
self.report_progress(percent_str, data_len_str, speed_str, eta_str) self.report_progress(percent_str, data_len_str, speed_str, eta_str)
# Apply rate limit # Apply rate limit
self.slow_down(start, byte_counter) self.slow_down(start, byte_counter - resume_len)
stream.close()
self.report_finish() self.report_finish()
if data_len is not None and str(byte_counter) != data_len: if data_len is not None and byte_counter != data_len:
raise ContentTooShortError(byte_counter, long(data_len)) raise ContentTooShortError(byte_counter, long(data_len))
self.try_rename(tmpfilename, filename)
# Update file modification time
if self.params.get('updatetime', True):
self.try_utime(filename, data.info().get('last-modified', None))
return True return True
class InfoExtractor(object): class InfoExtractor(object):
@ -701,7 +844,7 @@ class InfoExtractor(object):
def set_downloader(self, downloader): def set_downloader(self, downloader):
"""Sets the downloader for this IE.""" """Sets the downloader for this IE."""
self._downloader = downloader self._downloader = downloader
def _real_initialize(self): def _real_initialize(self):
"""Real initialization process. Redefine in subclasses.""" """Real initialization process. Redefine in subclasses."""
pass pass
@ -713,7 +856,7 @@ class InfoExtractor(object):
class YoutubeIE(InfoExtractor): class YoutubeIE(InfoExtractor):
"""Information extractor for youtube.com.""" """Information extractor for youtube.com."""
_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$' _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en' _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
@ -742,31 +885,31 @@ class YoutubeIE(InfoExtractor):
def report_login(self): def report_login(self):
"""Report attempt to log in.""" """Report attempt to log in."""
self._downloader.to_screen(u'[youtube] Logging in') self._downloader.to_screen(u'[youtube] Logging in')
def report_age_confirmation(self): def report_age_confirmation(self):
"""Report attempt to confirm age.""" """Report attempt to confirm age."""
self._downloader.to_screen(u'[youtube] Confirming age') self._downloader.to_screen(u'[youtube] Confirming age')
def report_video_webpage_download(self, video_id): def report_video_webpage_download(self, video_id):
"""Report attempt to download video webpage.""" """Report attempt to download video webpage."""
self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id) self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
def report_video_info_webpage_download(self, video_id): def report_video_info_webpage_download(self, video_id):
"""Report attempt to download video info webpage.""" """Report attempt to download video info webpage."""
self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id) self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
def report_information_extraction(self, video_id): def report_information_extraction(self, video_id):
"""Report attempt to extract video information.""" """Report attempt to extract video information."""
self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id) self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
def report_unavailable_format(self, video_id, format): def report_unavailable_format(self, video_id, format):
"""Report extracted video URL.""" """Report extracted video URL."""
self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format)) self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
def report_rtmp_download(self): def report_rtmp_download(self):
"""Indicate the download will use the RTMP protocol.""" """Indicate the download will use the RTMP protocol."""
self._downloader.to_screen(u'[youtube] RTMP download detected') self._downloader.to_screen(u'[youtube] RTMP download detected')
def _real_initialize(self): def _real_initialize(self):
if self._downloader is None: if self._downloader is None:
return return
@ -792,7 +935,7 @@ class YoutubeIE(InfoExtractor):
return return
# Set language # Set language
request = urllib2.Request(self._LANG_URL, None, std_headers) request = urllib2.Request(self._LANG_URL)
try: try:
self.report_lang() self.report_lang()
urllib2.urlopen(request).read() urllib2.urlopen(request).read()
@ -812,7 +955,7 @@ class YoutubeIE(InfoExtractor):
'username': username, 'username': username,
'password': password, 'password': password,
} }
request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers) request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
try: try:
self.report_login() self.report_login()
login_results = urllib2.urlopen(request).read() login_results = urllib2.urlopen(request).read()
@ -822,13 +965,13 @@ class YoutubeIE(InfoExtractor):
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err)) self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
return return
# Confirm age # Confirm age
age_form = { age_form = {
'next_url': '/', 'next_url': '/',
'action_confirm': 'Confirm', 'action_confirm': 'Confirm',
} }
request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers) request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
try: try:
self.report_age_confirmation() self.report_age_confirmation()
age_results = urllib2.urlopen(request).read() age_results = urllib2.urlopen(request).read()
@ -846,7 +989,7 @@ class YoutubeIE(InfoExtractor):
# Get video webpage # Get video webpage
self.report_video_webpage_download(video_id) self.report_video_webpage_download(video_id)
request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id, None, std_headers) request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
try: try:
video_webpage = urllib2.urlopen(request).read() video_webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@ -865,7 +1008,7 @@ class YoutubeIE(InfoExtractor):
for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']: for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
% (video_id, el_type)) % (video_id, el_type))
request = urllib2.Request(video_info_url, None, std_headers) request = urllib2.Request(video_info_url)
try: try:
video_info_webpage = urllib2.urlopen(request).read() video_info_webpage = urllib2.urlopen(request).read()
video_info = parse_qs(video_info_webpage) video_info = parse_qs(video_info_webpage)
@ -913,18 +1056,13 @@ class YoutubeIE(InfoExtractor):
upload_date = u'NA' upload_date = u'NA'
mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL) mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL)
if mobj is not None: if mobj is not None:
try: upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
if ',' in mobj.group(1): format_expressions = ['%d %B %Y', '%B %d %Y']
# Month Day, Year for expression in format_expressions:
m, d, y = mobj.group(1).replace(',', '').split() try:
else: upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
# Day Month Year, we'll suppose except:
d, m, y = mobj.group(1).split() pass
m = month_name_to_number[m]
d = '%02d' % (long(d))
upload_date = '%s%s%s' % (y, m, d)
except:
upload_date = u'NA'
# description # description
video_description = 'No description available.' video_description = 'No description available.'
@ -937,8 +1075,7 @@ class YoutubeIE(InfoExtractor):
video_token = urllib.unquote_plus(video_info['token'][0]) video_token = urllib.unquote_plus(video_info['token'][0])
# Decide which formats to download # Decide which formats to download
requested_format = self._downloader.params.get('format', None) req_format = self._downloader.params.get('format', None)
get_video_template = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=&ps=&asv=&fmt=%%s' % (video_id, video_token)
if 'fmt_url_map' in video_info: if 'fmt_url_map' in video_info:
url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(',')) url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(','))
@ -951,12 +1088,16 @@ class YoutubeIE(InfoExtractor):
if len(existing_formats) == 0: if len(existing_formats) == 0:
self._downloader.trouble(u'ERROR: no known formats available for video') self._downloader.trouble(u'ERROR: no known formats available for video')
return return
if requested_format is None: if req_format is None:
video_url_list = [(existing_formats[0], get_video_template % existing_formats[0])] # Best quality video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
elif requested_format == '-1': elif req_format == '-1':
video_url_list = [(f, get_video_template % f) for f in existing_formats] # All formats video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
else: else:
video_url_list = [(requested_format, get_video_template % requested_format)] # Specific format # Specific format
if req_format not in url_map:
self._downloader.trouble(u'ERROR: requested format not available')
return
video_url_list = [(req_format, url_map[req_format])] # Specific format
elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'): elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
self.report_rtmp_download() self.report_rtmp_download()
@ -990,7 +1131,7 @@ class YoutubeIE(InfoExtractor):
'player_url': player_url, 'player_url': player_url,
}) })
except UnavailableVideoError, err: except UnavailableVideoError, err:
self._downloader.trouble(u'ERROR: unable to download video (format may not be available)') self._downloader.trouble(u'\nERROR: unable to download video')
class MetacafeIE(InfoExtractor): class MetacafeIE(InfoExtractor):
@ -1016,18 +1157,18 @@ class MetacafeIE(InfoExtractor):
def report_age_confirmation(self): def report_age_confirmation(self):
"""Report attempt to confirm age.""" """Report attempt to confirm age."""
self._downloader.to_screen(u'[metacafe] Confirming age') self._downloader.to_screen(u'[metacafe] Confirming age')
def report_download_webpage(self, video_id): def report_download_webpage(self, video_id):
"""Report webpage download.""" """Report webpage download."""
self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id) self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
def report_extraction(self, video_id): def report_extraction(self, video_id):
"""Report information extraction.""" """Report information extraction."""
self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id) self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
def _real_initialize(self): def _real_initialize(self):
# Retrieve disclaimer # Retrieve disclaimer
request = urllib2.Request(self._DISCLAIMER, None, std_headers) request = urllib2.Request(self._DISCLAIMER)
try: try:
self.report_disclaimer() self.report_disclaimer()
disclaimer = urllib2.urlopen(request).read() disclaimer = urllib2.urlopen(request).read()
@ -1040,14 +1181,14 @@ class MetacafeIE(InfoExtractor):
'filters': '0', 'filters': '0',
'submit': "Continue - I'm over 18", 'submit': "Continue - I'm over 18",
} }
request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers) request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
try: try:
self.report_age_confirmation() self.report_age_confirmation()
disclaimer = urllib2.urlopen(request).read() disclaimer = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err)) self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
return return
def _real_extract(self, url): def _real_extract(self, url):
# Extract id and simplified title from URL # Extract id and simplified title from URL
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
@ -1083,7 +1224,7 @@ class MetacafeIE(InfoExtractor):
if mobj is not None: if mobj is not None:
mediaURL = urllib.unquote(mobj.group(1)) mediaURL = urllib.unquote(mobj.group(1))
video_extension = mediaURL[-3:] video_extension = mediaURL[-3:]
# Extract gdaKey if available # Extract gdaKey if available
mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
if mobj is None: if mobj is None:
@ -1135,7 +1276,7 @@ class MetacafeIE(InfoExtractor):
'player_url': None, 'player_url': None,
}) })
except UnavailableVideoError: except UnavailableVideoError:
self._downloader.trouble(u'ERROR: unable to download video') self._downloader.trouble(u'\nERROR: unable to download video')
class DailymotionIE(InfoExtractor): class DailymotionIE(InfoExtractor):
@ -1153,7 +1294,7 @@ class DailymotionIE(InfoExtractor):
def report_download_webpage(self, video_id): def report_download_webpage(self, video_id):
"""Report webpage download.""" """Report webpage download."""
self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id) self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
def report_extraction(self, video_id): def report_extraction(self, video_id):
"""Report information extraction.""" """Report information extraction."""
self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id) self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
@ -1204,7 +1345,7 @@ class DailymotionIE(InfoExtractor):
video_title = mobj.group(1).decode('utf-8') video_title = mobj.group(1).decode('utf-8')
video_title = sanitize_title(video_title) video_title = sanitize_title(video_title)
mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a>', webpage) mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
if mobj is None: if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract uploader nickname') self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
return return
@ -1224,7 +1365,7 @@ class DailymotionIE(InfoExtractor):
'player_url': None, 'player_url': None,
}) })
except UnavailableVideoError: except UnavailableVideoError:
self._downloader.trouble(u'ERROR: unable to download video') self._downloader.trouble(u'\nERROR: unable to download video')
class GoogleIE(InfoExtractor): class GoogleIE(InfoExtractor):
"""Information extractor for video.google.com.""" """Information extractor for video.google.com."""
@ -1334,7 +1475,7 @@ class GoogleIE(InfoExtractor):
'player_url': None, 'player_url': None,
}) })
except UnavailableVideoError: except UnavailableVideoError:
self._downloader.trouble(u'ERROR: unable to download video') self._downloader.trouble(u'\nERROR: unable to download video')
class PhotobucketIE(InfoExtractor): class PhotobucketIE(InfoExtractor):
@ -1416,7 +1557,7 @@ class PhotobucketIE(InfoExtractor):
'player_url': None, 'player_url': None,
}) })
except UnavailableVideoError: except UnavailableVideoError:
self._downloader.trouble(u'ERROR: unable to download video') self._downloader.trouble(u'\nERROR: unable to download video')
class YahooIE(InfoExtractor): class YahooIE(InfoExtractor):
@ -1574,7 +1715,7 @@ class YahooIE(InfoExtractor):
'player_url': None, 'player_url': None,
}) })
except UnavailableVideoError: except UnavailableVideoError:
self._downloader.trouble(u'ERROR: unable to download video') self._downloader.trouble(u'\nERROR: unable to download video')
class GenericIE(InfoExtractor): class GenericIE(InfoExtractor):
@ -1617,6 +1758,7 @@ class GenericIE(InfoExtractor):
self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
return return
self.report_extraction(video_id)
# Start with something easy: JW Player in SWFObject # Start with something easy: JW Player in SWFObject
mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
if mobj is None: if mobj is None:
@ -1674,7 +1816,7 @@ class GenericIE(InfoExtractor):
'player_url': None, 'player_url': None,
}) })
except UnavailableVideoError, err: except UnavailableVideoError, err:
self._downloader.trouble(u'ERROR: unable to download video') self._downloader.trouble(u'\nERROR: unable to download video')
class YoutubeSearchIE(InfoExtractor): class YoutubeSearchIE(InfoExtractor):
@ -1689,7 +1831,7 @@ class YoutubeSearchIE(InfoExtractor):
def __init__(self, youtube_ie, downloader=None): def __init__(self, youtube_ie, downloader=None):
InfoExtractor.__init__(self, downloader) InfoExtractor.__init__(self, downloader)
self._youtube_ie = youtube_ie self._youtube_ie = youtube_ie
@staticmethod @staticmethod
def suitable(url): def suitable(url):
return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None) return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
@ -1701,7 +1843,7 @@ class YoutubeSearchIE(InfoExtractor):
def _real_initialize(self): def _real_initialize(self):
self._youtube_ie.initialize() self._youtube_ie.initialize()
def _real_extract(self, query): def _real_extract(self, query):
mobj = re.match(self._VALID_QUERY, query) mobj = re.match(self._VALID_QUERY, query)
if mobj is None: if mobj is None:
@ -1742,7 +1884,7 @@ class YoutubeSearchIE(InfoExtractor):
while True: while True:
self.report_download_page(query, pagenum) self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
request = urllib2.Request(result_url, None, std_headers) request = urllib2.Request(result_url)
try: try:
page = urllib2.urlopen(request).read() page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@ -1780,7 +1922,7 @@ class GoogleSearchIE(InfoExtractor):
def __init__(self, google_ie, downloader=None): def __init__(self, google_ie, downloader=None):
InfoExtractor.__init__(self, downloader) InfoExtractor.__init__(self, downloader)
self._google_ie = google_ie self._google_ie = google_ie
@staticmethod @staticmethod
def suitable(url): def suitable(url):
return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None) return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
@ -1792,7 +1934,7 @@ class GoogleSearchIE(InfoExtractor):
def _real_initialize(self): def _real_initialize(self):
self._google_ie.initialize() self._google_ie.initialize()
def _real_extract(self, query): def _real_extract(self, query):
mobj = re.match(self._VALID_QUERY, query) mobj = re.match(self._VALID_QUERY, query)
if mobj is None: if mobj is None:
@ -1833,7 +1975,7 @@ class GoogleSearchIE(InfoExtractor):
while True: while True:
self.report_download_page(query, pagenum) self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
request = urllib2.Request(result_url, None, std_headers) request = urllib2.Request(result_url)
try: try:
page = urllib2.urlopen(request).read() page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@ -1871,7 +2013,7 @@ class YahooSearchIE(InfoExtractor):
def __init__(self, yahoo_ie, downloader=None): def __init__(self, yahoo_ie, downloader=None):
InfoExtractor.__init__(self, downloader) InfoExtractor.__init__(self, downloader)
self._yahoo_ie = yahoo_ie self._yahoo_ie = yahoo_ie
@staticmethod @staticmethod
def suitable(url): def suitable(url):
return (re.match(YahooSearchIE._VALID_QUERY, url) is not None) return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
@ -1883,7 +2025,7 @@ class YahooSearchIE(InfoExtractor):
def _real_initialize(self): def _real_initialize(self):
self._yahoo_ie.initialize() self._yahoo_ie.initialize()
def _real_extract(self, query): def _real_extract(self, query):
mobj = re.match(self._VALID_QUERY, query) mobj = re.match(self._VALID_QUERY, query)
if mobj is None: if mobj is None:
@ -1924,7 +2066,7 @@ class YahooSearchIE(InfoExtractor):
while True: while True:
self.report_download_page(query, pagenum) self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
request = urllib2.Request(result_url, None, std_headers) request = urllib2.Request(result_url)
try: try:
page = urllib2.urlopen(request).read() page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@ -1953,7 +2095,7 @@ class YahooSearchIE(InfoExtractor):
class YoutubePlaylistIE(InfoExtractor): class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists.""" """Information Extractor for YouTube playlists."""
_VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*' _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/|p/)([^&]+).*'
_TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en' _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR = r'/watch\?v=(.+?)&' _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>' _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
@ -1962,7 +2104,7 @@ class YoutubePlaylistIE(InfoExtractor):
def __init__(self, youtube_ie, downloader=None): def __init__(self, youtube_ie, downloader=None):
InfoExtractor.__init__(self, downloader) InfoExtractor.__init__(self, downloader)
self._youtube_ie = youtube_ie self._youtube_ie = youtube_ie
@staticmethod @staticmethod
def suitable(url): def suitable(url):
return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None) return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
@ -1973,7 +2115,7 @@ class YoutubePlaylistIE(InfoExtractor):
def _real_initialize(self): def _real_initialize(self):
self._youtube_ie.initialize() self._youtube_ie.initialize()
def _real_extract(self, url): def _real_extract(self, url):
# Extract playlist id # Extract playlist id
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
@ -1988,7 +2130,7 @@ class YoutubePlaylistIE(InfoExtractor):
while True: while True:
self.report_download_page(playlist_id, pagenum) self.report_download_page(playlist_id, pagenum)
request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers) request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum))
try: try:
page = urllib2.urlopen(request).read() page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@ -2025,7 +2167,7 @@ class YoutubeUserIE(InfoExtractor):
def __init__(self, youtube_ie, downloader=None): def __init__(self, youtube_ie, downloader=None):
InfoExtractor.__init__(self, downloader) InfoExtractor.__init__(self, downloader)
self._youtube_ie = youtube_ie self._youtube_ie = youtube_ie
@staticmethod @staticmethod
def suitable(url): def suitable(url):
return (re.match(YoutubeUserIE._VALID_URL, url) is not None) return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
@ -2036,7 +2178,7 @@ class YoutubeUserIE(InfoExtractor):
def _real_initialize(self): def _real_initialize(self):
self._youtube_ie.initialize() self._youtube_ie.initialize()
def _real_extract(self, url): def _real_extract(self, url):
# Extract username # Extract username
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
@ -2050,7 +2192,7 @@ class YoutubeUserIE(InfoExtractor):
pagenum = 1 pagenum = 1
self.report_download_page(username) self.report_download_page(username)
request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers) request = urllib2.Request(self._TEMPLATE_URL % (username))
try: try:
page = urllib2.urlopen(request).read() page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err: except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@ -2073,6 +2215,85 @@ class YoutubeUserIE(InfoExtractor):
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id) self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
return return
class DepositFilesIE(InfoExtractor):
"""Information extractor for depositfiles.com"""
_VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
def __init__(self, downloader=None):
InfoExtractor.__init__(self, downloader)
@staticmethod
def suitable(url):
return (re.match(DepositFilesIE._VALID_URL, url) is not None)
def report_download_webpage(self, file_id):
"""Report webpage download."""
self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
def report_extraction(self, file_id):
"""Report information extraction."""
self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
def _real_initialize(self):
return
def _real_extract(self, url):
# At this point we have a new file
self._downloader.increment_downloads()
file_id = url.split('/')[-1]
# Rebuild url in english locale
url = 'http://depositfiles.com/en/files/' + file_id
# Retrieve file webpage with 'Free download' button pressed
free_download_indication = { 'gateway_result' : '1' }
request = urllib2.Request(url, urllib.urlencode(free_download_indication))
try:
self.report_download_webpage(file_id)
webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
return
# Search for the real file URL
mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
if (mobj is None) or (mobj.group(1) is None):
# Try to figure out reason of the error.
mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
if (mobj is not None) and (mobj.group(1) is not None):
restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
self._downloader.trouble(u'ERROR: %s' % restriction_message)
else:
self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
return
file_url = mobj.group(1)
file_extension = os.path.splitext(file_url)[1][1:]
# Search for file title
mobj = re.search(r'<b title="(.*?)">', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract title')
return
file_title = mobj.group(1).decode('utf-8')
try:
# Process file information
self._downloader.process_info({
'id': file_id.decode('utf-8'),
'url': file_url.decode('utf-8'),
'uploader': u'NA',
'upload_date': u'NA',
'title': file_title,
'stitle': file_title,
'ext': file_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
})
except UnavailableVideoError, err:
self._downloader.trouble(u'ERROR: unable to download file')
class PostProcessor(object): class PostProcessor(object):
"""Post Processor class. """Post Processor class.
@ -2098,7 +2319,7 @@ class PostProcessor(object):
def set_downloader(self, downloader): def set_downloader(self, downloader):
"""Sets the downloader for this PP.""" """Sets the downloader for this PP."""
self._downloader = downloader self._downloader = downloader
def run(self, information): def run(self, information):
"""Run the PostProcessor. """Run the PostProcessor.
@ -2118,7 +2339,7 @@ class PostProcessor(object):
it was called from. it was called from.
""" """
return information # by default, do nothing return information # by default, do nothing
### MAIN PROGRAM ### ### MAIN PROGRAM ###
if __name__ == '__main__': if __name__ == '__main__':
try: try:
@ -2126,26 +2347,32 @@ if __name__ == '__main__':
import getpass import getpass
import optparse import optparse
# Function to update the program file with the latest version from bitbucket.org # Function to update the program file with the latest version from the repository.
def update_self(downloader, filename): def update_self(downloader, filename):
# Note: downloader only used for options # Note: downloader only used for options
if not os.access (filename, os.W_OK): if not os.access(filename, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % filename) sys.exit('ERROR: no write permissions on %s' % filename)
downloader.to_screen('Updating to latest stable version...') downloader.to_screen('Updating to latest stable version...')
latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION' try:
latest_version = urllib.urlopen(latest_url).read().strip() latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version latest_version = urllib.urlopen(latest_url).read().strip()
newcontent = urllib.urlopen(prog_url).read() prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
stream = open(filename, 'w') newcontent = urllib.urlopen(prog_url).read()
stream.write(newcontent) except (IOError, OSError), err:
stream.close() sys.exit('ERROR: unable to download latest version')
try:
stream = open(filename, 'w')
stream.write(newcontent)
stream.close()
except (IOError, OSError), err:
sys.exit('ERROR: unable to overwrite current version')
downloader.to_screen('Updated to version %s' % latest_version) downloader.to_screen('Updated to version %s' % latest_version)
# Parse command line # Parse command line
parser = optparse.OptionParser( parser = optparse.OptionParser(
usage='Usage: %prog [options] url...', usage='Usage: %prog [options] url...',
version='2010.11.19', version='2010.12.09',
conflict_handler='resolve', conflict_handler='resolve',
) )
@ -2165,6 +2392,9 @@ if __name__ == '__main__':
dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1) dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
parser.add_option('--playlist-end', parser.add_option('--playlist-end',
dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
parser.add_option('--dump-user-agent',
action='store_true', dest='dump_user_agent',
help='display the current browser identification', default=False)
authentication = optparse.OptionGroup(parser, 'Authentication Options') authentication = optparse.OptionGroup(parser, 'Authentication Options')
authentication.add_option('-u', '--username', authentication.add_option('-u', '--username',
@ -2178,14 +2408,10 @@ if __name__ == '__main__':
video_format = optparse.OptionGroup(parser, 'Video Format Options') video_format = optparse.OptionGroup(parser, 'Video Format Options')
video_format.add_option('-f', '--format', video_format.add_option('-f', '--format',
action='store', dest='format', metavar='FORMAT', help='video format code') action='store', dest='format', metavar='FORMAT', help='video format code')
video_format.add_option('-m', '--mobile-version',
action='store_const', dest='format', help='alias for -f 17', const='17')
video_format.add_option('--all-formats', video_format.add_option('--all-formats',
action='store_const', dest='format', help='download all available video formats', const='-1') action='store_const', dest='format', help='download all available video formats', const='-1')
video_format.add_option('--max-quality', video_format.add_option('--max-quality',
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
video_format.add_option('-b', '--best-quality',
action='store_true', dest='bestquality', help='download the best video quality (DEPRECATED)')
parser.add_option_group(video_format) parser.add_option_group(video_format)
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
@ -2198,11 +2424,19 @@ if __name__ == '__main__':
verbosity.add_option('-e', '--get-title', verbosity.add_option('-e', '--get-title',
action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
verbosity.add_option('--get-thumbnail', verbosity.add_option('--get-thumbnail',
action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False) action='store_true', dest='getthumbnail',
help='simulate, quiet but print thumbnail URL', default=False)
verbosity.add_option('--get-description', verbosity.add_option('--get-description',
action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False) action='store_true', dest='getdescription',
help='simulate, quiet but print video description', default=False)
verbosity.add_option('--get-filename',
action='store_true', dest='getfilename',
help='simulate, quiet but print output filename', default=False)
verbosity.add_option('--no-progress', verbosity.add_option('--no-progress',
action='store_true', dest='noprogress', help='do not print progress bar', default=False) action='store_true', dest='noprogress', help='do not print progress bar', default=False)
verbosity.add_option('--console-title',
action='store_true', dest='consoletitle',
help='display progress in console titlebar', default=False)
parser.add_option_group(verbosity) parser.add_option_group(verbosity)
filesystem = optparse.OptionGroup(parser, 'Filesystem Options') filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
@ -2211,7 +2445,8 @@ if __name__ == '__main__':
filesystem.add_option('-l', '--literal', filesystem.add_option('-l', '--literal',
action='store_true', dest='useliteral', help='use literal title in file name', default=False) action='store_true', dest='useliteral', help='use literal title in file name', default=False)
filesystem.add_option('-A', '--auto-number', filesystem.add_option('-A', '--auto-number',
action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False) action='store_true', dest='autonumber',
help='number downloaded files starting from 00000', default=False)
filesystem.add_option('-o', '--output', filesystem.add_option('-o', '--output',
dest='outtmpl', metavar='TEMPLATE', help='output filename template') dest='outtmpl', metavar='TEMPLATE', help='output filename template')
filesystem.add_option('-a', '--batch-file', filesystem.add_option('-a', '--batch-file',
@ -2222,6 +2457,11 @@ if __name__ == '__main__':
action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False) action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
filesystem.add_option('--cookies', filesystem.add_option('--cookies',
dest='cookiefile', metavar='FILE', help='file to dump cookie jar to') dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
filesystem.add_option('--no-part',
action='store_true', dest='nopart', help='do not use .part files', default=False)
filesystem.add_option('--no-mtime',
action='store_false', dest='updatetime',
help='do not use the Last-modified header to set the file modification time', default=True)
parser.add_option_group(filesystem) parser.add_option_group(filesystem)
(opts, args) = parser.parse_args() (opts, args) = parser.parse_args()
@ -2237,10 +2477,14 @@ if __name__ == '__main__':
except (IOError, OSError), err: except (IOError, OSError), err:
sys.exit(u'ERROR: unable to open cookie file') sys.exit(u'ERROR: unable to open cookie file')
# Dump user agent
if opts.dump_user_agent:
print std_headers['User-Agent']
sys.exit(0)
# General configuration # General configuration
cookie_processor = urllib2.HTTPCookieProcessor(jar) cookie_processor = urllib2.HTTPCookieProcessor(jar)
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler())) urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()))
urllib2.install_opener(urllib2.build_opener(cookie_processor))
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
# Batch file verification # Batch file verification
@ -2259,8 +2503,6 @@ if __name__ == '__main__':
all_urls = batchurls + args all_urls = batchurls + args
# Conflicting, missing and erroneous options # Conflicting, missing and erroneous options
if opts.bestquality:
print >>sys.stderr, u'\nWARNING: -b/--best-quality IS DEPRECATED AS IT IS THE DEFAULT BEHAVIOR NOW\n'
if opts.usenetrc and (opts.username is not None or opts.password is not None): if opts.usenetrc and (opts.username is not None or opts.password is not None):
parser.error(u'using .netrc conflicts with giving username/password') parser.error(u'using .netrc conflicts with giving username/password')
if opts.password is not None and opts.username is None: if opts.password is not None and opts.username is None:
@ -2306,6 +2548,7 @@ if __name__ == '__main__':
photobucket_ie = PhotobucketIE() photobucket_ie = PhotobucketIE()
yahoo_ie = YahooIE() yahoo_ie = YahooIE()
yahoo_search_ie = YahooSearchIE(yahoo_ie) yahoo_search_ie = YahooSearchIE(yahoo_ie)
deposit_files_ie = DepositFilesIE()
generic_ie = GenericIE() generic_ie = GenericIE()
# File downloader # File downloader
@ -2313,12 +2556,13 @@ if __name__ == '__main__':
'usenetrc': opts.usenetrc, 'usenetrc': opts.usenetrc,
'username': opts.username, 'username': opts.username,
'password': opts.password, 'password': opts.password,
'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription), 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
'forceurl': opts.geturl, 'forceurl': opts.geturl,
'forcetitle': opts.gettitle, 'forcetitle': opts.gettitle,
'forcethumbnail': opts.getthumbnail, 'forcethumbnail': opts.getthumbnail,
'forcedescription': opts.getdescription, 'forcedescription': opts.getdescription,
'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription), 'forcefilename': opts.getfilename,
'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
'format': opts.format, 'format': opts.format,
'format_limit': opts.format_limit, 'format_limit': opts.format_limit,
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding())) 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
@ -2340,6 +2584,9 @@ if __name__ == '__main__':
'playliststart': opts.playliststart, 'playliststart': opts.playliststart,
'playlistend': opts.playlistend, 'playlistend': opts.playlistend,
'logtostderr': opts.outtmpl == '-', 'logtostderr': opts.outtmpl == '-',
'consoletitle': opts.consoletitle,
'nopart': opts.nopart,
'updatetime': opts.updatetime,
}) })
fd.add_info_extractor(youtube_search_ie) fd.add_info_extractor(youtube_search_ie)
fd.add_info_extractor(youtube_pl_ie) fd.add_info_extractor(youtube_pl_ie)
@ -2352,6 +2599,7 @@ if __name__ == '__main__':
fd.add_info_extractor(photobucket_ie) fd.add_info_extractor(photobucket_ie)
fd.add_info_extractor(yahoo_ie) fd.add_info_extractor(yahoo_ie)
fd.add_info_extractor(yahoo_search_ie) fd.add_info_extractor(yahoo_search_ie)
fd.add_info_extractor(deposit_files_ie)
# This must come last since it's the # This must come last since it's the
# fallback if none of the others work # fallback if none of the others work