wxcafe/youtube-dl
1
0
Fork 0
mirror of https://code.hackerspace.pl/q3k/youtube-dl synced 2025-03-16 04:51:59 +00:00
Code Releases Activity

Prepare urllib references for 2/3 compatibility

Browse Source
This commit is contained in:
Philipp Hagemeister 2012-11-27 23:54:09 +01:00
parent e08bee320e
commit 01ba00ca42
4 changed files with 238 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
youtube_dl/FileDownloader.py
View File

@ -9,7 +9,6 @@ import socket
import subprocess import subprocess
import sys import sys
import time import time
import urllib2
if os.name == 'nt': if os.name == 'nt':
import ctypes import ctypes
@ -461,7 +460,7 @@ class FileDownloader(object):
success = self._do_download(filename, info_dict) success = self._do_download(filename, info_dict)
except (OSError, IOError) as err: except (OSError, IOError) as err:
raise UnavailableVideoError raise UnavailableVideoError
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self.trouble(u'ERROR: unable to download video data: %s' % str(err)) self.trouble(u'ERROR: unable to download video data: %s' % str(err))
return return
except (ContentTooShortError, ) as err: except (ContentTooShortError, ) as err:
@ -585,8 +584,8 @@ class FileDownloader(object):
# Do not include the Accept-Encoding header # Do not include the Accept-Encoding header
headers = {'Youtubedl-no-compression': 'True'} headers = {'Youtubedl-no-compression': 'True'}
basic_request = urllib2.Request(url, None, headers) basic_request = compat_urllib_request.Request(url, None, headers)
request = urllib2.Request(url, None, headers) request = compat_urllib_request.Request(url, None, headers)
# Establish possible resume length # Establish possible resume length
if os.path.isfile(encodeFilename(tmpfilename)): if os.path.isfile(encodeFilename(tmpfilename)):
@ -610,9 +609,9 @@ class FileDownloader(object):
try: try:
if count == 0 and 'urlhandle' in info_dict: if count == 0 and 'urlhandle' in info_dict:
data = info_dict['urlhandle'] data = info_dict['urlhandle']
data = urllib2.urlopen(request) data = compat_urllib_request.urlopen(request)
break break
except (urllib2.HTTPError, ) as err: except (compat_urllib_error.HTTPError, ) as err:
if (err.code < 500 or err.code >= 600) and err.code != 416: if (err.code < 500 or err.code >= 600) and err.code != 416:
# Unexpected HTTP error # Unexpected HTTP error
raise raise
@ -620,9 +619,9 @@ class FileDownloader(object):
# Unable to resume (requested range not satisfiable) # Unable to resume (requested range not satisfiable)
try: try:
# Open the connection again without the range header # Open the connection again without the range header
data = urllib2.urlopen(basic_request) data = compat_urllib_request.urlopen(basic_request)
content_length = data.info()['Content-Length'] content_length = data.info()['Content-Length']
except (urllib2.HTTPError, ) as err: except (compat_urllib_error.HTTPError, ) as err:
if err.code < 500 or err.code >= 600: if err.code < 500 or err.code >= 600:
raise raise
else: else:

397
youtube_dl/InfoExtractors.py
View File

@ -9,8 +9,6 @@ import os
import re import re
import socket import socket
import time import time
import urllib
import urllib2
import email.utils import email.utils
import xml.etree.ElementTree import xml.etree.ElementTree
import random import random
@ -53,7 +51,7 @@ class InfoExtractor(object):
player_url: SWF Player URL (used for rtmpdump). player_url: SWF Player URL (used for rtmpdump).
subtitles: The .srt file contents. subtitles: The .srt file contents.
urlhandle: [internal] The urlHandle to be used to download the file, urlhandle: [internal] The urlHandle to be used to download the file,
like returned by urllib2.urlopen like returned by urllib.request.urlopen
The fields should all be Unicode strings. The fields should all be Unicode strings.
@ -257,11 +255,11 @@ class YoutubeIE(InfoExtractor):
return return
# Set language # Set language
request = urllib2.Request(self._LANG_URL) request = compat_urllib_request.Request(self._LANG_URL)
try: try:
self.report_lang() self.report_lang()
urllib2.urlopen(request).read() compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err)) self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
return return
@ -277,14 +275,14 @@ class YoutubeIE(InfoExtractor):
'username': username, 'username': username,
'password': password, 'password': password,
} }
request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form)) request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
try: try:
self.report_login() self.report_login()
login_results = urllib2.urlopen(request).read() login_results = compat_urllib_request.urlopen(request).read()
if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None: if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
return return
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
return return
@ -293,11 +291,11 @@ class YoutubeIE(InfoExtractor):
'next_url': '/', 'next_url': '/',
'action_confirm': 'Confirm', 'action_confirm': 'Confirm',
} }
request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form)) request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
try: try:
self.report_age_confirmation() self.report_age_confirmation()
age_results = urllib2.urlopen(request).read() age_results = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
return return
@ -305,7 +303,7 @@ class YoutubeIE(InfoExtractor):
# Extract original video URL from URL with redirection, like age verification, using next_url parameter # Extract original video URL from URL with redirection, like age verification, using next_url parameter
mobj = re.search(self._NEXT_URL_RE, url) mobj = re.search(self._NEXT_URL_RE, url)
if mobj: if mobj:
url = 'http://www.youtube.com/' + urllib.unquote(mobj.group(1)).lstrip('/') url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
# Extract video id from URL # Extract video id from URL
mobj = re.match(self._VALID_URL, url, re.VERBOSE) mobj = re.match(self._VALID_URL, url, re.VERBOSE)
@ -316,10 +314,10 @@ class YoutubeIE(InfoExtractor):
# Get video webpage # Get video webpage
self.report_video_webpage_download(video_id) self.report_video_webpage_download(video_id)
request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id) request = compat_urllib_request.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
try: try:
video_webpage = urllib2.urlopen(request).read() video_webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return return
@ -335,13 +333,13 @@ class YoutubeIE(InfoExtractor):
for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']: for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
% (video_id, el_type)) % (video_id, el_type))
request = urllib2.Request(video_info_url) request = compat_urllib_request.Request(video_info_url)
try: try:
video_info_webpage = urllib2.urlopen(request).read() video_info_webpage = compat_urllib_request.urlopen(request).read()
video_info = parse_qs(video_info_webpage) video_info = parse_qs(video_info_webpage)
if 'token' in video_info: if 'token' in video_info:
break break
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
return return
if 'token' not in video_info: if 'token' not in video_info:
@ -363,13 +361,13 @@ class YoutubeIE(InfoExtractor):
if 'author' not in video_info: if 'author' not in video_info:
self._downloader.trouble(u'ERROR: unable to extract uploader nickname') self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
return return
video_uploader = urllib.unquote_plus(video_info['author'][0]) video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
# title # title
if 'title' not in video_info: if 'title' not in video_info:
self._downloader.trouble(u'ERROR: unable to extract video title') self._downloader.trouble(u'ERROR: unable to extract video title')
return return
video_title = urllib.unquote_plus(video_info['title'][0]) video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
video_title = video_title.decode('utf-8') video_title = video_title.decode('utf-8')
# thumbnail image # thumbnail image
@ -377,7 +375,7 @@ class YoutubeIE(InfoExtractor):
self._downloader.trouble(u'WARNING: unable to extract video thumbnail') self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
video_thumbnail = '' video_thumbnail = ''
else: # don't panic if we can't find it else: # don't panic if we can't find it
video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0]) video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
# upload date # upload date
upload_date = None upload_date = None
@ -401,10 +399,10 @@ class YoutubeIE(InfoExtractor):
if self._downloader.params.get('writesubtitles', False): if self._downloader.params.get('writesubtitles', False):
try: try:
self.report_video_subtitles_download(video_id) self.report_video_subtitles_download(video_id)
request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
try: try:
srt_list = urllib2.urlopen(request).read() srt_list = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list) srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list) srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
@ -418,10 +416,10 @@ class YoutubeIE(InfoExtractor):
srt_lang = srt_lang_list.keys()[0] srt_lang = srt_lang_list.keys()[0]
if not srt_lang in srt_lang_list: if not srt_lang in srt_lang_list:
raise Trouble(u'WARNING: no closed captions found in the specified language') raise Trouble(u'WARNING: no closed captions found in the specified language')
request = urllib2.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id)) request = compat_urllib_request.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
try: try:
srt_xml = urllib2.urlopen(request).read() srt_xml = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
if not srt_xml: if not srt_xml:
raise Trouble(u'WARNING: unable to download video subtitles') raise Trouble(u'WARNING: unable to download video subtitles')
@ -433,10 +431,10 @@ class YoutubeIE(InfoExtractor):
self._downloader.trouble(u'WARNING: unable to extract video duration') self._downloader.trouble(u'WARNING: unable to extract video duration')
video_duration = '' video_duration = ''
else: else:
video_duration = urllib.unquote_plus(video_info['length_seconds'][0]) video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
# token # token
video_token = urllib.unquote_plus(video_info['token'][0]) video_token = compat_urllib_parse.unquote_plus(video_info['token'][0])
# Decide which formats to download # Decide which formats to download
req_format = self._downloader.params.get('format', None) req_format = self._downloader.params.get('format', None)
@ -539,11 +537,11 @@ class MetacafeIE(InfoExtractor):
def _real_initialize(self): def _real_initialize(self):
# Retrieve disclaimer # Retrieve disclaimer
request = urllib2.Request(self._DISCLAIMER) request = compat_urllib_request.Request(self._DISCLAIMER)
try: try:
self.report_disclaimer() self.report_disclaimer()
disclaimer = urllib2.urlopen(request).read() disclaimer = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
return return
@ -552,11 +550,11 @@ class MetacafeIE(InfoExtractor):
'filters': '0', 'filters': '0',
'submit': "Continue - I'm over 18", 'submit': "Continue - I'm over 18",
} }
request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form)) request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
try: try:
self.report_age_confirmation() self.report_age_confirmation()
disclaimer = urllib2.urlopen(request).read() disclaimer = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
return return
@ -576,11 +574,11 @@ class MetacafeIE(InfoExtractor):
return return
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id) request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
return return
@ -588,7 +586,7 @@ class MetacafeIE(InfoExtractor):
self.report_extraction(video_id) self.report_extraction(video_id)
mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
if mobj is not None: if mobj is not None:
mediaURL = urllib.unquote(mobj.group(1)) mediaURL = compat_urllib_parse.unquote(mobj.group(1))
video_extension = mediaURL[-3:] video_extension = mediaURL[-3:]
# Extract gdaKey if available # Extract gdaKey if available
@ -666,12 +664,12 @@ class DailymotionIE(InfoExtractor):
video_extension = 'mp4' video_extension = 'mp4'
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
request.add_header('Cookie', 'family_filter=off') request.add_header('Cookie', 'family_filter=off')
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
return return
@ -681,7 +679,7 @@ class DailymotionIE(InfoExtractor):
if mobj is None: if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract media URL') self._downloader.trouble(u'ERROR: unable to extract media URL')
return return
flashvars = urllib.unquote(mobj.group(1)) flashvars = compat_urllib_parse.unquote(mobj.group(1))
for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']: for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
if key in flashvars: if key in flashvars:
@ -697,7 +695,7 @@ class DailymotionIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract video URL') self._downloader.trouble(u'ERROR: unable to extract video URL')
return return
video_url = urllib.unquote(mobj.group(1)).replace('\\/', '/') video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/')
# TODO: support choosing qualities # TODO: support choosing qualities
@ -763,11 +761,11 @@ class GoogleIE(InfoExtractor):
video_extension = 'mp4' video_extension = 'mp4'
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id) request = compat_urllib_request.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -780,7 +778,7 @@ class GoogleIE(InfoExtractor):
if mobj is None: if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract media URL') self._downloader.trouble(u'ERROR: unable to extract media URL')
return return
mediaURL = urllib.unquote(mobj.group(1)) mediaURL = compat_urllib_parse.unquote(mobj.group(1))
mediaURL = mediaURL.replace('\\x3d', '\x3d') mediaURL = mediaURL.replace('\\x3d', '\x3d')
mediaURL = mediaURL.replace('\\x26', '\x26') mediaURL = mediaURL.replace('\\x26', '\x26')
@ -803,10 +801,10 @@ class GoogleIE(InfoExtractor):
# Extract video thumbnail # Extract video thumbnail
if self._downloader.params.get('forcethumbnail', False): if self._downloader.params.get('forcethumbnail', False):
request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id))) request = compat_urllib_request.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage) mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
@ -856,11 +854,11 @@ class PhotobucketIE(InfoExtractor):
video_extension = 'flv' video_extension = 'flv'
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -870,7 +868,7 @@ class PhotobucketIE(InfoExtractor):
if mobj is None: if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract media URL') self._downloader.trouble(u'ERROR: unable to extract media URL')
return return
mediaURL = urllib.unquote(mobj.group(1)) mediaURL = compat_urllib_parse.unquote(mobj.group(1))
video_url = mediaURL video_url = mediaURL
@ -925,10 +923,10 @@ class YahooIE(InfoExtractor):
# Rewrite valid but non-extractable URLs as # Rewrite valid but non-extractable URLs as
# extractable English language /watch/ URLs # extractable English language /watch/ URLs
if re.match(self._VPAGE_URL, url) is None: if re.match(self._VPAGE_URL, url) is None:
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -948,11 +946,11 @@ class YahooIE(InfoExtractor):
return self._real_extract(url, new_video=False) return self._real_extract(url, new_video=False)
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -1004,13 +1002,13 @@ class YahooIE(InfoExtractor):
# seem to need most of them, otherwise the server sends a 401. # seem to need most of them, otherwise the server sends a 401.
yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
yv_bitrate = '700' # according to Wikipedia this is hard-coded yv_bitrate = '700' # according to Wikipedia this is hard-coded
request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id + request = compat_urllib_request.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
'&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
'&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -1019,7 +1017,7 @@ class YahooIE(InfoExtractor):
if mobj is None: if mobj is None:
self._downloader.trouble(u'ERROR: Unable to extract media URL') self._downloader.trouble(u'ERROR: Unable to extract media URL')
return return
video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8') video_url = compat_urllib_parse.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
video_url = unescapeHTML(video_url) video_url = unescapeHTML(video_url)
return [{ return [{
@ -1062,11 +1060,11 @@ class VimeoIE(InfoExtractor):
video_id = mobj.group(1) video_id = mobj.group(1)
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
request = urllib2.Request(url, None, std_headers) request = compat_urllib_request.Request(url, None, std_headers)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -1168,11 +1166,11 @@ class ArteTvIE(InfoExtractor):
def fetch_webpage(self, url): def fetch_webpage(self, url):
self._downloader.increment_downloads() self._downloader.increment_downloads()
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
self.report_download_webpage(url) self.report_download_webpage(url)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
except ValueError as err: except ValueError as err:
@ -1209,7 +1207,7 @@ class ArteTvIE(InfoExtractor):
] ]
) )
http_host = url.split('/')[2] http_host = url.split('/')[2]
next_url = 'http://%s%s' % (http_host, urllib.unquote(info.get('url'))) next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url')))
info = self.grep_webpage( info = self.grep_webpage(
next_url, next_url,
r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' + r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
@ -1234,7 +1232,7 @@ class ArteTvIE(InfoExtractor):
(1, 'url', u'ERROR: Invalid URL: %s' % url) (1, 'url', u'ERROR: Invalid URL: %s' % url)
] ]
) )
next_url = urllib.unquote(info.get('url')) next_url = compat_urllib_parse.unquote(info.get('url'))
info = self.grep_webpage( info = self.grep_webpage(
next_url, next_url,
r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang, r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
@ -1243,7 +1241,7 @@ class ArteTvIE(InfoExtractor):
(1, 'url', u'ERROR: Could not find <video> tag: %s' % url) (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
] ]
) )
next_url = urllib.unquote(info.get('url')) next_url = compat_urllib_parse.unquote(info.get('url'))
info = self.grep_webpage( info = self.grep_webpage(
next_url, next_url,
@ -1262,7 +1260,7 @@ class ArteTvIE(InfoExtractor):
return { return {
'id': info.get('id'), 'id': info.get('id'),
'url': urllib.unquote(info.get('url')), 'url': compat_urllib_parse.unquote(info.get('url')),
'uploader': u'arte.tv', 'uploader': u'arte.tv',
'upload_date': info.get('date'), 'upload_date': info.get('date'),
'title': info.get('title'), 'title': info.get('title'),
@ -1308,11 +1306,11 @@ class GenericIE(InfoExtractor):
def _test_redirect(self, url): def _test_redirect(self, url):
"""Check if it is a redirect, like url shorteners, in case restart chain.""" """Check if it is a redirect, like url shorteners, in case restart chain."""
class HeadRequest(urllib2.Request): class HeadRequest(compat_urllib_request.Request):
def get_method(self): def get_method(self):
return "HEAD" return "HEAD"
class HEADRedirectHandler(urllib2.HTTPRedirectHandler): class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
""" """
Subclass the HTTPRedirectHandler to make it use our Subclass the HTTPRedirectHandler to make it use our
HeadRequest also on the redirected URL HeadRequest also on the redirected URL
@ -1327,9 +1325,9 @@ class GenericIE(InfoExtractor):
origin_req_host=req.get_origin_req_host(), origin_req_host=req.get_origin_req_host(),
unverifiable=True) unverifiable=True)
else: else:
raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
class HTTPMethodFallback(urllib2.BaseHandler): class HTTPMethodFallback(compat_urllib_request.BaseHandler):
""" """
Fallback to GET if HEAD is not allowed (405 HTTP error) Fallback to GET if HEAD is not allowed (405 HTTP error)
""" """
@ -1339,23 +1337,24 @@ class GenericIE(InfoExtractor):
newheaders = dict((k,v) for k,v in req.headers.items() newheaders = dict((k,v) for k,v in req.headers.items()
if k.lower() not in ("content-length", "content-type")) if k.lower() not in ("content-length", "content-type"))
return self.parent.open(urllib2.Request(req.get_full_url(), return self.parent.open(compat_urllib_request.Request(req.get_full_url(),
headers=newheaders, headers=newheaders,
origin_req_host=req.get_origin_req_host(), origin_req_host=req.get_origin_req_host(),
unverifiable=True)) unverifiable=True))
# Build our opener # Build our opener
opener = urllib2.OpenerDirector() opener = compat_urllib_request.OpenerDirector()
for handler in [urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler, for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
HTTPMethodFallback, HEADRedirectHandler, HTTPMethodFallback, HEADRedirectHandler,
urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler]: compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
opener.add_handler(handler()) opener.add_handler(handler())
response = opener.open(HeadRequest(url)) response = opener.open(HeadRequest(url))
new_url = response.geturl() new_url = response.geturl()
if url == new_url: return False if url == new_url:
return False
self.report_following_redirect(new_url) self.report_following_redirect(new_url)
self._downloader.download([new_url]) self._downloader.download([new_url])
return True return True
@ -1364,11 +1363,11 @@ class GenericIE(InfoExtractor):
if self._test_redirect(url): return if self._test_redirect(url): return
video_id = url.split('/')[-1] video_id = url.split('/')[-1]
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
except ValueError as err: except ValueError as err:
@ -1393,7 +1392,7 @@ class GenericIE(InfoExtractor):
self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
return return
video_url = urllib.unquote(mobj.group(1)) video_url = compat_urllib_parse.unquote(mobj.group(1))
video_id = os.path.basename(video_url) video_id = os.path.basename(video_url)
# here's a fun little line of code for you: # here's a fun little line of code for you:
@ -1483,11 +1482,11 @@ class YoutubeSearchIE(InfoExtractor):
while (50 * pagenum) < limit: while (50 * pagenum) < limit:
self.report_download_page(query, pagenum+1) self.report_download_page(query, pagenum+1)
result_url = self._API_URL % (urllib.quote_plus(query), (50*pagenum)+1) result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
request = urllib2.Request(result_url) request = compat_urllib_request.Request(result_url)
try: try:
data = urllib2.urlopen(request).read() data = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
return return
api_response = json.loads(data)['data'] api_response = json.loads(data)['data']
@ -1560,11 +1559,11 @@ class GoogleSearchIE(InfoExtractor):
while True: while True:
self.report_download_page(query, pagenum) self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10) result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum*10)
request = urllib2.Request(result_url) request = compat_urllib_request.Request(result_url)
try: try:
page = urllib2.urlopen(request).read() page = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
@ -1643,11 +1642,11 @@ class YahooSearchIE(InfoExtractor):
while True: while True:
self.report_download_page(query, pagenum) self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum)
request = urllib2.Request(result_url) request = compat_urllib_request.Request(result_url)
try: try:
page = urllib2.urlopen(request).read() page = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
@ -1714,10 +1713,10 @@ class YoutubePlaylistIE(InfoExtractor):
while True: while True:
self.report_download_page(playlist_id, pagenum) self.report_download_page(playlist_id, pagenum)
url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum) url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
page = urllib2.urlopen(request).read() page = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
@ -1771,10 +1770,10 @@ class YoutubeChannelIE(InfoExtractor):
while True: while True:
self.report_download_page(channel_id, pagenum) self.report_download_page(channel_id, pagenum)
url = self._TEMPLATE_URL % (channel_id, pagenum) url = self._TEMPLATE_URL % (channel_id, pagenum)
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
page = urllib2.urlopen(request).read() page = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
@ -1833,11 +1832,11 @@ class YoutubeUserIE(InfoExtractor):
start_index = pagenum * self._GDATA_PAGE_SIZE + 1 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
self.report_download_page(username, start_index) self.report_download_page(username, start_index)
request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)) request = compat_urllib_request.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
try: try:
page = urllib2.urlopen(request).read() page = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
@ -1903,13 +1902,13 @@ class BlipTVUserIE(InfoExtractor):
page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1' page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
page = urllib2.urlopen(request).read().decode('utf-8') page = compat_urllib_request.urlopen(request).read().decode('utf-8')
mobj = re.search(r'data-users-id="([^"]+)"', page) mobj = re.search(r'data-users-id="([^"]+)"', page)
page_base = page_base % mobj.group(1) page_base = page_base % mobj.group(1)
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
@ -1925,11 +1924,11 @@ class BlipTVUserIE(InfoExtractor):
while True: while True:
self.report_download_page(username, pagenum) self.report_download_page(username, pagenum)
request = urllib2.Request( page_base + "&page=" + str(pagenum) ) request = compat_urllib_request.Request( page_base + "&page=" + str(pagenum) )
try: try:
page = urllib2.urlopen(request).read().decode('utf-8') page = compat_urllib_request.urlopen(request).read().decode('utf-8')
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
return return
@ -1993,11 +1992,11 @@ class DepositFilesIE(InfoExtractor):
# Retrieve file webpage with 'Free download' button pressed # Retrieve file webpage with 'Free download' button pressed
free_download_indication = { 'gateway_result' : '1' } free_download_indication = { 'gateway_result' : '1' }
request = urllib2.Request(url, urllib.urlencode(free_download_indication)) request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication))
try: try:
self.report_download_webpage(file_id) self.report_download_webpage(file_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
return return
@ -2079,7 +2078,7 @@ class FacebookIE(InfoExtractor):
for piece in data.keys(): for piece in data.keys():
mobj = re.search(data[piece], video_webpage) mobj = re.search(data[piece], video_webpage)
if mobj is not None: if mobj is not None:
video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape")) video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
# Video urls # Video urls
video_urls = {} video_urls = {}
@ -2088,7 +2087,7 @@ class FacebookIE(InfoExtractor):
if mobj is not None: if mobj is not None:
# URL is in a Javascript segment inside an escaped Unicode format within # URL is in a Javascript segment inside an escaped Unicode format within
# the generally utf-8 page # the generally utf-8 page
video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape")) video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
video_info['video_urls'] = video_urls video_info['video_urls'] = video_urls
return video_info return video_info
@ -2126,14 +2125,14 @@ class FacebookIE(InfoExtractor):
'pass': password, 'pass': password,
'login': 'Log+In' 'login': 'Log+In'
} }
request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form)) request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
try: try:
self.report_login() self.report_login()
login_results = urllib2.urlopen(request).read() login_results = compat_urllib_request.urlopen(request).read()
if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None: if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
return return
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
return return
@ -2146,11 +2145,11 @@ class FacebookIE(InfoExtractor):
# Get video webpage # Get video webpage
self.report_video_webpage_download(video_id) self.report_video_webpage_download(video_id)
request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id) request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
try: try:
page = urllib2.urlopen(request) page = compat_urllib_request.urlopen(request)
video_webpage = page.read() video_webpage = page.read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return return
@ -2265,11 +2264,11 @@ class BlipTVIE(InfoExtractor):
else: else:
cchar = '?' cchar = '?'
json_url = url + cchar + 'skin=json&version=2&no_wrap=1' json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
request = urllib2.Request(json_url.encode('utf-8')) request = compat_urllib_request.Request(json_url.encode('utf-8'))
self.report_extraction(mobj.group(1)) self.report_extraction(mobj.group(1))
info = None info = None
try: try:
urlh = urllib2.urlopen(request) urlh = compat_urllib_request.urlopen(request)
if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
basename = url.split('/')[-1] basename = url.split('/')[-1]
title,ext = os.path.splitext(basename) title,ext = os.path.splitext(basename)
@ -2285,13 +2284,13 @@ class BlipTVIE(InfoExtractor):
'ext': ext, 'ext': ext,
'urlhandle': urlh 'urlhandle': urlh
} }
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
return return
if info is None: # Regular URL if info is None: # Regular URL
try: try:
json_code = urlh.read() json_code = urlh.read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
return return
@ -2355,11 +2354,11 @@ class MyVideoIE(InfoExtractor):
video_id = mobj.group(1) video_id = mobj.group(1)
# Get video webpage # Get video webpage
request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id) request = compat_urllib_request.Request('http://www.myvideo.de/watch/%s' % video_id)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -2451,12 +2450,12 @@ class ComedyCentralIE(InfoExtractor):
else: else:
epTitle = mobj.group('episode') epTitle = mobj.group('episode')
req = urllib2.Request(url) req = compat_urllib_request.Request(url)
self.report_extraction(epTitle) self.report_extraction(epTitle)
try: try:
htmlHandle = urllib2.urlopen(req) htmlHandle = compat_urllib_request.urlopen(req)
html = htmlHandle.read() html = htmlHandle.read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
if dlNewest: if dlNewest:
@ -2487,18 +2486,18 @@ class ComedyCentralIE(InfoExtractor):
playerUrl_raw = mMovieParams[0][0] playerUrl_raw = mMovieParams[0][0]
self.report_player_url(epTitle) self.report_player_url(epTitle)
try: try:
urlHandle = urllib2.urlopen(playerUrl_raw) urlHandle = compat_urllib_request.urlopen(playerUrl_raw)
playerUrl = urlHandle.geturl() playerUrl = urlHandle.geturl()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to find out player URL: ' + compat_str(err)) self._downloader.trouble(u'ERROR: unable to find out player URL: ' + compat_str(err))
return return
uri = mMovieParams[0][1] uri = mMovieParams[0][1]
indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri}) indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})
self.report_index_download(epTitle) self.report_index_download(epTitle)
try: try:
indexXml = urllib2.urlopen(indexUrl).read() indexXml = compat_urllib_request.urlopen(indexUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download episode index: ' + compat_str(err)) self._downloader.trouble(u'ERROR: unable to download episode index: ' + compat_str(err))
return return
@ -2514,12 +2513,12 @@ class ComedyCentralIE(InfoExtractor):
officialDate = itemEl.findall('./pubDate')[0].text officialDate = itemEl.findall('./pubDate')[0].text
configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' + configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
urllib.urlencode({'uri': mediaId})) compat_urllib_parse.urlencode({'uri': mediaId}))
configReq = urllib2.Request(configUrl) configReq = compat_urllib_request.Request(configUrl)
self.report_config_download(epTitle) self.report_config_download(epTitle)
try: try:
configXml = urllib2.urlopen(configReq).read() configXml = compat_urllib_request.urlopen(configReq).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return return
@ -2598,11 +2597,11 @@ class EscapistIE(InfoExtractor):
self.report_extraction(showName) self.report_extraction(showName)
try: try:
webPage = urllib2.urlopen(url) webPage = compat_urllib_request.urlopen(url)
webPageBytes = webPage.read() webPageBytes = webPage.read()
m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type']) m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
webPage = webPageBytes.decode(m.group(1) if m else 'utf-8') webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err)) self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err))
return return
@ -2613,12 +2612,12 @@ class EscapistIE(InfoExtractor):
playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage) playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
playerUrl = unescapeHTML(playerUrlMatch.group(1)) playerUrl = unescapeHTML(playerUrlMatch.group(1))
configUrlMatch = re.search('config=(.*)$', playerUrl) configUrlMatch = re.search('config=(.*)$', playerUrl)
configUrl = urllib2.unquote(configUrlMatch.group(1)) configUrl = compat_urllib_parse.unquote(configUrlMatch.group(1))
self.report_config_download(showName) self.report_config_download(showName)
try: try:
configJSON = urllib2.urlopen(configUrl).read() configJSON = compat_urllib_request.urlopen(configUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err)) self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err))
return return
@ -2671,10 +2670,10 @@ class CollegeHumorIE(InfoExtractor):
video_id = mobj.group('videoid') video_id = mobj.group('videoid')
self.report_webpage(video_id) self.report_webpage(video_id)
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return return
@ -2694,8 +2693,8 @@ class CollegeHumorIE(InfoExtractor):
self.report_extraction(video_id) self.report_extraction(video_id)
xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
try: try:
metaXml = urllib2.urlopen(xmlUrl).read() metaXml = compat_urllib_request.urlopen(xmlUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
return return
@ -2737,10 +2736,10 @@ class XVideosIE(InfoExtractor):
self.report_webpage(video_id) self.report_webpage(video_id)
request = urllib2.Request(r'http://www.xvideos.com/video' + video_id) request = compat_urllib_request.Request(r'http://www.xvideos.com/video' + video_id)
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return return
@ -2752,7 +2751,7 @@ class XVideosIE(InfoExtractor):
if mobj is None: if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract video url') self._downloader.trouble(u'ERROR: unable to extract video url')
return return
video_url = urllib2.unquote(mobj.group(1).decode('utf-8')) video_url = compat_urllib_parse.unquote(mobj.group(1).decode('utf-8'))
# Extract title # Extract title
@ -2821,10 +2820,10 @@ class SoundcloudIE(InfoExtractor):
self.report_webpage('%s/%s' % (uploader, slug_title)) self.report_webpage('%s/%s' % (uploader, slug_title))
request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title)) request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return return
@ -2863,7 +2862,7 @@ class SoundcloudIE(InfoExtractor):
self._downloader.to_stderr(compat_str(e)) self._downloader.to_stderr(compat_str(e))
# for soundcloud, a request to a cross domain is required for cookies # for soundcloud, a request to a cross domain is required for cookies
request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers) request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
return [{ return [{
'id': video_id.decode('utf-8'), 'id': video_id.decode('utf-8'),
@ -2898,10 +2897,10 @@ class InfoQIE(InfoExtractor):
self.report_webpage(url) self.report_webpage(url)
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return return
@ -2913,7 +2912,7 @@ class InfoQIE(InfoExtractor):
if mobj is None: if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract video url') self._downloader.trouble(u'ERROR: unable to extract video url')
return return
video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64')) video_url = 'rtmpe://video.infoq.com/cfx/st/' + compat_urllib_parse.unquote(mobj.group(1).decode('base64'))
# Extract title # Extract title
@ -2978,9 +2977,9 @@ class MixcloudIE(InfoExtractor):
"""Returns 1st active url from list""" """Returns 1st active url from list"""
for url in url_list: for url in url_list:
try: try:
urllib2.urlopen(url) compat_urllib_request.urlopen(url)
return url return url
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
url = None url = None
return None return None
@ -3009,11 +3008,11 @@ class MixcloudIE(InfoExtractor):
# construct API request # construct API request
file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json' file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
# retrieve .json file with links to files # retrieve .json file with links to files
request = urllib2.Request(file_url) request = compat_urllib_request.Request(file_url)
try: try:
self.report_download_json(file_url) self.report_download_json(file_url)
jsonData = urllib2.urlopen(request).read() jsonData = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
return return
@ -3091,8 +3090,8 @@ class StanfordOpenClassroomIE(InfoExtractor):
baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/' baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
xmlUrl = baseUrl + video + '.xml' xmlUrl = baseUrl + video + '.xml'
try: try:
metaXml = urllib2.urlopen(xmlUrl).read() metaXml = compat_urllib_request.urlopen(xmlUrl).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
return return
mdoc = xml.etree.ElementTree.fromstring(metaXml) mdoc = xml.etree.ElementTree.fromstring(metaXml)
@ -3115,8 +3114,8 @@ class StanfordOpenClassroomIE(InfoExtractor):
self.report_download_webpage(info['id']) self.report_download_webpage(info['id'])
try: try:
coursepage = urllib2.urlopen(url).read() coursepage = compat_urllib_request.urlopen(url).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err)) self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
return return
@ -3154,8 +3153,8 @@ class StanfordOpenClassroomIE(InfoExtractor):
self.report_download_webpage(info['id']) self.report_download_webpage(info['id'])
rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
try: try:
rootpage = urllib2.urlopen(rootURL).read() rootpage = compat_urllib_request.urlopen(rootURL).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err)) self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
return return
@ -3199,10 +3198,10 @@ class MTVIE(InfoExtractor):
video_id = mobj.group('videoid') video_id = mobj.group('videoid')
self.report_webpage(video_id) self.report_webpage(video_id)
request = urllib2.Request(url) request = compat_urllib_request.Request(url)
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return return
@ -3232,10 +3231,10 @@ class MTVIE(InfoExtractor):
videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
self.report_extraction(video_id) self.report_extraction(video_id)
request = urllib2.Request(videogen_url) request = compat_urllib_request.Request(videogen_url)
try: try:
metadataXml = urllib2.urlopen(request).read() metadataXml = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
return return
@ -3319,11 +3318,11 @@ class YoukuIE(InfoExtractor):
info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
request = urllib2.Request(info_url, None, std_headers) request = compat_urllib_request.Request(info_url, None, std_headers)
try: try:
self.report_download_webpage(video_id) self.report_download_webpage(video_id)
jsondata = urllib2.urlopen(request).read() jsondata = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
@ -3416,8 +3415,8 @@ class XNXXIE(InfoExtractor):
# Get webpage content # Get webpage content
try: try:
webpage = urllib2.urlopen(url).read() webpage = compat_urllib_request.urlopen(url).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err) self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
return return
@ -3425,7 +3424,7 @@ class XNXXIE(InfoExtractor):
if result is None: if result is None:
self._downloader.trouble(u'ERROR: unable to extract video url') self._downloader.trouble(u'ERROR: unable to extract video url')
return return
video_url = urllib.unquote(result.group(1).decode('utf-8')) video_url = compat_urllib_parse.unquote(result.group(1).decode('utf-8'))
result = re.search(self.VIDEO_TITLE_RE, webpage) result = re.search(self.VIDEO_TITLE_RE, webpage)
if result is None: if result is None:
@ -3494,10 +3493,10 @@ class GooglePlusIE(InfoExtractor):
# Step 1, Retrieve post webpage to extract further information # Step 1, Retrieve post webpage to extract further information
self.report_extract_entry(post_url) self.report_extract_entry(post_url)
request = urllib2.Request(post_url) request = compat_urllib_request.Request(post_url)
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
return return
@ -3536,10 +3535,10 @@ class GooglePlusIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract video page URL') self._downloader.trouble(u'ERROR: unable to extract video page URL')
video_page = mobj.group(1) video_page = mobj.group(1)
request = urllib2.Request(video_page) request = compat_urllib_request.Request(video_page)
try: try:
webpage = urllib2.urlopen(request).read() webpage = compat_urllib_request.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return return
self.report_extract_vid_page(video_page) self.report_extract_vid_page(video_page)

20
youtube_dl/__init__.py
View File

@ -29,7 +29,6 @@ UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSIO
UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe' UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe'
import cookielib
import getpass import getpass
import optparse import optparse
import os import os
@ -38,7 +37,6 @@ import shlex
import socket import socket
import subprocess import subprocess
import sys import sys
import urllib2
import warnings import warnings
from utils import * from utils import *
@ -55,7 +53,7 @@ def updateSelf(downloader, filename):
downloader.to_screen(u'Updating to latest version...') downloader.to_screen(u'Updating to latest version...')
urlv = urllib2.urlopen(UPDATE_URL_VERSION) urlv = compat_urllib_request.urlopen(UPDATE_URL_VERSION)
newversion = urlv.read().strip() newversion = urlv.read().strip()
if newversion == __version__: if newversion == __version__:
downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
@ -69,7 +67,7 @@ def updateSelf(downloader, filename):
sys.exit('ERROR: no write permissions on %s' % directory) sys.exit('ERROR: no write permissions on %s' % directory)
try: try:
urlh = urllib2.urlopen(UPDATE_URL_EXE) urlh = compat_urllib_request.urlopen(UPDATE_URL_EXE)
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
with open(exe + '.new', 'wb') as outf: with open(exe + '.new', 'wb') as outf:
@ -94,7 +92,7 @@ del "%s"
else: else:
try: try:
urlh = urllib2.urlopen(UPDATE_URL) urlh = compat_urllib_request.urlopen(UPDATE_URL)
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError) as err: except (IOError, OSError) as err:
@ -380,10 +378,10 @@ def _real_main():
# Open appropriate CookieJar # Open appropriate CookieJar
if opts.cookiefile is None: if opts.cookiefile is None:
jar = cookielib.CookieJar() jar = compat_cookiejar.CookieJar()
else: else:
try: try:
jar = cookielib.MozillaCookieJar(opts.cookiefile) jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK): if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
jar.load() jar.load()
except (IOError, OSError) as err: except (IOError, OSError) as err:
@ -414,10 +412,10 @@ def _real_main():
all_urls = map(lambda url: url.strip(), all_urls) all_urls = map(lambda url: url.strip(), all_urls)
# General configuration # General configuration
cookie_processor = urllib2.HTTPCookieProcessor(jar) cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
proxy_handler = urllib2.ProxyHandler() proxy_handler = compat_urllib_request.ProxyHandler()
opener = urllib2.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
urllib2.install_opener(opener) compat_urllib_request.install_opener(opener)
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
extractors = gen_extractors() extractors = gen_extractors()

29
youtube_dl/utils.py
View File

@ -9,7 +9,6 @@ import os
import re import re
import sys import sys
import zlib import zlib
import urllib2
import email.utils import email.utils
import json import json
@ -31,6 +30,26 @@ try:
except NameError: except NameError:
compat_str = str compat_str = str
try:
import urllib.request as compat_urllib_request
except ImportError: # Python 2
import urllib2 as compat_urllib_request
try:
import urllib.error as compat_urllib_error
except ImportError: # Python 2
import urllib2 as compat_urllib_error
try:
import urllib.parse as compat_urllib_parse
except ImportError: # Python 2
import urllib2 as compat_urllib_parse
try:
import http.cookiejar as compat_cookiejar
except ImportError: # Python 2
import cookielib as compat_cookiejar
def preferredencoding(): def preferredencoding():
"""Get preferred encoding. """Get preferred encoding.
@ -320,7 +339,7 @@ class Trouble(Exception):
FileDownloader.trouble FileDownloader.trouble
""" """
class YoutubeDLHandler(urllib2.HTTPHandler): class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
"""Handler for HTTP requests and responses. """Handler for HTTP requests and responses.
This class, when installed with an OpenerDirector, automatically adds This class, when installed with an OpenerDirector, automatically adds
@ -347,9 +366,9 @@ class YoutubeDLHandler(urllib2.HTTPHandler):
@staticmethod @staticmethod
def addinfourl_wrapper(stream, headers, url, code): def addinfourl_wrapper(stream, headers, url, code):
if hasattr(urllib2.addinfourl, 'getcode'): if hasattr(compat_urllib_request.addinfourl, 'getcode'):
return urllib2.addinfourl(stream, headers, url, code) return compat_urllib_request.addinfourl(stream, headers, url, code)
ret = urllib2.addinfourl(stream, headers, url) ret = compat_urllib_request.addinfourl(stream, headers, url)
ret.code = code ret.code = code
return ret return ret