|
|
@ -47,6 +47,7 @@ std_headers = {
|
|
|
|
|
|
|
|
|
|
|
|
simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
|
|
|
|
simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def preferredencoding():
|
|
|
|
def preferredencoding():
|
|
|
|
"""Get preferred encoding.
|
|
|
|
"""Get preferred encoding.
|
|
|
|
|
|
|
|
|
|
|
@ -63,6 +64,7 @@ def preferredencoding():
|
|
|
|
yield pref
|
|
|
|
yield pref
|
|
|
|
return yield_preferredencoding().next()
|
|
|
|
return yield_preferredencoding().next()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def htmlentity_transform(matchobj):
|
|
|
|
def htmlentity_transform(matchobj):
|
|
|
|
"""Transforms an HTML entity to a Unicode character.
|
|
|
|
"""Transforms an HTML entity to a Unicode character.
|
|
|
|
|
|
|
|
|
|
|
@ -89,11 +91,13 @@ def htmlentity_transform(matchobj):
|
|
|
|
# Unknown entity in name, return its literal representation
|
|
|
|
# Unknown entity in name, return its literal representation
|
|
|
|
return (u'&%s;' % entity)
|
|
|
|
return (u'&%s;' % entity)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sanitize_title(utitle):
|
|
|
|
def sanitize_title(utitle):
|
|
|
|
"""Sanitizes a video title so it could be used as part of a filename."""
|
|
|
|
"""Sanitizes a video title so it could be used as part of a filename."""
|
|
|
|
utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
|
|
|
|
utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
|
|
|
|
return utitle.replace(unicode(os.sep), u'%')
|
|
|
|
return utitle.replace(unicode(os.sep), u'%')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sanitize_open(filename, open_mode):
|
|
|
|
def sanitize_open(filename, open_mode):
|
|
|
|
"""Try to open the given filename, and slightly tweak it if this fails.
|
|
|
|
"""Try to open the given filename, and slightly tweak it if this fails.
|
|
|
|
|
|
|
|
|
|
|
@ -120,6 +124,7 @@ def sanitize_open(filename, open_mode):
|
|
|
|
stream = open(filename, open_mode)
|
|
|
|
stream = open(filename, open_mode)
|
|
|
|
return (stream, filename)
|
|
|
|
return (stream, filename)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def timeconvert(timestr):
|
|
|
|
def timeconvert(timestr):
|
|
|
|
"""Convert RFC 2822 defined time string into system timestamp"""
|
|
|
|
"""Convert RFC 2822 defined time string into system timestamp"""
|
|
|
|
timestamp = None
|
|
|
|
timestamp = None
|
|
|
@ -128,6 +133,7 @@ def timeconvert(timestr):
|
|
|
|
timestamp = email.utils.mktime_tz(timetuple)
|
|
|
|
timestamp = email.utils.mktime_tz(timetuple)
|
|
|
|
return timestamp
|
|
|
|
return timestamp
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class DownloadError(Exception):
|
|
|
|
class DownloadError(Exception):
|
|
|
|
"""Download Error exception.
|
|
|
|
"""Download Error exception.
|
|
|
|
|
|
|
|
|
|
|
@ -137,6 +143,7 @@ class DownloadError(Exception):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SameFileError(Exception):
|
|
|
|
class SameFileError(Exception):
|
|
|
|
"""Same File exception.
|
|
|
|
"""Same File exception.
|
|
|
|
|
|
|
|
|
|
|
@ -145,6 +152,7 @@ class SameFileError(Exception):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PostProcessingError(Exception):
|
|
|
|
class PostProcessingError(Exception):
|
|
|
|
"""Post Processing exception.
|
|
|
|
"""Post Processing exception.
|
|
|
|
|
|
|
|
|
|
|
@ -153,6 +161,7 @@ class PostProcessingError(Exception):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class UnavailableVideoError(Exception):
|
|
|
|
class UnavailableVideoError(Exception):
|
|
|
|
"""Unavailable Format exception.
|
|
|
|
"""Unavailable Format exception.
|
|
|
|
|
|
|
|
|
|
|
@ -161,6 +170,7 @@ class UnavailableVideoError(Exception):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ContentTooShortError(Exception):
|
|
|
|
class ContentTooShortError(Exception):
|
|
|
|
"""Content Too Short exception.
|
|
|
|
"""Content Too Short exception.
|
|
|
|
|
|
|
|
|
|
|
@ -176,6 +186,7 @@ class ContentTooShortError(Exception):
|
|
|
|
self.downloaded = downloaded
|
|
|
|
self.downloaded = downloaded
|
|
|
|
self.expected = expected
|
|
|
|
self.expected = expected
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeDLHandler(urllib2.HTTPHandler):
|
|
|
|
class YoutubeDLHandler(urllib2.HTTPHandler):
|
|
|
|
"""Handler for HTTP requests and responses.
|
|
|
|
"""Handler for HTTP requests and responses.
|
|
|
|
|
|
|
|
|
|
|
@ -234,6 +245,7 @@ class YoutubeDLHandler(urllib2.HTTPHandler):
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
return resp
|
|
|
|
return resp
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class FileDownloader(object):
|
|
|
|
class FileDownloader(object):
|
|
|
|
"""File Downloader class.
|
|
|
|
"""File Downloader class.
|
|
|
|
|
|
|
|
|
|
|
@ -325,7 +337,7 @@ class FileDownloader(object):
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
exponent = long(math.log(bytes, 1024.0))
|
|
|
|
exponent = long(math.log(bytes, 1024.0))
|
|
|
|
suffix = 'bkMGTPEZY'[exponent]
|
|
|
|
suffix = 'bkMGTPEZY'[exponent]
|
|
|
|
converted = float(bytes) / float(1024**exponent)
|
|
|
|
converted = float(bytes) / float(1024 ** exponent)
|
|
|
|
return '%.2f%s' % (converted, suffix)
|
|
|
|
return '%.2f%s' % (converted, suffix)
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
@staticmethod
|
|
|
@ -477,7 +489,7 @@ class FileDownloader(object):
|
|
|
|
if filetime is None:
|
|
|
|
if filetime is None:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
os.utime(filename,(time.time(), filetime))
|
|
|
|
os.utime(filename, (time.time(), filetime))
|
|
|
|
except:
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
@ -680,7 +692,7 @@ class FileDownloader(object):
|
|
|
|
# Request parameters in case of being able to resume
|
|
|
|
# Request parameters in case of being able to resume
|
|
|
|
if self.params.get('continuedl', False) and resume_len != 0:
|
|
|
|
if self.params.get('continuedl', False) and resume_len != 0:
|
|
|
|
self.report_resuming_byte(resume_len)
|
|
|
|
self.report_resuming_byte(resume_len)
|
|
|
|
request.add_header('Range','bytes=%d-' % resume_len)
|
|
|
|
request.add_header('Range', 'bytes=%d-' % resume_len)
|
|
|
|
open_mode = 'ab'
|
|
|
|
open_mode = 'ab'
|
|
|
|
|
|
|
|
|
|
|
|
count = 0
|
|
|
|
count = 0
|
|
|
@ -784,6 +796,7 @@ class FileDownloader(object):
|
|
|
|
|
|
|
|
|
|
|
|
return True
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class InfoExtractor(object):
|
|
|
|
class InfoExtractor(object):
|
|
|
|
"""Information Extractor class.
|
|
|
|
"""Information Extractor class.
|
|
|
|
|
|
|
|
|
|
|
@ -855,6 +868,7 @@ class InfoExtractor(object):
|
|
|
|
"""Real extraction process. Redefine in subclasses."""
|
|
|
|
"""Real extraction process. Redefine in subclasses."""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeIE(InfoExtractor):
|
|
|
|
class YoutubeIE(InfoExtractor):
|
|
|
|
"""Information extractor for youtube.com."""
|
|
|
|
"""Information extractor for youtube.com."""
|
|
|
|
|
|
|
|
|
|
|
@ -1371,6 +1385,7 @@ class DailymotionIE(InfoExtractor):
|
|
|
|
except UnavailableVideoError:
|
|
|
|
except UnavailableVideoError:
|
|
|
|
self._downloader.trouble(u'\nERROR: unable to download video')
|
|
|
|
self._downloader.trouble(u'\nERROR: unable to download video')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class GoogleIE(InfoExtractor):
|
|
|
|
class GoogleIE(InfoExtractor):
|
|
|
|
"""Information extractor for video.google.com."""
|
|
|
|
"""Information extractor for video.google.com."""
|
|
|
|
|
|
|
|
|
|
|
@ -1464,7 +1479,6 @@ class GoogleIE(InfoExtractor):
|
|
|
|
else: # we need something to pass to process_info
|
|
|
|
else: # we need something to pass to process_info
|
|
|
|
video_thumbnail = ''
|
|
|
|
video_thumbnail = ''
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
# Process video information
|
|
|
|
# Process video information
|
|
|
|
self._downloader.process_info({
|
|
|
|
self._downloader.process_info({
|
|
|
@ -1664,7 +1678,8 @@ class YahooIE(InfoExtractor):
|
|
|
|
self._downloader.trouble(u'ERROR: unable to extract video description')
|
|
|
|
self._downloader.trouble(u'ERROR: unable to extract video description')
|
|
|
|
return
|
|
|
|
return
|
|
|
|
video_description = mobj.group(1).decode('utf-8')
|
|
|
|
video_description = mobj.group(1).decode('utf-8')
|
|
|
|
if not video_description: video_description = 'No description available.'
|
|
|
|
if not video_description:
|
|
|
|
|
|
|
|
video_description = 'No description available.'
|
|
|
|
|
|
|
|
|
|
|
|
# Extract video height and width
|
|
|
|
# Extract video height and width
|
|
|
|
mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
|
|
|
|
mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
|
|
|
@ -1914,6 +1929,7 @@ class YoutubeSearchIE(InfoExtractor):
|
|
|
|
|
|
|
|
|
|
|
|
pagenum = pagenum + 1
|
|
|
|
pagenum = pagenum + 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class GoogleSearchIE(InfoExtractor):
|
|
|
|
class GoogleSearchIE(InfoExtractor):
|
|
|
|
"""Information Extractor for Google Video search queries."""
|
|
|
|
"""Information Extractor for Google Video search queries."""
|
|
|
|
_VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
|
|
|
|
_VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
|
|
|
@ -2005,6 +2021,7 @@ class GoogleSearchIE(InfoExtractor):
|
|
|
|
|
|
|
|
|
|
|
|
pagenum = pagenum + 1
|
|
|
|
pagenum = pagenum + 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YahooSearchIE(InfoExtractor):
|
|
|
|
class YahooSearchIE(InfoExtractor):
|
|
|
|
"""Information Extractor for Yahoo! Video search queries."""
|
|
|
|
"""Information Extractor for Yahoo! Video search queries."""
|
|
|
|
_VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
|
|
|
|
_VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
|
|
|
@ -2096,6 +2113,7 @@ class YahooSearchIE(InfoExtractor):
|
|
|
|
|
|
|
|
|
|
|
|
pagenum = pagenum + 1
|
|
|
|
pagenum = pagenum + 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubePlaylistIE(InfoExtractor):
|
|
|
|
class YoutubePlaylistIE(InfoExtractor):
|
|
|
|
"""Information Extractor for YouTube playlists."""
|
|
|
|
"""Information Extractor for YouTube playlists."""
|
|
|
|
|
|
|
|
|
|
|
@ -2172,6 +2190,7 @@ class YoutubePlaylistIE(InfoExtractor):
|
|
|
|
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
|
|
|
|
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
|
|
|
|
return
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeUserIE(InfoExtractor):
|
|
|
|
class YoutubeUserIE(InfoExtractor):
|
|
|
|
"""Information Extractor for YouTube users."""
|
|
|
|
"""Information Extractor for YouTube users."""
|
|
|
|
|
|
|
|
|
|
|
@ -2342,6 +2361,7 @@ class DepositFilesIE(InfoExtractor):
|
|
|
|
except UnavailableVideoError, err:
|
|
|
|
except UnavailableVideoError, err:
|
|
|
|
self._downloader.trouble(u'ERROR: unable to download file')
|
|
|
|
self._downloader.trouble(u'ERROR: unable to download file')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class FacebookIE(InfoExtractor):
|
|
|
|
class FacebookIE(InfoExtractor):
|
|
|
|
"""Information Extractor for Facebook"""
|
|
|
|
"""Information Extractor for Facebook"""
|
|
|
|
|
|
|
|
|
|
|
@ -2565,6 +2585,7 @@ class FacebookIE(InfoExtractor):
|
|
|
|
except UnavailableVideoError, err:
|
|
|
|
except UnavailableVideoError, err:
|
|
|
|
self._downloader.trouble(u'\nERROR: unable to download video')
|
|
|
|
self._downloader.trouble(u'\nERROR: unable to download video')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PostProcessor(object):
|
|
|
|
class PostProcessor(object):
|
|
|
|
"""Post Processor class.
|
|
|
|
"""Post Processor class.
|
|
|
|
|
|
|
|
|
|
|
@ -2611,6 +2632,7 @@ class PostProcessor(object):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
return information # by default, do nothing
|
|
|
|
return information # by default, do nothing
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class FFmpegExtractAudioPP(PostProcessor):
|
|
|
|
class FFmpegExtractAudioPP(PostProcessor):
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, downloader=None, preferredcodec=None):
|
|
|
|
def __init__(self, downloader=None, preferredcodec=None):
|
|
|
|