|
|
|
@ -12,490 +12,490 @@ import email.utils
|
|
|
|
import json
|
|
|
|
import json
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
import urllib.request as compat_urllib_request
|
|
|
|
import urllib.request as compat_urllib_request
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
import urllib2 as compat_urllib_request
|
|
|
|
import urllib2 as compat_urllib_request
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
import urllib.error as compat_urllib_error
|
|
|
|
import urllib.error as compat_urllib_error
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
import urllib2 as compat_urllib_error
|
|
|
|
import urllib2 as compat_urllib_error
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
import urllib.parse as compat_urllib_parse
|
|
|
|
import urllib.parse as compat_urllib_parse
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
import urllib as compat_urllib_parse
|
|
|
|
import urllib as compat_urllib_parse
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
import http.cookiejar as compat_cookiejar
|
|
|
|
import http.cookiejar as compat_cookiejar
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
import cookielib as compat_cookiejar
|
|
|
|
import cookielib as compat_cookiejar
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
import html.entities as compat_html_entities
|
|
|
|
import html.entities as compat_html_entities
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
import htmlentitydefs as compat_html_entities
|
|
|
|
import htmlentitydefs as compat_html_entities
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
import html.parser as compat_html_parser
|
|
|
|
import html.parser as compat_html_parser
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
import HTMLParser as compat_html_parser
|
|
|
|
import HTMLParser as compat_html_parser
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
import http.client as compat_http_client
|
|
|
|
import http.client as compat_http_client
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
import httplib as compat_http_client
|
|
|
|
import httplib as compat_http_client
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
from urllib.parse import parse_qs as compat_parse_qs
|
|
|
|
from urllib.parse import parse_qs as compat_parse_qs
|
|
|
|
except ImportError: # Python 2
|
|
|
|
except ImportError: # Python 2
|
|
|
|
# HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
|
|
|
|
# HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
|
|
|
|
# Python 2's version is apparently totally broken
|
|
|
|
# Python 2's version is apparently totally broken
|
|
|
|
def _unquote(string, encoding='utf-8', errors='replace'):
|
|
|
|
def _unquote(string, encoding='utf-8', errors='replace'):
|
|
|
|
if string == '':
|
|
|
|
if string == '':
|
|
|
|
return string
|
|
|
|
return string
|
|
|
|
res = string.split('%')
|
|
|
|
res = string.split('%')
|
|
|
|
if len(res) == 1:
|
|
|
|
if len(res) == 1:
|
|
|
|
return string
|
|
|
|
return string
|
|
|
|
if encoding is None:
|
|
|
|
if encoding is None:
|
|
|
|
encoding = 'utf-8'
|
|
|
|
encoding = 'utf-8'
|
|
|
|
if errors is None:
|
|
|
|
if errors is None:
|
|
|
|
errors = 'replace'
|
|
|
|
errors = 'replace'
|
|
|
|
# pct_sequence: contiguous sequence of percent-encoded bytes, decoded
|
|
|
|
# pct_sequence: contiguous sequence of percent-encoded bytes, decoded
|
|
|
|
pct_sequence = b''
|
|
|
|
pct_sequence = b''
|
|
|
|
string = res[0]
|
|
|
|
string = res[0]
|
|
|
|
for item in res[1:]:
|
|
|
|
for item in res[1:]:
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
if not item:
|
|
|
|
if not item:
|
|
|
|
raise ValueError
|
|
|
|
raise ValueError
|
|
|
|
pct_sequence += item[:2].decode('hex')
|
|
|
|
pct_sequence += item[:2].decode('hex')
|
|
|
|
rest = item[2:]
|
|
|
|
rest = item[2:]
|
|
|
|
if not rest:
|
|
|
|
if not rest:
|
|
|
|
# This segment was just a single percent-encoded character.
|
|
|
|
# This segment was just a single percent-encoded character.
|
|
|
|
# May be part of a sequence of code units, so delay decoding.
|
|
|
|
# May be part of a sequence of code units, so delay decoding.
|
|
|
|
# (Stored in pct_sequence).
|
|
|
|
# (Stored in pct_sequence).
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
except ValueError:
|
|
|
|
except ValueError:
|
|
|
|
rest = '%' + item
|
|
|
|
rest = '%' + item
|
|
|
|
# Encountered non-percent-encoded characters. Flush the current
|
|
|
|
# Encountered non-percent-encoded characters. Flush the current
|
|
|
|
# pct_sequence.
|
|
|
|
# pct_sequence.
|
|
|
|
string += pct_sequence.decode(encoding, errors) + rest
|
|
|
|
string += pct_sequence.decode(encoding, errors) + rest
|
|
|
|
pct_sequence = b''
|
|
|
|
pct_sequence = b''
|
|
|
|
if pct_sequence:
|
|
|
|
if pct_sequence:
|
|
|
|
# Flush the final pct_sequence
|
|
|
|
# Flush the final pct_sequence
|
|
|
|
string += pct_sequence.decode(encoding, errors)
|
|
|
|
string += pct_sequence.decode(encoding, errors)
|
|
|
|
return string
|
|
|
|
return string
|
|
|
|
|
|
|
|
|
|
|
|
def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
|
|
|
def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
|
|
|
encoding='utf-8', errors='replace'):
|
|
|
|
encoding='utf-8', errors='replace'):
|
|
|
|
qs, _coerce_result = qs, unicode
|
|
|
|
qs, _coerce_result = qs, unicode
|
|
|
|
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
|
|
|
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
|
|
|
r = []
|
|
|
|
r = []
|
|
|
|
for name_value in pairs:
|
|
|
|
for name_value in pairs:
|
|
|
|
if not name_value and not strict_parsing:
|
|
|
|
if not name_value and not strict_parsing:
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
nv = name_value.split('=', 1)
|
|
|
|
nv = name_value.split('=', 1)
|
|
|
|
if len(nv) != 2:
|
|
|
|
if len(nv) != 2:
|
|
|
|
if strict_parsing:
|
|
|
|
if strict_parsing:
|
|
|
|
raise ValueError("bad query field: %r" % (name_value,))
|
|
|
|
raise ValueError("bad query field: %r" % (name_value,))
|
|
|
|
# Handle case of a control-name with no equal sign
|
|
|
|
# Handle case of a control-name with no equal sign
|
|
|
|
if keep_blank_values:
|
|
|
|
if keep_blank_values:
|
|
|
|
nv.append('')
|
|
|
|
nv.append('')
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
if len(nv[1]) or keep_blank_values:
|
|
|
|
if len(nv[1]) or keep_blank_values:
|
|
|
|
name = nv[0].replace('+', ' ')
|
|
|
|
name = nv[0].replace('+', ' ')
|
|
|
|
name = _unquote(name, encoding=encoding, errors=errors)
|
|
|
|
name = _unquote(name, encoding=encoding, errors=errors)
|
|
|
|
name = _coerce_result(name)
|
|
|
|
name = _coerce_result(name)
|
|
|
|
value = nv[1].replace('+', ' ')
|
|
|
|
value = nv[1].replace('+', ' ')
|
|
|
|
value = _unquote(value, encoding=encoding, errors=errors)
|
|
|
|
value = _unquote(value, encoding=encoding, errors=errors)
|
|
|
|
value = _coerce_result(value)
|
|
|
|
value = _coerce_result(value)
|
|
|
|
r.append((name, value))
|
|
|
|
r.append((name, value))
|
|
|
|
return r
|
|
|
|
return r
|
|
|
|
|
|
|
|
|
|
|
|
def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
|
|
|
def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
|
|
|
encoding='utf-8', errors='replace'):
|
|
|
|
encoding='utf-8', errors='replace'):
|
|
|
|
parsed_result = {}
|
|
|
|
parsed_result = {}
|
|
|
|
pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
|
|
|
|
pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
|
|
|
|
encoding=encoding, errors=errors)
|
|
|
|
encoding=encoding, errors=errors)
|
|
|
|
for name, value in pairs:
|
|
|
|
for name, value in pairs:
|
|
|
|
if name in parsed_result:
|
|
|
|
if name in parsed_result:
|
|
|
|
parsed_result[name].append(value)
|
|
|
|
parsed_result[name].append(value)
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
parsed_result[name] = [value]
|
|
|
|
parsed_result[name] = [value]
|
|
|
|
return parsed_result
|
|
|
|
return parsed_result
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
compat_str = unicode # Python 2
|
|
|
|
compat_str = unicode # Python 2
|
|
|
|
except NameError:
|
|
|
|
except NameError:
|
|
|
|
compat_str = str
|
|
|
|
compat_str = str
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
compat_chr = unichr # Python 2
|
|
|
|
compat_chr = unichr # Python 2
|
|
|
|
except NameError:
|
|
|
|
except NameError:
|
|
|
|
compat_chr = chr
|
|
|
|
compat_chr = chr
|
|
|
|
|
|
|
|
|
|
|
|
std_headers = {
|
|
|
|
std_headers = {
|
|
|
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
|
|
|
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
|
|
|
|
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
|
|
|
|
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
|
|
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
|
|
'Accept-Language': 'en-us,en;q=0.5',
|
|
|
|
'Accept-Language': 'en-us,en;q=0.5',
|
|
|
|
}
|
|
|
|
}
|
|
|
|
def preferredencoding():
|
|
|
|
def preferredencoding():
|
|
|
|
"""Get preferred encoding.
|
|
|
|
"""Get preferred encoding.
|
|
|
|
|
|
|
|
|
|
|
|
Returns the best encoding scheme for the system, based on
|
|
|
|
Returns the best encoding scheme for the system, based on
|
|
|
|
locale.getpreferredencoding() and some further tweaks.
|
|
|
|
locale.getpreferredencoding() and some further tweaks.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
pref = locale.getpreferredencoding()
|
|
|
|
pref = locale.getpreferredencoding()
|
|
|
|
u'TEST'.encode(pref)
|
|
|
|
u'TEST'.encode(pref)
|
|
|
|
except:
|
|
|
|
except:
|
|
|
|
pref = 'UTF-8'
|
|
|
|
pref = 'UTF-8'
|
|
|
|
|
|
|
|
|
|
|
|
return pref
|
|
|
|
return pref
|
|
|
|
|
|
|
|
|
|
|
|
if sys.version_info < (3,0):
|
|
|
|
if sys.version_info < (3,0):
|
|
|
|
def compat_print(s):
|
|
|
|
def compat_print(s):
|
|
|
|
print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
|
|
|
|
print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
def compat_print(s):
|
|
|
|
def compat_print(s):
|
|
|
|
assert type(s) == type(u'')
|
|
|
|
assert type(s) == type(u'')
|
|
|
|
print(s)
|
|
|
|
print(s)
|
|
|
|
|
|
|
|
|
|
|
|
def htmlentity_transform(matchobj):
|
|
|
|
def htmlentity_transform(matchobj):
|
|
|
|
"""Transforms an HTML entity to a character.
|
|
|
|
"""Transforms an HTML entity to a character.
|
|
|
|
|
|
|
|
|
|
|
|
This function receives a match object and is intended to be used with
|
|
|
|
This function receives a match object and is intended to be used with
|
|
|
|
the re.sub() function.
|
|
|
|
the re.sub() function.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
entity = matchobj.group(1)
|
|
|
|
entity = matchobj.group(1)
|
|
|
|
|
|
|
|
|
|
|
|
# Known non-numeric HTML entity
|
|
|
|
# Known non-numeric HTML entity
|
|
|
|
if entity in compat_html_entities.name2codepoint:
|
|
|
|
if entity in compat_html_entities.name2codepoint:
|
|
|
|
return compat_chr(compat_html_entities.name2codepoint[entity])
|
|
|
|
return compat_chr(compat_html_entities.name2codepoint[entity])
|
|
|
|
|
|
|
|
|
|
|
|
mobj = re.match(u'(?u)#(x?\\d+)', entity)
|
|
|
|
mobj = re.match(u'(?u)#(x?\\d+)', entity)
|
|
|
|
if mobj is not None:
|
|
|
|
if mobj is not None:
|
|
|
|
numstr = mobj.group(1)
|
|
|
|
numstr = mobj.group(1)
|
|
|
|
if numstr.startswith(u'x'):
|
|
|
|
if numstr.startswith(u'x'):
|
|
|
|
base = 16
|
|
|
|
base = 16
|
|
|
|
numstr = u'0%s' % numstr
|
|
|
|
numstr = u'0%s' % numstr
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
base = 10
|
|
|
|
base = 10
|
|
|
|
return compat_chr(int(numstr, base))
|
|
|
|
return compat_chr(int(numstr, base))
|
|
|
|
|
|
|
|
|
|
|
|
# Unknown entity in name, return its literal representation
|
|
|
|
# Unknown entity in name, return its literal representation
|
|
|
|
return (u'&%s;' % entity)
|
|
|
|
return (u'&%s;' % entity)
|
|
|
|
|
|
|
|
|
|
|
|
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
|
|
|
|
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
|
|
|
|
class IDParser(compat_html_parser.HTMLParser):
|
|
|
|
class IDParser(compat_html_parser.HTMLParser):
|
|
|
|
"""Modified HTMLParser that isolates a tag with the specified id"""
|
|
|
|
"""Modified HTMLParser that isolates a tag with the specified id"""
|
|
|
|
def __init__(self, id):
|
|
|
|
def __init__(self, id):
|
|
|
|
self.id = id
|
|
|
|
self.id = id
|
|
|
|
self.result = None
|
|
|
|
self.result = None
|
|
|
|
self.started = False
|
|
|
|
self.started = False
|
|
|
|
self.depth = {}
|
|
|
|
self.depth = {}
|
|
|
|
self.html = None
|
|
|
|
self.html = None
|
|
|
|
self.watch_startpos = False
|
|
|
|
self.watch_startpos = False
|
|
|
|
self.error_count = 0
|
|
|
|
self.error_count = 0
|
|
|
|
compat_html_parser.HTMLParser.__init__(self)
|
|
|
|
compat_html_parser.HTMLParser.__init__(self)
|
|
|
|
|
|
|
|
|
|
|
|
def error(self, message):
|
|
|
|
def error(self, message):
|
|
|
|
if self.error_count > 10 or self.started:
|
|
|
|
if self.error_count > 10 or self.started:
|
|
|
|
raise compat_html_parser.HTMLParseError(message, self.getpos())
|
|
|
|
raise compat_html_parser.HTMLParseError(message, self.getpos())
|
|
|
|
self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
|
|
|
|
self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
|
|
|
|
self.error_count += 1
|
|
|
|
self.error_count += 1
|
|
|
|
self.goahead(1)
|
|
|
|
self.goahead(1)
|
|
|
|
|
|
|
|
|
|
|
|
def loads(self, html):
|
|
|
|
def loads(self, html):
|
|
|
|
self.html = html
|
|
|
|
self.html = html
|
|
|
|
self.feed(html)
|
|
|
|
self.feed(html)
|
|
|
|
self.close()
|
|
|
|
self.close()
|
|
|
|
|
|
|
|
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
|
|
attrs = dict(attrs)
|
|
|
|
attrs = dict(attrs)
|
|
|
|
if self.started:
|
|
|
|
if self.started:
|
|
|
|
self.find_startpos(None)
|
|
|
|
self.find_startpos(None)
|
|
|
|
if 'id' in attrs and attrs['id'] == self.id:
|
|
|
|
if 'id' in attrs and attrs['id'] == self.id:
|
|
|
|
self.result = [tag]
|
|
|
|
self.result = [tag]
|
|
|
|
self.started = True
|
|
|
|
self.started = True
|
|
|
|
self.watch_startpos = True
|
|
|
|
self.watch_startpos = True
|
|
|
|
if self.started:
|
|
|
|
if self.started:
|
|
|
|
if not tag in self.depth: self.depth[tag] = 0
|
|
|
|
if not tag in self.depth: self.depth[tag] = 0
|
|
|
|
self.depth[tag] += 1
|
|
|
|
self.depth[tag] += 1
|
|
|
|
|
|
|
|
|
|
|
|
def handle_endtag(self, tag):
|
|
|
|
def handle_endtag(self, tag):
|
|
|
|
if self.started:
|
|
|
|
if self.started:
|
|
|
|
if tag in self.depth: self.depth[tag] -= 1
|
|
|
|
if tag in self.depth: self.depth[tag] -= 1
|
|
|
|
if self.depth[self.result[0]] == 0:
|
|
|
|
if self.depth[self.result[0]] == 0:
|
|
|
|
self.started = False
|
|
|
|
self.started = False
|
|
|
|
self.result.append(self.getpos())
|
|
|
|
self.result.append(self.getpos())
|
|
|
|
|
|
|
|
|
|
|
|
def find_startpos(self, x):
|
|
|
|
def find_startpos(self, x):
|
|
|
|
"""Needed to put the start position of the result (self.result[1])
|
|
|
|
"""Needed to put the start position of the result (self.result[1])
|
|
|
|
after the opening tag with the requested id"""
|
|
|
|
after the opening tag with the requested id"""
|
|
|
|
if self.watch_startpos:
|
|
|
|
if self.watch_startpos:
|
|
|
|
self.watch_startpos = False
|
|
|
|
self.watch_startpos = False
|
|
|
|
self.result.append(self.getpos())
|
|
|
|
self.result.append(self.getpos())
|
|
|
|
handle_entityref = handle_charref = handle_data = handle_comment = \
|
|
|
|
handle_entityref = handle_charref = handle_data = handle_comment = \
|
|
|
|
handle_decl = handle_pi = unknown_decl = find_startpos
|
|
|
|
handle_decl = handle_pi = unknown_decl = find_startpos
|
|
|
|
|
|
|
|
|
|
|
|
def get_result(self):
|
|
|
|
def get_result(self):
|
|
|
|
if self.result is None:
|
|
|
|
if self.result is None:
|
|
|
|
return None
|
|
|
|
return None
|
|
|
|
if len(self.result) != 3:
|
|
|
|
if len(self.result) != 3:
|
|
|
|
return None
|
|
|
|
return None
|
|
|
|
lines = self.html.split('\n')
|
|
|
|
lines = self.html.split('\n')
|
|
|
|
lines = lines[self.result[1][0]-1:self.result[2][0]]
|
|
|
|
lines = lines[self.result[1][0]-1:self.result[2][0]]
|
|
|
|
lines[0] = lines[0][self.result[1][1]:]
|
|
|
|
lines[0] = lines[0][self.result[1][1]:]
|
|
|
|
if len(lines) == 1:
|
|
|
|
if len(lines) == 1:
|
|
|
|
lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
|
|
|
|
lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
|
|
|
|
lines[-1] = lines[-1][:self.result[2][1]]
|
|
|
|
lines[-1] = lines[-1][:self.result[2][1]]
|
|
|
|
return '\n'.join(lines).strip()
|
|
|
|
return '\n'.join(lines).strip()
|
|
|
|
|
|
|
|
|
|
|
|
def get_element_by_id(id, html):
|
|
|
|
def get_element_by_id(id, html):
|
|
|
|
"""Return the content of the tag with the specified id in the passed HTML document"""
|
|
|
|
"""Return the content of the tag with the specified id in the passed HTML document"""
|
|
|
|
parser = IDParser(id)
|
|
|
|
parser = IDParser(id)
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
parser.loads(html)
|
|
|
|
parser.loads(html)
|
|
|
|
except compat_html_parser.HTMLParseError:
|
|
|
|
except compat_html_parser.HTMLParseError:
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
return parser.get_result()
|
|
|
|
return parser.get_result()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def clean_html(html):
|
|
|
|
def clean_html(html):
|
|
|
|
"""Clean an HTML snippet into a readable string"""
|
|
|
|
"""Clean an HTML snippet into a readable string"""
|
|
|
|
# Newline vs <br />
|
|
|
|
# Newline vs <br />
|
|
|
|
html = html.replace('\n', ' ')
|
|
|
|
html = html.replace('\n', ' ')
|
|
|
|
html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
|
|
|
|
html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
|
|
|
|
# Strip html tags
|
|
|
|
# Strip html tags
|
|
|
|
html = re.sub('<.*?>', '', html)
|
|
|
|
html = re.sub('<.*?>', '', html)
|
|
|
|
# Replace html entities
|
|
|
|
# Replace html entities
|
|
|
|
html = unescapeHTML(html)
|
|
|
|
html = unescapeHTML(html)
|
|
|
|
return html
|
|
|
|
return html
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sanitize_open(filename, open_mode):
|
|
|
|
def sanitize_open(filename, open_mode):
|
|
|
|
"""Try to open the given filename, and slightly tweak it if this fails.
|
|
|
|
"""Try to open the given filename, and slightly tweak it if this fails.
|
|
|
|
|
|
|
|
|
|
|
|
Attempts to open the given filename. If this fails, it tries to change
|
|
|
|
Attempts to open the given filename. If this fails, it tries to change
|
|
|
|
the filename slightly, step by step, until it's either able to open it
|
|
|
|
the filename slightly, step by step, until it's either able to open it
|
|
|
|
or it fails and raises a final exception, like the standard open()
|
|
|
|
or it fails and raises a final exception, like the standard open()
|
|
|
|
function.
|
|
|
|
function.
|
|
|
|
|
|
|
|
|
|
|
|
It returns the tuple (stream, definitive_file_name).
|
|
|
|
It returns the tuple (stream, definitive_file_name).
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
if filename == u'-':
|
|
|
|
if filename == u'-':
|
|
|
|
if sys.platform == 'win32':
|
|
|
|
if sys.platform == 'win32':
|
|
|
|
import msvcrt
|
|
|
|
import msvcrt
|
|
|
|
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
|
|
|
|
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
|
|
|
|
return (sys.stdout, filename)
|
|
|
|
return (sys.stdout, filename)
|
|
|
|
stream = open(encodeFilename(filename), open_mode)
|
|
|
|
stream = open(encodeFilename(filename), open_mode)
|
|
|
|
return (stream, filename)
|
|
|
|
return (stream, filename)
|
|
|
|
except (IOError, OSError) as err:
|
|
|
|
except (IOError, OSError) as err:
|
|
|
|
# In case of error, try to remove win32 forbidden chars
|
|
|
|
# In case of error, try to remove win32 forbidden chars
|
|
|
|
filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
|
|
|
|
filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
|
|
|
|
|
|
|
|
|
|
|
|
# An exception here should be caught in the caller
|
|
|
|
# An exception here should be caught in the caller
|
|
|
|
stream = open(encodeFilename(filename), open_mode)
|
|
|
|
stream = open(encodeFilename(filename), open_mode)
|
|
|
|
return (stream, filename)
|
|
|
|
return (stream, filename)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def timeconvert(timestr):
|
|
|
|
def timeconvert(timestr):
|
|
|
|
"""Convert RFC 2822 defined time string into system timestamp"""
|
|
|
|
"""Convert RFC 2822 defined time string into system timestamp"""
|
|
|
|
timestamp = None
|
|
|
|
timestamp = None
|
|
|
|
timetuple = email.utils.parsedate_tz(timestr)
|
|
|
|
timetuple = email.utils.parsedate_tz(timestr)
|
|
|
|
if timetuple is not None:
|
|
|
|
if timetuple is not None:
|
|
|
|
timestamp = email.utils.mktime_tz(timetuple)
|
|
|
|
timestamp = email.utils.mktime_tz(timetuple)
|
|
|
|
return timestamp
|
|
|
|
return timestamp
|
|
|
|
|
|
|
|
|
|
|
|
def sanitize_filename(s, restricted=False):
|
|
|
|
def sanitize_filename(s, restricted=False):
|
|
|
|
"""Sanitizes a string so it could be used as part of a filename.
|
|
|
|
"""Sanitizes a string so it could be used as part of a filename.
|
|
|
|
If restricted is set, use a stricter subset of allowed characters.
|
|
|
|
If restricted is set, use a stricter subset of allowed characters.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
def replace_insane(char):
|
|
|
|
def replace_insane(char):
|
|
|
|
if char == '?' or ord(char) < 32 or ord(char) == 127:
|
|
|
|
if char == '?' or ord(char) < 32 or ord(char) == 127:
|
|
|
|
return ''
|
|
|
|
return ''
|
|
|
|
elif char == '"':
|
|
|
|
elif char == '"':
|
|
|
|
return '' if restricted else '\''
|
|
|
|
return '' if restricted else '\''
|
|
|
|
elif char == ':':
|
|
|
|
elif char == ':':
|
|
|
|
return '_-' if restricted else ' -'
|
|
|
|
return '_-' if restricted else ' -'
|
|
|
|
elif char in '\\/|*<>':
|
|
|
|
elif char in '\\/|*<>':
|
|
|
|
return '_'
|
|
|
|
return '_'
|
|
|
|
if restricted and (char in '!&\'' or char.isspace()):
|
|
|
|
if restricted and (char in '!&\'' or char.isspace()):
|
|
|
|
return '_'
|
|
|
|
return '_'
|
|
|
|
if restricted and ord(char) > 127:
|
|
|
|
if restricted and ord(char) > 127:
|
|
|
|
return '_'
|
|
|
|
return '_'
|
|
|
|
return char
|
|
|
|
return char
|
|
|
|
|
|
|
|
|
|
|
|
result = u''.join(map(replace_insane, s))
|
|
|
|
result = u''.join(map(replace_insane, s))
|
|
|
|
while '__' in result:
|
|
|
|
while '__' in result:
|
|
|
|
result = result.replace('__', '_')
|
|
|
|
result = result.replace('__', '_')
|
|
|
|
result = result.strip('_')
|
|
|
|
result = result.strip('_')
|
|
|
|
# Common case of "Foreign band name - English song title"
|
|
|
|
# Common case of "Foreign band name - English song title"
|
|
|
|
if restricted and result.startswith('-_'):
|
|
|
|
if restricted and result.startswith('-_'):
|
|
|
|
result = result[2:]
|
|
|
|
result = result[2:]
|
|
|
|
if not result:
|
|
|
|
if not result:
|
|
|
|
result = '_'
|
|
|
|
result = '_'
|
|
|
|
return result
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
|
|
def orderedSet(iterable):
|
|
|
|
def orderedSet(iterable):
|
|
|
|
""" Remove all duplicates from the input iterable """
|
|
|
|
""" Remove all duplicates from the input iterable """
|
|
|
|
res = []
|
|
|
|
res = []
|
|
|
|
for el in iterable:
|
|
|
|
for el in iterable:
|
|
|
|
if el not in res:
|
|
|
|
if el not in res:
|
|
|
|
res.append(el)
|
|
|
|
res.append(el)
|
|
|
|
return res
|
|
|
|
return res
|
|
|
|
|
|
|
|
|
|
|
|
def unescapeHTML(s):
|
|
|
|
def unescapeHTML(s):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
@param s a string
|
|
|
|
@param s a string
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
assert type(s) == type(u'')
|
|
|
|
assert type(s) == type(u'')
|
|
|
|
|
|
|
|
|
|
|
|
result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
|
|
|
|
result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
|
|
|
|
return result
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
|
|
def encodeFilename(s):
|
|
|
|
def encodeFilename(s):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
@param s The name of the file
|
|
|
|
@param s The name of the file
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
assert type(s) == type(u'')
|
|
|
|
assert type(s) == type(u'')
|
|
|
|
|
|
|
|
|
|
|
|
# Python 3 has a Unicode API
|
|
|
|
# Python 3 has a Unicode API
|
|
|
|
if sys.version_info >= (3, 0):
|
|
|
|
if sys.version_info >= (3, 0):
|
|
|
|
return s
|
|
|
|
return s
|
|
|
|
|
|
|
|
|
|
|
|
if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
|
|
|
|
if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
|
|
|
|
# Pass u'' directly to use Unicode APIs on Windows 2000 and up
|
|
|
|
# Pass u'' directly to use Unicode APIs on Windows 2000 and up
|
|
|
|
# (Detecting Windows NT 4 is tricky because 'major >= 4' would
|
|
|
|
# (Detecting Windows NT 4 is tricky because 'major >= 4' would
|
|
|
|
# match Windows 9x series as well. Besides, NT 4 is obsolete.)
|
|
|
|
# match Windows 9x series as well. Besides, NT 4 is obsolete.)
|
|
|
|
return s
|
|
|
|
return s
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
return s.encode(sys.getfilesystemencoding(), 'ignore')
|
|
|
|
return s.encode(sys.getfilesystemencoding(), 'ignore')
|
|
|
|
|
|
|
|
|
|
|
|
class DownloadError(Exception):
|
|
|
|
class DownloadError(Exception):
|
|
|
|
"""Download Error exception.
|
|
|
|
"""Download Error exception.
|
|
|
|
|
|
|
|
|
|
|
|
This exception may be thrown by FileDownloader objects if they are not
|
|
|
|
This exception may be thrown by FileDownloader objects if they are not
|
|
|
|
configured to continue on errors. They will contain the appropriate
|
|
|
|
configured to continue on errors. They will contain the appropriate
|
|
|
|
error message.
|
|
|
|
error message.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SameFileError(Exception):
|
|
|
|
class SameFileError(Exception):
|
|
|
|
"""Same File exception.
|
|
|
|
"""Same File exception.
|
|
|
|
|
|
|
|
|
|
|
|
This exception will be thrown by FileDownloader objects if they detect
|
|
|
|
This exception will be thrown by FileDownloader objects if they detect
|
|
|
|
multiple files would have to be downloaded to the same file on disk.
|
|
|
|
multiple files would have to be downloaded to the same file on disk.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PostProcessingError(Exception):
|
|
|
|
class PostProcessingError(Exception):
|
|
|
|
"""Post Processing exception.
|
|
|
|
"""Post Processing exception.
|
|
|
|
|
|
|
|
|
|
|
|
This exception may be raised by PostProcessor's .run() method to
|
|
|
|
This exception may be raised by PostProcessor's .run() method to
|
|
|
|
indicate an error in the postprocessing task.
|
|
|
|
indicate an error in the postprocessing task.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
class MaxDownloadsReached(Exception):
|
|
|
|
class MaxDownloadsReached(Exception):
|
|
|
|
""" --max-downloads limit has been reached. """
|
|
|
|
""" --max-downloads limit has been reached. """
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class UnavailableVideoError(Exception):
|
|
|
|
class UnavailableVideoError(Exception):
|
|
|
|
"""Unavailable Format exception.
|
|
|
|
"""Unavailable Format exception.
|
|
|
|
|
|
|
|
|
|
|
|
This exception will be thrown when a video is requested
|
|
|
|
This exception will be thrown when a video is requested
|
|
|
|
in a format that is not available for that video.
|
|
|
|
in a format that is not available for that video.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ContentTooShortError(Exception):
|
|
|
|
class ContentTooShortError(Exception):
|
|
|
|
"""Content Too Short exception.
|
|
|
|
"""Content Too Short exception.
|
|
|
|
|
|
|
|
|
|
|
|
This exception may be raised by FileDownloader objects when a file they
|
|
|
|
This exception may be raised by FileDownloader objects when a file they
|
|
|
|
download is too small for what the server announced first, indicating
|
|
|
|
download is too small for what the server announced first, indicating
|
|
|
|
the connection was probably interrupted.
|
|
|
|
the connection was probably interrupted.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
# Both in bytes
|
|
|
|
# Both in bytes
|
|
|
|
downloaded = None
|
|
|
|
downloaded = None
|
|
|
|
expected = None
|
|
|
|
expected = None
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, downloaded, expected):
|
|
|
|
def __init__(self, downloaded, expected):
|
|
|
|
self.downloaded = downloaded
|
|
|
|
self.downloaded = downloaded
|
|
|
|
self.expected = expected
|
|
|
|
self.expected = expected
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Trouble(Exception):
|
|
|
|
class Trouble(Exception):
|
|
|
|
"""Trouble helper exception
|
|
|
|
"""Trouble helper exception
|
|
|
|
|
|
|
|
|
|
|
|
This is an exception to be handled with
|
|
|
|
This is an exception to be handled with
|
|
|
|
FileDownloader.trouble
|
|
|
|
FileDownloader.trouble
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|
|
|
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|
|
|
"""Handler for HTTP requests and responses.
|
|
|
|
"""Handler for HTTP requests and responses.
|
|
|
|
|
|
|
|
|
|
|
|
This class, when installed with an OpenerDirector, automatically adds
|
|
|
|
This class, when installed with an OpenerDirector, automatically adds
|
|
|
|
the standard headers to every HTTP request and handles gzipped and
|
|
|
|
the standard headers to every HTTP request and handles gzipped and
|
|
|
|
deflated responses from web servers. If compression is to be avoided in
|
|
|
|
deflated responses from web servers. If compression is to be avoided in
|
|
|
|
a particular request, the original request in the program code only has
|
|
|
|
a particular request, the original request in the program code only has
|
|
|
|
to include the HTTP header "Youtubedl-No-Compression", which will be
|
|
|
|
to include the HTTP header "Youtubedl-No-Compression", which will be
|
|
|
|
removed before making the real request.
|
|
|
|
removed before making the real request.
|
|
|
|
|
|
|
|
|
|
|
|
Part of this code was copied from:
|
|
|
|
Part of this code was copied from:
|
|
|
|
|
|
|
|
|
|
|
|
http://techknack.net/python-urllib2-handlers/
|
|
|
|
http://techknack.net/python-urllib2-handlers/
|
|
|
|
|
|
|
|
|
|
|
|
Andrew Rowls, the author of that code, agreed to release it to the
|
|
|
|
Andrew Rowls, the author of that code, agreed to release it to the
|
|
|
|
public domain.
|
|
|
|
public domain.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
@staticmethod
|
|
|
|
def deflate(data):
|
|
|
|
def deflate(data):
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
return zlib.decompress(data, -zlib.MAX_WBITS)
|
|
|
|
return zlib.decompress(data, -zlib.MAX_WBITS)
|
|
|
|
except zlib.error:
|
|
|
|
except zlib.error:
|
|
|
|
return zlib.decompress(data)
|
|
|
|
return zlib.decompress(data)
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
@staticmethod
|
|
|
|
def addinfourl_wrapper(stream, headers, url, code):
|
|
|
|
def addinfourl_wrapper(stream, headers, url, code):
|
|
|
|
if hasattr(compat_urllib_request.addinfourl, 'getcode'):
|
|
|
|
if hasattr(compat_urllib_request.addinfourl, 'getcode'):
|
|
|
|
return compat_urllib_request.addinfourl(stream, headers, url, code)
|
|
|
|
return compat_urllib_request.addinfourl(stream, headers, url, code)
|
|
|
|
ret = compat_urllib_request.addinfourl(stream, headers, url)
|
|
|
|
ret = compat_urllib_request.addinfourl(stream, headers, url)
|
|
|
|
ret.code = code
|
|
|
|
ret.code = code
|
|
|
|
return ret
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
def http_request(self, req):
|
|
|
|
def http_request(self, req):
|
|
|
|
for h in std_headers:
|
|
|
|
for h in std_headers:
|
|
|
|
if h in req.headers:
|
|
|
|
if h in req.headers:
|
|
|
|
del req.headers[h]
|
|
|
|
del req.headers[h]
|
|
|
|
req.add_header(h, std_headers[h])
|
|
|
|
req.add_header(h, std_headers[h])
|
|
|
|
if 'Youtubedl-no-compression' in req.headers:
|
|
|
|
if 'Youtubedl-no-compression' in req.headers:
|
|
|
|
if 'Accept-encoding' in req.headers:
|
|
|
|
if 'Accept-encoding' in req.headers:
|
|
|
|
del req.headers['Accept-encoding']
|
|
|
|
del req.headers['Accept-encoding']
|
|
|
|
del req.headers['Youtubedl-no-compression']
|
|
|
|
del req.headers['Youtubedl-no-compression']
|
|
|
|
return req
|
|
|
|
return req
|
|
|
|
|
|
|
|
|
|
|
|
def http_response(self, req, resp):
|
|
|
|
def http_response(self, req, resp):
|
|
|
|
old_resp = resp
|
|
|
|
old_resp = resp
|
|
|
|
# gzip
|
|
|
|
# gzip
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'gzip':
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'gzip':
|
|
|
|
gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
|
|
|
|
gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
|
|
|
|
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
# deflate
|
|
|
|
# deflate
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'deflate':
|
|
|
|
if resp.headers.get('Content-encoding', '') == 'deflate':
|
|
|
|
gz = io.BytesIO(self.deflate(resp.read()))
|
|
|
|
gz = io.BytesIO(self.deflate(resp.read()))
|
|
|
|
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
resp.msg = old_resp.msg
|
|
|
|
return resp
|
|
|
|
return resp
|
|
|
|
|
Philipp Hagemeister
12 years ago