|
|
|
@ -3,6 +3,7 @@ from __future__ import unicode_literals
|
|
|
|
|
|
|
|
|
|
|
|
import base64
|
|
|
|
import base64
|
|
|
|
import datetime
|
|
|
|
import datetime
|
|
|
|
|
|
|
|
import functools
|
|
|
|
import hashlib
|
|
|
|
import hashlib
|
|
|
|
import json
|
|
|
|
import json
|
|
|
|
import netrc
|
|
|
|
import netrc
|
|
|
|
@ -23,6 +24,7 @@ from ..compat import (
|
|
|
|
compat_getpass,
|
|
|
|
compat_getpass,
|
|
|
|
compat_integer_types,
|
|
|
|
compat_integer_types,
|
|
|
|
compat_http_client,
|
|
|
|
compat_http_client,
|
|
|
|
|
|
|
|
compat_map as map,
|
|
|
|
compat_os_name,
|
|
|
|
compat_os_name,
|
|
|
|
compat_str,
|
|
|
|
compat_str,
|
|
|
|
compat_urllib_error,
|
|
|
|
compat_urllib_error,
|
|
|
|
@ -31,6 +33,7 @@ from ..compat import (
|
|
|
|
compat_urllib_request,
|
|
|
|
compat_urllib_request,
|
|
|
|
compat_urlparse,
|
|
|
|
compat_urlparse,
|
|
|
|
compat_xml_parse_error,
|
|
|
|
compat_xml_parse_error,
|
|
|
|
|
|
|
|
compat_zip as zip,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
from ..downloader.f4m import (
|
|
|
|
from ..downloader.f4m import (
|
|
|
|
get_base_url,
|
|
|
|
get_base_url,
|
|
|
|
@ -70,6 +73,7 @@ from ..utils import (
|
|
|
|
str_or_none,
|
|
|
|
str_or_none,
|
|
|
|
str_to_int,
|
|
|
|
str_to_int,
|
|
|
|
strip_or_none,
|
|
|
|
strip_or_none,
|
|
|
|
|
|
|
|
traverse_obj,
|
|
|
|
try_get,
|
|
|
|
try_get,
|
|
|
|
unescapeHTML,
|
|
|
|
unescapeHTML,
|
|
|
|
unified_strdate,
|
|
|
|
unified_strdate,
|
|
|
|
@ -1349,6 +1353,44 @@ class InfoExtractor(object):
|
|
|
|
break
|
|
|
|
break
|
|
|
|
return dict((k, v) for k, v in info.items() if v is not None)
|
|
|
|
return dict((k, v) for k, v in info.items() if v is not None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _search_nextjs_data(self, webpage, video_id, **kw):
|
|
|
|
|
|
|
|
nkw = dict((k, v) for k, v in kw.items() if k in ('transform_source', 'fatal'))
|
|
|
|
|
|
|
|
kw.pop('transform_source', None)
|
|
|
|
|
|
|
|
next_data = self._search_regex(
|
|
|
|
|
|
|
|
r'''<script[^>]+\bid\s*=\s*('|")__NEXT_DATA__\1[^>]*>(?P<nd>[^<]+)</script>''',
|
|
|
|
|
|
|
|
webpage, 'next.js data', group='nd', **kw)
|
|
|
|
|
|
|
|
if not next_data:
|
|
|
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
return self._parse_json(next_data, video_id, **nkw)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _search_nuxt_data(self, webpage, video_id, *args, **kwargs):
|
|
|
|
|
|
|
|
"""Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)
|
|
|
|
|
|
|
|
context_name = args[0] if len(args) > 0 else kwargs.get('context_name', '__NUXT__')
|
|
|
|
|
|
|
|
fatal = kwargs.get('fatal', True)
|
|
|
|
|
|
|
|
traverse = kwargs.get('traverse', ('data', 0))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
re_ctx = re.escape(context_name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
FUNCTION_RE = (r'\(\s*function\s*\((?P<arg_keys>[\s\S]*?)\)\s*\{\s*'
|
|
|
|
|
|
|
|
r'return\s+(?P<js>\{[\s\S]*?})\s*;?\s*}\s*\((?P<arg_vals>[\s\S]*?)\)')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
js, arg_keys, arg_vals = self._search_regex(
|
|
|
|
|
|
|
|
(p.format(re_ctx, FUNCTION_RE) for p in
|
|
|
|
|
|
|
|
(r'<script>\s*window\s*\.\s*{0}\s*=\s*{1}\s*\)\s*;?\s*</script>',
|
|
|
|
|
|
|
|
r'{0}\s*\([\s\S]*?{1}')),
|
|
|
|
|
|
|
|
webpage, context_name, group=('js', 'arg_keys', 'arg_vals'),
|
|
|
|
|
|
|
|
default=NO_DEFAULT if fatal else (None, None, None))
|
|
|
|
|
|
|
|
if js is None:
|
|
|
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
args = dict(zip(arg_keys.split(','), map(json.dumps, self._parse_json(
|
|
|
|
|
|
|
|
'[{0}]'.format(arg_vals), video_id, transform_source=js_to_json, fatal=fatal) or ())))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal)
|
|
|
|
|
|
|
|
return traverse_obj(ret, traverse) or {}
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
@staticmethod
|
|
|
|
def _hidden_inputs(html):
|
|
|
|
def _hidden_inputs(html):
|
|
|
|
html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
|
|
|
|
html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
|
|
|
|
@ -2496,7 +2538,8 @@ class InfoExtractor(object):
|
|
|
|
return f
|
|
|
|
return f
|
|
|
|
return {}
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
|
|
def _media_formats(src, cur_media_type, type_info={}):
|
|
|
|
def _media_formats(src, cur_media_type, type_info=None):
|
|
|
|
|
|
|
|
type_info = type_info or {}
|
|
|
|
full_url = absolute_url(src)
|
|
|
|
full_url = absolute_url(src)
|
|
|
|
ext = type_info.get('ext') or determine_ext(full_url)
|
|
|
|
ext = type_info.get('ext') or determine_ext(full_url)
|
|
|
|
if ext == 'm3u8':
|
|
|
|
if ext == 'm3u8':
|
|
|
|
@ -2514,6 +2557,7 @@ class InfoExtractor(object):
|
|
|
|
formats = [{
|
|
|
|
formats = [{
|
|
|
|
'url': full_url,
|
|
|
|
'url': full_url,
|
|
|
|
'vcodec': 'none' if cur_media_type == 'audio' else None,
|
|
|
|
'vcodec': 'none' if cur_media_type == 'audio' else None,
|
|
|
|
|
|
|
|
'ext': ext,
|
|
|
|
}]
|
|
|
|
}]
|
|
|
|
return is_plain_url, formats
|
|
|
|
return is_plain_url, formats
|
|
|
|
|
|
|
|
|
|
|
|
@ -2522,7 +2566,7 @@ class InfoExtractor(object):
|
|
|
|
# so we wll include them right here (see
|
|
|
|
# so we wll include them right here (see
|
|
|
|
# https://www.ampproject.org/docs/reference/components/amp-video)
|
|
|
|
# https://www.ampproject.org/docs/reference/components/amp-video)
|
|
|
|
# For dl8-* tags see https://delight-vr.com/documentation/dl8-video/
|
|
|
|
# For dl8-* tags see https://delight-vr.com/documentation/dl8-video/
|
|
|
|
_MEDIA_TAG_NAME_RE = r'(?:(?:amp|dl8(?:-live)?)-)?(video|audio)'
|
|
|
|
_MEDIA_TAG_NAME_RE = r'(?:(?:amp|dl8(?:-live)?)-)?(video(?:-js)?|audio)'
|
|
|
|
media_tags = [(media_tag, media_tag_name, media_type, '')
|
|
|
|
media_tags = [(media_tag, media_tag_name, media_type, '')
|
|
|
|
for media_tag, media_tag_name, media_type
|
|
|
|
for media_tag, media_tag_name, media_type
|
|
|
|
in re.findall(r'(?s)(<(%s)[^>]*/>)' % _MEDIA_TAG_NAME_RE, webpage)]
|
|
|
|
in re.findall(r'(?s)(<(%s)[^>]*/>)' % _MEDIA_TAG_NAME_RE, webpage)]
|
|
|
|
@ -2540,7 +2584,8 @@ class InfoExtractor(object):
|
|
|
|
media_attributes = extract_attributes(media_tag)
|
|
|
|
media_attributes = extract_attributes(media_tag)
|
|
|
|
src = strip_or_none(media_attributes.get('src'))
|
|
|
|
src = strip_or_none(media_attributes.get('src'))
|
|
|
|
if src:
|
|
|
|
if src:
|
|
|
|
_, formats = _media_formats(src, media_type)
|
|
|
|
f = parse_content_type(media_attributes.get('type'))
|
|
|
|
|
|
|
|
_, formats = _media_formats(src, media_type, f)
|
|
|
|
media_info['formats'].extend(formats)
|
|
|
|
media_info['formats'].extend(formats)
|
|
|
|
media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
|
|
|
|
media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
|
|
|
|
if media_content:
|
|
|
|
if media_content:
|
|
|
|
|
dirkf
2 years ago