From 56f9c77f0e715a26b749d494c99a3f0b076bbbf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 14 May 2017 05:27:51 +0700 Subject: [PATCH] [mediaset] Improve extraction (closes #12708, closes #12964) --- youtube_dl/extractor/mediaset.py | 128 ++++++++++++++----------------- 1 file changed, 56 insertions(+), 72 deletions(-) diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py index 4b1489524..89e23ca9b 100644 --- a/youtube_dl/extractor/mediaset.py +++ b/youtube_dl/extractor/mediaset.py @@ -1,19 +1,25 @@ # coding: utf-8 from __future__ import unicode_literals -from .common import ( - InfoExtractor, - ExtractorError -) +from .common import InfoExtractor +from ..compat import compat_str from ..utils import ( determine_ext, parse_duration, - unified_strdate + try_get, + unified_strdate, ) class MediasetIE(InfoExtractor): - _VALID_URL = r'https?://www\.video\.mediaset\.it/(?:(?:video|on-demand)/(?:.+)_|player/playerIFrame(?:Twitter)?\.shtml\?id=)(?P[0-9]+)(?:.html|&.+)' + _VALID_URL = r'''(?x) + https?:// + (?:www\.)?video\.mediaset\.it/ + (?: + (?:video|on-demand)/(?:[^/]+/)+[^/]+_| + player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid= + )(?P[0-9]+) + ''' _TESTS = [{ # full episode 'url': 'http://www.video.mediaset.it/video/hello_goodbye/full/quarta-puntata_661824.html', @@ -22,97 +28,75 @@ class MediasetIE(InfoExtractor): 'id': '661824', 'ext': 'mp4', 'title': 'Quarta puntata', - 'thumbnail': 're:^https?://.*\.jpg$', 'description': 'md5:7183696d6df570e3412a5ef74b27c5e2', - 'uploader': 'mediaset' - } - }, { - # on demand - 'url': 'http://www.video.mediaset.it/video/domenica_live/interviste/il-fenomeno-elettra-lamborghini_716283.html', - 'md5': '81c57566bf2ee02e995f5342f079ca25', - 'info_dict': { - 'id': '716283', - 'ext': 'mp4', - 'title': 'Il fenomeno Elettra Lamborghini', - 'thumbnail': 're:^https?://.*\.jpg$', - 'description': 'md5:dabf0e7cf48fc6d0a3417b989028748a', - 'uploader': 'mediaset' - } + 'thumbnail': r're:^https?://.*\.jpg$', + 'duration': 1414, + 'creator': 'mediaset', + 'release_date': '20161107', + 'series': 'Hello Goodbye', + 'categories': ['reality'], + }, + 'expected_warnings': ['is not a supported codec'], }, { # clip 'url': 'http://www.video.mediaset.it/video/gogglebox/clip/un-grande-classico-della-commedia-sexy_661680.html', - 'md5': '189ca72fe399db80dbfa595a4abf42d0', - 'info_dict': { - 'id': '661680', - 'ext': 'mp4', - 'title': 'Un grande classico della commedia sexy', - 'thumbnail': 're:^https?://.*\.jpg$', - 'description': 'Un film che riesce a risvegliare i sensi di Gigi.', - 'uploader': 'mediaset' - } + 'only_matching': True, }, { # iframe simple 'url': 'http://www.video.mediaset.it/player/playerIFrame.shtml?id=665924&autoplay=true', - 'md5': '308430901e55e1ad83dddb4be2a4454a', - 'info_dict': { - 'id': '665924', - 'ext': 'mp4', - 'title': 'Gianna Nannini incontra i fan a Milano', - 'thumbnail': 're:^https?://.*\.jpg$', - 'description': 'La cantante parla del nuovo libro', - 'uploader': 'mediaset' - } + 'only_matching': True, }, { # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/) 'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true', - 'md5': '6f53a834b3b5eac1ebc2037ccf7194d0', - 'info_dict': { - 'id': '665104', - 'ext': 'mp4', - 'title': '\"Se prima mi fidavo zero...\"', - 'thumbnail': 're:^https?://.*\.jpg$', - 'description': 'Una piccola anteprima della prossima puntata del Trono Classico', - 'uploader': 'mediaset' - } + 'only_matching': True, }] def _real_extract(self, url): video_id = self._match_id(url) - formats = [] - uploader = None - categories = None - mediainfo = self._download_json( - 'http://plr.video.mediaset.it/html/metainfo.sjson?id=%s' % video_id, - video_id, 'Downloading video info JSON').get('video') + video_list = self._download_json( + 'http://cdnsel01.mediaset.net/GetCdn.aspx', + video_id, 'Downloading video CDN JSON', query={ + 'streamid': video_id, + 'format': 'json', + })['videoList'] - if 'brand-info' in mediainfo: - uploader = mediainfo.get('brand-info').get('publisher') - categories = [mediainfo.get('brand-info').get('category')] + formats = [] + for format_url in video_list: + if '.ism' in format_url: + formats.extend(self._extract_ism_formats( + format_url, video_id, ism_id='mss', fatal=False)) + else: + formats.append({ + 'url': format_url, + 'format_id': determine_ext(format_url), + }) + self._sort_formats(formats) - cnd = self._download_json( - 'http://cdnsel01.mediaset.net/GetCdn.aspx?streamid=%s&format=json' % video_id, - video_id, 'Downloading video CND JSON') + mediainfo = self._download_json( + 'http://plr.video.mediaset.it/html/metainfo.sjson', + video_id, 'Downloading video info JSON', query={ + 'id': video_id, + })['video'] - if not cnd.get('videoList'): - raise ExtractorError('Video not found') + title = mediainfo['title'] - for media_url in cnd.get('videoList'): - formats.append({ - 'url': media_url, - 'ext': determine_ext(media_url) - }) - self._sort_formats(formats) + creator = try_get( + mediainfo, lambda x: x['brand-info']['publisher'], compat_str) + category = try_get( + mediainfo, lambda x: x['brand-info']['category'], compat_str) + categories = [category] if category else None return { 'id': video_id, - 'title': mediainfo.get('title'), - 'formats': formats, + 'title': title, 'description': mediainfo.get('short-description'), - 'uploader': uploader, 'thumbnail': mediainfo.get('thumbnail'), 'duration': parse_duration(mediainfo.get('duration')), + 'creator': creator, 'release_date': unified_strdate(mediainfo.get('production-date')), 'webpage_url': mediainfo.get('url'), - 'categories': categories + 'series': mediainfo.get('brand-value'), + 'categories': categories, + 'formats': formats, }