[mediaset] Improve extraction (closes #12708, closes #12964)

youtube_dl/extractor/mediaset.py

@@ -1,19 +1,25 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import (
+from .common import InfoExtractor
-    InfoExtractor,
+from ..compat import compat_str
-    ExtractorError
-)
 from ..utils import (
     determine_ext,
     parse_duration,
-    unified_strdate
+    try_get,
+    unified_strdate,
 )
 
 
 class MediasetIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.video\.mediaset\.it/(?:(?:video|on-demand)/(?:.+)_|player/playerIFrame(?:Twitter)?\.shtml\?id=)(?P<id>[0-9]+)(?:.html|&.+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?video\.mediaset\.it/
+                        (?:
+                            (?:video|on-demand)/(?:[^/]+/)+[^/]+_|
+                            player/playerIFrame(?:Twitter)?\.shtml\?.*?\bid=
+                        )(?P<id>[0-9]+)
+                    '''
     _TESTS = [{
         # full episode
         'url': 'http://www.video.mediaset.it/video/hello_goodbye/full/quarta-puntata_661824.html',
@@ -22,97 +28,75 @@ class MediasetIE(InfoExtractor):
             'id': '661824',
             'ext': 'mp4',
             'title': 'Quarta puntata',
-            'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:7183696d6df570e3412a5ef74b27c5e2',
-            'uploader': 'mediaset'
+            'thumbnail': r're:^https?://.*\.jpg$',
-        }
+            'duration': 1414,
-    }, {
+            'creator': 'mediaset',
-        # on demand
+            'release_date': '20161107',
-        'url': 'http://www.video.mediaset.it/video/domenica_live/interviste/il-fenomeno-elettra-lamborghini_716283.html',
+            'series': 'Hello Goodbye',
-        'md5': '81c57566bf2ee02e995f5342f079ca25',
+            'categories': ['reality'],
-        'info_dict': {
+        },
-            'id': '716283',
+        'expected_warnings': ['is not a supported codec'],
-            'ext': 'mp4',
-            'title': 'Il fenomeno Elettra Lamborghini',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:dabf0e7cf48fc6d0a3417b989028748a',
-            'uploader': 'mediaset'
-        }
     }, {
         # clip
         'url': 'http://www.video.mediaset.it/video/gogglebox/clip/un-grande-classico-della-commedia-sexy_661680.html',
-        'md5': '189ca72fe399db80dbfa595a4abf42d0',
+        'only_matching': True,
-        'info_dict': {
-            'id': '661680',
-            'ext': 'mp4',
-            'title': 'Un grande classico della commedia sexy',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Un film che riesce a risvegliare i sensi di Gigi.',
-            'uploader': 'mediaset'
-        }
     }, {
         # iframe simple
         'url': 'http://www.video.mediaset.it/player/playerIFrame.shtml?id=665924&autoplay=true',
-        'md5': '308430901e55e1ad83dddb4be2a4454a',
+        'only_matching': True,
-        'info_dict': {
-            'id': '665924',
-            'ext': 'mp4',
-            'title': 'Gianna Nannini incontra i fan a Milano',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'La cantante parla del nuovo libro',
-            'uploader': 'mediaset'
-        }
     }, {
         # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
         'url': 'https://www.video.mediaset.it/player/playerIFrameTwitter.shtml?id=665104&playrelated=false&autoplay=false&related=true&hidesocial=true',
-        'md5': '6f53a834b3b5eac1ebc2037ccf7194d0',
+        'only_matching': True,
-        'info_dict': {
-            'id': '665104',
-            'ext': 'mp4',
-            'title': '\"Se prima mi fidavo zero...\"',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Una piccola anteprima della prossima puntata del Trono Classico',
-            'uploader': 'mediaset'
-        }
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        formats = []
-        uploader = None
-        categories = None
 
-        mediainfo = self._download_json(
+        video_list = self._download_json(
-            'http://plr.video.mediaset.it/html/metainfo.sjson?id=%s' % video_id,
+            'http://cdnsel01.mediaset.net/GetCdn.aspx',
-            video_id, 'Downloading video info JSON').get('video')
+            video_id, 'Downloading video CDN JSON', query={
+                'streamid': video_id,
+                'format': 'json',
+            })['videoList']
 
-        if 'brand-info' in mediainfo:
+        formats = []
-            uploader = mediainfo.get('brand-info').get('publisher')
+        for format_url in video_list:
-            categories = [mediainfo.get('brand-info').get('category')]
+            if '.ism' in format_url:
+                formats.extend(self._extract_ism_formats(
+                    format_url, video_id, ism_id='mss', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': determine_ext(format_url),
+                })
+        self._sort_formats(formats)
 
-        cnd = self._download_json(
+        mediainfo = self._download_json(
-            'http://cdnsel01.mediaset.net/GetCdn.aspx?streamid=%s&format=json' % video_id,
+            'http://plr.video.mediaset.it/html/metainfo.sjson',
-            video_id, 'Downloading video CND JSON')
+            video_id, 'Downloading video info JSON', query={
+                'id': video_id,
+            })['video']
 
-        if not cnd.get('videoList'):
+        title = mediainfo['title']
-            raise ExtractorError('Video not found')
 
-        for media_url in cnd.get('videoList'):
+        creator = try_get(
-            formats.append({
+            mediainfo, lambda x: x['brand-info']['publisher'], compat_str)
-                'url': media_url,
+        category = try_get(
-                'ext': determine_ext(media_url)
+            mediainfo, lambda x: x['brand-info']['category'], compat_str)
-            })
+        categories = [category] if category else None
-        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': mediainfo.get('title'),
+            'title': title,
-            'formats': formats,
             'description': mediainfo.get('short-description'),
-            'uploader': uploader,
             'thumbnail': mediainfo.get('thumbnail'),
             'duration': parse_duration(mediainfo.get('duration')),
+            'creator': creator,
             'release_date': unified_strdate(mediainfo.get('production-date')),
             'webpage_url': mediainfo.get('url'),
-            'categories': categories
+            'series': mediainfo.get('brand-value'),
+            'categories': categories,
+            'formats': formats,
         }