[zingmp3] fix extraction and add support for video clips(closes #10041)

youtube_dl/extractor/extractors.py

@@ -1114,7 +1114,4 @@ from .youtube import (
 )
 from .zapiks import ZapiksIE
 from .zdf import ZDFIE, ZDFChannelIE
-from .zingmp3 import (
+from .zingmp3 import ZingMp3IE
-    ZingMp3SongIE,
-    ZingMp3AlbumIE,
-)

youtube_dl/extractor/zingmp3.py

@@ -4,13 +4,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    update_url_query,
+)
 
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    def _extract_item(self, item, fatal=True):
+    def _extract_item(self, item, page_type, fatal=True):
-        error_message = item.find('./errormessage').text
+        error_message = item.get('msg')
         if error_message:
             if not fatal:
                 return
@@ -18,25 +22,48 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
                 '%s returned error: %s' % (self.IE_NAME, error_message),
                 expected=True)
 
-        title = item.find('./title').text.strip()
+        formats = []
-        source = item.find('./source').text
+        for quality, source_url in zip(item.get('qualities') or item.get('quality', []), item.get('source_list') or item.get('source', [])):
-        extension = item.attrib['type']
+            if not source_url or source_url == 'require vip':
-        thumbnail = item.find('./backimage').text
+                continue
+            if not re.match(r'https?://', source_url):
+                source_url = '//' + source_url
+            source_url = self._proto_relative_url(source_url, 'http:')
+            quality_num = int_or_none(quality)
+            f = {
+                'format_id': quality,
+                'url': source_url,
+            }
+            if page_type == 'video':
+                f.update({
+                    'height': quality_num,
+                    'ext': 'mp4',
+                })
+            else:
+                f.update({
+                    'abr': quality_num,
+                    'ext': 'mp3',
+                })
+            formats.append(f)
+
+        cover = item.get('cover')
 
         return {
-            'title': title,
+            'title': (item.get('name') or item.get('title')).strip(),
-            'url': source,
+            'formats': formats,
-            'ext': extension,
+            'thumbnail': 'http:/' + cover if cover else None,
-            'thumbnail': thumbnail,
+            'artist': item.get('artist'),
         }
 
-    def _extract_player_xml(self, player_xml_url, id, playlist_title=None):
+    def _extract_player_json(self, player_json_url, id, page_type, playlist_title=None):
-        player_xml = self._download_xml(player_xml_url, id, 'Downloading Player XML')
+        player_json = self._download_json(player_json_url, id, 'Downloading Player JSON')
-        items = player_xml.findall('./item')
+        items = player_json['data']
+        if 'item' in items:
+            items = items['item']
 
         if len(items) == 1:
             # one single song
-            data = self._extract_item(items[0])
+            data = self._extract_item(items[0], page_type)
             data['id'] = id
 
             return data
@@ -45,7 +72,7 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
             entries = []
 
             for i, item in enumerate(items, 1):
-                entry = self._extract_item(item, fatal=False)
+                entry = self._extract_item(item, page_type, fatal=False)
                 if not entry:
                     continue
                 entry['id'] = '%s-%d' % (id, i)
@@ -59,8 +86,8 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
             }
 
 
-class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
+class ZingMp3IE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/bai-hat/(?P<slug>[^/]+)/(?P<song_id>\w+)\.html'
+    _VALID_URL = r'https?://mp3\.zing\.vn/(?:bai-hat|album|playlist|video-clip)/[^/]+/(?P<id>\w+)\.html'
     _TESTS = [{
         'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
         'md5': 'ead7ae13693b3205cbc89536a077daed',
@@ -70,51 +97,47 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
             'ext': 'mp3',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-    }]
+    }, {
-    IE_NAME = 'zingmp3:song'
+        'url': 'http://mp3.zing.vn/video-clip/Let-It-Go-Frozen-OST-Sungha-Jung/ZW6BAEA0.html',
-    IE_DESC = 'mp3.zing.vn songs'
+        'md5': '870295a9cd8045c0e15663565902618d',
-
+        'info_dict': {
-    def _real_extract(self, url):
+            'id': 'ZW6BAEA0',
-        matched = re.match(self._VALID_URL, url)
+            'title': 'Let It Go (Frozen OST)',
-        slug = matched.group('slug')
+            'ext': 'mp4',
-        song_id = matched.group('song_id')
+        },
-
+    }, {
-        webpage = self._download_webpage(
-            'http://mp3.zing.vn/bai-hat/%s/%s.html' % (slug, song_id), song_id)
-
-        player_xml_url = self._search_regex(
-            r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
-
-        return self._extract_player_xml(player_xml_url, song_id)
-
-
-class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [{
         'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
         'info_dict': {
             '_type': 'playlist',
             'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+            'title': 'Lâu Đài Tình Ái - Bằng Kiều,Minh Tuyết | Album 320 lossless',
         },
         'playlist_count': 10,
+        'skip': 'removed at the request of the owner',
     }, {
         'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
         'only_matching': True,
     }]
-    IE_NAME = 'zingmp3:album'
+    IE_NAME = 'zingmp3'
-    IE_DESC = 'mp3.zing.vn albums'
+    IE_DESC = 'mp3.zing.vn'
 
     def _real_extract(self, url):
-        matched = re.match(self._VALID_URL, url)
+        page_id = self._match_id(url)
-        slug = matched.group('slug')
+
-        album_id = matched.group('album_id')
+        webpage = self._download_webpage(url, page_id)
-
+
-        webpage = self._download_webpage(
+        player_json_url = self._search_regex([
-            'http://mp3.zing.vn/album/%s/%s.html' % (slug, album_id), album_id)
+            r'data-xml="([^"]+)',
-        player_xml_url = self._search_regex(
+            r'&amp;xmlURL=([^&]+)&'
-            r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
+        ], webpage, 'player xml url')
-
+
-        return self._extract_player_xml(
+        playlist_title = None
-            player_xml_url, album_id,
+        page_type = self._search_regex(r'/(?:html5)?xml/([^/-]+)', player_json_url, 'page type')
-            playlist_title=self._og_search_title(webpage))
+        if page_type == 'video':
+            player_json_url = update_url_query(player_json_url, {'format': 'json'})
+        else:
+            player_json_url = player_json_url.replace('/xml/', '/html5xml/')
+            if page_type == 'album':
+                playlist_title = self._og_search_title(webpage)
+
+        return self._extract_player_json(player_json_url, page_id, page_type, playlist_title)