[mixcloud] Rewrite extractor (fixes #278)

youtube_dl/extractor/mixcloud.py

@@ -5,34 +5,27 @@ import socket
 from .common import InfoExtractor
 from ..utils import (
     compat_http_client,
-    compat_str,
     compat_urllib_error,
     compat_urllib_request,
-
+    unified_strdate,
-    ExtractorError,
 )
 
 
 class MixcloudIE(InfoExtractor):
-    _WORKING = False # New API, but it seems good http://www.mixcloud.com/developers/documentation/
     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
     IE_NAME = u'mixcloud'
 
-    def report_download_json(self, file_id):
+    _TEST = {
-        """Report JSON download."""
+        u'url': u'http://www.mixcloud.com/dholbach/cryptkeeper/',
-        self.to_screen(u'Downloading json')
+        u'file': u'dholbach-cryptkeeper.mp3',
-
+        u'info_dict': {
-    def get_urls(self, jsonData, fmt, bitrate='best'):
+            u'title': u'Cryptkeeper',
-        """Get urls from 'audio_formats' section in json"""
+            u'description': u'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
-        try:
+            u'uploader': u'Daniel Holbach',
-            bitrate_list = jsonData[fmt]
+            u'uploader_id': u'dholbach',
-            if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
+            u'upload_date': u'20111115',
-                bitrate = max(bitrate_list) # select highest
+        },
-
+    }
-            url_list = jsonData[fmt][bitrate]
-        except TypeError: # we have no bitrate info.
-            url_list = jsonData[fmt]
-        return url_list
 
     def check_urls(self, url_list):
         """Returns 1st active url from list"""
@@ -45,71 +38,32 @@ class MixcloudIE(InfoExtractor):
 
         return None
 
-    def _print_formats(self, formats):
-        print('Available formats:')
-        for fmt in formats.keys():
-            for b in formats[fmt]:
-                try:
-                    ext = formats[fmt][b][0]
-                    print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
-                except TypeError: # we have no bitrate info
-                    ext = formats[fmt][0]
-                    print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
-                    break
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-        # extract uploader & filename from url
-        uploader = mobj.group(1).decode('utf-8')
-        file_id = uploader + "-" + mobj.group(2).decode('utf-8')
-
-        # construct API request
-        file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
-        # retrieve .json file with links to files
-        request = compat_urllib_request.Request(file_url)
-        try:
-            self.report_download_json(file_url)
-            jsonData = compat_urllib_request.urlopen(request).read()
-        except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            raise ExtractorError(u'Unable to retrieve file: %s' % compat_str(err))
-
-        # parse JSON
-        json_data = json.loads(jsonData)
-        player_url = json_data['player_swf_url']
-        formats = dict(json_data['audio_formats'])
-
-        req_format = self._downloader.params.get('format', None)
-
-        if self._downloader.params.get('listformats', None):
-            self._print_formats(formats)
-            return
-
-        if req_format is None or req_format == 'best':
-            for format_param in formats.keys():
-                url_list = self.get_urls(formats, format_param)
-                # check urls
-                file_url = self.check_urls(url_list)
-                if file_url is not None:
-                    break # got it!
-        else:
-            if req_format not in formats:
-                raise ExtractorError(u'Format is not available')
-
-            url_list = self.get_urls(formats, req_format)
-            file_url = self.check_urls(url_list)
-            format_param = req_format
 
-        return [{
+        uploader = mobj.group(1)
-            'id': file_id.decode('utf-8'),
+        cloudcast_name = mobj.group(2)
-            'url': file_url.decode('utf-8'),
+        track_id = '-'.join((uploader, cloudcast_name))
-            'uploader': uploader.decode('utf-8'),
+        api_url = 'http://api.mixcloud.com/%s/%s/' % (uploader, cloudcast_name)
-            'upload_date': None,
+        webpage = self._download_webpage(url, track_id)
-            'title': json_data['name'],
+        json_data = self._download_webpage(api_url, track_id,
-            'ext': file_url.split('.')[-1].decode('utf-8'),
+            u'Downloading cloudcast info')
-            'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
+        info = json.loads(json_data)
-            'thumbnail': json_data['thumbnail_url'],
+
-            'description': json_data['description'],
+        preview_url = self._search_regex(r'data-preview-url="(.+?)"', webpage, u'preview url')
-            'player_url': player_url.decode('utf-8'),
+        song_url = preview_url.replace('/previews/', '/cloudcasts/originals/')
-        }]
+        template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
+        final_song_url = self.check_urls(template_url % i for i in range(30))
+
+        return {
+            'id': track_id,
+            'title': info['name'],
+            'url': final_song_url,
+            'ext': 'mp3',
+            'description': info['description'],
+            'thumbnail': info['pictures'].get('extra_large'),
+            'uploader': info['user']['name'],
+            'uploader_id': info['user']['username'],
+            'upload_date': unified_strdate(info['created_time']),
+            'view_count': info['play_count'],
+        }

youtube_dl/utils.py

@@ -700,7 +700,16 @@ def unified_strdate(date_str):
     date_str = date_str.replace(',',' ')
     # %z (UTC offset) is only supported in python>=3.2
     date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
-    format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S', '%d.%m.%Y %H:%M']
+    format_expressions = [
+        '%d %B %Y',
+        '%B %d %Y',
+        '%b %d %Y',
+        '%Y-%m-%d',
+        '%d/%m/%Y',
+        '%Y/%m/%d %H:%M:%S',
+        '%d.%m.%Y %H:%M',
+        '%Y-%m-%dT%H:%M:%SZ',
+    ]
     for expression in format_expressions:
         try:
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')