[crooksandliars] Improve embed extractor and remove article extractor

youtube_dl/extractor/__init__.py

@@ -90,7 +90,7 @@ from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
-from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
+from .crooksandliars import CrooksAndLiarsIE
 from .crunchyroll import (
     CrunchyrollIE,
     CrunchyrollShowPlaylistIE

youtube_dl/extractor/crooksandliars.py

@@ -1,71 +1,60 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 from ..utils import (
-    mimetype2ext,
+    int_or_none,
+    qualities,
 )
 
 
 class CrooksAndLiarsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
-
+    _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
         'info_dict': {
-            'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+            'id': '8RUoRhRi',
+            'ext': 'mp4',
             'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
-            'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+            'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1428207000,
-            'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
-            'uploader': "Heather",
+            'upload_date': '20150405',
+            'uploader': 'Heather',
+            'duration': 236,
         }
+    }, {
+        'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
+        webpage = self._download_webpage(
+            'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)
+
+        manifest = self._parse_json(
+            self._search_regex(
+                r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
+            video_id)
 
-        formats = []
-        for item in manifest['flavors']:
-            if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
-                continue
-            formats.append({
-                'format_id': item['type'],
-                'ext': mimetype2ext(item['mime']),
-                'url': item['url'],
-            })
+        quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))
+
+        formats = [{
+            'url': item['url'],
+            'format_id': item['type'],
+            'quality': quality(item['type']),
+        } for item in manifest['flavors'] if item['mime'].startswith('video/')]
+        self._sort_formats(formats)
 
-        # XXX: manifest['url']?
         return {
             'url': url,
             'id': video_id,
-            'uploader': manifest['author'],
             'title': manifest['title'],
-            'description': manifest['description'],
-            'thumbnail': self._proto_relative_url(manifest['poster']),
-            'duration': manifest['duration'],
-            'timestamp': int(manifest['created']),
+            'description': manifest.get('description'),
+            'thumbnail': self._proto_relative_url(manifest.get('poster')),
+            'timestamp': int_or_none(manifest.get('created')),
+            'uploader': manifest.get('author'),
+            'duration': int_or_none(manifest.get('duration')),
             'formats': formats,
         }
-
-class CrooksAndLiarsArticleIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
-
-    _TESTS = [{
-        'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
-
-        return {
-            '_type': 'url',
-            'url': player_url
-        }