From 725652e9247e1171110b624d748e20fa1c88260e Mon Sep 17 00:00:00 2001 From: Mister Hat Date: Sat, 16 May 2015 19:50:58 -0500 Subject: [PATCH 1/3] [karrierevideos] add support for www.karrierevideos.at (closes #5354) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/karrierevideos.py | 52 ++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 youtube_dl/extractor/karrierevideos.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index fb4f63ca3a..d131d3ec31 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -244,6 +244,7 @@ from .kaltura import KalturaIE from .kanalplay import KanalPlayIE from .kankan import KankanIE from .karaoketv import KaraoketvIE +from .karrierevideos import KarriereVideosIE from .keezmovies import KeezMoviesIE from .khanacademy import KhanAcademyIE from .kickstarter import KickStarterIE diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py new file mode 100644 index 0000000000..59d29e8457 --- /dev/null +++ b/youtube_dl/extractor/karrierevideos.py @@ -0,0 +1,52 @@ +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class KarriereVideosIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P[a-z-]+)' + _TEST = { + 'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin', + 'info_dict': { + 'id': 'altenpflegerin', + 'ext': 'mp4', + 'title': 'AltenpflegerIn', + 'thumbnail': 're:^http://.*\.png\?v=[0-9]+', + 'description': 'md5:dbadd1259fde2159a9b28667cb664ae2' + }, + 'params': { + 'skip_download': 'requires rtmpdump' + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + description = self._html_search_regex( + r'
\n{0,}?\s{0,}

(.*?)

', + webpage, 'description') + + playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist') + playlist = self._download_xml( + 'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist, + video_id) + + namespace = { + 'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats' + } + + item = playlist.find('tracklist/item') + streamer = item.find('jwplayer:streamer', namespace).text + + return { + 'id': video_id, + 'title': self._html_search_meta('title', webpage), + 'description': description, + 'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage), + 'protocol': 'rtmp', + 'url': streamer.replace('rtmpt', 'http'), + 'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text, + 'tc_url': streamer, + 'ext': 'mp4' + } From ba9d16291b8ace3bd412bcfc0c128c047545e509 Mon Sep 17 00:00:00 2001 From: Mister Hat Date: Sun, 17 May 2015 03:35:08 -0500 Subject: [PATCH 2/3] manually specify namespace --- youtube_dl/extractor/karrierevideos.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py index 59d29e8457..a05e8ab767 100644 --- a/youtube_dl/extractor/karrierevideos.py +++ b/youtube_dl/extractor/karrierevideos.py @@ -29,15 +29,13 @@ class KarriereVideosIE(InfoExtractor): playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist') playlist = self._download_xml( - 'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist, + 'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist, video_id) - namespace = { - 'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats' - } + namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats' item = playlist.find('tracklist/item') - streamer = item.find('jwplayer:streamer', namespace).text + streamer = item.find('{%s}streamer' % namespace).text return { 'id': video_id, @@ -46,7 +44,7 @@ class KarriereVideosIE(InfoExtractor): 'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage), 'protocol': 'rtmp', 'url': streamer.replace('rtmpt', 'http'), - 'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text, + 'play_path': 'mp4:' + item.find('{%s}file' % namespace).text, 'tc_url': streamer, 'ext': 'mp4' } From d78c834ead934e5532d2f5bc221bb11eedaef0e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 24 May 2015 20:04:13 +0600 Subject: [PATCH 3/3] [karrierevideos] Improve and simplify --- youtube_dl/extractor/karrierevideos.py | 94 +++++++++++++++++++------- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py index a05e8ab767..bed94bc933 100644 --- a/youtube_dl/extractor/karrierevideos.py +++ b/youtube_dl/extractor/karrierevideos.py @@ -1,50 +1,96 @@ +# coding: utf-8 from __future__ import unicode_literals from .common import InfoExtractor +from ..compat import compat_urlparse +from ..utils import ( + fix_xml_ampersands, + float_or_none, + xpath_with_ns, + xpath_text, +) class KarriereVideosIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P[a-z-]+)' - _TEST = { + _VALID_URL = r'http://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P[^/]+)' + _TESTS = [{ 'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin', 'info_dict': { - 'id': 'altenpflegerin', - 'ext': 'mp4', + 'id': '32c91', + 'ext': 'flv', 'title': 'AltenpflegerIn', - 'thumbnail': 're:^http://.*\.png\?v=[0-9]+', - 'description': 'md5:dbadd1259fde2159a9b28667cb664ae2' + 'description': 'md5:dbadd1259fde2159a9b28667cb664ae2', + 'thumbnail': 're:^http://.*\.png', }, 'params': { - 'skip_download': 'requires rtmpdump' + # rtmp download + 'skip_download': True, } - } + }, { + # broken ampersands + 'url': 'http://www.karrierevideos.at/orientierung/vaeterkarenz-und-neue-chancen-fuer-muetter-baby-was-nun', + 'info_dict': { + 'id': '5sniu', + 'ext': 'flv', + 'title': 'Väterkarenz und neue Chancen für Mütter - "Baby - was nun?"', + 'description': 'md5:97092c6ad1fd7d38e9d6a5fdeb2bcc33', + 'thumbnail': 're:^http://.*\.png', + }, + 'params': { + # rtmp download + 'skip_download': True, + } + }] def _real_extract(self, url): video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) - description = self._html_search_regex( - r'
\n{0,}?\s{0,}

(.*?)

', - webpage, 'description') + title = (self._html_search_meta('title', webpage, default=None) or + self._search_regex(r'

([^<]+)

')) - playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist') + video_id = self._search_regex( + r'/config/video/(.+?)\.xml', webpage, 'video id') playlist = self._download_xml( - 'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist, - video_id) + 'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % video_id, + video_id, transform_source=fix_xml_ampersands) - namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats' + NS_MAP = { + 'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats' + } + + def ns(path): + return xpath_with_ns(path, NS_MAP) + + item = playlist.find('./tracklist/item') + video_file = xpath_text( + item, ns('./jwplayer:file'), 'video url', fatal=True) + streamer = xpath_text( + item, ns('./jwplayer:streamer'), 'streamer', fatal=True) + + uploader = xpath_text( + item, ns('./jwplayer:author'), 'uploader') + duration = float_or_none( + xpath_text(item, ns('./jwplayer:duration'), 'duration')) + + description = self._html_search_regex( + r'(?s)
(.+?)
', + webpage, 'description') - item = playlist.find('tracklist/item') - streamer = item.find('{%s}streamer' % namespace).text + thumbnail = self._html_search_meta( + 'thumbnail', webpage, 'thumbnail') + if thumbnail: + thumbnail = compat_urlparse.urljoin(url, thumbnail) return { 'id': video_id, - 'title': self._html_search_meta('title', webpage), + 'url': streamer.replace('rtmpt', 'rtmp'), + 'play_path': 'mp4:%s' % video_file, + 'ext': 'flv', + 'title': title, 'description': description, - 'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage), - 'protocol': 'rtmp', - 'url': streamer.replace('rtmpt', 'http'), - 'play_path': 'mp4:' + item.find('{%s}file' % namespace).text, - 'tc_url': streamer, - 'ext': 'mp4' + 'thumbnail': thumbnail, + 'uploader': uploader, + 'duration': duration, }