diff --git a/README.md b/README.md index b218fc3eca..2ca0211259 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,10 @@ which means you can modify it, redistribute it or use it however you like. %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal - percent. Use - to output to stdout. + percent. Use - to output to stdout. Can also be + used to download to a different directory, for + example with -o '/my/downloads/%(uploader)s/%(title + )s-%(id)s.%(ext)s' . --restrict-filenames Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames -a, --batch-file FILE file containing URLs to download ('-' for stdin) @@ -194,3 +197,5 @@ Please include: * The output of `youtube-dl --version` * The output of `python --version` * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough). + +For discussions, join us in the irc channel #youtube-dl on freenode. diff --git a/test/gentests.py b/test/gentests.py index c3cca51561..9352d6d949 100755 --- a/test/gentests.py +++ b/test/gentests.py @@ -34,31 +34,19 @@ import youtube_dl.InfoExtractors def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() - -def md5_for_file(filename, block_size=2**20): - with open(filename) as f: - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() -_file_md5 = md5_for_file - - try: _skip_unless = unittest.skipUnless except AttributeError: # Python 2.6 def _skip_unless(cond, reason='No reason given'): def resfunc(f): - def wfunc(*args, **kwargs): + # Start the function name with test to appease nosetests-2.6 + def test_wfunc(*args, **kwargs): if cond: return f(*args, **kwargs) else: print('Skipped test') return - return wfunc + return test_wfunc return resfunc _skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs) diff --git a/test/test_download.py b/test/test_download.py index b7cb6cebb1..6926dd04e6 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -20,31 +20,19 @@ import youtube_dl.InfoExtractors def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() - -def md5_for_file(filename, block_size=2**20): - with open(filename) as f: - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() -_file_md5 = md5_for_file - - try: _skip_unless = unittest.skipUnless except AttributeError: # Python 2.6 def _skip_unless(cond, reason='No reason given'): def resfunc(f): - def wfunc(*args, **kwargs): + # Start the function name with test to appease nosetests-2.6 + def test_wfunc(*args, **kwargs): if cond: return f(*args, **kwargs) else: print('Skipped test') return - return wfunc + return test_wfunc return resfunc _skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs) @@ -79,7 +67,7 @@ class DownloadTest(unittest.TestCase): @_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING") def test_Metacafe(self): - filename = 'aUehQsCQtM.flv' + filename = '_aUehQsCQtM.flv' fd = FileDownloader(self.parameters) fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE()) fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE()) @@ -120,7 +108,7 @@ class DownloadTest(unittest.TestCase): @_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING") def test_Soundcloud(self): - filename = 'n6FLbx6ZzMiu.mp3' + filename = '62986583.mp3' fd = FileDownloader(self.parameters) fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE()) fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy']) @@ -159,26 +147,38 @@ class DownloadTest(unittest.TestCase): md5_for_file = _file_md5(filename) self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c') + @_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING") + def test_Youku(self): + filename = 'XNDgyMDQ2NTQw_part00.flv' + fd = FileDownloader(self.parameters) + fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE()) + fd.download(['http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html']) + self.assertTrue(os.path.exists(filename)) + md5_for_file = _file_md5(filename) + self.assertEqual(md5_for_file, 'ffe3f2e435663dc2d1eea34faeff5b5b') + def tearDown(self): if os.path.exists('BaW_jenozKc.mp4'): os.remove('BaW_jenozKc.mp4') if os.path.exists('x33vw9.mp4'): os.remove('x33vw9.mp4') - if os.path.exists('aUehQsCQtM.flv'): - os.remove('aUehQsCQtM.flv') + if os.path.exists('_aUehQsCQtM.flv'): + os.remove('_aUehQsCQtM.flv') if os.path.exists('5779306.m4v'): os.remove('5779306.m4v') if os.path.exists('939581.flv'): os.remove('939581.flv') # No file specified for Vimeo - if os.path.exists('n6FLbx6ZzMiu.mp3'): - os.remove('n6FLbx6ZzMiu.mp3') + if os.path.exists('62986583.mp3'): + os.remove('62986583.mp3') if os.path.exists('PracticalUnix_intro-environment.mp4'): os.remove('PracticalUnix_intro-environment.mp4') # No file specified for CollegeHumor if os.path.exists('1135332.flv'): os.remove('1135332.flv') + if os.path.exists('XNDgyMDQ2NTQw_part00.flv'): + os.remove('XNDgyMDQ2NTQw_part00.flv') diff --git a/test/test_execution.py b/test/test_execution.py new file mode 100644 index 0000000000..2b115fb31a --- /dev/null +++ b/test/test_execution.py @@ -0,0 +1,26 @@ +import unittest + +import sys +import os +import subprocess + +rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +try: + _DEV_NULL = subprocess.DEVNULL +except AttributeError: + _DEV_NULL = open(os.devnull, 'wb') + +class TestExecution(unittest.TestCase): + def test_import(self): + subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir) + + def test_module_exec(self): + if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution + subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL) + + def test_main_exec(self): + subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_import.py b/test/test_import.py deleted file mode 100644 index acf95444df..0000000000 --- a/test/test_import.py +++ /dev/null @@ -1,13 +0,0 @@ -import unittest - -import sys -import os.path -import subprocess - -class TestImport(unittest.TestCase): - def test_import(self): - rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir) - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index fd8190f510..796ed4ab27 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Various small unit tests import sys @@ -79,6 +81,11 @@ class TestUtil(unittest.TestCase): self.assertTrue(sanitize_filename('-', restricted=True) != '') self.assertTrue(sanitize_filename(':', restricted=True) != '') + def test_sanitize_ids(self): + self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw') + self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw') + self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI') + def test_ordered_set(self): self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) diff --git a/test/test_youtube_playlist_ids.py b/test/test_youtube_playlist_ids.py new file mode 100644 index 0000000000..b4dcedb454 --- /dev/null +++ b/test/test_youtube_playlist_ids.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +import sys +import unittest + +# Allow direct execution +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE + +class TestYoutubePlaylistMatching(unittest.TestCase): + def test_playlist_matching(self): + assert YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') + assert YoutubePlaylistIE().suitable(u'PL63F0C78739B09958') + assert not YoutubePlaylistIE().suitable(u'PLtS2H6bU1M') + + def test_youtube_matching(self): + assert YoutubeIE().suitable(u'PLtS2H6bU1M') + +if __name__ == '__main__': + unittest.main() diff --git a/test/tests.json b/test/tests.json index 731c91718d..cb29f2053e 100644 --- a/test/tests.json +++ b/test/tests.json @@ -16,7 +16,7 @@ "size": 5754305, "addIEs": ["Youtube"], "url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/", - "file": "aUehQsCQtM.flv" + "file": "_aUehQsCQtM.flv" }, { "name": "BlipTV", @@ -40,7 +40,7 @@ "name": "Soundcloud", "md5": "c1b9b9ea8bfd620b96b2628664576e1c", "url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy", - "file": "n6FLbx6ZzMiu.mp3" + "file": "62986583.mp3" }, { "name": "StanfordOpenClassroom", @@ -54,10 +54,16 @@ "url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody", "file": "" }, - { + { "name": "XNXX", "md5": "c5c67df477eb0d9b058200351448ba4c", "url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_", "file": "1135332.flv" + }, + { + "name": "Youku", + "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html", + "file": "XNDgyMDQ2NTQw_part00.flv", + "md5": "ffe3f2e435663dc2d1eea34faeff5b5b" } -] \ No newline at end of file +] diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 48c8eb1261..c471cc1608 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -334,8 +334,11 @@ class FileDownloader(object): template_dict['epoch'] = int(time.time()) template_dict['autonumber'] = u'%05d' % self._num_downloads - template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items()) - template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items()) + sanitize = lambda k,v: sanitize_filename( + u'NA' if v is None else compat_str(v), + restricted=self.params.get('restrictfilenames'), + is_id=(k==u'id')) + template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items()) filename = self.params['outtmpl'] % template_dict return filename diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 925dabc96e..49976c25b2 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1671,7 +1671,7 @@ class YahooSearchIE(InfoExtractor): class YoutubePlaylistIE(InfoExtractor): """Information Extractor for YouTube playlists.""" - _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*' + _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*' _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en' _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s' _MORE_PAGES_INDICATOR = r'yt-uix-pager-next' @@ -2803,13 +2803,13 @@ class SoundcloudIE(InfoExtractor): def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) - def report_webpage(self, video_id): + def report_resolve(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id)) def report_extraction(self, video_id): """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id)) def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -2818,65 +2818,47 @@ class SoundcloudIE(InfoExtractor): return # extract uploader (which is in the url) - uploader = mobj.group(1).decode('utf-8') + uploader = mobj.group(1) # extract simple title (uploader + slug of song title) - slug_title = mobj.group(2).decode('utf-8') + slug_title = mobj.group(2) simple_title = uploader + u'-' + slug_title - self.report_webpage('%s/%s' % (uploader, slug_title)) + self.report_resolve('%s/%s' % (uploader, slug_title)) - request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title)) + url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title) + resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28' + request = compat_urllib_request.Request(resolv_url) try: - webpage = compat_urllib_request.urlopen(request).read() + info_json_bytes = compat_urllib_request.urlopen(request).read() + info_json = info_json_bytes.decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) return + info = json.loads(info_json) + video_id = info['id'] self.report_extraction('%s/%s' % (uploader, slug_title)) - # extract uid and stream token that soundcloud hands out for access - mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage) - if mobj: - video_id = mobj.group(1) - stream_token = mobj.group(2) - - # extract unsimplified title - mobj = re.search('"title":"(.*?)",', webpage) - if mobj: - title = mobj.group(1).decode('utf-8') - else: - title = simple_title - - # construct media url (with uid/token) - mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s" - mediaURL = mediaURL % (video_id, stream_token) - - # description - description = u'No description available' - mobj = re.search('track-description-value">

(.*?)

', webpage) - if mobj: - description = mobj.group(1) - - # upload date - upload_date = None - mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)", webpage) - if mobj: - try: - upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d') - except Exception as err: - self._downloader.to_stderr(compat_str(err)) + streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28' + request = compat_urllib_request.Request(streams_url) + try: + stream_json_bytes = compat_urllib_request.urlopen(request).read() + stream_json = stream_json_bytes.decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) + return - # for soundcloud, a request to a cross domain is required for cookies - request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers) + streams = json.loads(stream_json) + mediaURL = streams['http_mp3_128_url'] return [{ - 'id': video_id.decode('utf-8'), + 'id': info['id'], 'url': mediaURL, - 'uploader': uploader.decode('utf-8'), - 'upload_date': upload_date, - 'title': title, + 'uploader': info['user']['username'], + 'upload_date': info['created_at'], + 'title': info['title'], 'ext': u'mp3', - 'description': description.decode('utf-8') + 'description': info['description'], }] diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index f953f1fb9c..8e46dfea94 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -305,7 +305,7 @@ def parseOpts(): action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False) filesystem.add_option('-o', '--output', - dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.') + dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .') filesystem.add_option('--restrict-filenames', action='store_true', dest='restrictfilenames', help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 7f73b84761..44f939053a 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -317,9 +317,10 @@ def timeconvert(timestr): timestamp = email.utils.mktime_tz(timetuple) return timestamp -def sanitize_filename(s, restricted=False): +def sanitize_filename(s, restricted=False, is_id=False): """Sanitizes a string so it could be used as part of a filename. If restricted is set, use a stricter subset of allowed characters. + Set is_id if this is not an arbitrary string, but an ID that should be kept if possible """ def replace_insane(char): if char == '?' or ord(char) < 32 or ord(char) == 127: @@ -337,14 +338,15 @@ def sanitize_filename(s, restricted=False): return char result = u''.join(map(replace_insane, s)) - while '__' in result: - result = result.replace('__', '_') - result = result.strip('_') - # Common case of "Foreign band name - English song title" - if restricted and result.startswith('-_'): - result = result[2:] - if not result: - result = '_' + if not is_id: + while '__' in result: + result = result.replace('__', '_') + result = result.strip('_') + # Common case of "Foreign band name - English song title" + if restricted and result.startswith('-_'): + result = result[2:] + if not result: + result = '_' return result def orderedSet(iterable): @@ -504,3 +506,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler): resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp + + https_request = http_request + https_response = http_response