From 60351178a59123ef6917bbfa366f0a58fa812be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fernando=20Rodr=C3=ADguez=20Var=C3=B3n?= Date: Sun, 1 Nov 2020 21:25:34 -0500 Subject: [PATCH 1/3] [TMZ] Fix TMZ.com extractor --- youtube_dlc/extractor/tmz.py | 73 ++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/youtube_dlc/extractor/tmz.py b/youtube_dlc/extractor/tmz.py index 419f9d92e..dae6eab9d 100644 --- a/youtube_dlc/extractor/tmz.py +++ b/youtube_dlc/extractor/tmz.py @@ -5,7 +5,7 @@ from .common import InfoExtractor class TMZIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P[^/?#]+)' + _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/.*(?P[^/?#]{10,10})' _TESTS = [{ 'url': 'http://www.tmz.com/videos/0_okj015ty/', 'md5': '4d22a51ef205b6c06395d8394f72d560', @@ -13,14 +13,30 @@ class TMZIE(InfoExtractor): 'id': '0_okj015ty', 'ext': 'mp4', 'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!', - 'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie??? Or is she just showing off her amazing boobs?', 'timestamp': 1394747163, 'uploader_id': 'batchUser', 'upload_date': '20140313', } }, { 'url': 'http://www.tmz.com/videos/0-cegprt2p/', - 'only_matching': True, + 'info_dict': { + 'id': '0_cegprt2p', + 'ext': 'mp4', + 'title': "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet", + 'timestamp': 1467831837, + 'uploader_id': 'batchUser', + 'upload_date': '20160706', + } + }, { + 'url': 'https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/', + 'info_dict': { + 'id': '0_zcsejvcr', + 'ext': 'mxf', + 'title': "Angry Bagel Shop Guy Says He Doesn't Trust Women", + 'timestamp': 1562889485, + 'uploader_id': 'batchUser', + 'upload_date': '20190711', + } }] def _real_extract(self, url): @@ -30,27 +46,54 @@ class TMZIE(InfoExtractor): class TMZArticleIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P[^/]+)/?' - _TEST = { + _TESTS = [{ 'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert', - 'md5': '3316ff838ae5bb7f642537825e1e90d2', + 'md5': '5429c85db8bde39a473a56ca8c4c5602', 'info_dict': { 'id': '0_6snoelag', - 'ext': 'mov', + 'ext': 'mp4', 'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake', - 'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake. She\'s watching me."', 'timestamp': 1429467813, 'upload_date': '20150419', 'uploader_id': 'batchUser', } - } + }, { + 'url': 'http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/', + 'info_dict': { + 'id': '0_jerz7s3l', + 'ext': 'mp4', + 'title': 'Patti LaBelle -- Goes Nuclear On Stripping Fan', + 'timestamp': 1442683746, + 'upload_date': '20150919', + 'uploader_id': 'batchUser', + } + }, { + 'url': 'http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/', + 'info_dict': { + 'id': '0_ytz87kk7', + 'ext': 'mp4', + 'title': "NBA's Adam Silver -- Blake Griffin's a Great Guy ... He'll Learn from This", + 'timestamp': 1454010989, + 'upload_date': '20160128', + 'uploader_id': 'batchUser', + } + }, { + 'url': 'http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/', + 'info_dict': { + 'id': '0_isigfatu', + 'ext': 'mp4', + 'title': "Trump Star Vandal -- I'm Not Afraid of Donald or the Cops!", + 'timestamp': 1477500095, + 'upload_date': '20161026', + 'uploader_id': 'batchUser', + } + }] def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - embedded_video_info = self._parse_json(self._html_search_regex( - r'tmzVideoEmbed\(({.+?})\);', webpage, 'embedded video info'), - video_id) - - return self.url_result( - 'http://www.tmz.com/videos/%s/' % embedded_video_info['id']) + params = self._html_search_regex(r'TMZ.actions.clickLink\(([\s\S]+?)\)', + webpage, 'embedded video info').split(',') + new_url = params[0].strip("'\"") + if new_url != url: + return self.url_result(new_url) From fff50711120b1a1c0477550748768d1e5b1fb755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fernando=20Rodr=C3=ADguez=20Var=C3=B3n?= Date: Sun, 8 Nov 2020 15:36:41 -0500 Subject: [PATCH 2/3] [TMZ] Add support for new page structure using JSON-LD --- youtube_dlc/extractor/extractors.py | 5 +- youtube_dlc/extractor/tmz.py | 188 +++++++++++++++------------- 2 files changed, 99 insertions(+), 94 deletions(-) diff --git a/youtube_dlc/extractor/extractors.py b/youtube_dlc/extractor/extractors.py index 666134d86..5dde2965b 100644 --- a/youtube_dlc/extractor/extractors.py +++ b/youtube_dlc/extractor/extractors.py @@ -1178,10 +1178,7 @@ from .thisoldhouse import ThisOldHouseIE from .threeqsdn import ThreeQSDNIE from .tiktok import TikTokIE from .tinypic import TinyPicIE -from .tmz import ( - TMZIE, - TMZArticleIE, -) +from .tmz import TMZIE from .tnaflix import ( TNAFlixNetworkEmbedIE, TNAFlixIE, diff --git a/youtube_dlc/extractor/tmz.py b/youtube_dlc/extractor/tmz.py index dae6eab9d..a2f100922 100644 --- a/youtube_dlc/extractor/tmz.py +++ b/youtube_dlc/extractor/tmz.py @@ -5,95 +5,103 @@ from .common import InfoExtractor class TMZIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/.*(?P[^/?#]{10,10})' - _TESTS = [{ - 'url': 'http://www.tmz.com/videos/0_okj015ty/', - 'md5': '4d22a51ef205b6c06395d8394f72d560', - 'info_dict': { - 'id': '0_okj015ty', - 'ext': 'mp4', - 'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!', - 'timestamp': 1394747163, - 'uploader_id': 'batchUser', - 'upload_date': '20140313', - } - }, { - 'url': 'http://www.tmz.com/videos/0-cegprt2p/', - 'info_dict': { - 'id': '0_cegprt2p', - 'ext': 'mp4', - 'title': "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet", - 'timestamp': 1467831837, - 'uploader_id': 'batchUser', - 'upload_date': '20160706', - } - }, { - 'url': 'https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/', - 'info_dict': { - 'id': '0_zcsejvcr', - 'ext': 'mxf', - 'title': "Angry Bagel Shop Guy Says He Doesn't Trust Women", - 'timestamp': 1562889485, - 'uploader_id': 'batchUser', - 'upload_date': '20190711', - } - }] + _VALID_URL = r"https?://(?:www\.)?tmz\.com/.*" + _TESTS = [ + { + "url": "http://www.tmz.com/videos/0-cegprt2p/", + "info_dict": { + "id": "http://www.tmz.com/videos/0-cegprt2p/", + "ext": "mp4", + "title": "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet", + "description": "Harvey talks about Director Comey’s decision not to prosecute Hillary Clinton.", + "timestamp": 1467831837, + "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}", + "upload_date": "20160706", + }, + }, + { + "url": "https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/", + "info_dict": { + "id": "https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/", + "ext": "mp4", + "title": "Angry Bagel Shop Guy Says He Doesn't Trust Women", + "description": "The enraged man who went viral for ranting about women on dating sites before getting ragdolled in a bagel shop is defending his misogyny ... he says it's women's fault in the first place.", + "timestamp": 1562889485, + "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}", + "upload_date": "20190711", + }, + }, + { + "url": "http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert", + "md5": "5429c85db8bde39a473a56ca8c4c5602", + "info_dict": { + "id": "http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert", + "ext": "mp4", + "title": "Bobby Brown Tells Crowd ... Bobbi Kristina is Awake", + "description": 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake. She\'s watching me."', + "timestamp": 1429467813, + "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}", + "upload_date": "20150419", + }, + }, + { + "url": "http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/", + "info_dict": { + "id": "http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/", + "ext": "mp4", + "title": "Patti LaBelle -- Goes Nuclear On Stripping Fan", + "description": "Patti LaBelle made it known loud and clear last night ... NO " + "ONE gets on her stage and strips down.", + "timestamp": 1442683746, + "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}", + "upload_date": "20150919", + }, + }, + { + "url": "http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/", + "info_dict": { + "id": "http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/", + "ext": "mp4", + "title": "NBA's Adam Silver -- Blake Griffin's a Great Guy ... He'll Learn from This", + "description": "Two pretty parts of this video with NBA Commish Adam Silver.", + "timestamp": 1454010989, + "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}", + "upload_date": "20160128", + }, + }, + { + "url": "http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/", + "info_dict": { + "id": "http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/", + "ext": "mp4", + "title": "Trump Star Vandal -- I'm Not Afraid of Donald or the Cops!", + "description": "James Otis is the the guy who took a pickaxe to Donald Trump's star on the Walk of Fame, and he tells TMZ .. he's ready and willing to go to jail for the crime.", + "timestamp": 1477500095, + "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}", + "upload_date": "20161026", + }, + }, + { + "url": "https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/", + "info_dict": { + "id": "https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/", + "ext": "mp4", + "title": "Cops Use Billy Clubs Against Pro-Trump and Anti-Fascist " + "Demonstrators", + "description": "Beverly Hills may be an omen of what's coming next week, " + "because things got crazy on the streets and cops started " + "swinging their billy clubs at both Anti-Fascist and Pro-Trump " + "demonstrators.", + "timestamp": 1604182772, + "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}", + "upload_date": "20201031", + }, + }, + ] def _real_extract(self, url): - video_id = self._match_id(url).replace('-', '_') - return self.url_result('kaltura:591531:%s' % video_id, 'Kaltura', video_id) - - -class TMZArticleIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P[^/]+)/?' - _TESTS = [{ - 'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert', - 'md5': '5429c85db8bde39a473a56ca8c4c5602', - 'info_dict': { - 'id': '0_6snoelag', - 'ext': 'mp4', - 'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake', - 'timestamp': 1429467813, - 'upload_date': '20150419', - 'uploader_id': 'batchUser', - } - }, { - 'url': 'http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/', - 'info_dict': { - 'id': '0_jerz7s3l', - 'ext': 'mp4', - 'title': 'Patti LaBelle -- Goes Nuclear On Stripping Fan', - 'timestamp': 1442683746, - 'upload_date': '20150919', - 'uploader_id': 'batchUser', - } - }, { - 'url': 'http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/', - 'info_dict': { - 'id': '0_ytz87kk7', - 'ext': 'mp4', - 'title': "NBA's Adam Silver -- Blake Griffin's a Great Guy ... He'll Learn from This", - 'timestamp': 1454010989, - 'upload_date': '20160128', - 'uploader_id': 'batchUser', - } - }, { - 'url': 'http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/', - 'info_dict': { - 'id': '0_isigfatu', - 'ext': 'mp4', - 'title': "Trump Star Vandal -- I'm Not Afraid of Donald or the Cops!", - 'timestamp': 1477500095, - 'upload_date': '20161026', - 'uploader_id': 'batchUser', - } - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - params = self._html_search_regex(r'TMZ.actions.clickLink\(([\s\S]+?)\)', - webpage, 'embedded video info').split(',') - new_url = params[0].strip("'\"") - if new_url != url: - return self.url_result(new_url) + webpage = self._download_webpage(url, url) + jsonld = self._search_json_ld(webpage, url) + if id not in jsonld: + jsonld["id"] = url + return jsonld From d71eb83b057d4933c3a0c655951ea4ad7a36c132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fernando=20Rodr=C3=ADguez=20Var=C3=B3n?= Date: Thu, 19 Nov 2020 23:51:43 -0500 Subject: [PATCH 3/3] Extract embedded youtube and twitter videos --- youtube_dlc/extractor/tmz.py | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/youtube_dlc/extractor/tmz.py b/youtube_dlc/extractor/tmz.py index a2f100922..aee2273b8 100644 --- a/youtube_dlc/extractor/tmz.py +++ b/youtube_dlc/extractor/tmz.py @@ -1,7 +1,13 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor +from ..utils import ( + ExtractorError, + get_element_by_attribute, +) class TMZIE(InfoExtractor): @@ -97,11 +103,55 @@ class TMZIE(InfoExtractor): "upload_date": "20201031", }, }, + { + "url": "https://www.tmz.com/2020/11/05/gervonta-davis-car-crash-hit-and-run-police/", + "info_dict": { + "id": "Dddb6IGe-ws", + "ext": "mp4", + "title": "SICK LAMBO GERVONTA DAVIS IN HIS NEW RIDE RIGHT AFTER KO AFTER LEO EsNews Boxing", + "uploader": "ESNEWS", + "description": "md5:49675bc58883ccf80474b8aa701e1064", + "upload_date": "20201101", + "uploader_id": "ESNEWS", + }, + }, + { + "url": "https://www.tmz.com/2020/11/19/conor-mcgregor-dustin-poirier-contract-fight-ufc-257-fight-island/", + "info_dict": { + "id": "1329450007125225473", + "ext": "mp4", + "title": "TheMacLife - BREAKING: Conor McGregor (@thenotoriousmma) has signed his bout agreement for his rematch with Dustin Poirier for January 23.", + "uploader": "TheMacLife", + "description": "md5:56e6009bbc3d12498e10d08a8e1f1c69", + "upload_date": "20201119", + "uploader_id": "Maclifeofficial", + "timestamp": 1605800556, + }, + }, ] def _real_extract(self, url): webpage = self._download_webpage(url, url) jsonld = self._search_json_ld(webpage, url) + if not jsonld or "url" not in jsonld: + # try to extract from YouTube Player API + # see https://developers.google.com/youtube/iframe_api_reference#Video_Queueing_Functions + match_obj = re.search(r'\.cueVideoById\(\s*(?P[\'"])(?P.*?)(?P=quote)', webpage) + if match_obj: + res = self.url_result(match_obj.group("id")) + return res + # try to extract from twitter + blockquote_el = get_element_by_attribute("class", "twitter-tweet", webpage) + if blockquote_el: + matches = re.findall( + r']+href=\s*(?P[\'"])(?P.*?)(?P=quote)', + blockquote_el) + if matches: + for _, match in matches: + if "/status/" in match: + res = self.url_result(match) + return res + raise ExtractorError("No video found!") if id not in jsonld: jsonld["id"] = url return jsonld