mirror of https://github.com/yt-dlp/yt-dlp
Merge branch 'yt-dlp:master' into playsuisse-handle-locale-parameters-from-urls
v3DJG6GL
4 months ago
committed by
GitHub
commit
c2709a75c4
@ -0,0 +1,84 @@
|
||||
import json
|
||||
import time
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
float_or_none,
|
||||
jwt_decode_hs256,
|
||||
parse_iso8601,
|
||||
url_or_none,
|
||||
variadic,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class CanalsurmasIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?canalsurmas\.es/videos/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.canalsurmas.es/videos/44006-el-gran-queo-1-lora-del-rio-sevilla-20072014',
|
||||
'md5': '861f86fdc1221175e15523047d0087ef',
|
||||
'info_dict': {
|
||||
'id': '44006',
|
||||
'ext': 'mp4',
|
||||
'title': 'Lora del Río (Sevilla)',
|
||||
'description': 'md5:3d9ee40a9b1b26ed8259e6b71ed27b8b',
|
||||
'thumbnail': 'https://cdn2.rtva.interactvty.com/content_cards/00f3e8f67b0a4f3b90a4a14618a48b0d.jpg',
|
||||
'timestamp': 1648123182,
|
||||
'upload_date': '20220324',
|
||||
},
|
||||
}]
|
||||
_API_BASE = 'https://api-rtva.interactvty.com'
|
||||
_access_token = None
|
||||
|
||||
@staticmethod
|
||||
def _is_jwt_expired(token):
|
||||
return jwt_decode_hs256(token)['exp'] - time.time() < 300
|
||||
|
||||
def _call_api(self, endpoint, video_id, fields=None):
|
||||
if not self._access_token or self._is_jwt_expired(self._access_token):
|
||||
self._access_token = self._download_json(
|
||||
f'{self._API_BASE}/jwt/token/', None,
|
||||
'Downloading access token', 'Failed to download access token',
|
||||
headers={'Content-Type': 'application/json'},
|
||||
data=json.dumps({
|
||||
'username': 'canalsur_demo',
|
||||
'password': 'dsUBXUcI',
|
||||
}).encode())['access']
|
||||
|
||||
return self._download_json(
|
||||
f'{self._API_BASE}/api/2.0/contents/{endpoint}/{video_id}/', video_id,
|
||||
f'Downloading {endpoint} API JSON', f'Failed to download {endpoint} API JSON',
|
||||
headers={'Authorization': f'jwtok {self._access_token}'},
|
||||
query={'optional_fields': ','.join(variadic(fields))} if fields else None)
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video_info = self._call_api('content', video_id, fields=[
|
||||
'description', 'image', 'duration', 'created_at', 'tags',
|
||||
])
|
||||
stream_info = self._call_api('content_resources', video_id, 'media_url')
|
||||
|
||||
formats, subtitles = [], {}
|
||||
for stream_url in traverse_obj(stream_info, ('results', ..., 'media_url', {url_or_none})):
|
||||
if determine_ext(stream_url) == 'm3u8':
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
stream_url, video_id, m3u8_id='hls', fatal=False)
|
||||
formats.extend(fmts)
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
else:
|
||||
formats.append({'url': stream_url})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(video_info, {
|
||||
'title': ('name', {str.strip}),
|
||||
'description': ('description', {str}),
|
||||
'thumbnail': ('image', {url_or_none}),
|
||||
'duration': ('duration', {float_or_none}),
|
||||
'timestamp': ('created_at', {parse_iso8601}),
|
||||
'tags': ('tags', ..., {str}),
|
||||
}),
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
urlencode_postdata,
|
||||
)
|
||||
|
||||
|
||||
class GigyaBaseIE(InfoExtractor):
|
||||
def _gigya_login(self, auth_data):
|
||||
auth_info = self._download_json(
|
||||
'https://accounts.eu1.gigya.com/accounts.login', None,
|
||||
note='Logging in', errnote='Unable to log in',
|
||||
data=urlencode_postdata(auth_data))
|
||||
|
||||
error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage')
|
||||
if error_message:
|
||||
raise ExtractorError(
|
||||
f'Unable to login: {error_message}', expected=True)
|
||||
return auth_info
|
||||
@ -0,0 +1,50 @@
|
||||
# flake8: noqa: F401
|
||||
from ._base import YoutubeBaseInfoExtractor
|
||||
from ._clip import YoutubeClipIE
|
||||
from ._mistakes import YoutubeTruncatedIDIE, YoutubeTruncatedURLIE
|
||||
from ._notifications import YoutubeNotificationsIE
|
||||
from ._redirect import (
|
||||
YoutubeConsentRedirectIE,
|
||||
YoutubeFavouritesIE,
|
||||
YoutubeFeedsInfoExtractor,
|
||||
YoutubeHistoryIE,
|
||||
YoutubeLivestreamEmbedIE,
|
||||
YoutubeRecommendedIE,
|
||||
YoutubeShortsAudioPivotIE,
|
||||
YoutubeSubscriptionsIE,
|
||||
YoutubeWatchLaterIE,
|
||||
YoutubeYtBeIE,
|
||||
YoutubeYtUserIE,
|
||||
)
|
||||
from ._search import YoutubeMusicSearchURLIE, YoutubeSearchDateIE, YoutubeSearchIE, YoutubeSearchURLIE
|
||||
from ._tab import YoutubePlaylistIE, YoutubeTabBaseInfoExtractor, YoutubeTabIE
|
||||
from ._video import YoutubeIE
|
||||
|
||||
# Hack to allow plugin overrides work
|
||||
for _cls in [
|
||||
YoutubeBaseInfoExtractor,
|
||||
YoutubeClipIE,
|
||||
YoutubeTruncatedIDIE,
|
||||
YoutubeTruncatedURLIE,
|
||||
YoutubeNotificationsIE,
|
||||
YoutubeConsentRedirectIE,
|
||||
YoutubeFavouritesIE,
|
||||
YoutubeFeedsInfoExtractor,
|
||||
YoutubeHistoryIE,
|
||||
YoutubeLivestreamEmbedIE,
|
||||
YoutubeRecommendedIE,
|
||||
YoutubeShortsAudioPivotIE,
|
||||
YoutubeSubscriptionsIE,
|
||||
YoutubeWatchLaterIE,
|
||||
YoutubeYtBeIE,
|
||||
YoutubeYtUserIE,
|
||||
YoutubeMusicSearchURLIE,
|
||||
YoutubeSearchDateIE,
|
||||
YoutubeSearchIE,
|
||||
YoutubeSearchURLIE,
|
||||
YoutubePlaylistIE,
|
||||
YoutubeTabBaseInfoExtractor,
|
||||
YoutubeTabIE,
|
||||
YoutubeIE,
|
||||
]:
|
||||
_cls.__module__ = 'yt_dlp.extractor.youtube'
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,66 @@
|
||||
from ._tab import YoutubeTabBaseInfoExtractor
|
||||
from ._video import YoutubeIE
|
||||
from ...utils import ExtractorError, traverse_obj
|
||||
|
||||
|
||||
class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
|
||||
IE_NAME = 'youtube:clip'
|
||||
_VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
|
||||
_TESTS = [{
|
||||
# FIXME: Other metadata should be extracted from the clip, not from the base video
|
||||
'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
|
||||
'info_dict': {
|
||||
'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
|
||||
'ext': 'mp4',
|
||||
'section_start': 29.0,
|
||||
'section_end': 39.7,
|
||||
'duration': 10.7,
|
||||
'age_limit': 0,
|
||||
'availability': 'public',
|
||||
'categories': ['Gaming'],
|
||||
'channel': 'Scott The Woz',
|
||||
'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
|
||||
'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
|
||||
'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
|
||||
'like_count': int,
|
||||
'playable_in_embed': True,
|
||||
'tags': 'count:17',
|
||||
'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
|
||||
'title': 'Mobile Games on Console - Scott The Woz',
|
||||
'upload_date': '20210920',
|
||||
'uploader': 'Scott The Woz',
|
||||
'uploader_id': '@ScottTheWoz',
|
||||
'uploader_url': 'https://www.youtube.com/@ScottTheWoz',
|
||||
'view_count': int,
|
||||
'live_status': 'not_live',
|
||||
'channel_follower_count': int,
|
||||
'chapters': 'count:20',
|
||||
'comment_count': int,
|
||||
'heatmap': 'count:100',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
clip_id = self._match_id(url)
|
||||
_, data = self._extract_webpage(url, clip_id)
|
||||
|
||||
video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
|
||||
if not video_id:
|
||||
raise ExtractorError('Unable to find video ID')
|
||||
|
||||
clip_data = traverse_obj(data, (
|
||||
'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
|
||||
'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
|
||||
'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
|
||||
'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
|
||||
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'url': f'https://www.youtube.com/watch?v={video_id}',
|
||||
'ie_key': YoutubeIE.ie_key(),
|
||||
'id': clip_id,
|
||||
'section_start': int(clip_data['startTimeMs']) / 1000,
|
||||
'section_end': int(clip_data['endTimeMs']) / 1000,
|
||||
'_format_sort_fields': ( # https protocol is prioritized for ffmpeg compatibility
|
||||
'proto:https', 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec', 'channels', 'acodec', 'lang'),
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
|
||||
from ._base import YoutubeBaseInfoExtractor
|
||||
from ...utils import ExtractorError
|
||||
|
||||
|
||||
class YoutubeTruncatedURLIE(YoutubeBaseInfoExtractor):
|
||||
IE_NAME = 'youtube:truncated_url'
|
||||
IE_DESC = False # Do not list
|
||||
_VALID_URL = r'''(?x)
|
||||
(?:https?://)?
|
||||
(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
|
||||
(?:watch\?(?:
|
||||
feature=[a-z_]+|
|
||||
annotation_id=annotation_[^&]+|
|
||||
x-yt-cl=[0-9]+|
|
||||
hl=[^&]*|
|
||||
t=[0-9]+
|
||||
)?
|
||||
|
|
||||
attribution_link\?a=[^&]+
|
||||
)
|
||||
$
|
||||
'''
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.youtube.com/watch?',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.youtube.com/watch?feature=foo',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.youtube.com/watch?hl=en-GB',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.youtube.com/watch?t=2372',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
raise ExtractorError(
|
||||
'Did you forget to quote the URL? Remember that & is a meta '
|
||||
'character in most shells, so you want to put the URL in quotes, '
|
||||
'like yt-dlp '
|
||||
'"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
|
||||
' or simply yt-dlp BaW_jenozKc .',
|
||||
expected=True)
|
||||
|
||||
|
||||
class YoutubeTruncatedIDIE(YoutubeBaseInfoExtractor):
|
||||
IE_NAME = 'youtube:truncated_id'
|
||||
IE_DESC = False # Do not list
|
||||
_VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
raise ExtractorError(
|
||||
f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
|
||||
expected=True)
|
||||
@ -0,0 +1,98 @@
|
||||
import itertools
|
||||
import re
|
||||
|
||||
from ._tab import YoutubeTabBaseInfoExtractor, YoutubeTabIE
|
||||
from ._video import YoutubeIE
|
||||
from ...utils import traverse_obj
|
||||
|
||||
|
||||
class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
|
||||
IE_NAME = 'youtube:notif'
|
||||
IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
|
||||
_VALID_URL = r':ytnotif(?:ication)?s?'
|
||||
_LOGIN_REQUIRED = True
|
||||
_TESTS = [{
|
||||
'url': ':ytnotif',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': ':ytnotifications',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _extract_notification_menu(self, response, continuation_list):
|
||||
notification_list = traverse_obj(
|
||||
response,
|
||||
('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
|
||||
('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
|
||||
expected_type=list) or []
|
||||
continuation_list[0] = None
|
||||
for item in notification_list:
|
||||
entry = self._extract_notification_renderer(item.get('notificationRenderer'))
|
||||
if entry:
|
||||
yield entry
|
||||
continuation = item.get('continuationItemRenderer')
|
||||
if continuation:
|
||||
continuation_list[0] = continuation
|
||||
|
||||
def _extract_notification_renderer(self, notification):
|
||||
video_id = traverse_obj(
|
||||
notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
|
||||
url = f'https://www.youtube.com/watch?v={video_id}'
|
||||
channel_id = None
|
||||
if not video_id:
|
||||
browse_ep = traverse_obj(
|
||||
notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
|
||||
channel_id = self.ucid_or_none(traverse_obj(browse_ep, 'browseId', expected_type=str))
|
||||
post_id = self._search_regex(
|
||||
r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
|
||||
'post id', default=None)
|
||||
if not channel_id or not post_id:
|
||||
return
|
||||
# The direct /post url redirects to this in the browser
|
||||
url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
|
||||
|
||||
channel = traverse_obj(
|
||||
notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
|
||||
expected_type=str)
|
||||
notification_title = self._get_text(notification, 'shortMessage')
|
||||
if notification_title:
|
||||
notification_title = notification_title.replace('\xad', '') # remove soft hyphens
|
||||
# TODO: handle recommended videos
|
||||
title = self._search_regex(
|
||||
rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
|
||||
'video title', default=None)
|
||||
timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
|
||||
if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
|
||||
else None)
|
||||
return {
|
||||
'_type': 'url',
|
||||
'url': url,
|
||||
'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
|
||||
'video_id': video_id,
|
||||
'title': title,
|
||||
'channel_id': channel_id,
|
||||
'channel': channel,
|
||||
'uploader': channel,
|
||||
'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
|
||||
'timestamp': timestamp,
|
||||
}
|
||||
|
||||
def _notification_menu_entries(self, ytcfg):
|
||||
continuation_list = [None]
|
||||
response = None
|
||||
for page in itertools.count(1):
|
||||
ctoken = traverse_obj(
|
||||
continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
|
||||
response = self._extract_response(
|
||||
item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
|
||||
ep='notification/get_notification_menu', check_get_keys='actions',
|
||||
headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
|
||||
yield from self._extract_notification_menu(response, continuation_list)
|
||||
if not continuation_list[0]:
|
||||
break
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = 'notifications'
|
||||
ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
|
||||
self._report_playlist_authcheck(ytcfg)
|
||||
return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue