From 7c97ead1b0560bd4dab6a636b1a518aa0a81893d Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:48:21 -0400 Subject: [PATCH 01/12] migrate from old rplay branch --- yt_dlp/extractor/_extractors.py | 5 + yt_dlp/extractor/rplaylive.py | 329 ++++++++++++++++++++++++++++++++ 2 files changed, 334 insertions(+) create mode 100644 yt_dlp/extractor/rplaylive.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 9b73fcd75e..14007d1d80 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -1728,6 +1728,11 @@ from .rozhlas import ( RozhlasIE, RozhlasVltavaIE, ) +from .rplaylive import ( + RPlayLiveIE, + RPlayUserIE, + RPlayVideoIE, +) from .rte import ( RteIE, RteRadioIE, diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py new file mode 100644 index 0000000000..208b175837 --- /dev/null +++ b/yt_dlp/extractor/rplaylive.py @@ -0,0 +1,329 @@ +import base64 +import datetime as dt +import hashlib +import hmac +import json +import random +import re +import time + +from .common import InfoExtractor +from ..aes import aes_cbc_encrypt_bytes +from ..utils import ( + ExtractorError, + UserNotLive, + encode_data_uri, + float_or_none, + parse_iso8601, + parse_qs, + traverse_obj, + url_or_none, +) + + +class RPlayBaseIE(InfoExtractor): + _NETRC_MACHINE = 'rplaylive' + _TOKEN_CACHE = {} + _user_id = None + _login_type = None + _jwt_token = None + + @property + def user_id(self): + return self._user_id + + @property + def login_type(self): + return self._login_type + + @property + def jwt_token(self): + return self._jwt_token + + @property + def requestor_query(self): + return { + 'requestorOid': self.user_id, + 'loginType': self.login_type, + } if self.user_id else {} + + @property + def jwt_header(self): + return { + 'Referer': 'https://rplay.live/', + 'Authorization': self.jwt_token or 'null', + } + + def _jwt_encode_hs256(self, payload: dict, key: str): + # yt_dlp.utils.jwt_encode_hs256() uses slightly different details that would fails + # and we need to re-implement it with minor changes + b64encode = lambda x: base64.urlsafe_b64encode( + json.dumps(x, separators=(',', ':')).encode()).strip(b'=') + + header_b64 = b64encode({'alg': 'HS256', 'typ': 'JWT'}) + payload_b64 = b64encode(payload) + h = hmac.new(key.encode(), header_b64 + b'.' + payload_b64, hashlib.sha256) + signature_b64 = base64.urlsafe_b64encode(h.digest()).strip(b'=') + return header_b64 + b'.' + payload_b64 + b'.' + signature_b64 + + def _perform_login(self, username, password): + payload = { + 'eml': username, + 'dat': dt.datetime.now(dt.timezone.utc).isoformat(timespec='milliseconds').replace('+00:00', 'Z'), + 'iat': int(time.time()), + } + key = hashlib.sha256(password.encode()).hexdigest() + self._login_by_token(self._jwt_encode_hs256(payload, key).decode()) + + def _login_by_token(self, jwt_token): + user_info = self._download_json( + 'https://api.rplay.live/account/login', 'login', note='performing login', errnote='Failed to login', + data=f'{{"token":"{jwt_token}","loginType":null,"checkAdmin":null}}'.encode(), + headers={'Content-Type': 'application/json', 'Authorization': 'null'}, fatal=False) + + if user_info: + self._user_id = traverse_obj(user_info, 'oid') + self._login_type = traverse_obj(user_info, 'accountType') + self._jwt_token = jwt_token if self._user_id else None + if not self._user_id: + self.report_warning('Failed to login, possibly due to wrong password or website change') + + def get_butter_token(self): + salt = 'QWI@(!WAS)Dj1AA(!@*DJ#@$@~1)P' + key = 'S%M@#H#B(!@()a2@' + ts_value = str(int(time.time() / 360)) + enc = aes_cbc_encrypt_bytes(f'{salt}https://rplay.live{ts_value}', key, ts_value.zfill(16)) + return enc.hex() + + +class RPlayVideoIE(RPlayBaseIE): + _VALID_URL = r'https://rplay.live/play/(?P[\d\w]+)' + _TESTS = [{ + 'url': 'https://rplay.live/play/669203d25223214e67579dc3/', + 'info_dict': { + 'id': '669203d25223214e67579dc3', + 'ext': 'mp4', + 'title': 'md5:6ab0a76410b40b1f5fb48a2ad7571264', + 'description': 'md5:d2fb2f74a623be439cf454df5ff3344a', + 'timestamp': 1720845266, + 'upload_date': '20240713', + 'release_timestamp': 1720846360, + 'release_date': '20240713', + 'duration': 5349.0, + 'thumbnail': r're:https://[\w\d]+.cloudfront.net/.*', + 'uploader': '杏都める', + 'uploader_id': '667adc9e9aa7f739a2158ff3', + 'tags': ['杏都める', 'めいどるーちぇ', '無料', '耳舐め', 'ASMR'], + }, + }, { + 'url': 'https://rplay.live/play/660bee4fd3c1d09d69db6870/', + 'info_dict': { + 'id': '660bee4fd3c1d09d69db6870', + 'ext': 'mp4', + 'title': 'md5:7de162a0f1c2266ec428234620a124fc', + 'description': 'md5:c6d12cc8110b748d5588d5f00787cd35', + 'timestamp': 1712057935, + 'upload_date': '20240402', + 'release_timestamp': 1712061900, + 'release_date': '20240402', + 'duration': 6791.0, + 'thumbnail': r're:https://[\w\d]+.cloudfront.net/.*', + 'uploader': '狐月れんげ', + 'uploader_id': '65eeb4b237043dc0b5654f86', + 'tags': 'count:10', + 'age_limit': 18, + }, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + + self.get_butter_token() + + playlist_id = traverse_obj(parse_qs(url), ('playlist', ..., any)) + if playlist_id and self._yes_playlist(playlist_id, video_id): + playlist_info = self._download_json( + 'https://api.rplay.live/content/playlist', playlist_id, + query={'playlistOid': playlist_id, **self.requestor_query}, + headers=self.jwt_header, fatal=False) + if playlist_info: + entries = traverse_obj(playlist_info, ('contentData', ..., '_id', { + lambda x: self.url_result(f'https://rplay.live/play/{x}/', ie=RPlayVideoIE, video_id=x)})) + return self.playlist_result(entries, playlist_id, playlist_info.get('name')) + else: + self.report_warning('Failed to get playlist, downloading video only') + + video_info = self._download_json('https://api.rplay.live/content', video_id, query={ + 'contentOid': video_id, + 'status': 'published', + 'withComments': True, + 'requestCanView': True, + **self.requestor_query, + }, headers=self.jwt_header) + if video_info.get('drm'): + raise ExtractorError('This video is DRM-protected') + + metainfo = traverse_obj(video_info, { + 'title': ('title', {str}), + 'description': ('introText', {str}), + 'release_timestamp': ('publishedAt', {parse_iso8601}), + 'timestamp': ('createdAt', {parse_iso8601}), + 'duration': ('length', {float_or_none}), + 'uploader': ('nickname', {str}), + 'uploader_id': ('creatorOid', {str}), + 'tags': ('hashtags', lambda _, v: v[0] != '_'), + 'age_limit': (('hideContent', 'isAdultContent'), {lambda x: 18 if x else None}, any), + }) + + m3u8_url = traverse_obj(video_info, ('canView', 'url', {url_or_none})) + if not m3u8_url: + msg = 'You do not have access to this video' + if traverse_obj(video_info, ('viewableTiers', 'free')): + msg = 'This video requires a free subscription to access' + if not self.user_id: + # browser credential is stored in localStorage + msg += f'. {self._login_hint(method="password")}' + raise ExtractorError(msg, expected=True) + + thumbnail_key = traverse_obj(video_info, ( + 'streamables', lambda _, v: v['type'].startswith('image/'), 's3key', any)) + if thumbnail_key: + metainfo['thumbnail'] = url_or_none(self._download_webpage( + 'https://api.rplay.live/upload/privateasset', video_id, 'getting cover url', query={ + 'key': thumbnail_key, + 'contentOid': video_id, + 'creatorOid': metainfo.get('uploader_id'), + **self.requestor_query, + }, fatal=False)) + + formats = self._extract_m3u8_formats(m3u8_url, video_id, headers={ + 'Referer': 'https://rplay.live/', 'Butter': self.get_butter_token()}) + for fmt in formats: + m3u8_doc = self._download_webpage(fmt['url'], video_id, 'getting m3u8 contents', headers={ + 'Referer': 'https://rplay.live/', 'Butter': self.get_butter_token()}) + fmt['url'] = encode_data_uri(m3u8_doc.encode(), 'application/x-mpegurl') + match = re.search(r'^#EXT-X-KEY.*?URI="([^"]+)"', m3u8_doc, flags=re.M) + if match: + urlh = self._request_webpage(match[1], video_id, 'getting hls key', headers={ + 'Referer': 'https://rplay.live/', + 'rplay-private-content-requestor': self.user_id or 'not-logged-in', + 'age': random.randint(1, 4999), + }) + fmt['hls_aes'] = {'key': urlh.read().hex()} + + return { + 'id': video_id, + 'formats': formats, + **metainfo, + 'http_headers': {'Referer': 'https://rplay.live/'}, + } + + +class RPlayUserIE(InfoExtractor): + _VALID_URL = r'https://rplay.live/(?Pc|creatorhome)/(?P[\d\w]+)/?(?:[#?]|$)' + _TESTS = [{ + 'url': 'https://rplay.live/creatorhome/667adc9e9aa7f739a2158ff3?page=contents', + 'info_dict': { + 'id': '667adc9e9aa7f739a2158ff3', + 'title': '杏都める', + }, + 'playlist_mincount': 34, + }, { + 'url': 'https://rplay.live/c/furachi?page=contents', + 'info_dict': { + 'id': '65e07e60850f4527aab74757', + 'title': '逢瀬ふらち OuseFurachi', + }, + 'playlist_mincount': 77, + }] + + def _real_extract(self, url): + user_id, short = self._match_valid_url(url).group('id', 'short') + key = 'customUrl' if short == 'c' else 'userOid' + + user_info = self._download_json( + f'https://api.rplay.live/account/getuser?{key}={user_id}&filter[]=nickname&filter[]=published', user_id) + replays = self._download_json( + 'https://api.rplay.live/live/replays?=667e4cd99aa7f739a2c91852', user_id, query={ + 'creatorOid': user_info.get('_id')}) + + entries = traverse_obj(user_info, ('published', ..., { + lambda x: self.url_result(f'https://rplay.live/play/{x}/', ie=RPlayVideoIE, video_id=x)})) + for entry_id in traverse_obj(replays, (..., '_id', {str})): + if entry_id in user_info.get('published', []): + continue + entries.append(self.url_result(f'https://rplay.live/play/{entry_id}/', ie=RPlayVideoIE, video_id=entry_id)) + + return self.playlist_result(entries, user_info.get('_id', user_id), user_info.get('nickname')) + + +class RPlayLiveIE(RPlayBaseIE): + _VALID_URL = [ + r'https://rplay.live/(?Pc)/(?P[\d\w]+)/live', + r'https://rplay.live/(?Plive)/(?P[\d\w]+)', + ] + _TESTS = [{ + 'url': 'https://rplay.live/c/chachamaru/live', + 'info_dict': { + 'id': '667e4cd99aa7f739a2c91852', + 'ext': 'mp4', + 'title': r're:【ASMR】ん～っやば//スキスキ耐久.*', + 'description': 'md5:7f88ac0a7a3d5d0b926a0baecd1d40e1', + 'timestamp': 1721739947, + 'upload_date': '20240723', + 'live_status': 'is_live', + 'thumbnail': 'https://pb.rplay.live/liveChannelThumbnails/667e4cd99aa7f739a2c91852', + 'uploader': '愛犬茶々丸', + 'uploader_id': '667e4cd99aa7f739a2c91852', + 'tags': 'count:9', + }, + 'skip': 'live', + }, { + 'url': 'https://rplay.live/live/667adc9e9aa7f739a2158ff3', + 'only_matching': True, + }] + + def _real_extract(self, url): + user_id, short = self._match_valid_url(url).group('id', 'short') + + if short == 'c': + user_info = self._download_json(f'https://api.rplay.live/account/getuser?customUrl={user_id}', user_id) + user_id = user_info['_id'] + else: + user_info = self._download_json(f'https://api.rplay.live/account/getuser?userOid={user_id}', user_id) + + live_info = self._download_json('https://api.rplay.live/live/play', user_id, query={'creatorOid': user_id}) + + stream_state = live_info['streamState'] + if stream_state == 'youtube': + return self.url_result(f'https://www.youtube.com/watch?v={live_info["liveStreamId"]}') + elif stream_state == 'live': + if not self.user_id and not live_info.get('allowAnonymous'): + self.raise_login_required(method='password') + key2 = self._download_webpage( + 'https://api.rplay.live/live/key2', user_id, 'getting live key', + headers=self.jwt_header, query=self.requestor_query) if self.user_id else '' + formats = self._extract_m3u8_formats( + 'https://api.rplay.live/live/stream/playlist.m3u8', user_id, + query={'creatorOid': user_id, 'key2': key2}) + + return { + 'id': user_id, + 'formats': formats, + 'is_live': True, + 'http_headers': {'Referer': 'https://rplay.live'}, + 'thumbnail': f'https://pb.rplay.live/liveChannelThumbnails/{user_id}', + 'uploader': traverse_obj(user_info, ('nickname', {str})), + 'uploader_id': user_id, + **traverse_obj(live_info, { + 'title': ('title', {str}), + 'description': ('description', {str}), + 'timestamp': ('streamStartTime', {parse_iso8601}), + 'tags': ('hashtags', ..., {str}), + 'age_limit': ('isAdultContent', {lambda x: 18 if x else None}), + }), + } + elif stream_state == 'offline': + raise UserNotLive + else: + raise ExtractorError(f'Unknow streamState: {stream_state}') From ee838b418c2e006a24c5df7cb036112cb247604e Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Thu, 8 Aug 2024 17:14:42 -0400 Subject: [PATCH 02/12] vod fix --- yt_dlp/extractor/rplaylive.py | 38 ++++++++++++++--------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index 208b175837..cec0ae6701 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -110,28 +110,29 @@ class RPlayVideoIE(RPlayBaseIE): 'release_timestamp': 1720846360, 'release_date': '20240713', 'duration': 5349.0, - 'thumbnail': r're:https://[\w\d]+.cloudfront.net/.*', + 'thumbnail': 'https://pb.rplay.live/thumbnail/669203d25223214e67579dc3', 'uploader': '杏都める', 'uploader_id': '667adc9e9aa7f739a2158ff3', 'tags': ['杏都める', 'めいどるーちぇ', '無料', '耳舐め', 'ASMR'], }, }, { - 'url': 'https://rplay.live/play/660bee4fd3c1d09d69db6870/', + 'url': 'https://rplay.live/play/66783c65dcd1c768a8a69f24/', 'info_dict': { - 'id': '660bee4fd3c1d09d69db6870', + 'id': '66783c65dcd1c768a8a69f24', 'ext': 'mp4', - 'title': 'md5:7de162a0f1c2266ec428234620a124fc', - 'description': 'md5:c6d12cc8110b748d5588d5f00787cd35', - 'timestamp': 1712057935, - 'upload_date': '20240402', - 'release_timestamp': 1712061900, - 'release_date': '20240402', - 'duration': 6791.0, - 'thumbnail': r're:https://[\w\d]+.cloudfront.net/.*', + 'title': 'md5:9be2febe48cee1b7536e3e9d4d5f8e56', + 'description': 'md5:a71374d3dcd1db0f852b96a69b41b699', + 'timestamp': 1719155813, + 'upload_date': '20240623', + 'release_timestamp': 1719155813, + 'release_date': '20240623', + 'duration': 4237.0, + 'thumbnail': 'https://pb.rplay.live/thumbnail/66783c65dcd1c768a8a69f24', 'uploader': '狐月れんげ', 'uploader_id': '65eeb4b237043dc0b5654f86', - 'tags': 'count:10', + 'tags': 'count:4', 'age_limit': 18, + 'live_status': 'was_live', }, }] @@ -173,6 +174,7 @@ class RPlayVideoIE(RPlayBaseIE): 'uploader_id': ('creatorOid', {str}), 'tags': ('hashtags', lambda _, v: v[0] != '_'), 'age_limit': (('hideContent', 'isAdultContent'), {lambda x: 18 if x else None}, any), + 'live_status': ('isReplayContent', {lambda x: 'was_live' if x else None}), }) m3u8_url = traverse_obj(video_info, ('canView', 'url', {url_or_none})) @@ -185,17 +187,6 @@ class RPlayVideoIE(RPlayBaseIE): msg += f'. {self._login_hint(method="password")}' raise ExtractorError(msg, expected=True) - thumbnail_key = traverse_obj(video_info, ( - 'streamables', lambda _, v: v['type'].startswith('image/'), 's3key', any)) - if thumbnail_key: - metainfo['thumbnail'] = url_or_none(self._download_webpage( - 'https://api.rplay.live/upload/privateasset', video_id, 'getting cover url', query={ - 'key': thumbnail_key, - 'contentOid': video_id, - 'creatorOid': metainfo.get('uploader_id'), - **self.requestor_query, - }, fatal=False)) - formats = self._extract_m3u8_formats(m3u8_url, video_id, headers={ 'Referer': 'https://rplay.live/', 'Butter': self.get_butter_token()}) for fmt in formats: @@ -215,6 +206,7 @@ class RPlayVideoIE(RPlayBaseIE): 'id': video_id, 'formats': formats, **metainfo, + 'thumbnail': f'https://pb.rplay.live/thumbnail/{video_id}', 'http_headers': {'Referer': 'https://rplay.live/'}, } From d92f6fbaea9811d035b73b7801a54e0ba67c2ca3 Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Fri, 9 Aug 2024 18:23:25 -0400 Subject: [PATCH 03/12] fix --- yt_dlp/extractor/rplaylive.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index cec0ae6701..b90e9ef80d 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -139,8 +139,6 @@ class RPlayVideoIE(RPlayBaseIE): def _real_extract(self, url): video_id = self._match_id(url) - self.get_butter_token() - playlist_id = traverse_obj(parse_qs(url), ('playlist', ..., any)) if playlist_id and self._yes_playlist(playlist_id, video_id): playlist_info = self._download_json( @@ -170,7 +168,7 @@ class RPlayVideoIE(RPlayBaseIE): 'release_timestamp': ('publishedAt', {parse_iso8601}), 'timestamp': ('createdAt', {parse_iso8601}), 'duration': ('length', {float_or_none}), - 'uploader': ('nickname', {str}), + 'uploader': ('creatorInfo', 'nickname', {str}), 'uploader_id': ('creatorOid', {str}), 'tags': ('hashtags', lambda _, v: v[0] != '_'), 'age_limit': (('hideContent', 'isAdultContent'), {lambda x: 18 if x else None}, any), From d9fd3dbdfaec5e8ad02ec352189978f1018fb665 Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Sun, 11 Aug 2024 03:38:25 -0400 Subject: [PATCH 04/12] improve user video list --- yt_dlp/extractor/rplaylive.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index b90e9ef80d..1df2d90009 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -217,34 +217,36 @@ class RPlayUserIE(InfoExtractor): 'id': '667adc9e9aa7f739a2158ff3', 'title': '杏都める', }, - 'playlist_mincount': 34, + 'playlist_mincount': 35, }, { 'url': 'https://rplay.live/c/furachi?page=contents', 'info_dict': { 'id': '65e07e60850f4527aab74757', 'title': '逢瀬ふらち OuseFurachi', }, - 'playlist_mincount': 77, + 'playlist_mincount': 94, }] def _real_extract(self, url): user_id, short = self._match_valid_url(url).group('id', 'short') key = 'customUrl' if short == 'c' else 'userOid' - user_info = self._download_json( - f'https://api.rplay.live/account/getuser?{key}={user_id}&filter[]=nickname&filter[]=published', user_id) + user_info = self._download_json('https://api.rplay.live/account/getuser', user_id, query={ + key: user_id, 'filter[]': ['nickname', 'published', 'publishedClips'], + 'options': '{"includeContentMetadata":true}'}) replays = self._download_json( - 'https://api.rplay.live/live/replays?=667e4cd99aa7f739a2c91852', user_id, query={ - 'creatorOid': user_info.get('_id')}) - - entries = traverse_obj(user_info, ('published', ..., { - lambda x: self.url_result(f'https://rplay.live/play/{x}/', ie=RPlayVideoIE, video_id=x)})) - for entry_id in traverse_obj(replays, (..., '_id', {str})): - if entry_id in user_info.get('published', []): - continue - entries.append(self.url_result(f'https://rplay.live/play/{entry_id}/', ie=RPlayVideoIE, video_id=entry_id)) - - return self.playlist_result(entries, user_info.get('_id', user_id), user_info.get('nickname')) + 'https://api.rplay.live/live/replays', user_id, query={'creatorOid': user_info.get('_id')}) + + def _entries(): + def _entry_ids(): + for entry in traverse_obj(user_info, ('metadataSet', ..., lambda _, v: v['_id'])): + yield entry['_id'], entry.get('title') + for entry in traverse_obj(replays, lambda _, v: v['_id']): + yield entry['_id'], entry.get('title') + for vid, title in dict(_entry_ids()).items(): + yield self.url_result(f'https://rplay.live/play/{vid}', ie=RPlayVideoIE, id=vid, title=title) + + return self.playlist_result(_entries(), user_info.get('_id', user_id), user_info.get('nickname')) class RPlayLiveIE(RPlayBaseIE): From 388dc541da1bc9d188160b2e1d95db53ecfae612 Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Sun, 11 Aug 2024 04:36:38 -0400 Subject: [PATCH 05/12] improve --- yt_dlp/extractor/rplaylive.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index 1df2d90009..8286307d01 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -181,7 +181,7 @@ class RPlayVideoIE(RPlayBaseIE): if traverse_obj(video_info, ('viewableTiers', 'free')): msg = 'This video requires a free subscription to access' if not self.user_id: - # browser credential is stored in localStorage + # credential is in browser localStorage only, no cookies to use msg += f'. {self._login_hint(method="password")}' raise ExtractorError(msg, expected=True) @@ -219,7 +219,7 @@ class RPlayUserIE(InfoExtractor): }, 'playlist_mincount': 35, }, { - 'url': 'https://rplay.live/c/furachi?page=contents', + 'url': 'https://rplay.live/c/furachi', 'info_dict': { 'id': '65e07e60850f4527aab74757', 'title': '逢瀬ふらち OuseFurachi', @@ -229,11 +229,9 @@ class RPlayUserIE(InfoExtractor): def _real_extract(self, url): user_id, short = self._match_valid_url(url).group('id', 'short') - key = 'customUrl' if short == 'c' else 'userOid' user_info = self._download_json('https://api.rplay.live/account/getuser', user_id, query={ - key: user_id, 'filter[]': ['nickname', 'published', 'publishedClips'], - 'options': '{"includeContentMetadata":true}'}) + 'customUrl' if short == 'c' else 'userOid': user_id, 'options': '{"includeContentMetadata":true}'}) replays = self._download_json( 'https://api.rplay.live/live/replays', user_id, query={'creatorOid': user_info.get('_id')}) @@ -278,11 +276,11 @@ class RPlayLiveIE(RPlayBaseIE): def _real_extract(self, url): user_id, short = self._match_valid_url(url).group('id', 'short') - if short == 'c': - user_info = self._download_json(f'https://api.rplay.live/account/getuser?customUrl={user_id}', user_id) - user_id = user_info['_id'] - else: - user_info = self._download_json(f'https://api.rplay.live/account/getuser?userOid={user_id}', user_id) + user_info = self._download_json('https://api.rplay.live/account/getuser', user_id, query={ + 'customUrl' if short == 'c' else 'userOid': user_id}) + if user_info.get('isLive') is False: + raise UserNotLive + user_id = user_info['_id'] live_info = self._download_json('https://api.rplay.live/live/play', user_id, query={'creatorOid': user_id}) From 02f4b25227929dd3b903b493df262b48128901ea Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Sun, 11 Aug 2024 10:45:10 -0400 Subject: [PATCH 06/12] update --- yt_dlp/extractor/rplaylive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index 8286307d01..180c77d269 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -287,6 +287,8 @@ class RPlayLiveIE(RPlayBaseIE): stream_state = live_info['streamState'] if stream_state == 'youtube': return self.url_result(f'https://www.youtube.com/watch?v={live_info["liveStreamId"]}') + elif stream_state == 'twitch': + return self.url_result(f'https://www.twitch.tv/{live_info["twitchLogin"]}') elif stream_state == 'live': if not self.user_id and not live_info.get('allowAnonymous'): self.raise_login_required(method='password') From dcd2e93fa0f585c434c68e54cc603ae79725a69c Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Thu, 19 Sep 2024 20:09:48 -0400 Subject: [PATCH 07/12] support jwt arg --- README.md | 4 ++++ yt_dlp/extractor/rplaylive.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ca32e09bfb..a061a87073 100644 --- a/README.md +++ b/README.md @@ -1864,6 +1864,10 @@ The following extractors use this feature: #### digitalconcerthall * `prefer_combined_hls`: Prefer extracting combined/pre-merged video and audio HLS formats. This will exclude 4K/HEVC video and lossless/FLAC audio formats, which are only available as split video/audio HLS formats +#### rplaylive + +* `jwt_token`: JWT token that can be found as value of `_AUTHORIZATION_` entry from the browser local storage. This can be used as an alternative login method. + **Note**: These options may be changed/removed in the future without concern for backward compatibility diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index 180c77d269..c43e403d15 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -27,17 +27,27 @@ class RPlayBaseIE(InfoExtractor): _user_id = None _login_type = None _jwt_token = None + _tested_jwt = False + + def _check_jwt_args(self): + jwt_arg = self._configuration_arg('jwt_token', ie_key='rplaylive', casesense=True) + if self._jwt_token is None and jwt_arg and not self._tested_jwt: + self._login_by_token(jwt_arg[0], raw_token_hint=True) + self._tested_jwt = True @property def user_id(self): + self._check_jwt_args() return self._user_id @property def login_type(self): + self._check_jwt_args() return self._login_type @property def jwt_token(self): + self._check_jwt_args() return self._jwt_token @property @@ -54,6 +64,10 @@ class RPlayBaseIE(InfoExtractor): 'Authorization': self.jwt_token or 'null', } + def _login_hint(self, **kwargs): + return (f'Use --username and --password, --netrc-cmd, --netrc ({self._NETRC_MACHINE}) ' + 'or --extractor-args "rplaylive:jwt_token=xxx" to provide account credentials') + def _jwt_encode_hs256(self, payload: dict, key: str): # yt_dlp.utils.jwt_encode_hs256() uses slightly different details that would fails # and we need to re-implement it with minor changes @@ -75,9 +89,9 @@ class RPlayBaseIE(InfoExtractor): key = hashlib.sha256(password.encode()).hexdigest() self._login_by_token(self._jwt_encode_hs256(payload, key).decode()) - def _login_by_token(self, jwt_token): + def _login_by_token(self, jwt_token, raw_token_hint=False): user_info = self._download_json( - 'https://api.rplay.live/account/login', 'login', note='performing login', errnote='Failed to login', + 'https://api.rplay.live/account/login', 'login', note='performing login', errnote='login failed', data=f'{{"token":"{jwt_token}","loginType":null,"checkAdmin":null}}'.encode(), headers={'Content-Type': 'application/json', 'Authorization': 'null'}, fatal=False) @@ -86,7 +100,10 @@ class RPlayBaseIE(InfoExtractor): self._login_type = traverse_obj(user_info, 'accountType') self._jwt_token = jwt_token if self._user_id else None if not self._user_id: - self.report_warning('Failed to login, possibly due to wrong password or website change') + if raw_token_hint: + self.report_warning('Login failed, possibly due to wrong or expired JWT token') + else: + self.report_warning('Login failed, possibly due to wrong password or website change') def get_butter_token(self): salt = 'QWI@(!WAS)Dj1AA(!@*DJ#@$@~1)P' @@ -181,8 +198,7 @@ class RPlayVideoIE(RPlayBaseIE): if traverse_obj(video_info, ('viewableTiers', 'free')): msg = 'This video requires a free subscription to access' if not self.user_id: - # credential is in browser localStorage only, no cookies to use - msg += f'. {self._login_hint(method="password")}' + msg += f'. {self._login_hint()}' raise ExtractorError(msg, expected=True) formats = self._extract_m3u8_formats(m3u8_url, video_id, headers={ From 9d4497e13e0fb3cd0dd0f2ef7d6a0facbb8e8812 Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Sun, 6 Oct 2024 10:45:54 -0400 Subject: [PATCH 08/12] fix for api change --- yt_dlp/extractor/rplaylive.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index c43e403d15..935430d730 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -308,12 +308,14 @@ class RPlayLiveIE(RPlayBaseIE): elif stream_state == 'live': if not self.user_id and not live_info.get('allowAnonymous'): self.raise_login_required(method='password') - key2 = self._download_webpage( + key2 = traverse_obj(self._download_json( 'https://api.rplay.live/live/key2', user_id, 'getting live key', - headers=self.jwt_header, query=self.requestor_query) if self.user_id else '' + headers=self.jwt_header, query=self.requestor_query), ('authKey', {str})) if self.user_id else '' + if key2 is None: + raise ExtractorError('Failed to get playlist key') formats = self._extract_m3u8_formats( 'https://api.rplay.live/live/stream/playlist.m3u8', user_id, - query={'creatorOid': user_id, 'key2': key2}) + query={'creatorOid': user_id, 'key2': key2}, headers={'Referer': 'https://rplay.live'}) return { 'id': user_id, From 501c9d526779444d9d8e6a69ec46a760a7cbf48a Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:47:47 -0400 Subject: [PATCH 09/12] update headers --- yt_dlp/extractor/rplaylive.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index 935430d730..c75f8a83cd 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -60,10 +60,19 @@ class RPlayBaseIE(InfoExtractor): @property def jwt_header(self): return { + 'Origin': 'https://rplay.live', 'Referer': 'https://rplay.live/', 'Authorization': self.jwt_token or 'null', } + @property + def butter_header(self): + return { + 'Origin': 'https://rplay.live', + 'Referer': 'https://rplay.live/', + 'Butter': self.get_butter_token(), + } + def _login_hint(self, **kwargs): return (f'Use --username and --password, --netrc-cmd, --netrc ({self._NETRC_MACHINE}) ' 'or --extractor-args "rplaylive:jwt_token=xxx" to provide account credentials') @@ -201,15 +210,14 @@ class RPlayVideoIE(RPlayBaseIE): msg += f'. {self._login_hint()}' raise ExtractorError(msg, expected=True) - formats = self._extract_m3u8_formats(m3u8_url, video_id, headers={ - 'Referer': 'https://rplay.live/', 'Butter': self.get_butter_token()}) + formats = self._extract_m3u8_formats(m3u8_url, video_id, headers=self.butter_header) for fmt in formats: - m3u8_doc = self._download_webpage(fmt['url'], video_id, 'getting m3u8 contents', headers={ - 'Referer': 'https://rplay.live/', 'Butter': self.get_butter_token()}) + m3u8_doc = self._download_webpage(fmt['url'], video_id, 'getting m3u8 contents', headers=self.butter_header) fmt['url'] = encode_data_uri(m3u8_doc.encode(), 'application/x-mpegurl') match = re.search(r'^#EXT-X-KEY.*?URI="([^"]+)"', m3u8_doc, flags=re.M) if match: urlh = self._request_webpage(match[1], video_id, 'getting hls key', headers={ + 'Origin': 'https://rplay.live', 'Referer': 'https://rplay.live/', 'rplay-private-content-requestor': self.user_id or 'not-logged-in', 'age': random.randint(1, 4999), From 70fdfb3116bebadcc0257a9583658120ab1ee7df Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Sun, 29 Dec 2024 03:24:11 -0500 Subject: [PATCH 10/12] hardcode logintype --- yt_dlp/extractor/rplaylive.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index c75f8a83cd..a574fc42c6 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -25,7 +25,6 @@ class RPlayBaseIE(InfoExtractor): _NETRC_MACHINE = 'rplaylive' _TOKEN_CACHE = {} _user_id = None - _login_type = None _jwt_token = None _tested_jwt = False @@ -40,11 +39,6 @@ class RPlayBaseIE(InfoExtractor): self._check_jwt_args() return self._user_id - @property - def login_type(self): - self._check_jwt_args() - return self._login_type - @property def jwt_token(self): self._check_jwt_args() @@ -54,7 +48,7 @@ class RPlayBaseIE(InfoExtractor): def requestor_query(self): return { 'requestorOid': self.user_id, - 'loginType': self.login_type, + 'loginType': 'plax', } if self.user_id else {} @property @@ -106,7 +100,6 @@ class RPlayBaseIE(InfoExtractor): if user_info: self._user_id = traverse_obj(user_info, 'oid') - self._login_type = traverse_obj(user_info, 'accountType') self._jwt_token = jwt_token if self._user_id else None if not self._user_id: if raw_token_hint: From 8ce308d519c77e6a1b433e879d992451cfb679f6 Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Tue, 25 Feb 2025 22:07:31 -0500 Subject: [PATCH 11/12] update api endpoint --- yt_dlp/extractor/rplaylive.py | 39 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index a574fc42c6..4eca948e8d 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -94,7 +94,7 @@ class RPlayBaseIE(InfoExtractor): def _login_by_token(self, jwt_token, raw_token_hint=False): user_info = self._download_json( - 'https://api.rplay.live/account/login', 'login', note='performing login', errnote='login failed', + 'https://api.rplay-cdn.com/account/login', 'login', note='performing login', errnote='login failed', data=f'{{"token":"{jwt_token}","loginType":null,"checkAdmin":null}}'.encode(), headers={'Content-Type': 'application/json', 'Authorization': 'null'}, fatal=False) @@ -153,6 +153,25 @@ class RPlayVideoIE(RPlayBaseIE): 'age_limit': 18, 'live_status': 'was_live', }, + }, { + 'url': 'https://rplay.live/play/664f6dbe8ff72ac8bb0aecfc', + 'info_dict': { + 'id': '664f6dbe8ff72ac8bb0aecfc', + 'ext': 'mp4', + 'title': 'md5:b47c5094854a84e3318f8b0bd70fdee8', + 'description': 'md5:edc7641e1bbb195e788a9695883f6ab9', + 'timestamp': 1716481470, + 'upload_date': '20240523', + 'release_timestamp': 1716485243, + 'release_date': '20240523', + 'duration': 7273.433333, + 'thumbnail': 'https://pb.rplay.live/thumbnail/664f6dbe8ff72ac8bb0aecfc', + 'uploader': 'ミス・ネフェルー', + 'uploader_id': '6640ce9db293d7d82bf76cfd', + 'tags': 'count:3', + 'age_limit': 18, + }, + 'skip': 'subscribe required', }] def _real_extract(self, url): @@ -161,7 +180,7 @@ class RPlayVideoIE(RPlayBaseIE): playlist_id = traverse_obj(parse_qs(url), ('playlist', ..., any)) if playlist_id and self._yes_playlist(playlist_id, video_id): playlist_info = self._download_json( - 'https://api.rplay.live/content/playlist', playlist_id, + 'https://api.rplay-cdn.com/content/playlist', playlist_id, query={'playlistOid': playlist_id, **self.requestor_query}, headers=self.jwt_header, fatal=False) if playlist_info: @@ -171,7 +190,7 @@ class RPlayVideoIE(RPlayBaseIE): else: self.report_warning('Failed to get playlist, downloading video only') - video_info = self._download_json('https://api.rplay.live/content', video_id, query={ + video_info = self._download_json('https://api.rplay-cdn.com/content', video_id, query={ 'contentOid': video_id, 'status': 'published', 'withComments': True, @@ -239,7 +258,7 @@ class RPlayUserIE(InfoExtractor): 'url': 'https://rplay.live/c/furachi', 'info_dict': { 'id': '65e07e60850f4527aab74757', - 'title': '逢瀬ふらち OuseFurachi', + 'title': '桜彗ふらち OuseFurachi', }, 'playlist_mincount': 94, }] @@ -247,10 +266,10 @@ class RPlayUserIE(InfoExtractor): def _real_extract(self, url): user_id, short = self._match_valid_url(url).group('id', 'short') - user_info = self._download_json('https://api.rplay.live/account/getuser', user_id, query={ + user_info = self._download_json('https://api.rplay-cdn.com/account/getuser', user_id, query={ 'customUrl' if short == 'c' else 'userOid': user_id, 'options': '{"includeContentMetadata":true}'}) replays = self._download_json( - 'https://api.rplay.live/live/replays', user_id, query={'creatorOid': user_info.get('_id')}) + 'https://api.rplay-cdn.com/live/replays', user_id, query={'creatorOid': user_info.get('_id')}) def _entries(): def _entry_ids(): @@ -293,13 +312,13 @@ class RPlayLiveIE(RPlayBaseIE): def _real_extract(self, url): user_id, short = self._match_valid_url(url).group('id', 'short') - user_info = self._download_json('https://api.rplay.live/account/getuser', user_id, query={ + user_info = self._download_json('https://api.rplay-cdn.com/account/getuser', user_id, query={ 'customUrl' if short == 'c' else 'userOid': user_id}) if user_info.get('isLive') is False: raise UserNotLive user_id = user_info['_id'] - live_info = self._download_json('https://api.rplay.live/live/play', user_id, query={'creatorOid': user_id}) + live_info = self._download_json('https://api.rplay-cdn.com/live/play', user_id, query={'creatorOid': user_id}) stream_state = live_info['streamState'] if stream_state == 'youtube': @@ -310,12 +329,12 @@ class RPlayLiveIE(RPlayBaseIE): if not self.user_id and not live_info.get('allowAnonymous'): self.raise_login_required(method='password') key2 = traverse_obj(self._download_json( - 'https://api.rplay.live/live/key2', user_id, 'getting live key', + 'https://api.rplay-cdn.com/live/key2', user_id, 'getting live key', headers=self.jwt_header, query=self.requestor_query), ('authKey', {str})) if self.user_id else '' if key2 is None: raise ExtractorError('Failed to get playlist key') formats = self._extract_m3u8_formats( - 'https://api.rplay.live/live/stream/playlist.m3u8', user_id, + 'https://api.rplay-cdn.com/live/stream/playlist.m3u8', user_id, query={'creatorOid': user_id, 'key2': key2}, headers={'Referer': 'https://rplay.live'}) return { From 6beae46c0f2b82c28a9ca3c6c18bdeb19ab75db2 Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:23:31 -0500 Subject: [PATCH 12/12] fixes for livestream --- yt_dlp/extractor/rplaylive.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/yt_dlp/extractor/rplaylive.py b/yt_dlp/extractor/rplaylive.py index 4eca948e8d..58f86e94be 100644 --- a/yt_dlp/extractor/rplaylive.py +++ b/yt_dlp/extractor/rplaylive.py @@ -67,7 +67,7 @@ class RPlayBaseIE(InfoExtractor): 'Butter': self.get_butter_token(), } - def _login_hint(self, **kwargs): + def _login_hint(self, *args, **kwargs): return (f'Use --username and --password, --netrc-cmd, --netrc ({self._NETRC_MACHINE}) ' 'or --extractor-args "rplaylive:jwt_token=xxx" to provide account credentials') @@ -291,17 +291,17 @@ class RPlayLiveIE(RPlayBaseIE): _TESTS = [{ 'url': 'https://rplay.live/c/chachamaru/live', 'info_dict': { - 'id': '667e4cd99aa7f739a2c91852', + 'id': '667e511a6f7cead36a00e7b1', 'ext': 'mp4', - 'title': r're:【ASMR】ん～っやば//スキスキ耐久.*', - 'description': 'md5:7f88ac0a7a3d5d0b926a0baecd1d40e1', - 'timestamp': 1721739947, - 'upload_date': '20240723', + 'title': r're:【ASMR】やばっ*', + 'description': 'md5:de9d0f8e8b80ee93678bebad5b43254e', + 'timestamp': 1740578497, + 'upload_date': '20250226', 'live_status': 'is_live', 'thumbnail': 'https://pb.rplay.live/liveChannelThumbnails/667e4cd99aa7f739a2c91852', 'uploader': '愛犬茶々丸', 'uploader_id': '667e4cd99aa7f739a2c91852', - 'tags': 'count:9', + 'tags': list, }, 'skip': 'live', }, { @@ -314,20 +314,25 @@ class RPlayLiveIE(RPlayBaseIE): user_info = self._download_json('https://api.rplay-cdn.com/account/getuser', user_id, query={ 'customUrl' if short == 'c' else 'userOid': user_id}) - if user_info.get('isLive') is False: - raise UserNotLive user_id = user_info['_id'] - live_info = self._download_json('https://api.rplay-cdn.com/live/play', user_id, query={'creatorOid': user_id}) + live_info = self._download_json('https://api.rplay-cdn.com/live/play', user_id, query={ + 'creatorOid': user_id, **self.requestor_query}, headers=self.jwt_header) stream_state = live_info['streamState'] - if stream_state == 'youtube': + if stream_state == 'offline': + raise UserNotLive + elif stream_state == 'youtube': return self.url_result(f'https://www.youtube.com/watch?v={live_info["liveStreamId"]}') elif stream_state == 'twitch': return self.url_result(f'https://www.twitch.tv/{live_info["twitchLogin"]}') elif stream_state == 'live': if not self.user_id and not live_info.get('allowAnonymous'): self.raise_login_required(method='password') + if not live_info.get('accessible'): + if traverse_obj(live_info, ('tierHashes', lambda _, v: v == 'free', any)): + raise ExtractorError('The livestream requires a free subscription to access', expected=True) + raise ExtractorError('You do not have access to the livestream', expected=True) key2 = traverse_obj(self._download_json( 'https://api.rplay-cdn.com/live/key2', user_id, 'getting live key', headers=self.jwt_header, query=self.requestor_query), ('authKey', {str})) if self.user_id else '' @@ -338,7 +343,7 @@ class RPlayLiveIE(RPlayBaseIE): query={'creatorOid': user_id, 'key2': key2}, headers={'Referer': 'https://rplay.live'}) return { - 'id': user_id, + 'id': live_info.get('oid') or user_id, 'formats': formats, 'is_live': True, 'http_headers': {'Referer': 'https://rplay.live'}, @@ -353,7 +358,5 @@ class RPlayLiveIE(RPlayBaseIE): 'age_limit': ('isAdultContent', {lambda x: 18 if x else None}), }), } - elif stream_state == 'offline': - raise UserNotLive else: raise ExtractorError(f'Unknow streamState: {stream_state}')