diff --git a/README.md b/README.md
index 6e27b0a348..1ee4223856 100644
--- a/README.md
+++ b/README.md
@@ -1866,6 +1866,9 @@ The following extractors use this feature:
#### sonylivseries
* `sort_order`: Episode sort order for series extraction - one of `asc` (ascending, oldest first) or `desc` (descending, newest first). Default is `asc`
+#### tver
+* `backend`: Backend API to use for extraction - one of `streaks` (default) or `brightcove` (deprecated)
+
**Note**: These options may be changed/removed in the future without concern for backward compatibility
diff --git a/pyproject.toml b/pyproject.toml
index b5d77d35b7..5e987a6fd3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -55,8 +55,7 @@ default = [
"websockets>=13.0",
]
curl-cffi = [
- "curl-cffi==0.5.10; os_name=='nt' and implementation_name=='cpython'",
- "curl-cffi>=0.5.10,!=0.6.*,<0.7.2; os_name!='nt' and implementation_name=='cpython'",
+ "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.11; implementation_name=='cpython'",
]
secretstorage = [
"cffi",
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 54f35ef552..c6ff6209a8 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -638,6 +638,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
'img_bipbop_adv_example_fmp4',
'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
[{
+ # 60kbps (bitrate not provided in m3u8); sorted as worst because it's grouped with lowest bitrate video track
'format_id': 'aud1-English',
'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a1/prog_index.m3u8',
'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
@@ -645,22 +646,27 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
'ext': 'mp4',
'protocol': 'm3u8_native',
'audio_ext': 'mp4',
+ 'source_preference': 0,
}, {
- 'format_id': 'aud2-English',
- 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a2/prog_index.m3u8',
+ # 192kbps (bitrate not provided in m3u8)
+ 'format_id': 'aud3-English',
+ 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a3/prog_index.m3u8',
'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
'language': 'en',
'ext': 'mp4',
'protocol': 'm3u8_native',
'audio_ext': 'mp4',
+ 'source_preference': 1,
}, {
- 'format_id': 'aud3-English',
- 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a3/prog_index.m3u8',
+ # 384kbps (bitrate not provided in m3u8); sorted as best because it's grouped with the highest bitrate video track
+ 'format_id': 'aud2-English',
+ 'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a2/prog_index.m3u8',
'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
'language': 'en',
'ext': 'mp4',
'protocol': 'm3u8_native',
'audio_ext': 'mp4',
+ 'source_preference': 2,
}, {
'format_id': '530',
'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v2/prog_index.m3u8',
diff --git a/test/test_networking.py b/test/test_networking.py
index 63914bc4ba..3ab60fe836 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -614,7 +614,6 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
rh, Request(f'http://127.0.0.1:{self.http_port}/source_address')).read().decode()
assert source_address == data
- # Not supported by CurlCFFI
@pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi')
def test_gzip_trailing_garbage(self, handler):
with handler() as rh:
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index f3b0056179..efd69b33d9 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -23,7 +23,6 @@ from yt_dlp.extractor import (
TedTalkIE,
ThePlatformFeedIE,
ThePlatformIE,
- VikiIE,
VimeoIE,
WallaIE,
YoutubeIE,
@@ -331,20 +330,6 @@ class TestRaiPlaySubtitles(BaseTestSubtitles):
self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd')
-@is_download_test
-@unittest.skip('IE broken - DRM only')
-class TestVikiSubtitles(BaseTestSubtitles):
- url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
- IE = VikiIE
-
- def test_allsubtitles(self):
- self.DL.params['writesubtitles'] = True
- self.DL.params['allsubtitles'] = True
- subtitles = self.getSubtitles()
- self.assertEqual(set(subtitles.keys()), {'en'})
- self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a')
-
-
@is_download_test
class TestThePlatformSubtitles(BaseTestSubtitles):
# from http://www.3playmedia.com/services-features/tools/integrations/theplatform/
diff --git a/test/test_utils.py b/test/test_utils.py
index 42dc7f937e..e60ceed8fd 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1260,6 +1260,7 @@ class TestUtil(unittest.TestCase):
def test_js_to_json_malformed(self):
self.assertEqual(js_to_json('42a1'), '42"a1"')
self.assertEqual(js_to_json('42a-1'), '42"a"-1')
+ self.assertEqual(js_to_json('{a: `${e("")}`}'), '{"a": "\\"e\\"(\\"\\")"}')
def test_js_to_json_template_literal(self):
self.assertEqual(js_to_json('`Hello ${name}`', {'name': '"world"'}), '"Hello world"')
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 45dc9113bd..453caacd65 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -83,6 +83,11 @@ _SIG_TESTS = [
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
'AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1',
),
+ (
+ 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
+ '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+ '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+ ),
]
_NSIG_TESTS = [
@@ -234,6 +239,10 @@ _NSIG_TESTS = [
'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js',
'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA',
),
+ (
+ 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
+ 'eWYu5d5YeY_4LyEDc', 'XJQqf-N7Xra3gg',
+ ),
]
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 8790b326b7..63e6e11b26 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -654,19 +654,21 @@ class YoutubeDL:
if not all_plugins_loaded.value:
load_all_plugins()
- try:
- windows_enable_vt_mode()
- except Exception as e:
- self.write_debug(f'Failed to enable VT mode: {e}')
-
stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout
self._out_files = Namespace(
out=stdout,
error=sys.stderr,
screen=sys.stderr if self.params.get('quiet') else stdout,
- console=next(filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None),
)
+ try:
+ windows_enable_vt_mode()
+ except Exception as e:
+ self.write_debug(f'Failed to enable VT mode: {e}')
+
+ # hehe "immutable" namespace
+ self._out_files.console = next(filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None)
+
if self.params.get('no_color'):
if self.params.get('color') is not None:
self.params.setdefault('_warnings', []).append(
@@ -4150,7 +4152,7 @@ class YoutubeDL:
(target, rh.RH_NAME)
for rh in self._request_director.handlers.values()
if isinstance(rh, ImpersonateRequestHandler)
- for target in rh.supported_targets
+ for target in reversed(rh.supported_targets)
]
def _impersonate_target_available(self, target):
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 7d8f100474..714d9ad5c2 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -1021,8 +1021,9 @@ def _real_main(argv=None):
# List of simplified targets we know are supported,
# to help users know what dependencies may be required.
(ImpersonateTarget('chrome'), 'curl_cffi'),
- (ImpersonateTarget('edge'), 'curl_cffi'),
(ImpersonateTarget('safari'), 'curl_cffi'),
+ (ImpersonateTarget('firefox'), 'curl_cffi>=0.10'),
+ (ImpersonateTarget('edge'), 'curl_cffi'),
]
available_targets = ydl._get_available_impersonate_targets()
@@ -1038,12 +1039,12 @@ def _real_main(argv=None):
for known_target, known_handler in known_targets:
if not any(
- known_target in target and handler == known_handler
+ known_target in target and known_handler.startswith(handler)
for target, handler in available_targets
):
- rows.append([
+ rows.insert(0, [
ydl._format_out(text, ydl.Styles.SUPPRESS)
- for text in make_row(known_target, f'{known_handler} (not available)')
+ for text in make_row(known_target, f'{known_handler} (unavailable)')
])
ydl.to_screen('[info] Available impersonate targets')
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 38e9673cd4..a44601b14d 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -496,10 +496,6 @@ from .daum import (
from .daystar import DaystarClipIE
from .dbtv import DBTVIE
from .dctp import DctpTvIE
-from .deezer import (
- DeezerAlbumIE,
- DeezerPlaylistIE,
-)
from .democracynow import DemocracynowIE
from .detik import DetikEmbedIE
from .deuxm import (
@@ -843,6 +839,7 @@ from .icareus import IcareusIE
from .ichinanalive import (
IchinanaLiveClipIE,
IchinanaLiveIE,
+ IchinanaLiveVODIE,
)
from .idolplus import IdolPlusIE
from .ign import (
@@ -1989,6 +1986,7 @@ from .storyfire import (
StoryFireSeriesIE,
StoryFireUserIE,
)
+from .streaks import StreaksIE
from .streamable import StreamableIE
from .streamcz import StreamCZIE
from .streetvoice import StreetVoiceIE
@@ -2352,10 +2350,6 @@ from .viewlift import (
ViewLiftIE,
)
from .viidea import ViideaIE
-from .viki import (
- VikiChannelIE,
- VikiIE,
-)
from .vimeo import (
VHXEmbedIE,
VimeoAlbumIE,
@@ -2400,6 +2394,12 @@ from .voxmedia import (
VoxMediaIE,
VoxMediaVolumeIE,
)
+from .vrsquare import (
+ VrSquareChannelIE,
+ VrSquareIE,
+ VrSquareSearchIE,
+ VrSquareSectionIE,
+)
from .vrt import (
VRTIE,
DagelijkseKostIE,
diff --git a/yt_dlp/extractor/chzzk.py b/yt_dlp/extractor/chzzk.py
index aec77ac454..a5daf5ca7c 100644
--- a/yt_dlp/extractor/chzzk.py
+++ b/yt_dlp/extractor/chzzk.py
@@ -21,7 +21,7 @@ class CHZZKLiveIE(InfoExtractor):
'channel': '진짜도현',
'channel_id': 'c68b8ef525fb3d2fa146344d84991753',
'channel_is_verified': False,
- 'thumbnail': r're:^https?://.*\.jpg$',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
'timestamp': 1705510344,
'upload_date': '20240117',
'live_status': 'is_live',
@@ -98,7 +98,7 @@ class CHZZKVideoIE(InfoExtractor):
'channel': '침착맨',
'channel_id': 'bb382c2c0cc9fa7c86ab3b037fb5799c',
'channel_is_verified': False,
- 'thumbnail': r're:^https?://.*\.jpg$',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
'duration': 15577,
'timestamp': 1702970505.417,
'upload_date': '20231219',
@@ -115,7 +115,7 @@ class CHZZKVideoIE(InfoExtractor):
'channel': '라디유radiyu',
'channel_id': '68f895c59a1043bc5019b5e08c83a5c5',
'channel_is_verified': False,
- 'thumbnail': r're:^https?://.*\.jpg$',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
'duration': 95,
'timestamp': 1703102631.722,
'upload_date': '20231220',
@@ -131,12 +131,30 @@ class CHZZKVideoIE(InfoExtractor):
'channel': '강지',
'channel_id': 'b5ed5db484d04faf4d150aedd362f34b',
'channel_is_verified': True,
- 'thumbnail': r're:^https?://.*\.jpg$',
+ 'thumbnail': r're:https?://.+/.+\.jpg',
'duration': 4433,
'timestamp': 1703307460.214,
'upload_date': '20231223',
'view_count': int,
},
+ }, {
+ # video_status == 'NONE' but is downloadable
+ 'url': 'https://chzzk.naver.com/video/6325166',
+ 'info_dict': {
+ 'id': '6325166',
+ 'ext': 'mp4',
+ 'title': '와이프 숙제빼주기',
+ 'channel': '이 다',
+ 'channel_id': '0076a519f147ee9fd0959bf02f9571ca',
+ 'channel_is_verified': False,
+ 'view_count': int,
+ 'duration': 28167,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'timestamp': 1742139216.86,
+ 'upload_date': '20250316',
+ 'live_status': 'was_live',
+ },
+ 'params': {'skip_download': 'm3u8'},
}]
def _real_extract(self, url):
@@ -147,11 +165,7 @@ class CHZZKVideoIE(InfoExtractor):
live_status = 'was_live' if video_meta.get('liveOpenDate') else 'not_live'
video_status = video_meta.get('vodStatus')
- if video_status == 'UPLOAD':
- playback = self._parse_json(video_meta['liveRewindPlaybackJson'], video_id)
- formats, subtitles = self._extract_m3u8_formats_and_subtitles(
- playback['media'][0]['path'], video_id, 'mp4', m3u8_id='hls')
- elif video_status == 'ABR_HLS':
+ if video_status == 'ABR_HLS':
formats, subtitles = self._extract_mpd_formats_and_subtitles(
f'https://apis.naver.com/neonplayer/vodplay/v1/playback/{video_meta["videoId"]}',
video_id, query={
@@ -161,10 +175,17 @@ class CHZZKVideoIE(InfoExtractor):
'cpl': 'en_US',
})
else:
- self.raise_no_formats(
- f'Unknown video status detected: "{video_status}"', expected=True, video_id=video_id)
- formats, subtitles = [], {}
- live_status = 'post_live' if live_status == 'was_live' else None
+ fatal = video_status == 'UPLOAD'
+ playback = self._parse_json(video_meta['liveRewindPlaybackJson'], video_id, fatal=fatal)
+ formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+ traverse_obj(playback, ('media', 0, 'path')), video_id, 'mp4', m3u8_id='hls', fatal=fatal)
+ if formats and video_status != 'UPLOAD':
+ self.write_debug(f'Video found with status: "{video_status}"')
+ elif not formats:
+ self.raise_no_formats(
+ f'Unknown video status detected: "{video_status}"', expected=True, video_id=video_id)
+ formats, subtitles = [], {}
+ live_status = 'post_live' if live_status == 'was_live' else None
return {
'id': video_id,
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 0119111816..4c1bc4cf47 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -78,6 +78,7 @@ from ..utils import (
parse_iso8601,
parse_m3u8_attributes,
parse_resolution,
+ qualities,
sanitize_url,
smuggle_url,
str_or_none,
@@ -2177,6 +2178,8 @@ class InfoExtractor:
media_url = media.get('URI')
if media_url:
manifest_url = format_url(media_url)
+ is_audio = media_type == 'AUDIO'
+ is_alternate = media.get('DEFAULT') == 'NO' or media.get('AUTOSELECT') == 'NO'
formats.extend({
'format_id': join_nonempty(m3u8_id, group_id, name, idx),
'format_note': name,
@@ -2189,7 +2192,11 @@ class InfoExtractor:
'preference': preference,
'quality': quality,
'has_drm': has_drm,
- 'vcodec': 'none' if media_type == 'AUDIO' else None,
+ 'vcodec': 'none' if is_audio else None,
+ # Alternate audio formats (e.g. audio description) should be deprioritized
+ 'source_preference': -2 if is_audio and is_alternate else None,
+ # Save this to assign source_preference based on associated video stream
+ '_audio_group_id': group_id if is_audio and not is_alternate else None,
} for idx in _extract_m3u8_playlist_indices(manifest_url))
def build_stream_name():
@@ -2284,6 +2291,8 @@ class InfoExtractor:
# ignore references to rendition groups and treat them
# as complete formats.
if audio_group_id and codecs and f.get('vcodec') != 'none':
+ # Save this to determine quality of audio formats that only have a GROUP-ID
+ f['_audio_group_id'] = audio_group_id
audio_group = groups.get(audio_group_id)
if audio_group and audio_group[0].get('URI'):
# TODO: update acodec for audio only formats with
@@ -2306,6 +2315,28 @@ class InfoExtractor:
formats.append(http_f)
last_stream_inf = {}
+
+ # Some audio-only formats only have a GROUP-ID without any other quality/bitrate/codec info
+ # Each audio GROUP-ID corresponds with one or more video formats' AUDIO attribute
+ # For sorting purposes, set source_preference based on the quality of the video formats they are grouped with
+ # See https://github.com/yt-dlp/yt-dlp/issues/11178
+ audio_groups_by_quality = orderedSet(f['_audio_group_id'] for f in sorted(
+ traverse_obj(formats, lambda _, v: v.get('vcodec') != 'none' and v['_audio_group_id']),
+ key=lambda x: (x.get('tbr') or 0, x.get('width') or 0)))
+ audio_quality_map = {
+ audio_groups_by_quality[0]: 'low',
+ audio_groups_by_quality[-1]: 'high',
+ } if len(audio_groups_by_quality) > 1 else None
+ audio_preference = qualities(audio_groups_by_quality)
+ for fmt in formats:
+ audio_group_id = fmt.pop('_audio_group_id', None)
+ if not audio_quality_map or not audio_group_id or fmt.get('vcodec') != 'none':
+ continue
+ # Use source_preference since quality and preference are set by params
+ fmt['source_preference'] = audio_preference(audio_group_id)
+ fmt['format_note'] = join_nonempty(
+ fmt.get('format_note'), audio_quality_map.get(audio_group_id), delim=', ')
+
return formats, subtitles
def _extract_m3u8_vod_duration(
diff --git a/yt_dlp/extractor/deezer.py b/yt_dlp/extractor/deezer.py
deleted file mode 100644
index 2ca8be5ca0..0000000000
--- a/yt_dlp/extractor/deezer.py
+++ /dev/null
@@ -1,142 +0,0 @@
-import json
-
-from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
- int_or_none,
- orderedSet,
-)
-
-
-class DeezerBaseInfoExtractor(InfoExtractor):
- def get_data(self, url):
- if not self.get_param('test'):
- self.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!')
-
- mobj = self._match_valid_url(url)
- data_id = mobj.group('id')
-
- webpage = self._download_webpage(url, data_id)
- geoblocking_msg = self._html_search_regex(
- r'(.*?)
', webpage, 'geoblocking message',
- default=None)
- if geoblocking_msg is not None:
- raise ExtractorError(
- f'Deezer said: {geoblocking_msg}', expected=True)
-
- data_json = self._search_regex(
- (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*',
- r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'),
- webpage, 'data JSON')
- data = json.loads(data_json)
- return data_id, webpage, data
-
-
-class DeezerPlaylistIE(DeezerBaseInfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?playlist/(?P[0-9]+)'
- _TEST = {
- 'url': 'http://www.deezer.com/playlist/176747451',
- 'info_dict': {
- 'id': '176747451',
- 'title': 'Best!',
- 'uploader': 'anonymous',
- 'thumbnail': r're:^https?://(e-)?cdns-images\.dzcdn\.net/images/cover/.*\.jpg$',
- },
- 'playlist_count': 29,
- }
-
- def _real_extract(self, url):
- playlist_id, webpage, data = self.get_data(url)
-
- playlist_title = data.get('DATA', {}).get('TITLE')
- playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME')
- playlist_thumbnail = self._search_regex(
- r'![]()
[0-9]+)'
- _TEST = {
- 'url': 'https://www.deezer.com/fr/album/67505622',
- 'info_dict': {
- 'id': '67505622',
- 'title': 'Last Week',
- 'uploader': 'Home Brew',
- 'thumbnail': r're:^https?://(e-)?cdns-images\.dzcdn\.net/images/cover/.*\.jpg$',
- },
- 'playlist_count': 7,
- }
-
- def _real_extract(self, url):
- album_id, webpage, data = self.get_data(url)
-
- album_title = data.get('DATA', {}).get('ALB_TITLE')
- album_uploader = data.get('DATA', {}).get('ART_NAME')
- album_thumbnail = self._search_regex(
- r'![]()
[^/?#]+)'
+ _TESTS = [{
+ 'url': 'https://17.live/ja/vod/27323042/2cf84520-e65e-4b22-891e-1d3a00b0f068',
+ 'md5': '3299b930d7457b069639486998a89580',
+ 'info_dict': {
+ 'id': '2cf84520-e65e-4b22-891e-1d3a00b0f068',
+ 'ext': 'mp4',
+ 'title': 'md5:b5f8cbf497d54cc6a60eb3b480182f01',
+ 'uploader': 'md5:29fb12122ab94b5a8495586e7c3085a5',
+ 'uploader_id': '27323042',
+ 'channel': '🌟オールナイトニッポン アーカイブ🌟',
+ 'channel_id': '2b4f85f1-d61e-429d-a901-68d32bdd8645',
+ 'like_count': int,
+ 'view_count': int,
+ 'thumbnail': r're:https?://.+/.+\.(?:jpe?g|png)',
+ 'duration': 549,
+ 'description': 'md5:116f326579700f00eaaf5581aae1192e',
+ 'timestamp': 1741058645,
+ 'upload_date': '20250304',
+ },
+ }, {
+ 'url': 'https://17.live/ja/vod/27323042/0de11bac-9bea-40b8-9eab-0239a7d88079',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ json_data = self._download_json(f'https://wap-api.17app.co/api/v1/vods/{video_id}', video_id)
+
+ return traverse_obj(json_data, {
+ 'id': ('vodID', {str}),
+ 'title': ('title', {str}),
+ 'formats': ('vodURL', {lambda x: self._extract_m3u8_formats(x, video_id)}),
+ 'uploader': ('userInfo', 'displayName', {str}),
+ 'uploader_id': ('userInfo', 'roomID', {int}, {str_or_none}),
+ 'channel': ('userInfo', 'name', {str}),
+ 'channel_id': ('userInfo', 'userID', {str}),
+ 'like_count': ('likeCount', {int_or_none}),
+ 'view_count': ('viewCount', {int_or_none}),
+ 'thumbnail': ('imageURL', {url_or_none}),
+ 'duration': ('duration', {int_or_none}),
+ 'description': ('description', {str}),
+ 'timestamp': ('createdAt', {int_or_none}),
+ })
diff --git a/yt_dlp/extractor/ninenow.py b/yt_dlp/extractor/ninenow.py
index f17531e622..7b0cb77a74 100644
--- a/yt_dlp/extractor/ninenow.py
+++ b/yt_dlp/extractor/ninenow.py
@@ -1,34 +1,46 @@
+import json
+import re
+
+from .brightcove import BrightcoveNewIE
from .common import InfoExtractor
from ..utils import (
- ExtractorError,
float_or_none,
int_or_none,
- smuggle_url,
+ parse_iso8601,
+ parse_resolution,
str_or_none,
- try_get,
- unified_strdate,
- unified_timestamp,
+ url_or_none,
)
+from ..utils.traversal import require, traverse_obj, value
class NineNowIE(InfoExtractor):
IE_NAME = '9now.com.au'
- _VALID_URL = r'https?://(?:www\.)?9now\.com\.au/(?:[^/]+/){2}(?P[^/?#]+)'
- _GEO_COUNTRIES = ['AU']
+ _VALID_URL = r'https?://(?:www\.)?9now\.com\.au/(?:[^/?#]+/){2}(?P(?Pclip|episode)-[^/?#]+)'
+ _GEO_BYPASS = False
_TESTS = [{
# clip
- 'url': 'https://www.9now.com.au/afl-footy-show/2016/clip-ciql02091000g0hp5oktrnytc',
- 'md5': '17cf47d63ec9323e562c9957a968b565',
+ 'url': 'https://www.9now.com.au/today/season-2025/clip-cm8hw9h5z00080hquqa5hszq7',
'info_dict': {
- 'id': '16801',
+ 'id': '6370295582112',
'ext': 'mp4',
- 'title': 'St. Kilda\'s Joey Montagna on the potential for a player\'s strike',
- 'description': 'Is a boycott of the NAB Cup "on the table"?',
+ 'title': 'Would Karl Stefanovic be able to land a plane?',
+ 'description': 'The Today host\'s skills are put to the test with the latest simulation tech.',
'uploader_id': '4460760524001',
- 'upload_date': '20160713',
- 'timestamp': 1468421266,
+ 'duration': 197.376,
+ 'tags': ['flights', 'technology', 'Karl Stefanovic'],
+ 'season': 'Season 2025',
+ 'season_number': 2025,
+ 'series': 'TODAY',
+ 'timestamp': 1742507988,
+ 'upload_date': '20250320',
+ 'release_timestamp': 1742507983,
+ 'release_date': '20250320',
+ 'thumbnail': r're:https?://.+/1920x0/.+\.jpg',
+ },
+ 'params': {
+ 'skip_download': 'HLS/DASH fragments and mp4 URLs are geo-restricted; only available in AU',
},
- 'skip': 'Only available in Australia',
}, {
# episode
'url': 'https://www.9now.com.au/afl-footy-show/2016/episode-19',
@@ -41,7 +53,7 @@ class NineNowIE(InfoExtractor):
# episode of series
'url': 'https://www.9now.com.au/lego-masters/season-3/episode-3',
'info_dict': {
- 'id': '6249614030001',
+ 'id': '6308830406112',
'title': 'Episode 3',
'ext': 'mp4',
'season_number': 3,
@@ -50,72 +62,87 @@ class NineNowIE(InfoExtractor):
'uploader_id': '4460760524001',
'timestamp': 1619002200,
'upload_date': '20210421',
+ 'duration': 3574.085,
+ 'thumbnail': r're:https?://.+/1920x0/.+\.jpg',
+ 'tags': ['episode'],
+ 'series': 'Lego Masters',
+ 'season': 'Season 3',
+ 'episode': 'Episode 3',
+ 'release_timestamp': 1619002200,
+ 'release_date': '20210421',
+ },
+ 'params': {
+ 'skip_download': 'HLS/DASH fragments and mp4 URLs are geo-restricted; only available in AU',
+ },
+ }, {
+ 'url': 'https://www.9now.com.au/married-at-first-sight/season-12/episode-1',
+ 'info_dict': {
+ 'id': '6367798770112',
+ 'ext': 'mp4',
+ 'title': 'Episode 1',
+ 'description': r're:The cultural sensation of Married At First Sight returns with our first weddings! .{90}$',
+ 'uploader_id': '4460760524001',
+ 'duration': 5415.079,
+ 'thumbnail': r're:https?://.+/1920x0/.+\.png',
+ 'tags': ['episode'],
+ 'season': 'Season 12',
+ 'season_number': 12,
+ 'episode': 'Episode 1',
+ 'episode_number': 1,
+ 'series': 'Married at First Sight',
+ 'timestamp': 1737973800,
+ 'upload_date': '20250127',
+ 'release_timestamp': 1737973800,
+ 'release_date': '20250127',
},
- 'expected_warnings': ['Ignoring subtitle tracks'],
'params': {
- 'skip_download': True,
+ 'skip_download': 'HLS/DASH fragments and mp4 URLs are geo-restricted; only available in AU',
},
}]
- BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4460760524001/default_default/index.html?videoId=%s'
+ BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4460760524001/default_default/index.html?videoId={}'
+
+ # XXX: For parsing next.js v15+ data; see also yt_dlp.extractor.francetv and yt_dlp.extractor.goplay
+ def _find_json(self, s):
+ return self._search_json(
+ r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None)
def _real_extract(self, url):
- display_id = self._match_id(url)
+ display_id, video_type = self._match_valid_url(url).group('id', 'type')
webpage = self._download_webpage(url, display_id)
- page_data = self._parse_json(self._search_regex(
- r'window\.__data\s*=\s*({.*?});', webpage,
- 'page data', default='{}'), display_id, fatal=False)
- if not page_data:
- page_data = self._parse_json(self._parse_json(self._search_regex(
- r'window\.__data\s*=\s*JSON\.parse\s*\(\s*(".+?")\s*\)\s*;',
- webpage, 'page data'), display_id), display_id)
- for kind in ('episode', 'clip'):
- current_key = page_data.get(kind, {}).get(
- f'current{kind.capitalize()}Key')
- if not current_key:
- continue
- cache = page_data.get(kind, {}).get(f'{kind}Cache', {})
- if not cache:
- continue
- common_data = {
- 'episode': (cache.get(current_key) or next(iter(cache.values())))[kind],
- 'season': (cache.get(current_key) or next(iter(cache.values()))).get('season', None),
- }
- break
- else:
- raise ExtractorError('Unable to find video data')
+ common_data = traverse_obj(
+ re.findall(r']*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*', webpage),
+ (..., {json.loads}, ..., {self._find_json},
+ lambda _, v: v['payload'][video_type]['slug'] == display_id,
+ 'payload', any, {require('video data')}))
- if not self.get_param('allow_unplayable_formats') and try_get(common_data, lambda x: x['episode']['video']['drm'], bool):
+ if traverse_obj(common_data, (video_type, 'video', 'drm', {bool})):
self.report_drm(display_id)
- brightcove_id = try_get(
- common_data, lambda x: x['episode']['video']['brightcoveId'], str) or 'ref:{}'.format(common_data['episode']['video']['referenceId'])
- video_id = str_or_none(try_get(common_data, lambda x: x['episode']['video']['id'])) or brightcove_id
-
- title = try_get(common_data, lambda x: x['episode']['name'], str)
- season_number = try_get(common_data, lambda x: x['season']['seasonNumber'], int)
- episode_number = try_get(common_data, lambda x: x['episode']['episodeNumber'], int)
- timestamp = unified_timestamp(try_get(common_data, lambda x: x['episode']['airDate'], str))
- release_date = unified_strdate(try_get(common_data, lambda x: x['episode']['availability'], str))
- thumbnails_data = try_get(common_data, lambda x: x['episode']['image']['sizes'], dict) or {}
- thumbnails = [{
- 'id': thumbnail_id,
- 'url': thumbnail_url,
- 'width': int_or_none(thumbnail_id[1:]),
- } for thumbnail_id, thumbnail_url in thumbnails_data.items()]
+ brightcove_id = traverse_obj(common_data, (
+ video_type, 'video', (
+ ('brightcoveId', {str}),
+ ('referenceId', {str}, {lambda x: f'ref:{x}' if x else None}),
+ ), any, {require('brightcove ID')}))
return {
'_type': 'url_transparent',
- 'url': smuggle_url(
- self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
- {'geo_countries': self._GEO_COUNTRIES}),
- 'id': video_id,
- 'title': title,
- 'description': try_get(common_data, lambda x: x['episode']['description'], str),
- 'duration': float_or_none(try_get(common_data, lambda x: x['episode']['video']['duration'], float), 1000),
- 'thumbnails': thumbnails,
- 'ie_key': 'BrightcoveNew',
- 'season_number': season_number,
- 'episode_number': episode_number,
- 'timestamp': timestamp,
- 'release_date': release_date,
+ 'ie_key': BrightcoveNewIE.ie_key(),
+ 'url': self.BRIGHTCOVE_URL_TEMPLATE.format(brightcove_id),
+ **traverse_obj(common_data, {
+ 'id': (video_type, 'video', 'id', {int}, ({str_or_none}, {value(brightcove_id)}), any),
+ 'title': (video_type, 'name', {str}),
+ 'description': (video_type, 'description', {str}),
+ 'duration': (video_type, 'video', 'duration', {float_or_none(scale=1000)}),
+ 'tags': (video_type, 'tags', ..., 'name', {str}, all, filter),
+ 'series': ('tvSeries', 'name', {str}),
+ 'season_number': ('season', 'seasonNumber', {int_or_none}),
+ 'episode_number': ('episode', 'episodeNumber', {int_or_none}),
+ 'timestamp': ('episode', 'airDate', {parse_iso8601}),
+ 'release_timestamp': (video_type, 'availability', {parse_iso8601}),
+ 'thumbnails': (video_type, 'image', 'sizes', {dict.items}, lambda _, v: url_or_none(v[1]), {
+ 'id': 0,
+ 'url': 1,
+ 'width': (1, {parse_resolution}, 'width'),
+ }),
+ }),
}
diff --git a/yt_dlp/extractor/streaks.py b/yt_dlp/extractor/streaks.py
new file mode 100644
index 0000000000..1b3718473d
--- /dev/null
+++ b/yt_dlp/extractor/streaks.py
@@ -0,0 +1,236 @@
+import json
+import urllib.parse
+
+from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
+from ..utils import (
+ ExtractorError,
+ filter_dict,
+ float_or_none,
+ join_nonempty,
+ mimetype2ext,
+ parse_iso8601,
+ unsmuggle_url,
+ update_url_query,
+ url_or_none,
+)
+from ..utils.traversal import traverse_obj
+
+
+class StreaksBaseIE(InfoExtractor):
+ _API_URL_TEMPLATE = 'https://{}.api.streaks.jp/v1/projects/{}/medias/{}{}'
+ _GEO_BYPASS = False
+ _GEO_COUNTRIES = ['JP']
+
+ def _extract_from_streaks_api(self, project_id, media_id, headers=None, query=None, ssai=False):
+ try:
+ response = self._download_json(
+ self._API_URL_TEMPLATE.format('playback', project_id, media_id, ''),
+ media_id, 'Downloading STREAKS playback API JSON', headers={
+ 'Accept': 'application/json',
+ 'Origin': 'https://players.streaks.jp',
+ **self.geo_verification_headers(),
+ **(headers or {}),
+ })
+ except ExtractorError as e:
+ if isinstance(e.cause, HTTPError) and e.cause.status in {403, 404}:
+ error = self._parse_json(e.cause.response.read().decode(), media_id, fatal=False)
+ message = traverse_obj(error, ('message', {str}))
+ code = traverse_obj(error, ('code', {str}))
+ if code == 'REQUEST_FAILED':
+ self.raise_geo_restricted(message, countries=self._GEO_COUNTRIES)
+ elif code == 'MEDIA_NOT_FOUND':
+ raise ExtractorError(message, expected=True)
+ elif code or message:
+ raise ExtractorError(join_nonempty(code, message, delim=': '))
+ raise
+
+ streaks_id = response['id']
+ live_status = {
+ 'clip': 'was_live',
+ 'file': 'not_live',
+ 'linear': 'is_live',
+ 'live': 'is_live',
+ }.get(response.get('type'))
+
+ formats, subtitles = [], {}
+ drm_formats = False
+
+ for source in traverse_obj(response, ('sources', lambda _, v: v['src'])):
+ if source.get('key_systems'):
+ drm_formats = True
+ continue
+
+ src_url = source['src']
+ is_live = live_status == 'is_live'
+ ext = mimetype2ext(source.get('type'))
+ if ext != 'm3u8':
+ self.report_warning(f'Unsupported stream type: {ext}')
+ continue
+
+ if is_live and ssai:
+ session_params = traverse_obj(self._download_json(
+ self._API_URL_TEMPLATE.format('ssai', project_id, streaks_id, '/ssai/session'),
+ media_id, 'Downloading session parameters',
+ headers={'Content-Type': 'application/json', 'Accept': 'application/json'},
+ data=json.dumps({'id': source['id']}).encode(),
+ ), (0, 'query', {urllib.parse.parse_qs}))
+ src_url = update_url_query(src_url, session_params)
+
+ fmts, subs = self._extract_m3u8_formats_and_subtitles(
+ src_url, media_id, 'mp4', m3u8_id='hls', fatal=False, live=is_live, query=query)
+ formats.extend(fmts)
+ self._merge_subtitles(subs, target=subtitles)
+
+ if not formats and drm_formats:
+ self.report_drm(media_id)
+ self._remove_duplicate_formats(formats)
+
+ for subs in traverse_obj(response, (
+ 'tracks', lambda _, v: v['kind'] in ('captions', 'subtitles') and url_or_none(v['src']),
+ )):
+ lang = traverse_obj(subs, ('srclang', {str.lower})) or 'ja'
+ subtitles.setdefault(lang, []).append({'url': subs['src']})
+
+ return {
+ 'id': streaks_id,
+ 'display_id': media_id,
+ 'formats': formats,
+ 'live_status': live_status,
+ 'subtitles': subtitles,
+ 'uploader_id': project_id,
+ **traverse_obj(response, {
+ 'title': ('name', {str}),
+ 'description': ('description', {str}, filter),
+ 'duration': ('duration', {float_or_none}),
+ 'modified_timestamp': ('updated_at', {parse_iso8601}),
+ 'tags': ('tags', ..., {str}),
+ 'thumbnails': (('poster', 'thumbnail'), 'src', {'url': {url_or_none}}),
+ 'timestamp': ('created_at', {parse_iso8601}),
+ }),
+ }
+
+
+class StreaksIE(StreaksBaseIE):
+ _VALID_URL = [
+ r'https?://players\.streaks\.jp/(?P[\w-]+)/[\da-f]+/index\.html\?(?:[^#]+&)?m=(?P(?:ref:)?[\w-]+)',
+ r'https?://playback\.api\.streaks\.jp/v1/projects/(?P[\w-]+)/medias/(?P(?:ref:)?[\w-]+)',
+ ]
+ _EMBED_REGEX = [rf']*\bsrc\s*=\s*["\'](?P{_VALID_URL[0]})']
+ _TESTS = [{
+ 'url': 'https://players.streaks.jp/tipness/08155cd19dc14c12bebefb69b92eafcc/index.html?m=dbdf2df35b4d483ebaeeaeb38c594647',
+ 'info_dict': {
+ 'id': 'dbdf2df35b4d483ebaeeaeb38c594647',
+ 'ext': 'mp4',
+ 'title': '3shunenCM_edit.mp4',
+ 'display_id': 'dbdf2df35b4d483ebaeeaeb38c594647',
+ 'duration': 47.533,
+ 'live_status': 'not_live',
+ 'modified_date': '20230726',
+ 'modified_timestamp': 1690356180,
+ 'timestamp': 1690355996,
+ 'upload_date': '20230726',
+ 'uploader_id': 'tipness',
+ },
+ }, {
+ 'url': 'https://players.streaks.jp/ktv-web/0298e8964c164ab384c07ef6e08c444b/index.html?m=ref:mycoffeetime_250317',
+ 'info_dict': {
+ 'id': 'dccdc079e3fd41f88b0c8435e2d453ab',
+ 'ext': 'mp4',
+ 'title': 'わたしの珈琲時間_250317',
+ 'display_id': 'ref:mycoffeetime_250317',
+ 'duration': 122.99,
+ 'live_status': 'not_live',
+ 'modified_date': '20250310',
+ 'modified_timestamp': 1741586302,
+ 'thumbnail': r're:https?://.+\.jpg',
+ 'timestamp': 1741585839,
+ 'upload_date': '20250310',
+ 'uploader_id': 'ktv-web',
+ },
+ }, {
+ 'url': 'https://playback.api.streaks.jp/v1/projects/ktv-web/medias/b5411938e1e5435dac71edf829dd4813',
+ 'info_dict': {
+ 'id': 'b5411938e1e5435dac71edf829dd4813',
+ 'ext': 'mp4',
+ 'title': 'KANTELE_SYUSEi_0630',
+ 'display_id': 'b5411938e1e5435dac71edf829dd4813',
+ 'live_status': 'not_live',
+ 'modified_date': '20250122',
+ 'modified_timestamp': 1737522999,
+ 'thumbnail': r're:https?://.+\.jpg',
+ 'timestamp': 1735205137,
+ 'upload_date': '20241226',
+ 'uploader_id': 'ktv-web',
+ },
+ }, {
+ # TVer Olympics: website already down, but api remains accessible
+ 'url': 'https://playback.api.streaks.jp/v1/projects/tver-olympic/medias/ref:sp_240806_1748_dvr',
+ 'info_dict': {
+ 'id': 'c10f7345adb648cf804d7578ab93b2e3',
+ 'ext': 'mp4',
+ 'title': 'サッカー 男子 準決勝_dvr',
+ 'display_id': 'ref:sp_240806_1748_dvr',
+ 'duration': 12960.0,
+ 'live_status': 'was_live',
+ 'modified_date': '20240805',
+ 'modified_timestamp': 1722896263,
+ 'timestamp': 1722777618,
+ 'upload_date': '20240804',
+ 'uploader_id': 'tver-olympic',
+ },
+ }, {
+ # TBS FREE: 24-hour stream
+ 'url': 'https://playback.api.streaks.jp/v1/projects/tbs/medias/ref:simul-02',
+ 'info_dict': {
+ 'id': 'c4e83a7b48f4409a96adacec674b4e22',
+ 'ext': 'mp4',
+ 'title': str,
+ 'display_id': 'ref:simul-02',
+ 'live_status': 'is_live',
+ 'modified_date': '20241031',
+ 'modified_timestamp': 1730339858,
+ 'timestamp': 1705466840,
+ 'upload_date': '20240117',
+ 'uploader_id': 'tbs',
+ },
+ }, {
+ # DRM protected
+ 'url': 'https://players.streaks.jp/sp-jbc/a12d7ee0f40c49d6a0a2bff520639677/index.html?m=5f89c62f37ee4a68be8e6e3b1396c7d8',
+ 'only_matching': True,
+ }]
+ _WEBPAGE_TESTS = [{
+ 'url': 'https://event.play.jp/playnext2023/',
+ 'info_dict': {
+ 'id': '2d975178293140dc8074a7fc536a7604',
+ 'ext': 'mp4',
+ 'title': 'PLAY NEXTキームービー（本番）',
+ 'uploader_id': 'play',
+ 'duration': 17.05,
+ 'thumbnail': r're:https?://.+\.jpg',
+ 'timestamp': 1668387517,
+ 'upload_date': '20221114',
+ 'modified_timestamp': 1739411523,
+ 'modified_date': '20250213',
+ 'live_status': 'not_live',
+ },
+ }, {
+ 'url': 'https://wowshop.jp/Page/special/cooking_goods/?bid=wowshop&srsltid=AfmBOor_phUNoPEE_UCPiGGSCMrJE5T2US397smvsbrSdLqUxwON0el4',
+ 'playlist_mincount': 2,
+ 'info_dict': {
+ 'id': '?bid=wowshop&srsltid=AfmBOor_phUNoPEE_UCPiGGSCMrJE5T2US397smvsbrSdLqUxwON0el4',
+ 'title': 'ワンランク上の料理道具でとびきりの“おいしい”を食卓へ｜wowshop',
+ 'description': 'md5:914b5cb8624fc69274c7fb7b2342958f',
+ 'age_limit': 0,
+ 'thumbnail': 'https://wowshop.jp/Page/special/cooking_goods/images/ogp.jpg',
+ },
+ }]
+
+ def _real_extract(self, url):
+ url, smuggled_data = unsmuggle_url(url, {})
+ project_id, media_id = self._match_valid_url(url).group('project_id', 'id')
+
+ return self._extract_from_streaks_api(
+ project_id, media_id, headers=filter_dict({
+ 'X-Streaks-Api-Key': smuggled_data.get('api_key'),
+ }))
diff --git a/yt_dlp/extractor/tver.py b/yt_dlp/extractor/tver.py
index f3daf89469..805150db45 100644
--- a/yt_dlp/extractor/tver.py
+++ b/yt_dlp/extractor/tver.py
@@ -1,31 +1,70 @@
-from .common import InfoExtractor
+from .streaks import StreaksBaseIE
from ..utils import (
ExtractorError,
+ int_or_none,
join_nonempty,
+ make_archive_id,
smuggle_url,
str_or_none,
strip_or_none,
- traverse_obj,
update_url_query,
)
+from ..utils.traversal import require, traverse_obj
-class TVerIE(InfoExtractor):
+class TVerIE(StreaksBaseIE):
_VALID_URL = r'https?://(?:www\.)?tver\.jp/(?:(?Plp|corner|series|episodes?|feature)/)+(?P[a-zA-Z0-9]+)'
+ _GEO_COUNTRIES = ['JP']
+ _GEO_BYPASS = False
_TESTS = [{
- 'skip': 'videos are only available for 7 days',
- 'url': 'https://tver.jp/episodes/ep83nf3w4p',
+ # via Streaks backend
+ 'url': 'https://tver.jp/episodes/epc1hdugbk',
'info_dict': {
- 'title': '家事ヤロウ!!! 売り場席巻のチーズSP＆財前直見×森泉親子の脱東京暮らし密着！',
- 'description': 'md5:dc2c06b6acc23f1e7c730c513737719b',
- 'series': '家事ヤロウ!!!',
- 'episode': '売り場席巻のチーズSP＆財前直見×森泉親子の脱東京暮らし密着！',
- 'alt_title': '売り場席巻のチーズSP＆財前直見×森泉親子の脱東京暮らし密着！',
- 'channel': 'テレビ朝日',
- 'id': 'ep83nf3w4p',
+ 'id': 'epc1hdugbk',
'ext': 'mp4',
+ 'display_id': 'ref:baeebeac-a2a6-4dbf-9eb3-c40d59b40068',
+ 'title': '神回だけ見せます！ #2 壮烈！車大騎馬戦（木曜スペシャル）',
+ 'alt_title': '神回だけ見せます！ #2 壮烈！車大騎馬戦（木曜スペシャル） 日テレ',
+ 'description': 'md5:2726f742d5e3886edeaf72fb6d740fef',
+ 'uploader_id': 'tver-ntv',
+ 'channel': '日テレ',
+ 'duration': 1158.024,
+ 'thumbnail': 'https://statics.tver.jp/images/content/thumbnail/episode/xlarge/epc1hdugbk.jpg?v=16',
+ 'series': '神回だけ見せます！',
+ 'episode': '#2 壮烈！車大騎馬戦（木曜スペシャル）',
+ 'episode_number': 2,
+ 'timestamp': 1736486036,
+ 'upload_date': '20250110',
+ 'modified_timestamp': 1736870264,
+ 'modified_date': '20250114',
+ 'live_status': 'not_live',
+ 'release_timestamp': 1651453200,
+ 'release_date': '20220502',
+ '_old_archive_ids': ['brightcovenew ref:baeebeac-a2a6-4dbf-9eb3-c40d59b40068'],
},
- 'add_ie': ['BrightcoveNew'],
+ }, {
+ # via Brightcove backend (deprecated)
+ 'url': 'https://tver.jp/episodes/epc1hdugbk',
+ 'info_dict': {
+ 'id': 'ref:baeebeac-a2a6-4dbf-9eb3-c40d59b40068',
+ 'ext': 'mp4',
+ 'title': '神回だけ見せます！ #2 壮烈！車大騎馬戦（木曜スペシャル）',
+ 'alt_title': '神回だけ見せます！ #2 壮烈！車大騎馬戦（木曜スペシャル） 日テレ',
+ 'description': 'md5:2726f742d5e3886edeaf72fb6d740fef',
+ 'uploader_id': '4394098882001',
+ 'channel': '日テレ',
+ 'duration': 1158.101,
+ 'thumbnail': 'https://statics.tver.jp/images/content/thumbnail/episode/xlarge/epc1hdugbk.jpg?v=16',
+ 'tags': [],
+ 'series': '神回だけ見せます！',
+ 'episode': '#2 壮烈！車大騎馬戦（木曜スペシャル）',
+ 'episode_number': 2,
+ 'timestamp': 1651388531,
+ 'upload_date': '20220501',
+ 'release_timestamp': 1651453200,
+ 'release_date': '20220502',
+ },
+ 'params': {'extractor_args': {'tver': {'backend': ['brightcove']}}},
}, {
'url': 'https://tver.jp/corner/f0103888',
'only_matching': True,
@@ -38,26 +77,7 @@ class TVerIE(InfoExtractor):
'id': 'srtxft431v',
'title': '名探偵コナン',
},
- 'playlist': [
- {
- 'md5': '779ffd97493ed59b0a6277ea726b389e',
- 'info_dict': {
- 'id': 'ref:conan-1137-241005',
- 'ext': 'mp4',
- 'title': '名探偵コナン #1137「行列店、味変の秘密」',
- 'uploader_id': '5330942432001',
- 'tags': [],
- 'channel': '読売テレビ',
- 'series': '名探偵コナン',
- 'description': 'md5:601fccc1d2430d942a2c8068c4b33eb5',
- 'episode': '#1137「行列店、味変の秘密」',
- 'duration': 1469.077,
- 'timestamp': 1728030405,
- 'upload_date': '20241004',
- 'alt_title': '名探偵コナン #1137「行列店、味変の秘密」 読売テレビ 10月5日(土)放送分',
- 'thumbnail': r're:https://.+\.jpg',
- },
- }],
+ 'playlist_mincount': 21,
}, {
'url': 'https://tver.jp/series/sru35hwdd2',
'info_dict': {
@@ -70,7 +90,11 @@ class TVerIE(InfoExtractor):
'only_matching': True,
}]
BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
- _HEADERS = {'x-tver-platform-type': 'web'}
+ _HEADERS = {
+ 'x-tver-platform-type': 'web',
+ 'Origin': 'https://tver.jp',
+ 'Referer': 'https://tver.jp/',
+ }
_PLATFORM_QUERY = {}
def _real_initialize(self):
@@ -103,6 +127,9 @@ class TVerIE(InfoExtractor):
def _real_extract(self, url):
video_id, video_type = self._match_valid_url(url).group('id', 'type')
+ backend = self._configuration_arg('backend', ['streaks'])[0]
+ if backend not in ('brightcove', 'streaks'):
+ raise ExtractorError(f'Invalid backend value: {backend}', expected=True)
if video_type == 'series':
series_info = self._call_platform_api(
@@ -129,12 +156,6 @@ class TVerIE(InfoExtractor):
video_info = self._download_json(
f'https://statics.tver.jp/content/episode/{video_id}.json', video_id, 'Downloading video info',
query={'v': version}, headers={'Referer': 'https://tver.jp/'})
- p_id = video_info['video']['accountID']
- r_id = traverse_obj(video_info, ('video', ('videoRefID', 'videoID')), get_all=False)
- if not r_id:
- raise ExtractorError('Failed to extract reference ID for Brightcove')
- if not r_id.isdigit():
- r_id = f'ref:{r_id}'
episode = strip_or_none(episode_content.get('title'))
series = str_or_none(episode_content.get('seriesTitle'))
@@ -161,17 +182,53 @@ class TVerIE(InfoExtractor):
]
]
- return {
- '_type': 'url_transparent',
+ metadata = {
'title': title,
'series': series,
'episode': episode,
# an another title which is considered "full title" for some viewers
'alt_title': join_nonempty(title, provider, onair_label, delim=' '),
'channel': provider,
- 'description': str_or_none(video_info.get('description')),
'thumbnails': thumbnails,
- 'url': smuggle_url(
- self.BRIGHTCOVE_URL_TEMPLATE % (p_id, r_id), {'geo_countries': ['JP']}),
- 'ie_key': 'BrightcoveNew',
+ **traverse_obj(video_info, {
+ 'description': ('description', {str}),
+ 'release_timestamp': ('viewStatus', 'startAt', {int_or_none}),
+ 'episode_number': ('no', {int_or_none}),
+ }),
+ }
+
+ brightcove_id = traverse_obj(video_info, ('video', ('videoRefID', 'videoID'), {str}, any))
+ if brightcove_id and not brightcove_id.isdecimal():
+ brightcove_id = f'ref:{brightcove_id}'
+
+ streaks_id = traverse_obj(video_info, ('streaks', 'videoRefID', {str}))
+ if streaks_id and not streaks_id.startswith('ref:'):
+ streaks_id = f'ref:{streaks_id}'
+
+ # Deprecated Brightcove extraction reachable w/extractor-arg or fallback; errors are expected
+ if backend == 'brightcove' or not streaks_id:
+ if backend != 'brightcove':
+ self.report_warning(
+ 'No STREAKS ID found; falling back to Brightcove extraction', video_id=video_id)
+ if not brightcove_id:
+ raise ExtractorError('Unable to extract brightcove reference ID', expected=True)
+ account_id = traverse_obj(video_info, (
+ 'video', 'accountID', {str}, {require('brightcove account ID', expected=True)}))
+ return {
+ **metadata,
+ '_type': 'url_transparent',
+ 'url': smuggle_url(
+ self.BRIGHTCOVE_URL_TEMPLATE % (account_id, brightcove_id),
+ {'geo_countries': ['JP']}),
+ 'ie_key': 'BrightcoveNew',
+ }
+
+ return {
+ **self._extract_from_streaks_api(video_info['streaks']['projectID'], streaks_id, {
+ 'Origin': 'https://tver.jp',
+ 'Referer': 'https://tver.jp/',
+ }),
+ **metadata,
+ 'id': video_id,
+ '_old_archive_ids': [make_archive_id('BrightcoveNew', brightcove_id)] if brightcove_id else None,
}
diff --git a/yt_dlp/extractor/unsupported.py b/yt_dlp/extractor/unsupported.py
index e8d6ae1289..31393b02a4 100644
--- a/yt_dlp/extractor/unsupported.py
+++ b/yt_dlp/extractor/unsupported.py
@@ -51,6 +51,8 @@ class KnownDRMIE(UnsupportedInfoExtractor):
r'(?:watch|front)\.njpwworld\.com',
r'qub\.ca/vrai',
r'(?:beta\.)?crunchyroll\.com',
+ r'viki\.com',
+ r'deezer\.com',
)
_TESTS = [{
@@ -160,6 +162,12 @@ class KnownDRMIE(UnsupportedInfoExtractor):
}, {
'url': 'https://beta.crunchyroll.com/pt-br/watch/G8WUN8VKP/the-ruler-of-conspiracy',
'only_matching': True,
+ }, {
+ 'url': 'https://www.viki.com/videos/1175236v-choosing-spouse-by-lottery-episode-1',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.deezer.com/playlist/176747451',
+ 'only_matching': True,
}]
def _real_extract(self, url):
diff --git a/yt_dlp/extractor/viki.py b/yt_dlp/extractor/viki.py
deleted file mode 100644
index 75f9cdf2ff..0000000000
--- a/yt_dlp/extractor/viki.py
+++ /dev/null
@@ -1,346 +0,0 @@
-import hashlib
-import hmac
-import json
-import time
-
-from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
- int_or_none,
- parse_age_limit,
- parse_iso8601,
- try_get,
-)
-
-
-class VikiBaseIE(InfoExtractor):
- _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
- _API_URL_TEMPLATE = 'https://api.viki.io%s'
-
- _DEVICE_ID = '112395910d'
- _APP = '100005a'
- _APP_VERSION = '6.11.3'
- _APP_SECRET = 'd96704b180208dbb2efa30fe44c48bd8690441af9f567ba8fd710a72badc85198f7472'
-
- _GEO_BYPASS = False
- _NETRC_MACHINE = 'viki'
-
- _token = None
-
- _ERRORS = {
- 'geo': 'Sorry, this content is not available in your region.',
- 'upcoming': 'Sorry, this content is not yet available.',
- 'paywall': 'Sorry, this content is only available to Viki Pass Plus subscribers',
- }
-
- def _stream_headers(self, timestamp, sig):
- return {
- 'X-Viki-manufacturer': 'vivo',
- 'X-Viki-device-model': 'vivo 1606',
- 'X-Viki-device-os-ver': '6.0.1',
- 'X-Viki-connection-type': 'WIFI',
- 'X-Viki-carrier': '',
- 'X-Viki-as-id': '100005a-1625321982-3932',
- 'timestamp': str(timestamp),
- 'signature': str(sig),
- 'x-viki-app-ver': self._APP_VERSION,
- }
-
- def _api_query(self, path, version=4, **kwargs):
- path += '?' if '?' not in path else '&'
- query = f'/v{version}/{path}app={self._APP}'
- if self._token:
- query += f'&token={self._token}'
- return query + ''.join(f'&{name}={val}' for name, val in kwargs.items())
-
- def _sign_query(self, path):
- timestamp = int(time.time())
- query = self._api_query(path, version=5)
- sig = hmac.new(
- self._APP_SECRET.encode('ascii'), f'{query}&t={timestamp}'.encode('ascii'), hashlib.sha1).hexdigest()
- return timestamp, sig, self._API_URL_TEMPLATE % query
-
- def _call_api(
- self, path, video_id, note='Downloading JSON metadata', data=None, query=None, fatal=True):
- if query is None:
- timestamp, sig, url = self._sign_query(path)
- else:
- url = self._API_URL_TEMPLATE % self._api_query(path, version=4)
- resp = self._download_json(
- url, video_id, note, fatal=fatal, query=query,
- data=json.dumps(data).encode() if data else None,
- headers=({'x-viki-app-ver': self._APP_VERSION} if data
- else self._stream_headers(timestamp, sig) if query is None
- else None), expected_status=400) or {}
-
- self._raise_error(resp.get('error'), fatal)
- return resp
-
- def _raise_error(self, error, fatal=True):
- if error is None:
- return
- msg = f'{self.IE_NAME} said: {error}'
- if fatal:
- raise ExtractorError(msg, expected=True)
- else:
- self.report_warning(msg)
-
- def _check_errors(self, data):
- for reason, status in (data.get('blocking') or {}).items():
- if status and reason in self._ERRORS:
- message = self._ERRORS[reason]
- if reason == 'geo':
- self.raise_geo_restricted(msg=message)
- elif reason == 'paywall':
- if try_get(data, lambda x: x['paywallable']['tvod']):
- self._raise_error('This video is for rent only or TVOD (Transactional Video On demand)')
- self.raise_login_required(message)
- self._raise_error(message)
-
- def _perform_login(self, username, password):
- self._token = self._call_api(
- 'sessions.json', None, 'Logging in', fatal=False,
- data={'username': username, 'password': password}).get('token')
- if not self._token:
- self.report_warning('Login Failed: Unable to get session token')
-
- @staticmethod
- def dict_selection(dict_obj, preferred_key):
- if preferred_key in dict_obj:
- return dict_obj[preferred_key]
- return (list(filter(None, dict_obj.values())) or [None])[0]
-
-
-class VikiIE(VikiBaseIE):
- IE_NAME = 'viki'
- _VALID_URL = rf'{VikiBaseIE._VALID_URL_BASE}(?:videos|player)/(?P[0-9]+v)'
- _TESTS = [{
- 'note': 'Free non-DRM video with storyboards in MPD',
- 'url': 'https://www.viki.com/videos/1175236v-choosing-spouse-by-lottery-episode-1',
- 'info_dict': {
- 'id': '1175236v',
- 'ext': 'mp4',
- 'title': 'Choosing Spouse by Lottery - Episode 1',
- 'timestamp': 1606463239,
- 'age_limit': 13,
- 'uploader': 'FCC',
- 'upload_date': '20201127',
- },
- }, {
- 'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
- 'info_dict': {
- 'id': '1023585v',
- 'ext': 'mp4',
- 'title': 'Heirs - Episode 14',
- 'uploader': 'SBS Contents Hub',
- 'timestamp': 1385047627,
- 'upload_date': '20131121',
- 'age_limit': 13,
- 'duration': 3570,
- 'episode_number': 14,
- },
- 'skip': 'Blocked in the US',
- }, {
- # clip
- 'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
- 'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
- 'info_dict': {
- 'id': '1067139v',
- 'ext': 'mp4',
- 'title': "'The Avengers: Age of Ultron' Press Conference",
- 'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
- 'duration': 352,
- 'timestamp': 1430380829,
- 'upload_date': '20150430',
- 'uploader': 'Arirang TV',
- 'like_count': int,
- 'age_limit': 0,
- },
- 'skip': 'Sorry. There was an error loading this video',
- }, {
- 'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
- 'info_dict': {
- 'id': '1048879v',
- 'ext': 'mp4',
- 'title': 'Ankhon Dekhi',
- 'duration': 6512,
- 'timestamp': 1408532356,
- 'upload_date': '20140820',
- 'uploader': 'Spuul',
- 'like_count': int,
- 'age_limit': 13,
- },
- 'skip': 'Blocked in the US',
- }, {
- # episode
- 'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
- 'md5': '0a53dc252e6e690feccd756861495a8c',
- 'info_dict': {
- 'id': '44699v',
- 'ext': 'mp4',
- 'title': 'Boys Over Flowers - Episode 1',
- 'description': 'md5:b89cf50038b480b88b5b3c93589a9076',
- 'duration': 4172,
- 'timestamp': 1270496524,
- 'upload_date': '20100405',
- 'uploader': 'group8',
- 'like_count': int,
- 'age_limit': 13,
- 'episode_number': 1,
- },
- }, {
- # youtube external
- 'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
- 'md5': '63f8600c1da6f01b7640eee7eca4f1da',
- 'info_dict': {
- 'id': '50562v',
- 'ext': 'webm',
- 'title': 'Poor Nastya [COMPLETE] - Episode 1',
- 'description': '',
- 'duration': 606,
- 'timestamp': 1274949505,
- 'upload_date': '20101213',
- 'uploader': 'ad14065n',
- 'uploader_id': 'ad14065n',
- 'like_count': int,
- 'age_limit': 13,
- },
- 'skip': 'Page not found!',
- }, {
- 'url': 'http://www.viki.com/player/44699v',
- 'only_matching': True,
- }, {
- # non-English description
- 'url': 'http://www.viki.com/videos/158036v-love-in-magic',
- 'md5': '41faaba0de90483fb4848952af7c7d0d',
- 'info_dict': {
- 'id': '158036v',
- 'ext': 'mp4',
- 'uploader': 'I Planet Entertainment',
- 'upload_date': '20111122',
- 'timestamp': 1321985454,
- 'description': 'md5:44b1e46619df3a072294645c770cef36',
- 'title': 'Love In Magic',
- 'age_limit': 13,
- },
- }]
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- video = self._call_api(f'videos/{video_id}.json', video_id, 'Downloading video JSON', query={})
- self._check_errors(video)
-
- title = try_get(video, lambda x: x['titles']['en'], str)
- episode_number = int_or_none(video.get('number'))
- if not title:
- title = f'Episode {episode_number}' if video.get('type') == 'episode' else video.get('id') or video_id
- container_titles = try_get(video, lambda x: x['container']['titles'], dict) or {}
- container_title = self.dict_selection(container_titles, 'en')
- title = f'{container_title} - {title}'
-
- thumbnails = [{
- 'id': thumbnail_id,
- 'url': thumbnail['url'],
- } for thumbnail_id, thumbnail in (video.get('images') or {}).items() if thumbnail.get('url')]
-
- resp = self._call_api(
- f'playback_streams/{video_id}.json?drms=dt3&device_id={self._DEVICE_ID}',
- video_id, 'Downloading video streams JSON')['main'][0]
-
- stream_id = try_get(resp, lambda x: x['properties']['track']['stream_id'])
- subtitles = dict((lang, [{
- 'ext': ext,
- 'url': self._API_URL_TEMPLATE % self._api_query(
- f'videos/{video_id}/auth_subtitles/{lang}.{ext}', stream_id=stream_id),
- } for ext in ('srt', 'vtt')]) for lang in (video.get('subtitle_completions') or {}))
-
- mpd_url = resp['url']
- # 720p is hidden in another MPD which can be found in the current manifest content
- mpd_content = self._download_webpage(mpd_url, video_id, note='Downloading initial MPD manifest')
- mpd_url = self._search_regex(
- r'(?mi)(http.+.mpd)', mpd_content, 'new manifest', default=mpd_url)
- if 'mpdhd_high' not in mpd_url and 'sig=' not in mpd_url:
- # Modify the URL to get 1080p
- mpd_url = mpd_url.replace('mpdhd', 'mpdhd_high')
- formats = self._extract_mpd_formats(mpd_url, video_id)
-
- return {
- 'id': video_id,
- 'formats': formats,
- 'title': title,
- 'description': self.dict_selection(video.get('descriptions', {}), 'en'),
- 'duration': int_or_none(video.get('duration')),
- 'timestamp': parse_iso8601(video.get('created_at')),
- 'uploader': video.get('author'),
- 'uploader_url': video.get('author_url'),
- 'like_count': int_or_none(try_get(video, lambda x: x['likes']['count'])),
- 'age_limit': parse_age_limit(video.get('rating')),
- 'thumbnails': thumbnails,
- 'subtitles': subtitles,
- 'episode_number': episode_number,
- }
-
-
-class VikiChannelIE(VikiBaseIE):
- IE_NAME = 'viki:channel'
- _VALID_URL = rf'{VikiBaseIE._VALID_URL_BASE}(?:tv|news|movies|artists)/(?P[0-9]+c)'
- _TESTS = [{
- 'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
- 'info_dict': {
- 'id': '50c',
- 'title': 'Boys Over Flowers',
- 'description': 'md5:804ce6e7837e1fd527ad2f25420f4d59',
- },
- 'playlist_mincount': 51,
- }, {
- 'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
- 'info_dict': {
- 'id': '1354c',
- 'title': 'Poor Nastya [COMPLETE]',
- 'description': 'md5:05bf5471385aa8b21c18ad450e350525',
- },
- 'playlist_count': 127,
- 'skip': 'Page not found',
- }, {
- 'url': 'http://www.viki.com/news/24569c-showbiz-korea',
- 'only_matching': True,
- }, {
- 'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
- 'only_matching': True,
- }, {
- 'url': 'http://www.viki.com/artists/2141c-shinee',
- 'only_matching': True,
- }]
-
- _video_types = ('episodes', 'movies', 'clips', 'trailers')
-
- def _entries(self, channel_id):
- params = {
- 'app': self._APP, 'token': self._token, 'only_ids': 'true',
- 'direction': 'asc', 'sort': 'number', 'per_page': 30,
- }
- video_types = self._configuration_arg('video_types') or self._video_types
- for video_type in video_types:
- if video_type not in self._video_types:
- self.report_warning(f'Unknown video_type: {video_type}')
- page_num = 0
- while True:
- page_num += 1
- params['page'] = page_num
- res = self._call_api(
- f'containers/{channel_id}/{video_type}.json', channel_id, query=params, fatal=False,
- note=f'Downloading {video_type.title()} JSON page {page_num}')
-
- for video_id in res.get('response') or []:
- yield self.url_result(f'https://www.viki.com/videos/{video_id}', VikiIE.ie_key(), video_id)
- if not res.get('more'):
- break
-
- def _real_extract(self, url):
- channel_id = self._match_id(url)
- channel = self._call_api(f'containers/{channel_id}.json', channel_id, 'Downloading channel JSON')
- self._check_errors(channel)
- return self.playlist_result(
- self._entries(channel_id), channel_id,
- self.dict_selection(channel['titles'], 'en'),
- self.dict_selection(channel['descriptions'], 'en'))
diff --git a/yt_dlp/extractor/vrsquare.py b/yt_dlp/extractor/vrsquare.py
new file mode 100644
index 0000000000..9e8740b421
--- /dev/null
+++ b/yt_dlp/extractor/vrsquare.py
@@ -0,0 +1,185 @@
+import itertools
+
+from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
+from ..utils import (
+ ExtractorError,
+ clean_html,
+ extract_attributes,
+ parse_duration,
+ parse_qs,
+)
+from ..utils.traversal import (
+ find_element,
+ find_elements,
+ traverse_obj,
+)
+
+
+class VrSquareIE(InfoExtractor):
+ IE_NAME = 'vrsquare'
+ IE_DESC = 'VR SQUARE'
+
+ _BASE_URL = 'https://livr.jp'
+ _VALID_URL = r'https?://livr\.jp/contents/(?P[\w-]+)'
+ _TESTS = [{
+ 'url': 'https://livr.jp/contents/P470896661',
+ 'info_dict': {
+ 'id': 'P470896661',
+ 'ext': 'mp4',
+ 'title': 'そこ曲がったら、櫻坂？ ７年間お疲れ様！菅井友香の卒業を祝う会！前半 2022年11月6日放送分',
+ 'description': 'md5:523726dc835aa8014dfe1e2b38d36cd1',
+ 'duration': 1515.0,
+ 'tags': 'count:2',
+ 'thumbnail': r're:https?://media\.livr\.jp/vod/img/.+\.jpg',
+ },
+ }, {
+ 'url': 'https://livr.jp/contents/P589523973',
+ 'info_dict': {
+ 'id': 'P589523973',
+ 'ext': 'mp4',
+ 'title': '薄闇に仰ぐ しだれ桜の妖艶',
+ 'description': 'md5:a042f517b2cbb4ed6746707afec4d306',
+ 'duration': 1084.0,
+ 'tags': list,
+ 'thumbnail': r're:https?://media\.livr\.jp/vod/img/.+\.jpg',
+ },
+ 'skip': 'Paid video',
+ }, {
+ 'url': 'https://livr.jp/contents/P316939908',
+ 'info_dict': {
+ 'id': 'P316939908',
+ 'ext': 'mp4',
+ 'title': '2024年5月16日（木） 「今日は誰に恋をする？」公演 小栗有以 生誕祭',
+ 'description': 'md5:2110bdcf947f28bd7d06ec420e51b619',
+ 'duration': 8559.0,
+ 'tags': list,
+ 'thumbnail': r're:https?://media\.livr\.jp/vod/img/.+\.jpg',
+ },
+ 'skip': 'Premium channel subscribers only',
+ }, {
+ # Accessible only in the VR SQUARE app
+ 'url': 'https://livr.jp/contents/P126481458',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ status = self._download_json(
+ f'{self._BASE_URL}/webApi/contentsStatus/{video_id}',
+ video_id, 'Checking contents status', fatal=False)
+ if traverse_obj(status, 'result_code') == '40407':
+ self.raise_login_required('Unable to access this video')
+
+ try:
+ web_api = self._download_json(
+ f'{self._BASE_URL}/webApi/play/url/{video_id}', video_id)
+ except ExtractorError as e:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 500:
+ raise ExtractorError('VR SQUARE app-only videos are not supported', expected=True)
+ raise
+
+ return {
+ 'id': video_id,
+ 'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
+ 'description': self._html_search_meta('description', webpage),
+ 'formats': self._extract_m3u8_formats(traverse_obj(web_api, (
+ 'urls', ..., 'url', any)), video_id, 'mp4', fatal=False),
+ 'thumbnail': self._html_search_meta('og:image', webpage),
+ **traverse_obj(webpage, {
+ 'duration': ({find_element(cls='layout-product-data-time')}, {parse_duration}),
+ 'tags': ({find_elements(cls='search-tag')}, ..., {clean_html}),
+ }),
+ }
+
+
+class VrSquarePlaylistBaseIE(InfoExtractor):
+ _BASE_URL = 'https://livr.jp'
+
+ def _fetch_vids(self, source, keys=()):
+ for url_path in traverse_obj(source, (
+ *keys, {find_elements(cls='video', html=True)}, ...,
+ {extract_attributes}, 'data-url', {str}, filter),
+ ):
+ yield self.url_result(
+ f'{self._BASE_URL}/contents/{url_path.removeprefix("/contents/")}', VrSquareIE)
+
+ def _entries(self, path, display_id, query=None):
+ for page in itertools.count(1):
+ ajax = self._download_json(
+ f'{self._BASE_URL}{path}', display_id,
+ f'Downloading playlist JSON page {page}',
+ query={'p': page, **(query or {})})
+ yield from self._fetch_vids(ajax, ('contents_render_list', ...))
+ if not traverse_obj(ajax, (('has_next', 'hasNext'), {bool}, any)):
+ break
+
+
+class VrSquareChannelIE(VrSquarePlaylistBaseIE):
+ IE_NAME = 'vrsquare:channel'
+
+ _VALID_URL = r'https?://livr\.jp/channel/(?P\w+)'
+ _TESTS = [{
+ 'url': 'https://livr.jp/channel/H372648599',
+ 'info_dict': {
+ 'id': 'H372648599',
+ 'title': 'AKB48＋チャンネル',
+ },
+ 'playlist_mincount': 502,
+ }]
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ webpage = self._download_webpage(url, playlist_id)
+
+ return self.playlist_result(
+ self._entries(f'/ajax/channel/{playlist_id}', playlist_id),
+ playlist_id, self._html_search_meta('og:title', webpage))
+
+
+class VrSquareSearchIE(VrSquarePlaylistBaseIE):
+ IE_NAME = 'vrsquare:search'
+
+ _VALID_URL = r'https?://livr\.jp/web-search/?\?(?:[^#]+&)?w=[^#]+'
+ _TESTS = [{
+ 'url': 'https://livr.jp/web-search?w=%23%E5%B0%8F%E6%A0%97%E6%9C%89%E4%BB%A5',
+ 'info_dict': {
+ 'id': '#小栗有以',
+ },
+ 'playlist_mincount': 60,
+ }]
+
+ def _real_extract(self, url):
+ search_query = parse_qs(url)['w'][0]
+
+ return self.playlist_result(
+ self._entries('/ajax/web-search', search_query, {'w': search_query}), search_query)
+
+
+class VrSquareSectionIE(VrSquarePlaylistBaseIE):
+ IE_NAME = 'vrsquare:section'
+
+ _VALID_URL = r'https?://livr\.jp/(?:category|headline)/(?P\w+)'
+ _TESTS = [{
+ 'url': 'https://livr.jp/category/C133936275',
+ 'info_dict': {
+ 'id': 'C133936275',
+ 'title': 'そこ曲がったら、櫻坂？VR',
+ },
+ 'playlist_mincount': 308,
+ }, {
+ 'url': 'https://livr.jp/headline/A296449604',
+ 'info_dict': {
+ 'id': 'A296449604',
+ 'title': 'AKB48 アフターVR',
+ },
+ 'playlist_mincount': 22,
+ }]
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ webpage = self._download_webpage(url, playlist_id)
+
+ return self.playlist_result(
+ self._fetch_vids(webpage), playlist_id, self._html_search_meta('og:title', webpage))
diff --git a/yt_dlp/extractor/youtube/_video.py b/yt_dlp/extractor/youtube/_video.py
index c773ba2f11..b8cc72ab1d 100644
--- a/yt_dlp/extractor/youtube/_video.py
+++ b/yt_dlp/extractor/youtube/_video.py
@@ -2176,10 +2176,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
"""Returns tuple of strings: variable assignment code, variable name, variable value code"""
return self._search_regex(
r'''(?x)
- \'use\s+strict\';\s*
+ (?P["\'])use\s+strict(?P=q1);\s*
(?P
var\s+(?P[a-zA-Z0-9_$]+)\s*=\s*
- (?P"(?:[^"\\]|\\.)+"\.split\("[^"]+"\))
+ (?P
+ (?P["\'])(?:(?!(?P=q2)).|\\.)+(?P=q2)
+ \.split\((?P["\'])(?:(?!(?P=q3)).)+(?P=q3)\)
+ )
)[;,]
''', jscode, 'global variable', group=('code', 'name', 'value'), default=(None, None, None))
@@ -2187,7 +2190,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
global_var, varname, _ = self._extract_player_js_global_var(full_code)
if global_var:
self.write_debug(f'Prepending n function code with global array variable "{varname}"')
- code = global_var + ', ' + code
+ code = global_var + '; ' + code
else:
self.write_debug('No global array variable found in player JS')
return argnames, re.sub(
@@ -2196,7 +2199,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
def _extract_n_function_code(self, video_id, player_url):
player_id = self._extract_player_info(player_url)
- func_code = self.cache.load('youtube-nsig', player_id, min_ver='2025.03.21')
+ func_code = self.cache.load('youtube-nsig', player_id, min_ver='2025.03.24')
jscode = func_code or self._load_player(video_id, player_url)
jsi = JSInterpreter(jscode)
diff --git a/yt_dlp/networking/_curlcffi.py b/yt_dlp/networking/_curlcffi.py
index 0643348e7e..c800f2c095 100644
--- a/yt_dlp/networking/_curlcffi.py
+++ b/yt_dlp/networking/_curlcffi.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import io
+import itertools
import math
import re
import urllib.parse
@@ -31,9 +32,9 @@ if curl_cffi is None:
curl_cffi_version = tuple(map(int, re.split(r'[^\d]+', curl_cffi.__version__)[:3]))
-if curl_cffi_version != (0, 5, 10) and not ((0, 7, 0) <= curl_cffi_version < (0, 7, 2)):
+if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version:
curl_cffi._yt_dlp__version = f'{curl_cffi.__version__} (unsupported)'
- raise ImportError('Only curl_cffi versions 0.5.10, 0.7.0 and 0.7.1 are supported')
+ raise ImportError('Only curl_cffi versions 0.5.10 and 0.10.x are supported')
import curl_cffi.requests
from curl_cffi.const import CurlECode, CurlOpt
@@ -97,7 +98,7 @@ class CurlCFFIResponseAdapter(Response):
return self.fp.read(amt)
except curl_cffi.requests.errors.RequestsError as e:
if e.code == CurlECode.PARTIAL_FILE:
- content_length = int_or_none(e.response.headers.get('Content-Length'))
+ content_length = e.response and int_or_none(e.response.headers.get('Content-Length'))
raise IncompleteRead(
partial=self.fp.bytes_read,
expected=content_length - self.fp.bytes_read if content_length is not None else None,
@@ -105,6 +106,51 @@ class CurlCFFIResponseAdapter(Response):
raise TransportError(cause=e) from e
+# See: https://github.com/lexiforest/curl_cffi?tab=readme-ov-file#supported-impersonate-browsers
+# https://github.com/lexiforest/curl-impersonate?tab=readme-ov-file#supported-browsers
+BROWSER_TARGETS: dict[tuple[int, ...], dict[str, ImpersonateTarget]] = {
+ (0, 5): {
+ 'chrome99': ImpersonateTarget('chrome', '99', 'windows', '10'),
+ 'chrome99_android': ImpersonateTarget('chrome', '99', 'android', '12'),
+ 'chrome100': ImpersonateTarget('chrome', '100', 'windows', '10'),
+ 'chrome101': ImpersonateTarget('chrome', '101', 'windows', '10'),
+ 'chrome104': ImpersonateTarget('chrome', '104', 'windows', '10'),
+ 'chrome107': ImpersonateTarget('chrome', '107', 'windows', '10'),
+ 'chrome110': ImpersonateTarget('chrome', '110', 'windows', '10'),
+ 'edge99': ImpersonateTarget('edge', '99', 'windows', '10'),
+ 'edge101': ImpersonateTarget('edge', '101', 'windows', '10'),
+ 'safari15_3': ImpersonateTarget('safari', '15.3', 'macos', '11'),
+ 'safari15_5': ImpersonateTarget('safari', '15.5', 'macos', '12'),
+ },
+ (0, 7): {
+ 'chrome116': ImpersonateTarget('chrome', '116', 'windows', '10'),
+ 'chrome119': ImpersonateTarget('chrome', '119', 'macos', '14'),
+ 'chrome120': ImpersonateTarget('chrome', '120', 'macos', '14'),
+ 'chrome123': ImpersonateTarget('chrome', '123', 'macos', '14'),
+ 'chrome124': ImpersonateTarget('chrome', '124', 'macos', '14'),
+ 'safari17_0': ImpersonateTarget('safari', '17.0', 'macos', '14'),
+ 'safari17_2_ios': ImpersonateTarget('safari', '17.2', 'ios', '17.2'),
+ },
+ (0, 9): {
+ 'safari15_3': ImpersonateTarget('safari', '15.3', 'macos', '14'),
+ 'safari15_5': ImpersonateTarget('safari', '15.5', 'macos', '14'),
+ 'chrome119': ImpersonateTarget('chrome', '119', 'macos', '14'),
+ 'chrome120': ImpersonateTarget('chrome', '120', 'macos', '14'),
+ 'chrome123': ImpersonateTarget('chrome', '123', 'macos', '14'),
+ 'chrome124': ImpersonateTarget('chrome', '124', 'macos', '14'),
+ 'chrome131': ImpersonateTarget('chrome', '131', 'macos', '14'),
+ 'chrome131_android': ImpersonateTarget('chrome', '131', 'android', '14'),
+ 'chrome133a': ImpersonateTarget('chrome', '133', 'macos', '15'),
+ 'firefox133': ImpersonateTarget('firefox', '133', 'macos', '14'),
+ 'safari18_0': ImpersonateTarget('safari', '18.0', 'macos', '15'),
+ 'safari18_0_ios': ImpersonateTarget('safari', '18.0', 'ios', '18.0'),
+ },
+ (0, 10): {
+ 'firefox135': ImpersonateTarget('firefox', '135', 'macos', '14'),
+ },
+}
+
+
@register_rh
class CurlCFFIRH(ImpersonateRequestHandler, InstanceStoreMixin):
RH_NAME = 'curl_cffi'
@@ -112,30 +158,21 @@ class CurlCFFIRH(ImpersonateRequestHandler, InstanceStoreMixin):
_SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY)
_SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h')
_SUPPORTED_IMPERSONATE_TARGET_MAP = {
- **({
- ImpersonateTarget('chrome', '124', 'macos', '14'): curl_cffi.requests.BrowserType.chrome124,
- ImpersonateTarget('chrome', '123', 'macos', '14'): curl_cffi.requests.BrowserType.chrome123,
- ImpersonateTarget('chrome', '120', 'macos', '14'): curl_cffi.requests.BrowserType.chrome120,
- ImpersonateTarget('chrome', '119', 'macos', '14'): curl_cffi.requests.BrowserType.chrome119,
- ImpersonateTarget('chrome', '116', 'windows', '10'): curl_cffi.requests.BrowserType.chrome116,
- } if curl_cffi_version >= (0, 7, 0) else {}),
- ImpersonateTarget('chrome', '110', 'windows', '10'): curl_cffi.requests.BrowserType.chrome110,
- ImpersonateTarget('chrome', '107', 'windows', '10'): curl_cffi.requests.BrowserType.chrome107,
- ImpersonateTarget('chrome', '104', 'windows', '10'): curl_cffi.requests.BrowserType.chrome104,
- ImpersonateTarget('chrome', '101', 'windows', '10'): curl_cffi.requests.BrowserType.chrome101,
- ImpersonateTarget('chrome', '100', 'windows', '10'): curl_cffi.requests.BrowserType.chrome100,
- ImpersonateTarget('chrome', '99', 'windows', '10'): curl_cffi.requests.BrowserType.chrome99,
- ImpersonateTarget('edge', '101', 'windows', '10'): curl_cffi.requests.BrowserType.edge101,
- ImpersonateTarget('edge', '99', 'windows', '10'): curl_cffi.requests.BrowserType.edge99,
- **({
- ImpersonateTarget('safari', '17.0', 'macos', '14'): curl_cffi.requests.BrowserType.safari17_0,
- } if curl_cffi_version >= (0, 7, 0) else {}),
- ImpersonateTarget('safari', '15.5', 'macos', '12'): curl_cffi.requests.BrowserType.safari15_5,
- ImpersonateTarget('safari', '15.3', 'macos', '11'): curl_cffi.requests.BrowserType.safari15_3,
- ImpersonateTarget('chrome', '99', 'android', '12'): curl_cffi.requests.BrowserType.chrome99_android,
- **({
- ImpersonateTarget('safari', '17.2', 'ios', '17.2'): curl_cffi.requests.BrowserType.safari17_2_ios,
- } if curl_cffi_version >= (0, 7, 0) else {}),
+ target: name if curl_cffi_version >= (0, 9) else curl_cffi.requests.BrowserType[name]
+ for name, target in dict(sorted(itertools.chain.from_iterable(
+ targets.items()
+ for version, targets in BROWSER_TARGETS.items()
+ if curl_cffi_version >= version
+ ), key=lambda x: (
+ # deprioritize mobile targets since they give very different behavior
+ x[1].os not in ('ios', 'android'),
+ # prioritize edge < firefox < safari < chrome
+ ('edge', 'firefox', 'safari', 'chrome').index(x[1].client),
+ # prioritize newest version
+ float(x[1].version) if x[1].version else 0,
+ # group by os name
+ x[1].os,
+ ), reverse=True)).items()
}
def _create_instance(self, cookiejar=None):
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 0140acaa3a..24525560ef 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -2767,7 +2767,8 @@ def js_to_json(code, vars={}, *, strict=False):
def template_substitute(match):
evaluated = js_to_json(match.group(1), vars, strict=strict)
if evaluated[0] == '"':
- return json.loads(evaluated)
+ with contextlib.suppress(json.JSONDecodeError):
+ return json.loads(evaluated)
return evaluated
def fix_kv(m):