mirror of https://github.com/yt-dlp/yt-dlp
Merge https://github.com/yt-dlp/yt-dlp into fix/ie/gedidigital
DTrombett
2 months ago
commit
09970da6da
@ -0,0 +1,41 @@
|
||||
name: Signature Tests
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- .github/workflows/signature-tests.yml
|
||||
- test/test_youtube_signature.py
|
||||
- yt_dlp/jsinterp.py
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/signature-tests.yml
|
||||
- test/test_youtube_signature.py
|
||||
- yt_dlp/jsinterp.py
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: signature-tests-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Signature Tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', pypy-3.10, pypy-3.11]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install test requirements
|
||||
run: python3 ./devscripts/install_deps.py --only-optional --include test
|
||||
- name: Run tests
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
python3 -m yt_dlp -v || true # Print debug head
|
||||
python3 ./devscripts/run_tests.py test/test_youtube_signature.py
|
||||
@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import unittest
|
||||
|
||||
from yt_dlp.utils.jslib import devalue
|
||||
|
||||
|
||||
TEST_CASES_EQUALS = [{
|
||||
'name': 'int',
|
||||
'unparsed': [-42],
|
||||
'parsed': -42,
|
||||
}, {
|
||||
'name': 'str',
|
||||
'unparsed': ['woo!!!'],
|
||||
'parsed': 'woo!!!',
|
||||
}, {
|
||||
'name': 'Number',
|
||||
'unparsed': [['Object', 42]],
|
||||
'parsed': 42,
|
||||
}, {
|
||||
'name': 'String',
|
||||
'unparsed': [['Object', 'yar']],
|
||||
'parsed': 'yar',
|
||||
}, {
|
||||
'name': 'Infinity',
|
||||
'unparsed': -4,
|
||||
'parsed': math.inf,
|
||||
}, {
|
||||
'name': 'negative Infinity',
|
||||
'unparsed': -5,
|
||||
'parsed': -math.inf,
|
||||
}, {
|
||||
'name': 'negative zero',
|
||||
'unparsed': -6,
|
||||
'parsed': -0.0,
|
||||
}, {
|
||||
'name': 'RegExp',
|
||||
'unparsed': [['RegExp', 'regexp', 'gim']], # XXX: flags are ignored
|
||||
'parsed': re.compile('regexp'),
|
||||
}, {
|
||||
'name': 'Date',
|
||||
'unparsed': [['Date', '2001-09-09T01:46:40.000Z']],
|
||||
'parsed': dt.datetime.fromtimestamp(1e9, tz=dt.timezone.utc),
|
||||
}, {
|
||||
'name': 'Array',
|
||||
'unparsed': [[1, 2, 3], 'a', 'b', 'c'],
|
||||
'parsed': ['a', 'b', 'c'],
|
||||
}, {
|
||||
'name': 'Array (empty)',
|
||||
'unparsed': [[]],
|
||||
'parsed': [],
|
||||
}, {
|
||||
'name': 'Array (sparse)',
|
||||
'unparsed': [[-2, 1, -2], 'b'],
|
||||
'parsed': [None, 'b', None],
|
||||
}, {
|
||||
'name': 'Object',
|
||||
'unparsed': [{'foo': 1, 'x-y': 2}, 'bar', 'z'],
|
||||
'parsed': {'foo': 'bar', 'x-y': 'z'},
|
||||
}, {
|
||||
'name': 'Set',
|
||||
'unparsed': [['Set', 1, 2, 3], 1, 2, 3],
|
||||
'parsed': [1, 2, 3],
|
||||
}, {
|
||||
'name': 'Map',
|
||||
'unparsed': [['Map', 1, 2], 'a', 'b'],
|
||||
'parsed': [['a', 'b']],
|
||||
}, {
|
||||
'name': 'BigInt',
|
||||
'unparsed': [['BigInt', '1']],
|
||||
'parsed': 1,
|
||||
}, {
|
||||
'name': 'Uint8Array',
|
||||
'unparsed': [['Uint8Array', 'AQID']],
|
||||
'parsed': [1, 2, 3],
|
||||
}, {
|
||||
'name': 'ArrayBuffer',
|
||||
'unparsed': [['ArrayBuffer', 'AQID']],
|
||||
'parsed': [1, 2, 3],
|
||||
}, {
|
||||
'name': 'str (repetition)',
|
||||
'unparsed': [[1, 1], 'a string'],
|
||||
'parsed': ['a string', 'a string'],
|
||||
}, {
|
||||
'name': 'None (repetition)',
|
||||
'unparsed': [[1, 1], None],
|
||||
'parsed': [None, None],
|
||||
}, {
|
||||
'name': 'dict (repetition)',
|
||||
'unparsed': [[1, 1], {}],
|
||||
'parsed': [{}, {}],
|
||||
}, {
|
||||
'name': 'Object without prototype',
|
||||
'unparsed': [['null']],
|
||||
'parsed': {},
|
||||
}, {
|
||||
'name': 'cross-realm POJO',
|
||||
'unparsed': [{}],
|
||||
'parsed': {},
|
||||
}]
|
||||
|
||||
TEST_CASES_IS = [{
|
||||
'name': 'bool',
|
||||
'unparsed': [True],
|
||||
'parsed': True,
|
||||
}, {
|
||||
'name': 'Boolean',
|
||||
'unparsed': [['Object', False]],
|
||||
'parsed': False,
|
||||
}, {
|
||||
'name': 'undefined',
|
||||
'unparsed': -1,
|
||||
'parsed': None,
|
||||
}, {
|
||||
'name': 'null',
|
||||
'unparsed': [None],
|
||||
'parsed': None,
|
||||
}, {
|
||||
'name': 'NaN',
|
||||
'unparsed': -3,
|
||||
'parsed': math.nan,
|
||||
}]
|
||||
|
||||
TEST_CASES_INVALID = [{
|
||||
'name': 'empty string',
|
||||
'unparsed': '',
|
||||
'error': ValueError,
|
||||
'pattern': r'expected int or list as input',
|
||||
}, {
|
||||
'name': 'hole',
|
||||
'unparsed': -2,
|
||||
'error': ValueError,
|
||||
'pattern': r'invalid integer input',
|
||||
}, {
|
||||
'name': 'string',
|
||||
'unparsed': 'hello',
|
||||
'error': ValueError,
|
||||
'pattern': r'expected int or list as input',
|
||||
}, {
|
||||
'name': 'number',
|
||||
'unparsed': 42,
|
||||
'error': ValueError,
|
||||
'pattern': r'invalid integer input',
|
||||
}, {
|
||||
'name': 'boolean',
|
||||
'unparsed': True,
|
||||
'error': ValueError,
|
||||
'pattern': r'expected int or list as input',
|
||||
}, {
|
||||
'name': 'null',
|
||||
'unparsed': None,
|
||||
'error': ValueError,
|
||||
'pattern': r'expected int or list as input',
|
||||
}, {
|
||||
'name': 'object',
|
||||
'unparsed': {},
|
||||
'error': ValueError,
|
||||
'pattern': r'expected int or list as input',
|
||||
}, {
|
||||
'name': 'empty array',
|
||||
'unparsed': [],
|
||||
'error': ValueError,
|
||||
'pattern': r'expected a non-empty list as input',
|
||||
}, {
|
||||
'name': 'Python negative indexing',
|
||||
'unparsed': [[1, 2, 3, 4, 5, 6, 7, -7], 1, 2, 3, 4, 5, 6, 7],
|
||||
'error': IndexError,
|
||||
'pattern': r'invalid index: -7',
|
||||
}]
|
||||
|
||||
|
||||
class TestDevalue(unittest.TestCase):
|
||||
def test_devalue_parse_equals(self):
|
||||
for tc in TEST_CASES_EQUALS:
|
||||
self.assertEqual(devalue.parse(tc['unparsed']), tc['parsed'], tc['name'])
|
||||
|
||||
def test_devalue_parse_is(self):
|
||||
for tc in TEST_CASES_IS:
|
||||
self.assertIs(devalue.parse(tc['unparsed']), tc['parsed'], tc['name'])
|
||||
|
||||
def test_devalue_parse_invalid(self):
|
||||
for tc in TEST_CASES_INVALID:
|
||||
with self.assertRaisesRegex(tc['error'], tc['pattern'], msg=tc['name']):
|
||||
devalue.parse(tc['unparsed'])
|
||||
|
||||
def test_devalue_parse_cyclical(self):
|
||||
name = 'Map (cyclical)'
|
||||
result = devalue.parse([['Map', 1, 0], 'self'])
|
||||
self.assertEqual(result[0][0], 'self', name)
|
||||
self.assertIs(result, result[0][1], name)
|
||||
|
||||
name = 'Set (cyclical)'
|
||||
result = devalue.parse([['Set', 0, 1], 42])
|
||||
self.assertEqual(result[1], 42, name)
|
||||
self.assertIs(result, result[0], name)
|
||||
|
||||
result = devalue.parse([[0]])
|
||||
self.assertIs(result, result[0], 'Array (cyclical)')
|
||||
|
||||
name = 'Object (cyclical)'
|
||||
result = devalue.parse([{'self': 0}])
|
||||
self.assertIs(result, result['self'], name)
|
||||
|
||||
name = 'Object with null prototype (cyclical)'
|
||||
result = devalue.parse([['null', 'self', 0]])
|
||||
self.assertIs(result, result['self'], name)
|
||||
|
||||
name = 'Objects (cyclical)'
|
||||
result = devalue.parse([[1, 2], {'second': 2}, {'first': 1}])
|
||||
self.assertIs(result[0], result[1]['first'], name)
|
||||
self.assertIs(result[1], result[0]['second'], name)
|
||||
|
||||
def test_devalue_parse_revivers(self):
|
||||
self.assertEqual(
|
||||
devalue.parse([['indirect', 1], {'a': 2}, 'b'], revivers={'indirect': lambda x: x}),
|
||||
{'a': 'b'}, 'revivers (indirect)')
|
||||
|
||||
self.assertEqual(
|
||||
devalue.parse([['parse', 1], '{"a":0}'], revivers={'parse': lambda x: json.loads(x)}),
|
||||
{'a': 0}, 'revivers (parse)')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 3.8 KiB |
@ -1,188 +0,0 @@
|
||||
from .adobepass import AdobePassIE
|
||||
from ..networking import HEADRequest
|
||||
from ..utils import (
|
||||
extract_attributes,
|
||||
float_or_none,
|
||||
get_element_html_by_class,
|
||||
int_or_none,
|
||||
merge_dicts,
|
||||
parse_age_limit,
|
||||
remove_end,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
unescapeHTML,
|
||||
unified_timestamp,
|
||||
update_url_query,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class BravoTVIE(AdobePassIE):
|
||||
_VALID_URL = r'https?://(?:www\.)?(?P<site>bravotv|oxygen)\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is',
|
||||
'info_dict': {
|
||||
'id': '3923059',
|
||||
'ext': 'mp4',
|
||||
'title': 'The Top Chef Season 16 Winner Is...',
|
||||
'description': 'Find out who takes the title of Top Chef!',
|
||||
'upload_date': '20190314',
|
||||
'timestamp': 1552591860,
|
||||
'season_number': 16,
|
||||
'episode_number': 15,
|
||||
'series': 'Top Chef',
|
||||
'episode': 'The Top Chef Season 16 Winner Is...',
|
||||
'duration': 190.357,
|
||||
'season': 'Season 16',
|
||||
'thumbnail': r're:^https://.+\.jpg',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://www.bravotv.com/top-chef/season-20/episode-1/london-calling',
|
||||
'info_dict': {
|
||||
'id': '9000234570',
|
||||
'ext': 'mp4',
|
||||
'title': 'London Calling',
|
||||
'description': 'md5:5af95a8cbac1856bd10e7562f86bb759',
|
||||
'upload_date': '20230310',
|
||||
'timestamp': 1678410000,
|
||||
'season_number': 20,
|
||||
'episode_number': 1,
|
||||
'series': 'Top Chef',
|
||||
'episode': 'London Calling',
|
||||
'duration': 3266.03,
|
||||
'season': 'Season 20',
|
||||
'chapters': 'count:7',
|
||||
'thumbnail': r're:^https://.+\.jpg',
|
||||
'age_limit': 14,
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
'skip': 'This video requires AdobePass MSO credentials',
|
||||
}, {
|
||||
'url': 'https://www.oxygen.com/in-ice-cold-blood/season-1/closing-night',
|
||||
'info_dict': {
|
||||
'id': '3692045',
|
||||
'ext': 'mp4',
|
||||
'title': 'Closing Night',
|
||||
'description': 'md5:3170065c5c2f19548d72a4cbc254af63',
|
||||
'upload_date': '20180401',
|
||||
'timestamp': 1522623600,
|
||||
'season_number': 1,
|
||||
'episode_number': 1,
|
||||
'series': 'In Ice Cold Blood',
|
||||
'episode': 'Closing Night',
|
||||
'duration': 2629.051,
|
||||
'season': 'Season 1',
|
||||
'chapters': 'count:6',
|
||||
'thumbnail': r're:^https://.+\.jpg',
|
||||
'age_limit': 14,
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
'skip': 'This video requires AdobePass MSO credentials',
|
||||
}, {
|
||||
'url': 'https://www.oxygen.com/in-ice-cold-blood/season-2/episode-16/videos/handling-the-horwitz-house-after-the-murder-season-2',
|
||||
'info_dict': {
|
||||
'id': '3974019',
|
||||
'ext': 'mp4',
|
||||
'title': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
|
||||
'description': 'md5:f9d638dd6946a1c1c0533a9c6100eae5',
|
||||
'upload_date': '20190617',
|
||||
'timestamp': 1560790800,
|
||||
'season_number': 2,
|
||||
'episode_number': 16,
|
||||
'series': 'In Ice Cold Blood',
|
||||
'episode': '\'Handling The Horwitz House After The Murder (Season 2, Episode 16)',
|
||||
'duration': 68.235,
|
||||
'season': 'Season 2',
|
||||
'thumbnail': r're:^https://.+\.jpg',
|
||||
'age_limit': 14,
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
site, display_id = self._match_valid_url(url).group('site', 'id')
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
settings = self._search_json(
|
||||
r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>', webpage, 'settings', display_id)
|
||||
tve = extract_attributes(get_element_html_by_class('tve-video-deck-app', webpage) or '')
|
||||
query = {
|
||||
'manifest': 'm3u',
|
||||
'formats': 'm3u,mpeg4',
|
||||
}
|
||||
|
||||
if tve:
|
||||
account_pid = tve.get('data-mpx-media-account-pid') or 'HNK2IC'
|
||||
account_id = tve['data-mpx-media-account-id']
|
||||
metadata = self._parse_json(
|
||||
tve.get('data-normalized-video', ''), display_id, fatal=False, transform_source=unescapeHTML)
|
||||
video_id = tve.get('data-guid') or metadata['guid']
|
||||
if tve.get('data-entitlement') == 'auth':
|
||||
auth = traverse_obj(settings, ('tve_adobe_auth', {dict})) or {}
|
||||
site = remove_end(site, 'tv')
|
||||
release_pid = tve['data-release-pid']
|
||||
resource = self._get_mvpd_resource(
|
||||
tve.get('data-adobe-pass-resource-id') or auth.get('adobePassResourceId') or site,
|
||||
tve['data-title'], release_pid, tve.get('data-rating'))
|
||||
query.update({
|
||||
'switch': 'HLSServiceSecure',
|
||||
'auth': self._extract_mvpd_auth(
|
||||
url, release_pid, auth.get('adobePassRequestorId') or site, resource),
|
||||
})
|
||||
|
||||
else:
|
||||
ls_playlist = traverse_obj(settings, ('ls_playlist', ..., {dict}), get_all=False) or {}
|
||||
account_pid = ls_playlist.get('mpxMediaAccountPid') or 'PHSl-B'
|
||||
account_id = ls_playlist['mpxMediaAccountId']
|
||||
video_id = ls_playlist['defaultGuid']
|
||||
metadata = traverse_obj(
|
||||
ls_playlist, ('videos', lambda _, v: v['guid'] == video_id, {dict}), get_all=False)
|
||||
|
||||
tp_url = f'https://link.theplatform.com/s/{account_pid}/media/guid/{account_id}/{video_id}'
|
||||
tp_metadata = self._download_json(
|
||||
update_url_query(tp_url, {'format': 'preview'}), video_id, fatal=False)
|
||||
|
||||
chapters = traverse_obj(tp_metadata, ('chapters', ..., {
|
||||
'start_time': ('startTime', {float_or_none(scale=1000)}),
|
||||
'end_time': ('endTime', {float_or_none(scale=1000)}),
|
||||
}))
|
||||
# prune pointless single chapters that span the entire duration from short videos
|
||||
if len(chapters) == 1 and not traverse_obj(chapters, (0, 'end_time')):
|
||||
chapters = None
|
||||
|
||||
m3u8_url = self._request_webpage(HEADRequest(
|
||||
update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').url
|
||||
if 'mpeg_cenc' in m3u8_url:
|
||||
self.report_drm(video_id)
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'chapters': chapters,
|
||||
**merge_dicts(traverse_obj(tp_metadata, {
|
||||
'title': 'title',
|
||||
'description': 'description',
|
||||
'duration': ('duration', {float_or_none(scale=1000)}),
|
||||
'timestamp': ('pubDate', {float_or_none(scale=1000)}),
|
||||
'season_number': (('pl1$seasonNumber', 'nbcu$seasonNumber'), {int_or_none}),
|
||||
'episode_number': (('pl1$episodeNumber', 'nbcu$episodeNumber'), {int_or_none}),
|
||||
'series': (('pl1$show', 'nbcu$show'), (None, ...), {str}),
|
||||
'episode': (('title', 'pl1$episodeNumber', 'nbcu$episodeNumber'), {str_or_none}),
|
||||
'age_limit': ('ratings', ..., 'rating', {parse_age_limit}),
|
||||
}, get_all=False), traverse_obj(metadata, {
|
||||
'title': 'title',
|
||||
'description': 'description',
|
||||
'duration': ('durationInSeconds', {int_or_none}),
|
||||
'timestamp': ('airDate', {unified_timestamp}),
|
||||
'thumbnail': ('thumbnailUrl', {url_or_none}),
|
||||
'season_number': ('seasonNumber', {int_or_none}),
|
||||
'episode_number': ('episodeNumber', {int_or_none}),
|
||||
'episode': 'episodeTitle',
|
||||
'series': 'show',
|
||||
})),
|
||||
}
|
||||
@ -1,32 +1,66 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import js_to_json, traverse_obj
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
clean_html,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import subs_list_to_dict, traverse_obj
|
||||
|
||||
|
||||
class MonsterSirenHypergryphMusicIE(InfoExtractor):
|
||||
IE_NAME = 'monstersiren'
|
||||
IE_DESC = '塞壬唱片'
|
||||
_API_BASE = 'https://monster-siren.hypergryph.com/api'
|
||||
_VALID_URL = r'https?://monster-siren\.hypergryph\.com/music/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://monster-siren.hypergryph.com/music/514562',
|
||||
'info_dict': {
|
||||
'id': '514562',
|
||||
'ext': 'wav',
|
||||
'artists': ['塞壬唱片-MSR'],
|
||||
'album': 'Flame Shadow',
|
||||
'title': 'Flame Shadow',
|
||||
'album': 'Flame Shadow',
|
||||
'artists': ['塞壬唱片-MSR'],
|
||||
'description': 'md5:19e2acfcd1b65b41b29e8079ab948053',
|
||||
'thumbnail': r're:https?://web\.hycdn\.cn/siren/pic/.+\.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://monster-siren.hypergryph.com/music/514518',
|
||||
'info_dict': {
|
||||
'id': '514518',
|
||||
'ext': 'wav',
|
||||
'title': 'Heavenly Me (Instrumental)',
|
||||
'album': 'Heavenly Me',
|
||||
'artists': ['塞壬唱片-MSR', 'AIYUE blessed : 理名'],
|
||||
'description': 'md5:ce790b41c932d1ad72eb791d1d8ae598',
|
||||
'thumbnail': r're:https?://web\.hycdn\.cn/siren/pic/.+\.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
audio_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, audio_id)
|
||||
json_data = self._search_json(
|
||||
r'window\.g_initialProps\s*=', webpage, 'data', audio_id, transform_source=js_to_json)
|
||||
song = self._download_json(f'{self._API_BASE}/song/{audio_id}', audio_id)
|
||||
if traverse_obj(song, 'code') != 0:
|
||||
msg = traverse_obj(song, ('msg', {str}, filter))
|
||||
raise ExtractorError(
|
||||
msg or 'API returned an error response', expected=bool(msg))
|
||||
|
||||
album = None
|
||||
if album_id := traverse_obj(song, ('data', 'albumCid', {str})):
|
||||
album = self._download_json(
|
||||
f'{self._API_BASE}/album/{album_id}/detail', album_id, fatal=False)
|
||||
|
||||
return {
|
||||
'id': audio_id,
|
||||
'title': traverse_obj(json_data, ('player', 'songDetail', 'name')),
|
||||
'url': traverse_obj(json_data, ('player', 'songDetail', 'sourceUrl')),
|
||||
'ext': 'wav',
|
||||
'vcodec': 'none',
|
||||
'artists': traverse_obj(json_data, ('player', 'songDetail', 'artists', ...)),
|
||||
'album': traverse_obj(json_data, ('musicPlay', 'albumDetail', 'name')),
|
||||
**traverse_obj(song, ('data', {
|
||||
'title': ('name', {str}),
|
||||
'artists': ('artists', ..., {str}),
|
||||
'subtitles': ({'url': 'lyricUrl'}, all, {subs_list_to_dict(lang='en')}),
|
||||
'url': ('sourceUrl', {url_or_none}),
|
||||
})),
|
||||
**traverse_obj(album, ('data', {
|
||||
'album': ('name', {str}),
|
||||
'description': ('intro', {clean_html}),
|
||||
'thumbnail': ('coverUrl', {url_or_none}),
|
||||
})),
|
||||
}
|
||||
|
||||
@ -1,408 +0,0 @@
|
||||
import base64
|
||||
import itertools
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import time
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
jwt_decode_hs256,
|
||||
parse_age_limit,
|
||||
try_call,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class JioCinemaBaseIE(InfoExtractor):
|
||||
_NETRC_MACHINE = 'jiocinema'
|
||||
_GEO_BYPASS = False
|
||||
_ACCESS_TOKEN = None
|
||||
_REFRESH_TOKEN = None
|
||||
_GUEST_TOKEN = None
|
||||
_USER_ID = None
|
||||
_DEVICE_ID = None
|
||||
_API_HEADERS = {'Origin': 'https://www.jiocinema.com', 'Referer': 'https://www.jiocinema.com/'}
|
||||
_APP_NAME = {'appName': 'RJIL_JioCinema'}
|
||||
_APP_VERSION = {'appVersion': '5.0.0'}
|
||||
_API_SIGNATURES = 'o668nxgzwff'
|
||||
_METADATA_API_BASE = 'https://content-jiovoot.voot.com/psapi'
|
||||
_ACCESS_HINT = 'the `accessToken` from your browser local storage'
|
||||
_LOGIN_HINT = (
|
||||
'Log in with "-u phone -p <PHONE_NUMBER>" to authenticate with OTP, '
|
||||
f'or use "-u token -p <ACCESS_TOKEN>" to log in with {_ACCESS_HINT}. '
|
||||
'If you have previously logged in with yt-dlp and your session '
|
||||
'has been cached, you can use "-u device -p <DEVICE_ID>"')
|
||||
|
||||
def _cache_token(self, token_type):
|
||||
assert token_type in ('access', 'refresh', 'all')
|
||||
if token_type in ('access', 'all'):
|
||||
self.cache.store(
|
||||
JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-access', JioCinemaBaseIE._ACCESS_TOKEN)
|
||||
if token_type in ('refresh', 'all'):
|
||||
self.cache.store(
|
||||
JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-refresh', JioCinemaBaseIE._REFRESH_TOKEN)
|
||||
|
||||
def _call_api(self, url, video_id, note='Downloading API JSON', headers={}, data={}):
|
||||
return self._download_json(
|
||||
url, video_id, note, data=json.dumps(data, separators=(',', ':')).encode(), headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
**self._API_HEADERS,
|
||||
**headers,
|
||||
}, expected_status=(400, 403, 474))
|
||||
|
||||
def _call_auth_api(self, service, endpoint, note, headers={}, data={}):
|
||||
return self._call_api(
|
||||
f'https://auth-jiocinema.voot.com/{service}service/apis/v4/{endpoint}',
|
||||
None, note=note, headers=headers, data=data)
|
||||
|
||||
def _refresh_token(self):
|
||||
if not JioCinemaBaseIE._REFRESH_TOKEN or not JioCinemaBaseIE._DEVICE_ID:
|
||||
raise ExtractorError('User token has expired', expected=True)
|
||||
response = self._call_auth_api(
|
||||
'token', 'refreshtoken', 'Refreshing token',
|
||||
headers={'accesstoken': self._ACCESS_TOKEN}, data={
|
||||
**self._APP_NAME,
|
||||
'deviceId': self._DEVICE_ID,
|
||||
'refreshToken': self._REFRESH_TOKEN,
|
||||
**self._APP_VERSION,
|
||||
})
|
||||
refresh_token = response.get('refreshTokenId')
|
||||
if refresh_token and refresh_token != JioCinemaBaseIE._REFRESH_TOKEN:
|
||||
JioCinemaBaseIE._REFRESH_TOKEN = refresh_token
|
||||
self._cache_token('refresh')
|
||||
JioCinemaBaseIE._ACCESS_TOKEN = response['authToken']
|
||||
self._cache_token('access')
|
||||
|
||||
def _fetch_guest_token(self):
|
||||
JioCinemaBaseIE._DEVICE_ID = ''.join(random.choices(string.digits, k=10))
|
||||
guest_token = self._call_auth_api(
|
||||
'token', 'guest', 'Downloading guest token', data={
|
||||
**self._APP_NAME,
|
||||
'deviceType': 'phone',
|
||||
'os': 'ios',
|
||||
'deviceId': self._DEVICE_ID,
|
||||
'freshLaunch': False,
|
||||
'adId': self._DEVICE_ID,
|
||||
**self._APP_VERSION,
|
||||
})
|
||||
self._GUEST_TOKEN = guest_token['authToken']
|
||||
self._USER_ID = guest_token['userId']
|
||||
|
||||
def _call_login_api(self, endpoint, guest_token, data, note):
|
||||
return self._call_auth_api(
|
||||
'user', f'loginotp/{endpoint}', note, headers={
|
||||
**self.geo_verification_headers(),
|
||||
'accesstoken': self._GUEST_TOKEN,
|
||||
**self._APP_NAME,
|
||||
**traverse_obj(guest_token, 'data', {
|
||||
'deviceType': ('deviceType', {str}),
|
||||
'os': ('os', {str}),
|
||||
})}, data=data)
|
||||
|
||||
def _is_token_expired(self, token):
|
||||
return (try_call(lambda: jwt_decode_hs256(token)['exp']) or 0) <= int(time.time() - 180)
|
||||
|
||||
def _perform_login(self, username, password):
|
||||
if self._ACCESS_TOKEN and not self._is_token_expired(self._ACCESS_TOKEN):
|
||||
return
|
||||
|
||||
UUID_RE = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
|
||||
|
||||
if username.lower() == 'token':
|
||||
if try_call(lambda: jwt_decode_hs256(password)):
|
||||
JioCinemaBaseIE._ACCESS_TOKEN = password
|
||||
refresh_hint = 'the `refreshToken` UUID from your browser local storage'
|
||||
refresh_token = self._configuration_arg('refresh_token', [''], ie_key=JioCinemaIE)[0]
|
||||
if not refresh_token:
|
||||
self.to_screen(
|
||||
'To extend the life of your login session, in addition to your access token, '
|
||||
'you can pass --extractor-args "jiocinema:refresh_token=REFRESH_TOKEN" '
|
||||
f'where REFRESH_TOKEN is {refresh_hint}')
|
||||
elif re.fullmatch(UUID_RE, refresh_token):
|
||||
JioCinemaBaseIE._REFRESH_TOKEN = refresh_token
|
||||
else:
|
||||
self.report_warning(f'Invalid refresh_token value. Use {refresh_hint}')
|
||||
else:
|
||||
raise ExtractorError(
|
||||
f'The password given could not be decoded as a token; use {self._ACCESS_HINT}', expected=True)
|
||||
|
||||
elif username.lower() == 'device' and re.fullmatch(rf'(?:{UUID_RE}|\d+)', password):
|
||||
JioCinemaBaseIE._REFRESH_TOKEN = self.cache.load(JioCinemaBaseIE._NETRC_MACHINE, f'{password}-refresh')
|
||||
JioCinemaBaseIE._ACCESS_TOKEN = self.cache.load(JioCinemaBaseIE._NETRC_MACHINE, f'{password}-access')
|
||||
if not JioCinemaBaseIE._REFRESH_TOKEN or not JioCinemaBaseIE._ACCESS_TOKEN:
|
||||
raise ExtractorError(f'Failed to load cached tokens for device ID "{password}"', expected=True)
|
||||
|
||||
elif username.lower() == 'phone' and re.fullmatch(r'\+?\d+', password):
|
||||
self._fetch_guest_token()
|
||||
guest_token = jwt_decode_hs256(self._GUEST_TOKEN)
|
||||
initial_data = {
|
||||
'number': base64.b64encode(password.encode()).decode(),
|
||||
**self._APP_VERSION,
|
||||
}
|
||||
response = self._call_login_api('send', guest_token, initial_data, 'Requesting OTP')
|
||||
if not traverse_obj(response, ('OTPInfo', {dict})):
|
||||
raise ExtractorError('There was a problem with the phone number login attempt')
|
||||
|
||||
is_iphone = guest_token.get('os') == 'ios'
|
||||
response = self._call_login_api('verify', guest_token, {
|
||||
'deviceInfo': {
|
||||
'consumptionDeviceName': 'iPhone' if is_iphone else 'Android',
|
||||
'info': {
|
||||
'platform': {'name': 'iPhone OS' if is_iphone else 'Android'},
|
||||
'androidId': self._DEVICE_ID,
|
||||
'type': 'iOS' if is_iphone else 'Android',
|
||||
},
|
||||
},
|
||||
**initial_data,
|
||||
'otp': self._get_tfa_info('the one-time password sent to your phone'),
|
||||
}, 'Submitting OTP')
|
||||
if traverse_obj(response, 'code') == 1043:
|
||||
raise ExtractorError('Wrong OTP', expected=True)
|
||||
JioCinemaBaseIE._REFRESH_TOKEN = response['refreshToken']
|
||||
JioCinemaBaseIE._ACCESS_TOKEN = response['authToken']
|
||||
|
||||
else:
|
||||
raise ExtractorError(self._LOGIN_HINT, expected=True)
|
||||
|
||||
user_token = jwt_decode_hs256(JioCinemaBaseIE._ACCESS_TOKEN)['data']
|
||||
JioCinemaBaseIE._USER_ID = user_token['userId']
|
||||
JioCinemaBaseIE._DEVICE_ID = user_token['deviceId']
|
||||
if JioCinemaBaseIE._REFRESH_TOKEN and username != 'device':
|
||||
self._cache_token('all')
|
||||
if self.get_param('cachedir') is not False:
|
||||
self.to_screen(
|
||||
f'NOTE: For subsequent logins you can use "-u device -p {JioCinemaBaseIE._DEVICE_ID}"')
|
||||
elif not JioCinemaBaseIE._REFRESH_TOKEN:
|
||||
JioCinemaBaseIE._REFRESH_TOKEN = self.cache.load(
|
||||
JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-refresh')
|
||||
if JioCinemaBaseIE._REFRESH_TOKEN:
|
||||
self._cache_token('access')
|
||||
self.to_screen(f'Logging in as device ID "{JioCinemaBaseIE._DEVICE_ID}"')
|
||||
if self._is_token_expired(JioCinemaBaseIE._ACCESS_TOKEN):
|
||||
self._refresh_token()
|
||||
|
||||
|
||||
class JioCinemaIE(JioCinemaBaseIE):
|
||||
IE_NAME = 'jiocinema'
|
||||
_VALID_URL = r'https?://(?:www\.)?jiocinema\.com/?(?:movies?/[^/?#]+/|tv-shows/(?:[^/?#]+/){3})(?P<id>\d{3,})'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.jiocinema.com/tv-shows/agnisakshi-ek-samjhauta/1/pradeep-to-stop-the-wedding/3759931',
|
||||
'info_dict': {
|
||||
'id': '3759931',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pradeep to stop the wedding?',
|
||||
'description': 'md5:75f72d1d1a66976633345a3de6d672b1',
|
||||
'episode': 'Pradeep to stop the wedding?',
|
||||
'episode_number': 89,
|
||||
'season': 'Agnisakshi…Ek Samjhauta-S1',
|
||||
'season_number': 1,
|
||||
'series': 'Agnisakshi Ek Samjhauta',
|
||||
'duration': 1238.0,
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
'age_limit': 13,
|
||||
'season_id': '3698031',
|
||||
'upload_date': '20230606',
|
||||
'timestamp': 1686009600,
|
||||
'release_date': '20230607',
|
||||
'genres': ['Drama'],
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}, {
|
||||
'url': 'https://www.jiocinema.com/movies/bhediya/3754021/watch',
|
||||
'info_dict': {
|
||||
'id': '3754021',
|
||||
'ext': 'mp4',
|
||||
'title': 'Bhediya',
|
||||
'description': 'md5:a6bf2900371ac2fc3f1447401a9f7bb0',
|
||||
'episode': 'Bhediya',
|
||||
'duration': 8500.0,
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
'age_limit': 13,
|
||||
'upload_date': '20230525',
|
||||
'timestamp': 1685026200,
|
||||
'release_date': '20230524',
|
||||
'genres': ['Comedy'],
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
|
||||
def _extract_formats_and_subtitles(self, playback, video_id):
|
||||
m3u8_url = traverse_obj(playback, (
|
||||
'data', 'playbackUrls', lambda _, v: v['streamtype'] == 'hls', 'url', {url_or_none}, any))
|
||||
if not m3u8_url: # DRM-only content only serves dash urls
|
||||
self.report_drm(video_id)
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, m3u8_id='hls')
|
||||
self._remove_duplicate_formats(formats)
|
||||
|
||||
return {
|
||||
# '/_definst_/smil:vod/' m3u8 manifests claim to have 720p+ formats but max out at 480p
|
||||
'formats': traverse_obj(formats, (
|
||||
lambda _, v: '/_definst_/smil:vod/' not in v['url'] or v['height'] <= 480)),
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
if not self._ACCESS_TOKEN and self._is_token_expired(self._GUEST_TOKEN):
|
||||
self._fetch_guest_token()
|
||||
elif self._ACCESS_TOKEN and self._is_token_expired(self._ACCESS_TOKEN):
|
||||
self._refresh_token()
|
||||
|
||||
playback = self._call_api(
|
||||
f'https://apis-jiovoot.voot.com/playbackjv/v3/{video_id}', video_id,
|
||||
'Downloading playback JSON', headers={
|
||||
**self.geo_verification_headers(),
|
||||
'accesstoken': self._ACCESS_TOKEN or self._GUEST_TOKEN,
|
||||
**self._APP_NAME,
|
||||
'deviceid': self._DEVICE_ID,
|
||||
'uniqueid': self._USER_ID,
|
||||
'x-apisignatures': self._API_SIGNATURES,
|
||||
'x-platform': 'androidweb',
|
||||
'x-platform-token': 'web',
|
||||
}, data={
|
||||
'4k': False,
|
||||
'ageGroup': '18+',
|
||||
'appVersion': '3.4.0',
|
||||
'bitrateProfile': 'xhdpi',
|
||||
'capability': {
|
||||
'drmCapability': {
|
||||
'aesSupport': 'yes',
|
||||
'fairPlayDrmSupport': 'none',
|
||||
'playreadyDrmSupport': 'none',
|
||||
'widevineDRMSupport': 'none',
|
||||
},
|
||||
'frameRateCapability': [{
|
||||
'frameRateSupport': '30fps',
|
||||
'videoQuality': '1440p',
|
||||
}],
|
||||
},
|
||||
'continueWatchingRequired': False,
|
||||
'dolby': False,
|
||||
'downloadRequest': False,
|
||||
'hevc': False,
|
||||
'kidsSafe': False,
|
||||
'manufacturer': 'Windows',
|
||||
'model': 'Windows',
|
||||
'multiAudioRequired': True,
|
||||
'osVersion': '10',
|
||||
'parentalPinValid': True,
|
||||
'x-apisignatures': self._API_SIGNATURES,
|
||||
})
|
||||
|
||||
status_code = traverse_obj(playback, ('code', {int}))
|
||||
if status_code == 474:
|
||||
self.raise_geo_restricted(countries=['IN'])
|
||||
elif status_code == 1008:
|
||||
error_msg = 'This content is only available for premium users'
|
||||
if self._ACCESS_TOKEN:
|
||||
raise ExtractorError(error_msg, expected=True)
|
||||
self.raise_login_required(f'{error_msg}. {self._LOGIN_HINT}', method=None)
|
||||
elif status_code == 400:
|
||||
raise ExtractorError('The requested content is not available', expected=True)
|
||||
elif status_code is not None and status_code != 200:
|
||||
raise ExtractorError(
|
||||
f'JioCinema says: {traverse_obj(playback, ("message", {str})) or status_code}')
|
||||
|
||||
metadata = self._download_json(
|
||||
f'{self._METADATA_API_BASE}/voot/v1/voot-web/content/query/asset-details',
|
||||
video_id, fatal=False, query={
|
||||
'ids': f'include:{video_id}',
|
||||
'responseType': 'common',
|
||||
'devicePlatformType': 'desktop',
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'http_headers': self._API_HEADERS,
|
||||
**self._extract_formats_and_subtitles(playback, video_id),
|
||||
**traverse_obj(playback, ('data', {
|
||||
# fallback metadata
|
||||
'title': ('name', {str}),
|
||||
'description': ('fullSynopsis', {str}),
|
||||
'series': ('show', 'name', {str}, filter),
|
||||
'season': ('tournamentName', {str}, {lambda x: x if x != 'Season 0' else None}),
|
||||
'season_number': ('episode', 'season', {int_or_none}, filter),
|
||||
'episode': ('fullTitle', {str}),
|
||||
'episode_number': ('episode', 'episodeNo', {int_or_none}, filter),
|
||||
'age_limit': ('ageNemonic', {parse_age_limit}),
|
||||
'duration': ('totalDuration', {float_or_none}),
|
||||
'thumbnail': ('images', {url_or_none}),
|
||||
})),
|
||||
**traverse_obj(metadata, ('result', 0, {
|
||||
'title': ('fullTitle', {str}),
|
||||
'description': ('fullSynopsis', {str}),
|
||||
'series': ('showName', {str}, filter),
|
||||
'season': ('seasonName', {str}, filter),
|
||||
'season_number': ('season', {int_or_none}),
|
||||
'season_id': ('seasonId', {str}, filter),
|
||||
'episode': ('fullTitle', {str}),
|
||||
'episode_number': ('episode', {int_or_none}),
|
||||
'timestamp': ('uploadTime', {int_or_none}),
|
||||
'release_date': ('telecastDate', {str}),
|
||||
'age_limit': ('ageNemonic', {parse_age_limit}),
|
||||
'duration': ('duration', {float_or_none}),
|
||||
'genres': ('genres', ..., {str}),
|
||||
'thumbnail': ('seo', 'ogImage', {url_or_none}),
|
||||
})),
|
||||
}
|
||||
|
||||
|
||||
class JioCinemaSeriesIE(JioCinemaBaseIE):
|
||||
IE_NAME = 'jiocinema:series'
|
||||
_VALID_URL = r'https?://(?:www\.)?jiocinema\.com/tv-shows/(?P<slug>[\w-]+)/(?P<id>\d{3,})'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.jiocinema.com/tv-shows/naagin/3499917',
|
||||
'info_dict': {
|
||||
'id': '3499917',
|
||||
'title': 'naagin',
|
||||
},
|
||||
'playlist_mincount': 120,
|
||||
}, {
|
||||
'url': 'https://www.jiocinema.com/tv-shows/mtv-splitsvilla-x5/3499820',
|
||||
'info_dict': {
|
||||
'id': '3499820',
|
||||
'title': 'mtv-splitsvilla-x5',
|
||||
},
|
||||
'playlist_mincount': 310,
|
||||
}]
|
||||
|
||||
def _entries(self, series_id):
|
||||
seasons = traverse_obj(self._download_json(
|
||||
f'{self._METADATA_API_BASE}/voot/v1/voot-web/view/show/{series_id}', series_id,
|
||||
'Downloading series metadata JSON', query={'responseType': 'common'}), (
|
||||
'trays', lambda _, v: v['trayId'] == 'season-by-show-multifilter',
|
||||
'trayTabs', lambda _, v: v['id']))
|
||||
|
||||
for season_num, season in enumerate(seasons, start=1):
|
||||
season_id = season['id']
|
||||
label = season.get('label') or season_num
|
||||
for page_num in itertools.count(1):
|
||||
episodes = traverse_obj(self._download_json(
|
||||
f'{self._METADATA_API_BASE}/voot/v1/voot-web/content/generic/series-wise-episode',
|
||||
season_id, f'Downloading season {label} page {page_num} JSON', query={
|
||||
'sort': 'episode:asc',
|
||||
'id': season_id,
|
||||
'responseType': 'common',
|
||||
'page': page_num,
|
||||
}), ('result', lambda _, v: v['id'] and url_or_none(v['slug'])))
|
||||
if not episodes:
|
||||
break
|
||||
for episode in episodes:
|
||||
yield self.url_result(
|
||||
episode['slug'], JioCinemaIE, **traverse_obj(episode, {
|
||||
'video_id': 'id',
|
||||
'video_title': ('fullTitle', {str}),
|
||||
'season_number': ('season', {int_or_none}),
|
||||
'episode_number': ('episode', {int_or_none}),
|
||||
}))
|
||||
|
||||
def _real_extract(self, url):
|
||||
slug, series_id = self._match_valid_url(url).group('slug', 'id')
|
||||
return self.playlist_result(self._entries(series_id), series_id, slug)
|
||||
@ -1,59 +1,57 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
get_element_by_attribute,
|
||||
UnsupportedError,
|
||||
clean_html,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
mimetype2ext,
|
||||
update_url_query,
|
||||
parse_duration,
|
||||
parse_qs,
|
||||
str_or_none,
|
||||
update_url,
|
||||
)
|
||||
from ..utils.traversal import find_element, traverse_obj
|
||||
|
||||
|
||||
class NobelPrizeIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://(?:www\.)?nobelprize\.org/mediaplayer.*?\bid=(?P<id>\d+)'
|
||||
_TEST = {
|
||||
'url': 'http://www.nobelprize.org/mediaplayer/?id=2636',
|
||||
'md5': '04c81e5714bb36cc4e2232fee1d8157f',
|
||||
_VALID_URL = r'https?://(?:(?:mediaplayer|www)\.)?nobelprize\.org/mediaplayer/'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.nobelprize.org/mediaplayer/?id=2636',
|
||||
'info_dict': {
|
||||
'id': '2636',
|
||||
'ext': 'mp4',
|
||||
'title': 'Announcement of the 2016 Nobel Prize in Physics',
|
||||
'description': 'md5:05beba57f4f5a4bbd4cf2ef28fcff739',
|
||||
'description': 'md5:1a2d8a6ca80c88fb3b9a326e0b0e8e43',
|
||||
'duration': 1560.0,
|
||||
'thumbnail': r're:https?://www\.nobelprize\.org/images/.+\.jpg',
|
||||
'timestamp': 1504883793,
|
||||
'upload_date': '20170908',
|
||||
},
|
||||
}
|
||||
}, {
|
||||
'url': 'https://mediaplayer.nobelprize.org/mediaplayer/?qid=12693',
|
||||
'info_dict': {
|
||||
'id': '12693',
|
||||
'ext': 'mp4',
|
||||
'title': 'Nobel Lecture by Peter Higgs',
|
||||
'description': 'md5:9b12e275dbe3a8138484e70e00673a05',
|
||||
'duration': 1800.0,
|
||||
'thumbnail': r're:https?://www\.nobelprize\.org/images/.+\.jpg',
|
||||
'timestamp': 1504883793,
|
||||
'upload_date': '20170908',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
media = self._parse_json(self._search_regex(
|
||||
r'(?s)var\s*config\s*=\s*({.+?});', webpage,
|
||||
'config'), video_id, js_to_json)['media']
|
||||
title = media['title']
|
||||
|
||||
formats = []
|
||||
for source in media.get('source', []):
|
||||
source_src = source.get('src')
|
||||
if not source_src:
|
||||
continue
|
||||
ext = mimetype2ext(source.get('type')) or determine_ext(source_src)
|
||||
if ext == 'm3u8':
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
source_src, video_id, 'mp4', 'm3u8_native',
|
||||
m3u8_id='hls', fatal=False))
|
||||
elif ext == 'f4m':
|
||||
formats.extend(self._extract_f4m_formats(
|
||||
update_url_query(source_src, {'hdcore': '3.7.0'}),
|
||||
video_id, f4m_id='hds', fatal=False))
|
||||
else:
|
||||
formats.append({
|
||||
'url': source_src,
|
||||
})
|
||||
video_id = traverse_obj(parse_qs(url), (
|
||||
('id', 'qid'), -1, {int_or_none}, {str_or_none}, any))
|
||||
if not video_id:
|
||||
raise UnsupportedError(url)
|
||||
webpage = self._download_webpage(
|
||||
update_url(url, netloc='mediaplayer.nobelprize.org'), video_id)
|
||||
|
||||
return {
|
||||
**self._search_json_ld(webpage, video_id),
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': get_element_by_attribute('itemprop', 'description', webpage),
|
||||
'duration': int_or_none(media.get('duration')),
|
||||
'formats': formats,
|
||||
'title': self._html_search_meta('caption', webpage),
|
||||
'description': traverse_obj(webpage, (
|
||||
{find_element(tag='span', attr='itemprop', value='description')}, {clean_html})),
|
||||
'duration': parse_duration(self._html_search_meta('duration', webpage)),
|
||||
}
|
||||
|
||||
@ -1,55 +1,82 @@
|
||||
from .common import InfoExtractor
|
||||
from .streaks import StreaksBaseIE
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
smuggle_url,
|
||||
traverse_obj,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
str_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import require, traverse_obj
|
||||
|
||||
|
||||
class NTVCoJpCUIE(InfoExtractor):
|
||||
class NTVCoJpCUIE(StreaksBaseIE):
|
||||
IE_NAME = 'cu.ntv.co.jp'
|
||||
IE_DESC = 'Nippon Television Network'
|
||||
_VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program)(?P<id>[^/?&#]+)'
|
||||
_TEST = {
|
||||
'url': 'https://cu.ntv.co.jp/televiva-chill-gohan_181031/',
|
||||
IE_DESC = '日テレ無料TADA!'
|
||||
_VALID_URL = r'https?://cu\.ntv\.co\.jp/(?!program-list|search)(?P<id>[\w-]+)/?(?:[?#]|$)'
|
||||
_TESTS = [{
|
||||
'url': 'https://cu.ntv.co.jp/gaki_20250525/',
|
||||
'info_dict': {
|
||||
'id': '5978891207001',
|
||||
'id': 'gaki_20250525',
|
||||
'ext': 'mp4',
|
||||
'title': '桜エビと炒り卵がポイント！ 「中華風 エビチリおにぎり」──『美虎』五十嵐美幸',
|
||||
'upload_date': '20181213',
|
||||
'description': 'md5:1985b51a9abc285df0104d982a325f2a',
|
||||
'uploader_id': '3855502814001',
|
||||
'timestamp': 1544669941,
|
||||
'title': '放送開始36年!方正ココリコが選ぶ神回&地獄回!',
|
||||
'cast': 'count:2',
|
||||
'description': 'md5:1e1db556224d627d4d2f74370c650927',
|
||||
'display_id': 'ref:gaki_20250525',
|
||||
'duration': 1450,
|
||||
'episode': '放送開始36年!方正ココリコが選ぶ神回&地獄回!',
|
||||
'episode_id': '000000010172808',
|
||||
'episode_number': 255,
|
||||
'genres': ['variety'],
|
||||
'live_status': 'not_live',
|
||||
'modified_date': '20250525',
|
||||
'modified_timestamp': 1748145537,
|
||||
'release_date': '20250525',
|
||||
'release_timestamp': 1748145539,
|
||||
'series': 'ダウンタウンのガキの使いやあらへんで！',
|
||||
'series_id': 'gaki',
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
'timestamp': 1748145197,
|
||||
'upload_date': '20250525',
|
||||
'uploader': '日本テレビ放送網',
|
||||
'uploader_id': '0x7FE2',
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
player_config = self._search_nuxt_data(webpage, display_id)
|
||||
video_id = traverse_obj(player_config, ('movie', 'video_id'))
|
||||
if not video_id:
|
||||
raise ExtractorError('Failed to extract video ID for Brightcove')
|
||||
account_id = traverse_obj(player_config, ('player', 'account')) or '3855502814001'
|
||||
title = traverse_obj(player_config, ('movie', 'name'))
|
||||
if not title:
|
||||
og_title = self._og_search_title(webpage, fatal=False) or traverse_obj(player_config, ('player', 'title'))
|
||||
if og_title:
|
||||
title = og_title.split('(', 1)[0].strip()
|
||||
description = (traverse_obj(player_config, ('movie', 'description'))
|
||||
or self._html_search_meta(['description', 'og:description'], webpage))
|
||||
|
||||
info = self._search_json(
|
||||
r'window\.app\s*=', webpage, 'video info',
|
||||
display_id)['falcorCache']['catalog']['episode'][display_id]['value']
|
||||
media_id = traverse_obj(info, (
|
||||
'streaks_data', 'mediaid', {str_or_none}, {require('Streaks media ID')}))
|
||||
non_phonetic = (lambda _, v: v['is_phonetic'] is False, 'value', {str})
|
||||
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, video_id), {'geo_countries': ['JP']}),
|
||||
'ie_key': 'BrightcoveNew',
|
||||
**self._extract_from_streaks_api('ntv-tada', media_id, headers={
|
||||
'X-Streaks-Api-Key': 'df497719056b44059a0483b8faad1f4a',
|
||||
}),
|
||||
**traverse_obj(info, {
|
||||
'id': ('content_id', {str_or_none}),
|
||||
'title': ('title', *non_phonetic, any),
|
||||
'age_limit': ('is_adult_only_content', {lambda x: 18 if x else None}),
|
||||
'cast': ('credit', ..., 'name', *non_phonetic),
|
||||
'genres': ('genre', ..., {str}),
|
||||
'release_timestamp': ('pub_date', {parse_iso8601}),
|
||||
'tags': ('tags', ..., {str}),
|
||||
'thumbnail': ('artwork', ..., 'url', any, {url_or_none}),
|
||||
}),
|
||||
**traverse_obj(info, ('tv_episode_info', {
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'episode_number': ('episode_number', {int}),
|
||||
'series': ('parent_show_title', *non_phonetic, any),
|
||||
'series_id': ('show_content_id', {str}),
|
||||
})),
|
||||
**traverse_obj(info, ('custom_data', {
|
||||
'description': ('program_detail', {str}),
|
||||
'episode': ('episode_title', {str}),
|
||||
'episode_id': ('episode_id', {str_or_none}),
|
||||
'uploader': ('network_name', {str}),
|
||||
'uploader_id': ('network_id', {str}),
|
||||
})),
|
||||
}
|
||||
|
||||
@ -0,0 +1,41 @@
|
||||
from .floatplane import FloatplaneBaseIE
|
||||
|
||||
|
||||
class SaucePlusIE(FloatplaneBaseIE):
|
||||
IE_DESC = 'Sauce+'
|
||||
_VALID_URL = r'https?://(?:(?:www|beta)\.)?sauceplus\.com/post/(?P<id>\w+)'
|
||||
_BASE_URL = 'https://www.sauceplus.com'
|
||||
_HEADERS = {
|
||||
'Origin': _BASE_URL,
|
||||
'Referer': f'{_BASE_URL}/',
|
||||
}
|
||||
_IMPERSONATE_TARGET = True
|
||||
_TESTS = [{
|
||||
'url': 'https://www.sauceplus.com/post/YbBwIa2A5g',
|
||||
'info_dict': {
|
||||
'id': 'eit4Ugu5TL',
|
||||
'ext': 'mp4',
|
||||
'display_id': 'YbBwIa2A5g',
|
||||
'title': 'Scare the Coyote - Episode 3',
|
||||
'description': '',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g$',
|
||||
'duration': 2975,
|
||||
'comment_count': int,
|
||||
'like_count': int,
|
||||
'dislike_count': int,
|
||||
'release_date': '20250627',
|
||||
'release_timestamp': 1750993500,
|
||||
'uploader': 'Scare The Coyote',
|
||||
'uploader_id': '683e0a3269688656a5a49a44',
|
||||
'uploader_url': 'https://www.sauceplus.com/channel/ScareTheCoyote/home',
|
||||
'channel': 'Scare The Coyote',
|
||||
'channel_id': '683e0a326968866ceba49a45',
|
||||
'channel_url': 'https://www.sauceplus.com/channel/ScareTheCoyote/home/main',
|
||||
'availability': 'subscriber_only',
|
||||
},
|
||||
'params': {'skip_download': 'm3u8'},
|
||||
}]
|
||||
|
||||
def _real_initialize(self):
|
||||
if not self._get_cookies(self._BASE_URL).get('__Host-sp-sess'):
|
||||
self.raise_login_required()
|
||||
@ -1,57 +1,102 @@
|
||||
from .ard import ARDMediathekBaseIE
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
get_element_by_attribute,
|
||||
clean_html,
|
||||
extract_attributes,
|
||||
parse_duration,
|
||||
parse_qs,
|
||||
unified_strdate,
|
||||
)
|
||||
from ..utils.traversal import (
|
||||
find_element,
|
||||
require,
|
||||
traverse_obj,
|
||||
)
|
||||
|
||||
|
||||
class SRMediathekIE(ARDMediathekBaseIE):
|
||||
_WORKING = False
|
||||
IE_NAME = 'sr:mediathek'
|
||||
IE_DESC = 'Saarländischer Rundfunk'
|
||||
_VALID_URL = r'https?://sr-mediathek(?:\.sr-online)?\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
|
||||
|
||||
_CLS_COMMON = 'teaser__image__caption__text teaser__image__caption__text--'
|
||||
_VALID_URL = r'https?://(?:www\.)?sr-mediathek\.de/index\.php\?.*?&id=(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
|
||||
'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=141317',
|
||||
'info_dict': {
|
||||
'id': '28455',
|
||||
'id': '141317',
|
||||
'ext': 'mp4',
|
||||
'title': 'sportarena (26.10.2014)',
|
||||
'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'title': 'Kärnten, da will ich hin!',
|
||||
'channel': 'SR Fernsehen',
|
||||
'description': 'md5:7732e71e803379a499732864a572a456',
|
||||
'duration': 1788.0,
|
||||
'release_date': '20250525',
|
||||
'series': 'da will ich hin!',
|
||||
'series_id': 'DWIH',
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
},
|
||||
'skip': 'no longer available',
|
||||
}, {
|
||||
'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=37682',
|
||||
'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=153853',
|
||||
'info_dict': {
|
||||
'id': '37682',
|
||||
'ext': 'mp4',
|
||||
'title': 'Love, Cakes and Rock\'n\'Roll',
|
||||
'description': 'md5:18bf9763631c7d326c22603681e1123d',
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
'id': '153853',
|
||||
'ext': 'mp3',
|
||||
'title': 'Kappes, Klöße, Kokosmilch: Bruschetta mit Nduja',
|
||||
'channel': 'SR 3',
|
||||
'description': 'md5:3935798de3562b10c4070b408a15e225',
|
||||
'duration': 139.0,
|
||||
'release_date': '20250523',
|
||||
'series': 'Kappes, Klöße, Kokosmilch',
|
||||
'series_id': 'SR3_KKK_A',
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://sr-mediathek.de/index.php?seite=7&id=7480',
|
||||
'only_matching': True,
|
||||
'url': 'https://www.sr-mediathek.de/index.php?seite=7&id=31406&pnr=&tbl=pf',
|
||||
'info_dict': {
|
||||
'id': '31406',
|
||||
'ext': 'mp3',
|
||||
'title': 'Das Leben schwer nehmen, ist einfach zu anstrengend',
|
||||
'channel': 'SR 1',
|
||||
'description': 'md5:3e03fd556af831ad984d0add7175fb0c',
|
||||
'duration': 1769.0,
|
||||
'release_date': '20230717',
|
||||
'series': 'Abendrot',
|
||||
'series_id': 'SR1_AB_P',
|
||||
'thumbnail': r're:https?://.+\.jpg',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
description = self._og_search_description(webpage)
|
||||
|
||||
if '>Der gewünschte Beitrag ist leider nicht mehr verfügbar.<' in webpage:
|
||||
if description == 'Der gewünschte Beitrag ist leider nicht mehr vorhanden.':
|
||||
raise ExtractorError(f'Video {video_id} is no longer available', expected=True)
|
||||
|
||||
media_collection_url = self._search_regex(
|
||||
r'data-mediacollection-ardplayer="([^"]+)"', webpage, 'media collection url')
|
||||
info = self._extract_media_info(media_collection_url, webpage, video_id)
|
||||
info.update({
|
||||
player_url = traverse_obj(webpage, (
|
||||
{find_element(tag='div', id=f'player{video_id}', html=True)},
|
||||
{extract_attributes}, 'data-mediacollection-ardplayer',
|
||||
{self._proto_relative_url}, {require('player URL')}))
|
||||
article = traverse_obj(webpage, (
|
||||
{find_element(cls='article__content')},
|
||||
{find_element(tag='p')}, {clean_html}))
|
||||
|
||||
return {
|
||||
**self._extract_media_info(player_url, webpage, video_id),
|
||||
'id': video_id,
|
||||
'title': get_element_by_attribute('class', 'ardplayer-title', webpage),
|
||||
'description': self._og_search_description(webpage),
|
||||
'title': traverse_obj(webpage, (
|
||||
{find_element(cls='ardplayer-title')}, {clean_html})),
|
||||
'channel': traverse_obj(webpage, (
|
||||
{find_element(cls=f'{self._CLS_COMMON}subheadline')},
|
||||
{lambda x: x.split('|')[0]}, {clean_html})),
|
||||
'description': description,
|
||||
'duration': parse_duration(self._search_regex(
|
||||
r'(\d{2}:\d{2}:\d{2})', article, 'duration')),
|
||||
'release_date': unified_strdate(self._search_regex(
|
||||
r'(\d{2}\.\d{2}\.\d{4})', article, 'release_date')),
|
||||
'series': traverse_obj(webpage, (
|
||||
{find_element(cls=f'{self._CLS_COMMON}headline')}, {clean_html})),
|
||||
'series_id': traverse_obj(webpage, (
|
||||
{find_element(cls='teaser__link', html=True)},
|
||||
{extract_attributes}, 'href', {parse_qs}, 'sen', ..., {str}, any)),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
})
|
||||
return info
|
||||
}
|
||||
|
||||
@ -1,76 +1,76 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none, urljoin
|
||||
from .youtube import YoutubeIE
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
parse_iso8601,
|
||||
update_url,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import subs_list_to_dict, traverse_obj
|
||||
|
||||
|
||||
class StarTrekIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'(?P<base>https?://(?:intl|www)\.startrek\.com)/videos/(?P<id>[^/]+)'
|
||||
IE_NAME = 'startrek'
|
||||
IE_DESC = 'STAR TREK'
|
||||
_VALID_URL = r'https?://(?:www\.)?startrek\.com(?:/en-(?:ca|un))?/videos/(?P<id>[^/?#]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://intl.startrek.com/videos/watch-welcoming-jess-bush-to-the-ready-room',
|
||||
'md5': '491df5035c9d4dc7f63c79caaf9c839e',
|
||||
'url': 'https://www.startrek.com/en-un/videos/official-trailer-star-trek-lower-decks-season-4',
|
||||
'info_dict': {
|
||||
'id': 'watch-welcoming-jess-bush-to-the-ready-room',
|
||||
'id': 'official-trailer-star-trek-lower-decks-season-4',
|
||||
'ext': 'mp4',
|
||||
'title': 'WATCH: Welcoming Jess Bush to The Ready Room',
|
||||
'duration': 1888,
|
||||
'timestamp': 1655388000,
|
||||
'upload_date': '20220616',
|
||||
'description': 'md5:1ffee884e3920afbdd6dd04e926a1221',
|
||||
'thumbnail': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/styles/video_1920x1080/public/images/2022-06/pp_14794_rr_thumb_107_yt_16x9\.jpg(?:\?.+)?',
|
||||
'subtitles': {'en-US': [{
|
||||
'url': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/video/captions/2022-06/TRR_SNW_107_v4\.vtt',
|
||||
}, {
|
||||
'url': 'https://media.startrek.com/2022/06/16/2043801155561/1069981_hls/trr_snw_107_v4-c4bfc25d/stream_vtt.m3u8',
|
||||
}]},
|
||||
'title': 'Official Trailer | Star Trek: Lower Decks - Season 4',
|
||||
'alt_title': 'md5:dd7e3191aaaf9e95db16fc3abd5ef68b',
|
||||
'categories': ['TRAILERS'],
|
||||
'description': 'md5:563d7856ddab99bee7a5e50f45531757',
|
||||
'release_date': '20230722',
|
||||
'release_timestamp': 1690033200,
|
||||
'series': 'Star Trek: Lower Decks',
|
||||
'series_id': 'star-trek-lower-decks',
|
||||
'thumbnail': r're:https?://.+\.(?:jpg|png)',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.startrek.com/videos/watch-ethan-peck-and-gia-sandhu-beam-down-to-the-ready-room',
|
||||
'md5': 'f5ad74fbb86e91e0882fc0a333178d1d',
|
||||
'url': 'https://www.startrek.com/en-ca/videos/my-first-contact-senator-cory-booker',
|
||||
'info_dict': {
|
||||
'id': 'watch-ethan-peck-and-gia-sandhu-beam-down-to-the-ready-room',
|
||||
'id': 'my-first-contact-senator-cory-booker',
|
||||
'ext': 'mp4',
|
||||
'title': 'WATCH: Ethan Peck and Gia Sandhu Beam Down to The Ready Room',
|
||||
'duration': 1986,
|
||||
'timestamp': 1654221600,
|
||||
'upload_date': '20220603',
|
||||
'description': 'md5:b3aa0edacfe119386567362dec8ed51b',
|
||||
'thumbnail': r're:https://www\.startrek\.com/sites/default/files/styles/video_1920x1080/public/images/2022-06/pp_14792_rr_thumb_105_yt_16x9_1.jpg(?:\?.+)?',
|
||||
'subtitles': {'en-US': [{
|
||||
'url': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/video/captions/2022-06/TRR_SNW_105_v5\.vtt',
|
||||
}]},
|
||||
'title': 'My First Contact: Senator Cory Booker',
|
||||
'alt_title': 'md5:fe74a8bdb0afab421c6e159a7680db4d',
|
||||
'categories': ['MY FIRST CONTACT'],
|
||||
'description': 'md5:a3992ab3b3e0395925d71156bbc018ce',
|
||||
'release_date': '20250401',
|
||||
'release_timestamp': 1743512400,
|
||||
'series': 'Star Trek: The Original Series',
|
||||
'series_id': 'star-trek-the-original-series',
|
||||
'thumbnail': r're:https?://.+\.(?:jpg|png)',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
urlbase, video_id = self._match_valid_url(url).group('base', 'id')
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
player = self._search_regex(
|
||||
r'(<\s*div\s+id\s*=\s*"cvp-player-[^<]+<\s*/div\s*>)', webpage, 'player')
|
||||
page_props = self._search_nextjs_data(webpage, video_id)['props']['pageProps']
|
||||
video_data = page_props['video']['data']
|
||||
if youtube_id := video_data.get('youtube_video_id'):
|
||||
return self.url_result(youtube_id, YoutubeIE)
|
||||
|
||||
hls = self._html_search_regex(r'\bdata-hls\s*=\s*"([^"]+)"', player, 'HLS URL')
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(hls, video_id, 'mp4')
|
||||
|
||||
captions = self._html_search_regex(
|
||||
r'\bdata-captions-url\s*=\s*"([^"]+)"', player, 'captions URL', fatal=False)
|
||||
if captions:
|
||||
subtitles.setdefault('en-US', [])[:0] = [{'url': urljoin(urlbase, captions)}]
|
||||
|
||||
# NB: Most of the data in the json_ld is undesirable
|
||||
json_ld = self._search_json_ld(webpage, video_id, fatal=False)
|
||||
series_id = traverse_obj(video_data, (
|
||||
'series_and_movies', ..., 'series_or_movie', 'slug', {str}, any))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._html_search_regex(
|
||||
r'\bdata-title\s*=\s*"([^"]+)"', player, 'title', json_ld.get('title')),
|
||||
'description': self._html_search_regex(
|
||||
r'(?s)<\s*div\s+class\s*=\s*"header-body"\s*>(.+?)<\s*/div\s*>',
|
||||
webpage, 'description', fatal=False),
|
||||
'duration': int_or_none(self._html_search_regex(
|
||||
r'\bdata-duration\s*=\s*"(\d+)"', player, 'duration', fatal=False)),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnail': urljoin(urlbase, self._html_search_regex(
|
||||
r'\bdata-poster-url\s*=\s*"([^"]+)"', player, 'thumbnail', fatal=False)),
|
||||
'timestamp': json_ld.get('timestamp'),
|
||||
'series': traverse_obj(page_props, (
|
||||
'queried', 'header', 'tab3', 'slices', ..., 'items',
|
||||
lambda _, v: v['link']['slug'] == series_id, 'link_copy', {str}, any)),
|
||||
'series_id': series_id,
|
||||
**traverse_obj(video_data, {
|
||||
'title': ('title', ..., 'text', {clean_html}, any),
|
||||
'alt_title': ('subhead', ..., 'text', {clean_html}, any),
|
||||
'categories': ('category', 'data', 'category_name', {str.upper}, filter, all),
|
||||
'description': ('slices', ..., 'primary', 'content', ..., 'text', {clean_html}, any),
|
||||
'release_timestamp': ('published', {parse_iso8601}),
|
||||
'subtitles': ({'url': 'legacy_subtitle_file'}, all, {subs_list_to_dict(lang='en')}),
|
||||
'thumbnail': ('poster_frame', 'url', {url_or_none}, {update_url(query=None)}),
|
||||
'url': ('legacy_video_url', {url_or_none}),
|
||||
}),
|
||||
}
|
||||
|
||||
@ -1,58 +0,0 @@
|
||||
from .adobepass import AdobePassIE
|
||||
from ..utils import (
|
||||
smuggle_url,
|
||||
update_url_query,
|
||||
)
|
||||
|
||||
|
||||
class SyfyIE(AdobePassIE):
|
||||
_WORKING = False
|
||||
_VALID_URL = r'https?://(?:www\.)?syfy\.com/(?:[^/]+/)?videos/(?P<id>[^/?#]+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.syfy.com/theinternetruinedmylife/videos/the-internet-ruined-my-life-season-1-trailer',
|
||||
'info_dict': {
|
||||
'id': '2968097',
|
||||
'ext': 'mp4',
|
||||
'title': 'The Internet Ruined My Life: Season 1 Trailer',
|
||||
'description': 'One tweet, one post, one click, can destroy everything.',
|
||||
'uploader': 'NBCU-MPAT',
|
||||
'upload_date': '20170113',
|
||||
'timestamp': 1484345640,
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
},
|
||||
'add_ie': ['ThePlatform'],
|
||||
'skip': 'Redirects to main page',
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
syfy_mpx = next(iter(self._parse_json(self._search_regex(
|
||||
r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', webpage, 'drupal settings'),
|
||||
display_id)['syfy']['syfy_mpx'].values()))
|
||||
video_id = syfy_mpx['mpxGUID']
|
||||
title = syfy_mpx['episodeTitle']
|
||||
query = {
|
||||
'mbr': 'true',
|
||||
'manifest': 'm3u',
|
||||
}
|
||||
if syfy_mpx.get('entitlement') == 'auth':
|
||||
resource = self._get_mvpd_resource(
|
||||
'syfy', title, video_id,
|
||||
syfy_mpx.get('mpxRating', 'TV-14'))
|
||||
query['auth'] = self._extract_mvpd_auth(
|
||||
url, video_id, 'syfy', resource)
|
||||
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'ie_key': 'ThePlatform',
|
||||
'url': smuggle_url(update_url_query(
|
||||
self._proto_relative_url(syfy_mpx['releaseURL']), query),
|
||||
{'force_smil_url': True}),
|
||||
'title': title,
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
}
|
||||
@ -1,98 +1,53 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
parse_filesize,
|
||||
parse_iso8601,
|
||||
)
|
||||
from ..utils import clean_html
|
||||
from ..utils.traversal import find_element, traverse_obj
|
||||
|
||||
|
||||
class UMGDeIE(InfoExtractor):
|
||||
_WORKING = False
|
||||
IE_NAME = 'umg:de'
|
||||
IE_DESC = 'Universal Music Deutschland'
|
||||
_VALID_URL = r'https?://(?:www\.)?universal-music\.de/[^/]+/videos/[^/?#]+-(?P<id>\d+)'
|
||||
_TEST = {
|
||||
_VALID_URL = r'https?://(?:www\.)?universal-music\.de/[^/?#]+/videos/(?P<slug>[^/?#]+-(?P<id>\d+))'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.universal-music.de/sido/videos/jedes-wort-ist-gold-wert-457803',
|
||||
'md5': 'ebd90f48c80dcc82f77251eb1902634f',
|
||||
'info_dict': {
|
||||
'id': '457803',
|
||||
'ext': 'mp4',
|
||||
'title': 'Jedes Wort ist Gold wert',
|
||||
'artists': ['Sido'],
|
||||
'description': 'md5:df2dbffcff1a74e0a7c9bef4b497aeec',
|
||||
'display_id': 'jedes-wort-ist-gold-wert-457803',
|
||||
'duration': 210.0,
|
||||
'thumbnail': r're:https?://images\.universal-music\.de/img/assets/.+\.jpg',
|
||||
'timestamp': 1513591800,
|
||||
'upload_date': '20171218',
|
||||
'view_count': int,
|
||||
},
|
||||
}
|
||||
}, {
|
||||
'url': 'https://www.universal-music.de/alexander-eder/videos/der-doktor-hat-gesagt-609533',
|
||||
'info_dict': {
|
||||
'id': '609533',
|
||||
'ext': 'mp4',
|
||||
'title': 'Der Doktor hat gesagt',
|
||||
'artists': ['Alexander Eder'],
|
||||
'display_id': 'der-doktor-hat-gesagt-609533',
|
||||
'duration': 146.0,
|
||||
'thumbnail': r're:https?://images\.universal-music\.de/img/assets/.+\.jpg',
|
||||
'timestamp': 1742982100,
|
||||
'upload_date': '20250326',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video_data = self._download_json(
|
||||
'https://graphql.universal-music.de/',
|
||||
video_id, query={
|
||||
'query': '''{
|
||||
universalMusic(channel:16) {
|
||||
video(id:%s) {
|
||||
headline
|
||||
formats {
|
||||
formatId
|
||||
url
|
||||
type
|
||||
width
|
||||
height
|
||||
mimeType
|
||||
fileSize
|
||||
}
|
||||
duration
|
||||
createdDate
|
||||
}
|
||||
}
|
||||
}''' % video_id})['data']['universalMusic']['video'] # noqa: UP031
|
||||
|
||||
title = video_data['headline']
|
||||
hls_url_template = 'http://mediadelivery.universal-music-services.de/vod/mp4:autofill/storage/' + '/'.join(list(video_id)) + '/content/%s/file/playlist.m3u8'
|
||||
|
||||
thumbnails = []
|
||||
formats = []
|
||||
|
||||
def add_m3u8_format(format_id):
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
hls_url_template % format_id, video_id, 'mp4',
|
||||
'm3u8_native', m3u8_id='hls', fatal=False))
|
||||
|
||||
for f in video_data.get('formats', []):
|
||||
f_url = f.get('url')
|
||||
mime_type = f.get('mimeType')
|
||||
if not f_url or mime_type == 'application/mxf':
|
||||
continue
|
||||
fmt = {
|
||||
'url': f_url,
|
||||
'width': int_or_none(f.get('width')),
|
||||
'height': int_or_none(f.get('height')),
|
||||
'filesize': parse_filesize(f.get('fileSize')),
|
||||
}
|
||||
f_type = f.get('type')
|
||||
if f_type == 'Image':
|
||||
thumbnails.append(fmt)
|
||||
elif f_type == 'Video':
|
||||
format_id = f.get('formatId')
|
||||
if format_id:
|
||||
fmt['format_id'] = format_id
|
||||
if mime_type == 'video/mp4':
|
||||
add_m3u8_format(format_id)
|
||||
urlh = self._request_webpage(f_url, video_id, fatal=False)
|
||||
if urlh:
|
||||
first_byte = urlh.read(1)
|
||||
if first_byte not in (b'F', b'\x00'):
|
||||
continue
|
||||
formats.append(fmt)
|
||||
if not formats:
|
||||
for format_id in (867, 836, 940):
|
||||
add_m3u8_format(format_id)
|
||||
display_id, video_id = self._match_valid_url(url).group('slug', 'id')
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
return {
|
||||
**self._search_json_ld(webpage, display_id),
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'duration': int_or_none(video_data.get('duration')),
|
||||
'timestamp': parse_iso8601(video_data.get('createdDate'), ' '),
|
||||
'thumbnails': thumbnails,
|
||||
'formats': formats,
|
||||
'artists': traverse_obj(self._html_search_meta('umg-artist-screenname', webpage), (filter, all)),
|
||||
# The JSON LD description duplicates the title
|
||||
'description': traverse_obj(webpage, ({find_element(cls='_3Y0Lj')}, {clean_html})),
|
||||
'display_id': display_id,
|
||||
'formats': self._extract_m3u8_formats(
|
||||
'https://hls.universal-music.de/get', display_id, 'mp4', query={'id': video_id}),
|
||||
}
|
||||
|
||||
@ -0,0 +1 @@
|
||||
# Utility functions for handling web input based on commonly used JavaScript libraries
|
||||
@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import array
|
||||
import base64
|
||||
import datetime as dt
|
||||
import math
|
||||
import re
|
||||
|
||||
from .._utils import parse_iso8601
|
||||
|
||||
TYPE_CHECKING = False
|
||||
if TYPE_CHECKING:
|
||||
import collections.abc
|
||||
import typing
|
||||
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
|
||||
_ARRAY_TYPE_LOOKUP = {
|
||||
'Int8Array': 'b',
|
||||
'Uint8Array': 'B',
|
||||
'Uint8ClampedArray': 'B',
|
||||
'Int16Array': 'h',
|
||||
'Uint16Array': 'H',
|
||||
'Int32Array': 'i',
|
||||
'Uint32Array': 'I',
|
||||
'Float32Array': 'f',
|
||||
'Float64Array': 'd',
|
||||
'BigInt64Array': 'l',
|
||||
'BigUint64Array': 'L',
|
||||
'ArrayBuffer': 'B',
|
||||
}
|
||||
|
||||
|
||||
def parse_iter(parsed: typing.Any, /, *, revivers: dict[str, collections.abc.Callable[[list], typing.Any]] | None = None):
|
||||
# based on https://github.com/Rich-Harris/devalue/blob/f3fd2aa93d79f21746555671f955a897335edb1b/src/parse.js
|
||||
resolved = {
|
||||
-1: None,
|
||||
-2: None,
|
||||
-3: math.nan,
|
||||
-4: math.inf,
|
||||
-5: -math.inf,
|
||||
-6: -0.0,
|
||||
}
|
||||
|
||||
if isinstance(parsed, int) and not isinstance(parsed, bool):
|
||||
if parsed not in resolved or parsed == -2:
|
||||
raise ValueError('invalid integer input')
|
||||
return resolved[parsed]
|
||||
elif not isinstance(parsed, list):
|
||||
raise ValueError('expected int or list as input')
|
||||
elif not parsed:
|
||||
raise ValueError('expected a non-empty list as input')
|
||||
|
||||
if revivers is None:
|
||||
revivers = {}
|
||||
return_value = [None]
|
||||
stack: list[tuple] = [(return_value, 0, 0)]
|
||||
|
||||
while stack:
|
||||
target, index, source = stack.pop()
|
||||
if isinstance(source, tuple):
|
||||
name, source, reviver = source
|
||||
try:
|
||||
resolved[source] = target[index] = reviver(target[index])
|
||||
except Exception as error:
|
||||
yield TypeError(f'failed to parse {source} as {name!r}: {error}')
|
||||
resolved[source] = target[index] = None
|
||||
continue
|
||||
|
||||
if source in resolved:
|
||||
target[index] = resolved[source]
|
||||
continue
|
||||
|
||||
# guard against Python negative indexing
|
||||
if source < 0:
|
||||
yield IndexError(f'invalid index: {source!r}')
|
||||
continue
|
||||
|
||||
try:
|
||||
value = parsed[source]
|
||||
except IndexError as error:
|
||||
yield error
|
||||
continue
|
||||
|
||||
if isinstance(value, list):
|
||||
if value and isinstance(value[0], str):
|
||||
# TODO: implement zips `strict=True`
|
||||
if reviver := revivers.get(value[0]):
|
||||
if value[1] == source:
|
||||
# XXX: avoid infinite loop
|
||||
yield IndexError(f'{value[0]!r} cannot point to itself (index: {source})')
|
||||
continue
|
||||
# inverse order: resolve index, revive value
|
||||
stack.append((target, index, (value[0], value[1], reviver)))
|
||||
stack.append((target, index, value[1]))
|
||||
continue
|
||||
|
||||
elif value[0] == 'Date':
|
||||
try:
|
||||
result = dt.datetime.fromtimestamp(parse_iso8601(value[1]), tz=dt.timezone.utc)
|
||||
except Exception:
|
||||
yield ValueError(f'invalid date: {value[1]!r}')
|
||||
result = None
|
||||
|
||||
elif value[0] == 'Set':
|
||||
result = [None] * (len(value) - 1)
|
||||
for offset, new_source in enumerate(value[1:]):
|
||||
stack.append((result, offset, new_source))
|
||||
|
||||
elif value[0] == 'Map':
|
||||
result = []
|
||||
for key, new_source in zip(*(iter(value[1:]),) * 2):
|
||||
pair = [None, None]
|
||||
stack.append((pair, 0, key))
|
||||
stack.append((pair, 1, new_source))
|
||||
result.append(pair)
|
||||
|
||||
elif value[0] == 'RegExp':
|
||||
# XXX: use jsinterp to translate regex flags
|
||||
# currently ignores `value[2]`
|
||||
result = re.compile(value[1])
|
||||
|
||||
elif value[0] == 'Object':
|
||||
result = value[1]
|
||||
|
||||
elif value[0] == 'BigInt':
|
||||
result = int(value[1])
|
||||
|
||||
elif value[0] == 'null':
|
||||
result = {}
|
||||
for key, new_source in zip(*(iter(value[1:]),) * 2):
|
||||
stack.append((result, key, new_source))
|
||||
|
||||
elif value[0] in _ARRAY_TYPE_LOOKUP:
|
||||
typecode = _ARRAY_TYPE_LOOKUP[value[0]]
|
||||
data = base64.b64decode(value[1])
|
||||
result = array.array(typecode, data).tolist()
|
||||
|
||||
else:
|
||||
yield TypeError(f'invalid type at {source}: {value[0]!r}')
|
||||
result = None
|
||||
else:
|
||||
result = len(value) * [None]
|
||||
for offset, new_source in enumerate(value):
|
||||
stack.append((result, offset, new_source))
|
||||
|
||||
elif isinstance(value, dict):
|
||||
result = {}
|
||||
for key, new_source in value.items():
|
||||
stack.append((result, key, new_source))
|
||||
|
||||
else:
|
||||
result = value
|
||||
|
||||
target[index] = resolved[source] = result
|
||||
|
||||
return return_value[0]
|
||||
|
||||
|
||||
def parse(parsed: typing.Any, /, *, revivers: dict[str, collections.abc.Callable[[typing.Any], typing.Any]] | None = None):
|
||||
generator = parse_iter(parsed, revivers=revivers)
|
||||
while True:
|
||||
try:
|
||||
raise generator.send(None)
|
||||
except StopIteration as error:
|
||||
return error.value
|
||||
Loading…
Reference in New Issue