mirror of https://github.com/yt-dlp/yt-dlp
Merge branch 'master' into yt-live-from-start-range
bashonly
1 year ago
committed by
GitHub
commit
6fc6349ef0
@ -1,10 +0,0 @@
|
||||
include AUTHORS
|
||||
include Changelog.md
|
||||
include LICENSE
|
||||
include README.md
|
||||
include completions/*/*
|
||||
include supportedsites.md
|
||||
include yt-dlp.1
|
||||
include requirements.txt
|
||||
recursive-include devscripts *
|
||||
recursive-include test *
|
||||
@ -0,0 +1 @@
|
||||
# Empty file
|
||||
@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow execution from anywhere
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import warnings
|
||||
|
||||
from py2exe import freeze
|
||||
|
||||
from devscripts.utils import read_version
|
||||
|
||||
VERSION = read_version()
|
||||
|
||||
|
||||
def main():
|
||||
warnings.warn(
|
||||
'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
|
||||
'It is recommended to run "pyinst.py" to build using pyinstaller instead')
|
||||
|
||||
return freeze(
|
||||
console=[{
|
||||
'script': './yt_dlp/__main__.py',
|
||||
'dest_base': 'yt-dlp',
|
||||
'icon_resources': [(1, 'devscripts/logo.ico')],
|
||||
}],
|
||||
version_info={
|
||||
'version': VERSION,
|
||||
'description': 'A youtube-dl fork with additional features and patches',
|
||||
'comments': 'Official repository: <https://github.com/yt-dlp/yt-dlp>',
|
||||
'product_name': 'yt-dlp',
|
||||
'product_version': VERSION,
|
||||
},
|
||||
options={
|
||||
'bundle_files': 0,
|
||||
'compressed': 1,
|
||||
'optimize': 2,
|
||||
'dist_dir': './dist',
|
||||
'excludes': [
|
||||
# py2exe cannot import Crypto
|
||||
'Crypto',
|
||||
'Cryptodome',
|
||||
# py2exe appears to confuse this with our socks library.
|
||||
# We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
|
||||
'urllib3.contrib.socks'
|
||||
],
|
||||
'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
|
||||
# Modules that are only imported dynamically must be added here
|
||||
'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
|
||||
'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
|
||||
},
|
||||
zipfile=None,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow execution from anywhere
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
from devscripts.tomlparse import parse_toml
|
||||
from devscripts.utils import read_file
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Install dependencies for yt-dlp')
|
||||
parser.add_argument(
|
||||
'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)')
|
||||
parser.add_argument(
|
||||
'-e', '--exclude', metavar='REQUIREMENT', action='append', help='Exclude a required dependency')
|
||||
parser.add_argument(
|
||||
'-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group')
|
||||
parser.add_argument(
|
||||
'-o', '--only-optional', action='store_true', help='Only install optional dependencies')
|
||||
parser.add_argument(
|
||||
'-p', '--print', action='store_true', help='Only print a requirements.txt to stdout')
|
||||
parser.add_argument(
|
||||
'-u', '--user', action='store_true', help='Install with pip as --user')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
toml_data = parse_toml(read_file(args.input))
|
||||
deps = toml_data['project']['dependencies']
|
||||
targets = deps.copy() if not args.only_optional else []
|
||||
|
||||
for exclude in args.exclude or []:
|
||||
for dep in deps:
|
||||
simplified_dep = re.match(r'[\w-]+', dep)[0]
|
||||
if dep in targets and (exclude.lower() == simplified_dep.lower() or exclude == dep):
|
||||
targets.remove(dep)
|
||||
|
||||
optional_deps = toml_data['project']['optional-dependencies']
|
||||
for include in args.include or []:
|
||||
group = optional_deps.get(include)
|
||||
if group:
|
||||
targets.extend(group)
|
||||
|
||||
if args.print:
|
||||
for target in targets:
|
||||
print(target)
|
||||
return
|
||||
|
||||
pip_args = [sys.executable, '-m', 'pip', 'install', '-U']
|
||||
if args.user:
|
||||
pip_args.append('--user')
|
||||
pip_args.extend(targets)
|
||||
|
||||
return subprocess.call(pip_args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Simple parser for spec compliant toml files
|
||||
|
||||
A simple toml parser for files that comply with the spec.
|
||||
Should only be used to parse `pyproject.toml` for `install_deps.py`.
|
||||
|
||||
IMPORTANT: INVALID FILES OR MULTILINE STRINGS ARE NOT SUPPORTED!
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
|
||||
WS = r'(?:[\ \t]*)'
|
||||
STRING_RE = re.compile(r'"(?:\\.|[^\\"\n])*"|\'[^\'\n]*\'')
|
||||
SINGLE_KEY_RE = re.compile(rf'{STRING_RE.pattern}|[A-Za-z0-9_-]+')
|
||||
KEY_RE = re.compile(rf'{WS}(?:{SINGLE_KEY_RE.pattern}){WS}(?:\.{WS}(?:{SINGLE_KEY_RE.pattern}){WS})*')
|
||||
EQUALS_RE = re.compile(rf'={WS}')
|
||||
WS_RE = re.compile(WS)
|
||||
|
||||
_SUBTABLE = rf'(?P<subtable>^\[(?P<is_list>\[)?(?P<path>{KEY_RE.pattern})\]\]?)'
|
||||
EXPRESSION_RE = re.compile(rf'^(?:{_SUBTABLE}|{KEY_RE.pattern}=)', re.MULTILINE)
|
||||
|
||||
LIST_WS_RE = re.compile(rf'{WS}((#[^\n]*)?\n{WS})*')
|
||||
LEFTOVER_VALUE_RE = re.compile(r'[^,}\]\t\n#]+')
|
||||
|
||||
|
||||
def parse_key(value: str):
|
||||
for match in SINGLE_KEY_RE.finditer(value):
|
||||
if match[0][0] == '"':
|
||||
yield json.loads(match[0])
|
||||
elif match[0][0] == '\'':
|
||||
yield match[0][1:-1]
|
||||
else:
|
||||
yield match[0]
|
||||
|
||||
|
||||
def get_target(root: dict, paths: list[str], is_list=False):
|
||||
target = root
|
||||
|
||||
for index, key in enumerate(paths, 1):
|
||||
use_list = is_list and index == len(paths)
|
||||
result = target.get(key)
|
||||
if result is None:
|
||||
result = [] if use_list else {}
|
||||
target[key] = result
|
||||
|
||||
if isinstance(result, dict):
|
||||
target = result
|
||||
elif use_list:
|
||||
target = {}
|
||||
result.append(target)
|
||||
else:
|
||||
target = result[-1]
|
||||
|
||||
assert isinstance(target, dict)
|
||||
return target
|
||||
|
||||
|
||||
def parse_enclosed(data: str, index: int, end: str, ws_re: re.Pattern):
|
||||
index += 1
|
||||
|
||||
if match := ws_re.match(data, index):
|
||||
index = match.end()
|
||||
|
||||
while data[index] != end:
|
||||
index = yield True, index
|
||||
|
||||
if match := ws_re.match(data, index):
|
||||
index = match.end()
|
||||
|
||||
if data[index] == ',':
|
||||
index += 1
|
||||
|
||||
if match := ws_re.match(data, index):
|
||||
index = match.end()
|
||||
|
||||
assert data[index] == end
|
||||
yield False, index + 1
|
||||
|
||||
|
||||
def parse_value(data: str, index: int):
|
||||
if data[index] == '[':
|
||||
result = []
|
||||
|
||||
indices = parse_enclosed(data, index, ']', LIST_WS_RE)
|
||||
valid, index = next(indices)
|
||||
while valid:
|
||||
index, value = parse_value(data, index)
|
||||
result.append(value)
|
||||
valid, index = indices.send(index)
|
||||
|
||||
return index, result
|
||||
|
||||
if data[index] == '{':
|
||||
result = {}
|
||||
|
||||
indices = parse_enclosed(data, index, '}', WS_RE)
|
||||
valid, index = next(indices)
|
||||
while valid:
|
||||
valid, index = indices.send(parse_kv_pair(data, index, result))
|
||||
|
||||
return index, result
|
||||
|
||||
if match := STRING_RE.match(data, index):
|
||||
return match.end(), json.loads(match[0]) if match[0][0] == '"' else match[0][1:-1]
|
||||
|
||||
match = LEFTOVER_VALUE_RE.match(data, index)
|
||||
assert match
|
||||
value = match[0].strip()
|
||||
for func in [
|
||||
int,
|
||||
float,
|
||||
datetime.time.fromisoformat,
|
||||
datetime.date.fromisoformat,
|
||||
datetime.datetime.fromisoformat,
|
||||
{'true': True, 'false': False}.get,
|
||||
]:
|
||||
try:
|
||||
value = func(value)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return match.end(), value
|
||||
|
||||
|
||||
def parse_kv_pair(data: str, index: int, target: dict):
|
||||
match = KEY_RE.match(data, index)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
*keys, key = parse_key(match[0])
|
||||
|
||||
match = EQUALS_RE.match(data, match.end())
|
||||
assert match
|
||||
index = match.end()
|
||||
|
||||
index, value = parse_value(data, index)
|
||||
get_target(target, keys)[key] = value
|
||||
return index
|
||||
|
||||
|
||||
def parse_toml(data: str):
|
||||
root = {}
|
||||
target = root
|
||||
|
||||
index = 0
|
||||
while True:
|
||||
match = EXPRESSION_RE.search(data, index)
|
||||
if not match:
|
||||
break
|
||||
|
||||
if match.group('subtable'):
|
||||
index = match.end()
|
||||
path, is_list = match.group('path', 'is_list')
|
||||
target = get_target(root, list(parse_key(path)), bool(is_list))
|
||||
continue
|
||||
|
||||
index = parse_kv_pair(data, match.start(), target)
|
||||
assert index is not None
|
||||
|
||||
return root
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile', type=Path, help='The TOML file to read as input')
|
||||
args = parser.parse_args()
|
||||
|
||||
with args.infile.open('r', encoding='utf-8') as file:
|
||||
data = file.read()
|
||||
|
||||
def default(obj):
|
||||
if isinstance(obj, (datetime.date, datetime.time, datetime.datetime)):
|
||||
return obj.isoformat()
|
||||
|
||||
print(json.dumps(parse_toml(data), default=default))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,5 +1,118 @@
|
||||
[build-system]
|
||||
build-backend = 'setuptools.build_meta'
|
||||
# https://github.com/yt-dlp/yt-dlp/issues/5941
|
||||
# https://github.com/pypa/distutils/issues/17
|
||||
requires = ['setuptools > 50']
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "yt-dlp"
|
||||
maintainers = [
|
||||
{name = "pukkandan", email = "pukkandan.ytdlp@gmail.com"},
|
||||
{name = "Grub4K", email = "contact@grub4k.xyz"},
|
||||
{name = "bashonly", email = "bashonly@protonmail.com"},
|
||||
]
|
||||
description = "A youtube-dl fork with additional features and patches"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
keywords = [
|
||||
"youtube-dl",
|
||||
"video-downloader",
|
||||
"youtube-downloader",
|
||||
"sponsorblock",
|
||||
"youtube-dlc",
|
||||
"yt-dlp",
|
||||
]
|
||||
license = {file = "LICENSE"}
|
||||
classifiers = [
|
||||
"Topic :: Multimedia :: Video",
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Environment :: Console",
|
||||
"Programming Language :: Python",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: Implementation",
|
||||
"Programming Language :: Python :: Implementation :: CPython",
|
||||
"Programming Language :: Python :: Implementation :: PyPy",
|
||||
"License :: OSI Approved :: The Unlicense (Unlicense)",
|
||||
"Operating System :: OS Independent",
|
||||
]
|
||||
dynamic = ["version"]
|
||||
dependencies = [
|
||||
"brotli; implementation_name=='cpython'",
|
||||
"brotlicffi; implementation_name!='cpython'",
|
||||
"certifi",
|
||||
"mutagen",
|
||||
"pycryptodomex",
|
||||
"requests>=2.31.0,<3",
|
||||
"urllib3>=1.26.17,<3",
|
||||
"websockets>=12.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
secretstorage = [
|
||||
"cffi",
|
||||
"secretstorage",
|
||||
]
|
||||
build = [
|
||||
"build",
|
||||
"hatchling",
|
||||
"pip",
|
||||
"wheel",
|
||||
]
|
||||
dev = [
|
||||
"flake8",
|
||||
"isort",
|
||||
"pytest",
|
||||
]
|
||||
pyinstaller = ["pyinstaller>=6.3"]
|
||||
py2exe = ["py2exe>=0.12"]
|
||||
|
||||
[project.urls]
|
||||
Documentation = "https://github.com/yt-dlp/yt-dlp#readme"
|
||||
Repository = "https://github.com/yt-dlp/yt-dlp"
|
||||
Tracker = "https://github.com/yt-dlp/yt-dlp/issues"
|
||||
Funding = "https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators"
|
||||
|
||||
[project.scripts]
|
||||
yt-dlp = "yt_dlp:main"
|
||||
|
||||
[project.entry-points.pyinstaller40]
|
||||
hook-dirs = "yt_dlp.__pyinstaller:get_hook_dirs"
|
||||
|
||||
[tool.hatch.build.targets.sdist]
|
||||
include = [
|
||||
"/yt_dlp",
|
||||
"/devscripts",
|
||||
"/test",
|
||||
"/.gitignore", # included by default, needed for auto-excludes
|
||||
"/Changelog.md",
|
||||
"/LICENSE", # included as license
|
||||
"/pyproject.toml", # included by default
|
||||
"/README.md", # included as readme
|
||||
"/setup.cfg",
|
||||
"/supportedsites.md",
|
||||
]
|
||||
artifacts = [
|
||||
"/yt_dlp/extractor/lazy_extractors.py",
|
||||
"/completions",
|
||||
"/AUTHORS", # included by default
|
||||
"/README.txt",
|
||||
"/yt-dlp.1",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["yt_dlp"]
|
||||
artifacts = ["/yt_dlp/extractor/lazy_extractors.py"]
|
||||
|
||||
[tool.hatch.build.targets.wheel.shared-data]
|
||||
"completions/bash/yt-dlp" = "share/bash-completion/completions/yt-dlp"
|
||||
"completions/zsh/_yt-dlp" = "share/zsh/site-functions/_yt-dlp"
|
||||
"completions/fish/yt-dlp.fish" = "share/fish/vendor_completions.d/yt-dlp.fish"
|
||||
"README.txt" = "share/doc/yt_dlp/README.txt"
|
||||
"yt-dlp.1" = "share/man/man1/yt-dlp.1"
|
||||
|
||||
[tool.hatch.version]
|
||||
path = "yt_dlp/version.py"
|
||||
pattern = "_pkg_version = '(?P<version>[^']+)'"
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
mutagen
|
||||
pycryptodomex
|
||||
brotli; implementation_name=='cpython'
|
||||
brotlicffi; implementation_name!='cpython'
|
||||
certifi
|
||||
requests>=2.31.0,<3
|
||||
urllib3>=1.26.17,<3
|
||||
websockets>=12.0
|
||||
@ -1,183 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow execution from anywhere
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import subprocess
|
||||
import warnings
|
||||
|
||||
try:
|
||||
from setuptools import Command, find_packages, setup
|
||||
setuptools_available = True
|
||||
except ImportError:
|
||||
from distutils.core import Command, setup
|
||||
setuptools_available = False
|
||||
|
||||
from devscripts.utils import read_file, read_version
|
||||
|
||||
VERSION = read_version(varname='_pkg_version')
|
||||
|
||||
DESCRIPTION = 'A youtube-dl fork with additional features and patches'
|
||||
|
||||
LONG_DESCRIPTION = '\n\n'.join((
|
||||
'Official repository: <https://github.com/yt-dlp/yt-dlp>',
|
||||
'**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
|
||||
read_file('README.md')))
|
||||
|
||||
REQUIREMENTS = read_file('requirements.txt').splitlines()
|
||||
|
||||
|
||||
def packages():
|
||||
if setuptools_available:
|
||||
return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'))
|
||||
|
||||
return [
|
||||
'yt_dlp', 'yt_dlp.extractor', 'yt_dlp.downloader', 'yt_dlp.postprocessor', 'yt_dlp.compat',
|
||||
]
|
||||
|
||||
|
||||
def py2exe_params():
|
||||
warnings.warn(
|
||||
'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
|
||||
'It is recommended to run "pyinst.py" to build using pyinstaller instead')
|
||||
|
||||
return {
|
||||
'console': [{
|
||||
'script': './yt_dlp/__main__.py',
|
||||
'dest_base': 'yt-dlp',
|
||||
'icon_resources': [(1, 'devscripts/logo.ico')],
|
||||
}],
|
||||
'version_info': {
|
||||
'version': VERSION,
|
||||
'description': DESCRIPTION,
|
||||
'comments': LONG_DESCRIPTION.split('\n')[0],
|
||||
'product_name': 'yt-dlp',
|
||||
'product_version': VERSION,
|
||||
},
|
||||
'options': {
|
||||
'bundle_files': 0,
|
||||
'compressed': 1,
|
||||
'optimize': 2,
|
||||
'dist_dir': './dist',
|
||||
'excludes': [
|
||||
# py2exe cannot import Crypto
|
||||
'Crypto',
|
||||
'Cryptodome',
|
||||
# py2exe appears to confuse this with our socks library.
|
||||
# We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
|
||||
'urllib3.contrib.socks'
|
||||
],
|
||||
'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
|
||||
# Modules that are only imported dynamically must be added here
|
||||
'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
|
||||
'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
|
||||
},
|
||||
'zipfile': None,
|
||||
}
|
||||
|
||||
|
||||
def build_params():
|
||||
files_spec = [
|
||||
('share/bash-completion/completions', ['completions/bash/yt-dlp']),
|
||||
('share/zsh/site-functions', ['completions/zsh/_yt-dlp']),
|
||||
('share/fish/vendor_completions.d', ['completions/fish/yt-dlp.fish']),
|
||||
('share/doc/yt_dlp', ['README.txt']),
|
||||
('share/man/man1', ['yt-dlp.1'])
|
||||
]
|
||||
data_files = []
|
||||
for dirname, files in files_spec:
|
||||
resfiles = []
|
||||
for fn in files:
|
||||
if not os.path.exists(fn):
|
||||
warnings.warn(f'Skipping file {fn} since it is not present. Try running " make pypi-files " first')
|
||||
else:
|
||||
resfiles.append(fn)
|
||||
data_files.append((dirname, resfiles))
|
||||
|
||||
params = {'data_files': data_files}
|
||||
|
||||
if setuptools_available:
|
||||
params['entry_points'] = {
|
||||
'console_scripts': ['yt-dlp = yt_dlp:main'],
|
||||
'pyinstaller40': ['hook-dirs = yt_dlp.__pyinstaller:get_hook_dirs'],
|
||||
}
|
||||
else:
|
||||
params['scripts'] = ['yt-dlp']
|
||||
return params
|
||||
|
||||
|
||||
class build_lazy_extractors(Command):
|
||||
description = 'Build the extractor lazy loading module'
|
||||
user_options = []
|
||||
|
||||
def initialize_options(self):
|
||||
pass
|
||||
|
||||
def finalize_options(self):
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
if self.dry_run:
|
||||
print('Skipping build of lazy extractors in dry run mode')
|
||||
return
|
||||
subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])
|
||||
|
||||
|
||||
def main():
|
||||
if sys.argv[1:2] == ['py2exe']:
|
||||
params = py2exe_params()
|
||||
try:
|
||||
from py2exe import freeze
|
||||
except ImportError:
|
||||
import py2exe # noqa: F401
|
||||
warnings.warn('You are using an outdated version of py2exe. Support for this version will be removed in the future')
|
||||
params['console'][0].update(params.pop('version_info'))
|
||||
params['options'] = {'py2exe': params.pop('options')}
|
||||
else:
|
||||
return freeze(**params)
|
||||
else:
|
||||
params = build_params()
|
||||
|
||||
setup(
|
||||
name='yt-dlp', # package name (do not change/remove comment)
|
||||
version=VERSION,
|
||||
maintainer='pukkandan',
|
||||
maintainer_email='pukkandan.ytdlp@gmail.com',
|
||||
description=DESCRIPTION,
|
||||
long_description=LONG_DESCRIPTION,
|
||||
long_description_content_type='text/markdown',
|
||||
url='https://github.com/yt-dlp/yt-dlp',
|
||||
packages=packages(),
|
||||
install_requires=REQUIREMENTS,
|
||||
python_requires='>=3.8',
|
||||
project_urls={
|
||||
'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
|
||||
'Source': 'https://github.com/yt-dlp/yt-dlp',
|
||||
'Tracker': 'https://github.com/yt-dlp/yt-dlp/issues',
|
||||
'Funding': 'https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators',
|
||||
},
|
||||
classifiers=[
|
||||
'Topic :: Multimedia :: Video',
|
||||
'Development Status :: 5 - Production/Stable',
|
||||
'Environment :: Console',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
'Programming Language :: Python :: 3.10',
|
||||
'Programming Language :: Python :: 3.11',
|
||||
'Programming Language :: Python :: 3.12',
|
||||
'Programming Language :: Python :: Implementation',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
'Programming Language :: Python :: Implementation :: PyPy',
|
||||
'License :: Public Domain',
|
||||
'Operating System :: OS Independent',
|
||||
],
|
||||
cmdclass={'build_lazy_extractors': build_lazy_extractors},
|
||||
**params
|
||||
)
|
||||
|
||||
|
||||
main()
|
||||
@ -0,0 +1,303 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import float_or_none, int_or_none, parse_iso8601, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class Art19IE(InfoExtractor):
|
||||
_UUID_REGEX = r'[\da-f]{8}-?[\da-f]{4}-?[\da-f]{4}-?[\da-f]{4}-?[\da-f]{12}'
|
||||
_VALID_URL = [
|
||||
rf'https?://(?:www\.)?art19\.com/shows/[^/#?]+/episodes/(?P<id>{_UUID_REGEX})',
|
||||
rf'https?://rss\.art19\.com/episodes/(?P<id>{_UUID_REGEX})\.mp3',
|
||||
]
|
||||
_EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL[0]})']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://rss.art19.com/episodes/5ba1413c-48b8-472b-9cc3-cfd952340bdb.mp3',
|
||||
'info_dict': {
|
||||
'id': '5ba1413c-48b8-472b-9cc3-cfd952340bdb',
|
||||
'ext': 'mp3',
|
||||
'title': 'Why Did DeSantis Drop Out?',
|
||||
'series': 'The Daily Briefing',
|
||||
'release_timestamp': 1705941275,
|
||||
'description': 'md5:da38961da4a3f7e419471365e3c6b49f',
|
||||
'episode': 'Episode 582',
|
||||
'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
|
||||
'series_id': 'ed52a0ab-08b1-4def-8afc-549e4d93296d',
|
||||
'upload_date': '20240122',
|
||||
'timestamp': 1705940815,
|
||||
'episode_number': 582,
|
||||
'modified_date': '20240122',
|
||||
'episode_id': '5ba1413c-48b8-472b-9cc3-cfd952340bdb',
|
||||
'modified_timestamp': 1705941275,
|
||||
'release_date': '20240122',
|
||||
'duration': 527.4,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://art19.com/shows/scamfluencers/episodes/8319b776-4153-4d22-8630-631f204a03dd',
|
||||
'info_dict': {
|
||||
'id': '8319b776-4153-4d22-8630-631f204a03dd',
|
||||
'ext': 'mp3',
|
||||
'title': 'Martha Stewart: The Homemaker Hustler Part 2',
|
||||
'modified_date': '20240116',
|
||||
'upload_date': '20240105',
|
||||
'modified_timestamp': 1705435802,
|
||||
'episode_id': '8319b776-4153-4d22-8630-631f204a03dd',
|
||||
'series_id': 'd3c9b8ca-26b3-42f4-9bd8-21d1a9031e75',
|
||||
'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
|
||||
'description': 'md5:4aa7cfd1358dc57e729835bc208d7893',
|
||||
'release_timestamp': 1705305660,
|
||||
'release_date': '20240115',
|
||||
'timestamp': 1704481536,
|
||||
'episode_number': 88,
|
||||
'series': 'Scamfluencers',
|
||||
'duration': 2588.37501,
|
||||
'episode': 'Episode 88',
|
||||
},
|
||||
}]
|
||||
_WEBPAGE_TESTS = [{
|
||||
'url': 'https://www.nu.nl/formule-1/6291456/verstappen-wordt-een-synoniem-voor-formule-1.html',
|
||||
'info_dict': {
|
||||
'id': '7d42626a-7301-47db-bb8a-3b6f054d77d7',
|
||||
'ext': 'mp3',
|
||||
'title': "'Verstappen wordt een synoniem voor Formule 1'",
|
||||
'season': 'Seizoen 6',
|
||||
'description': 'md5:39a7159a31c4cda312b2e893bdd5c071',
|
||||
'episode_id': '7d42626a-7301-47db-bb8a-3b6f054d77d7',
|
||||
'duration': 3061.82111,
|
||||
'series_id': '93f4e113-2a60-4609-a564-755058fa40d8',
|
||||
'release_date': '20231126',
|
||||
'modified_timestamp': 1701156004,
|
||||
'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
|
||||
'season_number': 6,
|
||||
'episode_number': 52,
|
||||
'modified_date': '20231128',
|
||||
'upload_date': '20231126',
|
||||
'timestamp': 1701025981,
|
||||
'season_id': '36097c1e-7455-490d-a2fe-e2f10b4d5f26',
|
||||
'series': 'De Boordradio',
|
||||
'release_timestamp': 1701026308,
|
||||
'episode': 'Episode 52',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.wishtv.com/podcast-episode/larry-bucshon-announces-retirement-from-congress/',
|
||||
'info_dict': {
|
||||
'id': '8da368bd-08d1-46d0-afaa-c134a4af7dc0',
|
||||
'ext': 'mp3',
|
||||
'title': 'Larry Bucshon announces retirement from congress',
|
||||
'upload_date': '20240115',
|
||||
'episode_number': 148,
|
||||
'episode': 'Episode 148',
|
||||
'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$',
|
||||
'release_date': '20240115',
|
||||
'timestamp': 1705328205,
|
||||
'release_timestamp': 1705329275,
|
||||
'series': 'All INdiana Politics',
|
||||
'modified_date': '20240117',
|
||||
'modified_timestamp': 1705458901,
|
||||
'series_id': 'c4af6c27-b10f-4ff2-9f84-0f407df86ff1',
|
||||
'episode_id': '8da368bd-08d1-46d0-afaa-c134a4af7dc0',
|
||||
'description': 'md5:53b5239e4d14973a87125c217c255b2a',
|
||||
'duration': 1256.18848,
|
||||
},
|
||||
}]
|
||||
|
||||
@classmethod
|
||||
def _extract_embed_urls(cls, url, webpage):
|
||||
yield from super()._extract_embed_urls(url, webpage)
|
||||
for episode_id in re.findall(
|
||||
rf'<div[^>]+\bclass=[\'"][^\'"]*art19-web-player[^\'"]*[\'"][^>]+\bdata-episode-id=[\'"]({cls._UUID_REGEX})[\'"]', webpage):
|
||||
yield f'https://rss.art19.com/episodes/{episode_id}.mp3'
|
||||
|
||||
def _real_extract(self, url):
|
||||
episode_id = self._match_id(url)
|
||||
|
||||
player_metadata = self._download_json(
|
||||
f'https://art19.com/episodes/{episode_id}', episode_id,
|
||||
note='Downloading player metadata', fatal=False,
|
||||
headers={'Accept': 'application/vnd.art19.v0+json'})
|
||||
rss_metadata = self._download_json(
|
||||
f'https://rss.art19.com/episodes/{episode_id}.json', episode_id, fatal=False,
|
||||
note='Downloading RSS metadata')
|
||||
|
||||
formats = [{
|
||||
'format_id': 'direct',
|
||||
'url': f'https://rss.art19.com/episodes/{episode_id}.mp3',
|
||||
'vcodec': 'none',
|
||||
'acodec': 'mp3',
|
||||
}]
|
||||
for fmt_id, fmt_data in traverse_obj(rss_metadata, ('content', 'media', {dict.items}, ...)):
|
||||
if fmt_id == 'waveform_bin':
|
||||
continue
|
||||
fmt_url = traverse_obj(fmt_data, ('url', {url_or_none}))
|
||||
if not fmt_url:
|
||||
continue
|
||||
formats.append({
|
||||
'format_id': fmt_id,
|
||||
'url': fmt_url,
|
||||
'vcodec': 'none',
|
||||
'acodec': fmt_id,
|
||||
'quality': -2 if fmt_id == 'ogg' else -1,
|
||||
})
|
||||
|
||||
return {
|
||||
'id': episode_id,
|
||||
'formats': formats,
|
||||
**traverse_obj(player_metadata, ('episode', {
|
||||
'title': ('title', {str}),
|
||||
'description': ('description_plain', {str}),
|
||||
'episode_id': ('id', {str}),
|
||||
'episode_number': ('episode_number', {int_or_none}),
|
||||
'season_id': ('season_id', {str}),
|
||||
'series_id': ('series_id', {str}),
|
||||
'timestamp': ('created_at', {parse_iso8601}),
|
||||
'release_timestamp': ('released_at', {parse_iso8601}),
|
||||
'modified_timestamp': ('updated_at', {parse_iso8601})
|
||||
})),
|
||||
**traverse_obj(rss_metadata, ('content', {
|
||||
'title': ('episode_title', {str}),
|
||||
'description': ('episode_description_plain', {str}),
|
||||
'episode_id': ('episode_id', {str}),
|
||||
'episode_number': ('episode_number', {int_or_none}),
|
||||
'season': ('season_title', {str}),
|
||||
'season_id': ('season_id', {str}),
|
||||
'season_number': ('season_number', {int_or_none}),
|
||||
'series': ('series_title', {str}),
|
||||
'series_id': ('series_id', {str}),
|
||||
'thumbnail': ('cover_image', {url_or_none}),
|
||||
'duration': ('duration', {float_or_none}),
|
||||
})),
|
||||
}
|
||||
|
||||
|
||||
class Art19ShowIE(InfoExtractor):
|
||||
_VALID_URL_BASE = r'https?://(?:www\.)?art19\.com/shows/(?P<id>[\w-]+)(?:/embed)?/?'
|
||||
_VALID_URL = [
|
||||
rf'{_VALID_URL_BASE}(?:$|[#?])',
|
||||
r'https?://rss\.art19\.com/(?P<id>[\w-]+)/?(?:$|[#?])',
|
||||
]
|
||||
_EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL_BASE}[^\'"])']
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.art19.com/shows/5898c087-a14f-48dc-b6fc-a2280a1ff6e0/',
|
||||
'info_dict': {
|
||||
'_type': 'playlist',
|
||||
'id': '5898c087-a14f-48dc-b6fc-a2280a1ff6e0',
|
||||
'display_id': 'echt-gebeurd',
|
||||
'title': 'Echt Gebeurd',
|
||||
'description': 'md5:5fd11dc80b76e51ffd34b6067fd5e560',
|
||||
'timestamp': 1492642167,
|
||||
'upload_date': '20170419',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
'tags': 'count:7',
|
||||
},
|
||||
'playlist_mincount': 425,
|
||||
}, {
|
||||
'url': 'https://www.art19.com/shows/echt-gebeurd',
|
||||
'info_dict': {
|
||||
'_type': 'playlist',
|
||||
'id': '5898c087-a14f-48dc-b6fc-a2280a1ff6e0',
|
||||
'display_id': 'echt-gebeurd',
|
||||
'title': 'Echt Gebeurd',
|
||||
'description': 'md5:5fd11dc80b76e51ffd34b6067fd5e560',
|
||||
'timestamp': 1492642167,
|
||||
'upload_date': '20170419',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
'tags': 'count:7',
|
||||
},
|
||||
'playlist_mincount': 425,
|
||||
}, {
|
||||
'url': 'https://rss.art19.com/scamfluencers',
|
||||
'info_dict': {
|
||||
'_type': 'playlist',
|
||||
'id': 'd3c9b8ca-26b3-42f4-9bd8-21d1a9031e75',
|
||||
'display_id': 'scamfluencers',
|
||||
'title': 'Scamfluencers',
|
||||
'description': 'md5:7d239d670c0ced6dadbf71c4caf764b7',
|
||||
'timestamp': 1647368573,
|
||||
'upload_date': '20220315',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
'tags': [],
|
||||
},
|
||||
'playlist_mincount': 90,
|
||||
}, {
|
||||
'url': 'https://art19.com/shows/enthuellt/embed',
|
||||
'info_dict': {
|
||||
'_type': 'playlist',
|
||||
'id': 'e2cacf57-bb8a-4263-aa81-719bcdd4f80c',
|
||||
'display_id': 'enthuellt',
|
||||
'title': 'Enthüllt',
|
||||
'description': 'md5:17752246643414a2fd51744fc9a1c08e',
|
||||
'timestamp': 1601645860,
|
||||
'upload_date': '20201002',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
'tags': 'count:10',
|
||||
},
|
||||
'playlist_mincount': 10,
|
||||
}]
|
||||
_WEBPAGE_TESTS = [{
|
||||
'url': 'https://deconstructingyourself.com/deconstructing-yourself-podcast',
|
||||
'info_dict': {
|
||||
'_type': 'playlist',
|
||||
'id': 'cfbb9b01-c295-4adb-8726-adde7c03cf21',
|
||||
'display_id': 'deconstructing-yourself',
|
||||
'title': 'Deconstructing Yourself',
|
||||
'description': 'md5:dab5082b28b248a35476abf64768854d',
|
||||
'timestamp': 1570581181,
|
||||
'upload_date': '20191009',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
'tags': 'count:5',
|
||||
},
|
||||
'playlist_mincount': 80,
|
||||
}, {
|
||||
'url': 'https://chicagoreader.com/columns-opinion/podcasts/ben-joravsky-show-podcast-episodes/',
|
||||
'info_dict': {
|
||||
'_type': 'playlist',
|
||||
'id': '9dfa2c37-ab87-4c13-8388-4897914313ec',
|
||||
'display_id': 'the-ben-joravsky-show',
|
||||
'title': 'The Ben Joravsky Show',
|
||||
'description': 'md5:c0f3ec0ee0dbea764390e521adc8780a',
|
||||
'timestamp': 1550875095,
|
||||
'upload_date': '20190222',
|
||||
'modified_timestamp': int,
|
||||
'modified_date': str,
|
||||
'tags': ['Chicago Politics', 'chicago', 'Ben Joravsky'],
|
||||
},
|
||||
'playlist_mincount': 1900,
|
||||
}]
|
||||
|
||||
@classmethod
|
||||
def _extract_embed_urls(cls, url, webpage):
|
||||
yield from super()._extract_embed_urls(url, webpage)
|
||||
for series_id in re.findall(
|
||||
r'<div[^>]+\bclass=[\'"][^\'"]*art19-web-player[^\'"]*[\'"][^>]+\bdata-series-id=[\'"]([\w-]+)[\'"]', webpage):
|
||||
yield f'https://art19.com/shows/{series_id}'
|
||||
|
||||
def _real_extract(self, url):
|
||||
series_id = self._match_id(url)
|
||||
series_metadata = self._download_json(
|
||||
f'https://art19.com/series/{series_id}', series_id, note='Downloading series metadata',
|
||||
headers={'Accept': 'application/vnd.art19.v0+json'})
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'entries': [
|
||||
self.url_result(f'https://rss.art19.com/episodes/{episode_id}.mp3', Art19IE)
|
||||
for episode_id in traverse_obj(series_metadata, ('series', 'episode_ids', ..., {str}))
|
||||
],
|
||||
**traverse_obj(series_metadata, ('series', {
|
||||
'id': ('id', {str}),
|
||||
'display_id': ('slug', {str}),
|
||||
'title': ('title', {str}),
|
||||
'description': ('description_plain', {str}),
|
||||
'timestamp': ('created_at', {parse_iso8601}),
|
||||
'modified_timestamp': ('updated_at', {parse_iso8601}),
|
||||
})),
|
||||
'tags': traverse_obj(series_metadata, ('tags', ..., 'name', {str})),
|
||||
}
|
||||
@ -0,0 +1,79 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class CloudyCDNIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:https?:)?//embed\.cloudycdn\.services/(?P<site_id>[^/?#]+)/media/(?P<id>[\w-]+)'
|
||||
_EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL})']
|
||||
_TESTS = [{
|
||||
'url': 'https://embed.cloudycdn.services/ltv/media/46k_d23-6000-105?',
|
||||
'md5': '64f72a360ca530d5ed89c77646c9eee5',
|
||||
'info_dict': {
|
||||
'id': '46k_d23-6000-105',
|
||||
'ext': 'mp4',
|
||||
'timestamp': 1700589151,
|
||||
'duration': 1442,
|
||||
'upload_date': '20231121',
|
||||
'title': 'D23-6000-105_cetstud',
|
||||
'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://embed.cloudycdn.services/izm/media/26e_lv-8-5-1',
|
||||
'md5': '798828a479151e2444d8dcfbec76e482',
|
||||
'info_dict': {
|
||||
'id': '26e_lv-8-5-1',
|
||||
'ext': 'mp4',
|
||||
'title': 'LV-8-5-1',
|
||||
'timestamp': 1669767167,
|
||||
'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/488306/placeholder1679423604.jpg',
|
||||
'duration': 1205,
|
||||
'upload_date': '20221130',
|
||||
}
|
||||
}]
|
||||
_WEBPAGE_TESTS = [{
|
||||
'url': 'https://www.tavaklase.lv/video/es-esmu-mina-um-2/',
|
||||
'md5': '63074e8e6c84ac2a01f2fb8bf03b8f43',
|
||||
'info_dict': {
|
||||
'id': 'cqd_lib-2',
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20230223',
|
||||
'duration': 629,
|
||||
'thumbnail': 'https://store.cloudycdn.services/tmsp00120/assets/media/518407/placeholder1678748124.jpg',
|
||||
'timestamp': 1677181513,
|
||||
'title': 'LIB-2',
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
site_id, video_id = self._match_valid_url(url).group('site_id', 'id')
|
||||
|
||||
data = self._download_json(
|
||||
f'https://player.cloudycdn.services/player/{site_id}/media/{video_id}/',
|
||||
video_id, data=urlencode_postdata({
|
||||
'version': '6.4.0',
|
||||
'referer': url,
|
||||
}))
|
||||
|
||||
formats, subtitles = [], {}
|
||||
for m3u8_url in traverse_obj(data, ('source', 'sources', ..., 'src', {url_or_none})):
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, fatal=False)
|
||||
formats.extend(fmts)
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(data, {
|
||||
'title': ('name', {str}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'timestamp': ('upload_date', {parse_iso8601}),
|
||||
'thumbnail': ('source', 'poster', {url_or_none}),
|
||||
}),
|
||||
}
|
||||
@ -1,68 +1,97 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import smuggle_url
|
||||
from ..utils import int_or_none, parse_iso8601, str_or_none, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class CNBCIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://video\.cnbc\.com/gallery/\?video=(?P<id>[0-9]+)'
|
||||
_TEST = {
|
||||
'url': 'http://video.cnbc.com/gallery/?video=3000503714',
|
||||
class CNBCVideoIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?cnbc\.com/video/(?:[^/?#]+/)+(?P<id>[^./?#&]+)\.html'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://www.cnbc.com/video/2023/12/07/mcdonalds-just-unveiled-cosmcsits-new-spinoff-brand.html',
|
||||
'info_dict': {
|
||||
'id': '3000503714',
|
||||
'ext': 'mp4',
|
||||
'title': 'Fighting zombies is big business',
|
||||
'description': 'md5:0c100d8e1a7947bd2feec9a5550e519e',
|
||||
'timestamp': 1459332000,
|
||||
'upload_date': '20160330',
|
||||
'uploader': 'NBCU-CNBC',
|
||||
'id': '107344774',
|
||||
'display_id': 'mcdonalds-just-unveiled-cosmcsits-new-spinoff-brand',
|
||||
'modified_timestamp': 1702053483,
|
||||
'timestamp': 1701977810,
|
||||
'channel': 'News Videos',
|
||||
'upload_date': '20231207',
|
||||
'description': 'md5:882c001d85cb43d7579b514307b3e78b',
|
||||
'release_timestamp': 1701977375,
|
||||
'modified_date': '20231208',
|
||||
'release_date': '20231207',
|
||||
'duration': 65,
|
||||
'author': 'Sean Conlon',
|
||||
'title': 'Here\'s a first look at McDonald\'s new spinoff brand, CosMc\'s',
|
||||
'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107344192-1701894812493-CosMcsskyHero_2336x1040_hero-desktop.jpg?v=1701894855',
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
},
|
||||
'skip': 'Dead link',
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'ie_key': 'ThePlatform',
|
||||
'url': smuggle_url(
|
||||
'http://link.theplatform.com/s/gZWlPC/media/guid/2408950221/%s?mbr=true&manifest=m3u' % video_id,
|
||||
{'force_smil_url': True}),
|
||||
'id': video_id,
|
||||
}
|
||||
|
||||
|
||||
class CNBCVideoIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?cnbc\.com(?P<path>/video/(?:[^/]+/)+(?P<id>[^./?#&]+)\.html)'
|
||||
_TEST = {
|
||||
'url': 'https://www.cnbc.com/video/2018/07/19/trump-i-dont-necessarily-agree-with-raising-rates.html',
|
||||
'expected_warnings': ['Unable to download f4m manifest'],
|
||||
}, {
|
||||
'url': 'https://www.cnbc.com/video/2023/12/08/jim-cramer-shares-his-take-on-seattles-tech-scene.html',
|
||||
'info_dict': {
|
||||
'id': '7000031301',
|
||||
'author': 'Jim Cramer',
|
||||
'channel': 'Mad Money with Jim Cramer',
|
||||
'description': 'md5:72925be21b952e95eba51178dddf4e3e',
|
||||
'duration': 299.0,
|
||||
'ext': 'mp4',
|
||||
'title': "Trump: I don't necessarily agree with raising rates",
|
||||
'description': 'md5:878d8f0b4ebb5bb1dda3514b91b49de3',
|
||||
'timestamp': 1531958400,
|
||||
'upload_date': '20180719',
|
||||
'uploader': 'NBCU-CNBC',
|
||||
'id': '107345451',
|
||||
'display_id': 'jim-cramer-shares-his-take-on-seattles-tech-scene',
|
||||
'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107345481-1702079431MM-B-120823.jpg?v=1702079430',
|
||||
'timestamp': 1702080139,
|
||||
'title': 'Jim Cramer shares his take on Seattle\'s tech scene',
|
||||
'release_date': '20231208',
|
||||
'upload_date': '20231209',
|
||||
'modified_timestamp': 1702080139,
|
||||
'modified_date': '20231209',
|
||||
'release_timestamp': 1702073551,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
'expected_warnings': ['Unable to download f4m manifest'],
|
||||
}, {
|
||||
'url': 'https://www.cnbc.com/video/2023/12/08/the-epicenter-of-ai-is-in-seattle-says-jim-cramer.html',
|
||||
'info_dict': {
|
||||
'author': 'Jim Cramer',
|
||||
'channel': 'Mad Money with Jim Cramer',
|
||||
'description': 'md5:72925be21b952e95eba51178dddf4e3e',
|
||||
'duration': 113.0,
|
||||
'ext': 'mp4',
|
||||
'id': '107345474',
|
||||
'display_id': 'the-epicenter-of-ai-is-in-seattle-says-jim-cramer',
|
||||
'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107345486-Screenshot_2023-12-08_at_70339_PM.png?v=1702080248',
|
||||
'timestamp': 1702080535,
|
||||
'title': 'The epicenter of AI is in Seattle, says Jim Cramer',
|
||||
'release_timestamp': 1702077347,
|
||||
'modified_timestamp': 1702080535,
|
||||
'release_date': '20231208',
|
||||
'upload_date': '20231209',
|
||||
'modified_date': '20231209',
|
||||
},
|
||||
'skip': 'Dead link',
|
||||
}
|
||||
'expected_warnings': ['Unable to download f4m manifest'],
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
path, display_id = self._match_valid_url(url).groups()
|
||||
video_id = self._download_json(
|
||||
'https://webql-redesign.cnbcfm.com/graphql', display_id, query={
|
||||
'query': '''{
|
||||
page(path: "%s") {
|
||||
vcpsId
|
||||
}
|
||||
}''' % path,
|
||||
})['data']['page']['vcpsId']
|
||||
return self.url_result(
|
||||
'http://video.cnbc.com/gallery/?video=%d' % video_id,
|
||||
CNBCIE.ie_key())
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
data = self._search_json(r'window\.__s_data=', webpage, 'video data', display_id)
|
||||
|
||||
player_data = traverse_obj(data, (
|
||||
'page', 'page', 'layout', ..., 'columns', ..., 'modules',
|
||||
lambda _, v: v['name'] == 'clipPlayer', 'data', {dict}), get_all=False)
|
||||
|
||||
return {
|
||||
'id': display_id,
|
||||
'display_id': display_id,
|
||||
'formats': self._extract_akamai_formats(player_data['playbackURL'], display_id),
|
||||
**self._search_json_ld(webpage, display_id, fatal=False),
|
||||
**traverse_obj(player_data, {
|
||||
'id': ('id', {str_or_none}),
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {str}),
|
||||
'author': ('author', ..., 'name', {str}),
|
||||
'timestamp': ('datePublished', {parse_iso8601}),
|
||||
'release_timestamp': ('uploadDate', {parse_iso8601}),
|
||||
'modified_timestamp': ('dateLastPublished', {parse_iso8601}),
|
||||
'thumbnail': ('thumbnail', {url_or_none}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
'channel': ('section', 'title', {str}),
|
||||
}, get_all=False),
|
||||
}
|
||||
|
||||
@ -0,0 +1,224 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
int_or_none,
|
||||
str_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class ERRJupiterIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:jupiter(?:pluss)?|lasteekraan)\.err\.ee/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'note': 'Jupiter: Movie: siin-me-oleme',
|
||||
'url': 'https://jupiter.err.ee/1211107/siin-me-oleme',
|
||||
'md5': '9b45d1682a98853acaa1e1b0c791f425',
|
||||
'info_dict': {
|
||||
'id': '1211107',
|
||||
'ext': 'mp4',
|
||||
'title': 'Siin me oleme!',
|
||||
'alt_title': '',
|
||||
'description': 'md5:1825b795f5f7584241aeb59e5bbb4f70',
|
||||
'release_date': '20231226',
|
||||
'upload_date': '20201217',
|
||||
'modified_date': '20201217',
|
||||
'release_timestamp': 1703577600,
|
||||
'timestamp': 1608210000,
|
||||
'modified_timestamp': 1608220800,
|
||||
'release_year': 1978,
|
||||
},
|
||||
}, {
|
||||
'note': 'Jupiter: Series: Impulss',
|
||||
'url': 'https://jupiter.err.ee/1609145945/impulss',
|
||||
'md5': 'a378486df07ed1ba74e46cc861886243',
|
||||
'info_dict': {
|
||||
'id': '1609145945',
|
||||
'ext': 'mp4',
|
||||
'title': 'Impulss',
|
||||
'alt_title': 'Loteriipilet hooldekodusse',
|
||||
'description': 'md5:fa8a2ed0cdccb130211513443ee4d571',
|
||||
'release_date': '20231107',
|
||||
'upload_date': '20231026',
|
||||
'modified_date': '20231118',
|
||||
'release_timestamp': 1699380000,
|
||||
'timestamp': 1698327601,
|
||||
'modified_timestamp': 1700311802,
|
||||
'series': 'Impulss',
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'episode': 'Loteriipilet hooldekodusse',
|
||||
'episode_number': 6,
|
||||
'series_id': '1609108187',
|
||||
'release_year': 2023,
|
||||
'episode_id': '1609145945',
|
||||
},
|
||||
}, {
|
||||
'note': 'Jupiter: Radio Show: mnemoturniir episode',
|
||||
'url': 'https://jupiter.err.ee/1037919/mnemoturniir',
|
||||
'md5': 'f1eb95fe66f9620ff84e81bbac37076a',
|
||||
'info_dict': {
|
||||
'id': '1037919',
|
||||
'ext': 'm4a',
|
||||
'title': 'Mnemoturniir',
|
||||
'alt_title': '',
|
||||
'description': 'md5:626db52394e7583c26ab74d6a34d9982',
|
||||
'release_date': '20240121',
|
||||
'upload_date': '20240108',
|
||||
'modified_date': '20240121',
|
||||
'release_timestamp': 1705827900,
|
||||
'timestamp': 1704675602,
|
||||
'modified_timestamp': 1705827601,
|
||||
'series': 'Mnemoturniir',
|
||||
'season': 'Season 0',
|
||||
'season_number': 0,
|
||||
'episode': 'Episode 0',
|
||||
'episode_number': 0,
|
||||
'series_id': '1037919',
|
||||
'release_year': 2024,
|
||||
'episode_id': '1609215101',
|
||||
},
|
||||
}, {
|
||||
'note': 'Jupiter+: Clip: bolee-zelenyj-tallinn',
|
||||
'url': 'https://jupiterpluss.err.ee/1609180445/bolee-zelenyj-tallinn',
|
||||
'md5': '1b812270c4daf6ce51c06bfeaf33ed95',
|
||||
'info_dict': {
|
||||
'id': '1609180445',
|
||||
'ext': 'mp4',
|
||||
'title': 'Более зеленый Таллинн',
|
||||
'alt_title': '',
|
||||
'description': 'md5:fd34d9bf939c28c4a725b19a7f0d6320',
|
||||
'release_date': '20231224',
|
||||
'upload_date': '20231130',
|
||||
'modified_date': '20231207',
|
||||
'release_timestamp': 1703423400,
|
||||
'timestamp': 1701338400,
|
||||
'modified_timestamp': 1701967200,
|
||||
'release_year': 2023,
|
||||
},
|
||||
}, {
|
||||
'note': 'Jupiter+: Series: The Sniffer',
|
||||
'url': 'https://jupiterpluss.err.ee/1608311387/njuhach',
|
||||
'md5': '2abdeb7131ce551bce49e8d0cea08536',
|
||||
'info_dict': {
|
||||
'id': '1608311387',
|
||||
'ext': 'mp4',
|
||||
'title': 'Нюхач',
|
||||
'alt_title': '',
|
||||
'description': 'md5:8c5c7d8f32ec6e54cd498c9e59ca83bc',
|
||||
'release_date': '20230601',
|
||||
'upload_date': '20210818',
|
||||
'modified_date': '20210903',
|
||||
'release_timestamp': 1685633400,
|
||||
'timestamp': 1629318000,
|
||||
'modified_timestamp': 1630686000,
|
||||
'release_year': 2013,
|
||||
'episode': 'Episode 1',
|
||||
'episode_id': '1608311390',
|
||||
'episode_number': 1,
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'series': 'Нюхач',
|
||||
'series_id': '1608311387',
|
||||
},
|
||||
}, {
|
||||
'note': 'Jupiter+: Podcast: lesnye-istorii-aisty',
|
||||
'url': 'https://jupiterpluss.err.ee/1608990335/lesnye-istorii-aisty',
|
||||
'md5': '8b46d7e4510b254a14b7a52211b5bf96',
|
||||
'info_dict': {
|
||||
'id': '1608990335',
|
||||
'ext': 'm4a',
|
||||
'title': 'Лесные истории | Аисты',
|
||||
'alt_title': '',
|
||||
'description': 'md5:065e721623e271e7a63e6540d409ca6b',
|
||||
'release_date': '20230609',
|
||||
'upload_date': '20230527',
|
||||
'modified_date': '20230608',
|
||||
'release_timestamp': 1686308700,
|
||||
'timestamp': 1685145600,
|
||||
'modified_timestamp': 1686252600,
|
||||
'release_year': 2023,
|
||||
'episode': 'Episode 0',
|
||||
'episode_id': '1608990335',
|
||||
'episode_number': 0,
|
||||
'season': 'Season 0',
|
||||
'season_number': 0,
|
||||
'series': 'Лесные истории | Аисты',
|
||||
'series_id': '1037497',
|
||||
}
|
||||
}, {
|
||||
'note': 'Lasteekraan: Pätu',
|
||||
'url': 'https://lasteekraan.err.ee/1092243/patu',
|
||||
'md5': 'a67eb9b9bcb3d201718c15d1638edf77',
|
||||
'info_dict': {
|
||||
'id': '1092243',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pätu',
|
||||
'alt_title': '',
|
||||
'description': 'md5:64a7b5a80afd7042d3f8ec48c77befd9',
|
||||
'release_date': '20230614',
|
||||
'upload_date': '20200520',
|
||||
'modified_date': '20200520',
|
||||
'release_timestamp': 1686745800,
|
||||
'timestamp': 1589975640,
|
||||
'modified_timestamp': 1589975640,
|
||||
'release_year': 1990,
|
||||
'episode': 'Episode 1',
|
||||
'episode_id': '1092243',
|
||||
'episode_number': 1,
|
||||
'season': 'Season 1',
|
||||
'season_number': 1,
|
||||
'series': 'Pätu',
|
||||
'series_id': '1092236',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
data = self._download_json(
|
||||
'https://services.err.ee/api/v2/vodContent/getContentPageData', video_id,
|
||||
query={'contentId': video_id})['data']['mainContent']
|
||||
|
||||
media_data = traverse_obj(data, ('medias', ..., {dict}), get_all=False)
|
||||
if traverse_obj(media_data, ('restrictions', 'drm', {bool})):
|
||||
self.report_drm(video_id)
|
||||
|
||||
formats, subtitles = [], {}
|
||||
for format_url in set(traverse_obj(media_data, ('src', ('hls', 'hls2', 'hlsNew'), {url_or_none}))):
|
||||
fmts, subs = self._extract_m3u8_formats_and_subtitles(
|
||||
format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
|
||||
formats.extend(fmts)
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
for format_url in set(traverse_obj(media_data, ('src', ('dash', 'dashNew'), {url_or_none}))):
|
||||
fmts, subs = self._extract_mpd_formats_and_subtitles(
|
||||
format_url, video_id, mpd_id='dash', fatal=False)
|
||||
formats.extend(fmts)
|
||||
self._merge_subtitles(subs, target=subtitles)
|
||||
if format_url := traverse_obj(media_data, ('src', 'file', {url_or_none})):
|
||||
formats.append({
|
||||
'url': format_url,
|
||||
'format_id': 'http',
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(data, {
|
||||
'title': ('heading', {str}),
|
||||
'alt_title': ('subHeading', {str}),
|
||||
'description': (('lead', 'body'), {clean_html}, {lambda x: x or None}),
|
||||
'timestamp': ('created', {int_or_none}),
|
||||
'modified_timestamp': ('updated', {int_or_none}),
|
||||
'release_timestamp': (('scheduleStart', 'publicStart'), {int_or_none}),
|
||||
'release_year': ('year', {int_or_none}),
|
||||
}, get_all=False),
|
||||
**(traverse_obj(data, {
|
||||
'series': ('heading', {str}),
|
||||
'series_id': ('rootContentId', {str_or_none}),
|
||||
'episode': ('subHeading', {str}),
|
||||
'season_number': ('season', {int_or_none}),
|
||||
'episode_number': ('episode', {int_or_none}),
|
||||
'episode_id': ('id', {str_or_none}),
|
||||
}) if data.get('type') == 'episode' else {}),
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
from .common import InfoExtractor
|
||||
from ..networking.exceptions import HTTPError
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
UserNotLive,
|
||||
parse_iso8601,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class FlexTVIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?flextv\.co\.kr/channels/(?P<id>\d+)/live'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.flextv.co.kr/channels/231638/live',
|
||||
'info_dict': {
|
||||
'id': '231638',
|
||||
'ext': 'mp4',
|
||||
'title': r're:^214하나만\.\.\. ',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'upload_date': r're:\d{8}',
|
||||
'timestamp': int,
|
||||
'live_status': 'is_live',
|
||||
'channel': 'Hi별',
|
||||
'channel_id': '244396',
|
||||
},
|
||||
'skip': 'The channel is offline',
|
||||
}, {
|
||||
'url': 'https://www.flextv.co.kr/channels/746/live',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel_id = self._match_id(url)
|
||||
|
||||
try:
|
||||
stream_data = self._download_json(
|
||||
f'https://api.flextv.co.kr/api/channels/{channel_id}/stream',
|
||||
channel_id, query={'option': 'all'})
|
||||
except ExtractorError as e:
|
||||
if isinstance(e.cause, HTTPError) and e.cause.status == 400:
|
||||
raise UserNotLive(video_id=channel_id)
|
||||
raise
|
||||
|
||||
playlist_url = stream_data['sources'][0]['url']
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
playlist_url, channel_id, 'mp4')
|
||||
|
||||
return {
|
||||
'id': channel_id,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'is_live': True,
|
||||
**traverse_obj(stream_data, {
|
||||
'title': ('stream', 'title', {str}),
|
||||
'timestamp': ('stream', 'createdAt', {parse_iso8601}),
|
||||
'thumbnail': ('thumbUrl', {url_or_none}),
|
||||
'channel': ('owner', 'name', {str}),
|
||||
'channel_id': ('owner', 'id', {str_or_none}),
|
||||
}),
|
||||
}
|
||||
@ -0,0 +1,282 @@
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
determine_ext,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
parse_iso8601,
|
||||
parse_qs,
|
||||
str_or_none,
|
||||
url_or_none,
|
||||
urljoin,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class LSMLREmbedIE(InfoExtractor):
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?:
|
||||
(?:latvijasradio|lr1|lr2|klasika|lr4|naba|radioteatris)\.lsm|
|
||||
pieci
|
||||
)\.lv/[^/?#]+/(?:
|
||||
pleijeris|embed
|
||||
)/?\?(?:[^#]+&)?(?:show|id)=(?P<id>\d+)'''
|
||||
_TESTS = [{
|
||||
'url': 'https://latvijasradio.lsm.lv/lv/embed/?theme=black&size=16x9&showCaptions=0&id=183522',
|
||||
'md5': '719b33875cd1429846eeeaeec6df2830',
|
||||
'info_dict': {
|
||||
'id': 'a342781',
|
||||
'ext': 'mp3',
|
||||
'duration': 1823,
|
||||
'title': '#138 Nepilnīgā kompensējamo zāļu sistēma pat mēnešiem dzenā pacientus pa aptiekām',
|
||||
'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/9/d/gallery_fd4675ac.jpg',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://radioteatris.lsm.lv/lv/embed/?id=&show=1270&theme=white&size=16x9',
|
||||
'info_dict': {
|
||||
'id': '1270',
|
||||
},
|
||||
'playlist_count': 3,
|
||||
'playlist': [{
|
||||
'md5': '2e61b6eceff00d14d57fdbbe6ab24cac',
|
||||
'info_dict': {
|
||||
'id': 'a297397',
|
||||
'ext': 'mp3',
|
||||
'title': 'Eriks Emanuels Šmits "Pilāta evaņģēlijs". 1. daļa',
|
||||
'thumbnail': 'https://radioteatris.lsm.lv/public/assets/shows/62f131ae81e3c.jpg',
|
||||
'duration': 3300,
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
'url': 'https://radioteatris.lsm.lv/lv/embed/?id=&show=1269&theme=white&size=16x9',
|
||||
'md5': '24810d4a961da2295d9860afdcaf4f5a',
|
||||
'info_dict': {
|
||||
'id': 'a230690',
|
||||
'ext': 'mp3',
|
||||
'title': 'Jens Ahlboms "Spārni". Radioizrāde ar Mārtiņa Freimaņa mūziku',
|
||||
'thumbnail': 'https://radioteatris.lsm.lv/public/assets/shows/62f13023a457c.jpg',
|
||||
'duration': 1788,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://lr1.lsm.lv/lv/embed/?id=166557&show=0&theme=white&size=16x9',
|
||||
'info_dict': {
|
||||
'id': '166557',
|
||||
},
|
||||
'playlist_count': 2,
|
||||
'playlist': [{
|
||||
'md5': '6a8b0927572f443f09c6e50a3ad65f2d',
|
||||
'info_dict': {
|
||||
'id': 'a303104',
|
||||
'ext': 'mp3',
|
||||
'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/c/5/gallery_a83ad2c2.jpg',
|
||||
'title': 'Krustpunktā Lielā intervija: Valsts prezidents Egils Levits',
|
||||
'duration': 3222,
|
||||
},
|
||||
}, {
|
||||
'md5': '5d5e191e718b7644e5118b7b4e093a6d',
|
||||
'info_dict': {
|
||||
'id': 'v303104',
|
||||
'ext': 'mp4',
|
||||
'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/c/5/gallery_a83ad2c2.jpg',
|
||||
'title': 'Krustpunktā Lielā intervija: Valsts prezidents Egils Levits - Video Version',
|
||||
'duration': 3222,
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
'url': 'https://lr1.lsm.lv/lv/embed/?id=183522&show=0&theme=white&size=16x9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://lr2.lsm.lv/lv/embed/?id=182126&show=0&theme=white&size=16x9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://klasika.lsm.lv/lv/embed/?id=110806&show=0&theme=white&size=16x9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://lr4.lsm.lv/lv/embed/?id=184282&show=0&theme=white&size=16x9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://pieci.lv/lv/embed/?id=168896&show=0&theme=white&size=16x9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://naba.lsm.lv/lv/embed/?id=182901&show=0&theme=white&size=16x9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://radioteatris.lsm.lv/lv/embed/?id=176439&show=0&theme=white&size=16x9',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://lr1.lsm.lv/lv/pleijeris/?embed=0&id=48205&time=00%3A00&idx=0',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
query = parse_qs(url)
|
||||
video_id = traverse_obj(query, (
|
||||
('show', 'id'), 0, {int_or_none}, {lambda x: x or None}, {str_or_none}), get_all=False)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
player_data, media_data = self._search_regex(
|
||||
r'LR\.audio\.Player\s*\([^{]*(?P<player>\{.*?\}),(?P<media>\{.*\})\);',
|
||||
webpage, 'player json', group=('player', 'media'))
|
||||
|
||||
player_json = self._parse_json(
|
||||
player_data, video_id, transform_source=js_to_json, fatal=False) or {}
|
||||
media_json = self._parse_json(media_data, video_id, transform_source=js_to_json)
|
||||
|
||||
entries = []
|
||||
for item in traverse_obj(media_json, (('audio', 'video'), lambda _, v: v['id'])):
|
||||
formats = []
|
||||
for source_url in traverse_obj(item, ('sources', ..., 'file', {url_or_none})):
|
||||
if determine_ext(source_url) == 'm3u8':
|
||||
formats.extend(self._extract_m3u8_formats(source_url, video_id, fatal=False))
|
||||
else:
|
||||
formats.append({'url': source_url})
|
||||
|
||||
id_ = item['id']
|
||||
title = item.get('title')
|
||||
if id_.startswith('v') and not title:
|
||||
title = traverse_obj(
|
||||
media_json, ('audio', lambda _, v: v['id'][1:] == id_[1:], 'title',
|
||||
{lambda x: x and f'{x} - Video Version'}), get_all=False)
|
||||
|
||||
entries.append({
|
||||
'formats': formats,
|
||||
'thumbnail': urljoin(url, player_json.get('poster')),
|
||||
'id': id_,
|
||||
'title': title,
|
||||
'duration': traverse_obj(item, ('duration', {int_or_none})),
|
||||
})
|
||||
|
||||
if len(entries) == 1:
|
||||
return entries[0]
|
||||
|
||||
return self.playlist_result(entries, video_id)
|
||||
|
||||
|
||||
class LSMLTVEmbedIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://ltv\.lsm\.lv/embed\?(?:[^#]+&)?c=(?P<id>[^#&]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://ltv.lsm.lv/embed?c=eyJpdiI6IjQzbHVUeHAyaDJiamFjcjdSUUFKdnc9PSIsInZhbHVlIjoiMHl3SnJNRmd2TmFIdnZwOGtGUUpzODFzUEZ4SVVsN2xoRjliSW9vckUyMWZIWG8vbWVzaFFkY0lhNmRjbjRpaCIsIm1hYyI6ImMzNjdhMzFhNTFhZmY1ZmE0NWI5YmFjZGI1YmJiNGEyNjgzNDM4MjUzMWEwM2FmMDMyZDMwYWM1MDFjZmM5MGIiLCJ0YWciOiIifQ==',
|
||||
'md5': '64f72a360ca530d5ed89c77646c9eee5',
|
||||
'info_dict': {
|
||||
'id': '46k_d23-6000-105',
|
||||
'ext': 'mp4',
|
||||
'timestamp': 1700589151,
|
||||
'duration': 1442,
|
||||
'upload_date': '20231121',
|
||||
'title': 'D23-6000-105_cetstud',
|
||||
'thumbnail': 'https://store.cloudycdn.services/tmsp00060/assets/media/660858/placeholder1700589200.jpg',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://ltv.lsm.lv/embed?enablesdkjs=1&c=eyJpdiI6IncwVzZmUFk2MU12enVWK1I3SUcwQ1E9PSIsInZhbHVlIjoid3FhV29vamc3T2sxL1RaRmJ5Rm1GTXozU0o2dVczdUtLK0cwZEZJMDQ2a3ZIRG5DK2pneGlnbktBQy9uazVleHN6VXhxdWIweWNvcHRDSnlISlNYOHlVZ1lpcTUrcWZSTUZPQW14TVdkMW9aOUtRWVNDcFF4eWpHNGcrT0VZbUNFQStKQk91cGpndW9FVjJIa0lpbkh3PT0iLCJtYWMiOiIyZGI1NDJlMWRlM2QyMGNhOGEwYTM2MmNlN2JlOGRhY2QyYjdkMmEzN2RlOTEzYTVkNzI1ODlhZDlhZjU4MjQ2IiwidGFnIjoiIn0=',
|
||||
'md5': 'a1711e190fe680fdb68fd8413b378e87',
|
||||
'info_dict': {
|
||||
'id': 'wUnFArIPDSY',
|
||||
'ext': 'mp4',
|
||||
'uploader': 'LTV_16plus',
|
||||
'release_date': '20220514',
|
||||
'channel_url': 'https://www.youtube.com/channel/UCNMrnafwXD2XKeeQOyfkFCw',
|
||||
'view_count': int,
|
||||
'availability': 'public',
|
||||
'thumbnail': 'https://i.ytimg.com/vi/wUnFArIPDSY/maxresdefault.jpg',
|
||||
'release_timestamp': 1652544074,
|
||||
'title': 'EIROVĪZIJA SALĀTOS',
|
||||
'live_status': 'was_live',
|
||||
'uploader_id': '@LTV16plus',
|
||||
'comment_count': int,
|
||||
'channel_id': 'UCNMrnafwXD2XKeeQOyfkFCw',
|
||||
'channel_follower_count': int,
|
||||
'categories': ['Entertainment'],
|
||||
'duration': 5269,
|
||||
'upload_date': '20220514',
|
||||
'age_limit': 0,
|
||||
'channel': 'LTV_16plus',
|
||||
'playable_in_embed': True,
|
||||
'tags': [],
|
||||
'uploader_url': 'https://www.youtube.com/@LTV16plus',
|
||||
'like_count': int,
|
||||
'description': 'md5:7ff0c42ba971e3c13e4b8a2ff03b70b5',
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = urllib.parse.unquote(self._match_id(url))
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
data = self._search_json(
|
||||
r'window\.ltvEmbedPayload\s*=', webpage, 'embed json', video_id)
|
||||
embed_type = traverse_obj(data, ('source', 'name', {str}))
|
||||
|
||||
if embed_type == 'telia':
|
||||
ie_key = 'CloudyCDN'
|
||||
embed_url = traverse_obj(data, ('source', 'embed_url', {url_or_none}))
|
||||
elif embed_type == 'youtube':
|
||||
ie_key = 'Youtube'
|
||||
embed_url = traverse_obj(data, ('source', 'id', {str}))
|
||||
else:
|
||||
raise ExtractorError(f'Unsupported embed type {embed_type!r}')
|
||||
|
||||
return self.url_result(
|
||||
embed_url, ie_key, video_id, **traverse_obj(data, {
|
||||
'title': ('parentInfo', 'title'),
|
||||
'duration': ('parentInfo', 'duration', {int_or_none}),
|
||||
'thumbnail': ('source', 'poster', {url_or_none}),
|
||||
}))
|
||||
|
||||
|
||||
class LSMReplayIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://replay\.lsm\.lv/[^/?#]+/(?:ieraksts|statja)/[^/?#]+/(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://replay.lsm.lv/lv/ieraksts/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
|
||||
'md5': '64f72a360ca530d5ed89c77646c9eee5',
|
||||
'info_dict': {
|
||||
'id': '46k_d23-6000-105',
|
||||
'ext': 'mp4',
|
||||
'timestamp': 1700586300,
|
||||
'description': 'md5:0f1b14798cc39e1ae578bd0eb268f759',
|
||||
'duration': 1442,
|
||||
'upload_date': '20231121',
|
||||
'title': '4. studija. Zolitūdes traģēdija un Inčupes stacija',
|
||||
'thumbnail': 'https://ltv.lsm.lv/storage/media/8/7/large/5/1f9604e1.jpg',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://replay.lsm.lv/lv/ieraksts/lr/183522/138-nepilniga-kompensejamo-zalu-sistema-pat-menesiem-dzena-pacientus-pa-aptiekam',
|
||||
'md5': '719b33875cd1429846eeeaeec6df2830',
|
||||
'info_dict': {
|
||||
'id': 'a342781',
|
||||
'ext': 'mp3',
|
||||
'duration': 1823,
|
||||
'title': '#138 Nepilnīgā kompensējamo zāļu sistēma pat mēnešiem dzenā pacientus pa aptiekām',
|
||||
'thumbnail': 'https://pic.latvijasradio.lv/public/assets/media/9/d/large_fd4675ac.jpg',
|
||||
'upload_date': '20231102',
|
||||
'timestamp': 1698921060,
|
||||
'description': 'md5:7bac3b2dd41e44325032943251c357b1',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://replay.lsm.lv/ru/statja/ltv/311130/4-studija-zolitudes-tragedija-un-incupes-stacija',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _fix_nuxt_data(self, webpage):
|
||||
return re.sub(r'Object\.create\(null(?:,(\{.+\}))?\)', lambda m: m.group(1) or 'null', webpage)
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
data = self._search_nuxt_data(
|
||||
self._fix_nuxt_data(webpage), video_id, context_name='__REPLAY__')
|
||||
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'id': video_id,
|
||||
**traverse_obj(data, {
|
||||
'url': ('playback', 'service', 'url', {url_or_none}),
|
||||
'title': ('mediaItem', 'title'),
|
||||
'description': ('mediaItem', ('lead', 'body')),
|
||||
'duration': ('mediaItem', 'duration', {int_or_none}),
|
||||
'timestamp': ('mediaItem', 'aired_at', {parse_iso8601}),
|
||||
'thumbnail': ('mediaItem', 'largeThumbnail', {url_or_none}),
|
||||
}, get_all=False),
|
||||
}
|
||||
@ -0,0 +1,171 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..networking import HEADRequest
|
||||
from ..utils import (
|
||||
get_element_by_class,
|
||||
int_or_none,
|
||||
try_call,
|
||||
url_or_none,
|
||||
urlhandle_detect_ext,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class Mx3BaseIE(InfoExtractor):
|
||||
_VALID_URL_TMPL = r'https?://(?:www\.)?%s/t/(?P<id>\w+)'
|
||||
_FORMATS = [{
|
||||
'url': 'player_asset',
|
||||
'format_id': 'default',
|
||||
'quality': 0,
|
||||
}, {
|
||||
'url': 'player_asset?quality=hd',
|
||||
'format_id': 'hd',
|
||||
'quality': 1,
|
||||
}, {
|
||||
'url': 'download',
|
||||
'format_id': 'download',
|
||||
'quality': 2,
|
||||
}, {
|
||||
'url': 'player_asset?quality=source',
|
||||
'format_id': 'source',
|
||||
'quality': 2,
|
||||
}]
|
||||
|
||||
def _extract_formats(self, track_id):
|
||||
formats = []
|
||||
for fmt in self._FORMATS:
|
||||
format_url = f'https://{self._DOMAIN}/tracks/{track_id}/{fmt["url"]}'
|
||||
urlh = self._request_webpage(
|
||||
HEADRequest(format_url), track_id, fatal=False, expected_status=404,
|
||||
note=f'Checking for format {fmt["format_id"]}')
|
||||
if urlh and urlh.status == 200:
|
||||
formats.append({
|
||||
**fmt,
|
||||
'url': format_url,
|
||||
'ext': urlhandle_detect_ext(urlh),
|
||||
'filesize': int_or_none(urlh.headers.get('Content-Length')),
|
||||
})
|
||||
return formats
|
||||
|
||||
def _real_extract(self, url):
|
||||
track_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, track_id)
|
||||
more_info = get_element_by_class('single-more-info', webpage)
|
||||
data = self._download_json(f'https://{self._DOMAIN}/t/{track_id}.json', track_id, fatal=False)
|
||||
|
||||
def get_info_field(name):
|
||||
return self._html_search_regex(
|
||||
rf'<dt[^>]*>\s*{name}\s*</dt>\s*<dd[^>]*>(.*?)</dd>',
|
||||
more_info, name, default=None, flags=re.DOTALL)
|
||||
|
||||
return {
|
||||
'id': track_id,
|
||||
'formats': self._extract_formats(track_id),
|
||||
'genre': self._html_search_regex(
|
||||
r'<div\b[^>]+class="single-band-genre"[^>]*>([^<]+)</div>', webpage, 'genre', default=None),
|
||||
'release_year': int_or_none(get_info_field('Year of creation')),
|
||||
'description': get_info_field('Description'),
|
||||
'tags': try_call(lambda: get_info_field('Tag').split(', '), list),
|
||||
**traverse_obj(data, {
|
||||
'title': ('title', {str}),
|
||||
'artist': (('performer_name', 'artist'), {str}),
|
||||
'album_artist': ('artist', {str}),
|
||||
'composer': ('composer_name', {str}),
|
||||
'thumbnail': (('picture_url_xlarge', 'picture_url'), {url_or_none}),
|
||||
}, get_all=False),
|
||||
}
|
||||
|
||||
|
||||
class Mx3IE(Mx3BaseIE):
|
||||
_DOMAIN = 'mx3.ch'
|
||||
_VALID_URL = Mx3BaseIE._VALID_URL_TMPL % re.escape(_DOMAIN)
|
||||
_TESTS = [{
|
||||
'url': 'https://mx3.ch/t/1Cru',
|
||||
'md5': '7ba09e9826b4447d4e1ce9d69e0e295f',
|
||||
'info_dict': {
|
||||
'id': '1Cru',
|
||||
'ext': 'wav',
|
||||
'artist': 'Godina',
|
||||
'album_artist': 'Tortue Tortue',
|
||||
'composer': 'Olivier Godinat',
|
||||
'genre': 'Rock',
|
||||
'thumbnail': 'https://mx3.ch/pictures/mx3/file/0101/4643/square_xlarge/1-s-envoler-1.jpg?1630272813',
|
||||
'title': "S'envoler",
|
||||
'release_year': 2021,
|
||||
'tags': [],
|
||||
}
|
||||
}, {
|
||||
'url': 'https://mx3.ch/t/1LIY',
|
||||
'md5': '48293cb908342547827f963a5a2e9118',
|
||||
'info_dict': {
|
||||
'id': '1LIY',
|
||||
'ext': 'mov',
|
||||
'artist': 'Tania Kimfumu',
|
||||
'album_artist': 'The Broots',
|
||||
'composer': 'Emmanuel Diserens',
|
||||
'genre': 'Electro',
|
||||
'thumbnail': 'https://mx3.ch/pictures/mx3/file/0110/0003/video_xlarge/frame_0000.png?1686963670',
|
||||
'title': 'The Broots-Larytta remix "Begging For Help"',
|
||||
'release_year': 2023,
|
||||
'tags': ['the broots', 'cassata records', 'larytta'],
|
||||
'description': '"Begging for Help" Larytta Remix Official Video\nRealized By Kali Donkilie in 2023',
|
||||
}
|
||||
}, {
|
||||
'url': 'https://mx3.ch/t/1C6E',
|
||||
'md5': '1afcd578493ddb8e5008e94bb6d97e25',
|
||||
'info_dict': {
|
||||
'id': '1C6E',
|
||||
'ext': 'wav',
|
||||
'artist': 'Alien Bubblegum',
|
||||
'album_artist': 'Alien Bubblegum',
|
||||
'composer': 'Alien Bubblegum',
|
||||
'genre': 'Punk',
|
||||
'thumbnail': 'https://mx3.ch/pictures/mx3/file/0101/1551/square_xlarge/pandora-s-box-cover-with-title.png?1627054733',
|
||||
'title': 'Wide Awake',
|
||||
'release_year': 2021,
|
||||
'tags': ['alien bubblegum', 'bubblegum', 'alien', 'pop punk', 'poppunk'],
|
||||
}
|
||||
}]
|
||||
|
||||
|
||||
class Mx3NeoIE(Mx3BaseIE):
|
||||
_DOMAIN = 'neo.mx3.ch'
|
||||
_VALID_URL = Mx3BaseIE._VALID_URL_TMPL % re.escape(_DOMAIN)
|
||||
_TESTS = [{
|
||||
'url': 'https://neo.mx3.ch/t/1hpd',
|
||||
'md5': '6d9986bbae5cac3296ec8813bf965eb2',
|
||||
'info_dict': {
|
||||
'id': '1hpd',
|
||||
'ext': 'wav',
|
||||
'artist': 'Baptiste Lopez',
|
||||
'album_artist': 'Kammerorchester Basel',
|
||||
'composer': 'Jannik Giger',
|
||||
'genre': 'Composition, Orchestra',
|
||||
'title': 'Troisième œil. Für Kammerorchester (2023)',
|
||||
'thumbnail': 'https://neo.mx3.ch/pictures/neo/file/0000/0241/square_xlarge/kammerorchester-basel-group-photo-2_c_-lukasz-rajchert.jpg?1560341252',
|
||||
'release_year': 2023,
|
||||
'tags': [],
|
||||
}
|
||||
}]
|
||||
|
||||
|
||||
class Mx3VolksmusikIE(Mx3BaseIE):
|
||||
_DOMAIN = 'volksmusik.mx3.ch'
|
||||
_VALID_URL = Mx3BaseIE._VALID_URL_TMPL % re.escape(_DOMAIN)
|
||||
_TESTS = [{
|
||||
'url': 'https://volksmusik.mx3.ch/t/Zx',
|
||||
'md5': 'dd967a7b0c1ef898f3e072cf9c2eae3c',
|
||||
'info_dict': {
|
||||
'id': 'Zx',
|
||||
'ext': 'mp3',
|
||||
'artist': 'Ländlerkapelle GrischArt',
|
||||
'album_artist': 'Ländlerkapelle GrischArt',
|
||||
'composer': 'Urs Glauser',
|
||||
'genre': 'Instrumental, Graubünden',
|
||||
'title': 'Chämilouf',
|
||||
'thumbnail': 'https://volksmusik.mx3.ch/pictures/vxm/file/0000/3815/square_xlarge/grischart1.jpg?1450530120',
|
||||
'release_year': 2012,
|
||||
'tags': [],
|
||||
}
|
||||
}]
|
||||
@ -1,33 +1,38 @@
|
||||
import datetime
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .youtube import YoutubeIE
|
||||
from ..utils import parse_iso8601, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NerdCubedFeedIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/feed\.json'
|
||||
_VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/?(?:$|[#?])'
|
||||
_TEST = {
|
||||
'url': 'http://www.nerdcubed.co.uk/feed.json',
|
||||
'url': 'http://www.nerdcubed.co.uk/',
|
||||
'info_dict': {
|
||||
'id': 'nerdcubed-feed',
|
||||
'title': 'nerdcubed.co.uk feed',
|
||||
},
|
||||
'playlist_mincount': 1300,
|
||||
'playlist_mincount': 5500,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
feed = self._download_json(url, url, 'Downloading NerdCubed JSON feed')
|
||||
def _extract_video(self, feed_entry):
|
||||
return self.url_result(
|
||||
f'https://www.youtube.com/watch?v={feed_entry["id"]}', YoutubeIE,
|
||||
**traverse_obj(feed_entry, {
|
||||
'id': ('id', {str}),
|
||||
'title': ('title', {str}),
|
||||
'description': ('description', {str}),
|
||||
'timestamp': ('publishedAt', {parse_iso8601}),
|
||||
'channel': ('source', 'name', {str}),
|
||||
'channel_id': ('source', 'id', {str}),
|
||||
'channel_url': ('source', 'url', {str}),
|
||||
'thumbnail': ('thumbnail', 'source', {url_or_none}),
|
||||
}), url_transparent=True)
|
||||
|
||||
entries = [{
|
||||
'_type': 'url',
|
||||
'title': feed_entry['title'],
|
||||
'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
|
||||
'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
|
||||
'url': 'http://www.youtube.com/watch?v=' + feed_entry['youtube_id'],
|
||||
} for feed_entry in feed]
|
||||
def _real_extract(self, url):
|
||||
video_id = 'nerdcubed-feed'
|
||||
feed = self._download_json('https://www.nerdcubed.co.uk/_/cdn/videos.json', video_id)
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'title': 'nerdcubed.co.uk feed',
|
||||
'id': 'nerdcubed-feed',
|
||||
'entries': entries,
|
||||
}
|
||||
return self.playlist_result(
|
||||
map(self._extract_video, traverse_obj(feed, ('videos', lambda _, v: v['id']))),
|
||||
video_id, 'nerdcubed.co.uk feed')
|
||||
|
||||
@ -0,0 +1,225 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none, mimetype2ext, parse_iso8601, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NinaProtocolIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?ninaprotocol\.com/releases/(?P<id>[^/#?]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.ninaprotocol.com/releases/3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
|
||||
'title': 'The Spatulas - March Chant',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'channel': 'ppm',
|
||||
'description': 'md5:bb9f9d39d8f786449cd5d0ff7c5772db',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'timestamp': 1701417610,
|
||||
'uploader': 'ppmrecs',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'display_id': 'the-spatulas-march-chant',
|
||||
'upload_date': '20231201',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_1',
|
||||
'title': 'March Chant In April',
|
||||
'track': 'March Chant In April',
|
||||
'ext': 'mp3',
|
||||
'duration': 152,
|
||||
'track_number': 1,
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'uploader': 'ppmrecs',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'timestamp': 1701417610,
|
||||
'channel': 'ppm',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'upload_date': '20231201',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_2',
|
||||
'title': 'Rescue Mission',
|
||||
'track': 'Rescue Mission',
|
||||
'ext': 'mp3',
|
||||
'duration': 212,
|
||||
'track_number': 2,
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'uploader': 'ppmrecs',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'channel': 'ppm',
|
||||
'upload_date': '20231201',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'timestamp': 1701417610,
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_3',
|
||||
'title': 'Slinger Style',
|
||||
'track': 'Slinger Style',
|
||||
'ext': 'mp3',
|
||||
'duration': 179,
|
||||
'track_number': 3,
|
||||
'timestamp': 1701417610,
|
||||
'upload_date': '20231201',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'uploader': 'ppmrecs',
|
||||
'channel': 'ppm',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_4',
|
||||
'title': 'Psychic Signal',
|
||||
'track': 'Psychic Signal',
|
||||
'ext': 'mp3',
|
||||
'duration': 220,
|
||||
'track_number': 4,
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'upload_date': '20231201',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'timestamp': 1701417610,
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'channel': 'ppm',
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'uploader': 'ppmrecs',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_5',
|
||||
'title': 'Curvy Color',
|
||||
'track': 'Curvy Color',
|
||||
'ext': 'mp3',
|
||||
'duration': 148,
|
||||
'track_number': 5,
|
||||
'timestamp': 1701417610,
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'channel': 'ppm',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'uploader': 'ppmrecs',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'upload_date': '20231201',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_6',
|
||||
'title': 'Caveman Star',
|
||||
'track': 'Caveman Star',
|
||||
'ext': 'mp3',
|
||||
'duration': 121,
|
||||
'track_number': 6,
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'uploader': 'ppmrecs',
|
||||
'timestamp': 1701417610,
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'channel': 'ppm',
|
||||
'upload_date': '20231201',
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
'url': 'https://www.ninaprotocol.com/releases/f-g-s-american-shield',
|
||||
'info_dict': {
|
||||
'id': '76PZnJwaMgViQHYfA4NYJXds7CmW6vHQKAtQUxGene6J',
|
||||
'description': 'md5:63f08d5db558b4b36e1896f317062721',
|
||||
'title': 'F.G.S. - American Shield',
|
||||
'uploader_id': 'Ej3rozs11wYqFk1Gs6oggGCkGLz8GzBhmJfnUxf6gPci',
|
||||
'channel_id': '6JuksCZPXuP16wJ1BUfwuukJzh42C7guhLrFPPkVJfyE',
|
||||
'channel': 'tinkscough',
|
||||
'tags': [],
|
||||
'album_artist': 'F.G.S.',
|
||||
'album': 'F.G.S. - American Shield',
|
||||
'thumbnail': 'https://www.arweave.net/YJpgImkXLT9SbpFb576KuZ5pm6bdvs452LMs3Rx6lm8',
|
||||
'display_id': 'f-g-s-american-shield',
|
||||
'uploader': 'flannerysilva',
|
||||
'timestamp': 1702395858,
|
||||
'upload_date': '20231212',
|
||||
},
|
||||
'playlist_count': 1,
|
||||
}, {
|
||||
'url': 'https://www.ninaprotocol.com/releases/time-to-figure-things-out',
|
||||
'info_dict': {
|
||||
'id': '6Zi1nC5hj6b13NkpxVYwRhFy6mYA7oLBbe9DMrgGDcYh',
|
||||
'display_id': 'time-to-figure-things-out',
|
||||
'description': 'md5:960202ed01c3134bb8958f1008527e35',
|
||||
'timestamp': 1706283607,
|
||||
'title': 'DJ STEPDAD - time to figure things out',
|
||||
'album_artist': 'DJ STEPDAD',
|
||||
'uploader': 'tddvsss',
|
||||
'upload_date': '20240126',
|
||||
'album': 'time to figure things out',
|
||||
'uploader_id': 'AXQNRgTyYsySyAMFDwxzumuGjfmoXshorCesjpquwCBi',
|
||||
'thumbnail': 'https://www.arweave.net/O4i8bcKVqJVZvNeHHFp6r8knpFGh9ZwEgbeYacr4nss',
|
||||
'tags': [],
|
||||
},
|
||||
'playlist_count': 4,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
release = self._download_json(
|
||||
f'https://api.ninaprotocol.com/v1/releases/{video_id}', video_id)['release']
|
||||
|
||||
video_id = release.get('publicKey') or video_id
|
||||
|
||||
common_info = traverse_obj(release, {
|
||||
'album': ('metadata', 'properties', 'title', {str}),
|
||||
'album_artist': ((('hub', 'data'), 'publisherAccount'), 'displayName', {str}),
|
||||
'timestamp': ('datetime', {parse_iso8601}),
|
||||
'thumbnail': ('metadata', 'image', {url_or_none}),
|
||||
'uploader': ('publisherAccount', 'handle', {str}),
|
||||
'uploader_id': ('publisherAccount', 'publicKey', {str}),
|
||||
'channel': ('hub', 'handle', {str}),
|
||||
'channel_id': ('hub', 'publicKey', {str}),
|
||||
}, get_all=False)
|
||||
common_info['tags'] = traverse_obj(release, ('metadata', 'properties', 'tags', ..., {str}))
|
||||
|
||||
entries = []
|
||||
for track_num, track in enumerate(traverse_obj(release, (
|
||||
'metadata', 'properties', 'files', lambda _, v: url_or_none(v['uri']))), 1):
|
||||
entries.append({
|
||||
'id': f'{video_id}_{track_num}',
|
||||
'url': track['uri'],
|
||||
**traverse_obj(track, {
|
||||
'title': ('track_title', {str}),
|
||||
'track': ('track_title', {str}),
|
||||
'ext': ('type', {mimetype2ext}),
|
||||
'track_number': ('track', {int_or_none}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
}),
|
||||
'vcodec': 'none',
|
||||
**common_info,
|
||||
})
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'id': video_id,
|
||||
'entries': entries,
|
||||
**traverse_obj(release, {
|
||||
'display_id': ('slug', {str}),
|
||||
'title': ('metadata', 'name', {str}),
|
||||
'description': ('metadata', 'description', {str}),
|
||||
}),
|
||||
**common_info,
|
||||
}
|
||||
@ -0,0 +1,199 @@
|
||||
import functools
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
OnDemandPagedList,
|
||||
UserNotLive,
|
||||
filter_dict,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
str_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NuumBaseIE(InfoExtractor):
|
||||
def _call_api(self, path, video_id, description, query={}):
|
||||
response = self._download_json(
|
||||
f'https://nuum.ru/api/v2/{path}', video_id, query=query,
|
||||
note=f'Downloading {description} metadata',
|
||||
errnote=f'Unable to download {description} metadata')
|
||||
if error := response.get('error'):
|
||||
raise ExtractorError(f'API returned error: {error!r}')
|
||||
return response['result']
|
||||
|
||||
def _get_channel_info(self, channel_name):
|
||||
return self._call_api(
|
||||
'broadcasts/public', video_id=channel_name, description='channel',
|
||||
query={
|
||||
'with_extra': 'true',
|
||||
'channel_name': channel_name,
|
||||
'with_deleted': 'true',
|
||||
})
|
||||
|
||||
def _parse_video_data(self, container, extract_formats=True):
|
||||
stream = traverse_obj(container, ('media_container_streams', 0, {dict})) or {}
|
||||
media = traverse_obj(stream, ('stream_media', 0, {dict})) or {}
|
||||
media_url = traverse_obj(media, (
|
||||
'media_meta', ('media_archive_url', 'media_url'), {url_or_none}), get_all=False)
|
||||
|
||||
video_id = str(container['media_container_id'])
|
||||
is_live = media.get('media_status') == 'RUNNING'
|
||||
|
||||
formats, subtitles = None, None
|
||||
if extract_formats:
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
media_url, video_id, 'mp4', live=is_live)
|
||||
|
||||
return filter_dict({
|
||||
'id': video_id,
|
||||
'is_live': is_live,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(container, {
|
||||
'title': ('media_container_name', {str}),
|
||||
'description': ('media_container_description', {str}),
|
||||
'timestamp': ('created_at', {parse_iso8601}),
|
||||
'channel': ('media_container_channel', 'channel_name', {str}),
|
||||
'channel_id': ('media_container_channel', 'channel_id', {str_or_none}),
|
||||
}),
|
||||
**traverse_obj(stream, {
|
||||
'view_count': ('stream_total_viewers', {int_or_none}),
|
||||
'concurrent_view_count': ('stream_current_viewers', {int_or_none}),
|
||||
}),
|
||||
**traverse_obj(media, {
|
||||
'duration': ('media_duration', {int_or_none}),
|
||||
'thumbnail': ('media_meta', ('media_preview_archive_url', 'media_preview_url'), {url_or_none}),
|
||||
}, get_all=False),
|
||||
})
|
||||
|
||||
|
||||
class NuumMediaIE(NuumBaseIE):
|
||||
IE_NAME = 'nuum:media'
|
||||
_VALID_URL = r'https?://nuum\.ru/(?:streams|videos|clips)/(?P<id>[\d]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://nuum.ru/streams/1592713-7-days-to-die',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://nuum.ru/videos/1567547-toxi-hurtz',
|
||||
'md5': 'f1d9118a30403e32b702a204eb03aca3',
|
||||
'info_dict': {
|
||||
'id': '1567547',
|
||||
'ext': 'mp4',
|
||||
'title': 'Toxi$ - Hurtz',
|
||||
'description': '',
|
||||
'timestamp': 1702631651,
|
||||
'upload_date': '20231215',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'view_count': int,
|
||||
'concurrent_view_count': int,
|
||||
'channel_id': '6911',
|
||||
'channel': 'toxis',
|
||||
'duration': 116,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://nuum.ru/clips/1552564-pro-misu',
|
||||
'md5': 'b248ae1565b1e55433188f11beeb0ca1',
|
||||
'info_dict': {
|
||||
'id': '1552564',
|
||||
'ext': 'mp4',
|
||||
'title': 'Про Мису 🙃',
|
||||
'timestamp': 1701971828,
|
||||
'upload_date': '20231207',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'view_count': int,
|
||||
'concurrent_view_count': int,
|
||||
'channel_id': '3320',
|
||||
'channel': 'Misalelik',
|
||||
'duration': 41,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video_data = self._call_api(f'media-containers/{video_id}', video_id, 'media')
|
||||
|
||||
return self._parse_video_data(video_data)
|
||||
|
||||
|
||||
class NuumLiveIE(NuumBaseIE):
|
||||
IE_NAME = 'nuum:live'
|
||||
_VALID_URL = r'https?://nuum\.ru/channel/(?P<id>[^/#?]+)/?(?:$|[#?])'
|
||||
_TESTS = [{
|
||||
'url': 'https://nuum.ru/channel/mts_live',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel = self._match_id(url)
|
||||
channel_info = self._get_channel_info(channel)
|
||||
if traverse_obj(channel_info, ('channel', 'channel_is_live')) is False:
|
||||
raise UserNotLive(video_id=channel)
|
||||
|
||||
info = self._parse_video_data(channel_info['media_container'])
|
||||
return {
|
||||
'webpage_url': f'https://nuum.ru/streams/{info["id"]}',
|
||||
'extractor_key': NuumMediaIE.ie_key(),
|
||||
'extractor': NuumMediaIE.IE_NAME,
|
||||
**info,
|
||||
}
|
||||
|
||||
|
||||
class NuumTabIE(NuumBaseIE):
|
||||
IE_NAME = 'nuum:tab'
|
||||
_VALID_URL = r'https?://nuum\.ru/channel/(?P<id>[^/#?]+)/(?P<type>streams|videos|clips)'
|
||||
_TESTS = [{
|
||||
'url': 'https://nuum.ru/channel/dankon_/clips',
|
||||
'info_dict': {
|
||||
'id': 'dankon__clips',
|
||||
'title': 'Dankon_',
|
||||
},
|
||||
'playlist_mincount': 29,
|
||||
}, {
|
||||
'url': 'https://nuum.ru/channel/dankon_/videos',
|
||||
'info_dict': {
|
||||
'id': 'dankon__videos',
|
||||
'title': 'Dankon_',
|
||||
},
|
||||
'playlist_mincount': 2,
|
||||
}, {
|
||||
'url': 'https://nuum.ru/channel/dankon_/streams',
|
||||
'info_dict': {
|
||||
'id': 'dankon__streams',
|
||||
'title': 'Dankon_',
|
||||
},
|
||||
'playlist_mincount': 1,
|
||||
}]
|
||||
|
||||
_PAGE_SIZE = 50
|
||||
|
||||
def _fetch_page(self, channel_id, tab_type, tab_id, page):
|
||||
CONTAINER_TYPES = {
|
||||
'clips': ['SHORT_VIDEO', 'REVIEW_VIDEO'],
|
||||
'videos': ['LONG_VIDEO'],
|
||||
'streams': ['SINGLE'],
|
||||
}
|
||||
|
||||
media_containers = self._call_api(
|
||||
'media-containers', video_id=tab_id, description=f'{tab_type} tab page {page + 1}',
|
||||
query={
|
||||
'limit': self._PAGE_SIZE,
|
||||
'offset': page * self._PAGE_SIZE,
|
||||
'channel_id': channel_id,
|
||||
'media_container_status': 'STOPPED',
|
||||
'media_container_type': CONTAINER_TYPES[tab_type],
|
||||
})
|
||||
for container in traverse_obj(media_containers, (..., {dict})):
|
||||
metadata = self._parse_video_data(container, extract_formats=False)
|
||||
yield self.url_result(f'https://nuum.ru/videos/{metadata["id"]}', NuumMediaIE, **metadata)
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel_name, tab_type = self._match_valid_url(url).group('id', 'type')
|
||||
tab_id = f'{channel_name}_{tab_type}'
|
||||
channel_data = self._get_channel_info(channel_name)['channel']
|
||||
|
||||
return self.playlist_result(OnDemandPagedList(functools.partial(
|
||||
self._fetch_page, channel_data['channel_id'], tab_type, tab_id), self._PAGE_SIZE),
|
||||
playlist_id=tab_id, playlist_title=channel_data.get('channel_name'))
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,135 @@
|
||||
import functools
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..networking import HEADRequest
|
||||
from ..utils import (
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
join_nonempty,
|
||||
parse_qs,
|
||||
update_url_query,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class RedCDNLivxIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://[^.]+\.(?:dcs\.redcdn|atmcdn)\.pl/(?:live(?:dash|hls|ss)|nvr)/o2/(?P<tenant>[^/?#]+)/(?P<id>[^?#]+)\.livx'
|
||||
IE_NAME = 'redcdnlivx'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'https://r.dcs.redcdn.pl/livedash/o2/senat/ENC02/channel.livx?indexMode=true&startTime=638272860000&stopTime=638292544000',
|
||||
'info_dict': {
|
||||
'id': 'ENC02-638272860000-638292544000',
|
||||
'ext': 'mp4',
|
||||
'title': 'ENC02',
|
||||
'duration': 19683.982,
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://r.dcs.redcdn.pl/livedash/o2/sejm/ENC18/live.livx?indexMode=true&startTime=722333096000&stopTime=722335562000',
|
||||
'info_dict': {
|
||||
'id': 'ENC18-722333096000-722335562000',
|
||||
'ext': 'mp4',
|
||||
'title': 'ENC18',
|
||||
'duration': 2463.995,
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://r.dcs.redcdn.pl/livehls/o2/sportevolution/live/triathlon2018/warsaw.livx/playlist.m3u8?startTime=550305000000&stopTime=550327620000',
|
||||
'info_dict': {
|
||||
'id': 'triathlon2018-warsaw-550305000000-550327620000',
|
||||
'ext': 'mp4',
|
||||
'title': 'triathlon2018/warsaw',
|
||||
'duration': 22619.98,
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://n-25-12.dcs.redcdn.pl/nvr/o2/sejm/Migacz-ENC01/1.livx?startTime=722347200000&stopTime=722367345000',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://redir.atmcdn.pl/nvr/o2/sejm/ENC08/1.livx?startTime=503831270000&stopTime=503840040000',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
"""
|
||||
Known methods (first in url path):
|
||||
- `livedash` - DASH MPD
|
||||
- `livehls` - HTTP Live Streaming
|
||||
- `livess` - IIS Smooth Streaming
|
||||
- `nvr` - CCTV mode, directly returns a file, typically flv, avc1, aac
|
||||
- `sc` - shoutcast/icecast (audio streams, like radio)
|
||||
"""
|
||||
|
||||
def _real_extract(self, url):
|
||||
tenant, path = self._match_valid_url(url).group('tenant', 'id')
|
||||
qs = parse_qs(url)
|
||||
start_time = traverse_obj(qs, ('startTime', 0, {int_or_none}))
|
||||
stop_time = traverse_obj(qs, ('stopTime', 0, {int_or_none}))
|
||||
|
||||
def livx_mode(mode):
|
||||
suffix = ''
|
||||
if mode == 'livess':
|
||||
suffix = '/manifest'
|
||||
elif mode == 'livehls':
|
||||
suffix = '/playlist.m3u8'
|
||||
file_qs = {}
|
||||
if start_time:
|
||||
file_qs['startTime'] = start_time
|
||||
if stop_time:
|
||||
file_qs['stopTime'] = stop_time
|
||||
if mode == 'nvr':
|
||||
file_qs['nolimit'] = 1
|
||||
elif mode != 'sc':
|
||||
file_qs['indexMode'] = 'true'
|
||||
return update_url_query(f'https://r.dcs.redcdn.pl/{mode}/o2/{tenant}/{path}.livx{suffix}', file_qs)
|
||||
|
||||
# no id or title for a transmission. making ones up.
|
||||
title = path \
|
||||
.replace('/live', '').replace('live/', '') \
|
||||
.replace('/channel', '').replace('channel/', '') \
|
||||
.strip('/')
|
||||
video_id = join_nonempty(title.replace('/', '-'), start_time, stop_time)
|
||||
|
||||
formats = []
|
||||
# downloading the manifest separately here instead of _extract_ism_formats to also get some stream metadata
|
||||
ism_res = self._download_xml_handle(
|
||||
livx_mode('livess'), video_id,
|
||||
note='Downloading ISM manifest',
|
||||
errnote='Failed to download ISM manifest',
|
||||
fatal=False)
|
||||
ism_doc = None
|
||||
if ism_res is not False:
|
||||
ism_doc, ism_urlh = ism_res
|
||||
formats, _ = self._parse_ism_formats_and_subtitles(ism_doc, ism_urlh.url, 'ss')
|
||||
|
||||
nvr_urlh = self._request_webpage(
|
||||
HEADRequest(livx_mode('nvr')), video_id, 'Follow flv file redirect', fatal=False,
|
||||
expected_status=lambda _: True)
|
||||
if nvr_urlh and nvr_urlh.status == 200:
|
||||
formats.append({
|
||||
'url': nvr_urlh.url,
|
||||
'ext': 'flv',
|
||||
'format_id': 'direct-0',
|
||||
'preference': -1, # might be slow
|
||||
})
|
||||
formats.extend(self._extract_mpd_formats(livx_mode('livedash'), video_id, mpd_id='dash', fatal=False))
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
livx_mode('livehls'), video_id, m3u8_id='hls', ext='mp4', fatal=False))
|
||||
|
||||
time_scale = traverse_obj(ism_doc, ('@TimeScale', {int_or_none})) or 10000000
|
||||
duration = traverse_obj(
|
||||
ism_doc, ('@Duration', {functools.partial(float_or_none, scale=time_scale)})) or None
|
||||
|
||||
live_status = None
|
||||
if traverse_obj(ism_doc, '@IsLive') == 'TRUE':
|
||||
live_status = 'is_live'
|
||||
elif duration:
|
||||
live_status = 'was_live'
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'duration': duration,
|
||||
'live_status': live_status,
|
||||
}
|
||||
@ -0,0 +1,218 @@
|
||||
import datetime
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .redge import RedCDNLivxIE
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
join_nonempty,
|
||||
js_to_json,
|
||||
strip_or_none,
|
||||
update_url_query,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
def is_dst(date):
|
||||
last_march = datetime.datetime(date.year, 3, 31)
|
||||
last_october = datetime.datetime(date.year, 10, 31)
|
||||
last_sunday_march = last_march - datetime.timedelta(days=last_march.isoweekday() % 7)
|
||||
last_sunday_october = last_october - datetime.timedelta(days=last_october.isoweekday() % 7)
|
||||
return last_sunday_march.replace(hour=2) <= date <= last_sunday_october.replace(hour=3)
|
||||
|
||||
|
||||
def rfc3339_to_atende(date):
|
||||
date = datetime.datetime.fromisoformat(date)
|
||||
date = date + datetime.timedelta(hours=1 if is_dst(date) else 0)
|
||||
return int((date.timestamp() - 978307200) * 1000)
|
||||
|
||||
|
||||
class SejmIE(InfoExtractor):
|
||||
_VALID_URL = (
|
||||
r'https?://(?:www\.)?sejm\.gov\.pl/[Ss]ejm(?P<term>\d+)\.nsf/transmisje(?:_arch)?\.xsp(?:\?[^#]*)?#(?P<id>[\dA-F]+)',
|
||||
r'https?://(?:www\.)?sejm\.gov\.pl/[Ss]ejm(?P<term>\d+)\.nsf/transmisje(?:_arch)?\.xsp\?(?:[^#]+&)?unid=(?P<id>[\dA-F]+)',
|
||||
r'https?://sejm-embed\.redcdn\.pl/[Ss]ejm(?P<term>\d+)\.nsf/VideoFrame\.xsp/(?P<id>[\dA-F]+)',
|
||||
)
|
||||
IE_NAME = 'sejm'
|
||||
|
||||
_TESTS = [{
|
||||
# multiple cameras, polish SL iterpreter
|
||||
'url': 'https://www.sejm.gov.pl/Sejm10.nsf/transmisje_arch.xsp#6181EF1AD9CEEBB5C1258A6D006452B5',
|
||||
'info_dict': {
|
||||
'id': '6181EF1AD9CEEBB5C1258A6D006452B5',
|
||||
'title': '1. posiedzenie Sejmu X kadencji',
|
||||
'duration': 20145,
|
||||
'live_status': 'was_live',
|
||||
'location': 'Sala Posiedzeń',
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': 'ENC01-722340000000-722360145000',
|
||||
'ext': 'mp4',
|
||||
'duration': 20145,
|
||||
'title': '1. posiedzenie Sejmu X kadencji - ENC01',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': 'ENC30-722340000000-722360145000',
|
||||
'ext': 'mp4',
|
||||
'duration': 20145,
|
||||
'title': '1. posiedzenie Sejmu X kadencji - ENC30',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': 'ENC31-722340000000-722360145000',
|
||||
'ext': 'mp4',
|
||||
'duration': 20145,
|
||||
'title': '1. posiedzenie Sejmu X kadencji - ENC31',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': 'ENC32-722340000000-722360145000',
|
||||
'ext': 'mp4',
|
||||
'duration': 20145,
|
||||
'title': '1. posiedzenie Sejmu X kadencji - ENC32',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}, {
|
||||
# sign lang interpreter
|
||||
'info_dict': {
|
||||
'id': 'Migacz-ENC01-1-722340000000-722360145000',
|
||||
'ext': 'mp4',
|
||||
'duration': 20145,
|
||||
'title': '1. posiedzenie Sejmu X kadencji - Migacz-ENC01',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
'url': 'https://www.sejm.gov.pl/Sejm8.nsf/transmisje.xsp?unid=9377A9D65518E9A5C125808E002E9FF2',
|
||||
'info_dict': {
|
||||
'id': '9377A9D65518E9A5C125808E002E9FF2',
|
||||
'title': 'Debata "Lepsza Polska: obywatelska"',
|
||||
'description': 'KP .Nowoczesna',
|
||||
'duration': 8770,
|
||||
'live_status': 'was_live',
|
||||
'location': 'sala kolumnowa im. Kazimierza Pużaka (bud. C-D)',
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': 'ENC08-1-503831270000-503840040000',
|
||||
'ext': 'mp4',
|
||||
'duration': 8770,
|
||||
'title': 'Debata "Lepsza Polska: obywatelska" - ENC08',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
# 7th term is very special, since it does not use redcdn livx
|
||||
'url': 'https://www.sejm.gov.pl/sejm7.nsf/transmisje_arch.xsp?rok=2015&month=11#A6E6D475ECCC6FE5C1257EF90034817F',
|
||||
'info_dict': {
|
||||
'id': 'A6E6D475ECCC6FE5C1257EF90034817F',
|
||||
'title': 'Konferencja prasowa - Stanowisko SLD ws. składu nowego rządu',
|
||||
'description': 'SLD - Biuro Prasowe Klubu',
|
||||
'duration': 514,
|
||||
'location': 'sala 101/bud. C',
|
||||
'live_status': 'was_live',
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': 'A6E6D475ECCC6FE5C1257EF90034817F',
|
||||
'ext': 'mp4',
|
||||
'title': 'Konferencja prasowa - Stanowisko SLD ws. składu nowego rządu',
|
||||
'duration': 514,
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
'url': 'https://sejm-embed.redcdn.pl/Sejm10.nsf/VideoFrame.xsp/FED58EABB97FBD53C1258A7400386492',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
term, video_id = self._match_valid_url(url).group('term', 'id')
|
||||
frame = self._download_webpage(
|
||||
f'https://sejm-embed.redcdn.pl/Sejm{term}.nsf/VideoFrame.xsp/{video_id}',
|
||||
video_id)
|
||||
# despite it says "transmisje_arch", it works for live streams too!
|
||||
data = self._download_json(
|
||||
f'https://www.sejm.gov.pl/Sejm{term}.nsf/transmisje_arch.xsp/json/{video_id}',
|
||||
video_id)
|
||||
params = data['params']
|
||||
|
||||
title = strip_or_none(data.get('title'))
|
||||
|
||||
if data.get('status') == 'VIDEO_ENDED':
|
||||
live_status = 'was_live'
|
||||
elif data.get('status') == 'VIDEO_PLAYING':
|
||||
live_status = 'is_live'
|
||||
else:
|
||||
live_status = None
|
||||
self.report_warning(f'unknown status: {data.get("status")}')
|
||||
|
||||
start_time = rfc3339_to_atende(params['start'])
|
||||
# current streams have a stop time of *expected* end of session, but actual times
|
||||
# can change during the transmission. setting a stop_time would artificially
|
||||
# end the stream at that time, while the session actually keeps going.
|
||||
if live_status == 'was_live':
|
||||
stop_time = rfc3339_to_atende(params['stop'])
|
||||
duration = (stop_time - start_time) // 1000
|
||||
else:
|
||||
stop_time, duration = None, None
|
||||
|
||||
entries = []
|
||||
|
||||
def add_entry(file, legacy_file=False):
|
||||
if not file:
|
||||
return
|
||||
file = self._proto_relative_url(file)
|
||||
if not legacy_file:
|
||||
file = update_url_query(file, {'startTime': start_time})
|
||||
if stop_time is not None:
|
||||
file = update_url_query(file, {'stopTime': stop_time})
|
||||
stream_id = self._search_regex(r'/o2/sejm/([^/]+)/[^./]+\.livx', file, 'stream id')
|
||||
common_info = {
|
||||
'url': file,
|
||||
'duration': duration,
|
||||
}
|
||||
if legacy_file:
|
||||
entries.append({
|
||||
**common_info,
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
})
|
||||
else:
|
||||
entries.append({
|
||||
**common_info,
|
||||
'_type': 'url_transparent',
|
||||
'ie_key': RedCDNLivxIE.ie_key(),
|
||||
'id': stream_id,
|
||||
'title': join_nonempty(title, stream_id, delim=' - '),
|
||||
})
|
||||
|
||||
cameras = self._search_json(
|
||||
r'var\s+cameras\s*=', frame, 'camera list', video_id,
|
||||
contains_pattern=r'\[(?s:.+)\]', transform_source=js_to_json,
|
||||
fatal=False) or []
|
||||
for camera_file in traverse_obj(cameras, (..., 'file', {dict})):
|
||||
if camera_file.get('flv'):
|
||||
add_entry(camera_file['flv'])
|
||||
elif camera_file.get('mp4'):
|
||||
# this is only a thing in 7th term. no streams before, and starting 8th it's redcdn livx
|
||||
add_entry(camera_file['mp4'], legacy_file=True)
|
||||
else:
|
||||
self.report_warning('Unknown camera stream type found')
|
||||
|
||||
if params.get('mig'):
|
||||
add_entry(self._search_regex(r"var sliUrl\s*=\s*'([^']+)'", frame, 'sign language interpreter url', fatal=False))
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'entries': entries,
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': clean_html(data.get('desc')) or None,
|
||||
'duration': duration,
|
||||
'live_status': live_status,
|
||||
'location': strip_or_none(data.get('location')),
|
||||
}
|
||||
@ -1,159 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
traverse_obj,
|
||||
try_get,
|
||||
)
|
||||
|
||||
|
||||
class WASDTVBaseIE(InfoExtractor):
|
||||
|
||||
def _fetch(self, path, video_id, description, query={}):
|
||||
response = self._download_json(
|
||||
f'https://wasd.tv/api/{path}', video_id, query=query,
|
||||
note=f'Downloading {description} metadata',
|
||||
errnote=f'Unable to download {description} metadata')
|
||||
error = response.get('error')
|
||||
if error:
|
||||
raise ExtractorError(f'{self.IE_NAME} returned error: {error}', expected=True)
|
||||
return response.get('result')
|
||||
|
||||
def _extract_thumbnails(self, thumbnails_dict):
|
||||
return [{
|
||||
'url': url,
|
||||
'preference': index,
|
||||
} for index, url in enumerate(
|
||||
traverse_obj(thumbnails_dict, (('small', 'medium', 'large'),))) if url]
|
||||
|
||||
def _real_extract(self, url):
|
||||
container = self._get_container(url)
|
||||
stream = traverse_obj(container, ('media_container_streams', 0))
|
||||
media = try_get(stream, lambda x: x['stream_media'][0])
|
||||
if not media:
|
||||
raise ExtractorError('Can not extract media data.', expected=True)
|
||||
media_meta = media.get('media_meta')
|
||||
media_url, is_live = self._get_media_url(media_meta)
|
||||
video_id = media.get('media_id') or container.get('media_container_id')
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(media_url, video_id, 'mp4')
|
||||
return {
|
||||
'id': str(video_id),
|
||||
'title': container.get('media_container_name') or self._og_search_title(self._download_webpage(url, video_id)),
|
||||
'description': container.get('media_container_description'),
|
||||
'thumbnails': self._extract_thumbnails(media_meta.get('media_preview_images')),
|
||||
'timestamp': parse_iso8601(container.get('created_at')),
|
||||
'view_count': int_or_none(stream.get('stream_current_viewers' if is_live else 'stream_total_viewers')),
|
||||
'is_live': is_live,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
|
||||
def _get_container(self, url):
|
||||
raise NotImplementedError('Subclass for get media container')
|
||||
|
||||
def _get_media_url(self, media_meta):
|
||||
raise NotImplementedError('Subclass for get media url')
|
||||
|
||||
|
||||
class WASDTVStreamIE(WASDTVBaseIE):
|
||||
IE_NAME = 'wasdtv:stream'
|
||||
_VALID_URL = r'https?://wasd\.tv/(?P<id>[^/#?]+)$'
|
||||
_TESTS = [{
|
||||
'url': 'https://wasd.tv/24_7',
|
||||
'info_dict': {
|
||||
'id': '559738',
|
||||
'ext': 'mp4',
|
||||
'title': 'Live 24/7 Music',
|
||||
'description': '24/7 Music',
|
||||
'timestamp': int,
|
||||
'upload_date': r're:^\d{8}$',
|
||||
'is_live': True,
|
||||
'view_count': int,
|
||||
},
|
||||
}]
|
||||
|
||||
def _get_container(self, url):
|
||||
nickname = self._match_id(url)
|
||||
channel = self._fetch(f'channels/nicknames/{nickname}', video_id=nickname, description='channel')
|
||||
channel_id = channel.get('channel_id')
|
||||
containers = self._fetch(
|
||||
'v2/media-containers', channel_id, 'running media containers',
|
||||
query={
|
||||
'channel_id': channel_id,
|
||||
'media_container_type': 'SINGLE',
|
||||
'media_container_status': 'RUNNING',
|
||||
})
|
||||
if not containers:
|
||||
raise ExtractorError(f'{nickname} is offline', expected=True)
|
||||
return containers[0]
|
||||
|
||||
def _get_media_url(self, media_meta):
|
||||
return media_meta['media_url'], True
|
||||
|
||||
|
||||
class WASDTVRecordIE(WASDTVBaseIE):
|
||||
IE_NAME = 'wasdtv:record'
|
||||
_VALID_URL = r'https?://wasd\.tv/[^/#?]+(?:/videos)?\?record=(?P<id>\d+)$'
|
||||
_TESTS = [{
|
||||
'url': 'https://wasd.tv/spacemita/videos?record=907755',
|
||||
'md5': 'c9899dd85be4cc997816ff9f9ca516ce',
|
||||
'info_dict': {
|
||||
'id': '906825',
|
||||
'ext': 'mp4',
|
||||
'title': 'Музыкальный',
|
||||
'description': 'md5:f510388d929ff60ae61d4c3cab3137cc',
|
||||
'timestamp': 1645812079,
|
||||
'upload_date': '20220225',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'is_live': False,
|
||||
'view_count': int,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://wasd.tv/spacemita?record=907755',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _get_container(self, url):
|
||||
container_id = self._match_id(url)
|
||||
return self._fetch(
|
||||
f'v2/media-containers/{container_id}', container_id, 'media container')
|
||||
|
||||
def _get_media_url(self, media_meta):
|
||||
media_archive_url = media_meta.get('media_archive_url')
|
||||
if media_archive_url:
|
||||
return media_archive_url, False
|
||||
return media_meta['media_url'], True
|
||||
|
||||
|
||||
class WASDTVClipIE(WASDTVBaseIE):
|
||||
IE_NAME = 'wasdtv:clip'
|
||||
_VALID_URL = r'https?://wasd\.tv/[^/#?]+/clips\?clip=(?P<id>\d+)$'
|
||||
_TESTS = [{
|
||||
'url': 'https://wasd.tv/spacemita/clips?clip=26804',
|
||||
'md5': '818885e720143d7a4e776ff66fcff148',
|
||||
'info_dict': {
|
||||
'id': '26804',
|
||||
'ext': 'mp4',
|
||||
'title': 'Пуш флексит на голове стримера',
|
||||
'timestamp': 1646682908,
|
||||
'upload_date': '20220307',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'view_count': int,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
clip_id = self._match_id(url)
|
||||
clip = self._fetch(f'v2/clips/{clip_id}', video_id=clip_id, description='clip')
|
||||
clip_data = clip.get('clip_data')
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(clip_data.get('url'), video_id=clip_id, ext='mp4')
|
||||
return {
|
||||
'id': clip_id,
|
||||
'title': clip.get('clip_title') or self._og_search_title(self._download_webpage(url, clip_id, fatal=False)),
|
||||
'thumbnails': self._extract_thumbnails(clip_data.get('preview')),
|
||||
'timestamp': parse_iso8601(clip.get('created_at')),
|
||||
'view_count': int_or_none(clip.get('clip_views_count')),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue