mirror of https://github.com/yt-dlp/yt-dlp
Merge branch 'yt-dlp:master' into pr/fb_parsedata_error
bashonly
11 months ago
commit
3302588c77
@ -1,10 +0,0 @@
|
||||
include AUTHORS
|
||||
include Changelog.md
|
||||
include LICENSE
|
||||
include README.md
|
||||
include completions/*/*
|
||||
include supportedsites.md
|
||||
include yt-dlp.1
|
||||
include requirements.txt
|
||||
recursive-include devscripts *
|
||||
recursive-include test *
|
||||
@ -0,0 +1 @@
|
||||
# Empty file
|
||||
@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow execution from anywhere
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import warnings
|
||||
|
||||
from py2exe import freeze
|
||||
|
||||
from devscripts.utils import read_version
|
||||
|
||||
VERSION = read_version()
|
||||
|
||||
|
||||
def main():
|
||||
warnings.warn(
|
||||
'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
|
||||
'It is recommended to run "pyinst.py" to build using pyinstaller instead')
|
||||
|
||||
return freeze(
|
||||
console=[{
|
||||
'script': './yt_dlp/__main__.py',
|
||||
'dest_base': 'yt-dlp',
|
||||
'icon_resources': [(1, 'devscripts/logo.ico')],
|
||||
}],
|
||||
version_info={
|
||||
'version': VERSION,
|
||||
'description': 'A youtube-dl fork with additional features and patches',
|
||||
'comments': 'Official repository: <https://github.com/yt-dlp/yt-dlp>',
|
||||
'product_name': 'yt-dlp',
|
||||
'product_version': VERSION,
|
||||
},
|
||||
options={
|
||||
'bundle_files': 0,
|
||||
'compressed': 1,
|
||||
'optimize': 2,
|
||||
'dist_dir': './dist',
|
||||
'excludes': [
|
||||
# py2exe cannot import Crypto
|
||||
'Crypto',
|
||||
'Cryptodome',
|
||||
# py2exe appears to confuse this with our socks library.
|
||||
# We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
|
||||
'urllib3.contrib.socks'
|
||||
],
|
||||
'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
|
||||
# Modules that are only imported dynamically must be added here
|
||||
'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
|
||||
'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
|
||||
},
|
||||
zipfile=None,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow execution from anywhere
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
from devscripts.tomlparse import parse_toml
|
||||
from devscripts.utils import read_file
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Install dependencies for yt-dlp')
|
||||
parser.add_argument(
|
||||
'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)')
|
||||
parser.add_argument(
|
||||
'-e', '--exclude', metavar='REQUIREMENT', action='append', help='Exclude a required dependency')
|
||||
parser.add_argument(
|
||||
'-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group')
|
||||
parser.add_argument(
|
||||
'-o', '--only-optional', action='store_true', help='Only install optional dependencies')
|
||||
parser.add_argument(
|
||||
'-p', '--print', action='store_true', help='Only print a requirements.txt to stdout')
|
||||
parser.add_argument(
|
||||
'-u', '--user', action='store_true', help='Install with pip as --user')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
toml_data = parse_toml(read_file(args.input))
|
||||
deps = toml_data['project']['dependencies']
|
||||
targets = deps.copy() if not args.only_optional else []
|
||||
|
||||
for exclude in args.exclude or []:
|
||||
for dep in deps:
|
||||
simplified_dep = re.match(r'[\w-]+', dep)[0]
|
||||
if dep in targets and (exclude.lower() == simplified_dep.lower() or exclude == dep):
|
||||
targets.remove(dep)
|
||||
|
||||
optional_deps = toml_data['project']['optional-dependencies']
|
||||
for include in args.include or []:
|
||||
group = optional_deps.get(include)
|
||||
if group:
|
||||
targets.extend(group)
|
||||
|
||||
if args.print:
|
||||
for target in targets:
|
||||
print(target)
|
||||
return
|
||||
|
||||
pip_args = [sys.executable, '-m', 'pip', 'install', '-U']
|
||||
if args.user:
|
||||
pip_args.append('--user')
|
||||
pip_args.extend(targets)
|
||||
|
||||
return subprocess.call(pip_args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Simple parser for spec compliant toml files
|
||||
|
||||
A simple toml parser for files that comply with the spec.
|
||||
Should only be used to parse `pyproject.toml` for `install_deps.py`.
|
||||
|
||||
IMPORTANT: INVALID FILES OR MULTILINE STRINGS ARE NOT SUPPORTED!
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
|
||||
WS = r'(?:[\ \t]*)'
|
||||
STRING_RE = re.compile(r'"(?:\\.|[^\\"\n])*"|\'[^\'\n]*\'')
|
||||
SINGLE_KEY_RE = re.compile(rf'{STRING_RE.pattern}|[A-Za-z0-9_-]+')
|
||||
KEY_RE = re.compile(rf'{WS}(?:{SINGLE_KEY_RE.pattern}){WS}(?:\.{WS}(?:{SINGLE_KEY_RE.pattern}){WS})*')
|
||||
EQUALS_RE = re.compile(rf'={WS}')
|
||||
WS_RE = re.compile(WS)
|
||||
|
||||
_SUBTABLE = rf'(?P<subtable>^\[(?P<is_list>\[)?(?P<path>{KEY_RE.pattern})\]\]?)'
|
||||
EXPRESSION_RE = re.compile(rf'^(?:{_SUBTABLE}|{KEY_RE.pattern}=)', re.MULTILINE)
|
||||
|
||||
LIST_WS_RE = re.compile(rf'{WS}((#[^\n]*)?\n{WS})*')
|
||||
LEFTOVER_VALUE_RE = re.compile(r'[^,}\]\t\n#]+')
|
||||
|
||||
|
||||
def parse_key(value: str):
|
||||
for match in SINGLE_KEY_RE.finditer(value):
|
||||
if match[0][0] == '"':
|
||||
yield json.loads(match[0])
|
||||
elif match[0][0] == '\'':
|
||||
yield match[0][1:-1]
|
||||
else:
|
||||
yield match[0]
|
||||
|
||||
|
||||
def get_target(root: dict, paths: list[str], is_list=False):
|
||||
target = root
|
||||
|
||||
for index, key in enumerate(paths, 1):
|
||||
use_list = is_list and index == len(paths)
|
||||
result = target.get(key)
|
||||
if result is None:
|
||||
result = [] if use_list else {}
|
||||
target[key] = result
|
||||
|
||||
if isinstance(result, dict):
|
||||
target = result
|
||||
elif use_list:
|
||||
target = {}
|
||||
result.append(target)
|
||||
else:
|
||||
target = result[-1]
|
||||
|
||||
assert isinstance(target, dict)
|
||||
return target
|
||||
|
||||
|
||||
def parse_enclosed(data: str, index: int, end: str, ws_re: re.Pattern):
|
||||
index += 1
|
||||
|
||||
if match := ws_re.match(data, index):
|
||||
index = match.end()
|
||||
|
||||
while data[index] != end:
|
||||
index = yield True, index
|
||||
|
||||
if match := ws_re.match(data, index):
|
||||
index = match.end()
|
||||
|
||||
if data[index] == ',':
|
||||
index += 1
|
||||
|
||||
if match := ws_re.match(data, index):
|
||||
index = match.end()
|
||||
|
||||
assert data[index] == end
|
||||
yield False, index + 1
|
||||
|
||||
|
||||
def parse_value(data: str, index: int):
|
||||
if data[index] == '[':
|
||||
result = []
|
||||
|
||||
indices = parse_enclosed(data, index, ']', LIST_WS_RE)
|
||||
valid, index = next(indices)
|
||||
while valid:
|
||||
index, value = parse_value(data, index)
|
||||
result.append(value)
|
||||
valid, index = indices.send(index)
|
||||
|
||||
return index, result
|
||||
|
||||
if data[index] == '{':
|
||||
result = {}
|
||||
|
||||
indices = parse_enclosed(data, index, '}', WS_RE)
|
||||
valid, index = next(indices)
|
||||
while valid:
|
||||
valid, index = indices.send(parse_kv_pair(data, index, result))
|
||||
|
||||
return index, result
|
||||
|
||||
if match := STRING_RE.match(data, index):
|
||||
return match.end(), json.loads(match[0]) if match[0][0] == '"' else match[0][1:-1]
|
||||
|
||||
match = LEFTOVER_VALUE_RE.match(data, index)
|
||||
assert match
|
||||
value = match[0].strip()
|
||||
for func in [
|
||||
int,
|
||||
float,
|
||||
datetime.time.fromisoformat,
|
||||
datetime.date.fromisoformat,
|
||||
datetime.datetime.fromisoformat,
|
||||
{'true': True, 'false': False}.get,
|
||||
]:
|
||||
try:
|
||||
value = func(value)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return match.end(), value
|
||||
|
||||
|
||||
def parse_kv_pair(data: str, index: int, target: dict):
|
||||
match = KEY_RE.match(data, index)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
*keys, key = parse_key(match[0])
|
||||
|
||||
match = EQUALS_RE.match(data, match.end())
|
||||
assert match
|
||||
index = match.end()
|
||||
|
||||
index, value = parse_value(data, index)
|
||||
get_target(target, keys)[key] = value
|
||||
return index
|
||||
|
||||
|
||||
def parse_toml(data: str):
|
||||
root = {}
|
||||
target = root
|
||||
|
||||
index = 0
|
||||
while True:
|
||||
match = EXPRESSION_RE.search(data, index)
|
||||
if not match:
|
||||
break
|
||||
|
||||
if match.group('subtable'):
|
||||
index = match.end()
|
||||
path, is_list = match.group('path', 'is_list')
|
||||
target = get_target(root, list(parse_key(path)), bool(is_list))
|
||||
continue
|
||||
|
||||
index = parse_kv_pair(data, match.start(), target)
|
||||
assert index is not None
|
||||
|
||||
return root
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile', type=Path, help='The TOML file to read as input')
|
||||
args = parser.parse_args()
|
||||
|
||||
with args.infile.open('r', encoding='utf-8') as file:
|
||||
data = file.read()
|
||||
|
||||
def default(obj):
|
||||
if isinstance(obj, (datetime.date, datetime.time, datetime.datetime)):
|
||||
return obj.isoformat()
|
||||
|
||||
print(json.dumps(parse_toml(data), default=default))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,5 +1,120 @@
|
||||
[build-system]
|
||||
build-backend = 'setuptools.build_meta'
|
||||
# https://github.com/yt-dlp/yt-dlp/issues/5941
|
||||
# https://github.com/pypa/distutils/issues/17
|
||||
requires = ['setuptools > 50']
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "yt-dlp"
|
||||
maintainers = [
|
||||
{name = "pukkandan", email = "pukkandan.ytdlp@gmail.com"},
|
||||
{name = "Grub4K", email = "contact@grub4k.xyz"},
|
||||
{name = "bashonly", email = "bashonly@protonmail.com"},
|
||||
]
|
||||
description = "A youtube-dl fork with additional features and patches"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
keywords = [
|
||||
"youtube-dl",
|
||||
"video-downloader",
|
||||
"youtube-downloader",
|
||||
"sponsorblock",
|
||||
"youtube-dlc",
|
||||
"yt-dlp",
|
||||
]
|
||||
license = {file = "LICENSE"}
|
||||
classifiers = [
|
||||
"Topic :: Multimedia :: Video",
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Environment :: Console",
|
||||
"Programming Language :: Python",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: Implementation",
|
||||
"Programming Language :: Python :: Implementation :: CPython",
|
||||
"Programming Language :: Python :: Implementation :: PyPy",
|
||||
"License :: OSI Approved :: The Unlicense (Unlicense)",
|
||||
"Operating System :: OS Independent",
|
||||
]
|
||||
dynamic = ["version"]
|
||||
dependencies = [
|
||||
"brotli; implementation_name=='cpython'",
|
||||
"brotlicffi; implementation_name!='cpython'",
|
||||
"certifi",
|
||||
"mutagen",
|
||||
"pycryptodomex",
|
||||
"requests>=2.31.0,<3",
|
||||
"urllib3>=1.26.17,<3",
|
||||
"websockets>=12.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
secretstorage = [
|
||||
"cffi",
|
||||
"secretstorage",
|
||||
]
|
||||
build = [
|
||||
"build",
|
||||
"hatchling",
|
||||
"pip",
|
||||
"wheel",
|
||||
]
|
||||
dev = [
|
||||
"flake8",
|
||||
"isort",
|
||||
"pytest",
|
||||
]
|
||||
pyinstaller = ["pyinstaller>=6.3"]
|
||||
py2exe = ["py2exe>=0.12"]
|
||||
|
||||
[project.urls]
|
||||
Documentation = "https://github.com/yt-dlp/yt-dlp#readme"
|
||||
Repository = "https://github.com/yt-dlp/yt-dlp"
|
||||
Tracker = "https://github.com/yt-dlp/yt-dlp/issues"
|
||||
Funding = "https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators"
|
||||
|
||||
[project.scripts]
|
||||
yt-dlp = "yt_dlp:main"
|
||||
|
||||
[project.entry-points.pyinstaller40]
|
||||
hook-dirs = "yt_dlp.__pyinstaller:get_hook_dirs"
|
||||
|
||||
[tool.hatch.build.targets.sdist]
|
||||
include = [
|
||||
"/yt_dlp",
|
||||
"/devscripts",
|
||||
"/test",
|
||||
"/.gitignore", # included by default, needed for auto-excludes
|
||||
"/Changelog.md",
|
||||
"/LICENSE", # included as license
|
||||
"/pyproject.toml", # included by default
|
||||
"/README.md", # included as readme
|
||||
"/setup.cfg",
|
||||
"/supportedsites.md",
|
||||
]
|
||||
exclude = ["/yt_dlp/__pyinstaller"]
|
||||
artifacts = [
|
||||
"/yt_dlp/extractor/lazy_extractors.py",
|
||||
"/completions",
|
||||
"/AUTHORS", # included by default
|
||||
"/README.txt",
|
||||
"/yt-dlp.1",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["yt_dlp"]
|
||||
exclude = ["/yt_dlp/__pyinstaller"]
|
||||
artifacts = ["/yt_dlp/extractor/lazy_extractors.py"]
|
||||
|
||||
[tool.hatch.build.targets.wheel.shared-data]
|
||||
"completions/bash/yt-dlp" = "share/bash-completion/completions/yt-dlp"
|
||||
"completions/zsh/_yt-dlp" = "share/zsh/site-functions/_yt-dlp"
|
||||
"completions/fish/yt-dlp.fish" = "share/fish/vendor_completions.d/yt-dlp.fish"
|
||||
"README.txt" = "share/doc/yt_dlp/README.txt"
|
||||
"yt-dlp.1" = "share/man/man1/yt-dlp.1"
|
||||
|
||||
[tool.hatch.version]
|
||||
path = "yt_dlp/version.py"
|
||||
pattern = "_pkg_version = '(?P<version>[^']+)'"
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
mutagen
|
||||
pycryptodomex
|
||||
brotli; implementation_name=='cpython'
|
||||
brotlicffi; implementation_name!='cpython'
|
||||
certifi
|
||||
requests>=2.31.0,<3
|
||||
urllib3>=1.26.17,<3
|
||||
websockets>=12.0
|
||||
@ -1,183 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Allow execution from anywhere
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import subprocess
|
||||
import warnings
|
||||
|
||||
try:
|
||||
from setuptools import Command, find_packages, setup
|
||||
setuptools_available = True
|
||||
except ImportError:
|
||||
from distutils.core import Command, setup
|
||||
setuptools_available = False
|
||||
|
||||
from devscripts.utils import read_file, read_version
|
||||
|
||||
VERSION = read_version(varname='_pkg_version')
|
||||
|
||||
DESCRIPTION = 'A youtube-dl fork with additional features and patches'
|
||||
|
||||
LONG_DESCRIPTION = '\n\n'.join((
|
||||
'Official repository: <https://github.com/yt-dlp/yt-dlp>',
|
||||
'**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
|
||||
read_file('README.md')))
|
||||
|
||||
REQUIREMENTS = read_file('requirements.txt').splitlines()
|
||||
|
||||
|
||||
def packages():
|
||||
if setuptools_available:
|
||||
return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins', 'devscripts'))
|
||||
|
||||
return [
|
||||
'yt_dlp', 'yt_dlp.extractor', 'yt_dlp.downloader', 'yt_dlp.postprocessor', 'yt_dlp.compat',
|
||||
]
|
||||
|
||||
|
||||
def py2exe_params():
|
||||
warnings.warn(
|
||||
'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
|
||||
'It is recommended to run "pyinst.py" to build using pyinstaller instead')
|
||||
|
||||
return {
|
||||
'console': [{
|
||||
'script': './yt_dlp/__main__.py',
|
||||
'dest_base': 'yt-dlp',
|
||||
'icon_resources': [(1, 'devscripts/logo.ico')],
|
||||
}],
|
||||
'version_info': {
|
||||
'version': VERSION,
|
||||
'description': DESCRIPTION,
|
||||
'comments': LONG_DESCRIPTION.split('\n')[0],
|
||||
'product_name': 'yt-dlp',
|
||||
'product_version': VERSION,
|
||||
},
|
||||
'options': {
|
||||
'bundle_files': 0,
|
||||
'compressed': 1,
|
||||
'optimize': 2,
|
||||
'dist_dir': './dist',
|
||||
'excludes': [
|
||||
# py2exe cannot import Crypto
|
||||
'Crypto',
|
||||
'Cryptodome',
|
||||
# py2exe appears to confuse this with our socks library.
|
||||
# We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
|
||||
'urllib3.contrib.socks'
|
||||
],
|
||||
'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
|
||||
# Modules that are only imported dynamically must be added here
|
||||
'includes': ['yt_dlp.compat._legacy', 'yt_dlp.compat._deprecated',
|
||||
'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
|
||||
},
|
||||
'zipfile': None,
|
||||
}
|
||||
|
||||
|
||||
def build_params():
|
||||
files_spec = [
|
||||
('share/bash-completion/completions', ['completions/bash/yt-dlp']),
|
||||
('share/zsh/site-functions', ['completions/zsh/_yt-dlp']),
|
||||
('share/fish/vendor_completions.d', ['completions/fish/yt-dlp.fish']),
|
||||
('share/doc/yt_dlp', ['README.txt']),
|
||||
('share/man/man1', ['yt-dlp.1'])
|
||||
]
|
||||
data_files = []
|
||||
for dirname, files in files_spec:
|
||||
resfiles = []
|
||||
for fn in files:
|
||||
if not os.path.exists(fn):
|
||||
warnings.warn(f'Skipping file {fn} since it is not present. Try running " make pypi-files " first')
|
||||
else:
|
||||
resfiles.append(fn)
|
||||
data_files.append((dirname, resfiles))
|
||||
|
||||
params = {'data_files': data_files}
|
||||
|
||||
if setuptools_available:
|
||||
params['entry_points'] = {
|
||||
'console_scripts': ['yt-dlp = yt_dlp:main'],
|
||||
'pyinstaller40': ['hook-dirs = yt_dlp.__pyinstaller:get_hook_dirs'],
|
||||
}
|
||||
else:
|
||||
params['scripts'] = ['yt-dlp']
|
||||
return params
|
||||
|
||||
|
||||
class build_lazy_extractors(Command):
|
||||
description = 'Build the extractor lazy loading module'
|
||||
user_options = []
|
||||
|
||||
def initialize_options(self):
|
||||
pass
|
||||
|
||||
def finalize_options(self):
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
if self.dry_run:
|
||||
print('Skipping build of lazy extractors in dry run mode')
|
||||
return
|
||||
subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])
|
||||
|
||||
|
||||
def main():
|
||||
if sys.argv[1:2] == ['py2exe']:
|
||||
params = py2exe_params()
|
||||
try:
|
||||
from py2exe import freeze
|
||||
except ImportError:
|
||||
import py2exe # noqa: F401
|
||||
warnings.warn('You are using an outdated version of py2exe. Support for this version will be removed in the future')
|
||||
params['console'][0].update(params.pop('version_info'))
|
||||
params['options'] = {'py2exe': params.pop('options')}
|
||||
else:
|
||||
return freeze(**params)
|
||||
else:
|
||||
params = build_params()
|
||||
|
||||
setup(
|
||||
name='yt-dlp', # package name (do not change/remove comment)
|
||||
version=VERSION,
|
||||
maintainer='pukkandan',
|
||||
maintainer_email='pukkandan.ytdlp@gmail.com',
|
||||
description=DESCRIPTION,
|
||||
long_description=LONG_DESCRIPTION,
|
||||
long_description_content_type='text/markdown',
|
||||
url='https://github.com/yt-dlp/yt-dlp',
|
||||
packages=packages(),
|
||||
install_requires=REQUIREMENTS,
|
||||
python_requires='>=3.8',
|
||||
project_urls={
|
||||
'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
|
||||
'Source': 'https://github.com/yt-dlp/yt-dlp',
|
||||
'Tracker': 'https://github.com/yt-dlp/yt-dlp/issues',
|
||||
'Funding': 'https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators',
|
||||
},
|
||||
classifiers=[
|
||||
'Topic :: Multimedia :: Video',
|
||||
'Development Status :: 5 - Production/Stable',
|
||||
'Environment :: Console',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
'Programming Language :: Python :: 3.10',
|
||||
'Programming Language :: Python :: 3.11',
|
||||
'Programming Language :: Python :: 3.12',
|
||||
'Programming Language :: Python :: Implementation',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
'Programming Language :: Python :: Implementation :: PyPy',
|
||||
'License :: Public Domain',
|
||||
'Operating System :: OS Independent',
|
||||
],
|
||||
cmdclass={'build_lazy_extractors': build_lazy_extractors},
|
||||
**params
|
||||
)
|
||||
|
||||
|
||||
main()
|
||||
@ -0,0 +1,225 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none, mimetype2ext, parse_iso8601, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NinaProtocolIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?ninaprotocol\.com/releases/(?P<id>[^/#?]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.ninaprotocol.com/releases/3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ',
|
||||
'title': 'The Spatulas - March Chant',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'channel': 'ppm',
|
||||
'description': 'md5:bb9f9d39d8f786449cd5d0ff7c5772db',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'timestamp': 1701417610,
|
||||
'uploader': 'ppmrecs',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'display_id': 'the-spatulas-march-chant',
|
||||
'upload_date': '20231201',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_1',
|
||||
'title': 'March Chant In April',
|
||||
'track': 'March Chant In April',
|
||||
'ext': 'mp3',
|
||||
'duration': 152,
|
||||
'track_number': 1,
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'uploader': 'ppmrecs',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'timestamp': 1701417610,
|
||||
'channel': 'ppm',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'upload_date': '20231201',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_2',
|
||||
'title': 'Rescue Mission',
|
||||
'track': 'Rescue Mission',
|
||||
'ext': 'mp3',
|
||||
'duration': 212,
|
||||
'track_number': 2,
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'uploader': 'ppmrecs',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'channel': 'ppm',
|
||||
'upload_date': '20231201',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'timestamp': 1701417610,
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_3',
|
||||
'title': 'Slinger Style',
|
||||
'track': 'Slinger Style',
|
||||
'ext': 'mp3',
|
||||
'duration': 179,
|
||||
'track_number': 3,
|
||||
'timestamp': 1701417610,
|
||||
'upload_date': '20231201',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'uploader': 'ppmrecs',
|
||||
'channel': 'ppm',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_4',
|
||||
'title': 'Psychic Signal',
|
||||
'track': 'Psychic Signal',
|
||||
'ext': 'mp3',
|
||||
'duration': 220,
|
||||
'track_number': 4,
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'upload_date': '20231201',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'timestamp': 1701417610,
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'channel': 'ppm',
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'uploader': 'ppmrecs',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_5',
|
||||
'title': 'Curvy Color',
|
||||
'track': 'Curvy Color',
|
||||
'ext': 'mp3',
|
||||
'duration': 148,
|
||||
'track_number': 5,
|
||||
'timestamp': 1701417610,
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'channel': 'ppm',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'uploader': 'ppmrecs',
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'upload_date': '20231201',
|
||||
}
|
||||
}, {
|
||||
'info_dict': {
|
||||
'id': '3SvsMM3y4oTPZ5DXFJnLkCAqkxz34hjzFxqms1vu9XBJ_6',
|
||||
'title': 'Caveman Star',
|
||||
'track': 'Caveman Star',
|
||||
'ext': 'mp3',
|
||||
'duration': 121,
|
||||
'track_number': 6,
|
||||
'channel_id': '4ceG4zsb7VVxBTGPtZMqDZWGHo3VUg2xRvzC2b17ymWP',
|
||||
'thumbnail': 'https://www.arweave.net/VyZA6CBeUuqP174khvSrD44Eosi3MLVyWN42uaQKg50',
|
||||
'tags': ['punk', 'postpresentmedium', 'cambridge'],
|
||||
'album_artist': 'Post Present Medium ',
|
||||
'uploader': 'ppmrecs',
|
||||
'timestamp': 1701417610,
|
||||
'uploader_id': '2bGjgdKUddJoj2shYGqfNcUfoSoABP21RJoiwGMZDq3A',
|
||||
'album': 'The Spatulas - March Chant',
|
||||
'channel': 'ppm',
|
||||
'upload_date': '20231201',
|
||||
},
|
||||
}],
|
||||
}, {
|
||||
'url': 'https://www.ninaprotocol.com/releases/f-g-s-american-shield',
|
||||
'info_dict': {
|
||||
'id': '76PZnJwaMgViQHYfA4NYJXds7CmW6vHQKAtQUxGene6J',
|
||||
'description': 'md5:63f08d5db558b4b36e1896f317062721',
|
||||
'title': 'F.G.S. - American Shield',
|
||||
'uploader_id': 'Ej3rozs11wYqFk1Gs6oggGCkGLz8GzBhmJfnUxf6gPci',
|
||||
'channel_id': '6JuksCZPXuP16wJ1BUfwuukJzh42C7guhLrFPPkVJfyE',
|
||||
'channel': 'tinkscough',
|
||||
'tags': [],
|
||||
'album_artist': 'F.G.S.',
|
||||
'album': 'F.G.S. - American Shield',
|
||||
'thumbnail': 'https://www.arweave.net/YJpgImkXLT9SbpFb576KuZ5pm6bdvs452LMs3Rx6lm8',
|
||||
'display_id': 'f-g-s-american-shield',
|
||||
'uploader': 'flannerysilva',
|
||||
'timestamp': 1702395858,
|
||||
'upload_date': '20231212',
|
||||
},
|
||||
'playlist_count': 1,
|
||||
}, {
|
||||
'url': 'https://www.ninaprotocol.com/releases/time-to-figure-things-out',
|
||||
'info_dict': {
|
||||
'id': '6Zi1nC5hj6b13NkpxVYwRhFy6mYA7oLBbe9DMrgGDcYh',
|
||||
'display_id': 'time-to-figure-things-out',
|
||||
'description': 'md5:960202ed01c3134bb8958f1008527e35',
|
||||
'timestamp': 1706283607,
|
||||
'title': 'DJ STEPDAD - time to figure things out',
|
||||
'album_artist': 'DJ STEPDAD',
|
||||
'uploader': 'tddvsss',
|
||||
'upload_date': '20240126',
|
||||
'album': 'time to figure things out',
|
||||
'uploader_id': 'AXQNRgTyYsySyAMFDwxzumuGjfmoXshorCesjpquwCBi',
|
||||
'thumbnail': 'https://www.arweave.net/O4i8bcKVqJVZvNeHHFp6r8knpFGh9ZwEgbeYacr4nss',
|
||||
'tags': [],
|
||||
},
|
||||
'playlist_count': 4,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
release = self._download_json(
|
||||
f'https://api.ninaprotocol.com/v1/releases/{video_id}', video_id)['release']
|
||||
|
||||
video_id = release.get('publicKey') or video_id
|
||||
|
||||
common_info = traverse_obj(release, {
|
||||
'album': ('metadata', 'properties', 'title', {str}),
|
||||
'album_artist': ((('hub', 'data'), 'publisherAccount'), 'displayName', {str}),
|
||||
'timestamp': ('datetime', {parse_iso8601}),
|
||||
'thumbnail': ('metadata', 'image', {url_or_none}),
|
||||
'uploader': ('publisherAccount', 'handle', {str}),
|
||||
'uploader_id': ('publisherAccount', 'publicKey', {str}),
|
||||
'channel': ('hub', 'handle', {str}),
|
||||
'channel_id': ('hub', 'publicKey', {str}),
|
||||
}, get_all=False)
|
||||
common_info['tags'] = traverse_obj(release, ('metadata', 'properties', 'tags', ..., {str}))
|
||||
|
||||
entries = []
|
||||
for track_num, track in enumerate(traverse_obj(release, (
|
||||
'metadata', 'properties', 'files', lambda _, v: url_or_none(v['uri']))), 1):
|
||||
entries.append({
|
||||
'id': f'{video_id}_{track_num}',
|
||||
'url': track['uri'],
|
||||
**traverse_obj(track, {
|
||||
'title': ('track_title', {str}),
|
||||
'track': ('track_title', {str}),
|
||||
'ext': ('type', {mimetype2ext}),
|
||||
'track_number': ('track', {int_or_none}),
|
||||
'duration': ('duration', {int_or_none}),
|
||||
}),
|
||||
'vcodec': 'none',
|
||||
**common_info,
|
||||
})
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'id': video_id,
|
||||
'entries': entries,
|
||||
**traverse_obj(release, {
|
||||
'display_id': ('slug', {str}),
|
||||
'title': ('metadata', 'name', {str}),
|
||||
'description': ('metadata', 'description', {str}),
|
||||
}),
|
||||
**common_info,
|
||||
}
|
||||
@ -0,0 +1,199 @@
|
||||
import functools
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
OnDemandPagedList,
|
||||
UserNotLive,
|
||||
filter_dict,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
str_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class NuumBaseIE(InfoExtractor):
|
||||
def _call_api(self, path, video_id, description, query={}):
|
||||
response = self._download_json(
|
||||
f'https://nuum.ru/api/v2/{path}', video_id, query=query,
|
||||
note=f'Downloading {description} metadata',
|
||||
errnote=f'Unable to download {description} metadata')
|
||||
if error := response.get('error'):
|
||||
raise ExtractorError(f'API returned error: {error!r}')
|
||||
return response['result']
|
||||
|
||||
def _get_channel_info(self, channel_name):
|
||||
return self._call_api(
|
||||
'broadcasts/public', video_id=channel_name, description='channel',
|
||||
query={
|
||||
'with_extra': 'true',
|
||||
'channel_name': channel_name,
|
||||
'with_deleted': 'true',
|
||||
})
|
||||
|
||||
def _parse_video_data(self, container, extract_formats=True):
|
||||
stream = traverse_obj(container, ('media_container_streams', 0, {dict})) or {}
|
||||
media = traverse_obj(stream, ('stream_media', 0, {dict})) or {}
|
||||
media_url = traverse_obj(media, (
|
||||
'media_meta', ('media_archive_url', 'media_url'), {url_or_none}), get_all=False)
|
||||
|
||||
video_id = str(container['media_container_id'])
|
||||
is_live = media.get('media_status') == 'RUNNING'
|
||||
|
||||
formats, subtitles = None, None
|
||||
if extract_formats:
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
media_url, video_id, 'mp4', live=is_live)
|
||||
|
||||
return filter_dict({
|
||||
'id': video_id,
|
||||
'is_live': is_live,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
**traverse_obj(container, {
|
||||
'title': ('media_container_name', {str}),
|
||||
'description': ('media_container_description', {str}),
|
||||
'timestamp': ('created_at', {parse_iso8601}),
|
||||
'channel': ('media_container_channel', 'channel_name', {str}),
|
||||
'channel_id': ('media_container_channel', 'channel_id', {str_or_none}),
|
||||
}),
|
||||
**traverse_obj(stream, {
|
||||
'view_count': ('stream_total_viewers', {int_or_none}),
|
||||
'concurrent_view_count': ('stream_current_viewers', {int_or_none}),
|
||||
}),
|
||||
**traverse_obj(media, {
|
||||
'duration': ('media_duration', {int_or_none}),
|
||||
'thumbnail': ('media_meta', ('media_preview_archive_url', 'media_preview_url'), {url_or_none}),
|
||||
}, get_all=False),
|
||||
})
|
||||
|
||||
|
||||
class NuumMediaIE(NuumBaseIE):
|
||||
IE_NAME = 'nuum:media'
|
||||
_VALID_URL = r'https?://nuum\.ru/(?:streams|videos|clips)/(?P<id>[\d]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://nuum.ru/streams/1592713-7-days-to-die',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://nuum.ru/videos/1567547-toxi-hurtz',
|
||||
'md5': 'f1d9118a30403e32b702a204eb03aca3',
|
||||
'info_dict': {
|
||||
'id': '1567547',
|
||||
'ext': 'mp4',
|
||||
'title': 'Toxi$ - Hurtz',
|
||||
'description': '',
|
||||
'timestamp': 1702631651,
|
||||
'upload_date': '20231215',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'view_count': int,
|
||||
'concurrent_view_count': int,
|
||||
'channel_id': '6911',
|
||||
'channel': 'toxis',
|
||||
'duration': 116,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://nuum.ru/clips/1552564-pro-misu',
|
||||
'md5': 'b248ae1565b1e55433188f11beeb0ca1',
|
||||
'info_dict': {
|
||||
'id': '1552564',
|
||||
'ext': 'mp4',
|
||||
'title': 'Про Мису 🙃',
|
||||
'timestamp': 1701971828,
|
||||
'upload_date': '20231207',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'view_count': int,
|
||||
'concurrent_view_count': int,
|
||||
'channel_id': '3320',
|
||||
'channel': 'Misalelik',
|
||||
'duration': 41,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
video_data = self._call_api(f'media-containers/{video_id}', video_id, 'media')
|
||||
|
||||
return self._parse_video_data(video_data)
|
||||
|
||||
|
||||
class NuumLiveIE(NuumBaseIE):
|
||||
IE_NAME = 'nuum:live'
|
||||
_VALID_URL = r'https?://nuum\.ru/channel/(?P<id>[^/#?]+)/?(?:$|[#?])'
|
||||
_TESTS = [{
|
||||
'url': 'https://nuum.ru/channel/mts_live',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel = self._match_id(url)
|
||||
channel_info = self._get_channel_info(channel)
|
||||
if traverse_obj(channel_info, ('channel', 'channel_is_live')) is False:
|
||||
raise UserNotLive(video_id=channel)
|
||||
|
||||
info = self._parse_video_data(channel_info['media_container'])
|
||||
return {
|
||||
'webpage_url': f'https://nuum.ru/streams/{info["id"]}',
|
||||
'extractor_key': NuumMediaIE.ie_key(),
|
||||
'extractor': NuumMediaIE.IE_NAME,
|
||||
**info,
|
||||
}
|
||||
|
||||
|
||||
class NuumTabIE(NuumBaseIE):
|
||||
IE_NAME = 'nuum:tab'
|
||||
_VALID_URL = r'https?://nuum\.ru/channel/(?P<id>[^/#?]+)/(?P<type>streams|videos|clips)'
|
||||
_TESTS = [{
|
||||
'url': 'https://nuum.ru/channel/dankon_/clips',
|
||||
'info_dict': {
|
||||
'id': 'dankon__clips',
|
||||
'title': 'Dankon_',
|
||||
},
|
||||
'playlist_mincount': 29,
|
||||
}, {
|
||||
'url': 'https://nuum.ru/channel/dankon_/videos',
|
||||
'info_dict': {
|
||||
'id': 'dankon__videos',
|
||||
'title': 'Dankon_',
|
||||
},
|
||||
'playlist_mincount': 2,
|
||||
}, {
|
||||
'url': 'https://nuum.ru/channel/dankon_/streams',
|
||||
'info_dict': {
|
||||
'id': 'dankon__streams',
|
||||
'title': 'Dankon_',
|
||||
},
|
||||
'playlist_mincount': 1,
|
||||
}]
|
||||
|
||||
_PAGE_SIZE = 50
|
||||
|
||||
def _fetch_page(self, channel_id, tab_type, tab_id, page):
|
||||
CONTAINER_TYPES = {
|
||||
'clips': ['SHORT_VIDEO', 'REVIEW_VIDEO'],
|
||||
'videos': ['LONG_VIDEO'],
|
||||
'streams': ['SINGLE'],
|
||||
}
|
||||
|
||||
media_containers = self._call_api(
|
||||
'media-containers', video_id=tab_id, description=f'{tab_type} tab page {page + 1}',
|
||||
query={
|
||||
'limit': self._PAGE_SIZE,
|
||||
'offset': page * self._PAGE_SIZE,
|
||||
'channel_id': channel_id,
|
||||
'media_container_status': 'STOPPED',
|
||||
'media_container_type': CONTAINER_TYPES[tab_type],
|
||||
})
|
||||
for container in traverse_obj(media_containers, (..., {dict})):
|
||||
metadata = self._parse_video_data(container, extract_formats=False)
|
||||
yield self.url_result(f'https://nuum.ru/videos/{metadata["id"]}', NuumMediaIE, **metadata)
|
||||
|
||||
def _real_extract(self, url):
|
||||
channel_name, tab_type = self._match_valid_url(url).group('id', 'type')
|
||||
tab_id = f'{channel_name}_{tab_type}'
|
||||
channel_data = self._get_channel_info(channel_name)['channel']
|
||||
|
||||
return self.playlist_result(OnDemandPagedList(functools.partial(
|
||||
self._fetch_page, channel_data['channel_id'], tab_type, tab_id), self._PAGE_SIZE),
|
||||
playlist_id=tab_id, playlist_title=channel_data.get('channel_name'))
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,159 +0,0 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
traverse_obj,
|
||||
try_get,
|
||||
)
|
||||
|
||||
|
||||
class WASDTVBaseIE(InfoExtractor):
|
||||
|
||||
def _fetch(self, path, video_id, description, query={}):
|
||||
response = self._download_json(
|
||||
f'https://wasd.tv/api/{path}', video_id, query=query,
|
||||
note=f'Downloading {description} metadata',
|
||||
errnote=f'Unable to download {description} metadata')
|
||||
error = response.get('error')
|
||||
if error:
|
||||
raise ExtractorError(f'{self.IE_NAME} returned error: {error}', expected=True)
|
||||
return response.get('result')
|
||||
|
||||
def _extract_thumbnails(self, thumbnails_dict):
|
||||
return [{
|
||||
'url': url,
|
||||
'preference': index,
|
||||
} for index, url in enumerate(
|
||||
traverse_obj(thumbnails_dict, (('small', 'medium', 'large'),))) if url]
|
||||
|
||||
def _real_extract(self, url):
|
||||
container = self._get_container(url)
|
||||
stream = traverse_obj(container, ('media_container_streams', 0))
|
||||
media = try_get(stream, lambda x: x['stream_media'][0])
|
||||
if not media:
|
||||
raise ExtractorError('Can not extract media data.', expected=True)
|
||||
media_meta = media.get('media_meta')
|
||||
media_url, is_live = self._get_media_url(media_meta)
|
||||
video_id = media.get('media_id') or container.get('media_container_id')
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(media_url, video_id, 'mp4')
|
||||
return {
|
||||
'id': str(video_id),
|
||||
'title': container.get('media_container_name') or self._og_search_title(self._download_webpage(url, video_id)),
|
||||
'description': container.get('media_container_description'),
|
||||
'thumbnails': self._extract_thumbnails(media_meta.get('media_preview_images')),
|
||||
'timestamp': parse_iso8601(container.get('created_at')),
|
||||
'view_count': int_or_none(stream.get('stream_current_viewers' if is_live else 'stream_total_viewers')),
|
||||
'is_live': is_live,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
|
||||
def _get_container(self, url):
|
||||
raise NotImplementedError('Subclass for get media container')
|
||||
|
||||
def _get_media_url(self, media_meta):
|
||||
raise NotImplementedError('Subclass for get media url')
|
||||
|
||||
|
||||
class WASDTVStreamIE(WASDTVBaseIE):
|
||||
IE_NAME = 'wasdtv:stream'
|
||||
_VALID_URL = r'https?://wasd\.tv/(?P<id>[^/#?]+)$'
|
||||
_TESTS = [{
|
||||
'url': 'https://wasd.tv/24_7',
|
||||
'info_dict': {
|
||||
'id': '559738',
|
||||
'ext': 'mp4',
|
||||
'title': 'Live 24/7 Music',
|
||||
'description': '24/7 Music',
|
||||
'timestamp': int,
|
||||
'upload_date': r're:^\d{8}$',
|
||||
'is_live': True,
|
||||
'view_count': int,
|
||||
},
|
||||
}]
|
||||
|
||||
def _get_container(self, url):
|
||||
nickname = self._match_id(url)
|
||||
channel = self._fetch(f'channels/nicknames/{nickname}', video_id=nickname, description='channel')
|
||||
channel_id = channel.get('channel_id')
|
||||
containers = self._fetch(
|
||||
'v2/media-containers', channel_id, 'running media containers',
|
||||
query={
|
||||
'channel_id': channel_id,
|
||||
'media_container_type': 'SINGLE',
|
||||
'media_container_status': 'RUNNING',
|
||||
})
|
||||
if not containers:
|
||||
raise ExtractorError(f'{nickname} is offline', expected=True)
|
||||
return containers[0]
|
||||
|
||||
def _get_media_url(self, media_meta):
|
||||
return media_meta['media_url'], True
|
||||
|
||||
|
||||
class WASDTVRecordIE(WASDTVBaseIE):
|
||||
IE_NAME = 'wasdtv:record'
|
||||
_VALID_URL = r'https?://wasd\.tv/[^/#?]+(?:/videos)?\?record=(?P<id>\d+)$'
|
||||
_TESTS = [{
|
||||
'url': 'https://wasd.tv/spacemita/videos?record=907755',
|
||||
'md5': 'c9899dd85be4cc997816ff9f9ca516ce',
|
||||
'info_dict': {
|
||||
'id': '906825',
|
||||
'ext': 'mp4',
|
||||
'title': 'Музыкальный',
|
||||
'description': 'md5:f510388d929ff60ae61d4c3cab3137cc',
|
||||
'timestamp': 1645812079,
|
||||
'upload_date': '20220225',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'is_live': False,
|
||||
'view_count': int,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://wasd.tv/spacemita?record=907755',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _get_container(self, url):
|
||||
container_id = self._match_id(url)
|
||||
return self._fetch(
|
||||
f'v2/media-containers/{container_id}', container_id, 'media container')
|
||||
|
||||
def _get_media_url(self, media_meta):
|
||||
media_archive_url = media_meta.get('media_archive_url')
|
||||
if media_archive_url:
|
||||
return media_archive_url, False
|
||||
return media_meta['media_url'], True
|
||||
|
||||
|
||||
class WASDTVClipIE(WASDTVBaseIE):
|
||||
IE_NAME = 'wasdtv:clip'
|
||||
_VALID_URL = r'https?://wasd\.tv/[^/#?]+/clips\?clip=(?P<id>\d+)$'
|
||||
_TESTS = [{
|
||||
'url': 'https://wasd.tv/spacemita/clips?clip=26804',
|
||||
'md5': '818885e720143d7a4e776ff66fcff148',
|
||||
'info_dict': {
|
||||
'id': '26804',
|
||||
'ext': 'mp4',
|
||||
'title': 'Пуш флексит на голове стримера',
|
||||
'timestamp': 1646682908,
|
||||
'upload_date': '20220307',
|
||||
'thumbnail': r're:^https?://.+\.jpg',
|
||||
'view_count': int,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
clip_id = self._match_id(url)
|
||||
clip = self._fetch(f'v2/clips/{clip_id}', video_id=clip_id, description='clip')
|
||||
clip_data = clip.get('clip_data')
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(clip_data.get('url'), video_id=clip_id, ext='mp4')
|
||||
return {
|
||||
'id': clip_id,
|
||||
'title': clip.get('clip_title') or self._og_search_title(self._download_webpage(url, clip_id, fatal=False)),
|
||||
'thumbnails': self._extract_thumbnails(clip_data.get('preview')),
|
||||
'timestamp': parse_iso8601(clip.get('created_at')),
|
||||
'view_count': int_or_none(clip.get('clip_views_count')),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
from .common import InfoExtractor
|
||||
from ..utils import merge_dicts, unified_timestamp, url_or_none
|
||||
from ..utils.traversal import traverse_obj
|
||||
|
||||
|
||||
class ZetlandDKArticleIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://www\.zetland\.dk/\w+/(?P<id>(?P<story_id>\w{8})-(?P<uploader_id>\w{8})-(?:\w{5}))'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.zetland.dk/historie/sO9aq2MY-a81VP3BY-66e69?utm_source=instagram&utm_medium=linkibio&utm_campaign=artikel',
|
||||
'info_dict': {
|
||||
'id': 'sO9aq2MY-a81VP3BY-66e69',
|
||||
'ext': 'mp3',
|
||||
'modified_date': '20240118',
|
||||
'title': 'Afsnit 1: “Det føltes som en kidnapning.” ',
|
||||
'upload_date': '20240116',
|
||||
'uploader_id': 'a81VP3BY',
|
||||
'modified_timestamp': 1705568739,
|
||||
'release_timestamp': 1705377592,
|
||||
'uploader_url': 'https://www.zetland.dk/skribent/a81VP3BY',
|
||||
'uploader': 'Helle Fuusager',
|
||||
'release_date': '20240116',
|
||||
'thumbnail': r're:https://zetland\.imgix\.net/2aafe500-b14e-11ee-bf83-65d5e1283a57/Zetland_Image_1\.jpg',
|
||||
'description': 'md5:9619d426772c133f5abb26db27f26a01',
|
||||
'timestamp': 1705377592,
|
||||
'series_id': '62d54630-e87b-4ab1-a255-8de58dbe1b14',
|
||||
}
|
||||
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
next_js_data = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
|
||||
story_data = traverse_obj(next_js_data, ('initialState', 'consume', 'story', 'story'))
|
||||
|
||||
formats = []
|
||||
for audio_url in traverse_obj(story_data, ('story_content', 'meta', 'audioFiles', ..., {url_or_none})):
|
||||
formats.append({
|
||||
'url': audio_url,
|
||||
'vcodec': 'none',
|
||||
})
|
||||
|
||||
return merge_dicts({
|
||||
'id': display_id,
|
||||
'formats': formats,
|
||||
'uploader_id': uploader_id
|
||||
}, traverse_obj(story_data, {
|
||||
'title': ((('story_content', 'content', 'title'), 'title'), {str}),
|
||||
'uploader': ('sharer', 'name'),
|
||||
'uploader_id': ('sharer', 'sharer_id'),
|
||||
'description': ('story_content', 'content', 'socialDescription'),
|
||||
'series_id': ('story_content', 'meta', 'seriesId'),
|
||||
'release_timestamp': ('published_at', {unified_timestamp}),
|
||||
'modified_timestamp': ('revised_at', {unified_timestamp}),
|
||||
}, get_all=False), traverse_obj(next_js_data, ('metaInfo', {
|
||||
'title': ((('meta', 'title'), ('ld', 'headline'), ('og', 'og:title'), ('og', 'twitter:title')), {str}),
|
||||
'description': ((('meta', 'description'), ('ld', 'description'), ('og', 'og:description'), ('og', 'twitter:description')), {str}),
|
||||
'uploader': ((('meta', 'author'), ('ld', 'author', 'name')), {str}),
|
||||
'uploader_url': ('ld', 'author', 'url', {url_or_none}),
|
||||
'thumbnail': ((('ld', 'image'), ('og', 'og:image'), ('og', 'twitter:image')), {url_or_none}),
|
||||
'modified_timestamp': ('ld', 'dateModified', {unified_timestamp}),
|
||||
'release_timestamp': ('ld', 'datePublished', {unified_timestamp}),
|
||||
'timestamp': ('ld', 'dateCreated', {unified_timestamp}),
|
||||
}), get_all=False), {
|
||||
'title': self._html_search_meta(['title', 'og:title', 'twitter:title'], webpage),
|
||||
'description': self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage),
|
||||
'thumbnail': self._html_search_meta(['og:image', 'twitter:image'], webpage),
|
||||
'uploader': self._html_search_meta(['author'], webpage),
|
||||
'release_timestamp': unified_timestamp(self._html_search_meta(['article:published_time'], webpage)),
|
||||
}, self._search_json_ld(webpage, display_id, fatal=False))
|
||||
Loading…
Reference in New Issue