From be4685ab7b6acc8ba0d383a6ae27453566046228 Mon Sep 17 00:00:00 2001
From: "Lesmiscore (Naoya Ozaki)" <nao20010128@gmail.com>
Date: Sat, 19 Mar 2022 10:15:01 +0900
Subject: [PATCH] [http] Reject broken range before request (#3079)
* And fix filesize estimate for byterange downloads
Closes #2001
Authored by: Lesmiscore, Jules-A, pukkandan
---
yt_dlp/downloader/http.py | 25 ++++++++++++++++---------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index c2a57726ec..8e096b76b5 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -18,6 +18,7 @@ from ..utils import (
parse_http_range,
sanitized_Request,
ThrottledDownload,
+ try_get,
write_xattr,
XAttrMetadataError,
XAttrUnavailableError,
@@ -55,7 +56,6 @@ class HttpFD(FileDownloader):
ctx.open_mode = 'wb'
ctx.resume_len = 0
- ctx.data_len = None
ctx.block_size = self.params.get('buffersize', 1024)
ctx.start_time = time.time()
ctx.chunk_size = None
@@ -102,6 +102,8 @@ class HttpFD(FileDownloader):
if ctx.is_resume:
self.report_resuming_byte(ctx.resume_len)
ctx.open_mode = 'ab'
+ elif req_start is not None:
+ range_start = req_start
elif ctx.chunk_size > 0:
range_start = 0
else:
@@ -118,11 +120,16 @@ class HttpFD(FileDownloader):
else:
range_end = None
- if range_end and ctx.data_len is not None and range_end >= ctx.data_len:
- range_end = ctx.data_len - 1
- has_range = range_start is not None
- ctx.has_range = has_range
+ if try_get(None, lambda _: range_start > range_end):
+ ctx.resume_len = 0
+ ctx.open_mode = 'wb'
+ raise RetryDownload(Exception(f'Conflicting range. (start={range_start} > end={range_end})'))
+
+ if try_get(None, lambda _: range_end >= ctx.content_len):
+ range_end = ctx.content_len - 1
+
request = sanitized_Request(url, request_data, headers)
+ has_range = range_start is not None
if has_range:
set_range(request, range_start, range_end)
# Establish connection
@@ -146,7 +153,8 @@ class HttpFD(FileDownloader):
or content_range_end == range_end
or content_len < range_end)
if accept_content_len:
- ctx.data_len = content_len
+ ctx.content_len = content_len
+ ctx.data_len = min(content_len, req_end or content_len) - (req_start or 0)
return
# Content-Range is either not present or invalid. Assuming remote webserver is
# trying to send the whole file, resume is not possible, so wiping the local file
@@ -154,8 +162,7 @@ class HttpFD(FileDownloader):
self.report_unable_to_resume()
ctx.resume_len = 0
ctx.open_mode = 'wb'
- ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))
- return
+ ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
except (compat_urllib_error.HTTPError, ) as err:
if err.code == 416:
# Unable to resume (requested range not satisfiable)
@@ -331,7 +338,7 @@ class HttpFD(FileDownloader):
elif speed:
throttle_start = None
- if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
+ if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
ctx.resume_len = byte_counter
# ctx.block_size = block_size
raise NextFragment()