|
|
|
@ -6,6 +6,7 @@ import hashlib
|
|
|
|
import json
|
|
|
|
import json
|
|
|
|
import netrc
|
|
|
|
import netrc
|
|
|
|
import os
|
|
|
|
import os
|
|
|
|
|
|
|
|
import random
|
|
|
|
import re
|
|
|
|
import re
|
|
|
|
import socket
|
|
|
|
import socket
|
|
|
|
import sys
|
|
|
|
import sys
|
|
|
|
@ -39,6 +40,8 @@ from ..utils import (
|
|
|
|
ExtractorError,
|
|
|
|
ExtractorError,
|
|
|
|
fix_xml_ampersands,
|
|
|
|
fix_xml_ampersands,
|
|
|
|
float_or_none,
|
|
|
|
float_or_none,
|
|
|
|
|
|
|
|
GeoRestrictedError,
|
|
|
|
|
|
|
|
GeoUtils,
|
|
|
|
int_or_none,
|
|
|
|
int_or_none,
|
|
|
|
js_to_json,
|
|
|
|
js_to_json,
|
|
|
|
parse_iso8601,
|
|
|
|
parse_iso8601,
|
|
|
|
@ -320,17 +323,25 @@ class InfoExtractor(object):
|
|
|
|
_real_extract() methods and define a _VALID_URL regexp.
|
|
|
|
_real_extract() methods and define a _VALID_URL regexp.
|
|
|
|
Probably, they should also be added to the list of extractors.
|
|
|
|
Probably, they should also be added to the list of extractors.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_BYPASS_GEO attribute may be set to False in order to disable
|
|
|
|
|
|
|
|
geo restriction bypass mechanisms for a particular extractor.
|
|
|
|
|
|
|
|
Though it won't disable explicit geo restriction bypass based on
|
|
|
|
|
|
|
|
country code provided with bypass_geo_restriction_as_country.
|
|
|
|
|
|
|
|
|
|
|
|
Finally, the _WORKING attribute should be set to False for broken IEs
|
|
|
|
Finally, the _WORKING attribute should be set to False for broken IEs
|
|
|
|
in order to warn the users and skip the tests.
|
|
|
|
in order to warn the users and skip the tests.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
_ready = False
|
|
|
|
_ready = False
|
|
|
|
_downloader = None
|
|
|
|
_downloader = None
|
|
|
|
|
|
|
|
_x_forwarded_for_ip = None
|
|
|
|
|
|
|
|
_BYPASS_GEO = True
|
|
|
|
_WORKING = True
|
|
|
|
_WORKING = True
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, downloader=None):
|
|
|
|
def __init__(self, downloader=None):
|
|
|
|
"""Constructor. Receives an optional downloader."""
|
|
|
|
"""Constructor. Receives an optional downloader."""
|
|
|
|
self._ready = False
|
|
|
|
self._ready = False
|
|
|
|
|
|
|
|
self._x_forwarded_for_ip = None
|
|
|
|
self.set_downloader(downloader)
|
|
|
|
self.set_downloader(downloader)
|
|
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
@classmethod
|
|
|
|
@ -359,6 +370,10 @@ class InfoExtractor(object):
|
|
|
|
|
|
|
|
|
|
|
|
def initialize(self):
|
|
|
|
def initialize(self):
|
|
|
|
"""Initializes an instance (authentication, etc)."""
|
|
|
|
"""Initializes an instance (authentication, etc)."""
|
|
|
|
|
|
|
|
if not self._x_forwarded_for_ip:
|
|
|
|
|
|
|
|
country_code = self._downloader.params.get('bypass_geo_restriction_as_country', None)
|
|
|
|
|
|
|
|
if country_code:
|
|
|
|
|
|
|
|
self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
|
|
|
|
if not self._ready:
|
|
|
|
if not self._ready:
|
|
|
|
self._real_initialize()
|
|
|
|
self._real_initialize()
|
|
|
|
self._ready = True
|
|
|
|
self._ready = True
|
|
|
|
@ -366,8 +381,22 @@ class InfoExtractor(object):
|
|
|
|
def extract(self, url):
|
|
|
|
def extract(self, url):
|
|
|
|
"""Extracts URL information and returns it in list of dicts."""
|
|
|
|
"""Extracts URL information and returns it in list of dicts."""
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
self.initialize()
|
|
|
|
for _ in range(2):
|
|
|
|
return self._real_extract(url)
|
|
|
|
try:
|
|
|
|
|
|
|
|
self.initialize()
|
|
|
|
|
|
|
|
return self._real_extract(url)
|
|
|
|
|
|
|
|
except GeoRestrictedError as e:
|
|
|
|
|
|
|
|
if (not self._downloader.params.get('bypass_geo_restriction_as_country', None) and
|
|
|
|
|
|
|
|
self._BYPASS_GEO and
|
|
|
|
|
|
|
|
self._downloader.params.get('bypass_geo_restriction', True) and
|
|
|
|
|
|
|
|
not self._x_forwarded_for_ip and
|
|
|
|
|
|
|
|
e.countries):
|
|
|
|
|
|
|
|
self._x_forwarded_for_ip = GeoUtils.random_ipv4(random.choice(e.countries))
|
|
|
|
|
|
|
|
if self._x_forwarded_for_ip:
|
|
|
|
|
|
|
|
self.report_warning(
|
|
|
|
|
|
|
|
'Video is geo restricted. Retrying extraction with fake %s IP as X-Forwarded-For.' % self._x_forwarded_for_ip)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
raise
|
|
|
|
except ExtractorError:
|
|
|
|
except ExtractorError:
|
|
|
|
raise
|
|
|
|
raise
|
|
|
|
except compat_http_client.IncompleteRead as e:
|
|
|
|
except compat_http_client.IncompleteRead as e:
|
|
|
|
@ -434,6 +463,15 @@ class InfoExtractor(object):
|
|
|
|
if isinstance(url_or_request, (compat_str, str)):
|
|
|
|
if isinstance(url_or_request, (compat_str, str)):
|
|
|
|
url_or_request = url_or_request.partition('#')[0]
|
|
|
|
url_or_request = url_or_request.partition('#')[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Some sites check X-Forwarded-For HTTP header in order to figure out
|
|
|
|
|
|
|
|
# the origin of the client behind proxy. This allows bypassing geo
|
|
|
|
|
|
|
|
# restriction by faking this header's value to IP that belongs to some
|
|
|
|
|
|
|
|
# geo unrestricted country. We will do so once we encounter any
|
|
|
|
|
|
|
|
# geo restriction error.
|
|
|
|
|
|
|
|
if self._x_forwarded_for_ip:
|
|
|
|
|
|
|
|
if 'X-Forwarded-For' not in headers:
|
|
|
|
|
|
|
|
headers['X-Forwarded-For'] = self._x_forwarded_for_ip
|
|
|
|
|
|
|
|
|
|
|
|
urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
|
|
|
|
urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
|
|
|
|
if urlh is False:
|
|
|
|
if urlh is False:
|
|
|
|
assert not fatal
|
|
|
|
assert not fatal
|
|
|
|
@ -609,10 +647,8 @@ class InfoExtractor(object):
|
|
|
|
expected=True)
|
|
|
|
expected=True)
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
@staticmethod
|
|
|
|
def raise_geo_restricted(msg='This video is not available from your location due to geo restriction'):
|
|
|
|
def raise_geo_restricted(msg='This video is not available from your location due to geo restriction', countries=None):
|
|
|
|
raise ExtractorError(
|
|
|
|
raise GeoRestrictedError(msg, countries=countries)
|
|
|
|
'%s. You might want to use --proxy to workaround.' % msg,
|
|
|
|
|
|
|
|
expected=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Methods for following #608
|
|
|
|
# Methods for following #608
|
|
|
|
@staticmethod
|
|
|
|
@staticmethod
|
|
|
|
|
Sergey M․
8 years ago
committed by