[youtube] Fix throttling by decrypting n-sig (#1437)

.gitignore

@@ -41,6 +41,7 @@ cookies
 *.webp
 *.annotations.xml
 *.description
+.cache/
 
 # Allow config/media files in testdata
 !test/**

test/test_jsinterp.py

@@ -112,6 +112,56 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_for_loop(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_signature.py

@@ -14,9 +14,10 @@ import string
 
 from test.helper import FakeYDL, is_download_test
 from yt_dlp.extractor import YoutubeIE
+from yt_dlp.jsinterp import JSInterpreter
 from yt_dlp.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,13 @@ _TESTS = [
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),  # TODO: Add more tests
+]
+
 
 @is_download_test
 class TestPlayerInfo(unittest.TestCase):
@@ -97,35 +105,49 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
+def t_factory(name, sig_func, url_pattern):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
+    def make_tfunc(url, sig_input, expected_sig):
-    assert m, '%r should follow URL format' % url
+        m = url_pattern.match(url)
-    test_id = m.group(1)
+        assert m, '%r should follow URL format' % url
+        test_id = m.group('id')
+
+        def test_func(self):
+            basename = f'player-{name}-{test_id}.js'
+            fn = os.path.join(self.TESTDATA_DIR, basename)
+
+            if not os.path.exists(fn):
+                compat_urlretrieve(url, fn)
+            with io.open(fn, encoding='utf-8') as testf:
+                jscode = testf.read()
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
+
+        test_func.__name__ = f'test_{name}_js_{test_id}'
+        setattr(TestSignature, test_func.__name__, test_func)
+    return make_tfunc
+
 
-    def test_func(self):
+def signature(jscode, sig_input):
-        basename = 'player-%s.js' % test_id
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
-        fn = os.path.join(self.TESTDATA_DIR, basename)
+    src_sig = (
+        compat_str(string.printable[:sig_input])
+        if isinstance(sig_input, int) else sig_input)
+    return func(src_sig)
 
-        if not os.path.exists(fn):
-            compat_urlretrieve(url, fn)
 
-        ydl = FakeYDL()
+def n_sig(jscode, sig_input):
-        ie = YoutubeIE(ydl)
+    funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
-        with io.open(fn, encoding='utf-8') as testf:
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
-            jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
-        src_sig = (
-            compat_str(string.printable[:sig_input])
-            if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
 
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
 
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
 
-for test_spec in _TESTS:
+make_nsig_test = t_factory(
-    make_tfunc(*test_spec)
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':

yt_dlp/extractor/youtube.py

@@ -1720,7 +1720,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             raise ExtractorError('Cannot identify player %r' % player_url)
         return id_m.group('id')
 
-    def _load_player(self, video_id, player_url, fatal=True) -> bool:
+    def _load_player(self, video_id, player_url, fatal=True):
         player_id = self._extract_player_info(player_url)
         if player_id not in self._code_cache:
             code = self._download_webpage(
@@ -1729,7 +1729,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 errnote='Download of %s failed' % player_url)
             if code:
                 self._code_cache[player_id] = code
-        return player_id in self._code_cache
+        return self._code_cache.get(player_id)
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         player_id = self._extract_player_info(player_url)
@@ -1743,8 +1743,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
-        if self._load_player(video_id, player_url):
+        code = self._load_player(video_id, player_url)
-            code = self._code_cache[player_id]
+        if code:
             res = self._parse_sig_js(code)
 
             test_string = ''.join(map(compat_chr, range(len(example_sig))))
@@ -1755,6 +1755,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return res
 
     def _print_sig_code(self, func, example_sig):
+        if not self.get_param('youtube_print_sig_code'):
+            return
+
         def gen_sig_code(idxs):
             def _genslice(start, end, step):
                 starts = '' if start == 0 else str(start)
@@ -1831,13 +1834,58 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 )
                 self._player_cache[player_id] = func
             func = self._player_cache[player_id]
-            if self.get_param('youtube_print_sig_code'):
+            self._print_sig_code(func, s)
-                self._print_sig_code(func, s)
             return func(s)
         except Exception as e:
-            tb = traceback.format_exc()
+            raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
-            raise ExtractorError(
+
-                'Signature extraction failed: ' + tb, cause=e)
+    def _decrypt_nsig(self, s, video_id, player_url):
+        """Turn the encrypted n field into a working signature"""
+        if player_url is None:
+            raise ExtractorError('Cannot decrypt nsig without player_url')
+        if player_url.startswith('//'):
+            player_url = 'https:' + player_url
+        elif not re.match(r'https?://', player_url):
+            player_url = compat_urlparse.urljoin(
+                'https://www.youtube.com', player_url)
+
+        sig_id = ('nsig_value', s)
+        if sig_id in self._player_cache:
+            return self._player_cache[sig_id]
+
+        try:
+            player_id = ('nsig', player_url)
+            if player_id not in self._player_cache:
+                self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
+            func = self._player_cache[player_id]
+            self._player_cache[sig_id] = func(s)
+            self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
+            return self._player_cache[sig_id]
+        except Exception as e:
+            raise ExtractorError(traceback.format_exc(), cause=e)
+
+    def _extract_n_function_name(self, jscode):
+        return self._search_regex(
+            (r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
+            jscode, 'Initial JS player n function name', group='nfunc')
+
+    def _extract_n_function(self, video_id, player_url):
+        player_id = self._extract_player_info(player_url)
+        func_code = self._downloader.cache.load('youtube-nsig', player_id)
+
+        if func_code:
+            jsi = JSInterpreter(func_code)
+        else:
+            jscode = self._load_player(video_id, player_url)
+            funcname = self._extract_n_function_name(jscode)
+            jsi = JSInterpreter(jscode)
+            func_code = jsi.extract_function_code(funcname)
+            self._downloader.cache.store('youtube-nsig', player_id, func_code)
+
+        if self.get_param('youtube_print_sig_code'):
+            self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
+
+        return lambda s: jsi.extract_function_from_code(*func_code)([s])
 
     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
         """
@@ -1856,9 +1904,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     raise ExtractorError(error_msg)
                 self.report_warning(error_msg)
                 return
-            if self._load_player(video_id, player_url, fatal=fatal):
+            code = self._load_player(video_id, player_url, fatal=fatal)
-                player_id = self._extract_player_info(player_url)
+            if code:
-                code = self._code_cache[player_id]
                 sts = int_or_none(self._search_regex(
                     r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
                     'JS player signature timestamp', group='sts', fatal=fatal))
@@ -2440,6 +2487,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
                 fmt_url += '&' + sp + '=' + signature
 
+            query = parse_qs(fmt_url)
+            throttled = False
+            if query.get('ratebypass') != ['yes'] and query.get('n'):
+                try:
+                    fmt_url = update_url_query(fmt_url, {
+                        'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
+                except ExtractorError as e:
+                    self.report_warning(f'nsig extraction failed: You may experience throttling for some formats\n{e}', only_once=True)
+                    throttled = True
+
             if itag:
                 itags.append(itag)
                 stream_ids.append(stream_id)
@@ -2453,7 +2510,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'format_note': ', '.join(filter(None, (
                     '%s%s' % (audio_track.get('displayName') or '',
                               ' (default)' if audio_track.get('audioIsDefault') else ''),
-                    fmt.get('qualityLabel') or quality.replace('audio_quality_', '')))),
+                    fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
+                    throttled and 'THROTTLED'))),
+                'source_preference': -10 if not throttled else -1,
                 'fps': int_or_none(fmt.get('fps')),
                 'height': height,
                 'quality': q(quality),
@@ -2645,12 +2704,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if reason:
                 self.raise_no_formats(reason, expected=True)
 
-        for f in formats:
-            if '&c=WEB&' in f['url'] and '&ratebypass=yes&' not in f['url']:  # throttled
-                f['source_preference'] = -10
-                # TODO: this method is not reliable
-                f['format_note'] = format_field(f, 'format_note', '%s ') + '(maybe throttled)'
-
         # Source is given priority since formats that throttle are given lower source_preference
         # When throttling issue is fully fixed, remove this
         self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang'))

yt_dlp/jsinterp.py

@@ -1,5 +1,4 @@
-from __future__ import unicode_literals
+from collections.abc import MutableMapping
-
 import json
 import operator
 import re
@@ -22,11 +21,54 @@ _OPERATORS = [
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
 
+class JS_Break(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid break')
+
+
+class JS_Continue(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid continue')
+
+
+class LocalNameSpace(MutableMapping):
+    def __init__(self, *stack):
+        self.stack = tuple(stack)
+
+    def __getitem__(self, key):
+        for scope in self.stack:
+            if key in scope:
+                return scope[key]
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        for scope in self.stack:
+            if key in scope:
+                scope[key] = value
+                break
+        else:
+            self.stack[0][key] = value
+        return value
+
+    def __delitem__(self, key):
+        raise NotImplementedError('Deleting is not supported')
+
+    def __iter__(self):
+        for scope in self.stack:
+            yield from scope
+
+    def __len__(self, key):
+        return len(iter(self))
+
+    def __repr__(self):
+        return f'LocalNameSpace{self.stack}'
+
+
 class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
@@ -34,11 +76,58 @@ class JSInterpreter(object):
         self.code = code
         self._functions = {}
         self._objects = objects
+        self.__named_object_counter = 0
+
+    def _named_object(self, namespace, obj):
+        self.__named_object_counter += 1
+        name = f'__yt_dlp_jsinterp_obj{self.__named_object_counter}'
+        namespace[name] = obj
+        return name
+
+    @staticmethod
+    def _seperate(expr, delim=',', max_split=None):
+        if not expr:
+            return
+        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
+        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        for idx, char in enumerate(expr):
+            if char in parens:
+                parens[char] += 1
+            is_in_parens = (parens['['] - parens[']']
+                            or parens['('] - parens[')']
+                            or parens['{'] - parens['}'])
+            if char == delim[pos] and not is_in_parens:
+                if pos == max_pos:
+                    pos = 0
+                    yield expr[start: idx - max_pos]
+                    start = idx + 1
+                    splits += 1
+                    if max_split and splits >= max_split:
+                        break
+                else:
+                    pos += 1
+            else:
+                pos = 0
+        yield expr[start:]
+
+    @staticmethod
+    def _seperate_at_paren(expr, delim):
+        seperated = list(JSInterpreter._seperate(expr, delim, 1))
+        if len(seperated) < 2:
+            raise ExtractorError(f'No terminating paren {delim} in {expr}')
+        return seperated[0][1:].strip(), seperated[1].strip()
 
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
+        sub_statements = list(self._seperate(stmt, ';'))
+        stmt = (sub_statements or ['']).pop()
+        for sub_stmt in sub_statements:
+            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+
         should_abort = False
         stmt = stmt.lstrip()
         stmt_m = re.match(r'var\s', stmt)
@@ -61,25 +150,118 @@ class JSInterpreter(object):
         if expr == '':  # Empty expression
             return None
 
+        if expr.startswith('{'):
+            inner, outer = self._seperate_at_paren(expr, '}')
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            if not outer or should_abort:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
         if expr.startswith('('):
-            parens_count = 0
+            inner, outer = self._seperate_at_paren(expr, ')')
-            for m in re.finditer(r'[()]', expr):
+            inner = self.interpret_expression(inner, local_vars, allow_recursion)
-                if m.group(0) == '(':
+            if not outer:
-                    parens_count += 1
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
+        if expr.startswith('['):
+            inner, outer = self._seperate_at_paren(expr, ']')
+            name = self._named_object(local_vars, [
+                self.interpret_expression(item, local_vars, allow_recursion)
+                for item in self._seperate(inner)])
+            expr = name + outer
+
+        m = re.match(r'try\s*', expr)
+        if m:
+            if expr[m.end()] == '{':
+                try_expr, expr = self._seperate_at_paren(expr[m.end():], '}')
+            else:
+                try_expr, expr = expr[m.end() - 1:], ''
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        m = re.match(r'catch\s*\(', expr)
+        if m:
+            # We ignore the catch block
+            _, expr = self._seperate_at_paren(expr, '}')
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        m = re.match(r'for\s*\(', expr)
+        if m:
+            constructor, remaining = self._seperate_at_paren(expr[m.end() - 1:], ')')
+            if remaining.startswith('{'):
+                body, expr = self._seperate_at_paren(remaining, '}')
+            else:
+                m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if m:
+                    switch_val, remaining = self._seperate_at_paren(remaining[m.end() - 1:], ')')
+                    body, expr = self._seperate_at_paren(remaining, '}')
+                    body = 'switch(%s){%s}' % (switch_val, body)
                 else:
-                    parens_count -= 1
+                    body, expr = remaining, ''
-                    if parens_count == 0:
+            start, cndn, increment = self._seperate(constructor, ';')
-                        sub_expr = expr[1:m.start()]
+            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
-                        sub_result = self.interpret_expression(
+                raise ExtractorError(
-                            sub_expr, local_vars, allow_recursion)
+                    f'Premature return in the initialization of a for loop in {constructor!r}')
-                        remaining_expr = expr[m.end():].strip()
+            while True:
-                        if not remaining_expr:
+                if not self.interpret_expression(cndn, local_vars, allow_recursion):
-                            return sub_result
+                    break
-                        else:
+                try:
-                            expr = json.dumps(sub_result) + remaining_expr
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    if should_abort:
+                        return ret
+                except JS_Break:
+                    break
+                except JS_Continue:
+                    pass
+                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
+                    raise ExtractorError(
+                        f'Premature return in the initialization of a for loop in {constructor!r}')
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        m = re.match(r'switch\s*\(', expr)
+        if m:
+            switch_val, remaining = self._seperate_at_paren(expr[m.end() - 1:], ')')
+            switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
+            body, expr = self._seperate_at_paren(remaining, '}')
+            body, default = body.split('default:') if 'default:' in body else (body, None)
+            items = body.split('case ')[1:]
+            if default:
+                items.append(f'default:{default}')
+            matched = False
+            for item in items:
+                case, stmt = [i.strip() for i in self._seperate(item, ':', 1)]
+                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
+                if matched:
+                    try:
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        if should_abort:
+                            return ret
+                    except JS_Break:
                         break
-            else:
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
-                raise ExtractorError('Premature end of parens in %r' % expr)
+
+        # Comma seperated statements
+        sub_expressions = list(self._seperate(expr))
+        expr = sub_expressions.pop().strip() if sub_expressions else ''
+        for sub_expr in sub_expressions:
+            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+
+        for m in re.finditer(rf'''(?x)
+                (?P<pre_sign>\+\+|--)(?P<var1>{_NAME_RE})|
+                (?P<var2>{_NAME_RE})(?P<post_sign>\+\+|--)''', expr):
+            var = m.group('var1') or m.group('var2')
+            start, end = m.span()
+            sign = m.group('pre_sign') or m.group('post_sign')
+            ret = local_vars[var]
+            local_vars[var] += 1 if sign[0] == '+' else -1
+            if m.group('pre_sign'):
+                ret = local_vars[var]
+            expr = expr[:start] + json.dumps(ret) + expr[end:]
 
         for op, opfunc in _ASSIGN_OPERATORS:
             m = re.match(r'''(?x)
@@ -88,14 +270,13 @@ class JSInterpreter(object):
                 (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
             if not m:
                 continue
-            right_val = self.interpret_expression(
+            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
-                m.group('expr'), local_vars, allow_recursion - 1)
 
             if m.groupdict().get('index'):
                 lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(
+                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
-                    m.group('index'), local_vars, allow_recursion)
+                if not isinstance(idx, int):
-                assert isinstance(idx, int)
+                    raise ExtractorError(f'List indices must be integers: {idx}')
                 cur = lvar[idx]
                 val = opfunc(cur, right_val)
                 lvar[idx] = val
@@ -109,8 +290,13 @@ class JSInterpreter(object):
         if expr.isdigit():
             return int(expr)
 
+        if expr == 'break':
+            raise JS_Break()
+        elif expr == 'continue':
+            raise JS_Continue()
+
         var_m = re.match(
-            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
             expr)
         if var_m:
             return local_vars[var_m.group('name')]
@@ -124,91 +310,154 @@ class JSInterpreter(object):
             r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
-            idx = self.interpret_expression(
+            idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
-                m.group('idx'), local_vars, allow_recursion - 1)
             return val[idx]
 
+        for op, opfunc in _OPERATORS:
+            seperated = list(self._seperate(expr, op))
+            if len(seperated) < 2:
+                continue
+            right_val = seperated.pop()
+            left_val = op.join(seperated)
+            left_val, should_abort = self.interpret_statement(
+                left_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise ExtractorError(f'Premature left-side return of {op} in {expr!r}')
+            right_val, should_abort = self.interpret_statement(
+                right_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise ExtractorError(f'Premature right-side return of {op} in {expr!r}')
+            return opfunc(left_val or 0, right_val)
+
         m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
+            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
             member = remove_quotes(m.group('member') or m.group('member2'))
-            arg_str = m.group('args')
+            arg_str = expr[m.end():]
-
+            if arg_str.startswith('('):
-            if variable in local_vars:
+                arg_str, remaining = self._seperate_at_paren(arg_str, ')')
-                obj = local_vars[variable]
-            else:
-                if variable not in self._objects:
-                    self._objects[variable] = self.extract_object(variable)
-                obj = self._objects[variable]
-
-            if arg_str is None:
-                # Member access
-                if member == 'length':
-                    return len(obj)
-                return obj[member]
-
-            assert expr.endswith(')')
-            # Function call
-            if arg_str == '':
-                argvals = tuple()
             else:
-                argvals = tuple([
+                arg_str, remaining = None, arg_str
+
+            def assertion(cndn, msg):
+                """ assert, but without risk of getting optimized out """
+                if not cndn:
+                    raise ExtractorError(f'{member} {msg}: {expr}')
+
+            def eval_method():
+                nonlocal member
+                if variable == 'String':
+                    obj = str
+                elif variable in local_vars:
+                    obj = local_vars[variable]
+                else:
+                    if variable not in self._objects:
+                        self._objects[variable] = self.extract_object(variable)
+                    obj = self._objects[variable]
+
+                if arg_str is None:
+                    # Member access
+                    if member == 'length':
+                        return len(obj)
+                    return obj[member]
+
+                # Function call
+                argvals = [
                     self.interpret_expression(v, local_vars, allow_recursion)
-                    for v in arg_str.split(',')])
+                    for v in self._seperate(arg_str)]
-
+
-            if member == 'split':
+                if obj == str:
-                assert argvals == ('',)
+                    if member == 'fromCharCode':
-                return list(obj)
+                        assertion(argvals, 'takes one or more arguments')
-            if member == 'join':
+                        return ''.join(map(chr, argvals))
-                assert len(argvals) == 1
+                    raise ExtractorError(f'Unsupported string method {member}')
-                return argvals[0].join(obj)
+
-            if member == 'reverse':
+                if member == 'split':
-                assert len(argvals) == 0
+                    assertion(argvals, 'takes one or more arguments')
-                obj.reverse()
+                    assertion(argvals == [''], 'with arguments is not implemented')
-                return obj
+                    return list(obj)
-            if member == 'slice':
+                elif member == 'join':
-                assert len(argvals) == 1
+                    assertion(isinstance(obj, list), 'must be applied on a list')
-                return obj[argvals[0]:]
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
-            if member == 'splice':
+                    return argvals[0].join(obj)
-                assert isinstance(obj, list)
+                elif member == 'reverse':
-                index, howMany = argvals
+                    assertion(not argvals, 'does not take any arguments')
-                res = []
+                    obj.reverse()
-                for i in range(index, min(index + howMany, len(obj))):
+                    return obj
-                    res.append(obj.pop(index))
+                elif member == 'slice':
-                return res
+                    assertion(isinstance(obj, list), 'must be applied on a list')
-
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
-            return obj[member](argvals)
+                    return obj[argvals[0]:]
-
+                elif member == 'splice':
-        for op, opfunc in _OPERATORS:
+                    assertion(isinstance(obj, list), 'must be applied on a list')
-            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
+                    assertion(argvals, 'takes one or more arguments')
-            if not m:
+                    index, howMany = (argvals + [len(obj)])[:2]
-                continue
+                    if index < 0:
-            x, abort = self.interpret_statement(
+                        index += len(obj)
-                m.group('x'), local_vars, allow_recursion - 1)
+                    add_items = argvals[2:]
-            if abort:
+                    res = []
-                raise ExtractorError(
+                    for i in range(index, min(index + howMany, len(obj))):
-                    'Premature left-side return of %s in %r' % (op, expr))
+                        res.append(obj.pop(index))
-            y, abort = self.interpret_statement(
+                    for i, item in enumerate(add_items):
-                m.group('y'), local_vars, allow_recursion - 1)
+                        obj.insert(index + i, item)
-            if abort:
+                    return res
-                raise ExtractorError(
+                elif member == 'unshift':
-                    'Premature right-side return of %s in %r' % (op, expr))
+                    assertion(isinstance(obj, list), 'must be applied on a list')
-            return opfunc(x, y)
+                    assertion(argvals, 'takes one or more arguments')
+                    for item in reversed(argvals):
+                        obj.insert(0, item)
+                    return obj
+                elif member == 'pop':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(not argvals, 'does not take any arguments')
+                    if not obj:
+                        return
+                    return obj.pop()
+                elif member == 'push':
+                    assertion(argvals, 'takes one or more arguments')
+                    obj.extend(argvals)
+                    return obj
+                elif member == 'forEach':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    f, this = (argvals + [''])[:2]
+                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                elif member == 'indexOf':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    idx, start = (argvals + [0])[:2]
+                    try:
+                        return obj.index(idx, start)
+                    except ValueError:
+                        return -1
+
+                if isinstance(obj, list):
+                    member = int(member)
+                return obj[member](argvals)
+
+            if remaining:
+                return self.interpret_expression(
+                    self._named_object(local_vars, eval_method()) + remaining,
+                    local_vars, allow_recursion)
+            else:
+                return eval_method()
 
-        m = re.match(
+        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
-            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
                 int(v) if v.isdigit() else local_vars[v]
-                for v in m.group('args').split(',')]) if len(m.group('args')) > 0 else tuple()
+                for v in self._seperate(m.group('args'))])
-            if fname not in self._functions:
+            if fname in local_vars:
+                return local_vars[fname](argvals)
+            elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
 
-        raise ExtractorError('Unsupported JS expression %r' % expr)
+        if expr:
+            raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -233,30 +482,55 @@ class JSInterpreter(object):
 
         return obj
 
-    def extract_function(self, funcname):
+    def extract_function_code(self, funcname):
+        """ @returns argnames, code """
         func_m = re.search(
             r'''(?x)
                 (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
-                \{(?P<code>[^}]+)\}''' % (
+                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % (
                 re.escape(funcname), re.escape(funcname), re.escape(funcname)),
             self.code)
+        code, _ = self._seperate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
-        argnames = func_m.group('args').split(',')
+        return func_m.group('args').split(','), code
 
-        return self.build_function(argnames, func_m.group('code'))
+    def extract_function(self, funcname):
+        return self.extract_function_from_code(*self.extract_function_code(funcname))
+
+    def extract_function_from_code(self, argnames, code, *global_stack):
+        local_vars = {}
+        while True:
+            mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
+            if mobj is None:
+                break
+            start, body_start = mobj.span()
+            body, remaining = self._seperate_at_paren(code[body_start - 1:], '}')
+            name = self._named_object(
+                local_vars,
+                self.extract_function_from_code(
+                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    body, local_vars, *global_stack))
+            code = code[:start] + name + remaining
+        return self.build_function(argnames, code, local_vars, *global_stack)
 
     def call_function(self, funcname, *args):
-        f = self.extract_function(funcname)
+        return self.extract_function(funcname)(args)
-        return f(args)
+
-
+    def build_function(self, argnames, code, *global_stack):
-    def build_function(self, argnames, code):
+        global_stack = list(global_stack) or [{}]
-        def resf(args):
+        local_vars = global_stack.pop(0)
-            local_vars = dict(zip(argnames, args))
+
-            for stmt in code.split(';'):
+        def resf(args, **kwargs):
-                res, abort = self.interpret_statement(stmt, local_vars)
+            local_vars.update({
-                if abort:
+                **dict(zip(argnames, args)),
+                **kwargs
+            })
+            var_stack = LocalNameSpace(local_vars, *global_stack)
+            for stmt in self._seperate(code.replace('\n', ''), ';'):
+                ret, should_abort = self.interpret_statement(stmt, var_stack)
+                if should_abort:
                     break
-            return res
+            return ret
         return resf