[cookies] Improve container support (#4806)

Closes #4800
Authored by: bashonly, pukkandan, coletdjnz

README.md

@@ -706,19 +706,20 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     and dump cookie jar in
     --no-cookies                    Do not read/dump cookies from/to file
                                     (default)
-    --cookies-from-browser BROWSER[+KEYRING][:PROFILE[:CONTAINER]]
+    --cookies-from-browser BROWSER[+KEYRING][:PROFILE][::CONTAINER]
-                                    The name of the browser and (optionally) the
+                                    The name of the browser to load cookies
-                                    name/path of the profile to load cookies
+                                    from. Currently supported browsers are:
-                                    from (and container name if Firefox)
+                                    brave, chrome, chromium, edge, firefox,
-                                    separated by a ":". Currently supported
+                                    opera, safari, vivaldi. Optionally, the
-                                    browsers are: brave, chrome, chromium, edge,
+                                    KEYRING used for decrypting Chromium cookies
-                                    firefox, opera, safari, vivaldi. By default,
+                                    on Linux, the name/path of the PROFILE to
-                                    the default container of the most recently
+                                    load cookies from, and the CONTAINER name
-                                    accessed profile is used. The keyring used
+                                    (if Firefox) ("none" for no container) can
-                                    for decrypting Chromium cookies on Linux can
+                                    be given with their respective seperators.
-                                    be (optionally) specified after the browser
+                                    By default, all containers of the most
-                                    name separated by a "+". Currently supported
+                                    recently accessed profile are used.
-                                    keyrings are: basictext, gnomekeyring, kwallet
+                                    Currently supported keyrings are: basictext,
+                                    gnomekeyring, kwallet
     --no-cookies-from-browser       Do not load cookies from browser (default)
     --cache-dir DIR                 Location in the filesystem where youtube-dl
                                     can store some downloaded information (such

yt_dlp/__init__.py

@@ -347,23 +347,25 @@ def validate_options(opts):
     # Cookies from browser
     if opts.cookiesfrombrowser:
         container = None
-        mobj = re.match(r'(?P<name>[^+:]+)(\s*\+\s*(?P<keyring>[^:]+))?(\s*:(?P<profile>.+))?', opts.cookiesfrombrowser)
+        mobj = re.fullmatch(r'''(?x)
+            (?P<name>[^+:]+)
+            (?:\s*\+\s*(?P<keyring>[^:]+))?
+            (?:\s*:\s*(?P<profile>.+?))?
+            (?:\s*::\s*(?P<container>.+))?
+        ''', opts.cookiesfrombrowser)
         if mobj is None:
             raise ValueError(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}')
-        browser_name, keyring, profile = mobj.group('name', 'keyring', 'profile')
+        browser_name, keyring, profile, container = mobj.group('name', 'keyring', 'profile', 'container')
         browser_name = browser_name.lower()
         if browser_name not in SUPPORTED_BROWSERS:
             raise ValueError(f'unsupported browser specified for cookies: "{browser_name}". '
                              f'Supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}')
-        elif profile and browser_name == 'firefox':
-            if ':' in profile and not os.path.exists(profile):
-                profile, container = profile.split(':', 1)
         if keyring is not None:
             keyring = keyring.upper()
             if keyring not in SUPPORTED_KEYRINGS:
                 raise ValueError(f'unsupported keyring specified for cookies: "{keyring}". '
                                  f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
-        opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
+        opts.cookiesfrombrowser = (browser_name, profile or None, keyring, container or None)
 
     # MetadataParser
     def metadataparser_actions(f):

yt_dlp/cookies.py

@@ -128,9 +128,14 @@ def _extract_firefox_cookies(profile, container, logger):
     else:
         search_root = os.path.join(_firefox_browser_dir(), profile)
 
+    cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger)
+    if cookie_database_path is None:
+        raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
+    logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
+
     container_id = None
-    if container is not None:
+    if container not in (None, 'none'):
-        containers_path = os.path.join(search_root, 'containers.json')
+        containers_path = os.path.join(os.path.dirname(cookie_database_path), 'containers.json')
         if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK):
             raise FileNotFoundError(f'could not read containers.json in {search_root}')
         with open(containers_path, 'r') as containers:
@@ -142,26 +147,21 @@ def _extract_firefox_cookies(profile, container, logger):
         if not isinstance(container_id, int):
             raise ValueError(f'could not find firefox container "{container}" in containers.json')
 
-    cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger)
-    if cookie_database_path is None:
-        raise FileNotFoundError(f'could not find firefox cookies database in {search_root}')
-    logger.debug(f'Extracting cookies from: "{cookie_database_path}"')
-
     with tempfile.TemporaryDirectory(prefix='yt_dlp') as tmpdir:
         cursor = None
         try:
             cursor = _open_database_copy(cookie_database_path, tmpdir)
-            origin_attributes = ''
             if isinstance(container_id, int):
-                origin_attributes = f'^userContextId={container_id}'
                 logger.debug(
                     f'Only loading cookies from firefox container "{container}", ID {container_id}')
-            try:
                 cursor.execute(
-                    'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE originAttributes=?',
+                    'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE originAttributes LIKE ? OR originAttributes LIKE ?',
-                    (origin_attributes, ))
+                    (f'%userContextId={container_id}', f'%userContextId={container_id}&%'))
-            except sqlite3.OperationalError:
+            elif container == 'none':
-                logger.debug('Database exception, loading all cookies')
+                logger.debug('Only loading cookies not belonging to any container')
+                cursor.execute(
+                    'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE NOT INSTR(originAttributes,"userContextId=")')
+            else:
                 cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
             jar = YoutubeDLCookieJar()
             with _create_progress_bar(logger) as progress_bar:

yt_dlp/options.py

@@ -1400,14 +1400,15 @@ def create_parser():
         help='Do not read/dump cookies from/to file (default)')
     filesystem.add_option(
         '--cookies-from-browser',
-        dest='cookiesfrombrowser', metavar='BROWSER[+KEYRING][:PROFILE[:CONTAINER]]',
+        dest='cookiesfrombrowser', metavar='BROWSER[+KEYRING][:PROFILE][::CONTAINER]',
         help=(
-            'The name of the browser and (optionally) the name/path of the profile to load cookies from '
+            'The name of the browser to load cookies from. '
-            '(and container name if Firefox) separated by a ":". '
             f'Currently supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}. '
-            'By default, the default container of the most recently accessed profile is used. '
+            'Optionally, the KEYRING used for decrypting Chromium cookies on Linux, '
-            'The keyring used for decrypting Chromium cookies on Linux can be '
+            'the name/path of the PROFILE to load cookies from, '
-            '(optionally) specified after the browser name separated by a "+". '
+            'and the CONTAINER name (if Firefox) ("none" for no container) '
+            'can be given with their respective seperators. '
+            'By default, all containers of the most recently accessed profile are used. '
             f'Currently supported keyrings are: {", ".join(map(str.lower, sorted(SUPPORTED_KEYRINGS)))}'))
     filesystem.add_option(
         '--no-cookies-from-browser',