From a7e999beeca17909bb0088d796c3181b4f35144e Mon Sep 17 00:00:00 2001
From: coletdjnz <colethedj@protonmail.com>
Date: Tue, 7 Sep 2021 20:59:20 +0000
Subject: [PATCH] [pbs] Fix subtitle extraction (#813)

Original PR: https://github.com/ytdl-org/youtube-dl/pull/24430, https://github.com/ytdl-org/youtube-dl/pull/17434
Closes: #836, https://github.com/ytdl-org/youtube-dl/issues/18796, https://github.com/ytdl-org/youtube-dl/issues/17273
Authored-by: coletdjnz, gesa, raphaeldore
---
 test/test_subtitles.py  | 38 ++++++++++++++++++++++++++++++++++++++
 yt_dlp/extractor/pbs.py | 31 ++++++++++---------------------
 2 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 0c5b49ee8..9b39dbd39 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -19,6 +19,7 @@ from yt_dlp.extractor import (
     CeskaTelevizeIE,
     LyndaIE,
     NPOIE,
+    PBSIE,
     ComedyCentralIE,
     NRKTVIE,
     RaiPlayIE,
@@ -372,5 +373,42 @@ class TestDemocracynowSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
 
 
+@is_download_test
+class TestPBSSubtitles(BaseTestSubtitles):
+    url = 'https://www.pbs.org/video/how-fantasy-reflects-our-world-picecq/'
+    IE = PBSIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+
+    def test_subtitles_dfxp_format(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitlesformat'] = 'dfxp'
+        subtitles = self.getSubtitles()
+        self.assertIn(md5(subtitles['en']), ['643b034254cdc3768ff1e750b6b5873b'])
+
+    def test_subtitles_vtt_format(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitlesformat'] = 'vtt'
+        subtitles = self.getSubtitles()
+        self.assertIn(
+            md5(subtitles['en']), ['937a05711555b165d4c55a9667017045', 'f49ea998d6824d94959c8152a368ff73'])
+
+    def test_subtitles_srt_format(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitlesformat'] = 'srt'
+        subtitles = self.getSubtitles()
+        self.assertIn(md5(subtitles['en']), ['2082c21b43759d9bf172931b2f2ca371'])
+
+    def test_subtitles_sami_format(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitlesformat'] = 'sami'
+        subtitles = self.getSubtitles()
+        self.assertIn(md5(subtitles['en']), ['4256b16ac7da6a6780fafd04294e85cd'])
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yt_dlp/extractor/pbs.py b/yt_dlp/extractor/pbs.py
index d68855d62..0eabf9bee 100644
--- a/yt_dlp/extractor/pbs.py
+++ b/yt_dlp/extractor/pbs.py
@@ -600,6 +600,7 @@ class PBSIE(InfoExtractor):
 
         formats = []
         http_url = None
+        hls_subs = {}
         for num, redirect in enumerate(redirects):
             redirect_id = redirect.get('eeid')
 
@@ -622,8 +623,9 @@ class PBSIE(InfoExtractor):
                 continue
 
             if determine_ext(format_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, display_id, 'mp4', m3u8_id='hls', fatal=False))
+                hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(
+                    format_url, display_id, 'mp4', m3u8_id='hls', fatal=False)
+                formats.extend(hls_formats)
             else:
                 formats.append({
                     'url': format_url,
@@ -666,25 +668,12 @@ class PBSIE(InfoExtractor):
         age_limit = US_RATINGS.get(rating_str)
 
         subtitles = {}
-        closed_captions_url = info.get('closed_captions_url')
-        if closed_captions_url:
-            subtitles['en'] = [{
-                'ext': 'ttml',
-                'url': closed_captions_url,
-            }]
-            mobj = re.search(r'/(\d+)_Encoded\.dfxp', closed_captions_url)
-            if mobj:
-                ttml_caption_suffix, ttml_caption_id = mobj.group(0, 1)
-                ttml_caption_id = int(ttml_caption_id)
-                subtitles['en'].extend([{
-                    'url': closed_captions_url.replace(
-                        ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)),
-                    'ext': 'srt',
-                }, {
-                    'url': closed_captions_url.replace(
-                        ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)),
-                    'ext': 'vtt',
-                }])
+        captions = info.get('cc') or {}
+        for caption_url in captions.values():
+            subtitles.setdefault('en', []).append({
+                'url': caption_url
+            })
+        subtitles = self._merge_subtitles(subtitles, hls_subs)
 
         # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc)
         # Try turning it to 'program - title' naming scheme if possible