From f5225737877a78f63b9a6f1de675c95c650f65d6 Mon Sep 17 00:00:00 2001 From: chris <6024426+iw0nderhow@users.noreply.github.com> Date: Sat, 1 Jan 2022 22:07:00 +0100 Subject: [PATCH] [extractor] Extract chapters from JSON-LD (#2031) Authored by: iw0nderhow, pukkandan --- test/test_InfoExtractor.py | 59 ++++++++++++++++++++++++++++++++++++++ yt_dlp/extractor/common.py | 18 ++++++++++++ 2 files changed, 77 insertions(+) diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 25bc00e0b..866ded243 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -208,6 +208,65 @@ class TestInfoExtractor(unittest.TestCase): }, {'expected_type': 'NewsArticle'}, ), + ( + r''' + ''', + { + 'chapters': [ + {"title": "Explosie Turnhout", "start_time": 70, "end_time": 440}, + {"title": "Jaarwisseling", "start_time": 440, "end_time": 1179}, + {"title": "Natuurbranden Colorado", "start_time": 1179, "end_time": 1263}, + {"title": "Klimaatverandering", "start_time": 1263, "end_time": 1367}, + {"title": "Zacht weer", "start_time": 1367, "end_time": 1383}, + {"title": "Financiƫle balans", "start_time": 1383, "end_time": 1484}, + {"title": "Club Brugge", "start_time": 1484, "end_time": 1575}, + {"title": "Mentale gezondheid bij topsporters", "start_time": 1575, "end_time": 1728}, + {"title": "Olympische Winterspelen", "start_time": 1728, "end_time": 1873}, + {"title": "Sober oudjaar in Nederland", "start_time": 1873, "end_time": 2079.23} + ], + 'title': 'Het journaal - Aflevering 365 (Seizoen 2021)' + }, {} + ), ( # test multiple thumbnails in a list r''' diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 1d694293e..79f53c9c2 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -1429,6 +1429,23 @@ class InfoExtractor(object): continue info[count_key] = interaction_count + def extract_chapter_information(e): + chapters = [{ + 'title': part.get('name'), + 'start_time': part.get('startOffset'), + 'end_time': part.get('endOffset'), + } for part in e.get('hasPart', []) if part.get('@type') == 'Clip'] + for idx, (last_c, current_c, next_c) in enumerate(zip( + [{'end_time': 0}] + chapters, chapters, chapters[1:])): + current_c['end_time'] = current_c['end_time'] or next_c['start_time'] + current_c['start_time'] = current_c['start_time'] or last_c['end_time'] + if None in current_c.values(): + self.report_warning(f'Chapter {idx} contains broken data. Not extracting chapters') + return + if chapters: + chapters[-1]['end_time'] = chapters[-1]['end_time'] or info['duration'] + info['chapters'] = chapters + def extract_video_object(e): assert e['@type'] == 'VideoObject' author = e.get('author') @@ -1452,6 +1469,7 @@ class InfoExtractor(object): 'view_count': int_or_none(e.get('interactionCount')), }) extract_interaction_statistic(e) + extract_chapter_information(e) def traverse_json_ld(json_ld, at_top_level=True): for e in json_ld: