From 6b4130962c158525ee78500d2c9f99a559059804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dor=C3=A9?= Date: Tue, 4 Sep 2018 14:15:16 -0400 Subject: [PATCH 1/4] PBS video subtitles should now be downloaded. --- youtube_dl/extractor/pbs.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py index 80340f595..0abcd9839 100644 --- a/youtube_dl/extractor/pbs.py +++ b/youtube_dl/extractor/pbs.py @@ -666,25 +666,18 @@ class PBSIE(InfoExtractor): age_limit = US_RATINGS.get(rating_str) subtitles = {} - closed_captions_url = info.get('closed_captions_url') - if closed_captions_url: - subtitles['en'] = [{ - 'ext': 'ttml', - 'url': closed_captions_url, - }] - mobj = re.search(r'/(\d+)_Encoded\.dfxp', closed_captions_url) - if mobj: - ttml_caption_suffix, ttml_caption_id = mobj.group(0, 1) - ttml_caption_id = int(ttml_caption_id) - subtitles['en'].extend([{ - 'url': closed_captions_url.replace( - ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)), - 'ext': 'srt', - }, { - 'url': closed_captions_url.replace( - ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)), - 'ext': 'vtt', - }]) + + closed_captions_urls = info.get('cc') + if closed_captions_urls: + for (subtitle_format_name, subtitle_url) in closed_captions_urls.items(): + # There seems to be issues when embedding converted .sami subtitles. Not sure why. + if subtitle_format_name == 'Caption-SAMI': + continue + + subtitles.setdefault('en', []).append({ + 'ext': determine_ext(subtitle_url), + 'url': subtitle_url, + }) # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc) # Try turning it to 'program - title' naming scheme if possible From 81474531ff52256d92e9acb30ef0434a1d6c57c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dor=C3=A9?= Date: Tue, 4 Sep 2018 14:39:31 -0400 Subject: [PATCH 2/4] Add PBS extractor subtitle tests --- test/test_subtitles.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 7d57a628e..64122cce7 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -28,6 +28,7 @@ from youtube_dl.extractor import ( RTVEALaCartaIE, FunnyOrDieIE, DemocracynowIE, + PBSIE ) @@ -354,5 +355,36 @@ class TestDemocracynowSubtitles(BaseTestSubtitles): self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c') +class TestPBSSubtitles(BaseTestSubtitles): + url = 'https://www.pbs.org/video/how-fantasy-reflects-our-world-picecq/' + IE = PBSIE + + def test_allsubtitles(self): + self.DL.params['writesubtitles'] = True + self.DL.params['allsubtitles'] = True + subtitles = self.getSubtitles() + self.assertEqual(len(subtitles.keys()), 1) + self.assertIn('en', subtitles) + self.assertTrue(len(subtitles['en']) > 20000) + + def test_subtitles_dfxp_format(self): + self.DL.params['writesubtitles'] = True + self.DL.params['subtitlesformat'] = 'dfxp' + subtitles = self.getSubtitles() + self.assertEqual(md5(subtitles['en']), '643b034254cdc3768ff1e750b6b5873b') + + def test_subtitles_vtt_format(self): + self.DL.params['writesubtitles'] = True + self.DL.params['subtitlesformat'] = 'vtt' + subtitles = self.getSubtitles() + self.assertEqual(md5(subtitles['en']), '937a05711555b165d4c55a9667017045') + + def test_subtitles_srt_format(self): + self.DL.params['writesubtitles'] = True + self.DL.params['subtitlesformat'] = 'srt' + subtitles = self.getSubtitles() + self.assertEqual(md5(subtitles['en']), '2082c21b43759d9bf172931b2f2ca371') + + if __name__ == '__main__': unittest.main() From 0620d817c36b8d8edf15e9217d328e9875341109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dor=C3=A9?= Date: Thu, 6 Sep 2018 20:39:14 -0400 Subject: [PATCH 3/4] Re-add previous subtitle download code, alongside new code. --- youtube_dl/extractor/pbs.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py index 0abcd9839..f70db9058 100644 --- a/youtube_dl/extractor/pbs.py +++ b/youtube_dl/extractor/pbs.py @@ -667,8 +667,27 @@ class PBSIE(InfoExtractor): subtitles = {} - closed_captions_urls = info.get('cc') - if closed_captions_urls: + if info.get("closed_captions_url"): + closed_captions_url = info.get('closed_captions_url') + + subtitles['en'] = [{ + 'ext': 'ttml', + 'url': closed_captions_url, + }] + mobj = re.search(r'/(\d+)_Encoded\.dfxp', closed_captions_url) + if mobj: + ttml_caption_suffix, ttml_caption_id = mobj.group(0, 1) + ttml_caption_id = int(ttml_caption_id) + subtitles['en'].extend([{ + 'url': closed_captions_url.replace( + ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)), 'ext': 'srt', + }, { + 'url': closed_captions_url.replace(ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)), + 'ext': 'vtt', + }]) + elif info.get('cc'): + closed_captions_urls = info.get('cc') + for (subtitle_format_name, subtitle_url) in closed_captions_urls.items(): # There seems to be issues when embedding converted .sami subtitles. Not sure why. if subtitle_format_name == 'Caption-SAMI': From 01bc8a1b99db350c95e6835b2f050c8545c34604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dor=C3=A9?= Date: Thu, 6 Sep 2018 20:43:26 -0400 Subject: [PATCH 4/4] Formatting... --- youtube_dl/extractor/pbs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py index f70db9058..c5a7452c3 100644 --- a/youtube_dl/extractor/pbs.py +++ b/youtube_dl/extractor/pbs.py @@ -680,9 +680,11 @@ class PBSIE(InfoExtractor): ttml_caption_id = int(ttml_caption_id) subtitles['en'].extend([{ 'url': closed_captions_url.replace( - ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)), 'ext': 'srt', + ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)), + 'ext': 'srt', }, { - 'url': closed_captions_url.replace(ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)), + 'url': closed_captions_url.replace( + ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)), 'ext': 'vtt', }]) elif info.get('cc'):