From ad6b8f8e58b2540091731670590cdd54dad044c9 Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 06:47:32 +0000 Subject: [PATCH 1/4] do nothing i tried to do two different pull requests on my master branch, whoops --- youtube_dl/extractor/twitch.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index 3f0f7e277..e211cd4c8 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -380,13 +380,11 @@ class TwitchPlaylistBaseIE(TwitchBaseIE): _PLAYLIST_PATH = 'kraken/channels/%s/videos/?offset=%d&limit=%d' _PAGE_LIMIT = 100 - def _extract_playlist(self, channel_name): + def _extract_playlist(self, channel_id): info = self._call_api( - 'kraken/users?login=%s' % channel_name, - channel_name, 'Downloading channel info JSON') - info = info['users'][0] - channel_id = info['_id'] - channel_name = info.get('display_name') or info.get('name') or channel_name + 'kraken/channels/%s' % channel_id, + channel_id, 'Downloading channel info JSON') + channel_name = info.get('display_name') or info.get('name') entries = [] offset = 0 limit = self._PAGE_LIMIT @@ -446,7 +444,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE): _TESTS = [{ 'url': 'http://www.twitch.tv/vanillatv/profile', 'info_dict': { - 'id': '22744919', + 'id': 'vanillatv', 'title': 'VanillaTV', }, 'playlist_mincount': 412, @@ -470,7 +468,7 @@ class TwitchAllVideosIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/all', 'info_dict': { - 'id': '497952', + 'id': 'spamfish', 'title': 'Spamfish', }, 'playlist_mincount': 869, @@ -489,7 +487,7 @@ class TwitchUploadsIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/uploads', 'info_dict': { - 'id': '497952', + 'id': 'spamfish', 'title': 'Spamfish', }, 'playlist_mincount': 0, @@ -508,7 +506,7 @@ class TwitchPastBroadcastsIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/past-broadcasts', 'info_dict': { - 'id': '497952', + 'id': 'spamfish', 'title': 'Spamfish', }, 'playlist_mincount': 0, @@ -527,7 +525,7 @@ class TwitchHighlightsIE(TwitchVideosBaseIE): _TESTS = [{ 'url': 'https://www.twitch.tv/spamfish/videos/highlights', 'info_dict': { - 'id': '497952', + 'id': 'spamfish', 'title': 'Spamfish', }, 'playlist_mincount': 805, From 4d44560896ec5b8729973087de22adceecf1c85c Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 06:56:46 +0000 Subject: [PATCH 2/4] github is not working for me, this is a workaround --- youtube_dl/extractor/screencast.py | 123 ----------------------------- 1 file changed, 123 deletions(-) delete mode 100644 youtube_dl/extractor/screencast.py diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py deleted file mode 100644 index 69a0d01f3..000000000 --- a/youtube_dl/extractor/screencast.py +++ /dev/null @@ -1,123 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .common import InfoExtractor -from ..compat import ( - compat_parse_qs, - compat_urllib_request, -) -from ..utils import ( - ExtractorError, -) - - -class ScreencastIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P[a-zA-Z0-9]+)' - _TESTS = [{ - 'url': 'http://www.screencast.com/t/3ZEjQXlT', - 'md5': '917df1c13798a3e96211dd1561fded83', - 'info_dict': { - 'id': '3ZEjQXlT', - 'ext': 'm4v', - 'title': 'Color Measurement with Ocean Optics Spectrometers', - 'description': 'md5:240369cde69d8bed61349a199c5fb153', - 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', - } - }, { - 'url': 'http://www.screencast.com/t/V2uXehPJa1ZI', - 'md5': 'e8e4b375a7660a9e7e35c33973410d34', - 'info_dict': { - 'id': 'V2uXehPJa1ZI', - 'ext': 'mov', - 'title': 'The Amadeus Spectrometer', - 'description': 're:^In this video, our friends at.*To learn more about Amadeus, visit', - 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', - } - }, { - 'url': 'http://www.screencast.com/t/aAB3iowa', - 'md5': 'dedb2734ed00c9755761ccaee88527cd', - 'info_dict': { - 'id': 'aAB3iowa', - 'ext': 'mp4', - 'title': 'Google Earth Export', - 'description': 'Provides a demo of a CommunityViz export to Google Earth, one of the 3D viewing options.', - 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', - } - }, { - 'url': 'http://www.screencast.com/t/X3ddTrYh', - 'md5': '669ee55ff9c51988b4ebc0877cc8b159', - 'info_dict': { - 'id': 'X3ddTrYh', - 'ext': 'wmv', - 'title': 'Toolkit 6 User Group Webinar (2014-03-04) - Default Judgment and First Impression', - 'description': 'md5:7b9f393bc92af02326a5c5889639eab0', - 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', - } - }, { - 'url': 'http://screencast.com/t/aAB3iowa', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - video_url = self._html_search_regex( - r'(?:(?!\1).)+)\1', - webpage, 'video url', default=None, group='url') - - if video_url is None: - video_url = self._html_search_meta( - 'og:video', webpage, default=None) - - if video_url is None: - raise ExtractorError('Cannot find video') - - title = self._og_search_title(webpage, default=None) - if title is None: - title = self._html_search_regex( - [r'Title: ([^<]+)', - r'class="tabSeperator">>(.+?)<', - r'([^<]+)'], - webpage, 'title') - thumbnail = self._og_search_thumbnail(webpage) - description = self._og_search_description(webpage, default=None) - if description is None: - description = self._html_search_meta('description', webpage) - - return { - 'id': video_id, - 'url': video_url, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - } From 762fe4e670a06dc42425445e97b85dba53ec8de8 Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 06:57:55 +0000 Subject: [PATCH 3/4] Prefer API to scraping HTML when possible --- youtube_dl/extractor/screencast.py | 151 +++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 youtube_dl/extractor/screencast.py diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py new file mode 100644 index 000000000..859c9dc0d --- /dev/null +++ b/youtube_dl/extractor/screencast.py @@ -0,0 +1,151 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import ( + compat_parse_qs, + compat_urllib_request, +) +from ..utils import ( + ExtractorError, +) + + +class ScreencastIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P[a-zA-Z0-9]+)' + _API_URL = 'https://www.screencast.com/api/external/oembed?url=%s&format=json' + + _TESTS = [{ + 'url': 'http://www.screencast.com/t/3ZEjQXlT', + 'md5': '917df1c13798a3e96211dd1561fded83', + 'info_dict': { + 'id': '3ZEjQXlT', + 'ext': 'm4v', + 'title': 'Color Measurement with Ocean Optics Spectrometers', + 'description': 'md5:240369cde69d8bed61349a199c5fb153', + 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', + } + }, { + 'url': 'http://www.screencast.com/t/V2uXehPJa1ZI', + 'md5': 'e8e4b375a7660a9e7e35c33973410d34', + 'info_dict': { + 'id': 'V2uXehPJa1ZI', + 'ext': 'mov', + 'title': 'The Amadeus Spectrometer', + 'description': 're:^In this video, our friends at.*To learn more about Amadeus, visit', + 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', + } + }, { + 'url': 'http://www.screencast.com/t/aAB3iowa', + 'md5': 'dedb2734ed00c9755761ccaee88527cd', + 'info_dict': { + 'id': 'aAB3iowa', + 'ext': 'mp4', + 'title': 'Google Earth Export', + 'description': 'Provides a demo of a CommunityViz export to Google Earth, one of the 3D viewing options.', + 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', + } + }, { + 'url': 'http://www.screencast.com/t/X3ddTrYh', + 'md5': '669ee55ff9c51988b4ebc0877cc8b159', + 'info_dict': { + 'id': 'X3ddTrYh', + 'ext': 'wmv', + 'title': 'Toolkit 6 User Group Webinar (2014-03-04) - Default Judgment and First Impression', + 'description': 'md5:7b9f393bc92af02326a5c5889639eab0', + 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', + } + }, { + 'url': 'http://screencast.com/t/aAB3iowa', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + + # The JSON given by the API has a thumbnail URL, + # but it's inferior to the webpage's thumbnail. + # It also has no video description, so we + # definitely still need to get the webpage. + + info = self._download_json( + self._API_URL % url, video_id, + 'Downloading video info JSON') + + video_url = info.get('url') + if video_url != None: + video_url_raw = compat_urllib_request.quote(video_url) + video_url = re.sub(r'^(?Phttps|http)%3A', + lambda match: '%s:' % match.group('proto'), + video_url_raw) + + title = info.get('title') + webpage = self._download_webpage(url, video_id) + + if video_url == None: + video_url = self._html_search_regex( + r'http|https)%3A', + lambda match: '%s:' % match.group('proto'), + video_url_raw) + + if video_url == None: + video_meta = self._html_search_meta( + 'og:video', webpage, default=None) + if video_meta: + video_url = self._search_regex( + r'src=(.*?)(?:$|&)', video_meta, + 'meta tag video URL', default=None) + + if video_url == None: + video_url = self._html_search_regex( + r'MediaContentUrl["\']\s*:(["\'])(?P(?:(?!\1).)+)\1', + webpage, 'video url', default=None, group='url') + + if video_url == None: + video_url = self._html_search_meta( + 'og:video', webpage, default=None) + + if video_url == None: + raise ExtractorError('Cannot find video') + + if title == None: + title = self._og_search_title(webpage, default=None) + + if title == None: + title = self._html_search_regex( + [r'Title: ([^<]+)', + r'class="tabSeperator">>(.+?)<', + r'([^<]+)'], + webpage, 'title') + + thumbnail = self._og_search_thumbnail(webpage) + description = self._og_search_description(webpage, default=None) + if description == None: + description = self._html_search_meta('description', webpage) + + return { + 'id': video_id, + 'url': video_url, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + } From da78a4f3eeab37023aad09c36f68a7328363d120 Mon Sep 17 00:00:00 2001 From: geauxlo <66712139+geauxlo@users.noreply.github.com> Date: Wed, 10 Jun 2020 07:01:52 +0000 Subject: [PATCH 4/4] Recognize more valid URLs URLs like `https://www.screencast.com/users/cindyhailes/folders/Jing/media/c9be177c-5808-4c4f-af56-eadceb3a7c82` weren't being accepted before --- youtube_dl/extractor/screencast.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py index 859c9dc0d..d23a53706 100644 --- a/youtube_dl/extractor/screencast.py +++ b/youtube_dl/extractor/screencast.py @@ -14,7 +14,7 @@ from ..utils import ( class ScreencastIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P[a-zA-Z0-9]+)' + _VALID_URL = r'https?://(?:www\.)?screencast\.com/(?:t|users/[^/]+/folders/[^/]+/media)/(?P[a-zA-Z0-9\-]+)' _API_URL = 'https://www.screencast.com/api/external/oembed?url=%s&format=json' _TESTS = [{ @@ -60,6 +60,16 @@ class ScreencastIE(InfoExtractor): }, { 'url': 'http://screencast.com/t/aAB3iowa', 'only_matching': True, + }, { + 'url': 'https://www.screencast.com/users/cindyhailes/folders/Jing/media/c9be177c-5808-4c4f-af56-eadceb3a7c82', + 'md5': '589d37a28d2add53c8bf16b9126d9dc2', + 'info_dict': { + 'id': 'c9be177c-5808-4c4f-af56-eadceb3a7c82', + 'ext': 'swf', + 'title': '2020-05-31_1737', + 'description': 'Shared from Screencast.com', + 'thumbnail': r're:^https?://.*\.(?:gif|jpg)$', + } }] def _real_extract(self, url):