From 3bef9683c46823a01ca7975079ef57d748d4d0db Mon Sep 17 00:00:00 2001 From: Andrew Udvare Date: Sat, 20 Oct 2018 21:33:32 -0400 Subject: [PATCH 1/6] [Minds] Add new extractor --- youtube_dl/extractor/extractors.py | 5 + youtube_dl/extractor/minds.py | 151 +++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 youtube_dl/extractor/minds.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 17b576df3..f9cc369f0 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -623,6 +623,11 @@ from .microsoftvirtualacademy import ( MicrosoftVirtualAcademyIE, MicrosoftVirtualAcademyCourseIE, ) +from .minds import ( + MindsIE, + MindsActivityIE, + MindsChannelIE, +) from .minhateca import MinhatecaIE from .ministrygrid import MinistryGridIE from .minoto import MinotoIE diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py new file mode 100644 index 000000000..ef2fcea20 --- /dev/null +++ b/youtube_dl/extractor/minds.py @@ -0,0 +1,151 @@ +# coding: utf-8 +from __future__ import unicode_literals +import re + +from .common import InfoExtractor +from ..utils import (int_or_none, sanitized_Request, str_or_none, + unified_strdate) + + +class MindsIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?minds\.com/media/(?P[0-9]+)' + _TEST = { + 'url': 'https://www.minds.com/media/100000000000086822', + 'md5': '215a658184a419764852239d4970b045', + 'info_dict': { + 'id': '100000000000086822', + 'ext': 'mp4', + 'title': 'Minds intro sequence', + 'thumbnail': 'https://cdn-cinemr.minds.com/cinemr_com/334128440657580032/thumbnail-00001.png', + 'uploader_id': '100000000000000341', + 'description': '', + 'upload_date': '20130524', + 'timestamp': 1369404826, + }, + 'params': { + 'skip_download': True, + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + video_api_url = 'https://www.minds.com/api/v1/media/%s' % video_id + token = self._get_cookies(url).get('XSRF-TOKEN') + headers = { + 'authority': 'www.minds.com', + 'referer': url, + 'x-xsrf-token': token.value if token else '', + } + data = self._download_json(video_api_url, video_id, headers=headers) + formats = [] + owner = data.get('ownerObj', {}) + + transcodes = data.get('transcodes', {}) + # These keys are the width so keep the highest width last + keys = sorted(transcodes.keys()) + + for format_id in keys: + is_numeric = re.match('^[0-9]+\.mp4', format_id) + video_url = transcodes[format_id] + info = { + 'url': video_url, + 'format_id': format_id, + 'http_headers': headers, + } + if is_numeric: + info['width'] = int(format_id.split('.')[0]) + formats.append(info) + + uploader_id = str_or_none(owner.get('guid') or + data.get('owner_guid') or + owner.get('legacy_guid') or + owner.get('owner_guid')) + description = str_or_none(data.get('description')) + if description: + description = description.strip() + uploader_url = age_limit = None + + if owner.get('username'): + uploader_url = 'https://www.minds.com/%s' % owner.get('username') + if data.get('mature') is True: + age_limit = 18 + + thumbnail_api_url = data.get('thumbnail_src') + thumbnail = None + if thumbnail_api_url: + req = sanitized_Request(thumbnail_api_url, method='HEAD') + res = self._request_webpage(req, video_id) + thumbnail = getattr(res, 'url', None) + + return { + 'id': video_id, + 'title': data['title'], + 'formats': formats, + 'description': description, + 'license': str_or_none(data.get('license')), + 'creator': str_or_none(owner.get('name') or owner.get('username')), + 'release_date': unified_strdate(data.get('time_created')), + 'timestamp': int_or_none(data.get('time_created')), + 'uploader_id': uploader_id, + 'uploader_url': uploader_url, + 'view_count': int_or_none(data.get('play:count')), + 'like_count': int_or_none(data.get('thumbs:up:count')), + 'dislike_count': int_or_none(data.get('thumbs:down:count')), + 'average_rating': int_or_none(data.get('rating')), + 'age_limit': age_limit, + 'categories': [str_or_none(data.get('category'))], + 'tags': [x.strip() for x in data.get('tags', '').split(',')], + # As of 20181020 the API is returning `false` for this value both + # at top level and within the entity.comments:count path. The only + # other way to get this is to fetch all comments and count. + 'comment_count': int_or_none(data.get('comments:count')), + 'thumbnail': thumbnail, + } + + +class MindsActivityIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?minds\.com/newsfeed/(?P[0-9]+)' + + def _real_extract(self, url): + guid = self._match_id(url) + api_url = 'https://www.minds.com/api/v1/newsfeed/single/%s' % guid + token = self._get_cookies(url).get('XSRF-TOKEN') + headers = { + 'authority': 'www.minds.com', + 'referer': url, + 'x-xsrf-token': token.value if token else '', + } + data = self._download_json(api_url, guid, headers=headers) + return self.url_result('https://www.minds.com/media/%s' % data['activity']['entity_guid']) + + +class MindsChannelIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?minds\.com/(?P[^/]+)' + + def _real_extract(self, url): + channel_name = self._match_id(url) + api_url = 'https://www.minds.com/api/v1/channel/%s' % channel_name + token = self._get_cookies(url).get('XSRF-TOKEN') + headers = { + 'authority': 'www.minds.com', + 'referer': url, + 'x-xsrf-token': token.value if token else '', + } + data = self._download_json(api_url, channel_name, headers=headers) + channel = data.get('channel', {}) + params = {'limit': 12, 'offset': ''} + api_url = 'https://www.minds.com/api/v1/newsfeed/personal/%s' % channel['guid'] + entries = [] + while True: + data = self._download_json(api_url, channel['guid'], + headers=headers, query=params) + activity = data.get('activity', []) + if len(activity) == 0 or not data.get('load-next'): + break + for info in activity: + if info.get('custom_type') != 'video': + continue + entries.append(self.url_result('https://www.minds.com/media/%s' % info['entity_guid'])) + params['offset'] = data['load-next'] + return self.playlist_result(entries, + playlist_title='%s activity' % channel_name) From a83b0a291bad024cd4da1a1647561abd426a25c7 Mon Sep 17 00:00:00 2001 From: Andrew Udvare Date: Sat, 20 Oct 2018 21:54:27 -0400 Subject: [PATCH 2/6] [Minds] Improve tag handling --- youtube_dl/extractor/minds.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py index ef2fcea20..ca885d40f 100644 --- a/youtube_dl/extractor/minds.py +++ b/youtube_dl/extractor/minds.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..compat import compat_str from ..utils import (int_or_none, sanitized_Request, str_or_none, unified_strdate) @@ -63,7 +64,7 @@ class MindsIE(InfoExtractor): description = str_or_none(data.get('description')) if description: description = description.strip() - uploader_url = age_limit = None + uploader_url = age_limit = thumbnail = None if owner.get('username'): uploader_url = 'https://www.minds.com/%s' % owner.get('username') @@ -71,11 +72,13 @@ class MindsIE(InfoExtractor): age_limit = 18 thumbnail_api_url = data.get('thumbnail_src') - thumbnail = None if thumbnail_api_url: req = sanitized_Request(thumbnail_api_url, method='HEAD') res = self._request_webpage(req, video_id) thumbnail = getattr(res, 'url', None) + tags = data.get('tags', '').strip() + if isinstance(tags, compat_str): + tags = [x.strip() for x in tags.split(',')] return { 'id': video_id, @@ -94,7 +97,7 @@ class MindsIE(InfoExtractor): 'average_rating': int_or_none(data.get('rating')), 'age_limit': age_limit, 'categories': [str_or_none(data.get('category'))], - 'tags': [x.strip() for x in data.get('tags', '').split(',')], + 'tags': tags, # As of 20181020 the API is returning `false` for this value both # at top level and within the entity.comments:count path. The only # other way to get this is to fetch all comments and count. From 11f239840b07b151a851656082bd8b3587d0f548 Mon Sep 17 00:00:00 2001 From: Andrew Udvare Date: Sat, 20 Oct 2018 22:03:52 -0400 Subject: [PATCH 3/6] [Minds] Ignore invalid thumbnails --- youtube_dl/extractor/minds.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py index ca885d40f..73c3792d3 100644 --- a/youtube_dl/extractor/minds.py +++ b/youtube_dl/extractor/minds.py @@ -75,7 +75,8 @@ class MindsIE(InfoExtractor): if thumbnail_api_url: req = sanitized_Request(thumbnail_api_url, method='HEAD') res = self._request_webpage(req, video_id) - thumbnail = getattr(res, 'url', None) + if res.headers.get('content-type', '').startswith('image/'): + thumbnail = getattr(res, 'url', None) tags = data.get('tags', '').strip() if isinstance(tags, compat_str): tags = [x.strip() for x in tags.split(',')] From a11f9bbf0cd1600dbaa75c50ba4e79b5edd0b056 Mon Sep 17 00:00:00 2001 From: Andrew Udvare Date: Sat, 20 Oct 2018 22:29:53 -0400 Subject: [PATCH 4/6] [Minds] Fix handling of non-string values --- youtube_dl/extractor/minds.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py index 73c3792d3..7b0e174e2 100644 --- a/youtube_dl/extractor/minds.py +++ b/youtube_dl/extractor/minds.py @@ -78,8 +78,15 @@ class MindsIE(InfoExtractor): if res.headers.get('content-type', '').startswith('image/'): thumbnail = getattr(res, 'url', None) tags = data.get('tags', '').strip() - if isinstance(tags, compat_str): + if isinstance(tags, compat_str) and tags: tags = [x.strip() for x in tags.split(',')] + else: + tags = None + category = data.get('category') + if isinstance(category, compat_str) and category: + category = [category] + else: + category = None return { 'id': video_id, From e797b0e7322c49595ad80c0c11cf98adb91fad64 Mon Sep 17 00:00:00 2001 From: Andrew Udvare Date: Sat, 24 Nov 2018 04:00:15 -0500 Subject: [PATCH 5/6] [Minds] "media" is not a valid channel name Pass children=false to video API URL like the site Fix getting thumbnail --- youtube_dl/extractor/minds.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py index 7b0e174e2..4517fc794 100644 --- a/youtube_dl/extractor/minds.py +++ b/youtube_dl/extractor/minds.py @@ -37,7 +37,8 @@ class MindsIE(InfoExtractor): 'referer': url, 'x-xsrf-token': token.value if token else '', } - data = self._download_json(video_api_url, video_id, headers=headers) + data = self._download_json(video_api_url, video_id, headers=headers, + query={'children': 'false'}) formats = [] owner = data.get('ownerObj', {}) @@ -73,7 +74,8 @@ class MindsIE(InfoExtractor): thumbnail_api_url = data.get('thumbnail_src') if thumbnail_api_url: - req = sanitized_Request(thumbnail_api_url, method='HEAD') + req = sanitized_Request(thumbnail_api_url) + req.get_method = lambda: 'HEAD' res = self._request_webpage(req, video_id) if res.headers.get('content-type', '').startswith('image/'): thumbnail = getattr(res, 'url', None) @@ -136,6 +138,8 @@ class MindsChannelIE(InfoExtractor): def _real_extract(self, url): channel_name = self._match_id(url) api_url = 'https://www.minds.com/api/v1/channel/%s' % channel_name + if channel_name == 'media': + return self.url_result(url, ie='Minds') token = self._get_cookies(url).get('XSRF-TOKEN') headers = { 'authority': 'www.minds.com', From 7719ef4681addaf43e2c68881b269e366386e124 Mon Sep 17 00:00:00 2001 From: Andrew Udvare Date: Sat, 24 Nov 2018 04:05:38 -0500 Subject: [PATCH 6/6] [Minds] Use negative lookahead in channel IE to ignore invalid names --- youtube_dl/extractor/minds.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py index 4517fc794..4523d0938 100644 --- a/youtube_dl/extractor/minds.py +++ b/youtube_dl/extractor/minds.py @@ -133,13 +133,11 @@ class MindsActivityIE(InfoExtractor): class MindsChannelIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?minds\.com/(?P[^/]+)' + _VALID_URL = r'https?://(?:www\.)?minds\.com/(?!newsfeed|media|api)(?P[^/]+)' def _real_extract(self, url): channel_name = self._match_id(url) api_url = 'https://www.minds.com/api/v1/channel/%s' % channel_name - if channel_name == 'media': - return self.url_result(url, ie='Minds') token = self._get_cookies(url).get('XSRF-TOKEN') headers = { 'authority': 'www.minds.com',