From 7191f5ec30c17e0800977a02c5430ca4640aa2f8 Mon Sep 17 00:00:00 2001 From: renalid Date: Sat, 2 Jul 2016 18:49:39 +0200 Subject: [PATCH 1/3] [FranceInter] Extractor Update --- youtube_dl/extractor/franceinter.py | 47 +++++++++++++++++------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py index 2369f868d..ccf460293 100644 --- a/youtube_dl/extractor/franceinter.py +++ b/youtube_dl/extractor/franceinter.py @@ -2,21 +2,27 @@ from __future__ import unicode_literals from .common import InfoExtractor -from ..utils import int_or_none +from ..utils import ( + int_or_none, + unified_strdate, + unified_timestamp, +) +import re class FranceInterIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/emissions/(?P[^?#]+)' + _TEST = { - 'url': 'http://www.franceinter.fr/player/reecouter?play=793962', - 'md5': '4764932e466e6f6c79c317d2e74f6884', + 'url': 'https://www.franceinter.fr/emissions/la-tete-au-carre/la-tete-au-carre-30-juin-2016', + 'md5': 'f13e4371662cf5a829f64d829ae78062', 'info_dict': { - 'id': '793962', + 'id': 'la-tete-au-carre/la-tete-au-carre-30-juin-2016', 'ext': 'mp3', - 'title': 'L’Histoire dans les jeux vidéo', - 'description': 'md5:7e93ddb4451e7530022792240a3049c7', - 'timestamp': 1387369800, - 'upload_date': '20131218', + 'title': 'Regards sur le sport du 30 juin 2016 - France Inter', + 'description': 'UEFA Europa, Jeux Olympiques... La période est aux sports, dans les gradins ou devant les écrans. Mais quel est le regard des spécialistes sur cette pratique? ', + 'timestamp': 1467244800, + 'upload_date': '20160630', }, } @@ -25,17 +31,18 @@ class FranceInterIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - path = self._search_regex( - r'(.+?)', webpage, 'title') - description = self._html_search_regex( - r'(.*?)', - webpage, 'description', fatal=False) - timestamp = int_or_none(self._search_regex( - r'data-date="(\d+)"', webpage, 'upload date', fatal=False)) + title = self._og_search_title(webpage) + description = self._og_search_description(webpage) + + extractdate = self._search_regex( + r'([0-9]+[.][0-9]+[.][0-9]+)', video_url, 'extractdate', fatal=False) + + timestamp = unified_timestamp(extractdate) + + upload_date = (unified_strdate(extractdate)) return { 'id': video_id, @@ -46,4 +53,4 @@ class FranceInterIE(InfoExtractor): 'url': video_url, 'vcodec': 'none', }], - } + } \ No newline at end of file From b0c9070d68bb1baef87e1a82951c4cc5f9e198c6 Mon Sep 17 00:00:00 2001 From: renalid Date: Sun, 3 Jul 2016 16:08:59 +0200 Subject: [PATCH 2/3] [FranceInter] Extractor Update (corrections) - correct the test with the same original content (the URL and HTML had change) - correct how to get the right date of the content using adaptation of french month name --- youtube_dl/extractor/franceinter.py | 27 ++++++++++++++++----------- youtube_dl/utils.py | 11 +++++++++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py index ccf460293..9a4684091 100644 --- a/youtube_dl/extractor/franceinter.py +++ b/youtube_dl/extractor/franceinter.py @@ -6,6 +6,7 @@ from ..utils import ( int_or_none, unified_strdate, unified_timestamp, + month_by_french_name, ) import re @@ -14,15 +15,15 @@ class FranceInterIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/emissions/(?P[^?#]+)' _TEST = { - 'url': 'https://www.franceinter.fr/emissions/la-tete-au-carre/la-tete-au-carre-30-juin-2016', - 'md5': 'f13e4371662cf5a829f64d829ae78062', + 'url': 'https://www.franceinter.fr/emissions/la-marche-de-l-histoire/la-marche-de-l-histoire-18-decembre-2013', + 'md5': '4764932e466e6f6c79c317d2e74f6884', 'info_dict': { - 'id': 'la-tete-au-carre/la-tete-au-carre-30-juin-2016', + 'id': 'la-marche-de-l-histoire/la-marche-de-l-histoire-18-decembre-2013', 'ext': 'mp3', - 'title': 'Regards sur le sport du 30 juin 2016 - France Inter', - 'description': 'UEFA Europa, Jeux Olympiques... La période est aux sports, dans les gradins ou devant les écrans. Mais quel est le regard des spécialistes sur cette pratique? ', - 'timestamp': 1467244800, - 'upload_date': '20160630', + 'title': 'L’Histoire dans les jeux vidéo du 18 décembre 2013 - France Inter', + 'description': 'L’Histoire dans les jeux vidéo du 18 décembre 2013 par Jean Lebrun en replay sur France Inter. Retrouvez l\'émission en réécoute gratuite et abonnez-vous au podcast !', + 'timestamp': 1387324800, + 'upload_date': '20131218', }, } @@ -37,12 +38,16 @@ class FranceInterIE(InfoExtractor): title = self._og_search_title(webpage) description = self._og_search_description(webpage) - extractdate = self._search_regex( - r'([0-9]+[.][0-9]+[.][0-9]+)', video_url, 'extractdate', fatal=False) + extractdate = self._html_search_regex( + r'(.*?)', webpage, 'extractdate', fatal=False) + + extractdate = extractdate.split() + + extractdate = extractdate[3]+","+str(month_by_french_name(extractdate[2]))+","+extractdate[1] + + upload_date = unified_strdate(extractdate) timestamp = unified_timestamp(extractdate) - - upload_date = (unified_strdate(extractdate)) return { 'id': video_id, diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 495878a0e..a954f72f0 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -89,6 +89,10 @@ NO_DEFAULT = object() ENGLISH_MONTH_NAMES = [ 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + +FRENCH_MONTH_NAMES = [ + 'janvier', 'février', 'mars', 'avril', 'mai', 'juin', + 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'] KNOWN_EXTENSIONS = ( 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac', @@ -1558,6 +1562,13 @@ def month_by_name(name): except ValueError: return None +def month_by_french_name(name): + """ Return the number of a month by French name """ + + try: + return FRENCH_MONTH_NAMES.index(name) + 1 + except ValueError: + return None def month_by_abbreviation(abbrev): """ Return the number of a month by (locale-independently) English From 9c8fa7c63c35109cbcd21320f9e515272122cb43 Mon Sep 17 00:00:00 2001 From: renalid Date: Tue, 5 Jul 2016 16:33:57 +0200 Subject: [PATCH 3/3] [FranceInter] update after flake8 code review --- youtube_dl/extractor/franceinter.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py index 9a4684091..863879bf1 100644 --- a/youtube_dl/extractor/franceinter.py +++ b/youtube_dl/extractor/franceinter.py @@ -3,13 +3,10 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( - int_or_none, - unified_strdate, unified_timestamp, month_by_french_name, ) -import re class FranceInterIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/emissions/(?P[^?#]+)' @@ -37,16 +34,14 @@ class FranceInterIE(InfoExtractor): title = self._og_search_title(webpage) description = self._og_search_description(webpage) - + extractdate = self._html_search_regex( r'(.*?)', webpage, 'extractdate', fatal=False) - + extractdate = extractdate.split() - - extractdate = extractdate[3]+","+str(month_by_french_name(extractdate[2]))+","+extractdate[1] - - upload_date = unified_strdate(extractdate) - + + extractdate = extractdate[3] + "," + str(month_by_french_name(extractdate[2])) + "," + extractdate[1] + timestamp = unified_timestamp(extractdate) return { @@ -58,4 +53,4 @@ class FranceInterIE(InfoExtractor): 'url': video_url, 'vcodec': 'none', }], - } \ No newline at end of file + }