From 8207626bbe5c35a553ec63c81d075c1ba042aab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 28 Feb 2013 22:07:29 +0100 Subject: [PATCH 01/12] Use color when printing warning messages --- youtube_dl/FileDownloader.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 192ad37d2..73f8dbd5f 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -104,7 +104,7 @@ class FileDownloader(object): self.params = params if '%(stitle)s' in self.params['outtmpl']: - self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') + self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') @staticmethod def format_bytes(bytes): @@ -234,6 +234,18 @@ class FileDownloader(object): raise DownloadError(message) self._download_retcode = 1 + def report_warning(self, message): + ''' + Print the message to stderr, it will be prefixed with 'WARNING:' + If stderr is a tty file the 'WARNING:' will be colored + ''' + if sys.stderr.isatty(): + _msg_header=u'\033[0;33mWARNING:\033[0m' + else: + _msg_header=u'WARNING:' + warning_message=u'%s %s' % (_msg_header,message) + self.to_stderr(warning_message) + def slow_down(self, start_time, byte_counter): """Sleep if the download speed is over the rate limit.""" rate_limit = self.params.get('ratelimit', None) @@ -496,8 +508,8 @@ class FileDownloader(object): # Warn if the _WORKING attribute is False if not ie.working(): - self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, ' - u'and will probably not work. If you want to go on, use the -i option.') + self.report_warning(u'the program functionality for this site has been marked as broken, ' + u'and will probably not work. If you want to go on, use the -i option.') # Suitable InfoExtractor found suitable_found = True @@ -555,7 +567,7 @@ class FileDownloader(object): self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename) os.remove(encodeFilename(filename)) except (IOError, OSError): - self.to_stderr(u'WARNING: Unable to remove downloaded video file') + self.report_warning(u'Unable to remove downloaded video file') def _download_with_rtmpdump(self, filename, url, player_url, page_url): self.report_destination(filename) From 7f9d41a55edb8e939c1bb69f0c3ad29380cad478 Mon Sep 17 00:00:00 2001 From: Juan Carlos Garcia Segovia Date: Fri, 1 Mar 2013 10:22:16 +0000 Subject: [PATCH 02/12] Allow downloading http://blip.tv/play/ embeded URLs --- youtube_dl/InfoExtractors.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index a94648dcf..64079d1fd 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2165,6 +2165,21 @@ class BlipTVIE(InfoExtractor): self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return + if '/play/' in url: + request = compat_urllib_request.Request(url) + response = compat_urllib_request.urlopen(request) + redirecturl = response.geturl() + hashindex = redirecturl.find('#') + if hashindex!=-1: + hash = redirecturl[hashindex+1:] + params = compat_parse_qs(hash) + files = params['file'] + for file in files: + match = re.search('/(\d+)',file) + if match: + file_id = match.group(1) + url = 'http://blip.tv/a/a-'+file_id + if '?' in url: cchar = '&' else: From 2e5457be1d2db836bfbd0c7e18b30ca4a0bbfef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 2 Mar 2013 11:24:07 +0100 Subject: [PATCH 03/12] Use report_warning in InfoExtractors --- youtube_dl/InfoExtractors.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index a94648dcf..c696bb968 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -308,7 +308,7 @@ class YoutubeIE(InfoExtractor): else: raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) except (IOError, netrc.NetrcParseError) as err: - self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) + self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) return # Set language @@ -317,7 +317,7 @@ class YoutubeIE(InfoExtractor): self.report_lang() compat_urllib_request.urlopen(request).read() except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err)) + self._downloader.report_warning(u'unable to set language: %s' % compat_str(err)) return # No authentication to be performed @@ -328,7 +328,7 @@ class YoutubeIE(InfoExtractor): try: login_page = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.to_stderr(u'WARNING: unable to fetch login page: %s' % compat_str(err)) + self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err)) return galx = None @@ -372,10 +372,10 @@ class YoutubeIE(InfoExtractor): self.report_login() login_results = compat_urllib_request.urlopen(request).read().decode('utf-8') if re.search(r'(?i)]* id="gaia_loginform"', login_results) is not None: - self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') + self._downloader.report_warning(u'unable to log in: bad username or password') return except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) + self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) return # Confirm age @@ -1456,7 +1456,7 @@ class YoutubeSearchIE(InfoExtractor): self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) return elif n > self._max_youtube_results: - self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) + self._downloader.report_warning(u'ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) n = self._max_youtube_results self._download_n_results(query, n) return @@ -1538,7 +1538,7 @@ class GoogleSearchIE(InfoExtractor): self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) return elif n > self._max_google_results: - self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) + self._downloader.report_warning(u'gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) n = self._max_google_results self._download_n_results(query, n) return @@ -1622,7 +1622,7 @@ class YahooSearchIE(InfoExtractor): self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) return elif n > self._max_yahoo_results: - self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) + self._downloader.report_warning(u'yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) n = self._max_yahoo_results self._download_n_results(query, n) return @@ -2080,7 +2080,7 @@ class FacebookIE(InfoExtractor): else: raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) except (IOError, netrc.NetrcParseError) as err: - self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) + self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) return if useremail is None: @@ -2097,10 +2097,10 @@ class FacebookIE(InfoExtractor): self.report_login() login_results = compat_urllib_request.urlopen(request).read() if re.search(r'', login_results) is not None: - self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') + self._downloader.report_warning(u'unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') return except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) + self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) return def _real_extract(self, url): @@ -3767,7 +3767,7 @@ class YouPornIE(InfoExtractor): # Get the video date result = re.search(r'Date:(?P.*) ', webpage) if result is None: - self._downloader.to_stderr(u'WARNING: unable to extract video date') + self._downloader.report_warning(u'unable to extract video date') upload_date = None else: upload_date = result.group('date').strip() @@ -3775,7 +3775,7 @@ class YouPornIE(InfoExtractor): # Get the video uploader result = re.search(r'Submitted:(?P.*)', webpage) if result is None: - self._downloader.to_stderr(u'WARNING: unable to extract uploader') + self._downloader.report_warning(u'unable to extract uploader') video_uploader = None else: video_uploader = result.group('uploader').strip() From f7b567ff84b4be37099d7907157829f8642074dd Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 3 Mar 2013 22:09:44 +0100 Subject: [PATCH 04/12] Use proper urlparse functions and simplify a bit --- youtube_dl/InfoExtractors.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index d19efe93f..4b6a1c097 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2165,20 +2165,16 @@ class BlipTVIE(InfoExtractor): self._downloader.trouble(u'ERROR: invalid URL: %s' % url) return - if '/play/' in url: + urlp = compat_urllib_parse_urlparse(url) + if urlp.path.startswith('/play/'): request = compat_urllib_request.Request(url) response = compat_urllib_request.urlopen(request) redirecturl = response.geturl() - hashindex = redirecturl.find('#') - if hashindex!=-1: - hash = redirecturl[hashindex+1:] - params = compat_parse_qs(hash) - files = params['file'] - for file in files: - match = re.search('/(\d+)',file) - if match: - file_id = match.group(1) - url = 'http://blip.tv/a/a-'+file_id + rurlp = compat_urllib_parse_urlparse(redirecturl) + file_id = compat_parse_qs(rurlp.fragment)['file'][0].rpartition('/')[2] + url = 'http://blip.tv/a/a-' + file_id + return self._real_extract(url) + if '?' in url: cchar = '&' From 7ae153ee9c55257369c4ad584578e93be4c108a3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 3 Mar 2013 22:15:06 +0100 Subject: [PATCH 05/12] Remove tweetreel - it has shut down --- test/tests.json | 12 --------- youtube_dl/InfoExtractors.py | 50 ------------------------------------ 2 files changed, 62 deletions(-) diff --git a/test/tests.json b/test/tests.json index fd7eb2d65..7af3c2892 100644 --- a/test/tests.json +++ b/test/tests.json @@ -128,18 +128,6 @@ "file": "0732f586d7.mp4", "md5": "f647e9e90064b53b6e046e75d0241fbd" }, - { - "name": "TweetReel", - "url": "http://tweetreel.com/?77smq", - "file": "77smq.mov", - "md5": "56b4d9ca9de467920f3f99a6d91255d6", - "info_dict": { - "uploader": "itszero", - "uploader_id": "itszero", - "upload_date": "20091225", - "description": "Installing Gentoo Linux on Powerbook G4, it turns out the sleep indicator becomes HDD activity indicator :D" - } - }, { "name": "Steam", "url": "http://store.steampowered.com/video/105600/", diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 4b6a1c097..7ce84fe79 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -3591,55 +3591,6 @@ class FunnyOrDieIE(InfoExtractor): } return [info] -class TweetReelIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P[0-9a-z]+)$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - video_id = mobj.group('id') - webpage = self._download_webpage(url, video_id) - - m = re.search(r'
', webpage) - if not m: - self._downloader.trouble(u'ERROR: Cannot find status ID') - status_id = m.group(1) - - m = re.search(r'
(.*?)
', webpage, flags=re.DOTALL) - if not m: - self._downloader.trouble(u'WARNING: Cannot find description') - desc = unescapeHTML(re.sub('', '', m.group(1))).strip() - - m = re.search(r'
.*?from (?P.+?)', webpage, flags=re.DOTALL) - if not m: - self._downloader.trouble(u'ERROR: Cannot find uploader') - uploader = unescapeHTML(m.group('uploader')) - uploader_id = unescapeHTML(m.group('uploader_id')) - - m = re.search(r'video|app)/ #If the page is only for videos or for a game @@ -4184,7 +4135,6 @@ def gen_extractors(): NBAIE(), JustinTVIE(), FunnyOrDieIE(), - TweetReelIE(), SteamIE(), UstreamIE(), RBMARadioIE(), From a2ae43a55fa954c8ca2a0d3dc4c1724d7e6c54a2 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 3 Mar 2013 22:19:19 +0100 Subject: [PATCH 06/12] Remove changed playlist test (#661) --- test/test_youtube_lists.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 9c2e82ea3..f4705bc5b 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -41,12 +41,6 @@ class TestYoutubeLists(unittest.TestCase): ytie_results = [YoutubeIE()._extract_id(r[0]) for r in dl.result] self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE']) - def test_issue_661(self): - dl = FakeDownloader() - ie = YoutubePlaylistIE(dl) - ie.extract('PLMCmkNmxw6Z9eduM7BZjSEh7HiU543Ig0') - self.assertTrue(len(dl.result) > 20) - def test_issue_673(self): dl = FakeDownloader() ie = YoutubePlaylistIE(dl) From f918ec7ea29a37521d1fc22fb9f900283c5a2c49 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 3 Mar 2013 22:35:26 +0100 Subject: [PATCH 07/12] Clarify rate limit documentation (Closes #723) --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 23e3c2ac2..3983e2f0e 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -126,7 +126,7 @@ def parseOpts(): general.add_option('-i', '--ignore-errors', action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) general.add_option('-r', '--rate-limit', - dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)') + dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)') general.add_option('-R', '--retries', dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) general.add_option('--buffer-size', From 967897fd227c85ebe3368b359a4f8c471e513de6 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 3 Mar 2013 22:38:38 +0100 Subject: [PATCH 08/12] Fix Python 3 errors with rmtp downloads --- youtube_dl/FileDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 73f8dbd5f..57f741c30 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -575,7 +575,7 @@ class FileDownloader(object): # Check for rtmpdump first try: - subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) except (OSError, IOError): self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') return False From 47dcd621c0ee23018ba306ee7f6ba6e338ef06da Mon Sep 17 00:00:00 2001 From: Tim Douglas Date: Wed, 6 Mar 2013 12:46:45 -0500 Subject: [PATCH 09/12] Escapist videos are acutally .mp4, not .flv --- youtube_dl/InfoExtractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 7ce84fe79..6b03bf307 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -2557,7 +2557,7 @@ class EscapistIE(InfoExtractor): 'uploader': showName, 'upload_date': None, 'title': showName, - 'ext': 'flv', + 'ext': 'mp4', 'thumbnail': imgUrl, 'description': description, 'player_url': playerUrl, From e5edd51de458d52f3824e6d8fc7c0713659694a4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 8 Mar 2013 20:12:05 +0100 Subject: [PATCH 10/12] Clear up error messages (#734) --- youtube_dl/FileDownloader.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 57f741c30..3b2adf84b 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -372,8 +372,11 @@ class FileDownloader(object): filename = self.params['outtmpl'] % template_dict return filename - except (ValueError, KeyError) as err: - self.trouble(u'ERROR: invalid system charset or erroneous output template') + except KeyError as err: + self.trouble(u'ERROR: Erroneous output template') + return None + except ValueError as err: + self.trouble(u'ERROR: Insufficient system charset ' + repr(preferredencoding())) return None def _match_entry(self, info_dict): From 3d3423574d35a0fe71062f21dd57ada02a5225b4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 8 Mar 2013 20:47:06 +0100 Subject: [PATCH 11/12] Fix Unicode handling GenericIE (Fixes #734) --- test/tests.json | 6 ++++++ youtube_dl/InfoExtractors.py | 10 +++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/test/tests.json b/test/tests.json index 7af3c2892..e4ea0b41e 100644 --- a/test/tests.json +++ b/test/tests.json @@ -293,5 +293,11 @@ "info_dict": { "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2" } + }, + { + "name": "Generic", + "url": "http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html", + "file": "13601338388002.mp4", + "md5": "85b90ccc9d73b4acd9138d3af4c27f89" } ] diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 6b03bf307..baba4a9a2 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1281,7 +1281,8 @@ class GenericIE(InfoExtractor): def report_download_webpage(self, video_id): """Report webpage download.""" - self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') + if not self._downloader.params.get('test', False): + self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id) def report_extraction(self, video_id): @@ -1351,13 +1352,8 @@ class GenericIE(InfoExtractor): if self._test_redirect(url): return video_id = url.split('/')[-1] - request = compat_urllib_request.Request(url) try: - self.report_download_webpage(video_id) - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) - return + webpage = self._download_webpage(url, video_id) except ValueError as err: # since this is the last-resort InfoExtractor, if # this error is thrown, it'll be thrown here From 8c42c506cdaab6f8e1cc65a2f3f2f756188a68fe Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 12 Mar 2013 00:10:05 +0100 Subject: [PATCH 12/12] Add configuration to -v output --- youtube_dl/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3983e2f0e..8a7aab7ac 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -274,12 +274,20 @@ def parseOpts(): xdg_config_home = os.environ.get('XDG_CONFIG_HOME') if xdg_config_home: - userConf = os.path.join(xdg_config_home, 'youtube-dl.conf') + userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') else: - userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') - argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:] + userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') + systemConf = _readOptions('/etc/youtube-dl.conf') + userConf = _readOptions(userConfFile) + commandLineConf = sys.argv[1:] + argv = systemConf + userConf + commandLineConf opts, args = parser.parse_args(argv) + if opts.verbose: + print(u'[debug] System config: ' + repr(systemConf)) + print(u'[debug] User config: ' + repr(userConf)) + print(u'[debug] Command-line args: ' + repr(commandLineConf)) + return parser, opts, args def _real_main():