mirror of
https://github.com/l1ving/youtube-dl
synced 2020-11-18 19:53:54 -08:00
Merge branch 'master' into use-other-downloaders
This commit is contained in:
Rogério Brito
commit
6cc4678e98
@ -41,12 +41,6 @@ class TestYoutubeLists(unittest.TestCase):
|
||||
ytie_results = [YoutubeIE()._extract_id(r[0]) for r in dl.result]
|
||||
self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])
|
||||
|
||||
def test_issue_661(self):
|
||||
dl = FakeDownloader()
|
||||
ie = YoutubePlaylistIE(dl)
|
||||
ie.extract('PLMCmkNmxw6Z9eduM7BZjSEh7HiU543Ig0')
|
||||
self.assertTrue(len(dl.result) > 20)
|
||||
|
||||
def test_issue_673(self):
|
||||
dl = FakeDownloader()
|
||||
ie = YoutubePlaylistIE(dl)
|
||||
|
||||
@ -128,18 +128,6 @@
|
||||
"file": "0732f586d7.mp4",
|
||||
"md5": "f647e9e90064b53b6e046e75d0241fbd"
|
||||
},
|
||||
{
|
||||
"name": "TweetReel",
|
||||
"url": "http://tweetreel.com/?77smq",
|
||||
"file": "77smq.mov",
|
||||
"md5": "56b4d9ca9de467920f3f99a6d91255d6",
|
||||
"info_dict": {
|
||||
"uploader": "itszero",
|
||||
"uploader_id": "itszero",
|
||||
"upload_date": "20091225",
|
||||
"description": "Installing Gentoo Linux on Powerbook G4, it turns out the sleep indicator becomes HDD activity indicator :D"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Steam",
|
||||
"url": "http://store.steampowered.com/video/105600/",
|
||||
@ -305,5 +293,11 @@
|
||||
"info_dict": {
|
||||
"title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Generic",
|
||||
"url": "http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html",
|
||||
"file": "13601338388002.mp4",
|
||||
"md5": "85b90ccc9d73b4acd9138d3af4c27f89"
|
||||
}
|
||||
]
|
||||
|
||||
@ -104,7 +104,7 @@ class FileDownloader(object):
|
||||
self.params = params
|
||||
|
||||
if '%(stitle)s' in self.params['outtmpl']:
|
||||
self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
|
||||
self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
|
||||
|
||||
@staticmethod
|
||||
def format_bytes(bytes):
|
||||
@ -234,6 +234,18 @@ class FileDownloader(object):
|
||||
raise DownloadError(message)
|
||||
self._download_retcode = 1
|
||||
|
||||
def report_warning(self, message):
|
||||
'''
|
||||
Print the message to stderr, it will be prefixed with 'WARNING:'
|
||||
If stderr is a tty file the 'WARNING:' will be colored
|
||||
'''
|
||||
if sys.stderr.isatty():
|
||||
_msg_header=u'\033[0;33mWARNING:\033[0m'
|
||||
else:
|
||||
_msg_header=u'WARNING:'
|
||||
warning_message=u'%s %s' % (_msg_header,message)
|
||||
self.to_stderr(warning_message)
|
||||
|
||||
def slow_down(self, start_time, byte_counter):
|
||||
"""Sleep if the download speed is over the rate limit."""
|
||||
rate_limit = self.params.get('ratelimit', None)
|
||||
@ -360,8 +372,11 @@ class FileDownloader(object):
|
||||
|
||||
filename = self.params['outtmpl'] % template_dict
|
||||
return filename
|
||||
except (ValueError, KeyError) as err:
|
||||
self.trouble(u'ERROR: invalid system charset or erroneous output template')
|
||||
except KeyError as err:
|
||||
self.trouble(u'ERROR: Erroneous output template')
|
||||
return None
|
||||
except ValueError as err:
|
||||
self.trouble(u'ERROR: Insufficient system charset ' + repr(preferredencoding()))
|
||||
return None
|
||||
|
||||
def _match_entry(self, info_dict):
|
||||
@ -496,8 +511,8 @@ class FileDownloader(object):
|
||||
|
||||
# Warn if the _WORKING attribute is False
|
||||
if not ie.working():
|
||||
self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
|
||||
u'and will probably not work. If you want to go on, use the -i option.')
|
||||
self.report_warning(u'the program functionality for this site has been marked as broken, '
|
||||
u'and will probably not work. If you want to go on, use the -i option.')
|
||||
|
||||
# Suitable InfoExtractor found
|
||||
suitable_found = True
|
||||
@ -555,7 +570,7 @@ class FileDownloader(object):
|
||||
self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
|
||||
os.remove(encodeFilename(filename))
|
||||
except (IOError, OSError):
|
||||
self.to_stderr(u'WARNING: Unable to remove downloaded video file')
|
||||
self.report_warning(u'Unable to remove downloaded video file')
|
||||
|
||||
def _download_with_rtmpdump(self, filename, url, player_url, page_url):
|
||||
self.report_destination(filename)
|
||||
@ -563,7 +578,7 @@ class FileDownloader(object):
|
||||
|
||||
# Check for rtmpdump first
|
||||
try:
|
||||
subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
|
||||
subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
|
||||
except (OSError, IOError):
|
||||
self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
|
||||
return False
|
||||
|
||||
@ -308,7 +308,7 @@ class YoutubeIE(InfoExtractor):
|
||||
else:
|
||||
raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
|
||||
except (IOError, netrc.NetrcParseError) as err:
|
||||
self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
|
||||
self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
# Set language
|
||||
@ -317,7 +317,7 @@ class YoutubeIE(InfoExtractor):
|
||||
self.report_lang()
|
||||
compat_urllib_request.urlopen(request).read()
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
|
||||
self._downloader.report_warning(u'unable to set language: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
# No authentication to be performed
|
||||
@ -328,7 +328,7 @@ class YoutubeIE(InfoExtractor):
|
||||
try:
|
||||
login_page = compat_urllib_request.urlopen(request).read().decode('utf-8')
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.to_stderr(u'WARNING: unable to fetch login page: %s' % compat_str(err))
|
||||
self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
galx = None
|
||||
@ -372,10 +372,10 @@ class YoutubeIE(InfoExtractor):
|
||||
self.report_login()
|
||||
login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
|
||||
if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
|
||||
self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
|
||||
self._downloader.report_warning(u'unable to log in: bad username or password')
|
||||
return
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
|
||||
self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
# Confirm age
|
||||
@ -1281,7 +1281,8 @@ class GenericIE(InfoExtractor):
|
||||
|
||||
def report_download_webpage(self, video_id):
|
||||
"""Report webpage download."""
|
||||
self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
|
||||
if not self._downloader.params.get('test', False):
|
||||
self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
|
||||
self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
|
||||
|
||||
def report_extraction(self, video_id):
|
||||
@ -1351,13 +1352,8 @@ class GenericIE(InfoExtractor):
|
||||
if self._test_redirect(url): return
|
||||
|
||||
video_id = url.split('/')[-1]
|
||||
request = compat_urllib_request.Request(url)
|
||||
try:
|
||||
self.report_download_webpage(video_id)
|
||||
webpage = compat_urllib_request.urlopen(request).read()
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
|
||||
return
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
except ValueError as err:
|
||||
# since this is the last-resort InfoExtractor, if
|
||||
# this error is thrown, it'll be thrown here
|
||||
@ -1456,7 +1452,7 @@ class YoutubeSearchIE(InfoExtractor):
|
||||
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
|
||||
return
|
||||
elif n > self._max_youtube_results:
|
||||
self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
|
||||
self._downloader.report_warning(u'ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
|
||||
n = self._max_youtube_results
|
||||
self._download_n_results(query, n)
|
||||
return
|
||||
@ -1538,7 +1534,7 @@ class GoogleSearchIE(InfoExtractor):
|
||||
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
|
||||
return
|
||||
elif n > self._max_google_results:
|
||||
self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
|
||||
self._downloader.report_warning(u'gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
|
||||
n = self._max_google_results
|
||||
self._download_n_results(query, n)
|
||||
return
|
||||
@ -1622,7 +1618,7 @@ class YahooSearchIE(InfoExtractor):
|
||||
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
|
||||
return
|
||||
elif n > self._max_yahoo_results:
|
||||
self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
|
||||
self._downloader.report_warning(u'yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
|
||||
n = self._max_yahoo_results
|
||||
self._download_n_results(query, n)
|
||||
return
|
||||
@ -2080,7 +2076,7 @@ class FacebookIE(InfoExtractor):
|
||||
else:
|
||||
raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
|
||||
except (IOError, netrc.NetrcParseError) as err:
|
||||
self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
|
||||
self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
if useremail is None:
|
||||
@ -2097,10 +2093,10 @@ class FacebookIE(InfoExtractor):
|
||||
self.report_login()
|
||||
login_results = compat_urllib_request.urlopen(request).read()
|
||||
if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
|
||||
self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
|
||||
self._downloader.report_warning(u'unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
|
||||
return
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
|
||||
self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
def _real_extract(self, url):
|
||||
@ -2165,6 +2161,17 @@ class BlipTVIE(InfoExtractor):
|
||||
self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
|
||||
return
|
||||
|
||||
urlp = compat_urllib_parse_urlparse(url)
|
||||
if urlp.path.startswith('/play/'):
|
||||
request = compat_urllib_request.Request(url)
|
||||
response = compat_urllib_request.urlopen(request)
|
||||
redirecturl = response.geturl()
|
||||
rurlp = compat_urllib_parse_urlparse(redirecturl)
|
||||
file_id = compat_parse_qs(rurlp.fragment)['file'][0].rpartition('/')[2]
|
||||
url = 'http://blip.tv/a/a-' + file_id
|
||||
return self._real_extract(url)
|
||||
|
||||
|
||||
if '?' in url:
|
||||
cchar = '&'
|
||||
else:
|
||||
@ -2546,7 +2553,7 @@ class EscapistIE(InfoExtractor):
|
||||
'uploader': showName,
|
||||
'upload_date': None,
|
||||
'title': showName,
|
||||
'ext': 'flv',
|
||||
'ext': 'mp4',
|
||||
'thumbnail': imgUrl,
|
||||
'description': description,
|
||||
'player_url': playerUrl,
|
||||
@ -3580,55 +3587,6 @@ class FunnyOrDieIE(InfoExtractor):
|
||||
}
|
||||
return [info]
|
||||
|
||||
class TweetReelIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P<id>[0-9a-z]+)$'
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
if mobj is None:
|
||||
self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
|
||||
return
|
||||
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
m = re.search(r'<div id="left" status_id="([0-9]+)">', webpage)
|
||||
if not m:
|
||||
self._downloader.trouble(u'ERROR: Cannot find status ID')
|
||||
status_id = m.group(1)
|
||||
|
||||
m = re.search(r'<div class="tweet_text">(.*?)</div>', webpage, flags=re.DOTALL)
|
||||
if not m:
|
||||
self._downloader.trouble(u'WARNING: Cannot find description')
|
||||
desc = unescapeHTML(re.sub('<a.*?</a>', '', m.group(1))).strip()
|
||||
|
||||
m = re.search(r'<div class="tweet_info">.*?from <a target="_blank" href="https?://twitter.com/(?P<uploader_id>.+?)">(?P<uploader>.+?)</a>', webpage, flags=re.DOTALL)
|
||||
if not m:
|
||||
self._downloader.trouble(u'ERROR: Cannot find uploader')
|
||||
uploader = unescapeHTML(m.group('uploader'))
|
||||
uploader_id = unescapeHTML(m.group('uploader_id'))
|
||||
|
||||
m = re.search(r'<span unixtime="([0-9]+)"', webpage)
|
||||
if not m:
|
||||
self._downloader.trouble(u'ERROR: Cannot find upload date')
|
||||
upload_date = datetime.datetime.fromtimestamp(int(m.group(1))).strftime('%Y%m%d')
|
||||
|
||||
title = desc
|
||||
video_url = 'http://files.tweetreel.com/video/' + status_id + '.mov'
|
||||
|
||||
info = {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'mov',
|
||||
'title': title,
|
||||
'description': desc,
|
||||
'uploader': uploader,
|
||||
'uploader_id': uploader_id,
|
||||
'internal_id': status_id,
|
||||
'upload_date': upload_date
|
||||
}
|
||||
return [info]
|
||||
|
||||
class SteamIE(InfoExtractor):
|
||||
_VALID_URL = r"""http://store.steampowered.com/
|
||||
(?P<urltype>video|app)/ #If the page is only for videos or for a game
|
||||
@ -3767,7 +3725,7 @@ class YouPornIE(InfoExtractor):
|
||||
# Get the video date
|
||||
result = re.search(r'Date:</label>(?P<date>.*) </li>', webpage)
|
||||
if result is None:
|
||||
self._downloader.to_stderr(u'WARNING: unable to extract video date')
|
||||
self._downloader.report_warning(u'unable to extract video date')
|
||||
upload_date = None
|
||||
else:
|
||||
upload_date = result.group('date').strip()
|
||||
@ -3775,7 +3733,7 @@ class YouPornIE(InfoExtractor):
|
||||
# Get the video uploader
|
||||
result = re.search(r'Submitted:</label>(?P<uploader>.*)</li>', webpage)
|
||||
if result is None:
|
||||
self._downloader.to_stderr(u'WARNING: unable to extract uploader')
|
||||
self._downloader.report_warning(u'unable to extract uploader')
|
||||
video_uploader = None
|
||||
else:
|
||||
video_uploader = result.group('uploader').strip()
|
||||
@ -4173,7 +4131,6 @@ def gen_extractors():
|
||||
NBAIE(),
|
||||
JustinTVIE(),
|
||||
FunnyOrDieIE(),
|
||||
TweetReelIE(),
|
||||
SteamIE(),
|
||||
UstreamIE(),
|
||||
RBMARadioIE(),
|
||||
|
||||
@ -126,7 +126,7 @@ def parseOpts():
|
||||
general.add_option('-i', '--ignore-errors',
|
||||
action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
|
||||
general.add_option('-r', '--rate-limit',
|
||||
dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
|
||||
dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)')
|
||||
general.add_option('-R', '--retries',
|
||||
dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
|
||||
general.add_option('--buffer-size',
|
||||
@ -274,12 +274,20 @@ def parseOpts():
|
||||
|
||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
|
||||
if xdg_config_home:
|
||||
userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
|
||||
userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
|
||||
else:
|
||||
userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
|
||||
argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
|
||||
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
|
||||
systemConf = _readOptions('/etc/youtube-dl.conf')
|
||||
userConf = _readOptions(userConfFile)
|
||||
commandLineConf = sys.argv[1:]
|
||||
argv = systemConf + userConf + commandLineConf
|
||||
opts, args = parser.parse_args(argv)
|
||||
|
||||
if opts.verbose:
|
||||
print(u'[debug] System config: ' + repr(systemConf))
|
||||
print(u'[debug] User config: ' + repr(userConf))
|
||||
print(u'[debug] Command-line args: ' + repr(commandLineConf))
|
||||
|
||||
return parser, opts, args
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user