v/youtube-dl
1
0
Fork 0
mirror of https://github.com/l1ving/youtube-dl synced 2020-11-18 19:53:54 -08:00
Code Issues Releases Wiki Activity

Cache the download archive in a list (fast!)

Browse Source 
This gives over 100x speed increase on my setup.
This commit is contained in:
Jasper Rebane 2020-09-20 17:59:54 +03:00 committed by GitHub
parent b55715934b
commit 2dc3dd181f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
youtube_dl/YoutubeDL.py
View File

@ -339,6 +339,7 @@ class YoutubeDL(object):
_download_retcode = None _download_retcode = None
_num_downloads = None _num_downloads = None
_screen_file = None _screen_file = None
archive_list = []
def __init__(self, params=None, auto_init=True): def __init__(self, params=None, auto_init=True):
"""Create a FileDownloader object with the given options.""" """Create a FileDownloader object with the given options."""
@ -2102,14 +2103,16 @@ class YoutubeDL(object):
if not vid_id: if not vid_id:
return False # Incomplete video information return False # Incomplete video information
try: if len(self.archive_list) == 0:
with locked_file(fn, 'r', encoding='utf-8') as archive_file: try:
for line in archive_file: with locked_file(fn, 'r', encoding='utf-8') as archive_file:
if line.strip() == vid_id: for line in archive_file:
return True self.archive_list.append(line.strip())
except IOError as ioe: except IOError as ioe:
if ioe.errno != errno.ENOENT: if ioe.errno != errno.ENOENT:
raise raise
if vid_id in self.archive_list:
return True
return False return False
def record_download_archive(self, info_dict): def record_download_archive(self, info_dict):
@ -2120,6 +2123,7 @@ class YoutubeDL(object):
assert vid_id assert vid_id
with locked_file(fn, 'a', encoding='utf-8') as archive_file: with locked_file(fn, 'a', encoding='utf-8') as archive_file:
archive_file.write(vid_id + '\n') archive_file.write(vid_id + '\n')
self.archive_list.append(vid_id)
@staticmethod @staticmethod
def format_resolution(format, default='unknown'): def format_resolution(format, default='unknown'):