From 64903427530167778b312e840daa94a14da6a795 Mon Sep 17 00:00:00 2001 From: jakeogh Date: Thu, 27 Dec 2018 17:08:20 -0700 Subject: [PATCH 1/2] modify sanitize_open() to use locked_file(), preventing silent corruption when a second youtube-dl instance is attempting to write the same file. There is still a corner case, if a .part file is being used (--no-part is not enabled), in that the .part file is closed before it's renamed to remove the .part, in that window, another process could modify the .part file before it's renamed. Using --no-part prevents this corner case. --- youtube_dl/utils.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 62e769fd5..ec6343da8 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -455,7 +455,8 @@ def sanitize_open(filename, open_mode): import msvcrt msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename) - stream = open(encodeFilename(filename), open_mode) + stream = locked_file(encodeFilename(filename), open_mode, block=False) + stream = stream.__enter__() return (stream, filename) except (IOError, OSError) as err: if err.errno in (errno.EACCES,): @@ -467,7 +468,8 @@ def sanitize_open(filename, open_mode): raise else: # An exception here should be caught in the caller - stream = open(encodeFilename(alt_filename), open_mode) + stream = locked_file(encodeFilename(filename), open_mode, block=False) + stream = stream.__enter__() return (stream, alt_filename) @@ -1562,15 +1564,18 @@ else: try: import fcntl - def _lock_file(f, exclusive): - fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + def _lock_file(f, exclusive, block): + if block: + fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + else: + fcntl.flock(f, fcntl.LOCK_EX|fcntl.LOCK_NB if exclusive else fcntl.LOCK_SH) def _unlock_file(f): fcntl.flock(f, fcntl.LOCK_UN) except ImportError: UNSUPPORTED_MSG = 'file locking is not supported on this platform' - def _lock_file(f, exclusive): + def _lock_file(f, exclusive, block): raise IOError(UNSUPPORTED_MSG) def _unlock_file(f): @@ -1578,15 +1583,16 @@ else: class locked_file(object): - def __init__(self, filename, mode, encoding=None): - assert mode in ['r', 'a', 'w'] + def __init__(self, filename, mode, block=True, encoding=None): + assert mode in ['r', 'rb', 'a', 'ab', 'w', 'wb'] self.f = io.open(filename, mode, encoding=encoding) self.mode = mode + self.block = block def __enter__(self): - exclusive = self.mode != 'r' + exclusive = self.mode not in ['r', 'rb'] try: - _lock_file(self.f, exclusive) + _lock_file(self.f, exclusive, self.block) except IOError: self.f.close() raise @@ -1607,6 +1613,8 @@ class locked_file(object): def read(self, *args): return self.f.read(*args) + def close(self, *args): + self.__exit__(self, *args, value=False, traceback=False) def get_filesystem_encoding(): encoding = sys.getfilesystemencoding() From 1f30831ab9faf00ca3f78b96314a0ac453b0172b Mon Sep 17 00:00:00 2001 From: jakeogh Date: Sat, 29 Dec 2018 01:17:24 -0700 Subject: [PATCH 2/2] add locked_file.flush() --- youtube_dl/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index ec6343da8..12de61412 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1613,6 +1613,9 @@ class locked_file(object): def read(self, *args): return self.f.read(*args) + def flush(self, *args): + self.f.flush() + def close(self, *args): self.__exit__(self, *args, value=False, traceback=False)