Skip to content

Commit

Permalink
[performance] Fix block size being ignored when reading everything vi…
Browse files Browse the repository at this point in the history
…a io.BufferedReader
  • Loading branch information
mxmlnkn committed Sep 11, 2024
1 parent 4bd6eea commit 4a94f4a
Showing 1 changed file with 15 additions and 0 deletions.
15 changes: 15 additions & 0 deletions core/ratarmountcore/StenciledFile.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,21 @@ def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
def tell(self) -> int:
return self.offset

@overrides(io.RawIOBase)
def readall(self) -> bytes:
# It is necessary to implement this, or else the io.RawIOBase.readall implementation would use
# io.DEFAULT_BUFFER_SIZE (8 KiB). Notably, this would ignore the block size configured in BufferedReader,
# when calling read(-1) on it because it thinks that raw.readall is a fast-path, but in this case is ~100x
# slower than 4 MiB reads equal to the Lustre-advertised block size.
# https://github.com/python/cpython/issues/85624
chunks = []
while True:
result = self.read()
if not result:
break
chunks.append(result)
return b"".join(chunks)


class RawJoinedFileFromFactory(io.RawIOBase):
def __init__(self, file_object_factories: List[Callable[[], IO[bytes]]], file_lock=None) -> None:
Expand Down

0 comments on commit 4a94f4a

Please sign in to comment.