[wip][feature] Add buffering argument to enable/disable buffering or …

…set the buffer size to MountSource.open

core/ratarmountcore/AutoMountLayer.py

@@ -301,9 +301,9 @@ def fileVersions(self, path: str) -> int:
         return fileVersions
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
         _, mountSource, sourceFileInfo = self.getMountSource(fileInfo)
-        return mountSource.open(sourceFileInfo)
+        return mountSource.open(sourceFileInfo, buffer=buffering)
 
     @overrides(MountSource)
     def read(self, fileInfo: FileInfo, size: int, offset: int) -> bytes:

core/ratarmountcore/FileVersionLayer.py

@@ -222,8 +222,8 @@ def fileVersions(self, path: str) -> int:
         return self.mountSource.fileVersions(path)
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
-        return self.mountSource.open(fileInfo)
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
+        return self.mountSource.open(fileInfo, buffering=buffering)
 
     @overrides(MountSource)
     def read(self, fileInfo: FileInfo, size: int, offset: int) -> bytes:

core/ratarmountcore/FolderMountSource.py

@@ -159,10 +159,10 @@ def fileVersions(self, path: str) -> int:
         return 1 if self.exists(path) else 0
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
         realpath = self.getFilePath(fileInfo)
         try:
-            return open(realpath, 'rb')
+            return open(realpath, 'rb', buffering=buffering)
         except Exception as e:
             raise ValueError(f"Specified path '{realpath}' is not a file that can be read!") from e
 

core/ratarmountcore/LibarchiveMountSource.py

@@ -634,7 +634,7 @@ def __del__(self):
         pass
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo):
+    def open(self, fileInfo: FileInfo, buffering=-1):
         assert fileInfo.userdata
         tarFileInfo = fileInfo.userdata[-1]
         assert isinstance(tarFileInfo, SQLiteIndexedTarUserData)

core/ratarmountcore/MountSource.py

@@ -65,8 +65,12 @@ def getFileInfo(self, path: str, fileVersion: int = 0) -> Optional[FileInfo]:
         pass
 
     @abstractmethod
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
-        pass
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
+        """
+        buffering : Behaves similarly to Python's built-in open call. A value of 0 should disable buffering.
+                    Any value larger than 1 should be the buffer size. The default of -1 may result in
+                    a default buffer size equal to the file(system)'s block size or Python's io.DEFAULT_BUFFER_SIZE.
+        """
 
     def statfs(self) -> Dict[str, Any]:
         """

core/ratarmountcore/RarMountSource.py

@@ -192,7 +192,8 @@ def fileVersions(self, path: str) -> int:
         return len(self._getFileInfos(path))
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
+        # I do not see any obvious option to zipfile.ZipFile to apply the specified buffer size.
         info = fileInfo.userdata[-1][1]
         assert isinstance(info, rarfile.RarInfo)
         return self.fileObject.open(info, 'r')

core/ratarmountcore/SQLiteIndexedTar.py

@@ -1121,7 +1121,8 @@ def _createIndexRecursively(
             self.index.setFileInfo(fileInfo)
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
+        # TODO Apply buffering argument. Maybe extend StenciledFile to take a file descriptor as argument.
         assert fileInfo.userdata
         tarFileInfo = fileInfo.userdata[-1]
         assert isinstance(tarFileInfo, SQLiteIndexedTarUserData)

core/ratarmountcore/SingleFileMountSource.py

@@ -86,7 +86,8 @@ def getFileInfo(self, path: str, fileVersion: int = 0) -> Optional[FileInfo]:
         return self._createFileInfo() if path == self.path else None
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
+        # TODO Apply buffering argument. Maybe extend StenciledFile to take a file descriptor as argument.
         if fileInfo != self._createFileInfo():
             raise ValueError("Only files may be opened!")
         # Use StenciledFile so that the returned file objects can be independently seeked!

core/ratarmountcore/SquashFSMountSource.py

@@ -36,7 +36,7 @@
     # We need to define this for @overrides and pytype, but it also is a nice documentation
     # for the expected members in PySquashfsImage.SquashFsImage.
     class SquashFsImage:  # type: ignore
-        def __init__(self, fd, offset=0, closefd=True):
+        def __init__(self, fd, offset: int=0, closefd: bool=True) -> None:
             self._sblk: Any = None
 
         def __iter__(self):
@@ -415,7 +415,7 @@ def __init__(
             raise ValueError("Not a valid SquashFS image!")
 
         # fmt: off
-        self.fileObject             = SquashFSImage(self.rawFileObject, offset=offset)
+        self.image                  = SquashFSImage(self.rawFileObject, offset=offset)
         self.archiveFilePath        = fileOrPath if isinstance(fileOrPath, str) else None
         self.encoding               = encoding
         self.verifyModificationTime = verifyModificationTime
@@ -516,7 +516,7 @@ def _createIndex(self) -> None:
 
         # TODO Doing this in a chunked manner with generators would make it work better for large archives.
         fileInfos = []
-        for inodeOffset, info in self.fileObject:
+        for inodeOffset, info in self.image:
             fileInfos.append(self._convertToRow(inodeOffset, info))
         self.index.setFileInfos(fileInfos)
 
@@ -534,14 +534,16 @@ def _createIndex(self) -> None:
     def __exit__(self, exception_type, exception_value, exception_traceback):
         super().__exit__(exception_type, exception_value, exception_traceback)
         self.rawFileObject.close()
-        self.fileObject.close()
+        self.image.close()
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
+        # TODO Can we reasonably apply buffering? Maybe for uncompressed SquashFS images when we can delegate
+        #      to Stenciled file? Check whether the file blocks are contiguous and watch out for the tail.
         assert fileInfo.userdata
         extendedFileInfo = fileInfo.userdata[-1]
         assert isinstance(extendedFileInfo, SQLiteIndexedTarUserData)
-        return self.fileObject.open(self.fileObject.read_inode(extendedFileInfo.offsetheader))
+        return self.image.open(self.image.read_inode(extendedFileInfo.offsetheader))
 
     @overrides(MountSource)
     def statfs(self) -> Dict[str, Any]:
@@ -551,7 +553,7 @@ def statfs(self) -> Dict[str, Any]:
         except Exception:
             pass
 
-        blockSize = max(blockSize, self.fileObject._sblk.block_size)
+        blockSize = max(blockSize, self.image._sblk.block_size)
         return {
             'f_bsize': blockSize,
             'f_frsize': blockSize,

core/ratarmountcore/SubvolumesMountSource.py

@@ -86,12 +86,12 @@ def listDirModeOnly(self, path: str) -> Optional[Union[Iterable[str], Dict[str,
         return self._listDir(path, onlyMode=True)
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
         subvolume = fileInfo.userdata.pop()
         if subvolume is None:
             raise ValueError(f"Found subvolume is None for fileInfo: {fileInfo}")
         try:
-            return self.mountSources[subvolume].open(fileInfo)
+            return self.mountSources[subvolume].open(fileInfo, buffering=buffering)
         finally:
             fileInfo.userdata.append(subvolume)
 

core/ratarmountcore/UnionMountSource.py

@@ -192,11 +192,11 @@ def listDirModeOnly(self, path: str) -> Optional[Union[Iterable[str], Dict[str,
         return self._listDir(path, onlyMode=True)
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
         mountSource = fileInfo.userdata.pop()
         try:
             assert isinstance(mountSource, MountSource)
-            return mountSource.open(fileInfo)
+            return mountSource.open(fileInfo, buffering=buffering)
         finally:
             fileInfo.userdata.append(mountSource)
 

core/ratarmountcore/ZipMountSource.py

@@ -198,7 +198,8 @@ def __exit__(self, exception_type, exception_value, exception_traceback):
         self.fileObject.close()
 
     @overrides(MountSource)
-    def open(self, fileInfo: FileInfo) -> IO[bytes]:
+    def open(self, fileInfo: FileInfo, buffering=-1) -> IO[bytes]:
+        # I do not see any obvious option to zipfile.ZipFile to apply the specified buffer size.
         assert fileInfo.userdata
         extendedFileInfo = fileInfo.userdata[-1]
         assert isinstance(extendedFileInfo, SQLiteIndexedTarUserData)