Add optional file-based listings caching

docs/source/api.rst

@@ -47,7 +47,9 @@ Base Classes
    fsspec.core.OpenFiles
    fsspec.core.get_fs_token_paths
    fsspec.core.url_to_fs
-   fsspec.dircache.DirCache
+   fsspec.dircache.DisabledListingsCache
+   fsspec.dircache.MemoryListingsCache
+   fsspec.dircache.FileListingsCache
    fsspec.FSMap
    fsspec.generic.GenericFileSystem
    fsspec.registry.register_implementation
@@ -82,7 +84,13 @@ Base Classes
 
 .. autofunction:: fsspec.core.url_to_fs
 
-.. autoclass:: fsspec.dircache.DirCache
+.. autoclass:: fsspec.dircache.DisabledListingsCache
+   :members: __init__
+
+.. autoclass:: fsspec.dircache.MemoryListingsCache
+   :members: __init__
+
+.. autoclass:: fsspec.dircache.FileListingsCache
    :members: __init__
 
 .. autoclass:: fsspec.FSMap

docs/source/changelog.rst

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+Dev
+--------
+
+Enhancements
+
+- add file-based listing cache using diskcache (#895)
+  warning: use new ``listings_cache_options`` instead of ``use_listings_cache`` etc.
+
 2024.3.1
 --------
 

docs/source/features.rst

@@ -181,15 +181,26 @@ Listings Caching
 ----------------
 
 For some implementations, getting file listings (i.e., ``ls`` and anything that
-depends on it) is expensive. These implementations use dict-like instances of
-:class:`fsspec.dircache.DirCache` to manage the listings.
-
-The cache allows for time-based expiry of entries with the ``listings_expiry_time``
-parameter, or LRU expiry with the ``max_paths`` parameter. These can be
-set on any implementation instance that uses listings caching; or to skip the
-caching altogether, use ``use_listings_cache=False``. That would be appropriate
-when the target location is known to be volatile because it is being written
-to from other sources.
+depends on it) is expensive. These implementations maye use either dict-like instances of
+:class:`fsspec.dircache.MemoryListingsCache` or file-based caching with instances of
+:class:`fsspec.dircache.FileListingsCache` to manage the listings.
+
+The listings cache can be controlled via the keyword ``listings_cache_options`` which is a dictionary.
+The type of cache that is used, can be controlled via the keyword ``cache_type`` (`disabled`, `memory` or `file`).
+The cache allows for time-based expiry of entries with the keyword ``expiry_time``. If the target location is known to
+be volatile because e.g. it is being written to from other sources we recommend to disable the listings cache.
+If you want to use the file-based caching, you can also provide the argument
+``directory`` to determine where the cache file is stored.
+
+Example for ``listings_cache_options``:
+
+.. code-block:: json
+
+        {
+            "cache_type": "file",
+            "expiry_time": 3600,
+            "directory": "/tmp/cache"
+        }
 
 When the ``fsspec`` instance writes to the backend, the method ``invalidate_cache``
 is called, so that subsequent listing of the given paths will force a refresh. In

fsspec/archive.py

@@ -37,19 +37,19 @@ def _all_dirnames(self, paths):
     def info(self, path, **kwargs):
         self._get_dirs()
         path = self._strip_protocol(path)
-        if path in {"", "/"} and self.dir_cache:
+        if path in {"", "/"} and self.listings_cache:
             return {"name": "", "type": "directory", "size": 0}
-        if path in self.dir_cache:
-            return self.dir_cache[path]
-        elif path + "/" in self.dir_cache:
-            return self.dir_cache[path + "/"]
+        if path in self.listings_cache:
+            return self.listings_cache[path]
+        elif path + "/" in self.listings_cache:
+            return self.listings_cache[path + "/"]
         else:
             raise FileNotFoundError(path)
 
     def ls(self, path, detail=True, **kwargs):
         self._get_dirs()
         paths = {}
-        for p, f in self.dir_cache.items():
+        for p, f in self.listings_cache.items():
             p = p.rstrip("/")
             if "/" in p:
                 root = p.rsplit("/", 1)[0]

fsspec/asyn.py

@@ -312,15 +312,23 @@ class AsyncFileSystem(AbstractFileSystem):
     mirror_sync_methods = True
     disable_throttling = False
 
-    def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs):
+    def __init__(
+        self,
+        *args,
+        asynchronous=False,
+        loop=None,
+        batch_size=None,
+        listings_cache_options=None,
+        **kwargs,
+    ):
         self.asynchronous = asynchronous
         self._pid = os.getpid()
         if not asynchronous:
             self._loop = loop or get_loop()
         else:
             self._loop = None
         self.batch_size = batch_size
-        super().__init__(*args, **kwargs)
+        super().__init__(listings_cache_options, *args, **kwargs)
 
     @property
     def loop(self):

fsspec/implementations/dbfs.py

@@ -89,7 +89,7 @@ def ls(self, path, detail=True, **kwargs):
                 }
                 for o in files
             ]
-            self.dircache[path] = out
+            self.listings_cache[path] = out
 
         if detail:
             return out
@@ -380,9 +380,9 @@ def _get_data(self, path, start, end):
 
     def invalidate_cache(self, path=None):
         if path is None:
-            self.dircache.clear()
+            self.listings_cache.clear()
         else:
-            self.dircache.pop(path, None)
+            self.listings_cache.pop(path, None)
         super().invalidate_cache(path)
 
 

fsspec/implementations/ftp.py

@@ -95,7 +95,7 @@ def _get_kwargs_from_urls(urlpath):
     def ls(self, path, detail=True, **kwargs):
         path = self._strip_protocol(path)
         out = []
-        if path not in self.dircache:
+        if path not in self.listings_cache:
             try:
                 try:
                     out = [
@@ -116,15 +116,15 @@ def ls(self, path, detail=True, **kwargs):
                         details["size"] = 0
                     if details["type"] == "dir":
                         details["type"] = "directory"
-                self.dircache[path] = out
+                self.listings_cache[path] = out
             except Error:
                 try:
                     info = self.info(path)
                     if info["type"] == "file":
                         out = [(path, info)]
                 except (Error, IndexError):
                     raise FileNotFoundError(path)
-        files = self.dircache.get(path, out)
+        files = self.listings_cache.get(path, out)
         if not detail:
             return sorted([fn for fn, details in files])
         return [details for fn, details in files]
@@ -252,9 +252,9 @@ def __del__(self):
 
     def invalidate_cache(self, path=None):
         if path is None:
-            self.dircache.clear()
+            self.listings_cache.clear()
         else:
-            self.dircache.pop(path, None)
+            self.listings_cache.pop(path, None)
         super().invalidate_cache(path)
 
 

fsspec/implementations/github.py

@@ -155,7 +155,7 @@ def ls(self, path, detail=False, sha=None, _sha=None, **kwargs):
                     else:
                         return path
                 _sha = out["sha"]
-        if path not in self.dircache or sha not in [self.root, None]:
+        if path not in self.listings_cache or sha not in [self.root, None]:
             r = requests.get(
                 self.url.format(org=self.org, repo=self.repo, sha=_sha),
                 timeout=self.timeout,
@@ -177,16 +177,16 @@ def ls(self, path, detail=False, sha=None, _sha=None, **kwargs):
                 if f["type"] in types
             ]
             if sha in [self.root, None]:
-                self.dircache[path] = out
+                self.listings_cache[path] = out
         else:
-            out = self.dircache[path]
+            out = self.listings_cache[path]
         if detail:
             return out
         else:
             return sorted([f["name"] for f in out])
 
     def invalidate_cache(self, path=None):
-        self.dircache.clear()
+        self.listings_cache.clear()
 
     @classmethod
     def _strip_protocol(cls, path):