Merge pull request #77 from rabernat/incremental-download

use incremental downloading

pangeo_forge/recipe.py

@@ -236,9 +236,16 @@ def cache_input(self) -> Callable:
         def cache_func(input_key: Hashable) -> None:
             logger.info(f"Caching input {input_key}")
             fname = self._inputs[input_key]
+            # TODO: add a check for whether the input is already cached?
             with input_opener(fname, mode="rb", **self.fsspec_open_kwargs) as source:
                 with self.input_cache.open(fname, mode="wb") as target:
-                    target.write(source.read())
+                    # TODO: make this configurable? Would we ever want to change it?
+                    BLOCK_SIZE = 10_000_000  # 10 MB
+                    while True:
+                        data = source.read(BLOCK_SIZE)
+                        if not data:
+                            break
+                        target.write(data)
 
         return cache_func