fetch dependencies in parallel on sandcastle

Reviewed By: bigfootjon

Differential Revision: D66661978

fbshipit-source-id: 8ba5906b4e1e9273b3df834e3c50d984a987ba1c

build/fbcode_builder/getdeps/buildopts.py

@@ -523,7 +523,8 @@ def find_unused_drive_letter():
     return available[-1]
 
 
-def create_subst_path(path: str) -> str:
+def map_subst_path(path: str) -> str:
+    """find a short drive letter mapping for a path"""
     for _attempt in range(0, 24):
         drive = find_existing_win32_subst_for_path(
             path, subst_mapping=list_win32_subst_letters()
@@ -544,9 +545,11 @@ def create_subst_path(path: str) -> str:
         # other processes on the same host, so this may not succeed.
         try:
             subprocess.check_call(["subst", "%s:" % available, path])
-            return "%s:\\" % available
+            subst = "%s:\\" % available
+            print("Mapped scratch dir %s -> %s" % (path, subst), file=sys.stderr)
+            return subst
         except Exception:
-            print("Failed to map %s -> %s" % (available, path))
+            print("Failed to map %s -> %s" % (available, path), file=sys.stderr)
 
     raise Exception("failed to set up a subst path for %s" % path)
 
@@ -619,10 +622,7 @@ def setup_build_options(args, host_type=None) -> BuildOptions:
             os.makedirs(scratch_dir)
 
         if is_windows():
-            subst = create_subst_path(scratch_dir)
-            print(
-                "Mapping scratch dir %s -> %s" % (scratch_dir, subst), file=sys.stderr
-            )
+            subst = map_subst_path(scratch_dir)
             scratch_dir = subst
     else:
         if not os.path.exists(scratch_dir):

build/fbcode_builder/getdeps/fetcher.py

@@ -868,6 +868,7 @@ def update(self) -> ChangeStatus:
 
         if not os.path.exists(self.file_name):
             self._download()
+            self._verify_hash()
 
         if tarfile.is_tarfile(self.file_name):
             opener = tarfile.open
@@ -877,19 +878,20 @@ def update(self) -> ChangeStatus:
             raise Exception("don't know how to extract %s" % self.file_name)
         os.makedirs(self.src_dir)
         print("Extract %s -> %s" % (self.file_name, self.src_dir))
-        t = opener(self.file_name)
         if is_windows():
             # Ensure that we don't fall over when dealing with long paths
             # on windows
             src = r"\\?\%s" % os.path.normpath(self.src_dir)
         else:
             src = self.src_dir
-        # The `str` here is necessary to ensure that we don't pass a unicode
-        # object down to tarfile.extractall on python2.  When extracting
-        # the boost tarball it makes some assumptions and tries to convert
-        # a non-ascii path to ascii and throws.
-        src = str(src)
-        t.extractall(src)
+
+        with opener(self.file_name) as t:
+            # The `str` here is necessary to ensure that we don't pass a unicode
+            # object down to tarfile.extractall on python2.  When extracting
+            # the boost tarball it makes some assumptions and tries to convert
+            # a non-ascii path to ascii and throws.
+            src = str(src)
+            t.extractall(src)
 
         with open(self.hash_file, "w") as f:
             f.write(self.sha256)