run .t tests for getdeps sapling cli build

Summary:
Connect up the sapling *.t tests so that github CI has visible cli test status.

To get them to run:
 * some needed fbpython on path. I included a shim for that in the test makefile target.
 * test-rust-hooks: Command not found message,  added a glob
 * test-identity.t: add a glob for the sapling version
 * test-eolfilename.t: output order was unstable, added sorts to make stable
 * helpers-testrepo.sh: fix assumption that system hg would be able to read test repo, check if its Sapling first.
 * added a manifest for the hexdump utility some of the tests required
 * excluded a few remaining tests (see comments in Makefile for reason)
 * fixed getdeps support for generating actions steps for test only dependencies

NB  the tests run as "hg".  The expectations would need to be updated if we were to run as "sl"

X-link: facebook/sapling#963
X-link: facebookincubator/zstrong#1004

Reviewed By: quark-zju

Differential Revision: D63958737

Pulled By: ahornby

fbshipit-source-id: 75c0d39258c320100d8d02b31390994bc2f3a3ce

build/fbcode_builder/getdeps.py

@@ -963,6 +963,12 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
         self.process_project_dir_arguments(args, loader)
         manifest = loader.load_manifest(args.project)
         manifest_ctx = loader.ctx_gen.get_context(manifest.name)
+        run_tests = (
+            args.enable_tests
+            and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off"
+        )
+        if run_tests:
+            manifest_ctx.set("test", "on")
         run_on = self.get_run_on(args)
 
         # Some projects don't do anything "useful" as a leaf project, only
@@ -1075,10 +1081,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 free_up_disk = ""
 
             allow_sys_arg = ""
-            if (
-                build_opts.allow_system_packages
-                and build_opts.host_type.get_package_manager()
-            ):
+            if run_tests:
                 sudo_arg = "sudo "
                 allow_sys_arg = " --allow-system-packages"
                 if build_opts.host_type.get_package_manager() == "deb":
@@ -1097,6 +1100,19 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                     out.write(
                         f"      run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf\n"
                     )
+                required_locales = manifest.get(
+                    "github.actions", "required_locales", ctx=manifest_ctx
+                )
+                if (
+                    build_opts.host_type.get_package_manager() == "deb"
+                    and required_locales
+                ):
+                    # ubuntu doesn't include this by default
+                    out.write("    - name: Install locale-gen\n")
+                    out.write(f"      run: {sudo_arg}apt-get install locales\n")
+                    for loc in required_locales.split():
+                        out.write(f"    - name: Ensure {loc} locale present\n")
+                        out.write(f"      run: {sudo_arg}locale-gen {loc}\n")
 
             projects = loader.manifests_in_dependency_order()
 
@@ -1188,11 +1204,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             out.write("        name: %s\n" % manifest.name)
             out.write("        path: _artifacts\n")
 
-            if (
-                args.enable_tests
-                and manifest.get("github.actions", "run_tests", ctx=manifest_ctx)
-                != "off"
-            ):
+            if run_tests:
                 num_jobs_arg = ""
                 if args.num_jobs:
                     num_jobs_arg = f"--num-jobs {args.num_jobs} "
@@ -1203,6 +1215,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 )
             if build_opts.free_up_disk and not build_opts.is_windows():
                 out.write("    - name: Show disk space at end\n")
+                out.write("      if: always()\n")
                 out.write("      run: df -h\n")
 
     def setup_project_cmd_parser(self, parser):

build/fbcode_builder/getdeps/manifest.py

@@ -87,6 +87,7 @@
         "optional_section": True,
         "fields": {
             "run_tests": OPTIONAL,
+            "required_locales": OPTIONAL,
         },
     },
     "crate.pathmap": {"optional_section": True},

build/fbcode_builder/manifests/hexdump

@@ -0,0 +1,12 @@
+[manifest]
+name = hexdump
+
+[rpms]
+util-linux
+
+[debs]
+bsdmainutils
+
+# only used from system packages currently
+[build]
+builder = nop

build/fbcode_builder/manifests/sapling

@@ -4,6 +4,9 @@ fbsource_path = fbcode/eden
 shipit_project = eden
 shipit_fbcode_builder = true
 
+[github.actions]
+required_locales = en_US.UTF-8
+
 [git]
 repo_url = https://github.com/facebook/sapling.git
 
@@ -57,6 +60,9 @@ fb303
 fbthrift
 rust-shed
 
+[dependencies.test=on]
+hexdump
+
 [dependencies.not(os=windows)]
 python