fix(test): do not use build step for optional pipeline

Optional pipeline started failing during a build failure in the rust
benchmarks tests.

During the build step, we build REVISION_A (main) and REVISION_B (tip of
the PR commit) and pass it over to the subsequent pipeline steps that
run the test.

However, 'test_benchmarks.py' needs to build the benchmarks, running
`cargo bench`. That step started failing when building 'aws-lc-rs-sys'
because CMake is complaining that its CMakeCache.txt file is not in the
same path as the one initially created.

Since we anyway run `cargo bench` in the test there's not really need to
pre-build firecracker. So, modify the buildkite pipeline logic to allow
defining pipelines without the build step, and change the
pipeline_pr_no_block.py to not use it.

Signed-off-by: Babis Chalios <bchalios@amazon.es>

.buildkite/common.py

@@ -229,7 +229,7 @@ class BKPipeline:
 
     parser = COMMON_PARSER
 
-    def __init__(self, initial_steps=None, **kwargs):
+    def __init__(self, initial_steps=None, with_build_step=True, **kwargs):
         self.steps = []
         self.args = args = self.parser.parse_args()
         # Retry one time if agent was lost. This can happen if we terminate the
@@ -252,9 +252,12 @@ def __init__(self, initial_steps=None, **kwargs):
         self.per_arch["platforms"] = [("al2", "linux_5.10")]
         self.binary_dir = args.binary_dir
         # Build sharing
-        build_cmds, self.shared_build = shared_build()
-        step_build = group("🏗️ Build", build_cmds, **self.per_arch)
-        self.steps += [step_build, "wait"]
+        if with_build_step:
+            build_cmds, self.shared_build = shared_build()
+            step_build = group("🏗️ Build", build_cmds, **self.per_arch)
+            self.steps += [step_build, "wait"]
+        else:
+            self.shared_build = None
 
         # If we run initial_steps before the "wait" step above, then a failure of the initial steps
         # would result in the build not progressing past the "wait" step (as buildkite only proceeds past a wait step
@@ -284,10 +287,12 @@ def add_step(self, step, decorate=True):
 
     def _adapt_group(self, group):
         """"""
-        prepend = [
-            f'buildkite-agent artifact download "{self.shared_build}" .',
-            f"tar xzf {self.shared_build}",
-        ]
+        prepend = []
+        if self.shared_build is not None:
+            prepend = [
+                f'buildkite-agent artifact download "{self.shared_build}" .',
+                f"tar xzf {self.shared_build}",
+            ]
         if self.binary_dir is not None:
             prepend.extend(
                 [
@@ -324,10 +329,12 @@ def to_json(self):
         """Serialize the pipeline to JSON"""
         return json.dumps(self.to_dict(), indent=4, sort_keys=True, ensure_ascii=False)
 
-    def devtool_test(self, devtool_opts=None, pytest_opts=None):
+    def devtool_test(self, devtool_opts=None, pytest_opts=None, devtool_no_build=True):
         """Generate a `devtool test` command"""
         cmds = []
-        parts = ["./tools/devtool -y test", "--no-build"]
+        parts = ["./tools/devtool -y test"]
+        if devtoop_no_build:
+            parts.append("--no-build")
         if devtool_opts:
             parts.append(devtool_opts)
         parts.append("--")

.buildkite/pipeline_pr_no_block.py

@@ -11,6 +11,7 @@
 DEFAULT_PRIORITY = 1
 
 pipeline = BKPipeline(
+    with_build_step=False,
     timeout_in_minutes=45,
     # some non-blocking tests are performance, so make sure they get ag=1 instances
     priority=DEFAULT_PRIORITY + 1,
@@ -20,6 +21,7 @@
 pipeline.build_group(
     "❓ Optional",
     pipeline.devtool_test(
+        devtool_no_build=False,
         devtool_opts="--performance -c 1-10 -m 0",
         pytest_opts="integration_tests/ -m 'no_block_pr and not nonci' --log-cli-level=INFO",
     ),