Support test and input skipping with fuzztest::SkipTestsOrCurrentInput.

e2e_tests/functional_test.cc

@@ -603,6 +603,34 @@ TEST_F(UnitTestModeTest, TimeLimitFlagWorks) {
   EXPECT_THAT(status, Eq(Signal(SIGABRT)));
 }
 
+TEST_F(UnitTestModeTest, TestIsSkippedWhenRequestedInFixturePerTest) {
+  auto [status, std_out, std_err] =
+      Run("SkippedTestFixturePerTest.SkippedTest", kDefaultTargetBinary,
+          /*env=*/{},
+          /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}});
+  EXPECT_THAT(std_err,
+              HasSubstr("Skipping SkippedTestFixturePerTest.SkippedTest"));
+  EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest is executed")));
+  EXPECT_THAT(status, Eq(ExitCode(0)));
+}
+
+TEST_F(UnitTestModeTest, TestIsSkippedWhenRequestedInFixturePerIteration) {
+  auto [status, std_out, std_err] =
+      Run("SkippedTestFixturePerIteration.SkippedTest", kDefaultTargetBinary,
+          /*env=*/{},
+          /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}});
+  EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest is executed")));
+  EXPECT_THAT(status, Eq(ExitCode(0)));
+}
+
+TEST_F(UnitTestModeTest, InputsAreSkippedWhenRequestedInTests) {
+  auto [status, std_out, std_err] =
+      Run("MySuite.SkipInputs", kDefaultTargetBinary,
+          /*env=*/{},
+          /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}});
+  EXPECT_THAT(std_err, HasSubstr("Skipped input"));
+}
+
 class GetRandomValueTest : public UnitTestModeTest {
  protected:
   int GetValueFromInnerTest(
@@ -1166,6 +1194,25 @@ TEST_F(FuzzingModeCommandLineInterfaceTest,
   EXPECT_THAT(status, Eq(ExitCode(0)));
 }
 
+// This tests both the command line interface and the fuzzing logic. It is under
+// FuzzingModeCommandLineInterfaceTest so it can specify the command line.
+TEST_F(FuzzingModeCommandLineInterfaceTest, CorpusDoesNotContainSkippedInputs) {
+  TempDir corpus_dir;
+  // Although theoretically possible, it is extreme unlikely that the test would
+  // find the crash without saving some corpus.
+  auto [producer_status, producer_std_out, producer_std_err] =
+      RunWith({{"fuzz", "MySuite.SkipInputs"}, {"fuzz_for", "10s"}},
+              {{"FUZZTEST_TESTSUITE_OUT_DIR", corpus_dir.dirname()}});
+
+  ASSERT_THAT(producer_std_err, HasSubstr("Skipped input"));
+
+  auto [replayer_status, replayer_std_out, replayer_std_err] =
+      RunWith({{"fuzz", "MySuite.SkipInputs"}},
+              {{"FUZZTEST_REPLAY", corpus_dir.dirname()}});
+
+  EXPECT_THAT(replayer_std_err, Not(HasSubstr("Skipped input")));
+}
+
 std::string CentipedePath() {
   const auto test_srcdir = absl::NullSafeStringView(getenv("TEST_SRCDIR"));
   FUZZTEST_INTERNAL_CHECK_PRECONDITION(
@@ -1305,6 +1352,23 @@ TEST_P(FuzzingModeFixtureTest, GoogleTestStaticTestSuiteFunctionsCalledOnce) {
       CountSubstrs(std_err, "<<CallCountGoogleTest::TearDownTestSuite()>>"));
 }
 
+TEST_P(FuzzingModeFixtureTest, TestIsSkippedWhenRequestedInFixturePerTest) {
+  auto [status, std_out, std_err] =
+      Run("SkippedTestFixturePerTest.SkippedTest", /*iterations=*/10);
+  EXPECT_THAT(std_err,
+              HasSubstr("Skipping SkippedTestFixturePerTest.SkippedTest"));
+  EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest should not be run")));
+  EXPECT_THAT(status, Eq(ExitCode(0)));
+}
+
+TEST_P(FuzzingModeFixtureTest,
+       TestIsSkippedWhenRequestedInFixturePerIteration) {
+  auto [status, std_out, std_err] =
+      Run("SkippedTestFixturePerIteration.SkippedTest", /*iterations=*/10);
+  EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest should not be run")));
+  EXPECT_THAT(status, Eq(ExitCode(0)));
+}
+
 INSTANTIATE_TEST_SUITE_P(FuzzingModeFixtureTestWithExecutionModel,
                          FuzzingModeFixtureTest,
                          testing::ValuesIn(GetAvailableExecutionModels()));
@@ -1705,6 +1769,14 @@ TEST_P(FuzzingModeCrashFindingTest,
   ExpectTargetAbort(status, std_err);
 }
 
+TEST_P(FuzzingModeCrashFindingTest, InputsAreSkippedWhenRequestedInTests) {
+  auto [status, std_out, std_err] =
+      Run("MySuite.SkipInputs", kDefaultTargetBinary);
+  EXPECT_THAT(std_err, HasSubstr("Skipped input"));
+  EXPECT_THAT(std_err, HasSubstr("argument 0: 123456789"));
+  ExpectTargetAbort(status, std_err);
+}
+
 INSTANTIATE_TEST_SUITE_P(FuzzingModeCrashFindingTestWithExecutionModel,
                          FuzzingModeCrashFindingTest,
                          testing::ValuesIn(GetAvailableExecutionModels()));

e2e_tests/testdata/fuzz_tests_for_functional_testing.cc

@@ -785,6 +785,38 @@ FUZZ_TEST(MySuite, LargeHeapAllocation)
         // 1 GiB
         1ULL << 30));
 
+// A fuzz test that is expected to accept and skip some inputs before hitting
+// the crash.
+void SkipInputs(uint32_t input) {
+  static bool skipped_input = false;
+  static bool accepted_input = false;
+  // Crash only when `input` is 123456789.
+  if (input != 123456789) {
+    // The condition below should have enough chance to either pass or not.
+    //
+    // Note that we want the input to here be accepted at least once so that the
+    // fuzzing engine can learn about the branch above.
+    if (input % 7 % 2 == 0) {
+      if (!skipped_input) {
+        skipped_input = true;
+        std::cerr << "Skipped input" << std::endl;
+      }
+      fuzztest::SkipTestsOrCurrentInput();
+      return;
+    }
+    if (!accepted_input) accepted_input = true;
+    return;
+  }
+  // This introduces statefulness which is undesired in real fuzz tests, but
+  // here it makes it more reliable for functional testing.
+  if (skipped_input && accepted_input) {
+    std::abort();
+  }
+}
+// Due to the limitation of the fuzzing engine, there must be an accepted input
+// when initializing the corpus for fuzzing. So we provide one.
+FUZZ_TEST(MySuite, SkipInputs).WithSeeds({1});
+
 }  // namespace
 
 int main(int argc, char** argv) {

e2e_tests/testdata/fuzz_tests_using_googletest.cc

@@ -18,6 +18,7 @@
 // to show that regular FUZZ_TEST work without having to #include GoogleTest.
 
 #include <cstdio>
+#include <cstdlib>
 #include <limits>
 
 #include "gtest/gtest.h"
@@ -124,4 +125,41 @@ void CrashOnFailingTestInput(const std::string& input) {
 }
 FUZZ_TEST(MySuite, CrashOnFailingTestInput);
 
+class SkippedTestFixturePerTest
+    : public ::fuzztest::PerFuzzTestFixtureAdapter<testing::Test> {
+ public:
+  SkippedTestFixturePerTest() { fuzztest::SkipTestsOrCurrentInput(); }
+
+  void SkippedTest() {
+    fprintf(stderr, "SkippedTest is executed! Aborting\n");
+    std::abort();
+  }
+};
+FUZZ_TEST_F(SkippedTestFixturePerTest, SkippedTest);
+
+class SkippedTestFixturePerIteration
+    : public ::fuzztest::PerIterationFixtureAdapter<testing::Test> {
+ public:
+  // For the engine limitation, there must be at least one non-skipped input
+  // when initializing the corpus for fuzzing. So we always accept the first
+  // input.
+  SkippedTestFixturePerIteration() {
+    if (!first_iteration_) fuzztest::SkipTestsOrCurrentInput();
+  }
+
+  void SkippedTest() {
+    if (first_iteration_) {
+      first_iteration_ = false;
+      return;
+    }
+    fprintf(stderr, "SkippedTest is executed! Aborting\n");
+    std::abort();
+  }
+
+ private:
+  static bool first_iteration_;
+};
+bool SkippedTestFixturePerIteration::first_iteration_ = true;
+FUZZ_TEST_F(SkippedTestFixturePerIteration, SkippedTest);
+
 }  // namespace

fuzztest/fuzztest_macros.h

@@ -153,6 +153,18 @@ inline std::vector<uint8_t> ToByteArray(std::string_view str) {
   return std::vector<uint8_t>(str.begin(), str.end());
 }
 
+// When called during the fixture setup (in the constructor or SetUp()), skips
+// calling property functions until the matching teardown (destructor or
+// TearDown()). When called in a property function, skips adding the current
+// input to the corpus when fuzzing.
+//
+// Note that this function should not be called frequently due to engine
+// limitation and efficiency reasons. Consider refining the domain definitions
+// to restrict input generation if possible.
+inline void SkipTestsOrCurrentInput() {
+  internal::Runtime::instance().SetSkippingRequested(true);
+}
+
 }  // namespace fuzztest
 
 #endif  // FUZZTEST_FUZZTEST_FUZZTEST_MACROS_H_

fuzztest/internal/centipede_adaptor.cc

@@ -447,8 +447,10 @@ void PopulateTestLimitsToCentipedeRunner(const Configuration& configuration) {
 class CentipedeFixtureDriver : public UntypedFixtureDriver {
  public:
   CentipedeFixtureDriver(
+      Runtime& runtime,
       std::unique_ptr<UntypedFixtureDriver> orig_fixture_driver)
-      : orig_fixture_driver_(std::move(orig_fixture_driver)) {}
+      : runtime_(runtime),
+        orig_fixture_driver_(std::move(orig_fixture_driver)) {}
 
   void SetUpFuzzTest() override {
     orig_fixture_driver_->SetUpFuzzTest();
@@ -464,6 +466,9 @@ class CentipedeFixtureDriver : public UntypedFixtureDriver {
 
   void TearDownIteration() override {
     orig_fixture_driver_->TearDownIteration();
+    if (runtime_.skipping_requested()) {
+      CentipedeSetExecutionResult(nullptr, 0);
+    }
     if (!runner_mode) CentipedeFinalizeProcessing();
   }
 
@@ -487,6 +492,7 @@ class CentipedeFixtureDriver : public UntypedFixtureDriver {
 
  private:
   const Configuration* configuration_ = nullptr;
+  Runtime& runtime_;
   const bool runner_mode = getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr;
   std::unique_ptr<UntypedFixtureDriver> orig_fixture_driver_;
 };
@@ -495,7 +501,7 @@ CentipedeFuzzerAdaptor::CentipedeFuzzerAdaptor(
     const FuzzTest& test, std::unique_ptr<UntypedFixtureDriver> fixture_driver)
     : test_(test),
       centipede_fixture_driver_(
-          new CentipedeFixtureDriver(std::move(fixture_driver))),
+          new CentipedeFixtureDriver(runtime_, std::move(fixture_driver))),
       fuzzer_impl_(test_, absl::WrapUnique(centipede_fixture_driver_)) {
   FUZZTEST_INTERNAL_CHECK(centipede_fixture_driver_ != nullptr,
                           "Invalid fixture driver!");
@@ -513,6 +519,7 @@ int CentipedeFuzzerAdaptor::RunInFuzzingMode(
     int* argc, char*** argv, const Configuration& configuration) {
   centipede_fixture_driver_->set_configuration(&configuration);
   runtime_.SetRunMode(RunMode::kFuzz);
+  runtime_.SetSkippingRequested(false);
   runtime_.SetCurrentTest(&test_, &configuration);
   if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr();
   runtime_.EnableReporter(&fuzzer_impl_.stats_, [] { return absl::Now(); });
@@ -527,6 +534,12 @@ int CentipedeFuzzerAdaptor::RunInFuzzingMode(
   // and we should not run CentipedeMain in this process.
   const bool runner_mode = getenv("CENTIPEDE_RUNNER_FLAGS");
   const int result = ([&]() {
+    if (runtime_.skipping_requested()) {
+      absl::FPrintF(GetStderr(),
+                    "[.] Skipping %s per request from the test setup.\n",
+                    test_.full_name());
+      return 0;
+    }
     if (runner_mode) {
       CentipedeAdaptorRunnerCallbacks runner_callbacks(&runtime_, &fuzzer_impl_,
                                                        &configuration);

fuzztest/internal/centipede_adaptor.h

@@ -35,10 +35,10 @@ class CentipedeFuzzerAdaptor : public FuzzTestFuzzer {
                        const Configuration& configuration) override;
 
  private:
+  Runtime& runtime_ = Runtime::instance();
   const FuzzTest& test_;
   CentipedeFixtureDriver* centipede_fixture_driver_;
   FuzzTestFuzzerImpl fuzzer_impl_;
-  Runtime& runtime_ = Runtime::instance();
 };
 
 }  // namespace fuzztest::internal

fuzztest/internal/runtime.cc

@@ -827,10 +827,17 @@ void PopulateLimits(const Configuration& configuration,
 }
 
 void FuzzTestFuzzerImpl::RunInUnitTestMode(const Configuration& configuration) {
+  runtime_.SetSkippingRequested(false);
   fixture_driver_->SetUpFuzzTest();
-  runtime_.StartWatchdog();
-  PopulateLimits(configuration, execution_coverage_);
   [&] {
+    if (runtime_.skipping_requested()) {
+      absl::FPrintF(GetStderr(),
+                    "[.] Skipping %s per request from the test setup.\n",
+                    test_.full_name());
+      return;
+    }
+    runtime_.StartWatchdog();
+    PopulateLimits(configuration, execution_coverage_);
     runtime_.EnableReporter(&stats_, [] { return absl::Now(); });
     runtime_.SetCurrentTest(&test_, &configuration);
 
@@ -923,16 +930,19 @@ FuzzTestFuzzerImpl::RunResult FuzzTestFuzzerImpl::RunOneInput(
     execution_coverage_->SetIsTracing(true);
   }
 
+  runtime_.SetSkippingRequested(false);
   fixture_driver_->SetUpIteration();
-  fixture_driver_->Test(std::move(untyped_args));
+  if (!runtime_.skipping_requested()) {
+    fixture_driver_->Test(std::move(untyped_args));
+  }
   fixture_driver_->TearDownIteration();
   if (execution_coverage_ != nullptr) {
     execution_coverage_->SetIsTracing(false);
   }
   const absl::Duration run_time = absl::Now() - start;
 
   bool new_coverage = false;
-  if (execution_coverage_ != nullptr) {
+  if (execution_coverage_ != nullptr && !runtime_.skipping_requested()) {
     new_coverage = corpus_coverage_.Update(execution_coverage_);
     stats_.max_stack_used =
         std::max(stats_.max_stack_used, execution_coverage_->MaxStackUsed());
@@ -986,10 +996,17 @@ void FuzzTestFuzzerImpl::MinimizeNonFatalFailureLocally(absl::BitGenRef prng) {
 
 int FuzzTestFuzzerImpl::RunInFuzzingMode(int* /*argc*/, char*** /*argv*/,
                                          const Configuration& configuration) {
+  runtime_.SetSkippingRequested(false);
   fixture_driver_->SetUpFuzzTest();
-  runtime_.StartWatchdog();
-  PopulateLimits(configuration, execution_coverage_);
   const int exit_code = [&] {
+    if (runtime_.skipping_requested()) {
+      absl::FPrintF(GetStderr(),
+                    "[.] Skipping %s per request from the test setup.\n",
+                    test_.full_name());
+      return 0;
+    }
+    runtime_.StartWatchdog();
+    PopulateLimits(configuration, execution_coverage_);
     runtime_.SetRunMode(RunMode::kFuzz);
 
     if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr();

fuzztest/internal/runtime.h

@@ -126,6 +126,14 @@ class Runtime {
     return external_failure_was_detected_.load(std::memory_order_relaxed);
   }
 
+  void SetSkippingRequested(bool requested) {
+    skipping_requested_.store(requested, std::memory_order_relaxed);
+  }
+
+  bool skipping_requested() const {
+    return skipping_requested_.load(std::memory_order_relaxed);
+  }
+
   void SetShouldTerminateOnNonFatalFailure(bool v) {
     should_terminate_on_non_fatal_failure_ = v;
   }
@@ -196,15 +204,20 @@ class Runtime {
   // Note: Even though failures should happen within the code under test, they
   // could be set from other threads at any moment. We make it an atomic to
   // avoid a race condition.
-  std::atomic<bool> external_failure_was_detected_{false};
+  std::atomic<bool> external_failure_was_detected_ = false;
 
   // To support in-process minimization for non-fatal failures we signal
   // suppress termination until we believe minimization is complete.
   bool should_terminate_on_non_fatal_failure_ = true;
 
+  // If set to true in fixture setup, skips calling property functions
+  // utill the matching teardown is called; If set to true in a property
+  // function, skip adding the current input to the corpus when fuzzing.
+  std::atomic<bool> skipping_requested_ = false;
+
   // If true, fuzzing should terminate as soon as possible.
   // Atomic because it is set from signal handlers.
-  std::atomic<bool> termination_requested_{false};
+  std::atomic<bool> termination_requested_ = false;
 
   RunMode run_mode_ = RunMode::kUnitTest;
   std::atomic<bool> watchdog_thread_started = false;