✨ Add m command for max-failures (#116)

Adds a new `m` command for dynamically controlling the `--max-failures` option of `mix test`.

README.md

@@ -17,9 +17,9 @@ which tests should be run with a few keystrokes.
 
 It allows you to easily switch between running all tests, stale tests, or failed
 tests. Or, you can run only the tests whose filenames contain a substring. You
-can also control which tags are included or excluded and easily specify the test
-seed to use.
-Includes an optional "watch mode" which runs tests after every file change.
+can also control which tags are included or excluded, modify the maximum number
+of failures allowed, and specify the test seed to use. Includes an optional
+"watch mode" which runs tests after every file change.
 
 ## Installation
 
@@ -92,6 +92,8 @@ more patterns on the command-line, `mix test.interactive` will find all test
 files matching those patterns and pass them to `mix test` as if you had used the
 `p` command (described below).
 
+## Interactive Commands
+
 After the tests run, you can use the interactive commands to change which tests
 will run.
 
@@ -104,6 +106,9 @@ will run.
 - `i <tags...>`: Include tests tagged with the listed tags (equivalent to the
   `--include` option of `mix test`).
 - `i`: Clear any included tags.
+- `m <max>`: Specify the maximum number of failures allowed (equivalent to the
+  `--max-failures` option of `mix test`).
+- `m`: Clear any previously specified maximum number of failures.
 - `o <tags...>`: Run only tests tagged with the listed tags (equivalent to the
   `--only` option of `mix test`).
 - `o`: Clear any "only" tags.

lib/mix/tasks/test/interactive.ex

@@ -5,9 +5,9 @@ defmodule Mix.Tasks.Test.Interactive do
 
   `mix test.interactive` allows you to easily switch between running all tests,
   stale tests, or failed tests. Or, you can run only the tests whose filenames
-  contain a substring. You can also control which tags are included or excluded
-  and easily specify the test seed to use. Includes an optional "watch mode"
-  which runs tests after every file change.
+  contain a substring. You can also control which tags are included or excluded,
+  modify the maximum number of failures allowed, and specify the test seed to use.
+  Includes an optional "watch mode" which runs tests after every file change.
 
   ## Usage
 
@@ -81,6 +81,9 @@ defmodule Mix.Tasks.Test.Interactive do
   - `i <tags...>`: Include tests tagged with the listed tags (equivalent to the
     `--include` option of `mix test`).
   - `i`: Clear any included tags.
+  - `m <max>`: Specify the maximum number of failures allowed (equivalent to the
+    `--max-failures` option of `mix test`).
+  - `m`: Clear any previously specified maximum number of failures.
   - `o <tags...>`: Run only tests tagged with the listed tags (equivalent to the
     `--only` option of `mix test`).
   - `o`: Clear any "only" tags.

lib/mix_test_interactive/command/max_failures.ex

@@ -0,0 +1,25 @@
+defmodule MixTestInteractive.Command.MaxFailures do
+  @moduledoc """
+  Specify or clear the maximum number of failures during a test run.
+
+  Runs the tests with the given maximum failures if provided. If not provided,
+  the max is cleared and the tests will run until completion as usual.
+  """
+  use MixTestInteractive.Command, command: "m", desc: "set or clear the maximum number of failures"
+
+  alias MixTestInteractive.Command
+  alias MixTestInteractive.Settings
+
+  @impl Command
+  def name, do: "m [<max>]"
+
+  @impl Command
+  def run([], %Settings{} = settings) do
+    {:ok, Settings.clear_max_failures(settings)}
+  end
+
+  @impl Command
+  def run([max], %Settings{} = settings) do
+    {:ok, Settings.with_max_failures(settings, max)}
+  end
+end

lib/mix_test_interactive/command_line_parser.ex

@@ -167,6 +167,7 @@ defmodule MixTestInteractive.CommandLineParser do
     {failed?, mix_test_opts} = Keyword.pop(mix_test_opts, :failed, false)
     {includes, mix_test_opts} = Keyword.pop_values(mix_test_opts, :include)
     {only, mix_test_opts} = Keyword.pop_values(mix_test_opts, :only)
+    {max_failures, mix_test_opts} = Keyword.pop(mix_test_opts, :max_failures)
     {seed, mix_test_opts} = Keyword.pop(mix_test_opts, :seed)
     {stale?, mix_test_opts} = Keyword.pop(mix_test_opts, :stale, false)
     watching? = Keyword.get(mti_opts, :watch, true)
@@ -176,6 +177,7 @@ defmodule MixTestInteractive.CommandLineParser do
       failed?: no_patterns? && failed?,
       includes: includes,
       initial_cli_args: OptionParser.to_argv(mix_test_opts),
+      max_failures: max_failures && to_string(max_failures),
       only: only,
       patterns: patterns,
       seed: seed && to_string(seed),

lib/mix_test_interactive/command_processor.ex

@@ -9,6 +9,7 @@ defmodule MixTestInteractive.CommandProcessor do
   alias MixTestInteractive.Command.Failed
   alias MixTestInteractive.Command.Help
   alias MixTestInteractive.Command.Include
+  alias MixTestInteractive.Command.MaxFailures
   alias MixTestInteractive.Command.Only
   alias MixTestInteractive.Command.Pattern
   alias MixTestInteractive.Command.Quit
@@ -26,6 +27,7 @@ defmodule MixTestInteractive.CommandProcessor do
     Failed,
     Help,
     Include,
+    MaxFailures,
     Only,
     Pattern,
     Quit,

lib/mix_test_interactive/settings.ex

@@ -19,6 +19,7 @@ defmodule MixTestInteractive.Settings do
     field :includes, [String.t()], default: []
     field :initial_cli_args, [String.t()], default: []
     field :list_all_files, (-> [String.t()]), default: @default_list_all_files
+    field :max_failures, String.t()
     field :only, [String.t()], default: []
     field :patterns, [String.t()], default: []
     field :seed, String.t()
@@ -50,6 +51,15 @@ defmodule MixTestInteractive.Settings do
     %{settings | includes: []}
   end
 
+  @doc """
+  Update settings to run with unlimited failures, clearing any specified maximum
+  number of failures.
+  """
+  @spec clear_max_failures(t()) :: t()
+  def clear_max_failures(%__MODULE__{} = settings) do
+    %{settings | max_failures: nil}
+  end
+
   @doc """
   Update settings to clear any "only" tags.
   """
@@ -140,13 +150,23 @@ defmodule MixTestInteractive.Settings do
           "Ran all tests"
       end
 
-    case_result =
+    with_seed =
       case settings.seed do
         nil -> run_summary
         seed -> run_summary <> " with seed: #{seed}"
       end
 
-    append_tag_filters(case_result, settings)
+    with_seed
+    |> append_max_failures(settings)
+    |> append_tag_filters(settings)
+  end
+
+  defp append_max_failures(summary, %__MODULE__{max_failures: nil} = _settings) do
+    summary
+  end
+
+  defp append_max_failures(summary, %__MODULE__{} = settings) do
+    summary <> "\nMax failures: #{settings.max_failures}"
   end
 
   defp append_tag_filters(summary, %__MODULE__{} = settings) do
@@ -186,6 +206,16 @@ defmodule MixTestInteractive.Settings do
     %{settings | includes: includes}
   end
 
+  @doc """
+  Stop running tests after a maximum number of failures.
+
+  Corresponds to `mix test --max-failures <max>`.
+  """
+  @spec with_max_failures(t(), String.t()) :: t()
+  def with_max_failures(%__MODULE__{} = settings, max) do
+    %{settings | max_failures: max}
+  end
+
   @doc """
   Run only the tests with the specified tags.
 
@@ -249,6 +279,9 @@ defmodule MixTestInteractive.Settings do
     Enum.flat_map(includes, &["--include", &1])
   end
 
+  defp opts_from_single_setting({:max_failures, nil}), do: []
+  defp opts_from_single_setting({:max_failures, max}), do: ["--max-failures", max]
+
   defp opts_from_single_setting({:only, only}) do
     Enum.flat_map(only, &["--only", &1])
   end

test/mix_test_interactive/command_line_parser_test.exs

@@ -244,6 +244,12 @@ defmodule MixTestInteractive.CommandLineParserTest do
       assert settings.initial_cli_args == ["--trace", "--raise"]
     end
 
+    test "extracts max-failures from arguments" do
+      {:ok, %{settings: settings}} = CommandLineParser.parse(["--trace", "--max-failures", "7", "--raise"])
+      assert settings.max_failures == "7"
+      assert settings.initial_cli_args == ["--trace", "--raise"]
+    end
+
     test "extracts only from arguments" do
       {:ok, %{settings: settings}} =
         CommandLineParser.parse(["--only", "tag1", "--trace", "--only", "tag2", "--failed", "--raise", "--only", "tag3"])

test/mix_test_interactive/command_processor_test.exs

@@ -64,6 +64,20 @@ defmodule MixTestInteractive.CommandProcessorTest do
       assert {:ok, ^expected} = process_command("i", settings)
     end
 
+    test "m <max> sets max-failures" do
+      settings = %Settings{}
+      expected = Settings.with_max_failures(settings, "4")
+
+      assert {:ok, ^expected} = process_command("m 4", settings)
+    end
+
+    test "m with no seed clears max-failures" do
+      {:ok, settings} = process_command("m 1", %Settings{})
+      expected = Settings.clear_max_failures(settings)
+
+      assert {:ok, ^expected} = process_command("m", settings)
+    end
+
     test "o <tag...> runs with only the given tags" do
       settings = %Settings{}
       expected = Settings.with_only(settings, ["tag1", "tag2"])

test/mix_test_interactive/end_to_end_test.exs

@@ -63,6 +63,32 @@ defmodule MixTestInteractive.EndToEndTest do
     assert_ran_tests(["--stale"])
   end
 
+  test "max failures workflow", %{pid: pid} do
+    assert_ran_tests()
+
+    assert :ok = InteractiveMode.process_command(pid, "m 3")
+    assert_ran_tests(["--max-failures", "3"])
+
+    assert :ok = InteractiveMode.process_command(pid, "m")
+    assert_ran_tests()
+  end
+
+  test "seed workflow", %{pid: pid} do
+    assert_ran_tests()
+
+    assert :ok = InteractiveMode.process_command(pid, "d 4242")
+    assert_ran_tests(["--seed", "4242"])
+
+    assert :ok = InteractiveMode.note_file_changed(pid)
+    assert_ran_tests(["--seed", "4242"])
+
+    assert :ok = InteractiveMode.process_command(pid, "s")
+    assert_ran_tests(["--seed", "4242", "--stale"])
+
+    assert :ok = InteractiveMode.process_command(pid, "d")
+    assert_ran_tests(["--stale"])
+  end
+
   test "tag workflow", %{pid: pid} do
     assert_ran_tests()
 
@@ -97,22 +123,6 @@ defmodule MixTestInteractive.EndToEndTest do
     assert_ran_tests()
   end
 
-  test "seed workflow", %{pid: pid} do
-    assert_ran_tests()
-
-    assert :ok = InteractiveMode.process_command(pid, "d 4242")
-    assert_ran_tests(["--seed", "4242"])
-
-    assert :ok = InteractiveMode.note_file_changed(pid)
-    assert_ran_tests(["--seed", "4242"])
-
-    assert :ok = InteractiveMode.process_command(pid, "s")
-    assert_ran_tests(["--seed", "4242", "--stale"])
-
-    assert :ok = InteractiveMode.process_command(pid, "d")
-    assert_ran_tests(["--stale"])
-  end
-
   test "watch on/off workflow", %{pid: pid} do
     assert_ran_tests()
 

test/mix_test_interactive/settings_test.exs

@@ -194,6 +194,26 @@ defmodule MixTestInteractive.SettingsTest do
     end
   end
 
+  describe "specifying maximum failures" do
+    test "stops after a specified number of failures" do
+      max = "3"
+      settings = Settings.with_max_failures(%Settings{initial_cli_args: ["--trace"]}, max)
+
+      {:ok, args} = Settings.cli_args(settings)
+      assert args == ["--trace", "--max-failures", max]
+    end
+
+    test "clears maximum failures" do
+      settings =
+        %Settings{}
+        |> Settings.with_max_failures("2")
+        |> Settings.clear_max_failures()
+
+      {:ok, args} = Settings.cli_args(settings)
+      assert args == []
+    end
+  end
+
   describe "specifying the seed" do
     test "runs with seed" do
       seed = "5678"
@@ -273,6 +293,12 @@ defmodule MixTestInteractive.SettingsTest do
       assert Settings.summary(settings) == "Ran all test files matching p1, p2 with seed: #{seed}"
     end
 
+    test "appends max failures" do
+      settings = Settings.with_max_failures(%Settings{}, "6")
+
+      assert Settings.summary(settings) =~ "Max failures: 6"
+    end
+
     test "appends tag filters" do
       settings =
         %Settings{}