diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 97dcad51..c4caf436 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,6 +8,9 @@ on: jobs: Benchmark: runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write if: contains(github.event.pull_request.labels.*.name, 'run benchmark') steps: - uses: actions/checkout@v2 diff --git a/libs/HMMBenchmark/src/hiddenmarkovmodels.jl b/libs/HMMBenchmark/src/hiddenmarkovmodels.jl index 63ca122b..88d6dfa8 100644 --- a/libs/HMMBenchmark/src/hiddenmarkovmodels.jl +++ b/libs/HMMBenchmark/src/hiddenmarkovmodels.jl @@ -43,12 +43,12 @@ function build_benchmarkables( if "forward" in algos benchs["forward"] = @benchmarkable begin forward($hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "forward!" in algos benchs["forward!"] = @benchmarkable begin forward!(f_storage, $hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) - end evals = 1 samples = 10 setup = ( + end evals = 1 samples = 100 setup = ( f_storage = initialize_forward($hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) ) end @@ -56,12 +56,12 @@ function build_benchmarkables( if "viterbi" in algos benchs["viterbi"] = @benchmarkable begin viterbi($hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "viterbi!" in algos benchs["viterbi!"] = @benchmarkable begin viterbi!(v_storage, $hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) - end evals = 1 samples = 10 setup = ( + end evals = 1 samples = 100 setup = ( v_storage = initialize_viterbi($hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) ) end @@ -69,12 +69,12 @@ function build_benchmarkables( if "forward_backward" in algos benchs["forward_backward"] = @benchmarkable begin forward_backward($hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "forward_backward!" in algos benchs["forward_backward!"] = @benchmarkable begin forward_backward!(fb_storage, $hmm, $obs_seq, $control_seq; seq_ends=$seq_ends) - end evals = 1 samples = 10 setup = ( + end evals = 1 samples = 100 setup = ( fb_storage = initialize_forward_backward( $hmm, $obs_seq, $control_seq; seq_ends=$seq_ends ) @@ -92,7 +92,7 @@ function build_benchmarkables( atol=-Inf, loglikelihood_increasing=false, ) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "baum_welch!" in algos benchs["baum_welch!"] = @benchmarkable begin @@ -107,7 +107,7 @@ function build_benchmarkables( atol=-Inf, loglikelihood_increasing=false, ) - end evals = 1 samples = 10 setup = ( + end evals = 1 samples = 100 setup = ( hmm_guess = build_model($implem, $instance, $params); fb_storage = initialize_forward_backward( hmm_guess, $obs_seq, $control_seq; seq_ends=$seq_ends diff --git a/libs/HMMComparison/src/dynamax.jl b/libs/HMMComparison/src/dynamax.jl index d60e8ef3..4a845911 100644 --- a/libs/HMMComparison/src/dynamax.jl +++ b/libs/HMMComparison/src/dynamax.jl @@ -48,7 +48,7 @@ function HMMBenchmark.build_benchmarkables( filter_vmap = jax.jit(jax.vmap(hmm.filter; in_axes=pylist((pybuiltins.None, 0)))) benchs["forward"] = @benchmarkable begin $(filter_vmap)($dyn_params, $obs_tens_jax_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "viterbi" in algos @@ -57,7 +57,7 @@ function HMMBenchmark.build_benchmarkables( ) benchs["viterbi"] = @benchmarkable begin $(most_likely_states_vmap)($dyn_params, $obs_tens_jax_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "forward_backward" in algos @@ -66,7 +66,7 @@ function HMMBenchmark.build_benchmarkables( ) benchs["forward_backward"] = @benchmarkable begin $(smoother_vmap)($dyn_params, $obs_tens_jax_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "baum_welch" in algos @@ -78,7 +78,7 @@ function HMMBenchmark.build_benchmarkables( num_iters=$bw_iter, verbose=false, ) - end evals = 1 samples = 10 setup = ( + end evals = 1 samples = 100 setup = ( tup = build_model($implem, $instance, $params); hmm_guess = tup[1]; dyn_params_guess = tup[2]; diff --git a/libs/HMMComparison/src/hmmbase.jl b/libs/HMMComparison/src/hmmbase.jl index 2c260e89..4608071a 100644 --- a/libs/HMMComparison/src/hmmbase.jl +++ b/libs/HMMComparison/src/hmmbase.jl @@ -41,7 +41,7 @@ function HMMBenchmark.build_benchmarkables( @threads for k in eachindex($obs_mats) HMMBase.forward($hmm, $obs_mats[k]) end - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "viterbi" in algos @@ -49,7 +49,7 @@ function HMMBenchmark.build_benchmarkables( @threads for k in eachindex($obs_mats) HMMBase.viterbi($hmm, $obs_mats[k]) end - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "forward_backward" in algos @@ -57,13 +57,13 @@ function HMMBenchmark.build_benchmarkables( @threads for k in eachindex($obs_mats) HMMBase.posteriors($hmm, $obs_mats[k]) end - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "baum_welch" in algos benchs["baum_welch"] = @benchmarkable begin HMMBase.fit_mle($hmm, $obs_mat_concat; maxiter=$bw_iter, tol=-Inf) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end return benchs diff --git a/libs/HMMComparison/src/hmmlearn.jl b/libs/HMMComparison/src/hmmlearn.jl index 58b1f053..610c9129 100644 --- a/libs/HMMComparison/src/hmmlearn.jl +++ b/libs/HMMComparison/src/hmmlearn.jl @@ -45,25 +45,25 @@ function HMMBenchmark.build_benchmarkables( if "forward" in algos benchs["forward"] = @benchmarkable begin $(hmm.score)($obs_mat_concat_py, $obs_mat_len_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "viterbi" in algos benchs["viterbi"] = @benchmarkable begin $(hmm.decode)($obs_mat_concat_py, $obs_mat_len_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "forward_backward" in algos benchs["forward_backward"] = @benchmarkable begin $(hmm.predict_proba)($obs_mat_concat_py, $obs_mat_len_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "baum_welch" in algos benchs["baum_welch"] = @benchmarkable begin hmm_guess.fit($obs_mat_concat_py, $obs_mat_len_py) - end evals = 1 samples = 10 setup = ( + end evals = 1 samples = 100 setup = ( hmm_guess = build_model($implem, $instance, $params) ) end diff --git a/libs/HMMComparison/src/pomegranate.jl b/libs/HMMComparison/src/pomegranate.jl index adf27f53..d5939919 100644 --- a/libs/HMMComparison/src/pomegranate.jl +++ b/libs/HMMComparison/src/pomegranate.jl @@ -57,19 +57,19 @@ function HMMBenchmark.build_benchmarkables( if "forward" in algos benchs["forward"] = @benchmarkable begin $(hmm.forward)($obs_tens_torch_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "forward_backward" in algos benchs["forward_backward"] = @benchmarkable begin $(hmm.forward_backward)($obs_tens_torch_py) - end evals = 1 samples = 10 + end evals = 1 samples = 100 end if "baum_welch" in algos benchs["baum_welch"] = @benchmarkable begin hmm_guess.fit($obs_tens_torch_py) - end evals = 1 samples = 10 setup = ( + end evals = 1 samples = 100 setup = ( hmm_guess = build_model($implem, $instance, $params) ) end