From 08f5725a5aa3f5169305baafb88db4a592df5086 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 12 Oct 2023 10:25:16 +0200 Subject: [PATCH] Run solid_queue with SQLite Update the tests to run solid_queue on SQLite. They can be triggered by: ``` TARGET_DB=sqlite rails test ``` The adapter is prone to raising `SQLite3::BusyException`s when concurrent transactions occur. Preventing this requires a couple of patches to the adapter. 1. Implement retry with backoff - add a `retries` config setting to the adapter and sleep progressively longer for each retry. This setting is currently in the Rails main branch, but that implementation has no backoff. That doesn't work well in our case. 1. Always create immediate transactions - SQLite by default creates deferred transactions, which don't take a write lock. Then later if there is a write it tries to upgrade the lock. This won't work if the transaction has a stale read so retrying the write by itself is not possible. Starting with an immediate transaction moves the write lock to that point and ensures that we only get blocked on a retryable transaction. --- .github/workflows/main.yml | 2 +- test/dummy/config/database.yml | 18 ++++++++++++ test/dummy/config/initializers/sqlite3.rb | 30 ++++++++++++++++++++ test/dummy/config/solid_queue.yml | 2 +- test/integration/jobs_lifecycle_test.rb | 2 +- test/integration/processes_lifecycle_test.rb | 2 +- test/test_helper.rb | 10 +++++++ test/unit/supervisor_test.rb | 1 + 8 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 test/dummy/config/initializers/sqlite3.rb diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29819e64..1c798f83 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: ruby-version: [3.2.2] - database: [mysql] + database: [mysql, sqlite] services: mysql: image: mysql:8.0.31 diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index 8af24d93..418c780c 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -5,6 +5,22 @@ # gem "mysql2" # +<% if ENV["TARGET_DB"] == "sqlite" %> +default: &default + adapter: sqlite3 + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 50 } %> + retries: 100 + +development: + <<: *default + database: db/development.sqlite3 + +test: + <<: *default + pool: 20 + database: db/test.sqlite3 + +<% else %> default: &default adapter: mysql2 username: root @@ -20,3 +36,5 @@ test: <<: *default pool: 20 database: solid_queue_test +<% end %> + diff --git a/test/dummy/config/initializers/sqlite3.rb b/test/dummy/config/initializers/sqlite3.rb new file mode 100644 index 00000000..80d0fec0 --- /dev/null +++ b/test/dummy/config/initializers/sqlite3.rb @@ -0,0 +1,30 @@ +module SqliteImmediateTransactions + def begin_db_transaction + log("begin immediate transaction", "TRANSACTION") do + with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn| + conn.transaction(:immediate) + end + end + end +end + +module SQLite3Configuration + private + def configure_connection + super + + if @config[:retries] + retries = self.class.type_cast_config_to_integer(@config[:retries]) + raw_connection.busy_handler do |count| + (count <= retries).tap { |result| sleep count * 0.001 if result } + end + end + end +end + +ActiveSupport.on_load :active_record do + if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter) + ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend SqliteImmediateTransactions + ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend SQLite3Configuration + end +end diff --git a/test/dummy/config/solid_queue.yml b/test/dummy/config/solid_queue.yml index df780efe..9ce3cdad 100644 --- a/test/dummy/config/solid_queue.yml +++ b/test/dummy/config/solid_queue.yml @@ -5,7 +5,7 @@ default: &default default: pool_size: 5 scheduler: - polling_interval: 300 + polling_interval: 1 batch_size: 500 development: diff --git a/test/integration/jobs_lifecycle_test.rb b/test/integration/jobs_lifecycle_test.rb index 620e3e29..fa5eefdd 100644 --- a/test/integration/jobs_lifecycle_test.rb +++ b/test/integration/jobs_lifecycle_test.rb @@ -44,7 +44,7 @@ class JobsLifecycleTest < ActiveSupport::TestCase travel_to 5.days.from_now - wait_for_jobs_to_finish_for(0.5.seconds) + wait_for_jobs_to_finish_for(5.seconds) assert_equal 2, JobBuffer.size assert_equal "I'm scheduled later", JobBuffer.last_value diff --git a/test/integration/processes_lifecycle_test.rb b/test/integration/processes_lifecycle_test.rb index 17e47f7e..22bda248 100644 --- a/test/integration/processes_lifecycle_test.rb +++ b/test/integration/processes_lifecycle_test.rb @@ -113,7 +113,7 @@ class ProcessLifecycleTest < ActiveSupport::TestCase test "term supervisor exceeding timeout while there are jobs in-flight" do no_pause = enqueue_store_result_job("no pause") - pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 0.1.second) + pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 1.second) signal_process(@pid, :TERM, wait: 0.1.second) wait_for_jobs_to_finish_for(SolidQueue.shutdown_timeout + 0.1.second) diff --git a/test/test_helper.rb b/test/test_helper.rb index eb7974f6..7d0af8f0 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -15,6 +15,16 @@ ActiveSupport::TestCase.fixtures :all end +module BlockLogDeviceTimeoutExceptions + def write(...) + # Prevents Timeout exceptions from occurring during log writing, where they will be swallowed + # See https://bugs.ruby-lang.org/issues/9115 + Thread.handle_interrupt(Timeout::Error => :never, Timeout::ExitException => :never) { super } + end +end + +Logger::LogDevice.prepend(BlockLogDeviceTimeoutExceptions) + class ActiveSupport::TestCase setup do SolidQueue.logger = ActiveSupport::Logger.new(nil) diff --git a/test/unit/supervisor_test.rb b/test/unit/supervisor_test.rb index 6fb69e26..ddf74049 100644 --- a/test/unit/supervisor_test.rb +++ b/test/unit/supervisor_test.rb @@ -47,6 +47,7 @@ class SupervisorTest < ActiveSupport::TestCase end test "abort if there's already a pidfile for a supervisor" do + FileUtils.mkdir_p(File.dirname(@pidfile)) File.write(@pidfile, ::Process.pid.to_s) pid = run_supervisor_as_fork(mode: :all)