Merge branch 'main' into remove-rails-dependency

Author: rosa

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    solid_queue (0.2.1)
+    solid_queue (0.2.2)
       activejob (>= 7.1)
       activerecord (>= 7.1)
       railties (>= 7.1)

README.md

@@ -66,6 +66,8 @@ $ bundle exec rake solid_queue:start
 
 This will start processing jobs in all queues using the default configuration. See [below](#configuration) to learn more about configuring Solid Queue.
 
+For small projects, you can run Solid Queue on the same machine as your webserver. When you're ready to scale, Solid Queue supports horizontal scaling out-of-the-box. You can run Solid Queue on a separate server from your webserver, or even run `bundle exec rake solid_queue:start` on multiple machines at the same time. If you'd like to designate some machines to be only dispatchers or only workers, use `bundle exec rake solid_queue:dispatch` or `bundle exec rake solid_queue:work`, respectively.
+
 ## Requirements
 Besides Rails 7.1, Solid Queue works best with MySQL 8+ or PostgreSQL 9.5+, as they support `FOR UPDATE SKIP LOCKED`. You can use it with older versions, but in that case, you might run into lock waits if you run multiple workers for the same queue.
 
@@ -115,7 +117,7 @@ Everything is optional. If no configuration is provided, Solid Queue will run wi
   This will create a worker fetching jobs from all queues starting with `staging`. The wildcard `*` is only allowed on its own or at the end of a queue name; you can't specify queue names such as `*_some_queue`. These will be ignored.
 
   Finally, you can combine prefixes with exact names, like `[ staging*, background ]`, and the behaviour with respect to order will be the same as with only exact names.
-- `threads`: this is the max size of the thread pool that each worker will have to run jobs. Each worker will fetch this number of jobs from their queue(s), at most and will post them to the thread pool to be run. By default, this is `5`. Only workers have this setting.
+- `threads`: this is the max size of the thread pool that each worker will have to run jobs. Each worker will fetch this number of jobs from their queue(s), at most and will post them to the thread pool to be run. By default, this is `3`. Only workers have this setting.
 - `processes`: this is the number of worker processes that will be forked by the supervisor with the settings given. By default, this is `1`, just a single process. This setting is useful if you want to dedicate more than one CPU core to a queue or queues with the same configuration. Only workers have this setting.
 
 
@@ -131,7 +133,7 @@ We recommend not mixing queue order with priorities but either choosing one or t
 
 ### Threads, processes and signals
 
-Workers in Solid Queue use a thread pool to run work in multiple threads, configurable via the `threads` parameter above. Besides this, parallelism can be achieved via multiple processes, configurable via different workers or the `processes` parameter above.
+Workers in Solid Queue use a thread pool to run work in multiple threads, configurable via the `threads` parameter above. Besides this, parallelism can be achieved via multiple processes on one machine (configurable via different workers or the `processes` parameter above) or by horizontal scaling.
 
 The supervisor is in charge of managing these processes, and it responds to the following signals:
 - `TERM`, `INT`: starts graceful termination. The supervisor will send a `TERM` signal to its supervised processes, and it'll wait up to `SolidQueue.shutdown_timeout` time until they're done. If any supervised processes are still around by then, it'll send a `QUIT` signal to them to indicate they must exit.

app/models/solid_queue/blocked_execution.rb

@@ -30,10 +30,10 @@ def release_one(concurrency_key)
 
       private
         def releasable(concurrency_keys)
-          semaphores = Semaphore.where(key: concurrency_keys).select(:key, :value).index_by(&:key)
+          semaphores = Semaphore.where(key: concurrency_keys).pluck(:key, :value).to_h
 
           # Concurrency keys without semaphore + concurrency keys with open semaphore
-          (concurrency_keys - semaphores.keys) | semaphores.select { |key, semaphore| semaphore.value > 0 }.map(&:first)
+          (concurrency_keys - semaphores.keys) | semaphores.select { |_key, value| value > 0 }.keys
         end
     end
 

app/models/solid_queue/execution/dispatching.rb

@@ -10,7 +10,7 @@ def dispatch_jobs(job_ids)
           jobs = Job.where(id: job_ids)
 
           Job.dispatch_all(jobs).map(&:id).tap do |dispatched_job_ids|
-            where(job_id: dispatched_job_ids).delete_all
+            where(job_id: dispatched_job_ids).order(:job_id).delete_all
             SolidQueue.logger.info("[SolidQueue] Dispatched #{dispatched_job_ids.size} jobs")
           end
         end

app/models/solid_queue/job/clearable.rb

@@ -6,13 +6,13 @@ module Clearable
       extend ActiveSupport::Concern
 
       included do
-        scope :clearable, ->(finished_before: SolidQueue.clear_finished_jobs_after.ago) { where.not(finished_at: nil).where(finished_at: ...finished_before) }
+        scope :clearable, ->(finished_before: SolidQueue.clear_finished_jobs_after.ago, class_name: nil) { where.not(finished_at: nil).where(finished_at: ...finished_before).where(class_name.present? ? { class_name: class_name } : {}) }
       end
 
       class_methods do
-        def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago)
+        def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago, class_name: nil)
           loop do
-            records_deleted = clearable(finished_before: finished_before).limit(batch_size).delete_all
+            records_deleted = clearable(finished_before: finished_before, class_name: class_name).limit(batch_size).delete_all
             break if records_deleted == 0
           end
         end

app/models/solid_queue/scheduled_execution.rb

@@ -5,7 +5,7 @@ class ScheduledExecution < Execution
     include Dispatching
 
     scope :due, -> { where(scheduled_at: ..Time.current) }
-    scope :ordered, -> { order(scheduled_at: :asc, priority: :asc) }
+    scope :ordered, -> { order(scheduled_at: :asc, priority: :asc, job_id: :asc) }
     scope :next_batch, ->(batch_size) { due.ordered.limit(batch_size) }
 
     assumes_attributes_from_job :scheduled_at

lib/solid_queue/configuration.rb

@@ -4,7 +4,7 @@ module SolidQueue
   class Configuration
     WORKER_DEFAULTS = {
       queues: "*",
-      threads: 5,
+      threads: 3,
       processes: 1,
       polling_interval: 0.1
     }

lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

test/integration/jobs_lifecycle_test.rb

@@ -133,6 +133,21 @@ class JobsLifecycleTest < ActiveSupport::TestCase
       SolidQueue::Job.clear_finished_in_batches
     end
   end
+  
+  test "respect class name when clearing finished jobs" do
+    10.times { AddToBufferJob.perform_later(2) }
+    10.times { RaisingJob.perform_later(RuntimeError, "A") }
+    jobs = SolidQueue::Job.all
+
+    jobs.each(&:finished!)
+
+    travel_to 3.days.from_now
+
+    SolidQueue::Job.clear_finished_in_batches(class_name: "AddToBufferJob")
+
+    assert_equal 0, SolidQueue::Job.where(class_name: "AddToBufferJob").count
+    assert_equal 10, SolidQueue::Job.where(class_name: "RaisingJob").count
+  end
 
   private
     def deleting_finished_jobs

test/unit/dispatcher_test.rb

@@ -4,6 +4,8 @@
 class DispatcherTest < ActiveSupport::TestCase
   include ActiveSupport::Testing::MethodCallAssertions
 
+  self.use_transactional_tests = false
+
   setup do
     @dispatcher = SolidQueue::Dispatcher.new(polling_interval: 0.1, batch_size: 10)
   end
@@ -56,6 +58,7 @@ class DispatcherTest < ActiveSupport::TestCase
     assert_equal 15, SolidQueue::ScheduledExecution.count
 
     another_dispatcher = SolidQueue::Dispatcher.new(polling_interval: 0.1, batch_size: 10)
+
     @dispatcher.start
     another_dispatcher.start