Migrate to Solid Queue 0.8.x and extract tables

With Solid Queue 0.8.0, support for dedicated  databases were introduced. In fact, Rails recommends moving the database, which we perform with the migration.

Unfortunately, it is currently unclear how exactly schema migrations will be provided. Therefore, we cannot use the same GitHub actions workflow for now and will re-add it once a new migration has been published.

Author: MrSerth

.github/workflows/ci.yml

@@ -66,19 +66,10 @@ jobs:
           cp config/database.yml.ci config/database.yml
           cp config/mnemosyne.yml.ci config/mnemosyne.yml
 
-      - name: Create database
+      - name: Prepare database
         env:
           RAILS_ENV: test
-        run: bundler exec rake db:schema:load
-      - name: Temporarily create pending SolidQueue migrations
-        # Dependabot might update the SolidQueue gem, which might include new migrations.
-        # However, Dependabot won't add the migrations nor run SolidQueue at all.
-        # Consequently, all tests would still pass and the dependency update would be merged.
-        # To prevent this, we temporarily create the pending migrations *after* the schema has been loaded.
-        # If a new migration was added, a `PendingMigrationError` will be raised and the CI will fail.
-        env:
-          RAILS_ENV: test
-        run: bundler exec rails solid_queue:install:migrations
+        run: bundler exec rake db:prepare
       - name: Precompile assets
         env:
           RAILS_ENV: test

.rubocop.yml

@@ -23,7 +23,7 @@ AllCops:
   NewCops: enable
   Exclude:
     - 'bin/*'
-    - 'db/schema.rb'
+    - 'db/*schema.rb'
     - 'vendor/**/*'
     # Ignore local files for faster processing
     - 'tmp/**/*'

Gemfile.lock

@@ -528,7 +528,7 @@ GEM
       rexml (~> 3.2)
       rubocop (>= 1.0, < 2.0)
       slim (>= 3.0, < 6.0)
-    solid_queue (0.7.1)
+    solid_queue (0.8.2)
       activejob (>= 7.1)
       activerecord (>= 7.1)
       concurrent-ruby (>= 1.3.1)

config/database.yml.example

@@ -29,8 +29,13 @@ default: &default
   <% end %>
 
 development:
-  <<: *default
-  database: codeharbor_development
+  primary: &primary_development
+    <<: *default
+    database: codeharbor_development
+  queue:
+    <<: *primary_development
+    database: codeharbor_development_queue
+    migrations_paths: db/queue_migrate
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -56,5 +61,10 @@ test:
 # for a full overview on how database connection configuration can be specified.
 #
 production:
-  <<: *default
-  database: codeharbor_production
+  primary: &primary_production
+    <<: *default
+    database: codeharbor_production
+  queue:
+    <<: *primary_production
+    database: codeharbor_production_queue
+    migrations_paths: db/queue_migrate

config/environments/development.rb

@@ -53,6 +53,7 @@
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
   config.active_job.queue_adapter = :solid_queue
+  config.solid_queue.connects_to = {database: {writing: :queue}}
   config.active_job.queue_name_prefix = 'codeharbor_development'
 
   # Disable caching for Action Mailer templates even if Action Controller

config/environments/production.rb

@@ -86,6 +86,7 @@
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
   config.active_job.queue_adapter = :solid_queue
+  config.solid_queue.connects_to = {database: {writing: :queue}}
   config.active_job.queue_name_prefix = 'codeharbor_production'
 
   # Disable caching for Action Mailer templates even if Action Controller

config/solid_queue.yml

@@ -1,20 +1,20 @@
- default: &default
-   dispatchers:
-     - polling_interval: 1
-       batch_size: 500
-   workers:
-     - queues: "*"
-       threads: 3
-       processes: 1
-       polling_interval: 0.1
+default: &default
+  dispatchers:
+    - polling_interval: 1
+      batch_size: 500
+  workers:
+    - queues: "*"
+      threads: 3
+      processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %>
+      polling_interval: 0.1
 
- development:
+development:
   <<: *default
 
- test:
+test:
   <<: *default
 
- production:
+production:
   <<: *default
   dispatchers:
     - polling_interval: 1

db/migrate/20240907100647_move_solid_queue_to_dedicated_database.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative '../scripts/copy_data'
+
+class MoveSolidQueueToDedicatedDatabase < ActiveRecord::Migration[7.1]
+  include CopyData
+  include ActiveRecord::Tasks
+
+  class Queue < ApplicationRecord
+    self.abstract_class = true
+    connects_to database: {writing: :queue}
+  end
+
+  def up
+    create_database(:queue)
+    load_queue_schema(database: :queue)
+    copy_data(source_connection: connection, target_connection: Queue.connection, operation: :matches, condition: 'solid_queue_%')
+    drop_queue_tables(connection:)
+  ensure
+    Queue.connection&.disconnect!
+  end
+
+  def down
+    load_queue_schema(database: :primary)
+    copy_data(source_connection: Queue.connection, target_connection: connection, operation: :matches, condition: 'solid_queue_%')
+    drop_queue_tables(connection: Queue.connection)
+  rescue StandardError
+    Queue.connection&.disconnect!
+  else
+    Queue.connection&.disconnect!
+    drop_database(:queue)
+  end
+
+  def foreign_key_targets
+    %w[
+      solid_queue_jobs
+    ]
+  end
+
+  private
+
+  def load_queue_schema(database:)
+    with_connection(database:) do
+      DatabaseTasks.load_schema(configs_for(:queue), ActiveRecord.schema_format, 'db/queue_schema.rb')
+    end
+  end
+
+  def drop_queue_tables(connection:)
+    connection.drop_table :solid_queue_semaphores
+    connection.drop_table :solid_queue_scheduled_executions
+    connection.drop_table :solid_queue_recurring_tasks
+    connection.drop_table :solid_queue_recurring_executions
+    connection.drop_table :solid_queue_ready_executions
+    connection.drop_table :solid_queue_processes
+    connection.drop_table :solid_queue_pauses
+    connection.drop_table :solid_queue_failed_executions
+    connection.drop_table :solid_queue_claimed_executions
+    connection.drop_table :solid_queue_blocked_executions
+    connection.drop_table :solid_queue_jobs
+  end
+
+  def configs_for(name)
+    ActiveRecord::Base.configurations.configs_for(name: name.to_s, env_name: Rails.env)
+  end
+
+  def create_database(name)
+    database_name = configs_for(name).database
+    new_primary_connection.create_database(database_name)
+  rescue ActiveRecord::DatabaseAlreadyExists
+    # Database already exists, do nothing
+  end
+
+  def drop_database(name)
+    database_name = configs_for(name).database
+    new_primary_connection.drop_database(database_name)
+  end
+
+  def new_primary_connection
+    ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.new(configs_for(:primary).configuration_hash)
+  end
+
+  def with_connection(database:)
+    # We must not overwrite the `connection`, which is automatically overwritten by establishing a new connection.
+    # However, we need to specify another connection, i.e. for loading the schema to the desired database.
+    # Hence, we use this monkey patching workaround to change the connection temporary and then revert back.
+    klass = database == :queue ? Queue : ApplicationRecord
+    DatabaseTasks.alias_method :previous_migration_class, :migration_class
+    DatabaseTasks.define_method(:migration_class) { klass }
+    yield
+  ensure
+    DatabaseTasks.alias_method :migration_class, :previous_migration_class
+  end
+end