-
Notifications
You must be signed in to change notification settings - Fork 308
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
What is the best way to start sidekiq processes on system boot, or restart processes that die? #212
Comments
I was using monit for this, but the solution isn't encouraged by sidekiq's author anymore, with proper reasons behind it. I'm currently moving into
This also applies for system reboots. Right now I hit the wall because the |
Hey @jsantos, since there's no solution yet, have you found any workarounds for this? I too have multiple sidekiq-*.service files that I need to restart on deploy. |
As a hacky workaround, this is what I did: # config/deploy/my-environment.rb
server 'my-server.com', user: 'deploy', roles: %w[app appkiq]
# Custom sidekiq configuration
# https://github.com/seuros/capistrano-sidekiq/blob/v1.0.3/lib/capistrano/tasks/sidekiq.rake
# https://calvin.my/posts/setup-sidekiq-with-systemd-and-capistrano-integration
# https://tagrudev.com/2019/sidekiq-systemd-and-capistrano/
set :sidekiq_service_names, %w[sidekiq_first sidekiq_second]
namespace :appkiq do
task :quiet do
on roles(:appkiq) do
fetch(:sidekiq_service_names).each do |service_name|
execute :sudo, :systemctl, :kill, "-s", "USR1", service_name
end
end
end
task :stop do
on roles(:appkiq) do
fetch(:sidekiq_service_names).each do |service_name|
execute :sudo, :systemctl, :stop, service_name
end
end
end
task :restart do
on roles(:appkiq) do
fetch(:sidekiq_service_names).each do |service_name|
execute :sudo, :systemctl, :restart, service_name
end
end
end
task :start do
on roles(:appkiq) do
fetch(:sidekiq_service_names).each do |service_name|
execute :sudo, :systemctl, :start, service_name
end
end
end
end
after 'deploy:starting', 'appkiq:quiet'
after 'deploy:updated', 'appkiq:stop'
after 'deploy:published', 'appkiq:start'
after 'deploy:failed', 'appkiq:restart' |
@aesyondu Yeah, I've also tweaked my rake task to properly handle multiple processes with systemd. Let me paste it here: # lib/capistrano/tasks/sidekiq.rake
namespace :load do
task :defaults do
set :sidekiq_default_hooks, true
set :sidekiq_pid, -> { File.join(shared_path, 'tmp', 'pids', 'sidekiq.pid') }
set :sidekiq_env, -> { fetch(:rack_env, fetch(:rails_env, fetch(:stage))) }
set :sidekiq_log, -> { File.join(shared_path, 'log', 'sidekiq.log') }
set :sidekiq_timeout, 10
set :sidekiq_roles, fetch(:sidekiq_role, :app)
set :sidekiq_processes, 1
set :sidekiq_options_per_process, nil
set :sidekiq_user, nil
# Rbenv, Chruby, and RVM integration
set :rbenv_map_bins, fetch(:rbenv_map_bins).to_a.concat(%w[sidekiq sidekiqctl])
set :rvm_map_bins, fetch(:rvm_map_bins).to_a.concat(%w[sidekiq sidekiqctl])
set :chruby_map_bins, fetch(:chruby_map_bins).to_a.concat(%w[sidekiq sidekiqctl])
# Bundler integration
set :bundle_bins, fetch(:bundle_bins).to_a.concat(%w[sidekiq sidekiqctl])
# Init system integration
set :init_system, -> { nil }
# systemd integration
set :service_unit_name, "sidekiq.service"
set :upstart_service_name, "sidekiq"
end
end
namespace :deploy do
before :starting, :check_sidekiq_hooks do
invoke 'sidekiq:add_default_hooks' if fetch(:sidekiq_default_hooks)
end
end
namespace :sidekiq do
task :add_default_hooks do
after 'deploy:starting', 'sidekiq:quiet'
after 'deploy:updated', 'sidekiq:stop'
after 'deploy:published', 'sidekiq:start'
after 'deploy:failed', 'sidekiq:restart'
end
desc 'Quiet sidekiq (stop fetching new tasks from Redis)'
task :quiet do
on roles fetch(:sidekiq_roles) do |role|
switch_user(role) do
case fetch(:init_system)
when :systemd
if test("[ -d #{release_path} ]")
each_systemd_process_with_index(reverse: true) do |service_name, _idx|
sudo :systemctl, "reload", service_name, raise_on_non_zero_exit: false
end
end
when :upstart
sudo :service, fetch(:upstart_service_name), :reload
else
if test("[ -d #{release_path} ]")
each_process_with_index(reverse: true) do |pid_file, _idx|
if pid_file_exists?(pid_file) && process_exists?(pid_file)
quiet_sidekiq(pid_file)
end
end
end
end
end
end
end
desc 'Stop sidekiq (graceful shutdown within timeout, put unfinished tasks back to Redis)'
task :stop do
on roles fetch(:sidekiq_roles) do |role|
switch_user(role) do
case fetch(:init_system)
when :systemd
if test("[ -d #{release_path} ]")
each_systemd_process_with_index(reverse: true) do |service_name, _idx|
sudo :systemctl, "stop", service_name
end
end
when :upstart
sudo :service, fetch(:upstart_service_name), :stop
else
if test("[ -d #{release_path} ]")
each_process_with_index(reverse: true) do |pid_file, _idx|
if pid_file_exists?(pid_file) && process_exists?(pid_file)
stop_sidekiq(pid_file)
end
end
end
end
end
end
end
desc 'Start sidekiq'
task :start do
on roles fetch(:sidekiq_roles) do |role|
switch_user(role) do
case fetch(:init_system)
when :systemd
if test("[ -d #{release_path} ]")
each_systemd_process_with_index(reverse: true) do |service_name, _idx|
sudo :systemctl, 'start', service_name
end
end
when :upstart
sudo :service, fetch(:upstart_service_name), :start
else
each_process_with_index do |pid_file, idx|
unless pid_file_exists?(pid_file) && process_exists?(pid_file)
start_sidekiq(pid_file, idx)
end
end
end
end
end
end
desc 'Restart sidekiq'
task :restart do
invoke! 'sidekiq:stop'
invoke! 'sidekiq:start'
end
desc 'Rolling-restart sidekiq'
task :rolling_restart do
on roles fetch(:sidekiq_roles) do |role|
switch_user(role) do
each_process_with_index(reverse: true) do |pid_file, idx|
if pid_file_exists?(pid_file) && process_exists?(pid_file)
stop_sidekiq(pid_file)
end
start_sidekiq(pid_file, idx)
end
end
end
end
desc 'Delete any pid file not in use'
task :cleanup do
on roles fetch(:sidekiq_roles) do |role|
switch_user(role) do
each_process_with_index do |pid_file, _idx|
unless process_exists?(pid_file)
next unless pid_file_exists?(pid_file)
execute "rm #{pid_file}"
end
end
end
end
end
# TODO: Don't start if all processes are off, raise warning.
desc 'Respawn missing sidekiq processes'
task :respawn do
invoke 'sidekiq:cleanup'
on roles fetch(:sidekiq_roles) do |role|
switch_user(role) do
each_process_with_index do |pid_file, idx|
start_sidekiq(pid_file, idx) unless pid_file_exists?(pid_file)
end
end
end
end
desc 'Setup configuration files for master'
task :setup_master do
on roles(:master) do
template "sidekiq_master.yml.erb", "#{shared_path}/config/sidekiq.yml"
end
end
desc 'Setup configuration files for slave'
task :setup_slave do
on roles(:slave) do
template "sidekiq_slave.yml.erb", "#{shared_path}/config/sidekiq.yml"
end
end
desc 'Setup configuration files for both master and slave'
task :setup do
invoke 'sidekiq:setup_master'
invoke 'sidekiq:setup_slave'
end
def each_process_with_index(reverse: false)
pid_file_list = pid_files
pid_file_list.reverse! if reverse
pid_file_list.each_with_index do |pid_file, idx|
within release_path do
yield(pid_file, idx)
end
end
end
def each_systemd_process_with_index(reverse: false)
process_list = processes
process_list.reverse! if reverse
process_list.each.with_index(1) do |service_name, idx|
within release_path do
yield(service_name, idx)
end
end
end
def pid_files
sidekiq_roles = Array(fetch(:sidekiq_roles)).dup
sidekiq_roles.select! { |role| host.roles.include?(role) }
sidekiq_roles.flat_map do |role|
processes = fetch(:"#{role}_processes") || fetch(:sidekiq_processes)
Array.new(processes) { |idx| fetch(:sidekiq_pid).gsub(/\.pid$/, "-#{idx}.pid") }
end
end
def processes
sidekiq_roles = Array(fetch(:sidekiq_roles)).dup
sidekiq_roles.select! { |role| host.roles.include?(role) }
sidekiq_roles.flat_map do |role|
processes = fetch(:"#{role}_processes") || fetch(:sidekiq_processes)
Array.new(processes) { |idx| fetch(:service_unit_name).gsub(/\.service$/, "-#{idx}.service") }
end
end
def pid_file_exists?(pid_file)
test(*("[ -f #{pid_file} ]").split(' '))
end
def process_exists?(pid_file)
test(*("kill -0 $( cat #{pid_file} )").split(' '))
end
def quiet_sidekiq(pid_file)
execute :sidekiqctl, 'quiet', pid_file.to_s
rescue SSHKit::Command::Failed
# If gems are not installed (first deploy) and sidekiq_default_hooks is active
warn 'sidekiqctl not found (ignore if this is the first deploy)'
end
def stop_sidekiq(pid_file)
execute :sidekiqctl, 'stop', pid_file.to_s, fetch(:sidekiq_timeout)
end
def start_sidekiq(pid_file, idx = 0)
args = []
args.push "--index #{idx}"
args.push "--pidfile #{pid_file}"
args.push "--environment #{fetch(:sidekiq_env)}"
args.push "--logfile #{fetch(:sidekiq_log)}" if fetch(:sidekiq_log)
args.push "--require #{fetch(:sidekiq_require)}" if fetch(:sidekiq_require)
args.push "--tag #{fetch(:sidekiq_tag)}" if fetch(:sidekiq_tag)
Array(fetch(:sidekiq_queue)).each do |queue|
args.push "--queue #{queue}"
end
args.push "--config #{fetch(:sidekiq_config)}" if fetch(:sidekiq_config)
args.push "--concurrency #{fetch(:sidekiq_concurrency)}" if fetch(:sidekiq_concurrency)
if (process_options = fetch(:sidekiq_options_per_process))
args.push process_options[idx]
end
# use sidekiq_options for special options
args.push fetch(:sidekiq_options) if fetch(:sidekiq_options)
if defined?(JRUBY_VERSION)
args.push '>/dev/null 2>&1 &'
warn 'Since JRuby doesn\'t support Process.daemon, Sidekiq will not be running as a daemon.'
else
args.push '--daemon'
end
execute :sidekiq, args.compact.join(' ')
end
def switch_user(role)
su_user = sidekiq_user(role)
if su_user == role.user
yield
else
as su_user do
yield
end
end
end
def sidekiq_user(role)
properties = role.properties
properties.fetch(:sidekiq_user) || # local property for sidekiq only
fetch(:sidekiq_user) ||
properties.fetch(:run_as) || # global property across multiple capistrano gems
role.user
end
end Hope this helps! Maybe I should create a PR with this, it's been a while since I touched this library. |
I've been using capistrano-sidekiq in production for a while, and its configurability for multiple processes is great. I use 4 different processes, and 4 queues, with a
config/deploy.rb
that looks like this:The one problem I've run into is that, if the server gets restarted, there won't be any sidekiq processes running until the next deployment. There are many different options for starting sidekiq processes on system boot, but I haven't found a way to do in a way that will use the same configuration (without having to separately maintain the config for capistrano-deployed sidekiq processes and on-boot processes).
I also understand that the daemon approach used by capistrano-sidekiq means that if a sidekiq process dies, it won't be restarted until the next deploy.
Is there a way to, for example, set up a cron job will use the same config to start the sidekiq processes locally or restart any processes that are not running?
The text was updated successfully, but these errors were encountered: