![Гетерогенность
• return unless RUBY_PLATFORM =~ /mswin/
• Dir[File.join(work_catalog,"page*.pdf")]](https://image.slidesharecdn.com/batchprocessinginrails-110430042713-phpapp02-160315141846/85/Batch-processing-on-RoR-12-320.jpg)
![Аудит
acts_as_audited :except => [:name, :file_name]
https://gist.github.com/949049#file_book.rb
desc "Loads files into Production line"
task :process_batch => :environment do
Audit.as_user("rake:process_batch") do
Book.getFromBatch unless running?("process_batch")
end
end
https://gist.github.com/949049#file_production.rake
https://gist.github.com/949049#file_books_controller.rb
def restart
if @book.restartable?
Audit.as_user current_user do
@book.restart_production!
end
flash[:notice] = 'Обработка перезапущена.'
else
flash[:error] = 'Перезапуск невозможен.'
end
redirect_to request.referer
end
https://github.com/collectiveidea/acts_as_audited](https://image.slidesharecdn.com/batchprocessinginrails-110430042713-phpapp02-160315141846/85/Batch-processing-on-RoR-13-320.jpg)
![namespace :munin do
task :config do
puts <<-CONFIG
graph_title Produced pages in last 5 minutes
graph_args -l 0
graph_vlabel pages amount
graph_category App
graph_info This graph shows amount of pages produced
texts.label Text files
docs.label Doc files
views.label View files
quotes.label Quote files
CONFIG
exit 0
end
task :run => :environment do
res = ActiveRecord::Base.connection.execute("select count(nullif(text_ready_time>(now()-'5 minutes'::interval),false)) as
c0,count(nullif(doc_ready_time>(now()-'5 minutes'::interval),false)) as c1,count(nullif(view_ready_time>(now()-'5 minutes'::interval),false)) as
c2,count(nullif(quote_ready_time>(now()-'5 minutes'::interval),false)) as c3 from pages where updated_at > (now()-'5 minutes'::interval);")
if res
puts "texts.value #{res[0]['c0']}"
puts "docs.value #{res[0]['c1']}"
puts "views.value #{res[0]['c2']}"
puts "quotes.value #{res[0]['c3']}"
end
exit 0
end
end
https://gist.github.com/949049#file_munin.rake](https://image.slidesharecdn.com/batchprocessinginrails-110430042713-phpapp02-160315141846/85/Batch-processing-on-RoR-15-320.jpg)
![https://gist.github.com/949049#file_nagios.rake
namespace :nagios do
task :abbyy_activity, [:warn, :crit] => :environment do |t, args|
if args.warn and args.crit
states_list = ["loaded", "in_abbyy", "p2f_queue"]
warn_level = args.warn.to_i
crit_level = args.crit.to_i
warn_events = Audit.find(:all, :conditions=>["auditable_type = 'Book' and (username is null or username ilike 'rake:%') and action = 'update' and
created_at > ?", warn_level.minutes.ago])
crit_events = Audit.find(:all, :conditions=>["auditable_type = 'Book' and (username is null or username ilike 'rake:%') and action = 'update' and
created_at > ?", crit_level.minutes.ago])
if (crit_count = crit_events.select{|a| a['changes']['state'] and (a['changes']['state'] & states_list).size > 1 }).empty?
puts "CRITICAL No state changes at abbyy for #{crit_level} minutes"
exit 2
end
if (warn_count = warn_events.select{|a| a['changes']['state'] and (a['changes']['state'] & states_list).size > 1 }).empty?
puts "WARNING No state changes at abbyy for #{warn_level} minutes"
exit 1
end
puts "OK Abbyy: state changes #{warn_count.size} for #{warn_level} minutes; state changes #{crit_count.size} for #{crit_level} minutes."
exit 0
else
exit 3
end
end
http://www.nagios.org/](https://image.slidesharecdn.com/batchprocessinginrails-110430042713-phpapp02-160315141846/85/Batch-processing-on-RoR-16-320.jpg)
Batch processing on RoR
- 2. О себе • http://moiseev--sergey.moikrug.ru/ • пишу на Ruby с 2007 года.
- 3. Задача • Генерация контента (книг) для сайта. • Входной формат: PDF. • Выходной формат: SWF. • Генератор: print2flash. • Основной приоритет: скорость обработки (количество публикуемых книг в месяц).
- 4. Нюансы • Книги поступают разного качества, часть из них требует многократной переработки. • Print2flash может печатать не более одного задания за раз в пределах машины. • Требует windows окружения.
- 5. Архитектура • Множественные инсталляции Rails приложения. • Общая база. • Конечный автомат для контроля состояния задания. • Отдельные операции - rake таски запускаемые по крону или планировщику задач.
- 6. Конфигурация
- 9. include AASM aasm_column :state aasm_initial_state :loaded aasm_state :loaded, :enter => :reset_retry_count, :exit=>:apply_total_pages aasm_state :in_abbyy, :enter => :copy_to_abbyy aasm_state :p2f_queue, :enter=>:apply_pages_count aasm_state :in_p2fline, :enter => :copy_to_p2fline aasm_state :p2f_printing, :enter => :p2f_print aasm_state :print_failed, :enter => :free_p2fline aasm_state :wait_for_confirm, :enter => :free_p2fline aasm_state :confirmed aasm_state :confirmed_non_full aasm_state :on_server aasm_state :on_server_non_full aasm_state :published aasm_state :published_non_full aasm_state :hold aasm_state :mycop aasm_state :restarting, :after_enter => :check_avail_pdf aasm_state :restarting_full, :exit=>:restart_full Состояния конечного автомата https://github.com/rubyist/aasm
- 11. Почему не очередь • Так казалось проще. • Для каждого задания можно одназначно сказать его статус. • Единый каталог со всем готовым на текущий момент контентом.
- 12. Гетерогенность • return unless RUBY_PLATFORM =~ /mswin/ • Dir[File.join(work_catalog,"page*.pdf")]
- 13. Аудит acts_as_audited :except => [:name, :file_name] https://gist.github.com/949049#file_book.rb desc "Loads files into Production line" task :process_batch => :environment do Audit.as_user("rake:process_batch") do Book.getFromBatch unless running?("process_batch") end end https://gist.github.com/949049#file_production.rake https://gist.github.com/949049#file_books_controller.rb def restart if @book.restartable? Audit.as_user current_user do @book.restart_production! end flash[:notice] = 'Обработка перезапущена.' else flash[:error] = 'Перезапуск невозможен.' end redirect_to request.referer end https://github.com/collectiveidea/acts_as_audited
- 15. namespace :munin do task :config do puts <<-CONFIG graph_title Produced pages in last 5 minutes graph_args -l 0 graph_vlabel pages amount graph_category App graph_info This graph shows amount of pages produced texts.label Text files docs.label Doc files views.label View files quotes.label Quote files CONFIG exit 0 end task :run => :environment do res = ActiveRecord::Base.connection.execute("select count(nullif(text_ready_time>(now()-'5 minutes'::interval),false)) as c0,count(nullif(doc_ready_time>(now()-'5 minutes'::interval),false)) as c1,count(nullif(view_ready_time>(now()-'5 minutes'::interval),false)) as c2,count(nullif(quote_ready_time>(now()-'5 minutes'::interval),false)) as c3 from pages where updated_at > (now()-'5 minutes'::interval);") if res puts "texts.value #{res[0]['c0']}" puts "docs.value #{res[0]['c1']}" puts "views.value #{res[0]['c2']}" puts "quotes.value #{res[0]['c3']}" end exit 0 end end https://gist.github.com/949049#file_munin.rake
- 16. https://gist.github.com/949049#file_nagios.rake namespace :nagios do task :abbyy_activity, [:warn, :crit] => :environment do |t, args| if args.warn and args.crit states_list = ["loaded", "in_abbyy", "p2f_queue"] warn_level = args.warn.to_i crit_level = args.crit.to_i warn_events = Audit.find(:all, :conditions=>["auditable_type = 'Book' and (username is null or username ilike 'rake:%') and action = 'update' and created_at > ?", warn_level.minutes.ago]) crit_events = Audit.find(:all, :conditions=>["auditable_type = 'Book' and (username is null or username ilike 'rake:%') and action = 'update' and created_at > ?", crit_level.minutes.ago]) if (crit_count = crit_events.select{|a| a['changes']['state'] and (a['changes']['state'] & states_list).size > 1 }).empty? puts "CRITICAL No state changes at abbyy for #{crit_level} minutes" exit 2 end if (warn_count = warn_events.select{|a| a['changes']['state'] and (a['changes']['state'] & states_list).size > 1 }).empty? puts "WARNING No state changes at abbyy for #{warn_level} minutes" exit 1 end puts "OK Abbyy: state changes #{warn_count.size} for #{warn_level} minutes; state changes #{crit_count.size} for #{crit_level} minutes." exit 0 else exit 3 end end http://www.nagios.org/