Skip to content

Instantly share code, notes, and snippets.

@arman-h
Last active December 17, 2015 11:58
Show Gist options
  • Select an option

  • Save arman-h/5605848 to your computer and use it in GitHub Desktop.

Select an option

Save arman-h/5605848 to your computer and use it in GitHub Desktop.
Setting up Delayed Jobs and Workless with Rails and Heroku. This app creates delayed jobs to fetch large remote files using Mechanize.
# db/migrate/20130516013336_create_fetchers.rb
# Table to store fetched/unfetched files.
class CreateFetchers < ActiveRecord::Migration
def change
create_table :fetchers do |t|
t.string :url
t.binary :file
t.string :status
t.timestamps
end
end
end
# db/migrate/20130516013734_create_delayed_jobs.rb
# Table to store delayed jobs.
# This migration is generated using `rails generate delayed_job:upgrade`.
class CreateDelayedJobs < ActiveRecord::Migration
def self.up
create_table :delayed_jobs, :force => true do |table|
table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue
table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually.
table.text :handler # YAML-encoded string of the object that will do work
table.text :last_error # reason for last failure (See Note below)
table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
table.datetime :locked_at # Set when a client is working on this object
table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
table.string :locked_by # Who is working on this object (if locked)
table.string :queue # The name of the queue this job is in
table.timestamps
end
add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
end
def self.down
drop_table :delayed_jobs
end
end
# app/controllers/delayed_job_controller.rb
class DelayedJobController < ApplicationController
# Just renders the form.
def index
end
# Creates Fetcher object and enqueues a Delayed Job.
def start
@current_job = Fetcher.create params['fetcher']
Delayed::Job.enqueue FetchingJob.new(@current_job.id)
end
# Updater for JSON status check.
def status_update
@current_job = Fetcher.find params[:id], :select => 'id, url, status, created_at, updated_at'
if @current_job.status.to_sym == :fetched
output = { :status => @current_job.status, :url => @current_job.url }
render :json => output
else
head 202
end
end
end # class DelayedJobController
# config/environments/development.rb
config.after_initialize do
Delayed::Job.scaler = :local
Delayed::Worker.backend = :active_record
Delayed::Worker.max_attempts = 10 # default: 25
Delayed::Worker.sleep_delay = 10 # default: 5
Delayed::Worker.max_run_time = 10.minutes # default: 4 hours
Delayed::Worker.destroy_failed_jobs = false # default: true
Delayed::Worker.read_ahead = 1000 # default: read 5 jobs from the queue when finding an available job
end
# app/models/fetcher.rb
# Model stores unfetched and fetched files.
# Before the file is fetched, model only contains the URL.
# After the file is successfully fetched, model also includes the file binary.
class Fetcher < ActiveRecord::Base
attr_accessible :url, :file, :status
def fetch
agent = Mechanize.new
agent.log = Logger.new($stdout)
agent.pluggable_parser.default = Mechanize::Download
self.file = agent.get_file(self.url)#.save('tmp/tmp-file')
self.status = :fetched
save!
end
end
# app/jobs/fetching_job.rb
# The actual job class, gets called by the Delayed Job worker. Must respond to `perform` method.
# In turn, this job activates Fetcher's process-heavy `fetch` method, which does the actual hard work.
class FetchingJob < Struct.new(:fetcher_id)
def perform
fetcher = Fetcher.find(fetcher_id)
fetcher.fetch
end
end
source 'https://rubygems.org'
gem 'rails', '3.2.13'
gem 'pg'
gem 'jquery-rails'
group :assets do
gem 'sass-rails', '~> 3.2.3'
gem 'coffee-rails', '~> 3.2.1'
gem 'uglifier', '>= 1.0.3'
end
### MY STUFF
gem 'activerecord-postgresql-adapter'
gem 'delayed_job_active_record'
gem 'workless'
gem 'mechanize' # for fetching files
# these gems are not required for DJ, but nice to have...
gem 'thin'
gem 'haml-rails'
GEM
remote: https://rubygems.org/
specs:
actionmailer (3.2.13)
actionpack (= 3.2.13)
mail (~> 2.5.3)
actionpack (3.2.13)
activemodel (= 3.2.13)
activesupport (= 3.2.13)
builder (~> 3.0.0)
erubis (~> 2.7.0)
journey (~> 1.0.4)
rack (~> 1.4.5)
rack-cache (~> 1.2)
rack-test (~> 0.6.1)
sprockets (~> 2.2.1)
activemodel (3.2.13)
activesupport (= 3.2.13)
builder (~> 3.0.0)
activerecord (3.2.13)
activemodel (= 3.2.13)
activesupport (= 3.2.13)
arel (~> 3.0.2)
tzinfo (~> 0.3.29)
activerecord-postgresql-adapter (0.0.1)
pg
activeresource (3.2.13)
activemodel (= 3.2.13)
activesupport (= 3.2.13)
activesupport (3.2.13)
i18n (= 0.6.1)
multi_json (~> 1.0)
arel (3.0.2)
builder (3.0.4)
coffee-rails (3.2.2)
coffee-script (>= 2.2.0)
railties (~> 3.2.0)
coffee-script (2.2.0)
coffee-script-source
execjs
coffee-script-source (1.6.2)
daemons (1.1.9)
delayed_job (3.0.5)
activesupport (~> 3.0)
delayed_job_active_record (0.3.3)
activerecord (>= 2.1.0, < 4)
delayed_job (~> 3.0)
domain_name (0.5.11)
unf (>= 0.0.5, < 1.0.0)
erubis (2.7.0)
eventmachine (1.0.3)
excon (0.21.0)
execjs (1.4.0)
multi_json (~> 1.0)
fattr (2.2.1)
haml (4.0.2)
tilt
haml-rails (0.4)
actionpack (>= 3.1, < 4.1)
activesupport (>= 3.1, < 4.1)
haml (>= 3.1, < 4.1)
railties (>= 3.1, < 4.1)
heroku-api (0.3.10)
excon (~> 0.21.0)
hike (1.2.2)
i18n (0.6.1)
journey (1.0.4)
jquery-rails (2.2.1)
railties (>= 3.0, < 5.0)
thor (>= 0.14, < 2.0)
json (1.8.0)
mail (2.5.4)
mime-types (~> 1.16)
treetop (~> 1.4.8)
mechanize (2.6.0)
domain_name (~> 0.5, >= 0.5.1)
mime-types (~> 1.17, >= 1.17.2)
net-http-digest_auth (~> 1.1, >= 1.1.1)
net-http-persistent (~> 2.5, >= 2.5.2)
nokogiri (~> 1.4)
ntlm-http (~> 0.1, >= 0.1.1)
webrobots (>= 0.0.9, < 0.2)
mime-types (1.23)
multi_json (1.7.3)
net-http-digest_auth (1.3)
net-http-persistent (2.8)
nokogiri (1.5.9)
ntlm-http (0.1.1)
pg (0.15.1)
polyglot (0.3.3)
rack (1.4.5)
rack-cache (1.2)
rack (>= 0.4)
rack-ssl (1.3.3)
rack
rack-test (0.6.2)
rack (>= 1.0)
rails (3.2.13)
actionmailer (= 3.2.13)
actionpack (= 3.2.13)
activerecord (= 3.2.13)
activeresource (= 3.2.13)
activesupport (= 3.2.13)
bundler (~> 1.0)
railties (= 3.2.13)
railties (3.2.13)
actionpack (= 3.2.13)
activesupport (= 3.2.13)
rack-ssl (~> 1.3.2)
rake (>= 0.8.7)
rdoc (~> 3.4)
thor (>= 0.14.6, < 2.0)
rake (10.0.4)
rdoc (3.12.2)
json (~> 1.4)
rush (0.6.8)
session
sass (3.2.9)
sass-rails (3.2.6)
railties (~> 3.2.0)
sass (>= 3.1.10)
tilt (~> 1.3)
session (3.1.0)
fattr
sprockets (2.2.2)
hike (~> 1.2)
multi_json (~> 1.0)
rack (~> 1.0)
tilt (~> 1.1, != 1.3.0)
thin (1.5.1)
daemons (>= 1.0.9)
eventmachine (>= 0.12.6)
rack (>= 1.0.0)
thor (0.18.1)
tilt (1.4.1)
treetop (1.4.12)
polyglot
polyglot (>= 0.3.1)
tzinfo (0.3.37)
uglifier (2.1.0)
execjs (>= 0.3.0)
multi_json (~> 1.0, >= 1.0.2)
unf (0.1.1)
unf_ext
unf_ext (0.0.6)
webrobots (0.1.1)
workless (1.1.2)
delayed_job (>= 2.0.7)
heroku-api
rails
rush
PLATFORMS
ruby
DEPENDENCIES
activerecord-postgresql-adapter
coffee-rails (~> 3.2.1)
delayed_job_active_record
haml-rails
jquery-rails
mechanize
pg
rails (= 3.2.13)
sass-rails (~> 3.2.3)
thin
uglifier (>= 1.0.3)
workless (~> 1.1.1)
# These are required by Workless to be able to scale workers up and down on Heroku,
HEROKU_API_KEY: my_long_API_key
HEROKU_POSTGRESQL_BLACK_URL: postgres://username:password@ec2-instance.amazonaws.com:5432/database_name
# These tell Workless how many workers to spin up depending on the number of jobs,
WORKLESS_MAX_WORKERS: 100
WORKLESS_MIN_WORKERS: 0
WORKLESS_WORKERS_RATIO: 1
// views/delayed_Jobs/index.html.haml
.well
%h1 Current Job Status
%hr
%dl
%dd
%dt Job #
%dd= @current_job.id
%dt URL
%dd= @current_job.url
%dt Status
%dd#status
%strong
%em= @current_job.status.upcase
- unless @current_job.status == "fetched"
%img{:src=>"/loader.gif"}
%dt Created
%dd= @current_job.created_at
%dt Updated
%dd= @current_job.updated_at
:javascript
$.PeriodicalUpdater('/delayed_job/status-update/#{@current_job.id}', {
method: 'get', // method; get or post
data: '', // array of values to be passed to the page - e.g. {name: "John", greeting: "hello"}
minTimeout: 3333, // starting value for the timeout in milliseconds
maxTimeout: 3333*6, // maximum length of time between requests
multiplier: 3, // the amount to expand the timeout by if the response hasn't changed (up to maxTimeout)
type: 'json', // response type - text, xml, json, etc. See $.ajax config options
maxCalls: 10, // maximum number of calls. 0 = no limit.
autoStop: 0, // automatically stop requests after this many returns of the same data. 0 = disabled.
//autoStopCallback: function() { ... } // The callback to execute when the autoStop condition is reached
cookie: {}, // configuration for the timeout-storing cookie
verbose: 2 // Sets the console logging verbosity: 0=none, 1=some, 2=all
}, function(remoteData, success, xhr, handle) {
// Process the new data (only called when there was a change)
// For a description of "success", see $.ajax documentation
alert("success");
});
web: bundle exec rails server -p $PORT
# Used only for development environment.
# Activate using `$ foreman start`.
# Worker processes are not necessary to be declared here for production environment,
# since the Workless gem will spin up/down as many workers as needed.
# However, do make sure Heroku allows adding workers to your app before activating Workless.
worker: bundle exec rake jobs:work
# config/environments/production.rb
config.after_initialize do
Delayed::Job.scaler = :heroku_cedar # <=== THIS IS IMPORTANT!
Delayed::Worker.backend = :active_record
Delayed::Worker.max_attempts = 3
Delayed::Worker.sleep_delay = 0
Delayed::Worker.max_run_time = 10.minutes
Delayed::Worker.destroy_failed_jobs = false
Delayed::Worker.read_ahead = 1000 # default: read 5 jobs from the queue when finding an available job
end
# app/config/routes.rb
TestWorklessApp::Application.routes.draw do
get '/' => 'delayed_job#index'
get 'delayed_job/index'
post 'delayed_job/start'
get 'delayed_job/status-update/(:id)' => 'delayed_job#status_update'
end
// views/delayed_Jobs/start.html.haml
.well
%h1 Delayed Jobs
%hr
%form.form-horizontal{:method=>:post, :action=>:'/delayed_job/start'}
%input{:type=>:hidden, :name=>:'fetcher[status]', :value=>:new}
.control-group
%label.control-label{:for=>:'fetcher[url]'}
%strong Fetch File URL
.controls
%input{:type=>:text, :name=>:'fetcher[url]'}
%select{:name=>:'fetcher[url]', :id=>:'fetcher[url]'}
%option
%option{:value=>'http://download.thinkbroadband.com/5MB.zip'} Small (5MB)
%option{:value=>'http://download.thinkbroadband.com/10MB.zip'} Medium (10MB)
%option{:value=>'http://download.thinkbroadband.com/50MB.zip'} Large (50MB)
%option{:value=>'http://download.thinkbroadband.com/100MB.zip'} X-Large (100MB)
.control-group
.controls
%input.btn.btn-primary.btn-medium{:type=>:submit, :value=>'Fetch using Delayed Job'}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment