From Pojo.us
#!/usr/bin/env ruby
#
# Amazon EBS Snapshot creation and management
# http://blog.nrgup.net/2009/04/amazon-ec2-mysql-script-for-gfs-snapshots/
# Original Author: Jonathan Bradshaw
# Other Contributors:
# Ben La Monica - Modified to use a config file; Allow multiple EBS volumes per mount point (for LVM/Raid Volumes); Only flush database if it is running;
# Michael Glass - Fixed bug with config values and made general ruby improvements; Made db-user and db-password optional in config file;
%w(logger optparse rubygems uuidtools right_aws sdb/active_sdb mysql net/http).each { |l| require l }
# hack to eliminate the SSL certificate verification notification
class Net::HTTP
alias_method :old_initialize, :initialize
def initialize(*args)
old_initialize(*args)
@ssl_context = OpenSSL::SSL::SSLContext.new
@ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
end
end
# class used for logging snapshot meta data to simpledb
class Snapshot < RightAws::ActiveSdb::Base
set_domain_name :snapshots
end
class MySQLDatabaseType
@username
@password
@expire
@connection
@logger
@is_running
def initialize(username, password, expire, logger)
@username = username
@password = password
@expire = expire
@logger = logger
running = `ps aux |grep "/mysqld " | grep -v grep`
@is_running = (!running.nil? and running.strip != "")
end
def running?
return @is_running
end
def connect
@connection = Mysql::new('localhost', @username, @password)
end
def lock
self.connect if @connection.nil?
@logger.info("Locking and flushing MySQL tables")
@connection.query('FLUSH TABLES WITH READ LOCK;')
@connection.query('FLUSH LOGS;')
end
def unlock
self.connect if @connection.nil?
@logger.info("Unlocking MySQL tables")
@connection.query('UNLOCK TABLES;')
end
def cleanup
self.connect if @connection.nil?
purgetime = (Time.now - (86400 * @expire.to_i)).strftime('%Y-%m-%d %H:%M:%S')
@logger.info("Purging log files older than #{purgetime}")
@connection.query("PURGE BINARY LOGS BEFORE '#{purgetime}';")
end
def close
@connection.close unless @connection.nil?
end
end
#
# Main application
#
logger = Logger.new("/var/log/snapshot.log")
logger.datetime_format = "%Y-%m-%d %H:%M:%S"
logger.level = Logger::INFO
key_names = {"sdb-access-key" => :aws_sdb_access_key_id,
"sdb-secret-key" => :aws_sdb_secret_access_key,
"ebs-access-key" => :aws_ebs_access_key_id,
"ebs-secret-key" => :aws_ebs_secret_access_key,
"devices" => :ebs_devices,
"mountpoint" => :mountpoint,
"db-user" => :db_user,
"db-password" => :db_password,
"expire" => :expire}
options = {}
parser = OptionParser.new do |p|
p.banner = "Usage: snapshot.rb [options]"
p.separator ""
p.separator "Specific options:"
p.on("-c", "--config CONFIG_FILE", "name=value config file; Keys are:\n\t" + key_names.keys.join("\n\t")) do |conf|
File.open(conf).each do |line|
line.strip!
next if line.empty?
args = line.split("=");
val = args[1].strip
key = key_names[args[0]]
unless key.nil?
puts("Setting #{key}.")
options[key] = val
else
puts("Key '#{args[0]}' was not found")
end
end
# convert the comma separated list into an array
options[:ebs_devices] = options[:ebs_devices].squeeze(" ").strip.split(",")
end
p.on("-e",
"--expire number",
"The number of days for this snapshot to live (0 = forever)") do |expire|
options[:expire] = expire
end
p.on_tail("-h", "--help", "Show this message") {
puts(p)
exit
}
p.parse!(ARGV) rescue puts(p)
end
# check and make sure all of our parameters are set
all_vars_set = true
missing_keys = []
key_names.keys.each do |key_name|
unless options.key?(key_names[key_name])
if ['db-user', 'db-password'].include? key_name
next if !options.key?( key_names['db-user'] ) && !options.key?( key_names['db-password'] )
end
missing_keys << key_name
all_vars_set = false
end
end
# all_vars_set &= options.key?(key_names[key_name])
unless all_vars_set
puts(parser)
puts("Missing keys: #{missing_keys.join(', ')}\n")
exit(1)
end
# Determine our instance ID
instance_id = Net::HTTP.get('169.254.169.254', '/latest/meta-data/instance-id')
puts("InstanceId = #{instance_id}")
# Create SimpleDB connection to AWS
RightAws::ActiveSdb.establish_connection(options[:aws_sdb_access_key_id], options[:aws_sdb_secret_access_key], :logger => logger)
# Create (if it doesn't exist) our SimpleDB database domain
Snapshot.create_domain
# Create a connection to AWS
ec2 = RightAws::Ec2.new(options[:aws_ebs_access_key_id], options[:aws_ebs_secret_access_key], :logger => logger)
# Locate the volume attached to the specified device (e.g. /dev/sdj)
volumes = ec2.describe_volumes.find_all { |vol| vol[:aws_instance_id] == instance_id and
options[:ebs_devices].include?(vol[:aws_device]) }
if (volumes.nil? or volumes.length == 0)
logger.error("Error: Unable to find #{options[:ebs_devices]} device attached to this instance #{instance_id}")
exit(1)
end
# Determine which file system is mounted on the EBS device
mountpoint = options[:mountpoint]
# Log the volume id, devie and mountpoint
volumes.each do |volume|
logger.info("Volume #{volume[:aws_id]} attached as #{volume[:aws_device]} at #{mountpoint}")
end
db = MySQLDatabaseType.new(options[:db_user], options[:db_password], options[:expire], logger) if options.key? :db_user
begin
# if the database is running, lock it so that nobody can write to it
db.lock if db && db.running?
# freeze the filesystem, to prevent writes while the snapshot is taking place
logger.info("Freezing filesystem #{mountpoint}")
system("xfs_freeze -f #{mountpoint}")
# loop through the EBS volumes, there may be several for LVM or raided volumes
volumes.each do |volume|
# Create EBS snapshot
result = ec2.create_snapshot(volume[:aws_id])
# Create snapshot entry in domain with expiration date
if (options[:expire].to_i > 0)
expires = Time.now + (86400 * options[:expire].to_i)
Snapshot.create 'snapshotid' => result[:aws_id], 'volumeid' => result[:aws_volume_id], 'expires' => expires.to_i
logger.info("Created snapshot #{result[:aws_id]} for #{result[:aws_volume_id]} expires #{expires}")
else
logger.info("Created snapshot #{result[:aws_id]} for #{result[:aws_volume_id]} (no expiration)")
end
end
ensure
#make sure we make the filesystem writable again!
logger.info("Unfreezing filesystem #{mountpoint}")
system("xfs_freeze -u #{mountpoint}")
# and unfreeze the database as well if it is running
if (db && db.running?)
begin
db.unlock
db.cleanup
ensure
db.close
end
end
end
# Purge expired snapshots
volumes.each do |volume|
logger.info("Querying for old snapshots for volume: #{volume[:aws_id]}")
# only delete snapshots for the volumes that we were working on
Snapshot.select(:all, :conditions=> [ "expires<? and volumeid=?", Time.now.to_i, volume[:aws_id]]).each do |snap|
snap.reload
logger.info("Removing expired EBS snapshot #{snap[:snapshotid]}")
begin
ec2.delete_snapshot(snap[:snapshotid])
rescue
logger.error("Unable to delete snapshot #{snap[:snapshotid]}, please delete manually.")
logger.error($!);
ensure
snap.delete
end
end
end
Example Configuration File
sdb-access-key=<INSERT_YOUR_SIMPLE_DB_ACCESS_KEY_HERE> sdb-secret-key=<INSERT_YOUR_SIMPLE_DB_SECRET_ACCESS_KEY_HERE> ebs-access-key=<INSERT_YOUR_EC2_ACCESS_KEY_HERE> ebs-secret-key=<INSERT_YOUR_EC2_SECRET_ACCESS_KEY_HERE> devices=/dev/sdb,/dev/sdc,/dev/sdd mountpoint=/data db-user=<INSERT YOUR DATABASE DBA USER HERE> db-password=<INSERT YOUR DATABASE DBA PASSWORD HERE>
Crontab
Add this to your root crontab for automated backups. This script will take hourly/daily/weekly/monthly snapshots of your EBS storage. (make sure to change the path to your script and config file if it is in a different place)
# m h dom mon dow command # Keep 1 day of hourly, 7 days of daily, 4 weeks of weekly and 1 year of monthly snapshots 00 * * * * /data/backup/snapshot.rb -e 1 -c /data/backup/snapshot.cfg 15 0 * * * /data/backup/snapshot.rb -e 7 -c /data/backup/snapshot.cfg 30 0 * * 0 /data/backup/snapshot.rb -e 28 -c /data/backup/snapshot.cfg 45 0 1 * * /data/backup/snapshot.rb -e 365 -c /data/backup/snapshot.cfg