-
-
Save fkei/3433375 to your computer and use it in GitHub Desktop.
Revisions
-
rore revised this gist
Mar 8, 2012 . 1 changed file with 17 additions and 7 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -10,6 +10,7 @@ from boto.ec2.connection import EC2Connection from optparse import OptionParser from subprocess import check_call from time import gmtime, strftime def get_ec2_metadata(key): res = urllib2.urlopen("http://169.254.169.254/2008-02-01/meta-data/" + key) @@ -21,6 +22,7 @@ def build_parser(): parser.add_option("-k", "--awskeys", dest="awskeys", help="AWS key and secret key separated by a colon") parser.add_option("-l", "--locker", dest="lockers", help="Locker for specific services (mongo, mysql, ...) e.g. -l mongodb:port=27018", default=[], action="append") parser.add_option("-m", "--mount", dest="mount_point", help="Mount point of volume to snapshot") parser.add_option("-c", "--device", dest="given_device", help="The mounted device name") parser.add_option("-v", "--verbose", dest="verbose", default=False, action="store_true") return parser @@ -67,12 +69,12 @@ def __enter__(self): logging.info("Freezing XFS: %s", self.mount_point) if not self.dryrun: check_call(["/bin/sync"]) check_call(["/sbin/fsfreeze", "-f", self.mount_point]) def __exit__(self, exc_type, exc_val, exc_tb): logging.info("Unfreezing XFS: %s", self.mount_point) if not self.dryrun: check_call(["/sbin/fsfreeze", "-u", self.mount_point]) class MongoLocker(object): def __init__(self, host="localhost", port=27017, slaveonly=False, dryrun=False): @@ -132,28 +134,35 @@ def __exit__(self, exc_type, exc_val, exc_tb): 'mysql': MySQLLocker, } def backup(mount_point, given_device, aws_key, aws_secret_key, lockers=[], dryrun=False): logging.info("Mount point: %s", mount_point) logging.info("Given device: %s", given_device) if given_device: devices = [given_device] else: devices = [get_device_for_mount(mount_point)] if devices[0].startswith("/dev/md"): devices = get_devices_for_raid(devices[0]) logging.info("Devices: %s", ", ".join(devices)) instance_id = get_ec2_metadata('instance-id') logging.info("Instance ID: %s", instance_id) ec2 = EC2Connection(aws_key, aws_secret_key) instance = ec2.get_all_instances([instance_id])[0].instances[0] all_volumes = ec2.get_all_volumes() logging.info("All Volumes: %s", ", ".join(v.id for v in all_volumes)) volumes = [] for v in all_volumes: if v.attach_data.instance_id == instance_id: logging.info("Found instance, device: %s", v.attach_data.device) if v.attach_data.device in devices: volumes.append(v) if not volumes: sys.stderr.write("No EBS volumes found for devices %s\n" % devices) sys.exit(1) logging.info("Volumes: %s", ", ".join(v.id for v in volumes)) locker_instances = [] @@ -177,7 +186,7 @@ def backup(mount_point, aws_key, aws_secret_key, lockers=[], dryrun=False): with contextlib.nested(*locker_instances): for v in volumes: name = v.tags.get('Name') + "-" + strftime("%Y%m%d-%H%M%S", gmtime()) logging.info("Snapshoting %s (%s)", v.id, name or 'NONAME') if not dryrun: snap = v.create_snapshot() @@ -196,6 +205,7 @@ def main(): aws_key, aws_secret_key = options.awskeys.split(':') backup(options.mount_point, given_device = options.given_device, aws_key = aws_key, aws_secret_key = aws_secret_key, lockers = options.lockers, -
samuel revised this gist
Feb 15, 2011 . 1 changed file with 36 additions and 7 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -59,7 +59,10 @@ class XFSLocker(object): def __init__(self, mount_point, dryrun=False): self.mount_point = mount_point self.dryrun = dryrun def validate(self): return True def __enter__(self): logging.info("Freezing XFS: %s", self.mount_point) if not self.dryrun: @@ -72,11 +75,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): check_call(["/usr/sbin/xfs_freeze", "-u", self.mount_point]) class MongoLocker(object): def __init__(self, host="localhost", port=27017, slaveonly=False, dryrun=False): import pymongo.connection self.connection = pymongo.connection.Connection(host, int(port), slave_okay=True) self.dryrun = dryrun self.slaveonly = slaveonly def validate(self): if not self.slaveonly: return True info = self.connection.admin.command("isMaster") return not info["ismaster"] def __enter__(self): logging.info("Fsyncing and locking MongoDB") if not self.dryrun: @@ -92,15 +102,25 @@ def __init__(self, user, password, host="localhost", port=3306, dryrun=False): import MySQLdb self.connection = MySQLdb.connect(host=host, port=port, user=user, passwd=password) self.dryrun = dryrun def validate(self): return True def __enter__(self): logging.info("Flushing and locking MySQL") if not self.dryrun: c = self.connection.cursor() # Don't pass FLUSH TABLES statements on to replication slaves # as this can interfere with long-running queries on the slaves. c.execute("SET SQL_LOG_BIN=0") c.execute("FLUSH LOCAL TABLES") c.execute("FLUSH LOCAL TABLES WITH READ LOCK") c.execute("SHOW MASTER STATUS") c.execute("SET SQL_LOG_BIN=1") def __exit__(self, exc_type, exc_val, exc_tb): logging.info("Unlocking MySQL") if not self.dryrun: @@ -140,9 +160,18 @@ def backup(mount_point, aws_key, aws_secret_key, lockers=[], dryrun=False): for l in lockers: l = l.split(':') cls = LOCKER_CLASSES[l[0]] kwargs = {} for k, v in (x.split('=') for x in l[1:]): if v.lower() == "true": v = True elif v.lower() == "false": v = False kwargs[k] = v kwargs['dryrun'] = dryrun inst = cls(**kwargs) locker_instances.append(inst) if not inst.validate(): return locker_instances.append(XFSLocker(mount_point, dryrun=dryrun)) -
Nov 30, 2010 . 1 changed file with 1 addition and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -63,6 +63,7 @@ def __init__(self, mount_point, dryrun=False): def __enter__(self): logging.info("Freezing XFS: %s", self.mount_point) if not self.dryrun: check_call(["/bin/sync"]) check_call(["/usr/sbin/xfs_freeze", "-f", self.mount_point]) def __exit__(self, exc_type, exc_val, exc_tb): -
Nov 30, 2010 . 1 changed file with 22 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -5,7 +5,6 @@ import contextlib import logging import os import sys import urllib2 from boto.ec2.connection import EC2Connection @@ -73,6 +72,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class MongoLocker(object): def __init__(self, host="localhost", port=27017, dryrun=False): import pymongo.connection self.connection = pymongo.connection.Connection(host, int(port), slave_okay=True) self.dryrun = dryrun @@ -86,8 +86,29 @@ def __exit__(self, exc_type, exc_val, exc_tb): if not self.dryrun: self.connection.admin["$cmd"].sys.unlock.find_one() class MySQLLocker(object): def __init__(self, user, password, host="localhost", port=3306, dryrun=False): import MySQLdb self.connection = MySQLdb.connect(host=host, port=port, user=user, passwd=password) self.dryrun = dryrun def __enter__(self): logging.info("Flushing and locking MySQL") if not self.dryrun: c = self.connection.cursor() c.execute("FLUSH LOCAL TABLES") c.execute("FLUSH TABLES WITH READ LOCK") c.execute("SHOW MASTER STATUS") def __exit__(self, exc_type, exc_val, exc_tb): logging.info("Unlocking MySQL") if not self.dryrun: c = self.connection.cursor() c.execute("UNLOCK TABLES") LOCKER_CLASSES = { 'mongodb': MongoLocker, 'mysql': MySQLLocker, } def backup(mount_point, aws_key, aws_secret_key, lockers=[], dryrun=False): -
Nov 10, 2010 . 1 changed file with 7 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,5 +1,7 @@ #!/usr/bin/env python from __future__ import with_statement import contextlib import logging import os @@ -71,7 +73,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class MongoLocker(object): def __init__(self, host="localhost", port=27017, dryrun=False): self.connection = pymongo.connection.Connection(host, int(port), slave_okay=True) self.dryrun = dryrun def __enter__(self): @@ -104,6 +106,10 @@ def backup(mount_point, aws_key, aws_secret_key, lockers=[], dryrun=False): if v.attach_data.device in devices: volumes.append(v) if not volumes: sys.stderr.write("No EBS volumes found for devices %s\n" % devices) sys.exit(1) logging.info("Instance ID: %s", instance_id) logging.info("Devices: %s", ", ".join(devices)) logging.info("Volumes: %s", ", ".join(v.id for v in volumes)) -
Nov 8, 2010 . 1 changed file with 5 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -19,7 +19,7 @@ def build_parser(): parser.add_option("-d", "--dryrun", dest="dryrun", help="Show what would be done but don't take any action", default=False, action="store_true") parser.add_option("-k", "--awskeys", dest="awskeys", help="AWS key and secret key separated by a colon") parser.add_option("-l", "--locker", dest="lockers", help="Locker for specific services (mongo, mysql, ...) e.g. -l mongodb:port=27018", default=[], action="append") parser.add_option("-m", "--mount", dest="mount_point", help="Mount point of volume to snapshot") parser.add_option("-v", "--verbose", dest="verbose", default=False, action="store_true") return parser @@ -130,6 +130,10 @@ def backup(mount_point, aws_key, aws_secret_key, lockers=[], dryrun=False): def main(): parser = build_parser() options, args = parser.parse_args() if not options.awskeys: parser.error("must specify AWS keys") if not options.mount_point: parser.error("must specify a mount point") logging.basicConfig(level=logging.INFO if options.verbose else logging.WARNING) -
Nov 8, 2010 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,144 @@ #!/usr/bin/env python import contextlib import logging import os import pymongo.connection import sys import urllib2 from boto.ec2.connection import EC2Connection from optparse import OptionParser from subprocess import check_call def get_ec2_metadata(key): res = urllib2.urlopen("http://169.254.169.254/2008-02-01/meta-data/" + key) return res.read().strip() def build_parser(): parser = OptionParser(usage="Usage: %prog [options] <command> ...") parser.add_option("-d", "--dryrun", dest="dryrun", help="Show what would be done but don't take any action", default=False, action="store_true") parser.add_option("-k", "--awskeys", dest="awskeys", help="AWS key and secret key separated by a colon") parser.add_option("-l", "--locker", dest="lockers", help="Locker for specific services (mongo, mysql, ...) e.g. -l mongodb:port=27018", default=[], action="append") parser.add_option("-m", "--mount", dest="mount_point", help="Mount point of volume to snapshot", default="/vol") parser.add_option("-v", "--verbose", dest="verbose", default=False, action="store_true") return parser def get_device_for_mount(mount_point): if not os.path.exists(mount_point): sys.stderr.write("Mount point %s does not exist\n" % mount_point) sys.exit(1) with open("/proc/mounts", "r") as fp: for line in fp: line = line.strip().split(' ') if line[1] == mount_point: return line[0] sys.stderr.write("Path %s does not refer to a mount point\n" % mount_point) sys.exit(1) def get_devices_for_raid(device): devname = device.rsplit('/', 1)[-1] with open("/proc/mdstat", "r") as fp: for line in fp: line = [x for x in line.strip().split(' ') if x] if line and line[0] == devname: mdevs = [] for x in line[4:]: dev, num = x.split('[', 1) num = num[:-1] mdevs.append((int(num), dev if dev.startswith('/') else "/dev/"+dev)) mdevs.sort() return [x[1] for x in mdevs] sys.stderr.write("Can't figure out devices for RAID device %s\n" % device) sys.exit(1) class XFSLocker(object): def __init__(self, mount_point, dryrun=False): self.mount_point = mount_point self.dryrun = dryrun def __enter__(self): logging.info("Freezing XFS: %s", self.mount_point) if not self.dryrun: check_call(["/usr/sbin/xfs_freeze", "-f", self.mount_point]) def __exit__(self, exc_type, exc_val, exc_tb): logging.info("Unfreezing XFS: %s", self.mount_point) if not self.dryrun: check_call(["/usr/sbin/xfs_freeze", "-u", self.mount_point]) class MongoLocker(object): def __init__(self, host="localhost", port=27017, dryrun=False): self.connection = pymongo.connection.Connection("localhost", int(port)) self.dryrun = dryrun def __enter__(self): logging.info("Fsyncing and locking MongoDB") if not self.dryrun: self.connection.admin.command("fsync", 1, lock=1) def __exit__(self, exc_type, exc_val, exc_tb): logging.info("Unlocking MongoDB") if not self.dryrun: self.connection.admin["$cmd"].sys.unlock.find_one() LOCKER_CLASSES = { 'mongodb': MongoLocker, } def backup(mount_point, aws_key, aws_secret_key, lockers=[], dryrun=False): devices = [get_device_for_mount(mount_point)] if devices[0].startswith("/dev/md"): devices = get_devices_for_raid(devices[0]) instance_id = get_ec2_metadata('instance-id') ec2 = EC2Connection(aws_key, aws_secret_key) instance = ec2.get_all_instances([instance_id])[0].instances[0] all_volumes = ec2.get_all_volumes() volumes = [] for v in all_volumes: if v.attach_data.instance_id == instance_id: if v.attach_data.device in devices: volumes.append(v) logging.info("Instance ID: %s", instance_id) logging.info("Devices: %s", ", ".join(devices)) logging.info("Volumes: %s", ", ".join(v.id for v in volumes)) locker_instances = [] for l in lockers: l = l.split(':') cls = LOCKER_CLASSES[l[0]] kwargs = dict(x.split('=') for x in l[1:]) kwargs['dryrun'] = dryrun locker_instances.append(cls(**kwargs)) locker_instances.append(XFSLocker(mount_point, dryrun=dryrun)) with contextlib.nested(*locker_instances): for v in volumes: name = v.tags.get('Name') logging.info("Snapshoting %s (%s)", v.id, name or 'NONAME') if not dryrun: snap = v.create_snapshot() if name: snap.add_tag('Name', name) def main(): parser = build_parser() options, args = parser.parse_args() logging.basicConfig(level=logging.INFO if options.verbose else logging.WARNING) aws_key, aws_secret_key = options.awskeys.split(':') backup(options.mount_point, aws_key = aws_key, aws_secret_key = aws_secret_key, lockers = options.lockers, dryrun = options.dryrun) if __name__ == "__main__": main()