Last active
September 27, 2020 20:10
-
-
Save SeanHayes/4958077 to your computer and use it in GitHub Desktop.
Revisions
-
Seán Hayes revised this gist
Feb 15, 2013 . No changes.There are no files selected for viewing
-
Feb 15, 2013 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,284 @@ # -*- coding: utf-8 -*- #Copyright (C) 2013 Seán Hayes import my_project.settings as dj_settings from fabric.api import local, run, sudo, env, prompt, settings, cd, parallel, execute from fabric.contrib.files import exists from fabric.decorators import hosts, roles, runs_once import json import logging import os logging.getLogger('').setLevel(logging.INFO) logger = logging.getLogger(__name__) env.forward_agent = True env.user = 'username' role_list = ( 'web', 'cache', 'db', 'celery', ) host_dict = { '100.100.100.100': ('web', 'cache',), '100.100.100.101': ('db', 'celery',), } env.hosts = host_dict.keys() for r in role_list: env.roledefs[r] = [] for k_host, v_roles in host_dict.items(): for v_role in v_roles: env.roledefs[v_role].append(k_host) env.code_dir = '/srv/' env.package_name = dj_settings.PACKAGE_MODULE env.project_dir = '%s%s/' % (env.code_dir, dj_settings.PROJECT_NAME,) env.package_dir = '%s%s/' % (env.project_dir, dj_settings.PACKAGE_MODULE,) env.project_git_uri = 'git@github.com:some-user/my-project.git' env.config_dir = '%sconfig/generated/' % env.project_dir env.log_dir = '%slogs/' % env.project_dir env.pip_dir = '%spip/' % env.code_dir env.celery_script_dir = '%scelery/init.d/' % env.config_dir main_dirs = [ env.pip_dir, ] project_dirs = [ env.log_dir, ] apt_packages = [ 'debconf-utils', 'git', 'mercurial', 'subversion', 'ntp', 'gdebi-core', 'graphviz', 'graphviz-dev', 'libmemcached-tools', 'memcached', 'nginx', 'pkg-config', 'postfix', 'python-pip', 'python-virtualenv', 'python-all-dev', 'postgresql', #'rabbitmq-server', #TODO: try to get as many of these as possible in requirements.txt #'python-django-doc', #some require 1/4 GB of dependencies to build the PIP version, which is unacceptable for this kind of application 'python-imaging', 'python-psycopg2', #'python-exactimage', #'python-crypto', ] #Run the following to make binary eggs when setuptools isn't used #python -c "import setuptools; execfile('setup.py')" bdist_egg # tasks def create_user(): "Create admin user on fresh cloud instance." username = prompt('Enter username to create: ', default=env.user) with settings(user='root'): run('useradd --groups sudo,www-data -d /home/%s -m %s' % (username, username)) run('passwd %s' % username) def switch_to_bash(): "switch from dash (the Ubuntu default) to bash" with cd('/bin'): #has to be one command since each call to sudo() is a different session, #and you can't login if sh isn't set sudo('rm sh; ln -s bash sh') @roles('db') @runs_once def setup_pgsql(): "Sets up PostgreSQL user and databases." name = prompt('Enter PostgreSQL role/db to create: ', default=env.project_name) sudo('createuser -s -P %s' % name, user='postgres') sudo('createdb -O %s %s' % (name, name), user='postgres') def mkdirs(dirs): "Sets up the directories we need and sets the right permissions." for d in dirs: if not exists(d): sudo('mkdir %s' % d) sudo('chown %s:www-data %s' % (env.user, d)) sudo('chmod 775 %s' % d) def upgrade_ubuntu(): "Probably shouldn't run this through Fabric, but here's the commands for it anyway." sudo('apt-get install update-manager-core') #edit /etc/update-manager/release-upgrades, set Prompt=normal sudo('do-release-upgrade') @parallel def install_apt(): "Updates package list, upgrades all packages to latest available version, and installs Apt dependencies for this project." sudo('apt-get update') sudo('apt-get upgrade') sudo('apt-get install -f %s' % ' '.join(apt_packages)) @parallel def set_permissions(): sudo('chown :www-data %s' % (env.code_dir,)) sudo('chmod 775 %s' % env.code_dir) @parallel def install_pip(): "Installs the PIP requirements for this project." with cd(env.pip_dir): sudo('pip install -r %srequirements.txt' % env.project_dir) @parallel def install_project(): "Clones this project's Git repo if there's no copy on the target machine, else it pulls the latest version." if exists(env.project_dir): with cd(env.project_dir): run('git pull origin master') else: with cd(env.code_dir): run('git clone %s' % env.project_git_uri) sudo('chown -R %s:www-data %s' % (env.user, env.project_dir)) sudo('chmod 775 %s' % env.project_dir) mkdirs(project_dirs) @hosts('') def install(): "Runs the commands to create all necessary directories, install Apt and PIP dependencies, and install project files." execute(mkdirs, main_dirs) execute(install_apt) execute(set_permissions) execute(install_project) execute(install_pip) @roles('db') @runs_once def migrate(): with cd(env.package_dir): run('./manage.py syncdb --migrate') @roles('web') def collectstatic(): with cd(env.project_dir): run('./manage.py collectstatic -l') def refresh_config_files(): "Regenerates dynamic config files using django-config-gen." with cd(env.package_dir): run('./manage.py config_gen') def link_config_file(source, destination): with settings(warn_only=True): sudo('rm %s' % destination) sudo('ln -s %s %s' % (source, destination)) @roles('web') def config_nginx(): with settings(warn_only=True): sudo('rm /etc/nginx/sites-available/*') link_config_file(os.path.join(env.config_dir, 'nginx'), '/etc/nginx/sites-available/default') @roles('db') def config_postgresql(): link_config_file(os.path.join(env.config_dir, 'pg_hba.conf'), '/etc/postgresql/9.1/main/pg_hba.conf') @roles('cache') def config_memcached(): link_config_file(os.path.join(env.config_dir, 'memcached.conf'), '/etc/memcached.conf') @roles('celery') def config_celery(): "Links Celery's Debian init scripts to /etc/init.d/." init_list=run('ls %s' % env.celery_script_dir).split() for script in init_list: p = '/etc/init.d/%s' % script init_file = os.path.join(env.celery_script_dir, script) link_config_file(init_file, p) sudo('chmod +x %s' % init_file) link_config_file(os.path.join(env.config_dir, 'celery/celeryd_default'), '/etc/default/celeryd') def config_tzdata(): "Configures the time zone for the server." run('echo \'America/New_York\'| sudo tee /etc/timezone') sudo('dpkg-reconfigure -f noninteractive tzdata') @hosts('') def config(): "Runs the commands to generate config files using django-config-gen and symlinks the generated files to the normal config file locations for Apache, Nginx, Memcached, etc." execute(refresh_config_files) execute(config_nginx) execute(config_memcached) execute(config_celery) execute(config_tzdata) @roles('web') def reload_nginx(): sudo('/etc/init.d/nginx reload') @roles('web') def reload_uwsgi(): sudo('kill -HUP `cat %suwsgi.pid`' % env.project_dir) @roles('celery') def restart_celery(): sudo('/etc/init.d/celeryd restart') sudo('/etc/init.d/celerybeat restart') sudo('/etc/init.d/celeryevcam restart') @roles('cache') def restart_memcached(): sudo('/etc/init.d/memcached restart') @hosts('') def reload_servers(): "Reloads Apache, Nginx, Rabbit MQ, and Celery where possible, otherwise it restarts them. Reloading config files is faster than restarting the processes." execute(reload_nginx) execute(reload_uwsgi) #sudo('/etc/init.d/rabbitmq-server reload') execute(restart_celery) execute(restart_memcached) #local development scripts def setup_git_shortcuts(): #https://git.wiki.kernel.org/index.php/Aliases shortcuts = { 'st': 'status', } for shortcut in shortcuts.items(): local('git config --global alias.%s %s' % shortcut) def set_django_colors(): local('export DJANGO_COLORS="%s"' % dj_settings.DJANGO_COLORS) def check_for_pdb(): "Easily check for instances of pdb.set_trace() in your code before committing." local('find . -name \'*.py\'|xargs grep \'pdb.set_trace\'') #TODO: handle exit code default_dump_file_template = '%s-dump.sql' def dump_db(): name = prompt('Enter PostgreSQL db to dump: ', default=env.project_name) local('sudo -u postgres pg_dump %s > %s' % (name, default_dump_file_template % name)) def load_db_dump(): name = prompt('Enter PostgreSQL db to load: ', default=env.project_name) local('sudo -u postgres psql %s < %s' % (name, default_dump_file_template % name)) This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,30 @@ #Python imports import os import sys #Django imports import django.conf.global_settings as DEFAULT_SETTINGS from django.utils import timezone PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__)) PROJECT_ROOT = PACKAGE_PARENT_DIR = os.path.dirname(PACKAGE_ROOT) PROJECT_NAME = PROJECT_ROOT.split(os.sep)[-1] PACKAGE_MODULE = __name__[:__name__.rfind('.')] if '.' in __name__ else PACKAGE_ROOT.split(os.sep)[-1] LOG_DIR = os.path.join(PROJECT_ROOT, 'logs') _colors_dict = { 'error': ['red', 'bold'], 'notice': ['red'], 'http_info': ['cyan'], 'http_success': ['blue'], 'http_not_modified': ['cyan'], 'http_redirect': ['cyan'], 'http_not_found': ['red'], 'http_bad_request': ['red'], 'http_server_error': ['red'], } #needs to be exported as an environment variable DJANGO_COLORS = ''.join([''.join([k, '=', ','.join(_colors_dict[k]), ';']) for k in _colors_dict])
Seán Hayes
revised
this gist