def update_modules(beat):
    changes = []

    # Get desired state
    desired_modules = set(blueprint.get('{}.modules'.format(beat), []))

    # Get current state
    with silent():
        module_files = run('find /etc/{}/modules.d -iname "*.yml"'.format(beat)).split()
        enabled_modules = {os.path.basename(module).split('.')[0] for module in module_files}

    # Disable extra services
    for extra in enabled_modules - desired_modules:
        info('Disabling {} module: {}', beat, extra)
        changes.append(extra)

        with silent(), sudo():
            run('{} modules disable {}'.format(beat, extra))

    # Enable services
    for missing in desired_modules - enabled_modules:
        info('Enabling {} module: {}', beat, missing)
        changes.append(missing)

        with silent(), sudo():
            run('{} modules enable {}'.format(beat, missing))

    return changes

def update_filters():
    changes = []

    # Generate desired state as enabled_name => source_name
    config = blueprint.get('config', {})
    filters = {
        '{}-{}.conf'.format(str(weight).zfill(2), conf): "{}.conf".format(conf)
        for weight, conf in config.iteritems()
    }

    # Get current state
    with silent():
        enabled_filters = run('ls {}'.format(conf_enabled_path)).split()

    # Disable extra services
    if blueprint.get('auto_disable_conf', True):
        for link in set(enabled_filters) - set(filters.keys()):
            info('Disabling conf: {}', link)
            changes.append(link)

            with silent(), sudo(), cd(conf_enabled_path):
                debian.rm(link)

    # Enable services
    for target in set(filters.keys()) - set(enabled_filters):
        source = os.path.join(conf_available_path, filters[target])
        info('Enabling conf: {}', target)
        changes.append(source)

        with silent(), sudo(), cd(conf_enabled_path):
            debian.ln(source, target)

    return changes

def setup_schemas(drop=False):
    """
    Create database schemas and grant user privileges

    :param drop: Drop existing schemas before creation
    """
    schemas = blueprint.get('schemas', {})
    with sudo('postgres'):
        for schema, config in schemas.iteritems():
            user, password = config['user'], config.get('password')
            info('Creating user {}', user)
            if password:
                _client_exec("CREATE ROLE %(user)s WITH PASSWORD '%(password)s' LOGIN",
                             user=user, password=password)
            else:
                _client_exec("CREATE ROLE %(user)s LOGIN", user=user)
            if drop:
                info('Droping schema {}', schema)
                _client_exec('DROP DATABASE %(name)s', name=schema)
            info('Creating schema {}', schema)
            _client_exec('CREATE DATABASE %(name)s', name=schema)
            info('Granting user {} to schema {}'.format(user, schema))
            _client_exec("GRANT ALL PRIVILEGES ON DATABASE %(schema)s to %(user)s",
                         schema=schema, user=user)

            for ext in blueprint.get('extensions', []):
                info('Creating extension {}'.format(ext))
                _client_exec("CREATE EXTENSION IF NOT EXISTS %(ext)s", ext=ext, schema=schema)

def install():
    with sudo():
        info('Installing Ruby v1.9.3')
        debian.apt_get('install', 'ruby1.9.3')

    info('Installing Bundler')
    gem('install', 'bundler')

def install_forwarder():
    """
    TODO: Build from github

    - wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz
    - gunzip <go>
    - mv go /usr/local/
    - apt-get install unzip make ruby ruby-dev
    - wget https://github.com/elasticsearch/logstash-forwarder/archive/master.zip
    - unzip <forwarder>
    - cd <forwarder>
    - go build
    - gem install fpm
    - make deb
    - dpkg -i <forwarder>
    """
    with sudo():
        info("Adding apt repository for {}", "logstash forwarder")
        debian.add_apt_repository("http://packages.elasticsearch.org/logstashforwarder/debian stable main")

        info("Installing {}", "logstash forwarder")
        debian.apt_get("update")
        debian.apt_get("install", "logstash-forwarder")

        # Upload init script
        blueprint.upload("forwarder/init.d/logstash-forwarder", "/etc/init.d/")
        debian.chmod("/etc/init.d/logstash-forwarder", mode=755)

        # Enable on boot
        debian.add_rc_service("logstash-forwarder")

def install():
    with sudo():
        # Generate a root password and save it in root home
        root_conf_path = '/root/.my.cnf'
        if not fabric.contrib.files.exists(root_conf_path):
            root_pw = generate_password()
            blueprint.upload('root_my.cnf', '/root/.my.cnf', {'password': root_pw})
            debian.chmod('/root/.my.cnf', mode=600)
        else:
            # TODO: use fabric.operations.get instead of cat when up to date with upstream
            with silent():
                output = run('cat {}'.format(root_conf_path))
            fd = StringIO(output)
            config_parser = ConfigParser.RawConfigParser()
            config_parser.readfp(fd)
            root_pw = config_parser.get('client', 'password')

        # Install external PPA
        info('Adding apt key for {}', __name__)
        run("apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A")

        info('Adding apt repository for {}', __name__)
        debian.add_apt_repository('http://repo.percona.com/apt trusty main')
        debian.apt_get('update')

        # Percona/MySQL base dependencies
        dependencies = (
            'percona-server-server',
            'percona-server-client',
            'libmysqlclient-dev',
            'mysqltuner'
        )

        # Configure debconf to autoset root password on installation prompts
        server_package = dependencies[0]
        debian.debconf_communicate('PURGE', server_package)
        with silent():
            debian.debconf_set_selections(
                '{}/root_password password {}'.format(server_package, root_pw),
                '{}/root_password_again password {}'.format(server_package, root_pw)
            )

        # Install package
        info('Installing {}', __name__)
        debian.apt_get('install', *dependencies)
        debian.debconf_communicate('PURGE', server_package)

        # Auto-answer mysql_secure_installation prompts
        prompts = {
            'Enter current password for root (enter for none): ': root_pw,
            'Change the root password? [Y/n] ': 'n',
            'Remove anonymous users? [Y/n] ': 'Y',
            'Disallow root login remotely? [Y/n] ': 'Y',
            'Remove test database and access to it? [Y/n] ': 'Y',
            'Reload privilege tables now? [Y/n] ': 'Y'

        }
        # Run mysql_secure_installation to remove test-db and remote root login
        with settings(prompts=prompts):
            run('mysql_secure_installation')

def configure_infra():
    with sudo():
        info('Adding license key to config')
        context = {"newrelic_key": blueprint.get('newrelic_key', None)}
        blueprint.upload('newrelic-infra.yml',
                         '/etc/newrelic-infra.yml',
                         context=context)

def enable(site, do_reload=True):
    """
    Enable site

    :param site: Site to enable
    :param do_reload: Reload nginx service
    :return: Got enabled?
    """
    enabled = False
    site = site if site.endswith('.conf') or site == 'default' else '{}.conf'.format(site)

    with sudo():
        available_site = os.path.join(sites_available_path, site)
        if not files.exists(available_site):
            warn('Invalid site: {}'.format(site))
        else:
            with cd(sites_enabled_path):
                if not files.exists(site):
                    info('Enabling site: {}', site)
                    with silent():
                        debian.ln(available_site, site)
                        enabled = True
                    if do_reload:
                        reload()

    return enabled

def add_apt_ppa(name, accept=True, src=False):
    with sudo(), fabric.context_managers.cd('/etc/apt/sources.list.d'):
        source_list = '%s-%s.list' % (name.replace('/', '-'), lsb_codename())

        if not fabric.contrib.files.exists(source_list):
            add_apt_repository('ppa:{}'.format(name), accept=accept, src=src)
            apt_get_update()

def install_user():
    with sudo():
        user.create_service_user("solr", home=solr_home)
        # Home dir will be created by solr installation (symbolic link)

        debian.mkdir("/var/lib/solr", mode=755, owner="solr", group="solr")
        debian.mkdir("/var/log/solr", mode=755, owner="solr", group="solr")

def setup():
    """
    Setup Logstash server
    """
    from .elasticsearch import add_elastic_repo

    with sudo():
        branch = blueprint.get('branch', '6.x')
        add_elastic_repo(branch)

        version = blueprint.get('version', 'latest')
        info('Installing {} version {}', 'logstash', version)
        package = 'logstash' + ('={}'.format(version) if version != 'latest' else '')
        debian.apt_get('install', package)

        # Enable on boot
        debian.add_rc_service('logstash')

        # prep custom folders
        debian.mkdir(conf_available_path)
        debian.mkdir(conf_enabled_path)

        # Install plugins
        plugins = blueprint.get('plugins', [])
        for plugin in plugins:
            info('Installing logstash "{}" plugin...', plugin)
            install_plugin(plugin)

    configure()

def setup():
    """
    Install nfs-common
    """
    with sudo():
        debian.apt_get('install', 'nfs-common')
    configure()

def install_solr():
    with sudo():
        version = blueprint.get('version')
        version_tuple = tuple(map(int, version.split('.')))

        archive = 'solr-{}.tgz'.format(version)
        if version_tuple < (4, 1, 0):
            archive = 'apache-{}'.format(archive)
        url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(version, archive)

        with cd('/tmp'):
            info('Download {} ({})', 'Solr', version)
            run('wget {}'.format(url))

            info('Extracting archive...')
            with silent():
                run('tar xzf {}'.format(archive))
                solr_version_dir = os.path.splitext(archive)[0]
                solr_version_path = os.path.join('/usr', 'share', solr_version_dir)
                debian.chmod(solr_version_dir, 755, 'solr', 'solr', recursive=True)
                if files.exists(solr_version_path):
                    info('Found same existing version, removing it...')
                    debian.rm(solr_version_path, recursive=True)
                debian.mv(solr_version_dir, '/usr/share/')
                debian.ln(solr_version_path, solr_home)
                debian.rm(archive)

def generate_pgtune_conf(role='db'):
    """
    Run pgtune and create pgtune.conf

    :param role: Which fabric role to place local pgtune.conf template under
    """
    conf_path = postgres_root('postgresql.conf')
    with sudo(), silent():
        output = run('pgtune -T Web -i {}'.format(conf_path)).strip()

        def parse(c):
            lines = [l for l in c.splitlines() if '# pgtune' in l]
            for line in lines:
                try:
                    comment = line.index('#')
                    line = line[:comment]
                except ValueError:
                    pass
                clean = lambda s: s.strip('\n\r\t\'" ')
                key, _, value = line.partition('=')
                key, value = map(clean, (key, value))
                if key:
                    yield key, value or None

        tune_conf = dict(parse(output))
        tune_conf.update(blueprint.get('pgtune', {}))
        tune_conf = '\n'.join((' = '.join(item)) for item in tune_conf.iteritems())
        conf_dir = os.path.join(os.path.dirname(env['real_fabfile']), 'templates', role, 'postgres')
        conf_path = os.path.join(conf_dir, 'pgtune.conf')

        if not os.path.exists(conf_dir):
            os.makedirs(conf_dir)

        with open(conf_path, 'w+') as f:
            f.write(tune_conf)

def install():
    with sudo():
        from blues import java
        java.install()

        version = blueprint.get('version', '1.0')
        info('Adding apt repository for {} version {}', 'elasticsearch', version)
        repository = 'http://packages.elasticsearch.org/elasticsearch/{0}/debian stable main'.format(version)
        debian.add_apt_repository(repository)

        info('Adding apt key for', repository)
        debian.add_apt_key('http://packages.elasticsearch.org/GPG-KEY-elasticsearch')
        debian.apt_get('update')

        # Install elasticsearch (and java)
        info('Installing {} version {}', 'elasticsearch', version)
        debian.apt_get('install', 'elasticsearch')

        # Install plugins
        plugins = blueprint.get('plugins', [])
        for plugin in plugins:
            info('Installing elasticsearch "{}" plugin...', plugin)
            install_plugin(plugin)

        # Enable on boot
        debian.add_rc_service('elasticsearch', priorities='defaults 95 10')

def create_app_root():
    from .project import app_root

    with sudo():
        # Create global apps root
        root_path = app_root()
        debian.mkdir(root_path, recursive=True)

def install_project_user():
    """
    Create project user and groups.
    Create user home dir.
    Disable ssh host checking.
    Create log dir.
    """
    from .project import project_home

    with sudo():
        info('Install application user')
        username = blueprint.get('project')
        home_path = project_home()

        # Setup groups for project user
        project_user_groups = ['app-data', 'www-data']
        for group in project_user_groups:
            debian.groupadd(group, gid_min=10000)

        # Get UID for project user
        user.create_system_user(username, groups=project_user_groups,
                                home=home_path)

        # Create application log path
        application_log_path = os.path.join('/var', 'log', username)
        debian.mkdir(application_log_path, group='app-data', mode=1775)

        # Configure ssh for github
        user.set_strict_host_checking(username, 'github.com')

def add_fstab(filesystem=None, mount_point=None, type='auto', options='rw', dump='0', pazz='0'):
    """
    Add mount point configuration to /etc/fstab.
    If mount point already mounted on different file system then unmount.

    :param str filesystem: The partition or storage device to be mounted
    :param str mount_point: The mount point where <filesystem> is mounted to
    :param str type: The file system type (Default: auto)
    :param str options: Mount options of the filesystem (Default: rw)
    :param str dump: Used by the dump utility to decide when to make a backup, 0|1 (Default: 0)
    :param str pazz: Used by fsck to decide which order filesystems are to be checked (Default: 0)
    """
    with sudo():
        fstab_line = '{fs} {mount} {type} {options} {dump} {pazz}'.format(
            fs=filesystem,
            mount=mount_point,
            type=type,
            options=options,
            dump=dump,
            pazz=pazz
        )
        validate_boot_options(options)

        # Add mount to /etc/fstab if not already there (?)
        with silent():
            output = run('cat /etc/fstab')
            fstab = output.stdout
            if fstab_line not in fstab.split('\n'):  # TODO: Handle comments
                info('Adding fstab: {} on {}', filesystem, mount_point)
                fabric.contrib.files.append('/etc/fstab', fstab_line, use_sudo=True)

        # Unmount any previous mismatching mount point
        mounted_file_system = get_mount(mount_point)
        if mounted_file_system and mounted_file_system != filesystem:
            unmount(mount_point)

def install():
    with sudo():
        info('Adding apt repository for {}', 'docker')
        repository = '[arch=amd64] https://download.docker.com/linux/ubuntu {} stable'.format(
            debian.lsb_codename())
        debian.add_apt_repository(repository)

        gpg_key = 'https://download.docker.com/linux/ubuntu/gpg'
        info('Adding apt key from {}', gpg_key)
        debian.add_apt_key(gpg_key)
        debian.apt_get_update()

        # Install Docker
        version = blueprint.get('version', 'latest')
        info('Target version {}', version)
        package = 'docker-ce' + ('={}'.format(version) if version != 'latest' else '')

        info('Installing {}', package)
        debian.apt_get('install', package)

        if debian.lsb_release() == '14.04':
            blueprint.upload('./docker.default', '/etc/default/docker')

        else:
            blueprint.upload('./docker.service',
                             '/etc/systemd/system/docker.service')
            debian.ln(
                '/etc/systemd/system/docker.service',
                '/etc/systemd/system/multi-user.target.wants/docker.service',
            )
            debian.systemd_daemon_reload()

def configure():
    """
    Configure nginx and enable/disable sites
    """
    with sudo():
        # Upload templates
        context = {
            'num_cores': debian.nproc()
        }
        uploads = blueprint.upload('./', nginx_root, context)

        # Disable previously enabled sites not configured sites-enabled
        changes = []
        sites = blueprint.get('sites')
        auto_disable_sites = blueprint.get('auto_disable_sites', True)
        if auto_disable_sites:
            with silent():
                enabled_site_links = run('ls {}'.format(sites_enabled_path)).split()
            for link in enabled_site_links:
                link_name = os.path.splitext(link)[0]  # Without extension
                if link not in sites and link_name not in sites:
                    changed = disable(link, do_reload=False)
                    changes.append(changed)

        ### Enable sites from settings
        for site in sites:
            changed = enable(site, do_reload=False)
            changes.append(changed)

        ### Reload nginx if new templates or any site has been enabled/disabled
        if uploads or any(changes):
            reload()