def set_libvirt_secret(devstack_node, ceph_node):
    log.info("Setting libvirt secret...")

    cinder_key_stringio = StringIO()
    ceph_node.run(args=["ceph", "auth", "get-key", "client.cinder"], stdout=cinder_key_stringio)
    cinder_key = cinder_key_stringio.getvalue().strip()

    uuid_stringio = StringIO()
    devstack_node.run(args=["uuidgen"], stdout=uuid_stringio)
    uuid = uuid_stringio.getvalue().strip()

    secret_path = "/tmp/secret.xml"
    secret_template = textwrap.dedent(
        """
    <secret ephemeral='no' private='no'>
        <uuid>{uuid}</uuid>
        <usage type='ceph'>
            <name>client.cinder secret</name>
        </usage>
    </secret>"""
    )
    misc.sudo_write_file(devstack_node, secret_path, secret_template.format(uuid=uuid))
    devstack_node.run(args=["sudo", "virsh", "secret-define", "--file", secret_path])
    devstack_node.run(args=["sudo", "virsh", "secret-set-value", "--secret", uuid, "--base64", cinder_key])
    return uuid

def setup_dnsmasq(client, name):
    """
    Setup simple dnsmasq name eg: s3.ceph.com
    Local RGW host can then be used with whatever name has been setup with.
    """
    resolv_conf = "nameserver 127.0.0.1\n"
    dnsmasq_template = """address=/{name}/{ip_address}
server=8.8.8.8
server=8.8.4.4
""".format(name=name, ip_address=client.ip_address)
    dnsmasq_config_path = '/etc/dnsmasq.d/ceph'
    # point resolv.conf to local dnsmasq
    misc.sudo_write_file(
        remote=client,
        path='/etc/resolv.conf',
        data=resolv_conf,
    )
    misc.sudo_write_file(
        remote=client,
        path=dnsmasq_config_path,
        data=dnsmasq_template,
    )
    client.run(args=['cat', dnsmasq_config_path])
    # restart dnsmasq
    client.run(args=['sudo', 'systemctl', 'restart', 'dnsmasq'])
    client.run(args=['sudo', 'systemctl', 'status', 'dnsmasq'])
    time.sleep(5)
    # verify dns name is set
    client.run(args=['ping', '-c', '4', name])

def set_libvirt_secret(devstack_node, ceph_node):
    log.info("Setting libvirt secret...")

    cinder_key_stringio = StringIO()
    ceph_node.run(args=['sudo', 'ceph', 'auth', 'get-key', 'client.cinder'],
                  stdout=cinder_key_stringio)
    cinder_key = cinder_key_stringio.getvalue().strip()

    uuid_stringio = StringIO()
    devstack_node.run(args=['uuidgen'], stdout=uuid_stringio)
    uuid = uuid_stringio.getvalue().strip()

    secret_path = '/tmp/secret.xml'
    secret_template = textwrap.dedent("""
    <secret ephemeral='no' private='no'>
        <uuid>{uuid}</uuid>
        <usage type='ceph'>
            <name>client.cinder secret</name>
        </usage>
    </secret>""")
    misc.sudo_write_file(devstack_node, secret_path,
                         secret_template.format(uuid=uuid))
    devstack_node.run(args=['sudo', 'virsh', 'secret-define', '--file',
                            secret_path])
    devstack_node.run(args=['sudo', 'virsh', 'secret-set-value', '--secret',
                            uuid, '--base64', cinder_key])
    return uuid

 def copy_key(from_remote, key_name, to_remote, dest_path, owner):
     key_stringio = StringIO()
     from_remote.run(
         args=['sudo', 'ceph', 'auth', 'get-or-create', key_name],
         stdout=key_stringio)
     key_stringio.seek(0)
     misc.sudo_write_file(to_remote, dest_path,
                          key_stringio, owner=owner)

def set_apache_servername(node):
    # Apache complains: "Could not reliably determine the server's fully
    # qualified domain name, using 127.0.0.1 for ServerName"
    # So, let's make sure it knows its name.
    log.info("Setting Apache ServerName...")

    hostname = node.hostname
    config_file = "/etc/apache2/conf.d/servername"
    misc.sudo_write_file(node, config_file, "ServerName {name}".format(name=hostname))

def ship_utilities(ctx, config):
    assert config is None
    testdir = teuthology.get_testdir(ctx)
    filenames = []

    log.info('Shipping valgrind.supp...')
    with file(os.path.join(os.path.dirname(__file__), 'valgrind.supp'), 'rb') as f:
        fn = os.path.join(testdir, 'valgrind.supp')
        filenames.append(fn)
        for rem in ctx.cluster.remotes.iterkeys():
            teuthology.sudo_write_file(
                remote=rem,
                path=fn,
                data=f,
                )
            f.seek(0)

    FILES = ['daemon-helper', 'adjust-ulimits', 'kcon_most']
    destdir = '/usr/bin'
    for filename in FILES:
        log.info('Shipping %r...', filename)
        src = os.path.join(os.path.dirname(__file__), filename)
        dst = os.path.join(destdir, filename)
        filenames.append(dst)
        with file(src, 'rb') as f:
            for rem in ctx.cluster.remotes.iterkeys():
                teuthology.sudo_write_file(
                    remote=rem,
                    path=dst,
                    data=f,
                )
                f.seek(0)
                rem.run(
                    args=[
                        'sudo',
                        'chmod',
                        'a=rx',
                        '--',
                        dst,
                    ],
                )

    try:
        yield
    finally:
        log.info('Removing shipped files: %s...', ' '.join(filenames))
        run.wait(
            ctx.cluster.run(
                args=[
                    'sudo',
                    'rm',
                    '-f',
                    '--',
                ] + list(filenames),
                wait=False,
            ),
        )

def install_repo(remote, reposerver, pkgdir, username=None, password=None):
    """
    Install a package repo for reposerver on remote.
    URL will be http if username and password are none, otherwise https.
    pkgdir is the piece path between "reposerver" and "deb" or "rpm"
     (say, 'packages', or 'packages-staging/my-branch', for example).
    so:
        http[s]://[<username>:<password>@]<reposerver>/<pkgdir>/{deb|rpm}
    will be written to deb's apt inktank.list or rpm's inktank.repo
    """

    relmap = _get_relmap(remote)
    log.info('Installing repo on %s', remote)
    if username is None or password is None:
        repo_uri = 'http://{reposerver}/{pkgdir}'
    else:
        repo_uri = 'https://{username}:{password}@{reposerver}/{pkgdir}'

    if relmap['flavor'] == 'deb':
        contents = 'deb ' + repo_uri + '/deb {codename} main'
        contents = contents.format(username=username, password=password,
                                   reposerver=reposerver, pkgdir=pkgdir,
                                   codename=relmap['version'],)
        teuthology.sudo_write_file(remote,
                                   '/etc/apt/sources.list.d/inktank.list',
                                   contents)
        remote.run(args=['sudo',
                         'apt-get',
                         'install',
                         'apt-transport-https',
                         '-y'])
        result = remote.run(args=['sudo', 'apt-get', 'update', '-y'],
                            stdout=StringIO())
        return result

    elif relmap['flavor'] == 'rpm':
        baseurl = repo_uri + '/rpm/{release}{version}'
        contents = textwrap.dedent('''
            [inktank]
            name=Inktank Storage, Inc.
            baseurl={baseurl}
            gpgcheck=1
            enabled=1
            '''.format(baseurl=baseurl))
        contents = contents.format(username=username,
                                   password=password,
                                   pkgdir=pkgdir,
                                   release=relmap['release'],
                                   version=relmap['version'])
        teuthology.sudo_write_file(remote,
                                   '/etc/yum.repos.d/inktank.repo',
                                   contents)
        return remote.run(args=['sudo', 'yum', 'makecache'])

    else:
        return False

def install_distro_kernel(remote):
    """
    RPM: Find newest kernel on the machine and update grub to use kernel + reboot.
    DEB: Find newest kernel. Parse grub.cfg to figure out the entryname/subentry.
    then modify 01_ceph_kernel to have correct entry + updategrub + reboot.
    """
    system_type = teuthology.get_system_type(remote)
    distribution = ''
    if system_type == 'rpm':
        output, err_mess = StringIO(), StringIO()
        remote.run(args=['rpm', '-q', 'kernel', '--last' ], stdout=output, stderr=err_mess )
        newest=output.getvalue().split()[0].split('kernel-')[1]
        log.info('Distro Kernel Version: {version}'.format(version=newest))
        update_grub_rpm(remote, newest)
        remote.run( args=['sudo', 'shutdown', '-r', 'now'], wait=False )
        output.close()
        err_mess.close()
        return

    if system_type == 'deb':
        distribution = teuthology.get_system_type(remote, distro=True)
        newversion = get_version_from_pkg(remote, distribution)
        if 'ubuntu' in distribution:
            grub2conf = teuthology.get_file(remote, '/boot/grub/grub.cfg', True)
            submenu = ''
            menuentry = ''
            for line in grub2conf.split('\n'):
                if 'submenu' in line:
                    submenu = line.split('submenu ')[1]
                    # Ubuntu likes to be sneaky and change formatting of
                    # grub.cfg between quotes/doublequotes between versions
                    if submenu.startswith("'"):
                        submenu = submenu.split("'")[1]
                    if submenu.startswith('"'):
                        submenu = submenu.split('"')[1]
                if 'menuentry' in line:
                    if newversion in line and 'recovery' not in line:
                        menuentry = line.split('\'')[1]
                        break
            if submenu:
                grubvalue = submenu + '>' + menuentry
            else:
                grubvalue = menuentry
            grubfile = 'cat <<EOF\nset default="' + grubvalue + '"\nEOF'
            teuthology.delete_file(remote, '/etc/grub.d/01_ceph_kernel', sudo=True, force=True)
            teuthology.sudo_write_file(remote, '/etc/grub.d/01_ceph_kernel', StringIO(grubfile), '755')
            log.info('Distro Kernel Version: {version}'.format(version=newversion))
            remote.run(args=['sudo', 'update-grub'])
            remote.run(args=['sudo', 'shutdown', '-r', 'now'], wait=False )
            return

        if 'debian' in distribution:
            grub2_kernel_select_generic(remote, newversion, 'deb')
            log.info('Distro Kernel Version: {version}'.format(version=newversion))
            remote.run( args=['sudo', 'shutdown', '-r', 'now'], wait=False )
            return

def run_rbd_map(remote, image, iodepth):
    iodepth = max(iodepth, 128)  # RBD_QUEUE_DEPTH_DEFAULT
    out = StringIO.StringIO()
    remote.run(args=['sudo', 'rbd', 'map', '-o', 'queue_depth={}'.format(iodepth), image], stdout=out)
    dev = out.getvalue().rstrip('\n')
    teuthology.sudo_write_file(
        remote,
        '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)),
        str(iodepth))
    return dev

def fix_yum_repos(remote, distro):
    """
    For yum calamari installations, the repos.d directory should only
    contain a repo file named rhel<version-number>.repo
    """
    if distro.startswith('centos'):
        # hack alert: detour: install lttng for ceph
        # this works because epel is preinstalled on the vpms
        # this is not a generic solution
        # this is here solely to test the one-off 1.3.0 release for centos6
        remote.run(args="sudo yum -y install lttng-tools")
        cmds = [
            'sudo mkdir /etc/yum.repos.d.old'.split(),
            ['sudo', 'cp', run.Raw('/etc/yum.repos.d/*'),
             '/etc/yum.repos.d.old'],
            ['sudo', 'rm', run.Raw('/etc/yum.repos.d/epel*')],
        ]
        for cmd in cmds:
            if remote.run(args=cmd).exitstatus:
                return False
    else:
        cmds = [
            'sudo mv /etc/yum.repos.d /etc/yum.repos.d.old'.split(),
            'sudo mkdir /etc/yum.repos.d'.split(),
        ]
        for cmd in cmds:
            if remote.run(args=cmd).exitstatus:
                return False

        # map "distroversion" from Remote.os to a tuple of
        # (repo title, repo name descriptor, apt-mirror repo path chunk)
        yum_repo_params = {
            'rhel 6.4': ('rhel6-server', 'RHEL', 'rhel6repo-server'),
            'rhel 6.5': ('rhel6-server', 'RHEL', 'rhel6repo-server'),
            'rhel 7.0': ('rhel7-server', 'RHEL', 'rhel7repo/server'),
        }
        repotitle, reponame, path = yum_repo_params[distro]
        repopath = '/etc/yum.repos.d/%s.repo' % repotitle
        # TO DO:  Make this data configurable too
        repo_contents = '\n'.join(
            ('[%s]' % repotitle,
             'name=%s $releasever - $basearch' % reponame,
             'baseurl=http://apt-mirror.front.sepia.ceph.com/' + path,
             'gpgcheck=0',
             'enabled=1')
        )
        misc.sudo_write_file(remote, repopath, repo_contents)
    cmds = [
        'sudo yum clean all'.split(),
        'sudo yum makecache'.split(),
    ]
    for cmd in cmds:
        if remote.run(args=cmd).exitstatus:
            return False
    return True

def _get_local_dir(config, remote):
    """
    Extract local directory name from the task lists.
    Copy files over to the remote site.
    """
    ldir = config.get('local', None)
    if ldir:
        remote.run(args=['sudo', 'mkdir', '-p', ldir,])
        for fyle in os.listdir(ldir):
            fname = "%s/%s" % (ldir, fyle)
            teuthology.sudo_write_file(remote, fname, open(fname).read(), '644')
    return ldir

def _get_local_dir(config, remote):
    """
    Extract local directory name from the task lists.
    Copy files over to the remote site.
    """
    ldir = config.get("local", None)
    if ldir:
        remote.run(args=["sudo", "mkdir", "-p", ldir])
        for fyle in os.listdir(ldir):
            fname = "%s/%s" % (ldir, fyle)
            teuthology.sudo_write_file(remote, fname, open(fname).read(), "644")
    return ldir

def _disable_default_nginx(remote):
    """
    Fix up nginx values
    """
    script = textwrap.dedent('''
        if [ -f /etc/nginx/conf.d/default.conf ]; then
            mv /etc/nginx/conf.d/default.conf \
                /etc/nginx/conf.d/default.disabled
        fi
        if [ -f /etc/nginx/sites-enabled/default ] ; then
            rm /etc/nginx/sites-enabled/default
        fi
        service nginx restart
        service {service} restart
    ''')
    service = pkg.get_service_name('httpd', remote)
    script = script.format(service=service)
    teuthology.sudo_write_file(remote, '/tmp/disable.nginx', script)
    return remote.run(args=['sudo', 'bash', '/tmp/disable.nginx'],
                      stdout=StringIO())

def setup_dnsmasq(remote, cnames):
    """ configure dnsmasq on the given remote, adding each cname given """
    log.info('Configuring dnsmasq on remote %s..', remote.name)

    # back up existing resolv.conf
    resolv_conf = misc.get_file(remote, '/etc/resolv.conf')
    # point resolv.conf to local dnsmasq
    misc.sudo_write_file(remote, '/etc/resolv.conf',
                         "nameserver 127.0.0.1\n")

    # add address entries to /etc/dnsmasq.d/ceph
    dnsmasq = "server=8.8.8.8\nserver=8.8.4.4\n"
    address_template = "address=/{cname}/{ip_address}\n"
    for cname, ip_address in cnames.iteritems():
        dnsmasq += address_template.format(cname=cname, ip_address=ip_address)
    misc.sudo_write_file(remote, '/etc/dnsmasq.d/ceph', dnsmasq)

    remote.run(args=['cat', '/etc/dnsmasq.d/ceph'])
    # restart dnsmasq
    remote.run(args=['sudo', 'systemctl', 'restart', 'dnsmasq'])
    remote.run(args=['sudo', 'systemctl', 'status', 'dnsmasq'])
    # verify dns name is set
    remote.run(args=['ping', '-c', '4', cnames.keys()[0]])

    yield

    log.info('Removing dnsmasq configuration from remote %s..', remote.name)
    # restore resolv.conf
    misc.sudo_write_file(remote, '/etc/resolv.conf', resolv_conf)
    # restart dnsmasq
    remote.run(args=['sudo', 'systemctl', 'restart', 'dnsmasq'])

def ship_utilities(ctx, config):
    """
    Write a copy of valgrind.supp to each of the remote sites.  Set executables used
    by Ceph in /usr/local/bin.  When finished (upon exit of the teuthology run), remove
    these files.

    :param ctx: Context
    :param config: Configuration
    """
    assert config is None
    testdir = teuthology.get_testdir(ctx)
    filenames = []

    log.info("Shipping valgrind.supp...")
    with file(os.path.join(os.path.dirname(__file__), "valgrind.supp"), "rb") as f:
        fn = os.path.join(testdir, "valgrind.supp")
        filenames.append(fn)
        for rem in ctx.cluster.remotes.iterkeys():
            teuthology.sudo_write_file(remote=rem, path=fn, data=f)
            f.seek(0)

    FILES = ["daemon-helper", "adjust-ulimits"]
    destdir = "/usr/bin"
    for filename in FILES:
        log.info("Shipping %r...", filename)
        src = os.path.join(os.path.dirname(__file__), filename)
        dst = os.path.join(destdir, filename)
        filenames.append(dst)
        with file(src, "rb") as f:
            for rem in ctx.cluster.remotes.iterkeys():
                teuthology.sudo_write_file(remote=rem, path=dst, data=f)
                f.seek(0)
                rem.run(args=["sudo", "chmod", "a=rx", "--", dst])

    try:
        yield
    finally:
        log.info("Removing shipped files: %s...", " ".join(filenames))
        run.wait(ctx.cluster.run(args=["sudo", "rm", "-f", "--"] + list(filenames), wait=False))

def update_grub_rpm(remote, newversion):
    """
    Updates grub file to boot new kernel version on both legacy grub/grub2.
    """
    grub='grub2'
    # Check if grub2 is isntalled
    try:
        remote.run(args=['sudo', 'rpm', '-qi', 'grub2'])
    except Exception:
        grub = 'legacy'
    log.info('Updating Grub Version: {grub}'.format(grub=grub))
    if grub == 'legacy':
        data = ''
        #Write new legacy grub entry.
        newgrub = generate_legacy_grub_entry(remote, newversion)
        for line in newgrub:
            data += line + '\n'
        temp_file_path = remote.mktemp()
        teuthology.sudo_write_file(remote, temp_file_path, StringIO(data), '755')
        teuthology.move_file(remote, temp_file_path, '/boot/grub/grub.conf', True)
    else:
        #Update grub menu entry to new version.
        grub2_kernel_select_generic(remote, newversion, 'rpm')

#.........这里部分代码省略.........
        admin_keyring = teuthology.get_file(
            remote=mon0_remote,
            path=admin_keyring_path,
            sudo=True,
            )

        clients = ctx.cluster.only(teuthology.is_type('client'))
        for remot, roles_for_host in clients.remotes.iteritems():
            for id_ in teuthology.roles_of_type(roles_for_host, 'client'):
                client_keyring = '/etc/ceph/ceph.client.{id}.keyring'.format(id=id_)
                mon0_remote.run(
                    args=[
                        'cd',
                        '{tdir}'.format(tdir=testdir),
                        run.Raw('&&'),
                        'sudo','bash','-c',
                        run.Raw('"'),'ceph',
                        'auth',
                        'get-or-create',
                        'client.{id}'.format(id=id_),
                        'mds', 'allow',
                        'mon', 'allow *',
                        'osd', 'allow *',
                        run.Raw('>'),
                        client_keyring,
                        run.Raw('"'),
                        ],
                    )
                key_data = teuthology.get_file(
                    remote=mon0_remote,
                    path=client_keyring,
                    sudo=True,
                    )
                teuthology.sudo_write_file(
                    remote=remot,
                    path=client_keyring,
                    data=key_data,
                    perms='0644'
                )
                teuthology.sudo_write_file(
                    remote=remot,
                    path=admin_keyring_path,
                    data=admin_keyring,
                    perms='0644'
                )
                teuthology.sudo_write_file(
                    remote=remot,
                    path=conf_path,
                    data=conf_data,
                    perms='0644'
                )
    else:
        raise RuntimeError("The cluster is NOT operational due to insufficient OSDs")

    try:
        yield

    finally:
        log.info('Stopping ceph...')
        ctx.cluster.run(args=[
                'sudo', 'stop', 'ceph-all',
                run.Raw('||'),
                'sudo', 'service', 'ceph', 'stop'
                ])

        if ctx.archive is not None:

#.........这里部分代码省略.........
                remote=mon0_remote,
                path=admin_keyring_path,
                sudo=True,
                )

            clients = ctx.cluster.only(teuthology.is_type('client'))
            for remot, roles_for_host in clients.remotes.iteritems():
                for id_ in teuthology.roles_of_type(roles_for_host, 'client'):
                    client_keyring = \
                        '/etc/ceph/ceph.client.{id}.keyring'.format(id=id_)
                    mon0_remote.run(
                        args=[
                            'cd',
                            '{tdir}'.format(tdir=testdir),
                            run.Raw('&&'),
                            'sudo', 'bash', '-c',
                            run.Raw('"'), 'ceph',
                            'auth',
                            'get-or-create',
                            'client.{id}'.format(id=id_),
                            'mds', 'allow',
                            'mon', 'allow *',
                            'osd', 'allow *',
                            run.Raw('>'),
                            client_keyring,
                            run.Raw('"'),
                            ],
                        )
                    key_data = teuthology.get_file(
                        remote=mon0_remote,
                        path=client_keyring,
                        sudo=True,
                        )
                    teuthology.sudo_write_file(
                        remote=remot,
                        path=client_keyring,
                        data=key_data,
                        perms='0644'
                    )
                    teuthology.sudo_write_file(
                        remote=remot,
                        path=admin_keyring_path,
                        data=admin_keyring,
                        perms='0644'
                    )
                    teuthology.sudo_write_file(
                        remote=remot,
                        path=conf_path,
                        data=conf_data,
                        perms='0644'
                    )
        else:
            raise RuntimeError(
                "The cluster is NOT operational due to insufficient OSDs")
        yield

    finally:
        log.info('Stopping ceph...')
        ctx.cluster.run(args=['sudo', 'stop', 'ceph-all', run.Raw('||'),
                              'sudo', 'service', 'ceph', 'stop' ])

        # Are you really not running anymore?
        # try first with the init tooling
        # ignoring the status so this becomes informational only
        ctx.cluster.run(args=['sudo', 'status', 'ceph-all', run.Raw('||'),
                              'sudo', 'service',  'ceph', 'status'],

def install_kernel(remote, path=None, version=None):
    """
    A bit of misnomer perhaps - the actual kernel package is installed
    elsewhere, this function deals with initrd and grub.  Currently the
    following cases are handled:
      - local, gitbuilder, distro for rpm packages
      - distro for deb packages - see TODO in install_and_reboot()

    TODO: reboots should be issued from install_and_reboot()

    :param path:    package path (for local and gitbuilder cases)
    :param version: for RPM distro kernels, pass this to update_grub_rpm
    """
    templ = "install_kernel(remote={remote}, path={path}, version={version})"
    log.debug(templ.format(remote=remote, path=path, version=version))
    package_type = remote.os.package_type
    if package_type == 'rpm':
        if path:
            version = get_image_version(remote, path)
            # This is either a gitbuilder or a local package and both of these
            # could have been built with upstream rpm targets with specs that
            # don't have a %post section at all, which means no initrd.
            maybe_generate_initrd_rpm(remote, path, version)
        elif not version or version == 'distro':
            version = get_latest_image_version_rpm(remote)
        update_grub_rpm(remote, version)
        remote.run( args=['sudo', 'shutdown', '-r', 'now'], wait=False )
        return

    if package_type == 'deb':
        distribution = remote.os.name
        newversion = get_latest_image_version_deb(remote, distribution)
        if 'ubuntu' in distribution:
            grub2conf = teuthology.get_file(remote, '/boot/grub/grub.cfg', True)
            submenu = ''
            menuentry = ''
            for line in grub2conf.split('\n'):
                if 'submenu' in line:
                    submenu = line.split('submenu ')[1]
                    # Ubuntu likes to be sneaky and change formatting of
                    # grub.cfg between quotes/doublequotes between versions
                    if submenu.startswith("'"):
                        submenu = submenu.split("'")[1]
                    if submenu.startswith('"'):
                        submenu = submenu.split('"')[1]
                if 'menuentry' in line:
                    if newversion in line and 'recovery' not in line:
                        menuentry = line.split('\'')[1]
                        break
            if submenu:
                grubvalue = submenu + '>' + menuentry
            else:
                grubvalue = menuentry
            grubfile = 'cat <<EOF\nset default="' + grubvalue + '"\nEOF'
            teuthology.delete_file(remote, '/etc/grub.d/01_ceph_kernel', sudo=True, force=True)
            teuthology.sudo_write_file(remote, '/etc/grub.d/01_ceph_kernel', StringIO(grubfile), '755')
            log.info('Distro Kernel Version: {version}'.format(version=newversion))
            remote.run(args=['sudo', 'update-grub'])
            remote.run(args=['sudo', 'shutdown', '-r', 'now'], wait=False )
            return

        if 'debian' in distribution:
            grub2_kernel_select_generic(remote, newversion, 'deb')
            log.info('Distro Kernel Version: {version}'.format(version=newversion))
            remote.run( args=['sudo', 'shutdown', '-r', 'now'], wait=False )
            return

def distribute_ceph_conf(devstack_node, ceph_node):
    log.info("Copying ceph.conf to DevStack node...")

    ceph_conf_path = "/etc/ceph/ceph.conf"
    ceph_conf = misc.get_file(ceph_node, ceph_conf_path, sudo=True)
    misc.sudo_write_file(devstack_node, ceph_conf_path, ceph_conf)