def test_get_package_version_rpm_command(self):
     remote = Mock()
     remote.os.package_type = "rpm"
     packaging.get_package_version(remote, "httpd")
     args, kwargs = remote.run.call_args
     expected_args = ["rpm", "-q", "httpd", "--qf", "%{VERSION}"]
     assert expected_args == kwargs["args"]

 def test_get_package_version_deb_command(self):
     remote = Mock()
     remote.os.package_type = "deb"
     packaging.get_package_version(remote, "apache2")
     args, kwargs = remote.run.call_args
     expected_args = ['dpkg-query', '-W', '-f', '${Version}', 'apache2']
     assert expected_args == kwargs['args']

def rh_install_pkgs(ctx, remote, installed_version):
    """
    Installs RH build using ceph-deploy.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    """
    pkgs = ['ceph-deploy']
    # install ceph-selinux for 1.3.2 as its not dependency of any core packages
    if (installed_version == '1.3.2'):
        pkgs.append('ceph-selinux')
    # install ceph-fuse for 2.0 as its not dependency of any core packages
    if (installed_version == '2.0'):
        pkgs.append('ceph-fuse')
    rh_version_check = {'0.94.1': '1.3.0', '0.94.3': '1.3.1',
                        '0.94.5': '1.3.2', '10.1.0': '2.0'}
    log.info("Remove any epel packages installed on node %s", remote.shortname)
    remote.run(args=['sudo', 'yum', 'remove', run.Raw("leveldb xmlstarlet fcgi"), '-y'],check_status=False)
    for pkg in pkgs:
        log.info("Check if %s is already installed on node %s", pkg, remote.shortname)
        remote.run(args=['sudo', 'yum', 'clean', 'metadata'])
        r = remote.run(
             args=['yum', 'list', 'installed', run.Raw(pkg)],
             stdout=StringIO(),
             check_status=False,
            )
        if r.stdout.getvalue().find(pkg) == -1:
            log.info("Installing %s " % pkg)
            remote.run(args=['sudo', 'yum', 'install', pkg, '-y'])
        else:
            log.info("Removing and reinstalling %s on %s", pkg, remote.shortname)
            remote.run(args=['sudo', 'yum', 'remove', pkg, '-y'])
            remote.run(args=['sudo', 'yum', 'install', pkg, '-y'])

    log.info("Check if ceph is already installed on %s", remote.shortname)
    r = remote.run(
          args=['yum', 'list', 'installed','ceph'],
          stdout=StringIO(),
          check_status=False,
        )
    host = r.hostname
    if r.stdout.getvalue().find('ceph') == -1:
        log.info("Install ceph using ceph-deploy on %s", remote.shortname)
        remote.run(args=['sudo', 'ceph-deploy', 'install', run.Raw('--no-adjust-repos'), host])
        remote.run(args=['sudo', 'yum', 'install', 'ceph-test', '-y'])
    else:
        log.info("Removing and reinstalling Ceph on %s", remote.shortname)
        remote.run(args=['sudo', 'ceph-deploy', 'uninstall', host])
        remote.run(args=['sudo', 'ceph-deploy', 'purgedata', host])
        remote.run(args=['sudo', 'ceph-deploy', 'install', host])
        remote.run(args=['sudo', 'yum', 'remove', 'ceph-test', '-y'])
        remote.run(args=['sudo', 'yum', 'install', 'ceph-test', '-y'])

    # check package version
    version = packaging.get_package_version(remote, 'ceph-common')
    log.info("Node: {n} Ceph version installed is {v}".format(n=remote.shortname,v=version))
    if rh_version_check[version] == installed_version:
        log.info("Installed version matches on %s", remote.shortname)
    else:
        raise RuntimeError("Version check failed on node %s", remote.shortname)

def install_pkgs(ctx, remote, version, downstream_config):
    """
    Installs RH build using ceph-deploy.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    :param downstream_config the dict object that has downstream pkg info
    """
    rh_version_check = downstream_config.get('versions').get('rpm').get('mapped')
    rh_rpm_pkgs = downstream_config.get('pkgs').get('rpm')
    pkgs = str.join(' ', rh_rpm_pkgs)
    log.info("Remove any epel packages installed on node %s", remote.shortname)
    # below packages can come from epel and still work, ensure we use cdn pkgs
    remote.run(
        args=[
            'sudo',
            'yum',
            'remove',
            run.Raw("leveldb xmlstarlet fcgi"),
            '-y'],
        check_status=False)
    log.info("Installing redhat ceph packages")
    remote.run(args=['sudo', 'yum', '-y', 'install',
                     run.Raw(pkgs)])
    # check package version
    installed_version = packaging.get_package_version(remote, 'ceph-common')
    log.info(
        "Node: {n} Ceph version installed is {v}".format(
            n=remote.shortname,
            v=version))
    req_ver = rh_version_check[version]
    if installed_version.startswith(req_ver):
        log.info("Installed version matches on %s", remote.shortname)
    else:
        raise RuntimeError("Version check failed on node %s", remote.shortname)

def _downgrade_packages(ctx, remote, pkgs, pkg_version, config):
    """
    Downgrade packages listed by 'downgrade_packages'

    Downgrade specified packages to given version. The list of packages
    downgrade is provided by 'downgrade_packages' as a property of "install"
    task.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    :param pkgs: list of package names to install
    :param pkg_version: the version to which all packages will be downgraded
    :param config: the config dict
    :return: list of package names from 'pkgs' which are not yet
             installed/downgraded
    """
    downgrade_pkgs = config.get('downgrade_packages', [])
    if not downgrade_pkgs:
        return pkgs
    log.info('Downgrading packages: {pkglist}'.format(
        pkglist=', '.join(downgrade_pkgs)))
    # assuming we are going to downgrade packages with the same version
    first_pkg = downgrade_pkgs[0]
    installed_version = packaging.get_package_version(remote, first_pkg)
    assert installed_version, "failed to get version of {}".format(first_pkg)
    assert LooseVersion(installed_version) > LooseVersion(pkg_version)
    # to compose package name like "librados2-0.94.10-87.g116a558.el7"
    pkgs_opt = ['-'.join([pkg, pkg_version]) for pkg in downgrade_pkgs]
    remote.run(args='sudo yum -y downgrade {}'.format(' '.join(pkgs_opt)))
    return [pkg for pkg in pkgs if pkg not in downgrade_pkgs]

def verify_package_version(ctx, config, remote):
    """
    Ensures that the version of package installed is what
    was asked for in the config.

    For most cases this is for ceph, but we also install samba
    for example.
    """
    # Do not verify the version if the ceph-deploy task is being used to
    # install ceph. Verifying the ceph installed by ceph-deploy should work,
    # but the qa suites will need reorganized first to run ceph-deploy
    # before the install task.
    # see: http://tracker.ceph.com/issues/11248
    if config.get("extras"):
        log.info("Skipping version verification...")
        return True
    gitbuilder = _get_gitbuilder_project(ctx, remote, config)
    version = gitbuilder.version
    pkg_to_check = gitbuilder.project
    installed_ver = packaging.get_package_version(remote, pkg_to_check)
    if installed_ver and version in installed_ver:
        msg = "The correct {pkg} version {ver} is installed.".format(ver=version, pkg=pkg_to_check)
        log.info(msg)
    else:
        raise RuntimeError(
            "{pkg} version {ver} was not installed, found {installed}.".format(
                ver=version, installed=installed_ver, pkg=pkg_to_check
            )
        )

def install_deb_pkgs(
        ctx,
        remote,
        version,
        downstream_config,
        deb_repo,
        deb_gpg_key):
    """
    Setup debian repo, Install gpg key
    and Install on debian packages
    : param ctx
    : param remote
    : param downstream_config the dict object that has downstream pkg info
    : deb_repo - http path of downstream ubuntu repo
    : deb_gpg_key - gpg key for the ubuntu pkg
    """
    set_deb_repo(remote, deb_repo, deb_gpg_key)
    rh_version_check = downstream_config.get('versions').get('deb').get('mapped')
    rh_deb_pkgs = downstream_config.get('pkgs').get('deb')
    pkgs = str.join(' ', rh_deb_pkgs)
    log.info("Installing redhat ceph packages")
    remote.run(args=['sudo', 'apt-get', '-y', 'install',
                     run.Raw(pkgs)])
    # check package version
    installed_version = packaging.get_package_version(remote, 'ceph-common')
    log.info(
        "Node: {n} Ceph version installed is {v}".format(
            n=remote.shortname,
            v=version))
    req_ver = rh_version_check[version]
    if installed_version.startswith(req_ver):
        log.info("Installed version matches on %s", remote.shortname)
    else:
        raise RuntimeError("Version check failed on node %s", remote.shortname)

 def test_get_package_version_not_found(self):
     remote = Mock()
     remote.os.package_type = "rpm"
     proc = Mock()
     proc.exitstatus = 1
     proc.stdout.getvalue.return_value = "not installed"
     remote.run.return_value = proc
     result = packaging.get_package_version(remote, "httpd")
     assert result is None

 def test_get_package_version_rpm_found(self):
     remote = Mock()
     remote.os.package_type = "rpm"
     proc = Mock()
     proc.exitstatus = 0
     proc.stdout.getvalue.return_value = "2.2"
     remote.run.return_value = proc
     result = packaging.get_package_version(remote, "httpd")
     assert result == "2.2"

    def test_filelock(self):
        """
        Check that file lock doesn't get lost after an MDS restart
        """
        a_version_str = get_package_version(self.mount_a.client_remote, "fuse")
        b_version_str = get_package_version(self.mount_b.client_remote, "fuse")
        flock_version_str = "2.9"

        version_regex = re.compile(r"[0-9\.]+")
        a_result = version_regex.match(a_version_str)
        self.assertTrue(a_result)
        b_result = version_regex.match(b_version_str)
        self.assertTrue(b_result)
        a_version = version.StrictVersion(a_result.group())
        b_version = version.StrictVersion(b_result.group())
        flock_version=version.StrictVersion(flock_version_str)

        flockable = False
        if (a_version >= flock_version and b_version >= flock_version):
            log.info("testing flock locks")
            flockable = True
        else:
            log.info("not testing flock locks, machines have versions {av} and {bv}".format(
                av=a_version_str,bv=b_version_str))

        lock_holder = self.mount_a.lock_background(do_flock=flockable)

        self.mount_b.wait_for_visible("background_file-2")
        self.mount_b.check_filelock(do_flock=flockable)

        self.fs.mds_fail_restart()
        self.fs.wait_for_state('up:active', timeout=MDS_RESTART_GRACE)

        self.mount_b.check_filelock(do_flock=flockable)

        # Tear down the background process
        lock_holder.stdin.close()
        try:
            lock_holder.wait()
        except (CommandFailedError, ConnectionLostError):
            # We killed it, so it raises an error
            pass

 def test_get_package_version_invalid_version(self):
     # this tests the possibility that the package is not found
     # but the exitstatus is still 0.  Not entirely sure we'll ever
     # hit this condition, but I want to test the codepath regardless
     remote = Mock()
     remote.os.package_type = "rpm"
     proc = Mock()
     proc.exitstatus = 0
     proc.stdout.getvalue.return_value = "not installed"
     remote.run.return_value = proc
     result = packaging.get_package_version(remote, "httpd")
     assert result is None

    def _is_flockable(self):
        a_version_str = get_package_version(self.mount_a.client_remote, "fuse")
        b_version_str = get_package_version(self.mount_b.client_remote, "fuse")
        flock_version_str = "2.9"

        version_regex = re.compile(r"[0-9\.]+")
        a_result = version_regex.match(a_version_str)
        self.assertTrue(a_result)
        b_result = version_regex.match(b_version_str)
        self.assertTrue(b_result)
        a_version = version.StrictVersion(a_result.group())
        b_version = version.StrictVersion(b_result.group())
        flock_version=version.StrictVersion(flock_version_str)

        if (a_version >= flock_version and b_version >= flock_version):
            log.info("flock locks are available")
            return True
        else:
            log.info("not testing flock locks, machines have versions {av} and {bv}".format(
                av=a_version_str,bv=b_version_str))
            return False

def rh_install_pkgs(ctx, remote, installed_version):
    """
    Installs RH build using ceph-deploy.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    """
    pkgs = ['ceph-deploy']
    rh_version_check = {'0.94.1': '1.3.0', '0.94.3': '1.3.1'}
    for pkg in pkgs:
        log.info("Check if ceph-deploy is already installed on node %s", remote.shortname)
        remote.run(args=['sudo', 'yum', 'clean', 'metadata'])
        r = remote.run(
             args=['yum', 'list', 'installed', run.Raw(pkg)],
             stdout=StringIO(),
             check_status=False,
            )
        if r.stdout.getvalue().find(pkg) == -1:
            log.info("Installing %s " % pkg)
            remote.run(args=['sudo', 'yum', 'install', pkg, '-y'])
        else:
            log.info("Removing and reinstalling ceph-deploy on %s", remote.shortname)
            remote.run(args=['sudo', 'yum', 'remove', pkg, '-y'])
            remote.run(args=['sudo', 'yum', 'install', pkg, '-y'])

    log.info("Check if ceph is already installed on %s", remote.shortname)
    r = remote.run(
          args=['yum', 'list', 'installed','ceph'],
          stdout=StringIO(),
          check_status=False,
        )
    host = r.hostname
    if r.stdout.getvalue().find('ceph') == -1:
        log.info("Install ceph using ceph-deploy on %s", remote.shortname)
        remote.run(args=['sudo', 'ceph-deploy', 'install', run.Raw('--no-adjust-repos'), host])
        remote.run(args=['sudo', 'yum', 'install', 'ceph-test', '-y'])
    else:
        log.info("Removing and reinstalling Ceph on %s", remote.shortname)
        remote.run(args=['sudo', 'ceph-deploy', 'uninstall', host])
        remote.run(args=['sudo', 'ceph-deploy', 'purgedata', host])
        remote.run(args=['sudo', 'ceph-deploy', 'install', host])
        remote.run(args=['sudo', 'yum', 'remove', 'ceph-test', '-y'])
        remote.run(args=['sudo', 'yum', 'install', 'ceph-test', '-y'])

    # check package version
    version = packaging.get_package_version(remote, 'ceph')
    log.info("Node: {n} Ceph version installed is {v}".format(n=remote.shortname,v=version))
    if rh_version_check[version] == installed_version:
        log.info("Installed version matches on %s", remote.shortname)
    else:
        raise RuntimeError("Version check failed on node %s", remote.shortname)

def rh_install_pkgs(ctx, remote, installed_version):
    """
    Installs RH build using ceph-deploy.

    :param ctx: the argparse.Namespace object
    :param remote: the teuthology.orchestra.remote.Remote object
    """
    pkgs = ["ceph-deploy"]
    rh_version_check = {"0.94.1": "1.3.0", "0.94.2": "1.3.1"}
    for pkg in pkgs:
        log.info("Check if ceph-deploy is already installed on node %s", remote.shortname)
        remote.run(args=["sudo", "yum", "clean", "metadata"])
        r = remote.run(args=["yum", "list", "installed", run.Raw(pkg)], stdout=StringIO(), check_status=False)
        if r.stdout.getvalue().find(pkg) == -1:
            log.info("Installing %s " % pkg)
            remote.run(args=["sudo", "yum", "install", pkg, "-y"])
        else:
            log.info("Removing and reinstalling ceph-deploy on %s", remote.shortname)
            remote.run(args=["sudo", "yum", "remove", pkg, "-y"])
            remote.run(args=["sudo", "yum", "install", pkg, "-y"])

    log.info("Check if ceph is already installed on %s", remote.shortname)
    r = remote.run(args=["yum", "list", "installed", "ceph"], stdout=StringIO(), check_status=False)
    host = r.hostname
    if r.stdout.getvalue().find("ceph") == -1:
        log.info("Install ceph using ceph-deploy on %s", remote.shortname)
        remote.run(args=["sudo", "ceph-deploy", "install", run.Raw("--no-adjust-repos"), host])
        remote.run(args=["sudo", "yum", "install", "ceph-test", "-y"])
    else:
        log.info("Removing and reinstalling Ceph on %s", remote.shortname)
        remote.run(args=["sudo", "ceph-deploy", "uninstall", host])
        remote.run(args=["sudo", "ceph-deploy", "purgedata", host])
        remote.run(args=["sudo", "ceph-deploy", "install", host])
        remote.run(args=["sudo", "yum", "remove", "ceph-test", "-y"])
        remote.run(args=["sudo", "yum", "install", "ceph-test", "-y"])

    # check package version
    version = packaging.get_package_version(remote, "ceph")
    log.info("Node: {n} Ceph version installed is {v}".format(n=remote.shortname, v=version))
    if rh_version_check[version] == installed_version:
        log.info("Installed version matches on %s", remote.shortname)
    else:
        raise RuntimeError("Version check failed on node %s", remote.shortname)

def upgrade_common(ctx, config, deploy_style):
    """
    Common code for upgrading
    """
    remotes = upgrade_remote_to_config(ctx, config)
    project = config.get('project', 'ceph')

    extra_pkgs = config.get('extra_packages', [])
    log.info('extra packages: {packages}'.format(packages=extra_pkgs))

    for remote, node in remotes.iteritems():

        system_type = teuthology.get_system_type(remote)
        assert system_type in ('deb', 'rpm')
        pkgs = get_package_list(ctx, config)[system_type]
        log.info("Upgrading {proj} {system_type} packages: {pkgs}".format(
            proj=project, system_type=system_type, pkgs=', '.join(pkgs)))
        if isinstance(extra_pkgs, dict):
            pkgs += extra_pkgs.get(system_type, [])
        else:
            pkgs += extra_pkgs

        installed_version = packaging.get_package_version(remote, 'ceph-common')
        upgrade_version = get_upgrade_version(ctx, node, remote)
        log.info("Ceph {s} upgrade from {i} to {u}".format(
            s=system_type,
            i=installed_version,
            u=upgrade_version
        ))
	if _upgrade_is_downgrade(installed_version, upgrade_version):
            raise RuntimeError(
                "An attempt to upgrade from a higher version to a lower one "
                "will always fail. Hint: check tags in the target git branch."
            )


        deploy_style(ctx, node, remote, pkgs, system_type)
        verify_package_version(ctx, node, remote)
    return len(remotes)