def check_floating_ip_status(self, floating_ip, status):
        """Verifies floatingip reaches the given status

        :param dict floating_ip: floating IP dict to check status
        :param status: target status
        :raises: AssertionError if status doesn't match
        """
        floatingip_id = floating_ip['id']

        def refresh():
            result = (self.floating_ips_client.
                      show_floatingip(floatingip_id)['floatingip'])
            return status == result['status']

        test_utils.call_until_true(refresh,
                                   CONF.network.build_timeout,
                                   CONF.network.build_interval)
        floating_ip = self.floating_ips_client.show_floatingip(
            floatingip_id)['floatingip']
        self.assertEqual(status, floating_ip['status'],
                         message="FloatingIP: {fp} is at status: {cst}. "
                                 "failed  to reach status: {st}"
                         .format(fp=floating_ip, cst=floating_ip['status'],
                                 st=status))
        LOG.info("FloatingIP: {fp} is at status: {st}"
                 .format(fp=floating_ip, st=status))

 def wait_zone_delete(self, zone_client, zone_id, **kwargs):
     zone_client.delete_zone(zone_id, **kwargs)
     utils.call_until_true(self._check_zone_deleted,
                           CONF.dns.build_timeout,
                           CONF.dns.build_interval,
                           zone_client,
                           zone_id)

    def _hotplug_server(self):
        old_floating_ip, server = self.floating_ip_tuple
        ip_address = old_floating_ip['floating_ip_address']
        private_key = self._get_server_key(server)
        ssh_client = self.get_remote_client(
            ip_address, private_key=private_key, server=server)
        old_nic_list = self._get_server_nics(ssh_client)
        # get a port from a list of one item
        port_list = self.os_admin.ports_client.list_ports(
            device_id=server['id'])['ports']
        self.assertEqual(1, len(port_list))
        old_port = port_list[0]
        interface = self.interface_client.create_interface(
            server_id=server['id'],
            net_id=self.new_net['id'])['interfaceAttachment']
        self.addCleanup(self.ports_client.wait_for_resource_deletion,
                        interface['port_id'])
        self.addCleanup(test_utils.call_and_ignore_notfound_exc,
                        self.interface_client.delete_interface,
                        server['id'], interface['port_id'])

        def check_ports():
            self.new_port_list = [
                port for port in
                self.os_admin.ports_client.list_ports(
                    device_id=server['id'])['ports']
                if port['id'] != old_port['id']
            ]
            return len(self.new_port_list) == 1

        if not test_utils.call_until_true(
                check_ports, CONF.network.build_timeout,
                CONF.network.build_interval):
            raise exceptions.TimeoutException(
                "No new port attached to the server in time (%s sec)! "
                "Old port: %s. Number of new ports: %d" % (
                    CONF.network.build_timeout, old_port,
                    len(self.new_port_list)))
        new_port = self.new_port_list[0]

        def check_new_nic():
            new_nic_list = self._get_server_nics(ssh_client)
            self.diff_list = [n for n in new_nic_list if n not in old_nic_list]
            return len(self.diff_list) == 1

        if not test_utils.call_until_true(
                check_new_nic, CONF.network.build_timeout,
                CONF.network.build_interval):
            raise exceptions.TimeoutException("Interface not visible on the "
                                              "guest after %s sec"
                                              % CONF.network.build_timeout)

        _, new_nic = self.diff_list[0]
        ssh_client.exec_command("sudo ip addr add %s/%s dev %s" % (
                                new_port['fixed_ips'][0]['ip_address'],
                                CONF.network.project_network_mask_bits,
                                new_nic))
        ssh_client.exec_command("sudo ip link set %s up" % new_nic)

    def _prepare_and_test(self, address6_mode, n_subnets6=1, dualnet=False):
        net_list = self.prepare_network(address6_mode=address6_mode,
                                        n_subnets6=n_subnets6,
                                        dualnet=dualnet)

        sshv4_1, ips_from_api_1, sid1 = self.prepare_server(networks=net_list)
        sshv4_2, ips_from_api_2, sid2 = self.prepare_server(networks=net_list)

        def guest_has_address(ssh, addr):
            return addr in ssh.exec_command("ip address")

        # Turn on 2nd NIC for Cirros when dualnet
        if dualnet:
            _, network_v6 = net_list
            self.turn_nic6_on(sshv4_1, sid1, network_v6['id'])
            self.turn_nic6_on(sshv4_2, sid2, network_v6['id'])

        # get addresses assigned to vNIC as reported by 'ip address' utility
        ips_from_ip_1 = sshv4_1.exec_command("ip address")
        ips_from_ip_2 = sshv4_2.exec_command("ip address")
        self.assertIn(ips_from_api_1['4'], ips_from_ip_1)
        self.assertIn(ips_from_api_2['4'], ips_from_ip_2)
        for i in range(n_subnets6):
            # v6 should be configured since the image supports it
            # It can take time for ipv6 automatic address to get assigned
            srv1_v6_addr_assigned = functools.partial(
                guest_has_address, sshv4_1, ips_from_api_1['6'][i])

            srv2_v6_addr_assigned = functools.partial(
                guest_has_address, sshv4_2, ips_from_api_2['6'][i])

            self.assertTrue(test_utils.call_until_true(srv1_v6_addr_assigned,
                            CONF.validation.ping_timeout, 1))

            self.assertTrue(test_utils.call_until_true(srv2_v6_addr_assigned,
                            CONF.validation.ping_timeout, 1))

        self.check_remote_connectivity(sshv4_1, ips_from_api_2['4'])
        self.check_remote_connectivity(sshv4_2, ips_from_api_1['4'])

        for i in range(n_subnets6):
            self.check_remote_connectivity(sshv4_1,
                                           ips_from_api_2['6'][i])
            self.check_remote_connectivity(sshv4_1,
                                           self.subnets_v6[i]['gateway_ip'])
            self.check_remote_connectivity(sshv4_2,
                                           ips_from_api_1['6'][i])
            self.check_remote_connectivity(sshv4_2,
                                           self.subnets_v6[i]['gateway_ip'])

def wait_node_instance_association(client, instance_uuid, timeout=None,
                                   interval=None):
    """Waits for a node to be associated with instance_id.

    :param client: an instance of tempest plugin BaremetalClient.
    :param instance_uuid: UUID of the instance.
    :param timeout: the timeout after which the check is considered as failed.
        Defaults to CONF.baremetal.association_timeout.
    :param interval: an interval between show_node calls for status check.
        Defaults to client.build_interval.
    """
    timeout, interval = _determine_and_check_timeout_interval(
        timeout, CONF.baremetal.association_timeout,
        interval, client.build_interval)

    def is_some_node_associated():
        node = utils.get_node(client, instance_uuid=instance_uuid)
        return node is not None

    if not test_utils.call_until_true(is_some_node_associated, timeout,
                                      interval):
        msg = ('Timed out waiting to get Ironic node by instance UUID '
               '%(instance_uuid)s within the required time (%(timeout)s s).'
               % {'instance_uuid': instance_uuid, 'timeout': timeout})
        raise lib_exc.TimeoutException(msg)

    def get_and_reserve_node(cls, node=None):
        """Pick an available node for deployment and reserve it.

        Only one instance_uuid may be associated, use this behaviour as
        reservation node when tests are launched concurrently. If node is
        not passed directly pick random available for deployment node.

        :param node: Ironic node to associate instance_uuid with.
        :returns: Ironic node.
        """
        instance_uuid = uuidutils.generate_uuid()
        nodes = []

        def _try_to_associate_instance():
            n = node or cls.get_random_available_node()
            try:
                cls._associate_instance_with_node(n['uuid'], instance_uuid)
                nodes.append(n)
            except lib_exc.Conflict:
                return False
            return True

        if (not test_utils.call_until_true(_try_to_associate_instance,
            duration=CONF.baremetal.association_timeout, sleep_for=1)):
            msg = ('Timed out waiting to associate instance to ironic node '
                   'uuid %s' % instance_uuid)
            raise lib_exc.TimeoutException(msg)

        return nodes[0]

    def verify_metadata(self):
        if self.run_ssh and CONF.compute_feature_enabled.metadata_service:
            # Verify metadata service
            md_url = 'http://169.254.169.254/latest/meta-data/public-ipv4'

            def exec_cmd_and_verify_output():
                cmd = 'curl ' + md_url
                result = self.ssh_client.exec_command(cmd)
                if result:
                    msg = ('Failed while verifying metadata on server. Result '
                           'of command "%s" is NOT "%s".' % (cmd, self.fip))
                    self.assertEqual(self.fip, result, msg)
                    return 'Verification is successful!'

            if not test_utils.call_until_true(exec_cmd_and_verify_output,
                                              CONF.compute.build_timeout,
                                              CONF.compute.build_interval):
                raise exceptions.TimeoutException('Timed out while waiting to '
                                                  'verify metadata on server. '
                                                  '%s is empty.' % md_url)

            # Also, test a POST
            md_url = 'http://169.254.169.254/openstack/2013-10-17/password'
            data = data_utils.arbitrary_string(100)
            cmd = 'curl -X POST -d ' + data + ' ' + md_url
            self.ssh_client.exec_command(cmd)
            result = self.servers_client.show_password(self.instance['id'])
            self.assertEqual(data, result['password'])

 def test_call_until_true_when_f_returns_true(self, m_time, m_sleep):
     timeout = 42  # The value doesn't matter as we mock time.time()
     sleep = 60  # The value doesn't matter as we mock time.sleep()
     m_time.return_value = 0
     self.assertEqual(
         True, test_utils.call_until_true(lambda: True, timeout, sleep)
     )
     self.assertEqual(0, m_sleep.call_count)
     self.assertEqual(1, m_time.call_count)

 def test_call_until_true_when_f_never_returns_true(self, m_time, m_sleep):
     timeout = 42  # The value doesn't matter as we mock time.time()
     sleep = 60  # The value doesn't matter as we mock time.sleep()
     m_time.side_effect = utils.generate_timeout_series(timeout)
     self.assertEqual(
         False, test_utils.call_until_true(lambda: False, timeout, sleep)
     )
     m_sleep.call_args_list = [mock.call(sleep)] * 2
     m_time.call_args_list = [mock.call()] * 2

    def wait_disassociate(self):
        cli = self.manager.compute_floating_ips_client

        def func():
            floating = (cli.show_floating_ip(self.floating['id'])
                        ['floating_ip'])
            return floating['instance_id'] is None

        if not test_utils.call_until_true(func, self.check_timeout,
                                          self.check_interval):
            raise RuntimeError("IP disassociate timeout!")

    def check_icmp_echo(self):
        self.logger.info("%s(%s): Pinging..",
                         self.server_id, self.floating['ip'])

        def func():
            return self.ping_ip_address(self.floating['ip'])
        if not test_utils.call_until_true(func, self.check_timeout,
                                          self.check_interval):
            raise RuntimeError("%s(%s): Cannot ping the machine.",
                               self.server_id, self.floating['ip'])
        self.logger.info("%s(%s): pong :)",
                         self.server_id, self.floating['ip'])

    def _run_and_wait(self, key, data, version,
                      content_type='application/json', headers=None):

        headers = base._get_headers(headers, content_type)

        def wait():
            return self.logs_search_client.count_search_messages(key, headers) > 0

        self.assertEqual(0, self.logs_search_client.count_search_messages(key, headers),
                         'Find log message in elasticsearch: {0}'.format(key))

        headers = base._get_headers(headers, content_type)
        data = base._get_data(data, content_type, version=version)

        response, _ = self.logs_clients[version].send_single_log(data, headers)
        self.assertEqual(204, response.status)

        test_utils.call_until_true(wait, _RETRY_COUNT * _RETRY_WAIT, _RETRY_WAIT)
        response = self.logs_search_client.search_messages(key, headers)
        self.assertEqual(1, len(response))

        return response

    def _wait_for_volume_available_on_the_system(self, ip_address,
                                                 private_key):
        ssh = self.get_remote_client(ip_address, private_key=private_key)

        def _func():
            part = ssh.get_partitions()
            LOG.debug("Partitions:%s" % part)
            return CONF.compute.volume_device_name in part

        if not test_utils.call_until_true(_func,
                                          CONF.compute.build_timeout,
                                          CONF.compute.build_interval):
            raise lib_exc.TimeoutException

    def _test_call_until_true(self, return_values, duration, time_sequence,
                              args=None, kwargs=None):
        """Test call_until_true function

        :param return_values: list of booleans values to be returned
        each time given function is called. If any of these values
        is not consumed by calling the function the test fails.
        The list must contain a sequence of False items terminated
        by a single True or False
        :param duration: parameter passed to call_until_true function
        (a floating point value).
        :param time_sequence: sequence of time values returned by
        mocked time.time function used to trigger call_until_true
        behavior when handling timeout condition. The sequence must
        contain the exact number of values expected to be consumed by
        each time call_until_true calls time.time function.
        :param args: sequence of positional arguments to be passed
        to call_until_true function.
        :param kwargs: sequence of named arguments to be passed
        to call_until_true function.
        """

        # all values except the last are False
        self.assertEqual([False] * len(return_values[:-1]), return_values[:-1])
        # last value can be True or False
        self.assertIn(return_values[-1], [True, False])

        # GIVEN
        func = mock.Mock(side_effect=return_values)
        sleep = 10.  # this value has no effect as time.sleep is being mocked
        sleep_func = self.patch('time.sleep')
        time_func = self._patch_time(time_sequence)
        args = args or tuple()
        kwargs = kwargs or dict()

        # WHEN
        result = test_utils.call_until_true(func, duration, sleep,
                                            *args, **kwargs)
        # THEN

        # It must return last returned value
        self.assertIs(return_values[-1], result)

        self._test_func_calls(func, return_values, *args, **kwargs)
        self._test_sleep_calls(sleep_func, return_values, sleep)
        # The number of times time.time is called is not relevant as a
        # requirement of call_until_true. What is instead relevant is that
        # call_until_true use a mocked function to make the test reliable
        # and the test actually provide the right sequence of numbers to
        # reproduce the behavior has to be tested
        self._assert_called_n_times(time_func, len(time_sequence))

    def _prepare_and_test(self, address6_mode, n_subnets6=1, dualnet=False):
        net_list = self.prepare_network(address6_mode=address6_mode,
                                        n_subnets6=n_subnets6,
                                        dualnet=dualnet)

        sshv4_1, ips_from_api_1, srv1 = self.prepare_server(networks=net_list)
        sshv4_2, ips_from_api_2, srv2 = self.prepare_server(networks=net_list)

        def guest_has_address(ssh, addr):
            return addr in ssh.exec_command("ip address")

        # Turn on 2nd NIC for Cirros when dualnet
        if dualnet:
            _, network_v6 = net_list
            self.turn_nic6_on(sshv4_1, srv1['id'], network_v6['id'])
            self.turn_nic6_on(sshv4_2, srv2['id'], network_v6['id'])

        # get addresses assigned to vNIC as reported by 'ip address' utility
        ips_from_ip_1 = sshv4_1.exec_command("ip address")
        ips_from_ip_2 = sshv4_2.exec_command("ip address")
        self.assertIn(ips_from_api_1['4'], ips_from_ip_1)
        self.assertIn(ips_from_api_2['4'], ips_from_ip_2)
        for i in range(n_subnets6):
            # v6 should be configured since the image supports it
            # It can take time for ipv6 automatic address to get assigned
            for srv, ssh, ips in (
                    (srv1, sshv4_1, ips_from_api_1),
                    (srv2, sshv4_2, ips_from_api_2)):
                ip = ips['6'][i]
                result = test_utils.call_until_true(
                    guest_has_address,
                    CONF.validation.ping_timeout, 1, ssh, ip)
                if not result:
                    self._log_console_output(servers=[srv])
                    self.fail(
                        'Address %s not configured for instance %s, '
                        'ip address output is\n%s' %
                        (ip, srv['id'], ssh.exec_command("ip address")))

        self.check_remote_connectivity(sshv4_1, ips_from_api_2['4'])
        self.check_remote_connectivity(sshv4_2, ips_from_api_1['4'])

        for i in range(n_subnets6):
            self.check_remote_connectivity(sshv4_1,
                                           ips_from_api_2['6'][i])
            self.check_remote_connectivity(sshv4_1,
                                           self.subnets_v6[i]['gateway_ip'])
            self.check_remote_connectivity(sshv4_2,
                                           ips_from_api_1['6'][i])
            self.check_remote_connectivity(sshv4_2,
                                           self.subnets_v6[i]['gateway_ip'])

 def call_until_valid(self, func, duration, *args, **kwargs):
     # Call until get valid response for "duration"
     # because tenant usage doesn't become available immediately
     # after create VM.
     def is_valid():
         try:
             self.resp = func(*args, **kwargs)
             return True
         except e.InvalidHTTPResponseBody:
             return False
     self.assertEqual(test_utils.call_until_true(is_valid, duration, 1),
                      True, "%s not return valid response in %s secs" % (
                          func.__name__, duration))
     return self.resp

    def _wait_until_ready(self, fwg_id):
        target_states = ('ACTIVE', 'CREATED')

        def _wait():
            firewall_group = self.firewall_groups_client.show_firewall_group(
                fwg_id)
            firewall_group = firewall_group['firewall_group']
            return firewall_group['status'] in target_states

        if not test_utils.call_until_true(_wait, CONF.network.build_timeout,
                                          CONF.network.build_interval):
            m = ("Timed out waiting for firewall_group %s to reach %s "
                 "state(s)" %
                 (fwg_id, target_states))
            raise lib_exc.TimeoutException(m)

    def _wait_until_deleted(self, fwg_id):
        def _wait():
            try:
                fwg = self.firewall_groups_client.show_firewall_group(fwg_id)
            except lib_exc.NotFound:
                return True

            fwg_status = fwg['firewall_group']['status']
            if fwg_status == 'ERROR':
                raise lib_exc.DeleteErrorException(resource_id=fwg_id)

        if not test_utils.call_until_true(_wait, CONF.network.build_timeout,
                                          CONF.network.build_interval):
            m = ("Timed out waiting for firewall_group %s deleted" % fwg_id)
            raise lib_exc.TimeoutException(m)

 def part_wait(self, num_match):
     def _part_state():
         self.partitions = self.remote_client.get_partitions().split('\n')
         matching = 0
         for part_line in self.partitions[1:]:
             if self.part_line_re.match(part_line):
                 matching += 1
         return matching == num_match
     if test_utils.call_until_true(_part_state,
                                   CONF.compute.build_timeout,
                                   CONF.compute.build_interval):
         return
     else:
         raise RuntimeError("Unexpected partitions: %s",
                            str(self.partitions))

    def _check_remote_connectivity(self, source, dest, should_succeed=True,
                                   nic=None, mtu=None, fragmentation=True):
        """check ping server via source ssh connection

        :param source: RemoteClient: an ssh connection from which to ping
        :param dest: and IP to ping against
        :param should_succeed: boolean should ping succeed or not
        :param nic: specific network interface to ping from
        :param mtu: mtu size for the packet to be sent
        :param fragmentation: Flag for packet fragmentation
        :returns: boolean -- should_succeed == ping
        :returns: ping is false if ping failed
        """
        def ping_host(source, host, count=CONF.validation.ping_count,
                      size=CONF.validation.ping_size, nic=None, mtu=None,
                      fragmentation=True):
            addr = netaddr.IPAddress(host)
            cmd = 'ping6' if addr.version == 6 else 'ping'
            if nic:
                cmd = 'sudo {cmd} -I {nic}'.format(cmd=cmd, nic=nic)
            if mtu:
                if not fragmentation:
                    cmd += ' -M do'
                size = str(net_utils.get_ping_payload_size(
                    mtu=mtu, ip_version=addr.version))
            cmd += ' -c{0} -w{0} -s{1} {2}'.format(count, size, host)
            return source.exec_command(cmd)

        def ping_remote():
            try:
                result = ping_host(source, dest, nic=nic, mtu=mtu,
                                   fragmentation=fragmentation)

            except lib_exc.SSHExecCommandFailed:
                LOG.warning('Failed to ping IP: %s via a ssh connection '
                            'from: %s.', dest, source.host)
                return not should_succeed
            LOG.debug('ping result: %s', result)
            # Assert that the return traffic was from the correct
            # source address.
            from_source = 'from %s' % dest
            self.assertIn(from_source, result)
            return should_succeed

        return test_utils.call_until_true(ping_remote,
                                          CONF.validation.ping_timeout,
                                          1)